index

package
v0.8.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 7, 2024 License: MIT Imports: 42 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FeatureSortable = "sortable"
	FeaturePrefix   = "prefix"
	FeatureOnlyICU  = "only_icu"
)

Fields can depend on index settings (like custom normalisers). These won't be supported on old indexes, so instead of failing we set feature flags on the indexes so that we know what's supported.

View Source 
const (
	LogKeyIndexOperation = "index_operation"
	LogKeyResponseStatus = "response_status"
)
View Source 
const (
	DocumentEvent = "document"
	DeleteEvent   = "document_delete"
	ACLEvent      = "acl"
	StatusEvent   = "status"
)
View Source 
const (
	MappingBreaking = "breaking"
	MappingEqual    = "equal"
	MappingSuperset = "superset"
)
View Source 
const (
	ScopeIndexAdmin = "index_admin"
	ScopeSearch     = "search"
)
View Source 
const (
	DefaultSearchSize = 50
)
View Source 
const IndexerStopTimeout = 10 * time.Second

Variables

This section is empty.

Functions

func ElasticHandler 

func ElasticHandler(
	logger *slog.Logger,
	w http.ResponseWriter, r *http.Request,
	fn func(w http.ResponseWriter, r *http.Request) error,
)

func InterpretBulkResponse added in v0.5.0 

func InterpretBulkResponse(
	ctx context.Context, log *slog.Logger, r io.Reader,
) (map[string]int, error)

func ListenAndServe added in v0.3.0 

func ListenAndServe(ctx context.Context, addr string, h http.Handler) error

func RequireAnyScope added in v0.3.0 

func RequireAnyScope(ctx context.Context, scopes ...string) (*elephantine.AuthInfo, error)

func RunIndex added in v0.3.0 

func RunIndex(ctx context.Context, p Parameters) error

Types

type ACLEntry 

type ACLEntry struct {
	URI         string
	Permissions []string
}

type ActiveIndexGetter added in v0.3.0 

type ActiveIndexGetter interface {
	GetActiveIndex() (*opensearch.Client, string)
}

type BooleanQuery 

type BooleanQuery struct {
	Must    []json.RawMessage `json:"must,omitempty"`
	MustNot []ElasticQuery    `json:"must_not,omitempty"`
	Should  []ElasticQuery    `json:"should,omitempty"`
	Filter  []ElasticQuery    `json:"filter,omitempty"`
}

type ClusterAuth added in v0.3.0 

type ClusterAuth struct {
	IAM bool
}

type ClusterGetter added in v0.3.0 

type ClusterGetter interface {
	GetCluster(ctx context.Context, name string) (postgres.Cluster, error)
}

type Coordinator added in v0.3.0 

type Coordinator struct {
	// contains filtered or unexported fields
}

func NewCoordinator added in v0.3.0 

func NewCoordinator(
	db *pgxpool.Pool, opt CoordinatorOptions,
) (*Coordinator, error)

func (*Coordinator) EnsureDefaultIndexSet added in v0.3.0 

func (c *Coordinator) EnsureDefaultIndexSet(
	ctx context.Context,
	defaultClusterURL string,
	clusterAuth ClusterAuth,
) error

Ensure that we have a default cluster and index set. Starts with an ACCESS EXCLUSIVE lock on the cluster table, so only one instance will be running this check at any given time.

func (*Coordinator) GetActiveIndex added in v0.3.0 

func (c *Coordinator) GetActiveIndex() (*opensearch.Client, string)

GetActiveIndex the name of the currently active index set, and an OpenSearch client that can be used to access it.

func (*Coordinator) Run added in v0.3.0 

func (c *Coordinator) Run(ctx context.Context) error

Run the coordinator. A coordinator can only run once.

func (*Coordinator) Stop added in v0.3.0 

func (c *Coordinator) Stop(timeout time.Duration)

Stop the coordinator. Blocks until it has stopped or the timeout has been reached.

type CoordinatorOptions added in v0.3.0 

type CoordinatorOptions struct {
	Logger          *slog.Logger
	Metrics         *Metrics
	Documents       repository.Documents
	ClientGetter    OpenSearchClientFunc
	Validator       ValidatorSource
	DefaultLanguage string
	Sharding        ShardingPolicy
	NoIndexing      bool
}

type Document 

type Document struct {
	Fields map[string]Field
}

func BuildDocument 

func BuildDocument(
	validator *revisor.Validator, state *DocumentState,
	language OpenSeachIndexConfig, featureFlags map[string]bool,
) (*Document, error)

func NewDocument 

func NewDocument() *Document

func (*Document) AddField 

func (d *Document) AddField(name string, f Field)

func (*Document) AddInteger 

func (d *Document) AddInteger(name string, value int64)

func (*Document) AddTime 

func (d *Document) AddTime(name string, value time.Time)

func (*Document) Mappings 

func (d *Document) Mappings() Mappings

func (*Document) Values 

func (d *Document) Values() map[string][]string

type DocumentSource added in v0.2.0 

type DocumentSource struct {
	DocumentLanguage []string `json:"document.language"`
}

Some well-known field names used by the indexer.

type DocumentState 

type DocumentState struct {
	Created        time.Time         `json:"created"`
	Modified       time.Time         `json:"modified"`
	CurrentVersion int64             `json:"current_version"`
	ACL            []ACLEntry        `json:"acl"`
	Heads          map[string]Status `json:"heads"`
	Document       newsdoc.Document  `json:"document"`
	MetaDocument   *newsdoc.Document `json:"meta_document"`
}

DocumentState is the full state that we want to index.

type ElasticError 

type ElasticError struct {
	Reason  string           `json:"reason"`
	Details string           `json:"details"`
	Type    ElasticErrorType `json:"type"`
}

type ElasticErrorResponse 

type ElasticErrorResponse struct {
	ErrorInfo ElasticError `json:"error"`
	Status    int          `json:"status"`
}

func ElasticErrorf 

func ElasticErrorf(t ElasticErrorType, format string, a ...any) ElasticErrorResponse

func (ElasticErrorResponse) Error 

func (er ElasticErrorResponse) Error() string

type ElasticErrorType 

type ElasticErrorType string
const (
	ErrorTypeUnauthorized       ElasticErrorType = "elephant.Unauthorized"
	ErrorTypeAccessDenied       ElasticErrorType = "elephant.AccessDenied"
	ErrorTypeInternal           ElasticErrorType = "elephant.InternalError"
	ErrorTypeBadRequest         ElasticErrorType = "elephant.BadRequest"
	ErrorTypeNotFound           ElasticErrorType = "elephant.NotFound"
	ErrorTypeClusterUnavailable ElasticErrorType = "elephant.ClusterUnavailable"
)

func (ElasticErrorType) Reason 

func (et ElasticErrorType) Reason() string

func (ElasticErrorType) StatusCode 

func (et ElasticErrorType) StatusCode() int

type ElasticProxy 

type ElasticProxy struct {
	// contains filtered or unexported fields
}

func NewElasticProxy 

func NewElasticProxy(
	logger *slog.Logger,
	active ActiveIndexGetter,
	authInfoParser *elephantine.AuthInfoParser,
) *ElasticProxy

func (*ElasticProxy) ServeHTTP 

func (ep *ElasticProxy) ServeHTTP(w http.ResponseWriter, r *http.Request)

type ElasticQuery 

type ElasticQuery struct {
	Bool *BooleanQuery     `json:"bool,omitempty"`
	Term map[string]string `json:"term,omitempty"`
	IDs  *IDsQuery         `json:"ids,omitempty"`
}

type ElasticSearchRequest 

type ElasticSearchRequest struct {
	Query       ElasticQuery    `json:"query"`
	Fields      json.RawMessage `json:"fields,omitempty"`
	Sort        json.RawMessage `json:"sort,omitempty"`
	Source      *bool           `json:"_source,omitempty"`
	From        *int            `json:"from,omitempty"`
	Size        *int            `json:"size,omitempty"`
	SearchAfter json.RawMessage `json:"search_after,omitempty"`
}

type Field 

type Field struct {
	FieldOptions

	Values []string                `json:"values"`
	Fields map[string]FieldOptions `json:"fields,omitempty"`
}

func (*Field) AddSubField added in v0.5.1 

func (f *Field) AddSubField(name string, sf FieldOptions)

type FieldOptions added in v0.8.6 

type FieldOptions struct {
	Type           FieldType `json:"type"`
	Normalizer     string    `json:"normalizer,omitempty"`
	Analyzer       string    `json:"analyzer,omitempty"`
	SearchAnalyzer string    `json:"search_analyzer,omitempty"`
	Index          *bool     `json:"index,omitempty"`
	Language       string    `json:"language,omitempty"`
	Country        string    `json:"country,omitempty"`
	Variant        string    `json:"variant,omitempty"`
}

func (FieldOptions) Equal added in v0.8.6 

func (fo FieldOptions) Equal(other FieldOptions) bool

type FieldType 

type FieldType string
const (
	TypeUnknown    FieldType = ""
	TypeBoolean    FieldType = "boolean"
	TypeDouble     FieldType = "double"
	TypeLong       FieldType = "long"
	TypeDate       FieldType = "date"
	TypeText       FieldType = "text"
	TypeKeyword    FieldType = "keyword"
	TypeAlias      FieldType = "alias"
	TypePercolator FieldType = "percolator"
	TypeICUKeyword FieldType = "icu_collation_keyword"
)

func (FieldType) Priority 

func (ft FieldType) Priority() int

We should not have colliding types, but if something first is defined as text or keyword, and then has a more specific constraint in f.ex. an extension, then we should allow the more specific constraint to win out.

type IDsQuery added in v0.4.0 

type IDsQuery struct {
	Values []string `json:"values,omitempty"`
}

type Indexer 

type Indexer struct {
	// contains filtered or unexported fields
}

Indexer takes care of indexing to a named set of indexes in a cluster.

func NewIndexer 

func NewIndexer(ctx context.Context, opts IndexerOptions) (*Indexer, error)

func (*Indexer) Run 

func (idx *Indexer) Run(ctx context.Context) error

func (*Indexer) Stop added in v0.3.0 

func (idx *Indexer) Stop(timeout time.Duration) error

Stop the indexer. Blocks until it has stopped or the timeout has been reached.

func (*Indexer) Stopping added in v0.3.0 

func (idx *Indexer) Stopping() <-chan struct{}

type IndexerOptions 

type IndexerOptions struct {
	Logger            *slog.Logger
	SetName           string
	Database          *pgxpool.Pool
	Client            *opensearch.Client
	Documents         repository.Documents
	Validator         ValidatorSource
	Metrics           *Metrics
	DefaultLanguage   string
	DefaultRegions    map[string]string
	EnablePercolation bool
	Sharding          ShardingPolicy
}

type Language 

type Language struct {
	Code     string
	Language string
	Analyzer string
}

type ManagementService added in v0.3.0 

type ManagementService struct {
	// contains filtered or unexported fields
}

func NewManagementService added in v0.3.0 

func NewManagementService(
	logger *slog.Logger,
	db *pgxpool.Pool,
) (*ManagementService, error)

func (*ManagementService) DeleteCluster added in v0.3.0 

func (s *ManagementService) DeleteCluster(
	ctx context.Context, req *index.DeleteClusterRequest,
) (*index.DeleteClusterResponse, error)

DeleteCluster implements index.Management.

func (*ManagementService) DeleteIndexSet added in v0.3.0 

func (s *ManagementService) DeleteIndexSet(
	ctx context.Context, req *index.DeleteIndexSetRequest,
) (*index.DeleteIndexSetResponse, error)

DeleteIndexSet implements index.Management.

func (*ManagementService) ListClusters added in v0.3.0 

func (s *ManagementService) ListClusters(
	ctx context.Context, _ *index.ListClustersRequest,
) (*index.ListClustersResponse, error)

ListClusters implements index.Management.

func (*ManagementService) ListIndexSets added in v0.3.0 

func (s *ManagementService) ListIndexSets(
	ctx context.Context, req *index.ListIndexSetsRequest,
) (*index.ListIndexSetsResponse, error)

ListIndexSets implements index.Management.

func (*ManagementService) PartialReindex added in v0.3.0 

func (s *ManagementService) PartialReindex(
	ctx context.Context, _ *index.PartialReindexRequest,
) (*index.PartialReindexResponse, error)

PartialReindex implements index.Management.

func (*ManagementService) RegisterCluster added in v0.3.0 

func (s *ManagementService) RegisterCluster(
	ctx context.Context, req *index.RegisterClusterRequest,
) (*index.RegisterClusterResponse, error)

RegisterCluster implements index.Management.

func (*ManagementService) Reindex added in v0.3.0 

func (s *ManagementService) Reindex(
	ctx context.Context, req *index.ReindexRequest,
) (*index.ReindexResponse, error)

Reindex implements index.Management.

func (*ManagementService) SetIndexSetStatus added in v0.3.0 

func (s *ManagementService) SetIndexSetStatus(
	ctx context.Context, req *index.SetIndexSetStatusRequest,
) (*index.SetIndexSetStatusResponse, error)

SetIndexSetStatus implements index.Management.

type Mapping 

type Mapping struct {
	FieldOptions

	Path   string                  `json:"path,omitempty"`
	Fields map[string]FieldOptions `json:"fields,omitempty"`
}

func (Mapping) Compare added in v0.5.1 

func (m Mapping) Compare(other Mapping) MappingComparison

type MappingChange 

type MappingChange struct {
	Mapping

	Comparison MappingComparison `json:"comparison"`
}

type MappingChanges 

type MappingChanges map[string]MappingChange

func (MappingChanges) HasNew 

func (mc MappingChanges) HasNew() bool

func (MappingChanges) Superset 

func (mc MappingChanges) Superset(mappings Mappings) Mappings

type MappingComparison added in v0.5.1 

type MappingComparison string

type MappingSource added in v0.8.6 

type MappingSource interface {
	GetMappings(
		ctx context.Context,
		indexSet string,
		docType string,
	) (map[string]Mapping, error)
}

type Mappings 

type Mappings struct {
	Properties map[string]Mapping `json:"properties"`
}

func NewMappings 

func NewMappings() Mappings

func (*Mappings) ChangesFrom 

func (m *Mappings) ChangesFrom(mappings Mappings) MappingChanges

type Metrics 

type Metrics struct {
	// contains filtered or unexported fields
}

func NewMetrics 

func NewMetrics(reg prometheus.Registerer) (*Metrics, error)

type Notification added in v0.3.0 

type Notification struct {
	Type NotifyChannel
	Name string
}

func (Notification) Send added in v0.3.0 

func (n Notification) Send(ctx context.Context, q *postgres.Queries) error

type NotifyChannel added in v0.3.0 

type NotifyChannel string
const (
	NotifyIndexStatusChange NotifyChannel = "index_status_change"
)

type OSClientProvider added in v0.3.0 

type OSClientProvider struct {
	// contains filtered or unexported fields
}

func NewOSClientProvider added in v0.3.0 

func NewOSClientProvider(clusters ClusterGetter) *OSClientProvider

func (*OSClientProvider) GetClientForCluster added in v0.3.0 

func (o *OSClientProvider) GetClientForCluster(
	ctx context.Context, cluster string,
) (*opensearch.Client, error)

type OpenSeachIndexConfig added in v0.8.4 

type OpenSeachIndexConfig struct {
	NameSuffix string
	Language   string
	Region     string
	Settings   OpensearchSettings
}

func GetIndexConfig added in v0.8.4 

func GetIndexConfig(
	code string, defaultLanguage string,
	defaultRegions map[string]string,
) (OpenSeachIndexConfig, error)

type OpenSearchClientFunc added in v0.3.0 

type OpenSearchClientFunc func(
	ctx context.Context, cluster string,
) (*opensearch.Client, error)

type OpensearchAnalysis added in v0.5.1 

type OpensearchAnalysis struct {
	Analyzer   map[string]OpensearchAnalyzer   `json:"analyzer,omitempty"`
	Normalizer map[string]OpensearchNormaliser `json:"normalizer,omitempty"`
	Tokenizer  map[string]OpensearchTokenizer  `json:"tokenizer,omitempty"`
}

func (*OpensearchAnalysis) SetAnalyzer added in v0.5.1 

func (osa *OpensearchAnalysis) SetAnalyzer(name string, v OpensearchAnalyzer)

func (*OpensearchAnalysis) SetNormalizer added in v0.5.1 

func (osa *OpensearchAnalysis) SetNormalizer(name string, v OpensearchNormaliser)

func (*OpensearchAnalysis) SetTokenizer added in v0.8.4 

func (osa *OpensearchAnalysis) SetTokenizer(name string, v OpensearchTokenizer)

type OpensearchAnalyzer added in v0.5.1 

type OpensearchAnalyzer struct {
	Type      string   `json:"type"`
	Tokenizer string   `json:"tokenizer,omitempty"`
	Filter    []string `json:"filter,omitempty"`
}

type OpensearchNormaliser added in v0.5.1 

type OpensearchNormaliser struct {
	Type   string   `json:"type"`
	Filter []string `json:"filter"`
}

type OpensearchSettings 

type OpensearchSettings struct {
	Settings struct {
		Index    ShardingSettings   `json:"index"`
		Analysis OpensearchAnalysis `json:"analysis"`
	} `json:"settings"`
}

type OpensearchTokenizer added in v0.8.4 

type OpensearchTokenizer struct {
	Type       string   `json:"type"`
	MinGram    int      `json:"min_gram,omitempty"`
	MaxGram    int      `json:"max_gram,omitempty"`
	TokenChars []string `json:"token_chars,omitempty"`
}

type Parameters added in v0.3.0 

type Parameters struct {
	Addr               string
	ProfileAddr        string
	Logger             *slog.Logger
	Database           *pgxpool.Pool
	DefaultCluster     string
	ClusterAuth        ClusterAuth
	Client             OpenSearchClientFunc
	Documents          repository.Documents
	RepositoryEndpoint string
	Validator          ValidatorSource
	Metrics            *Metrics
	DefaultLanguage    string
	NoIndexer          bool
	AuthInfoParser     *elephantine.AuthInfoParser
	Sharding           ShardingPolicy
}

type PostgresMappingSource added in v0.8.6 

type PostgresMappingSource struct {
	// contains filtered or unexported fields
}

func NewPostgresMappingSource added in v0.8.6 

func NewPostgresMappingSource(
	q *postgres.Queries,
) *PostgresMappingSource

func (*PostgresMappingSource) GetMappings added in v0.8.6 

func (pms *PostgresMappingSource) GetMappings(
	ctx context.Context, setName string, contentType string,
) (map[string]Mapping, error)

type RawSearchRequest 

type RawSearchRequest struct {
	Query       json.RawMessage `json:"query"`
	Fields      json.RawMessage `json:"fields"`
	Sort        json.RawMessage `json:"sort"`
	Source      *bool           `json:"_source,omitempty"`
	From        *int            `json:"from,omitempty"`
	Size        *int            `json:"size,omitempty"`
	SearchAfter json.RawMessage `json:"search_after,omitempty"`
}

type SchemaLoader 

type SchemaLoader struct {
	// contains filtered or unexported fields
}

func NewSchemaLoader 

func NewSchemaLoader(
	ctx context.Context,
	logger *slog.Logger,
	client repository.Schemas,
) (*SchemaLoader, error)

func (*SchemaLoader) GetValidator 

func (sl *SchemaLoader) GetValidator() *revisor.Validator

type SearchResponseBody added in v0.2.0 

type SearchResponseBody struct {
	Hits struct {
		Hits []struct {
			ID     string         `json:"_id"`
			Index  string         `json:"_index"`
			Source DocumentSource `json:"_source"`
		} `json:"hits"`
	} `json:"hits"`
}

type SearchServiceV1 added in v0.8.0 

type SearchServiceV1 struct {
	// contains filtered or unexported fields
}

func NewSearchServiceV1 added in v0.8.0 

func NewSearchServiceV1(
	mappings MappingSource,
	active ActiveIndexGetter,
	repositoryEndpoint string,
) *SearchServiceV1

func (*SearchServiceV1) GetMappings added in v0.8.6 

func (s *SearchServiceV1) GetMappings(
	ctx context.Context, req *index.GetMappingsRequestV1,
) (*index.GetMappingsResponseV1, error)

GetMappings implements index.SearchV1.

func (*SearchServiceV1) Query added in v0.8.0 

func (s *SearchServiceV1) Query(
	ctx context.Context, req *index.QueryRequestV1,
) (_ *index.QueryResponseV1, outErr error)

Query implements index.SearchV1.

type ServerOptions added in v0.3.0 

type ServerOptions struct {
	Hooks          *twirp.ServerHooks
	AuthMiddleware func(
		w http.ResponseWriter, r *http.Request, next http.Handler,
	) error
}

func (*ServerOptions) SetJWTValidation added in v0.3.0 

func (so *ServerOptions) SetJWTValidation(parser *elephantine.AuthInfoParser)

type ShardingPolicy added in v0.6.0 

type ShardingPolicy struct {
	Default ShardingSettings
	Indexes []ShardingRule
}

func ParseShardingPolicy added in v0.6.0 

func ParseShardingPolicy(
	value string, defaultSettings ShardingSettings,
) (ShardingPolicy, error)

func (ShardingPolicy) GetSettings added in v0.6.0 

func (sc ShardingPolicy) GetSettings(index string) ShardingSettings

type ShardingRule added in v0.6.0 

type ShardingRule struct {
	Prefix   string
	Settings ShardingSettings
}

type ShardingSettings added in v0.6.0 

type ShardingSettings struct {
	Shards   int `json:"number_of_shards"`
	Replicas int `json:"number_of_replicas"`
}

type Status 

type Status struct {
	ID      int64
	Version int64
	Creator string
	Created time.Time
	Meta    newsdoc.DataMap
}

type ValidatorSource 

type ValidatorSource interface {
	GetValidator() *revisor.Validator
}

type ValueCollector 

type ValueCollector struct {
	// contains filtered or unexported fields
}

func NewValueCollector 

func NewValueCollector() *ValueCollector

func (*ValueCollector) CollectValue 

func (c *ValueCollector) CollectValue(a revisor.ValueAnnotation)

func (*ValueCollector) Values 

func (c *ValueCollector) Values() []revisor.ValueAnnotation

func (*ValueCollector) With 

func (c *ValueCollector) With(ref revisor.EntityRef) revisor.ValueCollector

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.