index

README

Package cloudeng.io/glean/crawlindex/index

import cloudeng.io/glean/crawlindex/index

Types

Type BulkFlags

type BulkFlags struct {
	config.FileFlags
	UploadID      string `subcmd:"upload-id,upload,id to use for this bulk upload"`
	ForceRestart  bool   `subcmd:"force-restart,false,restart the bulk upload"`
	ForceDeletion bool   `subcmd:"force-sync-deletion,false,synchronously delete stale documents on upload of last bulk indexing batch"`
	DryRun        bool   `subcmd:"dry-run,false,'process but not do index documents'"`
}

BulkFlags represents the flags to the bulk indexing command.

Type BulkIndexOption

type BulkIndexOption func(o *bulkOptions)

BulkIndexOption represents an option to NewBulkIndexer.

Functions

func WithBulkID(id string) BulkIndexOption

WithBulkID specifies a custom id to use for the bulk upload. If one is not specified the current date and time are used.

func WithDryRun(v bool) BulkIndexOption

func WithForceDelete(forceDelete bool) BulkIndexOption

WithForceDelete disables the deletion of too many documents test.

func WithForceRestart(forceRestart bool) BulkIndexOption

WithForceRestart sets the force restart options.

func WithReqSizes(doc, emp int) BulkIndexOption

func WithUsers(v bool) BulkIndexOption

Type DeleteAllFlags

type DeleteAllFlags struct {
	config.FileFlags
}

DeleteAllFlags represents the flags to the indexing delete-all command.

Type DeleteFlags

type DeleteFlags struct {
	config.FileFlags
	NumDocuments int    `subcmd:"num-documents,100,number of documents to return"`
	Type         string `subcmd:"doc-type,,type of object to delete"`
}

DeleteFlags represents the flags to the indexing delete command.

Type Indexer

type Indexer struct {
	// contains filtered or unexported fields
}

Indexer represents a Glean indexer.

Functions

func New(ctx context.Context, fv config.FileFlags, datasource string, resources Resources) (*Indexer, error)

Methods

func (idx *Indexer) Bulk(ctx context.Context, fv *BulkFlags) error

Bulk indexes a datasource in bulk mode.

func (idx *Indexer) Delete(ctx context.Context, fv *DeleteFlags, query string) error

func (idx *Indexer) DeleteAll(ctx context.Context, _ *DeleteAllFlags) error

func (idx *Indexer) ProcessNow(ctx context.Context, _ *ProcessNowFlags) error

func (idx *Indexer) Query(ctx context.Context, fv *QueryFlags, datasource string, query string) error

func (idx *Indexer) Stats(ctx context.Context, _ *StatsFlags) error

Type ProcessNowFlags

type ProcessNowFlags struct {
	config.FileFlags
}

ProcessNowFlags represents the flags to the indexing process now command.

Type QueryFlags

type QueryFlags struct {
	config.FileFlags
	NumDocuments       int  `subcmd:"num-documents,10,number of documents to return"`
	ShowIndexingStatus bool `subcmd:"show-indexing-status,true,show indexing status for all returned documents"`
}

QueryFlags represents the flags to the indexing query command.

Type Request

type Request struct {
	Documents []gleansdk.DocumentDefinition
	Users     []gleansdk.DatasourceUserDefinition
	LastPage  bool
}

Request contains the entities to be indexed and a flag indicating if these are the last entities in a bulk index operation. The indexer will stop when it receives a request with LastPage set. If LastPage flag is never set, the indexer will assume that the indexing operation is complete when its input channel is closed.

Type Resources

type Resources struct {
	IndexingToken      *apitokens.T
	ClientToken        *apitokens.T
	DocumentConverters *content.Registry[converters.Document]
	UserConverters     *content.Registry[converters.User]
	NewOperationsFS    func(ctx context.Context, cfg crawlcmd.CrawlCacheConfig) (operations.FS, error)
}

Resources represents the resources needed by an indexer.

Type StatsFlags

type StatsFlags struct {
	config.FileFlags
}

StatsFlags represents the flags to the bulk indexing command.

Documentation

Index

• type BulkFlags
• type BulkIndexOption
  • func WithBulkID(id string) BulkIndexOption
  • func WithDryRun(v bool) BulkIndexOption
  • func WithForceDelete(forceDelete bool) BulkIndexOption
  • func WithForceRestart(forceRestart bool) BulkIndexOption
  • func WithReqSizes(doc, emp int) BulkIndexOption
  • func WithUsers(v bool) BulkIndexOption
• type DeleteAllFlags
• type DeleteFlags
• type Indexer
  • func New(ctx context.Context, fv config.FileFlags, datasource string, ...) (*Indexer, error)
  • func (idx *Indexer) Bulk(ctx context.Context, fv *BulkFlags) error
  • func (idx *Indexer) Delete(ctx context.Context, fv *DeleteFlags, query string) error
  • func (idx *Indexer) DeleteAll(ctx context.Context, _ *DeleteAllFlags) error
  • func (idx *Indexer) ProcessNow(ctx context.Context, _ *ProcessNowFlags) error
  • func (idx *Indexer) Query(ctx context.Context, fv *QueryFlags, datasource string, query string) error
  • func (idx *Indexer) Stats(ctx context.Context, _ *StatsFlags) error
• type ProcessNowFlags
• type QueryFlags
• type Request
• type Resources
• type StatsFlags

Types

type BulkFlags

type BulkFlags struct {
	config.FileFlags
	UploadID      string `subcmd:"upload-id,upload,id to use for this bulk upload"`
	ForceRestart  bool   `subcmd:"force-restart,false,restart the bulk upload"`
	ForceDeletion bool   `subcmd:"force-sync-deletion,false,synchronously delete stale documents on upload of last bulk indexing batch"`
	DryRun        bool   `subcmd:"dry-run,false,'process but not do index documents'"`
}

BulkFlags represents the flags to the bulk indexing command.

type BulkIndexOption

type BulkIndexOption func(o *bulkOptions)

BulkIndexOption represents an option to NewBulkIndexer.

func WithBulkID

func WithBulkID(id string) BulkIndexOption

WithBulkID specifies a custom id to use for the bulk upload. If one is not specified the current date and time are used.

func WithDryRun

func WithDryRun(v bool) BulkIndexOption

func WithForceDelete

func WithForceDelete(forceDelete bool) BulkIndexOption

WithForceDelete disables the deletion of too many documents test.

func WithForceRestart

func WithForceRestart(forceRestart bool) BulkIndexOption

WithForceRestart sets the force restart options.

func WithReqSizes

func WithReqSizes(doc, emp int) BulkIndexOption

func WithUsers

func WithUsers(v bool) BulkIndexOption

type DeleteAllFlags

type DeleteAllFlags struct {
	config.FileFlags
}

DeleteAllFlags represents the flags to the indexing delete-all command.

type DeleteFlags

type DeleteFlags struct {
	config.FileFlags
	NumDocuments int    `subcmd:"num-documents,100,number of documents to return"`
	Type         string `subcmd:"doc-type,,type of object to delete"`
}

DeleteFlags represents the flags to the indexing delete command.

type Indexer

type Indexer struct {
	// contains filtered or unexported fields
}

Indexer represents a Glean indexer.

func New

func New(ctx context.Context, fv config.FileFlags, datasource string, resources Resources) (*Indexer, error)

func (*Indexer) Bulk

func (idx *Indexer) Bulk(ctx context.Context, fv *BulkFlags) error

Bulk indexes a datasource in bulk mode.

func (*Indexer) Delete

func (idx *Indexer) Delete(ctx context.Context, fv *DeleteFlags, query string) error

func (*Indexer) DeleteAll

func (idx *Indexer) DeleteAll(ctx context.Context, _ *DeleteAllFlags) error

func (*Indexer) ProcessNow

func (idx *Indexer) ProcessNow(ctx context.Context, _ *ProcessNowFlags) error

func (*Indexer) Query

func (idx *Indexer) Query(ctx context.Context, fv *QueryFlags, datasource string, query string) error

func (*Indexer) Stats

func (idx *Indexer) Stats(ctx context.Context, _ *StatsFlags) error

type ProcessNowFlags

type ProcessNowFlags struct {
	config.FileFlags
}

ProcessNowFlags represents the flags to the indexing process now command.

type QueryFlags

type QueryFlags struct {
	config.FileFlags
	NumDocuments       int  `subcmd:"num-documents,10,number of documents to return"`
	ShowIndexingStatus bool `subcmd:"show-indexing-status,true,show indexing status for all returned documents"`
}

QueryFlags represents the flags to the indexing query command.

type Request

type Request struct {
	Documents []gleansdk.DocumentDefinition
	Users     []gleansdk.DatasourceUserDefinition
	LastPage  bool
}

Request contains the entities to be indexed and a flag indicating if these are the last entities in a bulk index operation. The indexer will stop when it receives a request with LastPage set. If LastPage flag is never set, the indexer will assume that the indexing operation is complete when its input channel is closed.

type Resources

type Resources struct {
	IndexingToken      *apitokens.T
	ClientToken        *apitokens.T
	DocumentConverters *content.Registry[converters.Document]
	UserConverters     *content.Registry[converters.User]
	NewOperationsFS    func(ctx context.Context, cfg crawlcmd.CrawlCacheConfig) (operations.FS, error)
}

Resources represents the resources needed by an indexer.

type StatsFlags

type StatsFlags struct {
	config.FileFlags
}

StatsFlags represents the flags to the bulk indexing command.