indexshipper

Documentation

Index

• Constants
• type Config
  • func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
• type Index
• type IndexShipper
  • func NewIndexShipper(cfg Config, storageClient client.ObjectClient, limits downloads.Limits) (IndexShipper, error)
• type Mode

Constants

const (
	// ModeReadWrite is to allow both read and write
	ModeReadWrite = Mode("RW")
	// ModeReadOnly is to allow only read operations
	ModeReadOnly = Mode("RO")
	// ModeWriteOnly is to allow only write operations
	ModeWriteOnly = Mode("WO")

	// UploadInterval defines interval for when we check if there are new index files to upload.
	// It's also used to snapshot the currently written index tables so the snapshots can be used for reads.
	UploadInterval = 1 * time.Minute
)

Types

type Config

type Config struct {
	ActiveIndexDirectory string        `yaml:"active_index_directory"`
	SharedStoreType      string        `yaml:"shared_store"`
	SharedStoreKeyPrefix string        `yaml:"shared_store_key_prefix"`
	CacheLocation        string        `yaml:"cache_location"`
	CacheTTL             time.Duration `yaml:"cache_ttl"`
	ResyncInterval       time.Duration `yaml:"resync_interval"`
	QueryReadyNumDays    int           `yaml:"query_ready_num_days"`

	UploaderName           string
	IngesterName           string
	Mode                   Mode
	IngesterDBRetainPeriod time.Duration
}

func (*Config) RegisterFlagsWithPrefix

func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix registers flags.

type Index

type Index interface {
	Close() error
}

type IndexShipper

type IndexShipper interface {
	// AddIndex adds an immutable index to a logical table which would eventually get uploaded to the object store.
	AddIndex(tableName, userID string, index index.Index) error
	// ForEach lets us iterates through each index file in a table for a specific user.
	// On the write path, it would iterate on the files given to the shipper for uploading, until they eventually get dropped from local disk.
	// On the read path, it would iterate through the files if already downloaded else it would download and iterate through them.
	ForEach(ctx context.Context, tableName, userID string, callback func(index index.Index) error) error
	Stop()
}

func NewIndexShipper

func NewIndexShipper(cfg Config, storageClient client.ObjectClient, limits downloads.Limits) (IndexShipper, error)

NewIndexShipper creates a shipper for providing index store functionality using index files and object storage. It manages the whole life cycle of uploading the index and downloading the index at query time.

type Mode

type Mode string