indexheader

package
v0.0.0-...-457fe6c Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 20, 2024 License: AGPL-3.0 Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// BinaryFormatV1 represents first version of index-header file.
	BinaryFormatV1 = 1

	// MagicIndex are 4 bytes at the head of an index-header file.
	MagicIndex = 0xBAAAD792
)
View Source 
const (
	DefaultIndexHeaderLazyLoadingEnabled     = true
	DefaultIndexHeaderLazyLoadingIdleTimeout = 60 * time.Minute
)

Variables

View Source 
var NotFoundRangeErr = errors.New("range not found") //nolint:revive

NotFoundRangeErr is an error returned by PostingsOffset when there is no posting for given name and value pairs.

Functions

func RestoreLoadedBlocks 

func RestoreLoadedBlocks(directory string) (map[ulid.ULID]int64, error)

func WriteBinary 

func WriteBinary(ctx context.Context, bkt objstore.BucketReader, id ulid.ULID, filename string) (err error)

WriteBinary build index-header file from the pieces of index in object storage.

Types

type BinaryTOC 

type BinaryTOC struct {
	// Symbols holds start to the same symbols section as index related to this index header.
	Symbols uint64
	// PostingsOffsetTable holds start to the same Postings Offset Table section as index related to this index header.
	PostingsOffsetTable uint64
}

BinaryTOC is a table of content for index-header file.

type BlocksLoader 

type BlocksLoader interface {
	LoadedBlocks() map[ulid.ULID]int64
}

type Config 

type Config struct {
	MaxIdleFileHandles         uint `yaml:"max_idle_file_handles" category:"advanced"`
	EagerLoadingStartupEnabled bool `yaml:"eager_loading_startup_enabled" category:"experimental"`

	// Controls whether index-header lazy loading is enabled.
	LazyLoadingEnabled     bool          `yaml:"lazy_loading_enabled" category:"advanced"`
	LazyLoadingIdleTimeout time.Duration `yaml:"lazy_loading_idle_timeout" category:"advanced"`

	// Maximum index-headers loaded into store-gateway concurrently
	LazyLoadingConcurrency             int           `yaml:"lazy_loading_concurrency" category:"advanced"`
	LazyLoadingConcurrencyQueueTimeout time.Duration `yaml:"lazy_loading_concurrency_queue_timeout" category:"advanced"`

	VerifyOnLoad bool `yaml:"verify_on_load" category:"advanced"`

	// EagerLoadingPersistInterval is injected for testing purposes only.
	EagerLoadingPersistInterval time.Duration `yaml:"-" doc:"hidden"`
}

func (*Config) RegisterFlagsWithPrefix 

func (cfg *Config) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*Config) Validate 

func (cfg *Config) Validate() error

type FileWriter 

type FileWriter struct {
	// contains filtered or unexported fields
}

func NewFileWriter 

func NewFileWriter(name string, size int) (*FileWriter, error)

TODO(bwplotka): Added size to method, upstream this.

func (*FileWriter) AddPadding 

func (fw *FileWriter) AddPadding(size int) error

AddPadding adds zero byte padding until the file size is a multiple size.

func (*FileWriter) Close 

func (fw *FileWriter) Close() error

func (*FileWriter) Flush 

func (fw *FileWriter) Flush() error

func (*FileWriter) Pos 

func (fw *FileWriter) Pos() uint64

func (*FileWriter) Remove 

func (fw *FileWriter) Remove() error

func (*FileWriter) Write 

func (fw *FileWriter) Write(bufs ...[]byte) error

func (*FileWriter) WriteAt 

func (fw *FileWriter) WriteAt(buf []byte, pos uint64) error

type LazyBinaryReader 

type LazyBinaryReader struct {
	// contains filtered or unexported fields
}

LazyBinaryReader wraps BinaryReader and loads (mmap or streaming read) the index-header only upon the first Reader function is called.

func NewLazyBinaryReader 

func NewLazyBinaryReader(
	ctx context.Context,
	readerFactory func() (Reader, error),
	logger log.Logger,
	bkt objstore.BucketReader,
	dir string,
	id ulid.ULID,
	metrics *LazyBinaryReaderMetrics,
	onClosed func(*LazyBinaryReader),
	lazyLoadingGate gate.Gate,
) (*LazyBinaryReader, error)

NewLazyBinaryReader makes a new LazyBinaryReader. If the index-header does not exist on the local disk at dir location, this function will build it downloading required sections from the full index stored in the bucket. However, this function doesn't load (mmap or streaming read) the index-header; it will be loaded at first Reader function call.

func (*LazyBinaryReader) Close 

func (r *LazyBinaryReader) Close() error

Close implements Reader.

func (*LazyBinaryReader) IndexVersion 

func (r *LazyBinaryReader) IndexVersion(ctx context.Context) (int, error)

IndexVersion implements Reader.

func (*LazyBinaryReader) IsIdleSince 

func (r *LazyBinaryReader) IsIdleSince(ts int64) bool

IsIdleSince returns true if the reader is idle since given time (as unix nano).

func (*LazyBinaryReader) LabelNames 

func (r *LazyBinaryReader) LabelNames(ctx context.Context) ([]string, error)

LabelNames implements Reader.

func (*LazyBinaryReader) LabelValuesOffsets 

func (r *LazyBinaryReader) LabelValuesOffsets(ctx context.Context, name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)

LabelValuesOffsets implements Reader.

func (*LazyBinaryReader) LoadedLastUse 

func (r *LazyBinaryReader) LoadedLastUse() int64

LoadedLastUse returns 0 if the reader is not loaded. LoadedLastUse returns a timestamp in nanoseconds of the last time this reader was used.

func (*LazyBinaryReader) LookupSymbol 

func (r *LazyBinaryReader) LookupSymbol(ctx context.Context, o uint32) (string, error)

LookupSymbol implements Reader.

func (*LazyBinaryReader) PostingsOffset 

func (r *LazyBinaryReader) PostingsOffset(ctx context.Context, name string, value string) (index.Range, error)

PostingsOffset implements Reader.

func (*LazyBinaryReader) SymbolsReader 

func (r *LazyBinaryReader) SymbolsReader(ctx context.Context) (streamindex.SymbolsReader, error)

SymbolsReader implements Reader.

type LazyBinaryReaderMetrics 

type LazyBinaryReaderMetrics struct {
	// contains filtered or unexported fields
}

LazyBinaryReaderMetrics holds metrics tracked by LazyBinaryReader.

func NewLazyBinaryReaderMetrics 

func NewLazyBinaryReaderMetrics(reg prometheus.Registerer) *LazyBinaryReaderMetrics

NewLazyBinaryReaderMetrics makes new LazyBinaryReaderMetrics.

type Reader 

type Reader interface {
	// Close should be called when this instance of Reader will no longer be used.
	// It is illegal to call Close multiple times.
	Close() error

	// IndexVersion returns version of index.
	IndexVersion(context.Context) (int, error)

	// PostingsOffset returns start and end offsets of postings for given name and value.
	// The Start is inclusive and is the byte offset of the number_of_entries field of a posting list.
	// The End is exclusive and is typically the byte offset of the CRC32 field.
	// The End might be bigger than the actual posting ending, but not larger than the whole index file.
	// NotFoundRangeErr is returned when no index can be found for given name and value.
	PostingsOffset(ctx context.Context, name string, value string) (index.Range, error)

	// LookupSymbol returns string based on given reference.
	// Error is return if the symbol can't be found.
	LookupSymbol(ctx context.Context, o uint32) (string, error)

	SymbolsReader(ctx context.Context) (streamindex.SymbolsReader, error)

	// LabelValuesOffsets returns all label values and the offsets for their posting lists for given label name or error.
	// The returned label values are sorted lexicographically (which the same as sorted by posting offset).
	// The ranges of each posting list are the same as returned by PostingsOffset.
	// If no values are found for label name, or label name does not exists,
	// then empty slice is returned and no error.
	// If non-empty prefix is provided, only posting lists starting with the prefix are returned.
	// If non-nil filter is provided, then only posting lists for which filter returns true are returned.
	LabelValuesOffsets(ctx context.Context, name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)

	// LabelNames returns all label names in sorted order.
	LabelNames(ctx context.Context) ([]string, error)
}

Reader is an interface allowing to read essential, minimal number of index fields from the small portion of index file called header.

type ReaderPool 

type ReaderPool struct {
	services.Service
	// contains filtered or unexported fields
}

ReaderPool is used to istantiate new index-header readers and keep track of them. When the lazy reader is enabled, the pool keeps track of all instantiated readers and automatically close them once the idle timeout is reached. A closed lazy reader will be automatically re-opened upon next usage.

func NewReaderPool 

func NewReaderPool(logger log.Logger, indexHeaderConfig Config, lazyLoadingGate gate.Gate, metrics *ReaderPoolMetrics) *ReaderPool

NewReaderPool makes a new ReaderPool. If lazy-loading is enabled, NewReaderPool also starts a background task for unloading idle Readers.

func (*ReaderPool) LoadedBlocks 

func (p *ReaderPool) LoadedBlocks() map[ulid.ULID]int64

LoadedBlocks returns a new map of lazy-loaded block IDs and the last time they were used in milliseconds.

func (*ReaderPool) NewBinaryReader 

func (p *ReaderPool) NewBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int, cfg Config) (Reader, error)

NewBinaryReader creates and returns a new binary reader. If the pool has been configured with lazy reader enabled, this function will return a lazy reader. The returned lazy reader is tracked by the pool and automatically closed once the idle timeout expires.

type ReaderPoolMetrics 

type ReaderPoolMetrics struct {
	// contains filtered or unexported fields
}

ReaderPoolMetrics holds metrics tracked by ReaderPool.

func NewReaderPoolMetrics 

func NewReaderPoolMetrics(reg prometheus.Registerer) *ReaderPoolMetrics

NewReaderPoolMetrics makes new ReaderPoolMetrics.

type Snapshotter 

type Snapshotter struct {
	services.Service
	// contains filtered or unexported fields
}

Snapshotter manages the snapshots of lazy loaded blocks.

func NewSnapshotter 

func NewSnapshotter(logger log.Logger, conf SnapshotterConfig, bl BlocksLoader) *Snapshotter

func (*Snapshotter) PersistLoadedBlocks 

func (s *Snapshotter) PersistLoadedBlocks() error

type SnapshotterConfig 

type SnapshotterConfig struct {
	PersistInterval time.Duration
	// Path stores where lazy loaded blocks will be tracked in a single file per tenant
	Path   string
	UserID string
}

type StreamBinaryReader 

type StreamBinaryReader struct {
	// contains filtered or unexported fields
}

func NewStreamBinaryReader 

func NewStreamBinaryReader(ctx context.Context, logger log.Logger, bkt objstore.BucketReader, dir string, id ulid.ULID, postingOffsetsInMemSampling int, metrics *StreamBinaryReaderMetrics, cfg Config) (*StreamBinaryReader, error)

NewStreamBinaryReader loads or builds new index-header if not present on disk.

func (*StreamBinaryReader) Close 

func (r *StreamBinaryReader) Close() error

func (*StreamBinaryReader) IndexVersion 

func (r *StreamBinaryReader) IndexVersion(context.Context) (int, error)

func (*StreamBinaryReader) LabelNames 

func (r *StreamBinaryReader) LabelNames(context.Context) ([]string, error)

func (*StreamBinaryReader) LabelValuesOffsets 

func (r *StreamBinaryReader) LabelValuesOffsets(ctx context.Context, name string, prefix string, filter func(string) bool) ([]streamindex.PostingListOffset, error)

func (*StreamBinaryReader) LookupSymbol 

func (r *StreamBinaryReader) LookupSymbol(_ context.Context, o uint32) (string, error)

func (*StreamBinaryReader) PostingsOffset 

func (r *StreamBinaryReader) PostingsOffset(_ context.Context, name string, value string) (index.Range, error)

func (*StreamBinaryReader) SymbolsReader 

func (r *StreamBinaryReader) SymbolsReader(context.Context) (streamindex.SymbolsReader, error)

type StreamBinaryReaderMetrics 

type StreamBinaryReaderMetrics struct {
	// contains filtered or unexported fields
}

func NewStreamBinaryReaderMetrics 

func NewStreamBinaryReaderMetrics(reg prometheus.Registerer) *StreamBinaryReaderMetrics

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.