index

Documentation

Index

• Variables
• func NewStore() *store.IndexStore
• type Config
  • func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
• type Index
  • func NewIndex(logger log.Logger, s Store, cfg *Config) *Index
  • func (i *Index) FindBlocks(tx *bbolt.Tx, list *metastorev1.BlockList) ([]*metastorev1.BlockMeta, error)
  • func (i *Index) GetTenantStats(tenant string) *metastorev1.TenantStats
  • func (i *Index) Init(tx *bbolt.Tx) error
  • func (i *Index) InsertBlock(tx *bbolt.Tx, b *metastorev1.BlockMeta) error
  • func (i *Index) QueryMetadata(tx *bbolt.Tx, query MetadataQuery) iter.Iterator[*metastorev1.BlockMeta]
  • func (i *Index) ReplaceBlocks(tx *bbolt.Tx, compacted *metastorev1.CompactedBlocks) error
  • func (i *Index) Restore(tx *bbolt.Tx) error
• type InvalidQueryError
  • func (e *InvalidQueryError) Error() string
• type MetadataQuery
  • func (q *MetadataQuery) String() string
• type Store

Variables

var DefaultConfig = Config{
	PartitionDuration:     partitionDuration,
	QueryLookaroundPeriod: partitionDuration,
	PartitionCacheSize:    partitionTenantCacheSize,
}

var ErrBlockExists = fmt.Errorf("block already exists")

Functions

func NewStore

func NewStore() *store.IndexStore

Types

type Config

type Config struct {
	PartitionCacheSize    int `yaml:"partition_cache_size"`
	PartitionDuration     time.Duration
	QueryLookaroundPeriod time.Duration `yaml:"query_lookaround_period"`
}

func (*Config) RegisterFlagsWithPrefix

func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type Index

type Index struct {
	// contains filtered or unexported fields
}

func NewIndex

func NewIndex(logger log.Logger, s Store, cfg *Config) *Index

NewIndex initializes a new metastore index.

The index provides access to block metadata. The data is partitioned by time, shard and tenant. Partition identifiers contain the time period referenced by partitions, e.g., "20240923T16.1h" refers to a partition for the 1-hour period between 2024-09-23T16:00:00.000Z and 2024-09-23T16:59:59.999Z.

Partitions are mostly transparent for the end user, though PartitionMeta is at times used externally. Partition durations are configurable (at application level).

The index requires a backing Store for loading data in memory. Data is loaded directly via LoadPartitions() or when looking up blocks with FindBlock() or FindBlocksInRange().

func (*Index) FindBlocks

func (i *Index) FindBlocks(tx *bbolt.Tx, list *metastorev1.BlockList) ([]*metastorev1.BlockMeta, error)

func (*Index) GetTenantStats

func (i *Index) GetTenantStats(tenant string) *metastorev1.TenantStats

func (*Index) Init

func (i *Index) Init(tx *bbolt.Tx) error

func (*Index) InsertBlock

func (i *Index) InsertBlock(tx *bbolt.Tx, b *metastorev1.BlockMeta) error

func (*Index) QueryMetadata

func (i *Index) QueryMetadata(tx *bbolt.Tx, query MetadataQuery) iter.Iterator[*metastorev1.BlockMeta]

func (*Index) ReplaceBlocks

func (i *Index) ReplaceBlocks(tx *bbolt.Tx, compacted *metastorev1.CompactedBlocks) error

ReplaceBlocks removes source blocks from the index and inserts replacement blocks into the index. The intended usage is for block compaction. The replacement blocks could be added to the same or a different partition.

func (*Index) Restore

func (i *Index) Restore(tx *bbolt.Tx) error

type InvalidQueryError

type InvalidQueryError struct {
	Query MetadataQuery
	Err   error
}

func (*InvalidQueryError) Error

func (e *InvalidQueryError) Error() string

type MetadataQuery

type MetadataQuery struct {
	Expr      string
	StartTime time.Time
	EndTime   time.Time
	Tenant    []string
}

func (*MetadataQuery) String

func (q *MetadataQuery) String() string

type Store

type Store interface {
	CreateBuckets(*bbolt.Tx) error

	StoreBlock(tx *bbolt.Tx, shard *store.TenantShard, md *metastorev1.BlockMeta) error
	DeleteBlockList(*bbolt.Tx, store.PartitionKey, *metastorev1.BlockList) error

	ListPartitions(*bbolt.Tx) ([]*store.Partition, error)
	LoadTenantShard(*bbolt.Tx, store.PartitionKey, string, uint32) (*store.TenantShard, error)
}