common

Documentation

Index

• Constants
• Variables
• func BloomName(shard int) string
• func ShardKeyForSnapshotID(snapshotID []byte, shardCount int) int
• func ValidateConfig(b *BlockConfig) error
• func ValidateShardCount(shardCount int) int
• type BackendBlock
• type BlockConfig
• type CacheControl
• type CompactionOptions
• type Compactor
• type Finder
• type ID
• type IDMap
  • func NewIDMap[T any]() *IDMap[T]
  • func (m *IDMap[T]) EntriesSortedByID() []IDMapEntry[T]
  • func (m *IDMap[T]) Get(id ID) (T, bool)
  • func (m *IDMap[T]) Has(id ID) bool
  • func (m *IDMap[T]) Len() int
  • func (m *IDMap[T]) Set(id ID, val T)
• type IDMapEntry
• type Iterator
• type SearchOptions
  • func DefaultSearchOptions() SearchOptions
• type Searcher
• type ShardedBloomFilter
  • func NewBloom(fp float64, shardSize, estimatedObjects uint) *ShardedBloomFilter
  • func (b *ShardedBloomFilter) Add(snapshotID []byte)
  • func (b *ShardedBloomFilter) GetShardCount() int
  • func (b *ShardedBloomFilter) Marshal() ([][]byte, error)
  • func (b *ShardedBloomFilter) Test(snapshotID []byte) bool
• type TagCallback
• type TagCallbackV2
• type WALBlock

Constants

const (
	// NameObjects names the backend data object
	NameObjects = "data"
	// NameIndex names the backend index object
	NameIndex = "index"
)

Variables

var ErrUnsupported = errors.New("unsupported")

Functions

func BloomName

func BloomName(shard int) string

BloomName returns the backend bloom name for the given shard

func ShardKeyForSnapshotID

func ShardKeyForSnapshotID(snapshotID []byte, shardCount int) int

func ValidateConfig

func ValidateConfig(b *BlockConfig) error

ValidateConfig returns true if the config is valid

func ValidateShardCount

func ValidateShardCount(shardCount int) int

ValidateShardCount For backward compatibility

Types

type BackendBlock

type BackendBlock interface {
	Finder
	Searcher

	BlockMeta() *backend.BlockMeta
}

type BlockConfig

type BlockConfig struct {
	BloomFP             float64          `yaml:"bloom_filter_false_positive"`
	BloomShardSizeBytes int              `yaml:"bloom_filter_shard_size_bytes"`
	Version             string           `yaml:"version"`
	SearchEncoding      backend.Encoding `yaml:"search_encoding"`

	// v2 fields
	Encoding backend.Encoding `yaml:"v2_encoding"`

	// parquet fields
	RowGroupSizeBytes int `yaml:"parquet_row_group_size_bytes"`
}

BlockConfig holds configuration options for newly created blocks

type CacheControl

type CacheControl struct {
	Footer      bool
	ColumnIndex bool
	OffsetIndex bool
}

type CompactionOptions

type CompactionOptions struct {
	ChunkSizeBytes      uint32
	FlushSizeBytes      uint32
	IteratorBufferSize  int // How many snapshots to prefetch async.
	MaxBytesPerSnapshot int
	OutputBlocks        uint8
	BlockConfig         BlockConfig

	ObjectsWritten     func(compactionLevel, objects int)
	BytesWritten       func(compactionLevel, bytes int)
	SnapshotsDiscarded func(snapshotID string, count int)
}

type Compactor

type Compactor interface {
	Compact(ctx context.Context, l log.Logger, r backend.Reader, writerCallback func(*backend.BlockMeta, time.Time) backend.Writer, inputs []*backend.BlockMeta) ([]*backend.BlockMeta, error)
}

type Finder

type Finder interface {
	FindSnapshotByID(ctx context.Context, id ID, opts SearchOptions) (*deep_tp.Snapshot, error)
}

type ID

type ID []byte

ID in deepdb

type IDMap

type IDMap[T any] struct {
	// contains filtered or unexported fields
}

IDMap is a helper for recording and checking for IDs. Not safe for concurrent use.

func NewIDMap

func NewIDMap[T any]() *IDMap[T]

func (*IDMap[T]) EntriesSortedByID

func (m *IDMap[T]) EntriesSortedByID() []IDMapEntry[T]

func (*IDMap[T]) Get

func (m *IDMap[T]) Get(id ID) (T, bool)

func (*IDMap[T]) Has

func (m *IDMap[T]) Has(id ID) bool

func (*IDMap[T]) Len

func (m *IDMap[T]) Len() int

func (*IDMap[T]) Set

func (m *IDMap[T]) Set(id ID, val T)

type IDMapEntry

type IDMapEntry[T any] struct {
	ID    ID
	Entry T
}

type Iterator

type Iterator interface {
	Next(ctx context.Context) (ID, *deep_tp.Snapshot, error)
	Close()
}

type SearchOptions

type SearchOptions struct {
	ChunkSizeBytes        uint32 // Buffer size to read from backend storage.
	StartPage             int    // Controls searching only a subset of the block. Which page to begin searching at.
	TotalPages            int    // Controls searching only a subset of the block. How many pages to search.
	MaxBytes              int    // Max allowable snapshot size in bytes. Snapshots exceeding this are not searched.
	PrefetchSnapshotCount int    // How many snapshots to prefetch async.
	ReadBufferCount       int
	ReadBufferSize        int
	CacheControl          CacheControl
}

func DefaultSearchOptions

func DefaultSearchOptions() SearchOptions

DefaultSearchOptions is used in a lot of places such as local ingester searches. It is important in these cases to set a reasonable read buffer size and count to prevent constant tiny readranges against the local backend. TODO: Note that there is another method of creating "default search options" that looks like this: deepdb.SearchConfig{}.ApplyToOptions(&searchOpts). we should consolidate these.

type Searcher

type Searcher interface {
	Search(ctx context.Context, req *deeppb.SearchRequest, opts SearchOptions) (*deeppb.SearchResponse, error)
	SearchTags(ctx context.Context, cb TagCallback, opts SearchOptions) error
	SearchTagValues(ctx context.Context, tag string, cb TagCallback, opts SearchOptions) error
	SearchTagValuesV2(ctx context.Context, tag string, cb TagCallbackV2, opts SearchOptions) error

	Fetch(context.Context, deepql.FetchSnapshotRequest, SearchOptions) (deepql.FetchSnapshotResponse, error)
}

type ShardedBloomFilter

type ShardedBloomFilter struct {
	// contains filtered or unexported fields
}

func NewBloom

func NewBloom(fp float64, shardSize, estimatedObjects uint) *ShardedBloomFilter

NewBloom creates a ShardedBloomFilter

func (*ShardedBloomFilter) Add

func (b *ShardedBloomFilter) Add(snapshotID []byte)

func (*ShardedBloomFilter) GetShardCount

func (b *ShardedBloomFilter) GetShardCount() int

func (*ShardedBloomFilter) Marshal

func (b *ShardedBloomFilter) Marshal() ([][]byte, error)

Marshal is a wrapper around bloom.WriteTo

func (*ShardedBloomFilter) Test

func (b *ShardedBloomFilter) Test(snapshotID []byte) bool

Test implements bloom.Test -> required only for testing

type TagCallback

type TagCallback func(t string)

type TagCallbackV2

type TagCallbackV2 func(deepql.Static) (stop bool)

type WALBlock

type WALBlock interface {
	BackendBlock

	// Append the given snapshot to the block. Must be safe for concurrent use with read operations.
	Append(id ID, b []byte, start uint32) error

	// Flush any unbuffered data to disk.  Must be safe for concurrent use with read operations.
	Flush() error

	DataLength() uint64
	Iterator() (Iterator, error)
	Clear() error
}