bloomshipper

package
v3.3.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 18, 2024 License: AGPL-3.0 Imports: 45 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BloomPrefix  = "bloom"
	MetasPrefix  = "metas"
	BlocksPrefix = "blocks"
)

Variables

This section is empty.

Functions

func LoadBlocksDirIntoCache 

func LoadBlocksDirIntoCache(paths []string, c Cache, logger log.Logger) error

func ParseTime 

func ParseTime(s string) (model.Time, error)

ParseFingerprint parses the input string into a model.Time.

Types

type Block 

type Block struct {
	BlockRef
	Data io.ReadSeekCloser
}

func BlockFrom 

func BlockFrom(enc compression.Codec, tenant, table string, blk *v1.Block) (Block, error)

type BlockClient 

type BlockClient interface {
	KeyResolver
	GetBlock(ctx context.Context, ref BlockRef) (BlockDirectory, error)
	GetBlocks(ctx context.Context, refs []BlockRef) ([]BlockDirectory, error)
	PutBlock(ctx context.Context, block Block) error
	DeleteBlocks(ctx context.Context, refs []BlockRef) error
}

type BlockDirectory 

type BlockDirectory struct {
	BlockRef
	Path string
	// contains filtered or unexported fields
}

A BlockDirectory is a local file path that contains a bloom block. It maintains a counter for currently active readers.

func NewBlockDirectory 

func NewBlockDirectory(ref BlockRef, path string) BlockDirectory

NewBlockDirectory creates a new BlockDirectory. Must exist on disk.

func (BlockDirectory) Block 

func (b BlockDirectory) Block(metrics *v1.Metrics) *v1.Block

func (BlockDirectory) BlockQuerier 

func (b BlockDirectory) BlockQuerier(
	alloc mempool.Allocator,
	close func() error,
	maxPageSize int,
	metrics *v1.Metrics,
) *CloseableBlockQuerier

BlockQuerier returns a new block querier from the directory. The passed function `close` is called when the the returned querier is closed.

func (BlockDirectory) Size 

func (b BlockDirectory) Size() int64

type BlockRef 

type BlockRef struct {
	Ref
	compression.Codec
}

func BlockRefFromKey added in v3.1.0 

func BlockRefFromKey(k string) (BlockRef, error)

func BlocksForMetas 

func BlocksForMetas(metas []Meta, interval Interval, keyspaces []v1.FingerprintBounds) (refs []BlockRef)

BlocksForMetas returns all the blocks from all the metas listed that are within the requested bounds

func (BlockRef) String 

func (r BlockRef) String() string

type BlocksCache 

type BlocksCache struct {
	// contains filtered or unexported fields
}

BlocksCache is an in-memory cache that manages block directories on the filesystem.

func NewFsBlocksCache 

func NewFsBlocksCache(cfg config.BlocksCacheConfig, reg prometheus.Registerer, logger log.Logger) *BlocksCache

NewFsBlocksCache returns a new file-system mapping cache for bloom blocks, where entries map block directories on disk.

func (*BlocksCache) Get 

func (c *BlocksCache) Get(ctx context.Context, key string) (BlockDirectory, bool)

Get implements Cache. Get returns the stored value against the given key.

func (*BlocksCache) Put 

func (c *BlocksCache) Put(ctx context.Context, key string, value BlockDirectory) error

Put implements Cache. It stores a value with given key.

func (*BlocksCache) PutInc 

func (c *BlocksCache) PutInc(ctx context.Context, key string, value BlockDirectory) error

PutInc implements Cache. It stores a value with given key and increments the ref counter on that item.

func (*BlocksCache) PutMany 

func (c *BlocksCache) PutMany(ctx context.Context, keys []string, values []BlockDirectory) error

PutMany implements Cache.

func (*BlocksCache) Release 

func (c *BlocksCache) Release(ctx context.Context, key string) error

Release decrements the ref counter on the cached item with given key.

func (*BlocksCache) Stop 

func (c *BlocksCache) Stop()

Stop implements Cache.

type BloomClient 

type BloomClient struct {
	KeyResolver
	// contains filtered or unexported fields
}

func NewBloomClient 

func NewBloomClient(cfg bloomStoreConfig, client client.ObjectClient, logger log.Logger) (*BloomClient, error)

func (*BloomClient) DeleteBlocks 

func (b *BloomClient) DeleteBlocks(ctx context.Context, references []BlockRef) error

func (*BloomClient) DeleteMetas 

func (b *BloomClient) DeleteMetas(ctx context.Context, refs []MetaRef) error

func (*BloomClient) GetBlock 

func (b *BloomClient) GetBlock(ctx context.Context, ref BlockRef) (BlockDirectory, error)

GetBlock downloads the blocks from objectStorage and returns the directory in which the block data resides

func (*BloomClient) GetBlocks 

func (b *BloomClient) GetBlocks(ctx context.Context, refs []BlockRef) ([]BlockDirectory, error)

func (*BloomClient) GetMeta 

func (b *BloomClient) GetMeta(ctx context.Context, ref MetaRef) (Meta, error)

GetMeta fetches the meta file for given MetaRef from object storage and decodes the JSON data into a Meta. If the meta file is not found in storage or decoding fails, the empty Meta is returned along with the error.

func (*BloomClient) GetMetas 

func (b *BloomClient) GetMetas(ctx context.Context, refs []MetaRef) ([]Meta, error)

func (*BloomClient) IsObjectNotFoundErr 

func (b *BloomClient) IsObjectNotFoundErr(err error) bool

func (*BloomClient) ObjectClient 

func (b *BloomClient) ObjectClient() client.ObjectClient

func (*BloomClient) PutBlock 

func (b *BloomClient) PutBlock(ctx context.Context, block Block) error

func (*BloomClient) PutMeta 

func (b *BloomClient) PutMeta(ctx context.Context, meta Meta) error

func (*BloomClient) Stop 

func (b *BloomClient) Stop()

type BloomStore 

type BloomStore struct {
	// contains filtered or unexported fields
}

func NewBloomStore 

func NewBloomStore(
	periodicConfigs []config.PeriodConfig,
	storageConfig storage.Config,
	clientMetrics storage.ClientMetrics,
	metasCache cache.Cache,
	blocksCache Cache,
	allocator mempool.Allocator,
	reg prometheus.Registerer,
	logger log.Logger,
) (*BloomStore, error)

func (*BloomStore) Allocator added in v3.2.0 

func (b *BloomStore) Allocator() mempool.Allocator

Allocator implements Store.

func (*BloomStore) Block 

func (b *BloomStore) Block(ref BlockRef) (loc Location)

Impements KeyResolver

func (*BloomStore) BloomMetrics 

func (b *BloomStore) BloomMetrics() *v1.Metrics

func (*BloomStore) Client 

func (b *BloomStore) Client(ts model.Time) (Client, error)

Client implements Store.

func (*BloomStore) FetchBlocks 

func (b *BloomStore) FetchBlocks(ctx context.Context, blocks []BlockRef, opts ...FetchOption) ([]*CloseableBlockQuerier, error)

FetchBlocks implements Store.

func (*BloomStore) FetchMetas 

func (b *BloomStore) FetchMetas(ctx context.Context, params MetaSearchParams) ([]Meta, error)

FetchMetas implements Store.

func (*BloomStore) Fetcher 

func (b *BloomStore) Fetcher(ts model.Time) (*Fetcher, error)

Fetcher implements Store.

func (*BloomStore) Meta 

func (b *BloomStore) Meta(ref MetaRef) (loc Location)

Impements KeyResolver

func (BloomStore) ParseBlockKey 

func (BloomStore) ParseBlockKey(loc Location) (BlockRef, error)

func (BloomStore) ParseMetaKey 

func (BloomStore) ParseMetaKey(loc Location) (MetaRef, error)

func (*BloomStore) ResolveMetas 

func (b *BloomStore) ResolveMetas(ctx context.Context, params MetaSearchParams) ([][]MetaRef, []*Fetcher, error)

ResolveMetas implements Store.

func (*BloomStore) Stop 

func (b *BloomStore) Stop()

Stop implements Store.

func (BloomStore) Tenant 

func (BloomStore) Tenant(tenant, table string) Location

func (*BloomStore) TenantFilesForInterval 

func (b *BloomStore) TenantFilesForInterval(
	ctx context.Context,
	interval Interval,
	filter func(tenant string, object client.StorageObject) bool,
) (map[string][]client.StorageObject, error)

func (BloomStore) TenantPrefix 

func (BloomStore) TenantPrefix(loc Location) (string, error)

type Cache 

type Cache interface {
	Put(ctx context.Context, key string, value BlockDirectory) error
	PutInc(ctx context.Context, key string, value BlockDirectory) error
	PutMany(ctx context.Context, keys []string, values []BlockDirectory) error
	Get(ctx context.Context, key string) (BlockDirectory, bool)
	Release(ctx context.Context, key string) error
	Stop()
}

type Client 

type Client interface {
	MetaClient
	BlockClient
	IsObjectNotFoundErr(err error) bool
	ObjectClient() client.ObjectClient
	Stop()
}

type ClosableReadSeekerAdapter 

type ClosableReadSeekerAdapter struct {
	io.ReadSeeker
}

CloseableReadSeekerAdapter is a wrapper around io.ReadSeeker to make it io.Closer if it doesn't already implement it.

func (ClosableReadSeekerAdapter) Close 

func (c ClosableReadSeekerAdapter) Close() error

type CloseableBlockQuerier 

type CloseableBlockQuerier struct {
	BlockRef
	*v1.BlockQuerier
	// contains filtered or unexported fields
}

func (*CloseableBlockQuerier) Close 

func (c *CloseableBlockQuerier) Close() error

func (*CloseableBlockQuerier) SeriesIter 

func (c *CloseableBlockQuerier) SeriesIter() (iter.PeekIterator[*v1.SeriesWithBlooms], error)

type Entry 

type Entry struct {
	Key   string
	Value BlockDirectory
	// contains filtered or unexported fields
}

type FetchOption 

type FetchOption func(opts *options)

func WithFetchAsync 

func WithFetchAsync(v bool) FetchOption

func WithIgnoreNotFound 

func WithIgnoreNotFound(v bool) FetchOption

func WithPool 

func WithPool(v mempool.Allocator) FetchOption

type Fetcher 

type Fetcher struct {
	// contains filtered or unexported fields
}

func NewFetcher 

func NewFetcher(
	cfg bloomStoreConfig,
	client Client,
	metasCache cache.Cache,
	blocksCache Cache,
	reg prometheus.Registerer,
	logger log.Logger,
	bloomMetrics *v1.Metrics,
) (*Fetcher, error)

func (*Fetcher) Close 

func (f *Fetcher) Close()

func (*Fetcher) FetchBlocks 

func (f *Fetcher) FetchBlocks(ctx context.Context, refs []BlockRef, opts ...FetchOption) ([]*CloseableBlockQuerier, error)

FetchBlocks implements fetcher

func (*Fetcher) FetchMetas 

func (f *Fetcher) FetchMetas(ctx context.Context, refs []MetaRef) ([]Meta, error)

FetchMetas implements fetcher

type ForEachBlockCallback 

type ForEachBlockCallback func(bq *v1.BlockQuerier, bounds v1.FingerprintBounds) error

type Interface 

type Interface interface {
	ForEach(ctx context.Context, tenant string, blocks []BlockRef, callback ForEachBlockCallback) error
	Stop()
}

type Interval 

type Interval struct {
	Start, End model.Time
}

Interval defines a time range with start end end time where the start is inclusive, the end is non-inclusive.

func NewInterval 

func NewInterval(start, end model.Time) Interval

func ParseIntervalFromAddr 

func ParseIntervalFromAddr(s string) (Interval, error)

ParseBoundsFromAddr parses a fingerprint bounds from a string Does not support negative times (times prior to Unix epoch).

func ParseIntervalFromParts 

func ParseIntervalFromParts(a, b string) (Interval, error)

ParseIntervalFromParts parses a fingerprint bounds already separated strings

func (Interval) Cmp 

func (i Interval) Cmp(ts model.Time) v1.BoundsCheck

Cmp returns the position of a time relative to the interval

func (Interval) Hash 

func (i Interval) Hash(h hash.Hash32) error

func (Interval) Overlaps 

func (i Interval) Overlaps(target Interval) bool

Overlaps returns whether the interval overlaps (partially) with the target interval

func (Interval) Repr 

func (i Interval) Repr() string

func (Interval) String 

func (i Interval) String() string

func (Interval) Within 

func (i Interval) Within(target Interval) bool

Within returns whether the interval is fully within the target interval

type KeyResolver 

type KeyResolver interface {
	Meta(MetaRef) Location
	ParseMetaKey(Location) (MetaRef, error)
	Block(BlockRef) Location
	ParseBlockKey(Location) (BlockRef, error)
	Tenant(tenant, table string) Location
	TenantPrefix(loc Location) (string, error)
}

KeyResolver is an interface for resolving keys to locations. This is used to determine where items are stored in object storage _and_ on disk. Using an interface allows us to abstract away platform specifics (e.g. OS path-specifics, object storage difference, etc) TODO(owen-d): implement resolvers that schema-aware, allowing us to change the locations of data across schema boundaries (for instance to upgrade|improve).

func NewPrefixedResolver 

func NewPrefixedResolver(prefix string, resolver KeyResolver) KeyResolver

func NewShardedPrefixedResolver 

func NewShardedPrefixedResolver(prefixes []string, resolver KeyResolver) (KeyResolver, error)

type Location 

type Location interface {
	Addr() string      // object storage location
	LocalPath() string // local path version
}

type Meta 

type Meta struct {
	MetaRef `json:"-"`

	// The specific TSDB files used to generate the block.
	Sources []tsdb.SingleTenantTSDBIdentifier

	// A list of blocks that were generated
	Blocks []BlockRef
}

todo rename it

func FilterMetasOverlappingBounds added in v3.1.0 

func FilterMetasOverlappingBounds(metas []Meta, bounds v1.FingerprintBounds) []Meta

FilterMetasOverlappingBounds filters metas that are within the given bounds. the input metas are expected to be sorted by fingerprint.

func (Meta) MostRecentSource 

func (m Meta) MostRecentSource() (tsdb.SingleTenantTSDBIdentifier, bool)

type MetaClient 

type MetaClient interface {
	KeyResolver
	GetMeta(ctx context.Context, ref MetaRef) (Meta, error)
	GetMetas(ctx context.Context, refs []MetaRef) ([]Meta, error)
	PutMeta(ctx context.Context, meta Meta) error
	DeleteMetas(ctx context.Context, refs []MetaRef) error
}

type MetaRef 

type MetaRef struct {
	Ref
}

func MetaRefFrom 

func MetaRefFrom(
	tenant,
	table string,
	bounds v1.FingerprintBounds,
	sources []tsdb.SingleTenantTSDBIdentifier,
	blocks []BlockRef,
) (MetaRef, error)

func MetaRefFromKey added in v3.1.0 

func MetaRefFromKey(k string) (MetaRef, error)

func (MetaRef) String 

func (r MetaRef) String() string

type MetaSearchParams 

type MetaSearchParams struct {
	TenantID string
	Interval Interval
	Keyspace v1.FingerprintBounds
}

type PrefixedResolver 

type PrefixedResolver struct {
	KeyResolver
	// contains filtered or unexported fields
}

func (PrefixedResolver) Block 

func (p PrefixedResolver) Block(ref BlockRef) Location

func (PrefixedResolver) Meta 

func (p PrefixedResolver) Meta(ref MetaRef) Location

type Ref 

type Ref struct {
	TenantID                     string
	TableName                    string
	Bounds                       v1.FingerprintBounds
	StartTimestamp, EndTimestamp model.Time
	Checksum                     uint32
}

func (Ref) Cmp 

func (r Ref) Cmp(fp uint64) v1.BoundsCheck

Cmp returns the fingerprint's position relative to the bounds

func (Ref) Hash 

func (r Ref) Hash(h hash.Hash32) error

Hash hashes the ref NB(owen-d): we don't include the tenant in the hash as it's not included in the data and leaving it out gives flexibility for migrating data between tenants

func (Ref) Interval 

func (r Ref) Interval() Interval

type ShardedPrefixedResolver 

type ShardedPrefixedResolver struct {
	KeyResolver
	// contains filtered or unexported fields
}

func (ShardedPrefixedResolver) Block 

func (r ShardedPrefixedResolver) Block(ref BlockRef) Location

func (ShardedPrefixedResolver) Meta 

func (r ShardedPrefixedResolver) Meta(ref MetaRef) Location

type Shipper 

type Shipper struct {
	// contains filtered or unexported fields
}

func NewShipper 

func NewShipper(client StoreBase) *Shipper

func (*Shipper) ForEach 

func (s *Shipper) ForEach(ctx context.Context, refs []BlockRef, callback ForEachBlockCallback) error

ForEach is a convenience function that wraps the store's FetchBlocks function and automatically closes the block querier once the callback was run.

func (*Shipper) Stop 

func (s *Shipper) Stop()

type Store 

type Store interface {
	StoreBase
	BloomMetrics() *v1.Metrics
	Allocator() mempool.Allocator
}

type StoreBase added in v3.2.0 

type StoreBase interface {
	ResolveMetas(ctx context.Context, params MetaSearchParams) ([][]MetaRef, []*Fetcher, error)
	FetchMetas(ctx context.Context, params MetaSearchParams) ([]Meta, error)
	FetchBlocks(ctx context.Context, refs []BlockRef, opts ...FetchOption) ([]*CloseableBlockQuerier, error)
	TenantFilesForInterval(
		ctx context.Context, interval Interval,
		filter func(tenant string, object client.StorageObject) bool,
	) (map[string][]client.StorageObject, error)
	Fetcher(ts model.Time) (*Fetcher, error)
	Client(ts model.Time) (Client, error)
	Stop()
}

Source Files

View all Source files

Directories

Path Synopsis
package bloomshipperconfig resides in its own package to prevent circular imports with storage package
 package bloomshipperconfig resides in its own package to prevent circular imports with storage package

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.