storecache

package
v0.37.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 25, 2024 License: Apache-2.0 Imports: 36 Imported by: 5

Documentation

Index

Constants

View Source 
const (
	CacheTypePostings         string = "Postings"
	CacheTypeExpandedPostings string = "ExpandedPostings"
	CacheTypeSeries           string = "Series"
)

Variables

View Source 
var (
	DefaultInMemoryIndexCacheConfig = InMemoryIndexCacheConfig{
		MaxSize:     250 * 1024 * 1024,
		MaxItemSize: 125 * 1024 * 1024,
	}
)

Functions

func LabelMatchersToString added in v0.34.0 

func LabelMatchersToString(matchers []*labels.Matcher) string

LabelMatchersToString converts the given label matchers to string format.

func NewCachingBucketFromYaml added in v0.13.0 

func NewCachingBucketFromYaml(yamlContent []byte, bucket objstore.Bucket, logger log.Logger, reg prometheus.Registerer, r *route.Router, configPath string) (objstore.InstrumentedBucket, error)

NewCachingBucketFromYaml uses YAML configuration to create new caching bucket.

func ValidateEnabledItems added in v0.33.0 

func ValidateEnabledItems(enabledItems []string) error

Types

type BucketCacheProvider added in v0.13.0 

type BucketCacheProvider string

BucketCacheProvider is a type used to evaluate all bucket cache providers. 

const (
	InMemoryBucketCacheProvider   BucketCacheProvider = "IN-MEMORY"  // In-memory cache-provider for caching bucket.
	MemcachedBucketCacheProvider  BucketCacheProvider = "MEMCACHED"  // Memcached cache-provider for caching bucket.
	RedisBucketCacheProvider      BucketCacheProvider = "REDIS"      // Redis cache-provider for caching bucket.
	GroupcacheBucketCacheProvider BucketCacheProvider = "GROUPCACHE" // Groupcache cache-provider for caching bucket.
)

type CacheKey added in v0.34.0 

type CacheKey struct {
	Block string
	Key   interface{}

	Compression string
}

CacheKey defines cache key used in index cache.

func (CacheKey) KeyType added in v0.34.0 

func (c CacheKey) KeyType() string

KeyType returns cache key type.

func (CacheKey) Size added in v0.34.0 

func (c CacheKey) Size() uint64

Size returns the size bytes of the cache key.

func (CacheKey) String added in v0.34.0 

func (c CacheKey) String() string

type CacheKeyExpandedPostings added in v0.34.0 

type CacheKeyExpandedPostings string // We don't use []*labels.Matcher because it is not a hashable type so fail at inmemory cache.

type CacheKeyPostings added in v0.34.0 

type CacheKeyPostings labels.Label

type CacheKeySeries added in v0.34.0 

type CacheKeySeries uint64

type CachingBucket added in v0.13.0 

type CachingBucket struct {
	objstore.Bucket
	// contains filtered or unexported fields
}

CachingBucket implementation that provides some caching features, based on passed configuration.

func NewCachingBucket added in v0.13.0 

func NewCachingBucket(b objstore.Bucket, cfg *cache.CachingBucketConfig, logger log.Logger, reg prometheus.Registerer) (*CachingBucket, error)

NewCachingBucket creates new caching bucket with provided configuration. Configuration should not be changed after creating caching bucket.

func (*CachingBucket) Attributes added in v0.14.0 

func (cb *CachingBucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

func (*CachingBucket) Exists added in v0.13.0 

func (cb *CachingBucket) Exists(ctx context.Context, name string) (bool, error)

func (*CachingBucket) Get added in v0.13.0 

func (cb *CachingBucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

func (*CachingBucket) GetRange added in v0.13.0 

func (cb *CachingBucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

func (*CachingBucket) IsObjNotFoundErr added in v0.13.0 

func (cb *CachingBucket) IsObjNotFoundErr(err error) bool

func (*CachingBucket) Iter added in v0.13.0 

func (cb *CachingBucket) Iter(ctx context.Context, dir string, f func(string) error, options ...objstore.IterOption) error

func (*CachingBucket) Name added in v0.13.0 

func (cb *CachingBucket) Name() string

func (*CachingBucket) ReaderWithExpectedErrs added in v0.13.0 

func (cb *CachingBucket) ReaderWithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.BucketReader

func (*CachingBucket) WithExpectedErrs added in v0.13.0 

func (cb *CachingBucket) WithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.Bucket

type CachingWithBackendConfig added in v0.13.0 

type CachingWithBackendConfig struct {
	Type          BucketCacheProvider `yaml:"type"`
	BackendConfig interface{}         `yaml:"config"`

	// Basic unit used to cache chunks.
	ChunkSubrangeSize int64 `yaml:"chunk_subrange_size"`

	// Maximum number of GetRange requests issued by this bucket for single GetRange call. Zero or negative value = unlimited.
	MaxChunksGetRangeRequests int `yaml:"max_chunks_get_range_requests"`

	MetafileMaxSize model.Bytes `yaml:"metafile_max_size"`

	// TTLs for various cache items.
	ChunkObjectAttrsTTL time.Duration `yaml:"chunk_object_attrs_ttl"`
	ChunkSubrangeTTL    time.Duration `yaml:"chunk_subrange_ttl"`

	// How long to cache result of Iter call in root directory.
	BlocksIterTTL time.Duration `yaml:"blocks_iter_ttl"`

	// Config for Exists and Get operations for metadata files.
	MetafileExistsTTL      time.Duration `yaml:"metafile_exists_ttl"`
	MetafileDoesntExistTTL time.Duration `yaml:"metafile_doesnt_exist_ttl"`
	MetafileContentTTL     time.Duration `yaml:"metafile_content_ttl"`
}

CachingWithBackendConfig is a configuration of caching bucket used by Store component.

func (*CachingWithBackendConfig) Defaults added in v0.13.0 

func (cfg *CachingWithBackendConfig) Defaults()

type CommonMetrics added in v0.34.0 

type CommonMetrics struct {
	RequestTotal  *prometheus.CounterVec
	HitsTotal     *prometheus.CounterVec
	DataSizeBytes *prometheus.HistogramVec
	FetchLatency  *prometheus.HistogramVec
}

Common metrics that should be used by all cache implementations.

func NewCommonMetrics added in v0.34.0 

func NewCommonMetrics(reg prometheus.Registerer) *CommonMetrics

NewCommonMetrics initializes common metrics for index cache.

type FilteredIndexCache added in v0.33.0 

type FilteredIndexCache struct {
	// contains filtered or unexported fields
}

func NewFilteredIndexCache added in v0.33.0 

func NewFilteredIndexCache(cache IndexCache, enabledItems []string) *FilteredIndexCache

NewFilteredIndexCache creates a filtered index cache based on enabled items.

func (*FilteredIndexCache) FetchExpandedPostings added in v0.33.0 

func (c *FilteredIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) ([]byte, bool)

FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.

func (*FilteredIndexCache) FetchMultiPostings added in v0.33.0 

func (c *FilteredIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

FetchMultiPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys.

func (*FilteredIndexCache) FetchMultiSeries added in v0.33.0 

func (c *FilteredIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)

FetchMultiSeries fetches multiple series - each identified by ID - from the cache and returns a map containing cache hits, along with a list of missing IDs.

func (*FilteredIndexCache) StoreExpandedPostings added in v0.33.0 

func (c *FilteredIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string)

StoreExpandedPostings stores expanded postings for a set of label matchers.

func (*FilteredIndexCache) StorePostings added in v0.33.0 

func (c *FilteredIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

StorePostings sets the postings identified by the ulid and label to the value v, if the postings already exists in the cache it is not mutated.

func (*FilteredIndexCache) StoreSeries added in v0.33.0 

func (c *FilteredIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

StoreSeries sets the series identified by the ulid and id to the value v, if the series already exists in the cache it is not mutated.

type InMemoryIndexCache added in v0.10.0 

type InMemoryIndexCache struct {
	// contains filtered or unexported fields
}

func NewInMemoryIndexCache added in v0.10.0 

func NewInMemoryIndexCache(logger log.Logger, commonMetrics *CommonMetrics, reg prometheus.Registerer, conf []byte) (*InMemoryIndexCache, error)

NewInMemoryIndexCache creates a new thread-safe LRU cache for index entries and ensures the total cache size approximately does not exceed maxBytes.

func NewInMemoryIndexCacheWithConfig added in v0.10.0 

func NewInMemoryIndexCacheWithConfig(logger log.Logger, commonMetrics *CommonMetrics, reg prometheus.Registerer, config InMemoryIndexCacheConfig) (*InMemoryIndexCache, error)

NewInMemoryIndexCacheWithConfig creates a new thread-safe LRU cache for index entries and ensures the total cache size approximately does not exceed maxBytes.

func (*InMemoryIndexCache) FetchExpandedPostings added in v0.32.0 

func (c *InMemoryIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) ([]byte, bool)

FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.

func (*InMemoryIndexCache) FetchMultiPostings added in v0.10.0 

func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

FetchMultiPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys.

func (*InMemoryIndexCache) FetchMultiSeries added in v0.10.0 

func (c *InMemoryIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)

FetchMultiSeries fetches multiple series - each identified by ID - from the cache and returns a map containing cache hits, along with a list of missing IDs.

func (*InMemoryIndexCache) StoreExpandedPostings added in v0.32.0 

func (c *InMemoryIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string)

StoreExpandedPostings stores expanded postings for a set of label matchers.

func (*InMemoryIndexCache) StorePostings added in v0.10.0 

func (c *InMemoryIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

StorePostings sets the postings identified by the ulid and label to the value v, if the postings already exists in the cache it is not mutated.

func (*InMemoryIndexCache) StoreSeries added in v0.10.0 

func (c *InMemoryIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

StoreSeries sets the series identified by the ulid and id to the value v, if the series already exists in the cache it is not mutated.

type InMemoryIndexCacheConfig added in v0.10.0 

type InMemoryIndexCacheConfig struct {
	// MaxSize represents overall maximum number of bytes cache can contain.
	MaxSize model.Bytes `yaml:"max_size"`
	// MaxItemSize represents maximum size of single item.
	MaxItemSize model.Bytes `yaml:"max_item_size"`
}

InMemoryIndexCacheConfig holds the in-memory index cache config.

type IndexCache 

type IndexCache interface {
	// StorePostings stores postings for a single series.
	StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

	// FetchMultiPostings fetches multiple postings - each identified by a label -
	// and returns a map containing cache hits, along with a list of missing keys.
	FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

	// StoreExpandedPostings stores expanded postings for a set of label matchers.
	StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string)

	// FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.
	FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) ([]byte, bool)

	// StoreSeries stores a single series.
	StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

	// FetchMultiSeries fetches multiple series - each identified by ID - from the cache
	// and returns a map containing cache hits, along with a list of missing IDs.
	FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)
}

IndexCache is the interface exported by index cache backends. Store operations do not support context.Context, deadlines need to be supported by the backends themselves. This is because Set operations are run async and it does not make sense to attach same context (potentially with a deadline) as in the original user's request.

func NewIndexCache 

func NewIndexCache(logger log.Logger, confContentYaml []byte, reg prometheus.Registerer) (IndexCache, error)

NewIndexCache initializes and returns new index cache.

type IndexCacheConfig added in v0.10.0 

type IndexCacheConfig struct {
	Type   IndexCacheProvider `yaml:"type"`
	Config interface{}        `yaml:"config"`

	// Available item types are Postings, Series and ExpandedPostings.
	EnabledItems []string `yaml:"enabled_items"`
	// TTL for storing items in remote cache. Not supported for inmemory cache.
	// Default value is 24h.
	TTL time.Duration `yaml:"ttl"`
}

IndexCacheConfig specifies the index cache config.

type IndexCacheProvider added in v0.10.0 

type IndexCacheProvider string
const (
	INMEMORY  IndexCacheProvider = "IN-MEMORY"
	MEMCACHED IndexCacheProvider = "MEMCACHED"
	REDIS     IndexCacheProvider = "REDIS"
)

type JSONIterCodec added in v0.13.0 

type JSONIterCodec struct{}

JSONIterCodec encodes iter results into JSON. Suitable for root dir.

func (JSONIterCodec) Decode added in v0.13.0 

func (jic JSONIterCodec) Decode(data []byte) ([]string, error)

func (JSONIterCodec) Encode added in v0.13.0 

func (jic JSONIterCodec) Encode(files []string) ([]byte, error)

type RemoteIndexCache added in v0.25.0 

type RemoteIndexCache struct {
	// contains filtered or unexported fields
}

RemoteIndexCache is a memcached-based index cache.

func NewMemcachedIndexCache added in v0.10.0 

func NewMemcachedIndexCache(logger log.Logger, memcached cacheutil.RemoteCacheClient, reg prometheus.Registerer) (*RemoteIndexCache, error)

NewMemcachedIndexCache is alias NewRemoteIndexCache for compatible.

func NewRemoteIndexCache added in v0.25.0 

func NewRemoteIndexCache(logger log.Logger, cacheClient cacheutil.RemoteCacheClient, commonMetrics *CommonMetrics, reg prometheus.Registerer, ttl time.Duration) (*RemoteIndexCache, error)

NewRemoteIndexCache makes a new RemoteIndexCache.

func (*RemoteIndexCache) FetchExpandedPostings added in v0.32.0 

func (c *RemoteIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, lbls []*labels.Matcher, tenant string) ([]byte, bool)

FetchExpandedPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys. In case of error, it logs and return an empty cache hits map.

func (*RemoteIndexCache) FetchMultiPostings added in v0.25.0 

func (c *RemoteIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, lbls []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

FetchMultiPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys. In case of error, it logs and return an empty cache hits map.

func (*RemoteIndexCache) FetchMultiSeries added in v0.25.0 

func (c *RemoteIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)

FetchMultiSeries fetches multiple series - each identified by ID - from the cache and returns a map containing cache hits, along with a list of missing IDs. In case of error, it logs and return an empty cache hits map.

func (*RemoteIndexCache) StoreExpandedPostings added in v0.32.0 

func (c *RemoteIndexCache) StoreExpandedPostings(blockID ulid.ULID, keys []*labels.Matcher, v []byte, tenant string)

StoreExpandedPostings sets the postings identified by the ulid and label to the value v. The function enqueues the request and returns immediately: the entry will be asynchronously stored in the cache.

func (*RemoteIndexCache) StorePostings added in v0.25.0 

func (c *RemoteIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

StorePostings sets the postings identified by the ulid and label to the value v. The function enqueues the request and returns immediately: the entry will be asynchronously stored in the cache.

func (*RemoteIndexCache) StoreSeries added in v0.25.0 

func (c *RemoteIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

StoreSeries sets the series identified by the ulid and id to the value v. The function enqueues the request and returns immediately: the entry will be asynchronously stored in the cache.

type TracingIndexCache added in v0.34.0 

type TracingIndexCache struct {
	// contains filtered or unexported fields
}

func NewTracingIndexCache added in v0.34.0 

func NewTracingIndexCache(name string, cache IndexCache) *TracingIndexCache

NewTracingIndexCache creates an index cache wrapper with traces instrumentation.

func (*TracingIndexCache) FetchExpandedPostings added in v0.34.0 

func (c *TracingIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, matchers []*labels.Matcher, tenant string) (data []byte, exists bool)

FetchExpandedPostings fetches expanded postings and returns cached data and a boolean value representing whether it is a cache hit or not.

func (*TracingIndexCache) FetchMultiPostings added in v0.34.0 

func (c *TracingIndexCache) FetchMultiPostings(ctx context.Context, blockID ulid.ULID, keys []labels.Label, tenant string) (hits map[labels.Label][]byte, misses []labels.Label)

FetchMultiPostings fetches multiple postings - each identified by a label - and returns a map containing cache hits, along with a list of missing keys.

func (*TracingIndexCache) FetchMultiSeries added in v0.34.0 

func (c *TracingIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef, tenant string) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef)

FetchMultiSeries fetches multiple series - each identified by ID - from the cache and returns a map containing cache hits, along with a list of missing IDs.

func (*TracingIndexCache) StoreExpandedPostings added in v0.34.0 

func (c *TracingIndexCache) StoreExpandedPostings(blockID ulid.ULID, matchers []*labels.Matcher, v []byte, tenant string)

StoreExpandedPostings stores expanded postings for a set of label matchers.

func (*TracingIndexCache) StorePostings added in v0.34.0 

func (c *TracingIndexCache) StorePostings(blockID ulid.ULID, l labels.Label, v []byte, tenant string)

StorePostings stores postings for a single series.

func (*TracingIndexCache) StoreSeries added in v0.34.0 

func (c *TracingIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, v []byte, tenant string)

StoreSeries stores a single series. Skip instrumenting this method excessive spans as a single request can store millions of series.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.