storage

package
v3.3.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 18, 2024 License: AGPL-3.0 Imports: 68 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func IsBlockOverlapping 

func IsBlockOverlapping(b chunkenc.Block, with *LazyChunk, direction logproto.Direction) bool

func NewBucketClient 

func NewBucketClient(storageConfig Config) (index.BucketClient, error)

NewBucketClient makes a new bucket client based on the configuration.

func NewChunkClient 

func NewChunkClient(name, component string, cfg Config, schemaCfg config.SchemaConfig, cc congestion.Controller, registerer prometheus.Registerer, clientMetrics ClientMetrics, logger log.Logger) (client.Client, error)

NewChunkClient makes a new chunk.Client of the desired types.

func NewIndexClient 

func NewIndexClient(component string, periodCfg config.PeriodConfig, tableRange config.TableRange, cfg Config, schemaCfg config.SchemaConfig, limits StoreLimits, cm ClientMetrics, shardingStrategy indexgateway.ShardingStrategy, registerer prometheus.Registerer, logger log.Logger, metricsNamespace string) (index.Client, error)

NewIndexClient creates a new index client of the desired type specified in the PeriodConfig

func NewObjectClient 

func NewObjectClient(name, component string, cfg Config, clientMetrics ClientMetrics) (client.ObjectClient, error)

NewObjectClient makes a new StorageClient with the prefix in the front.

func NewTableClient 

func NewTableClient(name, component string, periodCfg config.PeriodConfig, cfg Config, cm ClientMetrics, registerer prometheus.Registerer, logger log.Logger) (index.TableClient, error)

NewTableClient makes a new table client based on the configuration.

func ResetBoltDBIndexClientsWithShipper 

func ResetBoltDBIndexClientsWithShipper()

ResetBoltDBIndexClientsWithShipper allows to reset the singletons. MUST ONLY BE USED IN TESTS

Types

type AsyncStore 

type AsyncStore struct {
	stores.Store
	// contains filtered or unexported fields
}

AsyncStore does querying to both ingesters and chunk store and combines the results after deduping them. This should be used when using an async store like boltdb-shipper. AsyncStore is meant to be used only in queriers or any other service other than ingesters. It should never be used in ingesters otherwise it would start spiraling around doing queries over and over again to other ingesters.

func NewAsyncStore 

func NewAsyncStore(cfg AsyncStoreCfg, store stores.Store, scfg config.SchemaConfig) *AsyncStore

func (*AsyncStore) GetChunks 

func (a *AsyncStore) GetChunks(ctx context.Context,
	userID string,
	from,
	through model.Time,
	predicate chunk.Predicate,
	storeChunksOverride *logproto.ChunkRefGroup,
) ([][]chunk.Chunk, []*fetcher.Fetcher, error)

func (*AsyncStore) GetShards added in v3.1.0 

func (a *AsyncStore) GetShards(
	ctx context.Context,
	userID string,
	from, through model.Time,
	targetBytesPerShard uint64,
	predicate chunk.Predicate,
) (*logproto.ShardsResponse, error)

func (*AsyncStore) Stats 

func (a *AsyncStore) Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*stats.Stats, error)

func (*AsyncStore) Volume 

func (a *AsyncStore) Volume(ctx context.Context, userID string, from, through model.Time, limit int32, targetLabels []string, aggregateBy string, matchers ...*labels.Matcher) (*logproto.VolumeResponse, error)

type AsyncStoreCfg 

type AsyncStoreCfg struct {
	IngesterQuerier IngesterQuerier
	// QueryIngestersWithin defines maximum lookback beyond which ingesters are not queried for chunk ids.
	QueryIngestersWithin time.Duration
}

type ChunkMetrics 

type ChunkMetrics struct {
	// contains filtered or unexported fields
}

func NewChunkMetrics 

func NewChunkMetrics(r prometheus.Registerer, maxBatchSize int) *ChunkMetrics

type ClientMetrics 

type ClientMetrics struct {
	AzureMetrics azure.BlobStorageMetrics
}

func NewClientMetrics 

func NewClientMetrics() ClientMetrics

func (*ClientMetrics) Unregister 

func (c *ClientMetrics) Unregister()

type Config 

type Config struct {
	AlibabaStorageConfig   alibaba.OssConfig         `yaml:"alibabacloud"`
	AWSStorageConfig       aws.StorageConfig         `yaml:"aws"`
	AzureStorageConfig     azure.BlobStorageConfig   `yaml:"azure"`
	BOSStorageConfig       baidubce.BOSStorageConfig `yaml:"bos"`
	GCPStorageConfig       gcp.Config                `` /* 151-byte string literal not displayed */
	GCSConfig              gcp.GCSConfig             `yaml:"gcs" doc:"description=Configures storing chunks in GCS. Required fields only required when gcs is defined in config."`
	CassandraStorageConfig cassandra.Config          `yaml:"cassandra" doc:"description=Deprecated: Configures storing chunks and/or the index in Cassandra."`
	BoltDBConfig           local.BoltDBConfig        `` /* 154-byte string literal not displayed */
	FSConfig               local.FSConfig            `` /* 170-byte string literal not displayed */
	Swift                  openstack.SwiftConfig     `yaml:"swift"`
	GrpcConfig             grpc.Config               `yaml:"grpc_store" doc:"deprecated"`
	Hedging                hedging.Config            `yaml:"hedging"`
	NamedStores            NamedStores               `yaml:"named_stores"`
	COSConfig              ibmcloud.COSConfig        `yaml:"cos"`
	IndexCacheValidity     time.Duration             `yaml:"index_cache_validity"`
	CongestionControl      congestion.Config         `yaml:"congestion_control,omitempty"`
	ObjectPrefix           string                    `` /* 133-byte string literal not displayed */

	IndexQueriesCacheConfig  cache.Config `yaml:"index_queries_cache_config"`
	DisableBroadIndexQueries bool         `yaml:"disable_broad_index_queries"`
	MaxParallelGetChunk      int          `yaml:"max_parallel_get_chunk"`

	UseThanosObjstore bool          `yaml:"use_thanos_objstore" doc:"hidden"`
	ObjectStore       bucket.Config `yaml:"object_store" doc:"hidden"`

	MaxChunkBatchSize   int                       `yaml:"max_chunk_batch_size"`
	BoltDBShipperConfig boltdb.IndexCfg           `` /* 220-byte string literal not displayed */
	TSDBShipperConfig   indexshipper.Config       `` /* 213-byte string literal not displayed */
	BloomShipperConfig  bloomshipperconfig.Config `` /* 213-byte string literal not displayed */

	// Config for using AsyncStore when using async index stores like `boltdb-shipper`.
	// It is required for getting chunk ids of recently flushed chunks from the ingesters.
	EnableAsyncStore bool          `yaml:"-"`
	AsyncStoreConfig AsyncStoreCfg `yaml:"-"`
}

Config chooses which storage client to use.

func (*Config) RegisterFlags 

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*Config) Validate 

func (cfg *Config) Validate() error

Validate config and returns error on failure

type IngesterQuerier 

type IngesterQuerier interface {
	GetChunkIDs(ctx context.Context, from, through model.Time, matchers ...*labels.Matcher) ([]string, error)
	Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*stats.Stats, error)
	Volume(ctx context.Context, userID string, from, through model.Time, limit int32, targetLabels []string, aggregateBy string, matchers ...*labels.Matcher) (*logproto.VolumeResponse, error)
}

type Instrumentable 

type Instrumentable interface {
	SetExtractorWrapper(wrapper lokilog.SampleExtractorWrapper)

	SetPipelineWrapper(wrapper lokilog.PipelineWrapper)
}

type LazyChunk 

type LazyChunk struct {
	Chunk   chunk.Chunk
	IsValid bool
	Fetcher *fetcher.Fetcher
	// contains filtered or unexported fields
}

LazyChunk loads the chunk when it is accessed.

func (*LazyChunk) IsOverlapping 

func (c *LazyChunk) IsOverlapping(with *LazyChunk, direction logproto.Direction) bool

func (*LazyChunk) Iterator 

func (c *LazyChunk) Iterator(
	ctx context.Context,
	from, through time.Time,
	direction logproto.Direction,
	pipeline log.StreamPipeline,
	nextChunk *LazyChunk,
) (iter.EntryIterator, error)

Iterator returns an entry iterator. The iterator returned will cache overlapping block's entries with the next chunk if passed. This way when we re-use them for ordering across batches we don't re-decompress the data again.

func (*LazyChunk) SampleIterator 

func (c *LazyChunk) SampleIterator(
	ctx context.Context,
	from, through time.Time,
	extractor log.StreamSampleExtractor,
	nextChunk *LazyChunk,
) (iter.SampleIterator, error)

SampleIterator returns an sample iterator. The iterator returned will cache overlapping block's entries with the next chunk if passed. This way when we re-use them for ordering across batches we don't re-decompress the data again.

type LokiStore 

type LokiStore struct {
	stores.Store
	// contains filtered or unexported fields
}

func NewStore 

func NewStore(cfg Config, storeCfg config.ChunkStoreConfig, schemaCfg config.SchemaConfig,
	limits StoreLimits, clientMetrics ClientMetrics, registerer prometheus.Registerer, logger log.Logger,
	metricsNamespace string,
) (*LokiStore, error)

NewStore creates a new Loki Store using configuration supplied.

func (*LokiStore) GetSchemaConfigs 

func (s *LokiStore) GetSchemaConfigs() []config.PeriodConfig

func (*LokiStore) SelectLogs 

func (s *LokiStore) SelectLogs(ctx context.Context, req logql.SelectLogParams) (iter.EntryIterator, error)

SelectLogs returns an iterator that will query the store for more chunks while iterating instead of fetching all chunks upfront for that request.

func (*LokiStore) SelectSamples 

func (s *LokiStore) SelectSamples(ctx context.Context, req logql.SelectSampleParams) (iter.SampleIterator, error)

func (*LokiStore) SelectSeries 

func (s *LokiStore) SelectSeries(ctx context.Context, req logql.SelectLogParams) ([]logproto.SeriesIdentifier, error)

func (*LokiStore) SetChunkFilterer 

func (s *LokiStore) SetChunkFilterer(chunkFilterer chunk.RequestChunkFilterer)

func (*LokiStore) SetExtractorWrapper 

func (s *LokiStore) SetExtractorWrapper(wrapper lokilog.SampleExtractorWrapper)

func (*LokiStore) SetPipelineWrapper 

func (s *LokiStore) SetPipelineWrapper(wrapper lokilog.PipelineWrapper)

type NamedAWSStorageConfig 

type NamedAWSStorageConfig aws.StorageConfig

func (*NamedAWSStorageConfig) UnmarshalYAML 

func (cfg *NamedAWSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedAWSStorageConfig) Validate 

func (cfg *NamedAWSStorageConfig) Validate() error

type NamedBOSStorageConfig 

type NamedBOSStorageConfig baidubce.BOSStorageConfig

func (*NamedBOSStorageConfig) UnmarshalYAML 

func (cfg *NamedBOSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedBlobStorageConfig 

type NamedBlobStorageConfig azure.BlobStorageConfig

func (*NamedBlobStorageConfig) UnmarshalYAML 

func (cfg *NamedBlobStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedBlobStorageConfig) Validate 

func (cfg *NamedBlobStorageConfig) Validate() error

type NamedCOSConfig 

type NamedCOSConfig ibmcloud.COSConfig

func (*NamedCOSConfig) UnmarshalYAML 

func (cfg *NamedCOSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedFSConfig 

type NamedFSConfig local.FSConfig

func (*NamedFSConfig) UnmarshalYAML 

func (cfg *NamedFSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedGCSConfig 

type NamedGCSConfig gcp.GCSConfig

func (*NamedGCSConfig) UnmarshalYAML 

func (cfg *NamedGCSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedOssConfig 

type NamedOssConfig alibaba.OssConfig

func (*NamedOssConfig) UnmarshalYAML 

func (cfg *NamedOssConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedStores 

type NamedStores struct {
	AWS          map[string]NamedAWSStorageConfig  `yaml:"aws"`
	Azure        map[string]NamedBlobStorageConfig `yaml:"azure"`
	BOS          map[string]NamedBOSStorageConfig  `yaml:"bos"`
	Filesystem   map[string]NamedFSConfig          `yaml:"filesystem"`
	GCS          map[string]NamedGCSConfig         `yaml:"gcs"`
	AlibabaCloud map[string]NamedOssConfig         `yaml:"alibabacloud"`
	Swift        map[string]NamedSwiftConfig       `yaml:"swift"`
	COS          map[string]NamedCOSConfig         `yaml:"cos"`
	// contains filtered or unexported fields
}

NamedStores helps configure additional object stores from a given storage provider

func (*NamedStores) Exists 

func (ns *NamedStores) Exists(name string) bool

func (*NamedStores) Validate 

func (ns *NamedStores) Validate() error

type NamedSwiftConfig 

type NamedSwiftConfig openstack.SwiftConfig

func (*NamedSwiftConfig) UnmarshalYAML 

func (cfg *NamedSwiftConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedSwiftConfig) Validate 

func (cfg *NamedSwiftConfig) Validate() error

type SchemaConfigProvider 

type SchemaConfigProvider interface {
	GetSchemaConfigs() []config.PeriodConfig
}

type SelectStore 

type SelectStore interface {
	SelectSamples(ctx context.Context, req logql.SelectSampleParams) (iter.SampleIterator, error)
	SelectLogs(ctx context.Context, req logql.SelectLogParams) (iter.EntryIterator, error)
	SelectSeries(ctx context.Context, req logql.SelectLogParams) ([]logproto.SeriesIdentifier, error)
}

type Store 

type Store interface {
	stores.Store
	SelectStore
	SchemaConfigProvider
	Instrumentable
}

type StoreLimits 

type StoreLimits interface {
	downloads.Limits
	stores.StoreLimits
	indexgateway.Limits
	CardinalityLimit(string) int
}

StoreLimits helps get Limits specific to Queries for Stores

Source Files

View all Source files

Directories

Path Synopsis
bloom
v1
v1/filter
azure
filesystem
gcs
http
s3
swift
cache
cache/resultscache
client
client/alibaba
client/aws
client/azure
client/baidubce
client/cassandra
client/congestion
client/gcp
client/grpc
client/hedging
client/ibmcloud
client/local
client/openstack
client/testutils
client/util
fetcher
common
aws
index
index/seriesvolume
index/stats
series
series/index
shipper/bloomshipper
shipper/bloomshipper/config
package bloomshipperconfig resides in its own package to prevent circular imports with storage package
 package bloomshipperconfig resides in its own package to prevent circular imports with storage package
shipper/indexshipper
shipper/indexshipper/boltdb
shipper/indexshipper/boltdb/compactor
shipper/indexshipper/downloads
shipper/indexshipper/index
shipper/indexshipper/storage
shipper/indexshipper/testutil
shipper/indexshipper/tsdb
shipper/indexshipper/tsdb/index
shipper/indexshipper/tsdb/sharding
shipper/indexshipper/tsdb/testutil
shipper/indexshipper/uploads
shipper/indexshipper/util
wal
chunks
Package chunks provides functionality for efficient storage and retrieval of log data and metrics.
 Package chunks provides functionality for efficient storage and retrieval of log data and metrics.
index

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.