storage

Documentation

Index

• func IsBlockOverlapping(b chunkenc.Block, with *LazyChunk, direction logproto.Direction) bool
• func NewBucketClient(storageConfig Config) (index.BucketClient, error)
• func NewChunkClient(name string, cfg Config, schemaCfg config.SchemaConfig, ...) (client.Client, error)
• func NewIndexClient(periodCfg config.PeriodConfig, tableRange config.TableRange, cfg Config, ...) (index.Client, error)
• func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (client.ObjectClient, error)
• func NewTableClient(name string, cfg Config, cm ClientMetrics, registerer prometheus.Registerer) (index.TableClient, error)
• func ResetBoltDBIndexClientsWithShipper()
• type AsyncStore
  • func NewAsyncStore(cfg AsyncStoreCfg, store Store, scfg config.SchemaConfig) *AsyncStore
  • func (a *AsyncStore) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, ...) ([][]chunk.Chunk, []*fetcher.Fetcher, error)
  • func (a *AsyncStore) LabelVolume(ctx context.Context, userID string, from, through model.Time, limit int32, ...) (*logproto.LabelVolumeResponse, error)
  • func (a *AsyncStore) Stats(ctx context.Context, userID string, from, through model.Time, ...) (*stats.Stats, error)
• type AsyncStoreCfg
• type ChunkMetrics
  • func NewChunkMetrics(r prometheus.Registerer, maxBatchSize int) *ChunkMetrics
• type ClientMetrics
  • func NewClientMetrics() ClientMetrics
  • func (c *ClientMetrics) Unregister()
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type IngesterQuerier
• type LazyChunk
  • func (c *LazyChunk) IsOverlapping(with *LazyChunk, direction logproto.Direction) bool
  • func (c *LazyChunk) Iterator(ctx context.Context, from, through time.Time, direction logproto.Direction, ...) (iter.EntryIterator, error)
  • func (c *LazyChunk) SampleIterator(ctx context.Context, from, through time.Time, ...) (iter.SampleIterator, error)
• type NamedAWSStorageConfig
  • func (cfg *NamedAWSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
  • func (cfg *NamedAWSStorageConfig) Validate() error
• type NamedBOSStorageConfig
  • func (cfg *NamedBOSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
• type NamedBlobStorageConfig
  • func (cfg *NamedBlobStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
  • func (cfg *NamedBlobStorageConfig) Validate() error
• type NamedCOSConfig
  • func (cfg *NamedCOSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
• type NamedFSConfig
  • func (cfg *NamedFSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
• type NamedGCSConfig
  • func (cfg *NamedGCSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
• type NamedOssConfig
  • func (cfg *NamedOssConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
• type NamedStores
• type NamedSwiftConfig
  • func (cfg *NamedSwiftConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
  • func (cfg *NamedSwiftConfig) Validate() error
• type Store
  • func NewStore(cfg Config, storeCfg config.ChunkStoreConfig, schemaCfg config.SchemaConfig, ...) (Store, error)
• type StoreLimits

Functions

func IsBlockOverlapping

func IsBlockOverlapping(b chunkenc.Block, with *LazyChunk, direction logproto.Direction) bool

func NewBucketClient

func NewBucketClient(storageConfig Config) (index.BucketClient, error)

NewBucketClient makes a new bucket client based on the configuration.

func NewChunkClient

func NewChunkClient(name string, cfg Config, schemaCfg config.SchemaConfig, clientMetrics ClientMetrics, registerer prometheus.Registerer) (client.Client, error)

NewChunkClient makes a new chunk.Client of the desired types.

func NewIndexClient

func NewIndexClient(periodCfg config.PeriodConfig, tableRange config.TableRange, cfg Config, schemaCfg config.SchemaConfig, limits StoreLimits, cm ClientMetrics, ownsTenantFn downloads.IndexGatewayOwnsTenant, registerer prometheus.Registerer, logger log.Logger) (index.Client, error)

NewIndexClient makes a new index client of the desired type.

func NewObjectClient

func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (client.ObjectClient, error)

NewObjectClient makes a new StorageClient of the desired types.

func NewTableClient

func NewTableClient(name string, cfg Config, cm ClientMetrics, registerer prometheus.Registerer) (index.TableClient, error)

NewTableClient makes a new table client based on the configuration.

func ResetBoltDBIndexClientsWithShipper

func ResetBoltDBIndexClientsWithShipper()

ResetBoltDBIndexClientsWithShipper allows to reset the singletons. MUST ONLY BE USED IN TESTS

Types

type AsyncStore

type AsyncStore struct {
	Store
	// contains filtered or unexported fields
}

AsyncStore does querying to both ingesters and chunk store and combines the results after deduping them. This should be used when using an async store like boltdb-shipper. AsyncStore is meant to be used only in queriers or any other service other than ingesters. It should never be used in ingesters otherwise it would start spiraling around doing queries over and over again to other ingesters.

func NewAsyncStore

func NewAsyncStore(cfg AsyncStoreCfg, store Store, scfg config.SchemaConfig) *AsyncStore

func (*AsyncStore) GetChunkRefs

func (a *AsyncStore) GetChunkRefs(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) ([][]chunk.Chunk, []*fetcher.Fetcher, error)

func (*AsyncStore) LabelVolume

func (a *AsyncStore) LabelVolume(ctx context.Context, userID string, from, through model.Time, limit int32, matchers ...*labels.Matcher) (*logproto.LabelVolumeResponse, error)

func (*AsyncStore) Stats

func (a *AsyncStore) Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*stats.Stats, error)

type AsyncStoreCfg

type AsyncStoreCfg struct {
	IngesterQuerier IngesterQuerier
	// QueryIngestersWithin defines maximum lookback beyond which ingesters are not queried for chunk ids.
	QueryIngestersWithin time.Duration
}

type ChunkMetrics

type ChunkMetrics struct {
	// contains filtered or unexported fields
}

func NewChunkMetrics

func NewChunkMetrics(r prometheus.Registerer, maxBatchSize int) *ChunkMetrics

type ClientMetrics

type ClientMetrics struct {
	AzureMetrics azure.BlobStorageMetrics
}

func NewClientMetrics

func NewClientMetrics() ClientMetrics

func (*ClientMetrics) Unregister

func (c *ClientMetrics) Unregister()

type Config

type Config struct {
	AlibabaStorageConfig   alibaba.OssConfig         `yaml:"alibabacloud"`
	AWSStorageConfig       aws.StorageConfig         `yaml:"aws"`
	AzureStorageConfig     azure.BlobStorageConfig   `yaml:"azure"`
	BOSStorageConfig       baidubce.BOSStorageConfig `yaml:"bos"`
	GCPStorageConfig       gcp.Config                `` /* 139-byte string literal not displayed */
	GCSConfig              gcp.GCSConfig             `yaml:"gcs" doc:"description=Configures storing chunks in GCS. Required fields only required when gcs is defined in config."`
	CassandraStorageConfig cassandra.Config          `yaml:"cassandra" doc:"description=Configures storing chunks and/or the index in Cassandra."`
	BoltDBConfig           local.BoltDBConfig        `` /* 142-byte string literal not displayed */
	FSConfig               local.FSConfig            `` /* 170-byte string literal not displayed */
	Swift                  openstack.SwiftConfig     `yaml:"swift"`
	GrpcConfig             grpc.Config               `yaml:"grpc_store"`
	Hedging                hedging.Config            `yaml:"hedging"`
	NamedStores            NamedStores               `yaml:"named_stores"`
	COSConfig              ibmcloud.COSConfig        `yaml:"cos"`
	IndexCacheValidity     time.Duration             `yaml:"index_cache_validity"`

	IndexQueriesCacheConfig  cache.Config `yaml:"index_queries_cache_config"`
	DisableBroadIndexQueries bool         `yaml:"disable_broad_index_queries"`
	MaxParallelGetChunk      int          `yaml:"max_parallel_get_chunk"`

	MaxChunkBatchSize   int                 `yaml:"max_chunk_batch_size"`
	BoltDBShipperConfig shipper.Config      `` /* 220-byte string literal not displayed */
	TSDBShipperConfig   indexshipper.Config `yaml:"tsdb_shipper"`

	// Config for using AsyncStore when using async index stores like `boltdb-shipper`.
	// It is required for getting chunk ids of recently flushed chunks from the ingesters.
	AsyncStoreConfig AsyncStoreCfg `yaml:"-"`
}

Config chooses which storage client to use.

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*Config) Validate

func (cfg *Config) Validate() error

Validate config and returns error on failure

type IngesterQuerier

type IngesterQuerier interface {
	GetChunkIDs(ctx context.Context, from, through model.Time, matchers ...*labels.Matcher) ([]string, error)
	Stats(ctx context.Context, userID string, from, through model.Time, matchers ...*labels.Matcher) (*stats.Stats, error)
	LabelVolume(ctx context.Context, userID string, from, through model.Time, limit int32, matchers ...*labels.Matcher) (*logproto.LabelVolumeResponse, error)
}

type LazyChunk

type LazyChunk struct {
	Chunk   chunk.Chunk
	IsValid bool
	Fetcher *fetcher.Fetcher
	// contains filtered or unexported fields
}

LazyChunk loads the chunk when it is accessed.

func (*LazyChunk) IsOverlapping

func (c *LazyChunk) IsOverlapping(with *LazyChunk, direction logproto.Direction) bool

func (*LazyChunk) Iterator

func (c *LazyChunk) Iterator(
	ctx context.Context,
	from, through time.Time,
	direction logproto.Direction,
	pipeline log.StreamPipeline,
	nextChunk *LazyChunk,
) (iter.EntryIterator, error)

Iterator returns an entry iterator. The iterator returned will cache overlapping block's entries with the next chunk if passed. This way when we re-use them for ordering across batches we don't re-decompress the data again.

func (*LazyChunk) SampleIterator

func (c *LazyChunk) SampleIterator(
	ctx context.Context,
	from, through time.Time,
	extractor log.StreamSampleExtractor,
	nextChunk *LazyChunk,
) (iter.SampleIterator, error)

SampleIterator returns an sample iterator. The iterator returned will cache overlapping block's entries with the next chunk if passed. This way when we re-use them for ordering across batches we don't re-decompress the data again.

type NamedAWSStorageConfig

type NamedAWSStorageConfig aws.StorageConfig

func (*NamedAWSStorageConfig) UnmarshalYAML

func (cfg *NamedAWSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedAWSStorageConfig) Validate

func (cfg *NamedAWSStorageConfig) Validate() error

type NamedBOSStorageConfig

type NamedBOSStorageConfig baidubce.BOSStorageConfig

func (*NamedBOSStorageConfig) UnmarshalYAML

func (cfg *NamedBOSStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedBlobStorageConfig

type NamedBlobStorageConfig azure.BlobStorageConfig

func (*NamedBlobStorageConfig) UnmarshalYAML

func (cfg *NamedBlobStorageConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedBlobStorageConfig) Validate

func (cfg *NamedBlobStorageConfig) Validate() error

type NamedCOSConfig

type NamedCOSConfig ibmcloud.COSConfig

func (*NamedCOSConfig) UnmarshalYAML

func (cfg *NamedCOSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedFSConfig

type NamedFSConfig local.FSConfig

func (*NamedFSConfig) UnmarshalYAML

func (cfg *NamedFSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedGCSConfig

type NamedGCSConfig gcp.GCSConfig

func (*NamedGCSConfig) UnmarshalYAML

func (cfg *NamedGCSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedOssConfig

type NamedOssConfig alibaba.OssConfig

func (*NamedOssConfig) UnmarshalYAML

func (cfg *NamedOssConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

type NamedStores

type NamedStores struct {
	AWS          map[string]NamedAWSStorageConfig  `yaml:"aws"`
	Azure        map[string]NamedBlobStorageConfig `yaml:"azure"`
	BOS          map[string]NamedBOSStorageConfig  `yaml:"bos"`
	Filesystem   map[string]NamedFSConfig          `yaml:"filesystem"`
	GCS          map[string]NamedGCSConfig         `yaml:"gcs"`
	AlibabaCloud map[string]NamedOssConfig         `yaml:"alibabacloud"`
	Swift        map[string]NamedSwiftConfig       `yaml:"swift"`
	COS          map[string]NamedCOSConfig         `yaml:"cos"`
	// contains filtered or unexported fields
}

NamedStores helps configure additional object stores from a given storage provider

type NamedSwiftConfig

type NamedSwiftConfig openstack.SwiftConfig

func (*NamedSwiftConfig) UnmarshalYAML

func (cfg *NamedSwiftConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*NamedSwiftConfig) Validate

func (cfg *NamedSwiftConfig) Validate() error

type Store

type Store interface {
	stores.Store
	stores.ChunkReader
	index.Filterable
	GetSchemaConfigs() []config.PeriodConfig
}

func NewStore

func NewStore(cfg Config, storeCfg config.ChunkStoreConfig, schemaCfg config.SchemaConfig,
	limits StoreLimits, clientMetrics ClientMetrics, registerer prometheus.Registerer, logger log.Logger,
) (Store, error)

NewStore creates a new Loki Store using configuration supplied.

type StoreLimits

type StoreLimits interface {
	downloads.Limits
	stores.StoreLimits
	CardinalityLimit(string) int
}

StoreLimits helps get Limits specific to Queries for Stores