storage

package
v2.4.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 21, 2022 License: AGPL-3.0 Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	StorageEngineChunks = "chunks"
	StorageEngineBlocks = "blocks"
)

Supported storage engines

View Source 
const (
	StorageTypeAWS            = "aws"
	StorageTypeAWSDynamo      = "aws-dynamo"
	StorageTypeAzure          = "azure"
	StorageTypeBoltDB         = "boltdb"
	StorageTypeCassandra      = "cassandra"
	StorageTypeInMemory       = "inmemory"
	StorageTypeBigTable       = "bigtable"
	StorageTypeBigTableHashed = "bigtable-hashed"
	StorageTypeFileSystem     = "filesystem"
	StorageTypeGCP            = "gcp"
	StorageTypeGCPColumnKey   = "gcp-columnkey"
	StorageTypeGCS            = "gcs"
	StorageTypeGrpc           = "grpc-store"
	StorageTypeS3             = "s3"
	StorageTypeSwift          = "swift"
)

Supported storage clients

Variables

View Source 
var (
	ErrInvalidLengthCachingIndexClient = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowCachingIndexClient   = fmt.Errorf("proto: integer overflow")
)
View Source 
var Fixtures = []testutils.Fixture{
	fixture{gcp.Fixtures[0]},
}

Fixtures for unit testing the caching storage.

Functions

func NewBucketClient 

func NewBucketClient(storageConfig Config) (chunk.BucketClient, error)

NewBucketClient makes a new bucket client based on the configuration.

func NewChunkClient 

func NewChunkClient(name string, cfg Config, schemaCfg chunk.SchemaConfig, clientMetrics ClientMetrics, registerer prometheus.Registerer) (chunk.Client, error)

NewChunkClient makes a new chunk.Client of the desired types.

func NewIndexClient 

func NewIndexClient(name string, cfg Config, schemaCfg chunk.SchemaConfig, limits StoreLimits, registerer prometheus.Registerer) (chunk.IndexClient, error)

NewIndexClient makes a new index client of the desired type.

func NewObjectClient 

func NewObjectClient(name string, cfg Config, clientMetrics ClientMetrics) (chunk.ObjectClient, error)

NewObjectClient makes a new StorageClient of the desired types.

func NewStore 

func NewStore(
	cfg Config,
	storeCfg chunk.StoreConfig,
	schemaCfg chunk.SchemaConfig,
	limits StoreLimits,
	clientMetrics ClientMetrics,
	reg prometheus.Registerer,
	cacheGenNumLoader chunk.CacheGenNumLoader,
	logger log.Logger,
) (chunk.Store, error)

NewStore makes the storage clients based on the configuration.

func NewTableClient 

func NewTableClient(name string, cfg Config, registerer prometheus.Registerer) (chunk.TableClient, error)

NewTableClient makes a new table client based on the configuration.

func RegisterIndexStore 

func RegisterIndexStore(name string, indexClientFactory IndexClientFactoryFunc, tableClientFactory TableClientFactoryFunc)

RegisterIndexStore is used for registering a custom index type. When an index type is registered here with same name as existing types, the registered one takes the precedence.

Types

type Bytes 

type Bytes []byte

Bytes exists to stop proto copying the byte array

func (*Bytes) Compare 

func (bs *Bytes) Compare(other Bytes) int

Compare Bytes to other

func (*Bytes) Equal 

func (bs *Bytes) Equal(other Bytes) bool

Equal returns true if other equals Bytes

func (*Bytes) Marshal 

func (bs *Bytes) Marshal() ([]byte, error)

Marshal just returns bs

func (*Bytes) MarshalTo 

func (bs *Bytes) MarshalTo(data []byte) (n int, err error)

MarshalTo copies Bytes to data

func (*Bytes) Size 

func (bs *Bytes) Size() int

Size returns the length of Bytes

func (*Bytes) Unmarshal 

func (bs *Bytes) Unmarshal(data []byte) error

Unmarshal updates Bytes to be data, without a copy

type ClientMetrics 

type ClientMetrics struct {
	AzureMetrics azure.BlobStorageMetrics
}

func NewClientMetrics 

func NewClientMetrics() ClientMetrics

func (*ClientMetrics) Unregister 

func (c *ClientMetrics) Unregister()

type Config 

type Config struct {
	Engine                 string                  `yaml:"engine"`
	AWSStorageConfig       aws.StorageConfig       `yaml:"aws"`
	AzureStorageConfig     azure.BlobStorageConfig `yaml:"azure"`
	GCPStorageConfig       gcp.Config              `yaml:"bigtable"`
	GCSConfig              gcp.GCSConfig           `yaml:"gcs"`
	CassandraStorageConfig cassandra.Config        `yaml:"cassandra"`
	BoltDBConfig           local.BoltDBConfig      `yaml:"boltdb"`
	FSConfig               local.FSConfig          `yaml:"filesystem"`
	Swift                  openstack.SwiftConfig   `yaml:"swift"`

	IndexCacheValidity time.Duration `yaml:"index_cache_validity"`

	IndexQueriesCacheConfig  cache.Config `yaml:"index_queries_cache_config"`
	DisableBroadIndexQueries bool         `yaml:"disable_broad_index_queries"`
	MaxParallelGetChunk      int          `yaml:"max_parallel_get_chunk"`

	GrpcConfig grpc.Config `yaml:"grpc_store"`

	Hedging hedging.Config `yaml:"hedging"`
}

Config chooses which storage client to use.

func (*Config) RegisterFlags 

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*Config) Validate 

func (cfg *Config) Validate() error

Validate config and returns error on failure

type Entry 

type Entry struct {
	Column Bytes `protobuf:"bytes,1,opt,name=Column,proto3,customtype=Bytes" json:"Column"`
	Value  Bytes `protobuf:"bytes,2,opt,name=Value,proto3,customtype=Bytes" json:"Value"`
}

func (*Entry) Descriptor 

func (*Entry) Descriptor() ([]byte, []int)

func (*Entry) Equal 

func (this *Entry) Equal(that interface{}) bool

func (*Entry) GoString 

func (this *Entry) GoString() string

func (*Entry) Marshal 

func (m *Entry) Marshal() (dAtA []byte, err error)

func (*Entry) MarshalTo 

func (m *Entry) MarshalTo(dAtA []byte) (int, error)

func (*Entry) MarshalToSizedBuffer 

func (m *Entry) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*Entry) ProtoMessage 

func (*Entry) ProtoMessage()

func (*Entry) Reset 

func (m *Entry) Reset()

func (*Entry) Size 

func (m *Entry) Size() (n int)

func (*Entry) String 

func (this *Entry) String() string

func (*Entry) Unmarshal 

func (m *Entry) Unmarshal(dAtA []byte) error

func (*Entry) XXX_DiscardUnknown 

func (m *Entry) XXX_DiscardUnknown()

func (*Entry) XXX_Marshal 

func (m *Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Entry) XXX_Merge 

func (m *Entry) XXX_Merge(src proto.Message)

func (*Entry) XXX_Size 

func (m *Entry) XXX_Size() int

func (*Entry) XXX_Unmarshal 

func (m *Entry) XXX_Unmarshal(b []byte) error

type IndexClientFactoryFunc 

type IndexClientFactoryFunc func(limits StoreLimits) (chunk.IndexClient, error)

IndexClientFactoryFunc defines signature of function which creates chunk.IndexClient for managing index in index store

type ReadBatch 

type ReadBatch struct {
	Entries []Entry `protobuf:"bytes,1,rep,name=entries,proto3" json:"entries"`
	Key     string  `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
	// The time at which the key expires.
	Expiry int64 `protobuf:"varint,3,opt,name=expiry,proto3" json:"expiry,omitempty"`
	// The number of entries; used for cardinality limiting.
	// entries will be empty when this is set.
	Cardinality int32 `protobuf:"varint,4,opt,name=cardinality,proto3" json:"cardinality,omitempty"`
}

func (*ReadBatch) Descriptor 

func (*ReadBatch) Descriptor() ([]byte, []int)

func (*ReadBatch) Equal 

func (this *ReadBatch) Equal(that interface{}) bool

func (*ReadBatch) GetCardinality 

func (m *ReadBatch) GetCardinality() int32

func (*ReadBatch) GetEntries 

func (m *ReadBatch) GetEntries() []Entry

func (*ReadBatch) GetExpiry 

func (m *ReadBatch) GetExpiry() int64

func (*ReadBatch) GetKey 

func (m *ReadBatch) GetKey() string

func (*ReadBatch) GoString 

func (this *ReadBatch) GoString() string

func (ReadBatch) Iterator 

func (b ReadBatch) Iterator() chunk.ReadBatchIterator

Iterator implements chunk.ReadBatch.

func (*ReadBatch) Marshal 

func (m *ReadBatch) Marshal() (dAtA []byte, err error)

func (*ReadBatch) MarshalTo 

func (m *ReadBatch) MarshalTo(dAtA []byte) (int, error)

func (*ReadBatch) MarshalToSizedBuffer 

func (m *ReadBatch) MarshalToSizedBuffer(dAtA []byte) (int, error)

func (*ReadBatch) ProtoMessage 

func (*ReadBatch) ProtoMessage()

func (*ReadBatch) Reset 

func (m *ReadBatch) Reset()

func (*ReadBatch) Size 

func (m *ReadBatch) Size() (n int)

func (*ReadBatch) String 

func (this *ReadBatch) String() string

func (*ReadBatch) Unmarshal 

func (m *ReadBatch) Unmarshal(dAtA []byte) error

func (*ReadBatch) XXX_DiscardUnknown 

func (m *ReadBatch) XXX_DiscardUnknown()

func (*ReadBatch) XXX_Marshal 

func (m *ReadBatch) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*ReadBatch) XXX_Merge 

func (m *ReadBatch) XXX_Merge(src proto.Message)

func (*ReadBatch) XXX_Size 

func (m *ReadBatch) XXX_Size() int

func (*ReadBatch) XXX_Unmarshal 

func (m *ReadBatch) XXX_Unmarshal(b []byte) error

type StoreLimits 

type StoreLimits interface {
	downloads.Limits
	CardinalityLimit(userID string) int
	MaxChunksPerQueryFromStore(userID string) int
	MaxQueryLength(userID string) time.Duration
}

StoreLimits helps get Limits specific to Queries for Stores

type TableClientFactoryFunc 

type TableClientFactoryFunc func() (chunk.TableClient, error)

TableClientFactoryFunc defines signature of function which creates chunk.TableClient for managing tables in index store

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.