chunk

package
v0.0.0-...-2371dfd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 10, 2017 License: Apache-2.0 Imports: 44 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ErrInvalidChunkID  = errors.Error("invalid chunk ID")
	ErrInvalidChecksum = errors.Error("invalid chunk checksum")
	ErrWrongMetadata   = errors.Error("wrong chunk metadata")
)

Errors that decode can return

Variables

View Source 
var (
	ErrNoMetricNameNotSupported = errors.New("metric name required for pre-v7 schemas")
)

Errors

Functions

func ChunksToMatrix 

func ChunksToMatrix(chunks []Chunk) (model.Matrix, error)

ChunksToMatrix converts a slice of chunks into a model.Matrix.

Types

type AWSStorageConfig 

type AWSStorageConfig struct {
	DynamoDBConfig
	S3 util.URLValue
}

AWSStorageConfig specifies config for storing data on AWS.

func (*AWSStorageConfig) RegisterFlags 

func (cfg *AWSStorageConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type Bucket 

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket describes a range of time with a tableName and hashKey

type ByKey 

type ByKey []Chunk

ByKey allow you to sort chunks by ID

func (ByKey) Len 

func (cs ByKey) Len() int

func (ByKey) Less 

func (cs ByKey) Less(i, j int) bool

func (ByKey) Swap 

func (cs ByKey) Swap(i, j int)

type Cache 

type Cache struct {
	// contains filtered or unexported fields
}

Cache type caches chunks

func NewCache 

func NewCache(cfg CacheConfig) *Cache

NewCache makes a new Cache

func (*Cache) BackgroundWrite 

func (c *Cache) BackgroundWrite(key string, buf []byte)

BackgroundWrite writes chunks for the cache in the background

func (*Cache) FetchChunkData 

func (c *Cache) FetchChunkData(ctx context.Context, chunks []Chunk) (found []Chunk, missing []Chunk, err error)

FetchChunkData gets chunks from the chunk cache.

func (*Cache) Stop 

func (c *Cache) Stop()

Stop the background flushing goroutines.

func (*Cache) StoreChunk 

func (c *Cache) StoreChunk(ctx context.Context, key string, buf []byte) error

StoreChunk serializes and stores a chunk in the chunk cache.

type CacheConfig 

type CacheConfig struct {
	Expiration          time.Duration
	WriteBackGoroutines int
	WriteBackBuffer     int
	// contains filtered or unexported fields
}

CacheConfig is config to make a Cache

func (*CacheConfig) RegisterFlags 

func (cfg *CacheConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type Chunk 

type Chunk struct {
	// These two fields will be missing from older chunks (as will the hash).
	// On fetch we will initialise these fields from the DynamoDB key.
	Fingerprint model.Fingerprint `json:"fingerprint"`
	UserID      string            `json:"userID"`

	// These fields will be in all chunks, including old ones.
	From    model.Time   `json:"from"`
	Through model.Time   `json:"through"`
	Metric  model.Metric `json:"metric"`

	// The hash is not written to the external storage either.  We use
	// crc32, Castagnoli table.  See http://www.evanjones.ca/crc32c.html.
	// For old chunks, ChecksumSet will be false.
	ChecksumSet bool   `json:"-"`
	Checksum    uint32 `json:"-"`

	// We never use Delta encoding (the zero value), so if this entry is
	// missing, we default to DoubleDelta.
	Encoding prom_chunk.Encoding `json:"encoding"`
	Data     prom_chunk.Chunk    `json:"-"`
	// contains filtered or unexported fields
}

Chunk contains encoded timeseries data

func NewChunk 

func NewChunk(userID string, fp model.Fingerprint, metric model.Metric, c prom_chunk.Chunk, from, through model.Time) Chunk

NewChunk creates a new chunk

type DynamoDBConfig 

type DynamoDBConfig struct {
	DynamoDB util.URLValue
	APILimit float64
}

DynamoDBConfig specifies config for a DynamoDB database.

func (*DynamoDBConfig) RegisterFlags 

func (cfg *DynamoDBConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type IndexEntry 

type IndexEntry struct {
	TableName string
	HashValue string

	// For writes, RangeValue will always be set.
	RangeValue []byte

	// New for v6 schema, label value is not written as part of the range key.
	Value []byte
}

IndexEntry describes an entry in the chunk index

type IndexQuery 

type IndexQuery struct {
	TableName string
	HashValue string

	// One of RangeValuePrefix or RangeValueStart might be set:
	// - If RangeValuePrefix is not nil, must read all keys with that prefix.
	// - If RangeValueStart is not nil, must read all keys from there onwards.
	// - If neither is set, must read all keys for that row.
	RangeValuePrefix []byte
	RangeValueStart  []byte

	// Filters for querying
	ValueEqual []byte
}

IndexQuery describes a query for entries

type Memcache 

type Memcache interface {
	GetMulti(keys []string) (map[string]*memcache.Item, error)
	Set(item *memcache.Item) error
}

Memcache caches things

type MemcacheClient 

type MemcacheClient struct {
	*memcache.Client
	// contains filtered or unexported fields
}

MemcacheClient is a memcache client that gets its server list from SRV records, and periodically updates that ServerList.

func NewMemcacheClient 

func NewMemcacheClient(cfg MemcacheConfig) *MemcacheClient

NewMemcacheClient creates a new MemcacheClient that gets its server list from SRV and updates the server list on a regular basis.

func (*MemcacheClient) Stop 

func (c *MemcacheClient) Stop()

Stop the memcache client.

type MemcacheConfig 

type MemcacheConfig struct {
	Host           string
	Service        string
	Timeout        time.Duration
	UpdateInterval time.Duration
}

MemcacheConfig defines how a MemcacheClient should be constructed.

func (*MemcacheConfig) RegisterFlags 

func (cfg *MemcacheConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type MockStorage 

type MockStorage struct {
	// contains filtered or unexported fields
}

MockStorage is a fake in-memory StorageClient.

func NewMockStorage 

func NewMockStorage() *MockStorage

NewMockStorage creates a new MockStorage.

func (*MockStorage) BatchWrite 

func (m *MockStorage) BatchWrite(ctx context.Context, batch WriteBatch) error

BatchWrite implements StorageClient.

func (*MockStorage) CreateTable 

func (m *MockStorage) CreateTable(_ context.Context, desc TableDesc) error

CreateTable implements StorageClient.

func (*MockStorage) DescribeTable 

func (m *MockStorage) DescribeTable(_ context.Context, name string) (desc TableDesc, status string, err error)

DescribeTable implements StorageClient.

func (*MockStorage) GetChunks 

func (m *MockStorage) GetChunks(ctx context.Context, chunkSet []Chunk) ([]Chunk, error)

GetChunks implements StorageClient.

func (*MockStorage) ListTables 

func (m *MockStorage) ListTables(_ context.Context) ([]string, error)

ListTables implements StorageClient.

func (*MockStorage) NewWriteBatch 

func (m *MockStorage) NewWriteBatch() WriteBatch

NewWriteBatch implements StorageClient.

func (*MockStorage) PutChunks 

func (m *MockStorage) PutChunks(_ context.Context, chunks []Chunk) error

PutChunks implements StorageClient.

func (*MockStorage) QueryPages 

func (m *MockStorage) QueryPages(ctx context.Context, query IndexQuery, callback func(result ReadBatch, lastPage bool) (shouldContinue bool)) error

QueryPages implements StorageClient.

func (*MockStorage) UpdateTable 

func (m *MockStorage) UpdateTable(_ context.Context, _, desc TableDesc) error

UpdateTable implements StorageClient.

type ReadBatch 

type ReadBatch interface {
	Len() int
	RangeValue(index int) []byte
	Value(index int) []byte
}

ReadBatch represents the results of a QueryPages.

type Schema 

type Schema interface {
	// When doing a write, use this method to return the list of entries you should write to.
	GetWriteEntries(from, through model.Time, userID string, metricName model.LabelValue, labels model.Metric, chunkID string) ([]IndexEntry, error)

	// When doing a read, use these methods to return the list of entries you should query
	GetReadQueries(from, through model.Time, userID string) ([]IndexQuery, error)
	GetReadQueriesForMetric(from, through model.Time, userID string, metricName model.LabelValue) ([]IndexQuery, error)
	GetReadQueriesForMetricLabel(from, through model.Time, userID string, metricName model.LabelValue, labelName model.LabelName) ([]IndexQuery, error)
	GetReadQueriesForMetricLabelValue(from, through model.Time, userID string, metricName model.LabelValue, labelName model.LabelName, labelValue model.LabelValue) ([]IndexQuery, error)
}

Schema interface defines methods to calculate the hash and range keys needed to write or read chunks from the external index.

type SchemaConfig 

type SchemaConfig struct {
	// After midnight on this day, we start bucketing indexes by day instead of by
	// hour.  Only the day matters, not the time within the day.
	DailyBucketsFrom util.DayValue
	Base64ValuesFrom util.DayValue
	V4SchemaFrom     util.DayValue
	V5SchemaFrom     util.DayValue
	V6SchemaFrom     util.DayValue
	V7SchemaFrom     util.DayValue

	// Period with which the table manager will poll for tables.
	DynamoDBPollInterval time.Duration

	// duration a table will be created before it is needed.
	CreationGracePeriod time.Duration
	MaxChunkAge         time.Duration

	// Config for the index & chunk tables.
	OriginalTableName string
	UsePeriodicTables bool
	IndexTables       periodicTableConfig
	ChunkTables       periodicTableConfig

	// Deprecated configuration for setting tags on all tables.
	Tags Tags
}

SchemaConfig contains the config for our chunk index schemas

func (*SchemaConfig) RegisterFlags 

func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type StorageClient 

type StorageClient interface {
	// For the write path.
	NewWriteBatch() WriteBatch
	BatchWrite(context.Context, WriteBatch) error

	// For the read path.
	QueryPages(ctx context.Context, query IndexQuery, callback func(result ReadBatch, lastPage bool) (shouldContinue bool)) error

	// For storing and retrieving chunks.
	PutChunks(ctx context.Context, chunks []Chunk) error
	GetChunks(ctx context.Context, chunks []Chunk) ([]Chunk, error)
}

StorageClient is a client for the persistent storage for Cortex. (e.g. DynamoDB + S3).

func NewAWSStorageClient 

func NewAWSStorageClient(cfg AWSStorageConfig, schemaCfg SchemaConfig) (StorageClient, error)

NewAWSStorageClient makes a new AWS-backed StorageClient.

func NewStorageClient 

func NewStorageClient(cfg StorageClientConfig, schemaCfg SchemaConfig) (StorageClient, error)

NewStorageClient makes a storage client based on the configuration.

type StorageClientConfig 

type StorageClientConfig struct {
	StorageClient string
	AWSStorageConfig
}

StorageClientConfig chooses which storage client to use.

func (*StorageClientConfig) RegisterFlags 

func (cfg *StorageClientConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

type Store 

type Store struct {
	// contains filtered or unexported fields
}

Store implements Store

func NewStore 

func NewStore(cfg StoreConfig, schemaCfg SchemaConfig, storage StorageClient) (*Store, error)

NewStore makes a new ChunkStore

func (*Store) Get 

func (c *Store) Get(ctx context.Context, from, through model.Time, allMatchers ...*metric.LabelMatcher) ([]Chunk, error)

Get implements ChunkStore

func (*Store) Put 

func (c *Store) Put(ctx context.Context, chunks []Chunk) error

Put implements ChunkStore

func (*Store) Stop 

func (c *Store) Stop()

Stop any background goroutines (ie in the cache.)

type StoreConfig 

type StoreConfig struct {
	CacheConfig
	// contains filtered or unexported fields
}

StoreConfig specifies config for a ChunkStore

func (*StoreConfig) RegisterFlags 

func (cfg *StoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type TableClient 

type TableClient interface {
	ListTables(ctx context.Context) ([]string, error)
	CreateTable(ctx context.Context, desc TableDesc) error
	DescribeTable(ctx context.Context, name string) (desc TableDesc, status string, err error)
	UpdateTable(ctx context.Context, current, expected TableDesc) error
}

TableClient is a client for telling Dynamo what to do with tables.

func NewDynamoDBTableClient 

func NewDynamoDBTableClient(cfg DynamoDBConfig) (TableClient, error)

NewDynamoDBTableClient makes a new DynamoTableClient.

type TableDesc 

type TableDesc struct {
	Name             string
	ProvisionedRead  int64
	ProvisionedWrite int64
	Tags             Tags
}

TableDesc describes a table.

func (TableDesc) Equals 

func (desc TableDesc) Equals(other TableDesc) bool

Equals returns true if other matches desc.

type TableManager 

type TableManager struct {
	// contains filtered or unexported fields
}

TableManager creates and manages the provisioned throughput on DynamoDB tables

func NewTableManager 

func NewTableManager(cfg SchemaConfig, tableClient TableClient) (*TableManager, error)

NewTableManager makes a new TableManager

func (*TableManager) Start 

func (m *TableManager) Start()

Start the TableManager

func (*TableManager) Stop 

func (m *TableManager) Stop()

Stop the TableManager

type Tags 

type Tags map[string]string

Tags is a string-string map that implements flag.Value.

func (Tags) AWSTags 

func (ts Tags) AWSTags() []*dynamodb.Tag

AWSTags converts ts into a []*dynamodb.Tag.

func (Tags) Equals 

func (ts Tags) Equals(other Tags) bool

Equals returns true is other matches ts.

func (*Tags) Set 

func (ts *Tags) Set(s string) error

Set implements flag.Value

func (Tags) String 

func (ts Tags) String() string

String implements flag.Value

type WriteBatch 

type WriteBatch interface {
	Add(tableName, hashValue string, rangeValue []byte, value []byte)
}

WriteBatch represents a batch of writes.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.