tsdb

package
v1.0.0-rc.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 31, 2020 License: Apache-2.0 Imports: 28 Imported by: 3

Documentation

Index

Constants

View Source 
const (
	// BackendS3 is the value for the S3 storage backend
	BackendS3 = "s3"

	// BackendGCS is the value for the GCS storage backend
	BackendGCS = "gcs"

	// BackendAzure is the value for the Azure storage backend
	BackendAzure = "azure"

	// BackendFilesystem is the value for the filesystem storge backend
	BackendFilesystem = "filesystem"

	// TenantIDExternalLabel is the external label set when shipping blocks to the storage
	TenantIDExternalLabel = "__org_id__"
)
View Source 
const (
	// IndexCacheBackendInMemory is the value for the in-memory index cache backend.
	IndexCacheBackendInMemory = "inmemory"

	// IndexCacheBackendMemcached is the value for the memcached index cache backend.
	IndexCacheBackendMemcached = "memcached"

	// IndexCacheBackendDefault is the value for the default index cache backend.
	IndexCacheBackendDefault = IndexCacheBackendInMemory
)
View Source 
const (
	// DefaultRefCacheTTL is the default RefCache purge TTL. We use a reasonable
	// value that should cover most use cases. The cache would be ineffective if
	// the scrape interval of a series is greater than this TTL.
	DefaultRefCacheTTL = 10 * time.Minute
)

Variables

This section is empty.

Functions

func MustNewIndexCacheMetrics added in v1.0.0 

func MustNewIndexCacheMetrics(backend string, reg *prometheus.Registry) prometheus.Collector

func NewBucketClient 

func NewBucketClient(ctx context.Context, cfg Config, name string, logger log.Logger) (objstore.Bucket, error)

NewBucketClient creates a new bucket client based on the configured backend

func NewIndexCache added in v0.7.0 

func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error)

NewIndexCache creates a new index cache based on the input configuration.

Types

type BucketClientMock added in v0.6.0 

type BucketClientMock struct {
	mock.Mock
}

BucketClientMock mocks objstore.Bucket

func (*BucketClientMock) Close added in v0.6.0 

func (m *BucketClientMock) Close() error

Close mocks objstore.Bucket.Close()

func (*BucketClientMock) Delete added in v0.6.0 

func (m *BucketClientMock) Delete(ctx context.Context, name string) error

Delete mocks objstore.Bucket.Delete()

func (*BucketClientMock) Exists added in v0.6.0 

func (m *BucketClientMock) Exists(ctx context.Context, name string) (bool, error)

Exists mocks objstore.Bucket.Exists()

func (*BucketClientMock) Get added in v0.6.0 

func (m *BucketClientMock) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get mocks objstore.Bucket.Get()

func (*BucketClientMock) GetRange added in v0.6.0 

func (m *BucketClientMock) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange mocks objstore.Bucket.GetRange()

func (*BucketClientMock) IsObjNotFoundErr added in v0.6.0 

func (m *BucketClientMock) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr mocks objstore.Bucket.IsObjNotFoundErr()

func (*BucketClientMock) Iter added in v0.6.0 

func (m *BucketClientMock) Iter(ctx context.Context, dir string, f func(string) error) error

Iter mocks objstore.Bucket.Iter()

func (*BucketClientMock) MockDelete added in v1.0.0 

func (m *BucketClientMock) MockDelete(name string, err error)

func (*BucketClientMock) MockGet added in v0.6.0 

func (m *BucketClientMock) MockGet(name, content string, err error)

MockGet is a convenient method to mock Get() and Exists()

func (*BucketClientMock) MockIter added in v0.6.0 

func (m *BucketClientMock) MockIter(prefix string, objects []string, err error)

MockIter is a convenient method to mock Iter()

func (*BucketClientMock) Name added in v0.6.0 

func (m *BucketClientMock) Name() string

Name mocks objstore.Bucket.Name()

func (*BucketClientMock) ObjectSize added in v0.6.0 

func (m *BucketClientMock) ObjectSize(ctx context.Context, name string) (uint64, error)

ObjectSize mocks objstore.Bucket.ObjectSize()

func (*BucketClientMock) Upload added in v0.6.0 

func (m *BucketClientMock) Upload(ctx context.Context, name string, r io.Reader) error

Upload mocks objstore.Bucket.Upload()

type BucketStoreConfig added in v0.6.0 

type BucketStoreConfig struct {
	SyncDir                  string           `yaml:"sync_dir"`
	SyncInterval             time.Duration    `yaml:"sync_interval"`
	MaxChunkPoolBytes        uint64           `yaml:"max_chunk_pool_bytes"`
	MaxSampleCount           uint64           `yaml:"max_sample_count"`
	MaxConcurrent            int              `yaml:"max_concurrent"`
	TenantSyncConcurrency    int              `yaml:"tenant_sync_concurrency"`
	BlockSyncConcurrency     int              `yaml:"block_sync_concurrency"`
	MetaSyncConcurrency      int              `yaml:"meta_sync_concurrency"`
	BinaryIndexHeader        bool             `yaml:"binary_index_header_enabled"`
	ConsistencyDelay         time.Duration    `yaml:"consistency_delay"`
	IndexCache               IndexCacheConfig `yaml:"index_cache"`
	IgnoreDeletionMarksDelay time.Duration    `yaml:"ignore_deletion_mark_delay"`
}

BucketStoreConfig holds the config information for Bucket Stores used by the querier

func (*BucketStoreConfig) RegisterFlags added in v0.6.0 

func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the BucketStore flags

func (*BucketStoreConfig) Validate added in v1.0.0 

func (cfg *BucketStoreConfig) Validate() error

Validate the config.

type Config 

type Config struct {
	Dir                       string            `yaml:"dir"`
	BlockRanges               DurationList      `yaml:"block_ranges_period"`
	Retention                 time.Duration     `yaml:"retention_period"`
	ShipInterval              time.Duration     `yaml:"ship_interval"`
	ShipConcurrency           int               `yaml:"ship_concurrency"`
	Backend                   string            `yaml:"backend"`
	BucketStore               BucketStoreConfig `yaml:"bucket_store"`
	HeadCompactionInterval    time.Duration     `yaml:"head_compaction_interval"`
	HeadCompactionConcurrency int               `yaml:"head_compaction_concurrency"`
	StripeSize                int               `yaml:"stripe_size"`

	// MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup
	MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`

	// Backends
	S3         s3.Config         `yaml:"s3"`
	GCS        gcs.Config        `yaml:"gcs"`
	Azure      azure.Config      `yaml:"azure"`
	Filesystem filesystem.Config `yaml:"filesystem"`
}

Config holds the config information for TSDB storage

func (*Config) BlocksDir added in v0.6.0 

func (cfg *Config) BlocksDir(userID string) string

BlocksDir returns the directory path where TSDB blocks and wal should be stored by the ingester

func (*Config) RegisterFlags 

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the TSDB flags

func (*Config) Validate 

func (cfg *Config) Validate() error

Validate the config.

type DurationList 

type DurationList []time.Duration

DurationList is the block ranges for a tsdb

func (*DurationList) Set 

func (d *DurationList) Set(s string) error

Set implements the flag.Value interface

func (*DurationList) String 

func (d *DurationList) String() string

String implements the flag.Value interface

func (*DurationList) ToMilliseconds added in v0.6.0 

func (d *DurationList) ToMilliseconds() []int64

ToMilliseconds returns the duration list in milliseconds

type InMemoryIndexCacheConfig added in v1.0.0 

type InMemoryIndexCacheConfig struct {
	MaxSizeBytes uint64 `yaml:"max_size_bytes"`
}

func (*InMemoryIndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *InMemoryIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

type InMemoryIndexCacheMetrics added in v1.0.0 

type InMemoryIndexCacheMetrics struct {
	// contains filtered or unexported fields
}

InMemoryIndexCacheMetrics aggregates metrics exported by Thanos in-memory index cache and re-exports them as Cortex metrics.

func NewInMemoryIndexCacheMetrics added in v1.0.0 

func NewInMemoryIndexCacheMetrics(reg *prometheus.Registry) *InMemoryIndexCacheMetrics

NewInMemoryIndexCacheMetrics makes InMemoryIndexCacheMetrics.

func (*InMemoryIndexCacheMetrics) Collect added in v1.0.0 

func (m *InMemoryIndexCacheMetrics) Collect(out chan<- prometheus.Metric)

func (*InMemoryIndexCacheMetrics) Describe added in v1.0.0 

func (m *InMemoryIndexCacheMetrics) Describe(out chan<- *prometheus.Desc)

type IndexCacheConfig added in v1.0.0 

type IndexCacheConfig struct {
	Backend             string                    `yaml:"backend"`
	InMemory            InMemoryIndexCacheConfig  `yaml:"inmemory"`
	Memcached           MemcachedIndexCacheConfig `yaml:"memcached"`
	PostingsCompression bool                      `yaml:"postings_compression_enabled"`
}

func (*IndexCacheConfig) RegisterFlags added in v1.0.0 

func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet)

func (*IndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*IndexCacheConfig) Validate added in v1.0.0 

func (cfg *IndexCacheConfig) Validate() error

Validate the config.

type MemcachedIndexCacheConfig added in v1.0.0 

type MemcachedIndexCacheConfig struct {
	Addresses              string        `yaml:"addresses"`
	Timeout                time.Duration `yaml:"timeout"`
	MaxIdleConnections     int           `yaml:"max_idle_connections"`
	MaxAsyncConcurrency    int           `yaml:"max_async_concurrency"`
	MaxAsyncBufferSize     int           `yaml:"max_async_buffer_size"`
	MaxGetMultiConcurrency int           `yaml:"max_get_multi_concurrency"`
	MaxGetMultiBatchSize   int           `yaml:"max_get_multi_batch_size"`
	MaxItemSize            int           `yaml:"max_item_size"`
}

func (*MemcachedIndexCacheConfig) GetAddresses added in v1.0.0 

func (cfg *MemcachedIndexCacheConfig) GetAddresses() []string

func (*MemcachedIndexCacheConfig) RegisterFlagsWithPrefix added in v1.0.0 

func (cfg *MemcachedIndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string)

func (*MemcachedIndexCacheConfig) Validate added in v1.0.0 

func (cfg *MemcachedIndexCacheConfig) Validate() error

Validate the config.

type MemcachedIndexCacheMetrics added in v1.0.0 

type MemcachedIndexCacheMetrics struct {
	// contains filtered or unexported fields
}

MemcachedIndexCacheMetrics aggregates metrics exported by Thanos memcached index cache and re-exports them as Cortex metrics.

func NewMemcachedIndexCacheMetrics added in v1.0.0 

func NewMemcachedIndexCacheMetrics(reg *prometheus.Registry) *MemcachedIndexCacheMetrics

NewMemcachedIndexCacheMetrics makes MemcachedIndexCacheMetrics.

func (*MemcachedIndexCacheMetrics) Collect added in v1.0.0 

func (m *MemcachedIndexCacheMetrics) Collect(out chan<- prometheus.Metric)

func (*MemcachedIndexCacheMetrics) Describe added in v1.0.0 

func (m *MemcachedIndexCacheMetrics) Describe(out chan<- *prometheus.Desc)

type RefCache added in v0.7.0 

type RefCache struct {
	// contains filtered or unexported fields
}

RefCache is a single-tenant cache mapping a labels set with the reference ID in TSDB, in order to be able to append samples to the TSDB head without having to copy write request series labels each time (because the memory buffers used to unmarshal the write request is reused).

func NewRefCache added in v0.7.0 

func NewRefCache() *RefCache

NewRefCache makes a new RefCache.

func (*RefCache) Purge added in v0.7.0 

func (c *RefCache) Purge(keepUntil time.Time)

Purge removes expired entries from the cache. This function should be called periodically to avoid memory leaks.

func (*RefCache) Ref added in v0.7.0 

func (c *RefCache) Ref(now time.Time, series labels.Labels) (uint64, bool)

Ref returns the cached series reference, and guarantees the input labels set is NOT retained.

func (*RefCache) SetRef added in v0.7.0 

func (c *RefCache) SetRef(now time.Time, series labels.Labels, ref uint64)

SetRef sets/updates the cached series reference. The input labels set IS retained.

type UserBucketClient added in v0.6.0 

type UserBucketClient struct {
	// contains filtered or unexported fields
}

UserBucketClient is a wrapper around a objstore.Bucket that prepends writes with a userID

func NewUserBucketClient added in v0.6.0 

func NewUserBucketClient(userID string, bucket objstore.Bucket) *UserBucketClient

NewUserBucketClient makes a new UserBucketClient.

func (*UserBucketClient) Close added in v0.6.0 

func (b *UserBucketClient) Close() error

Close implements io.Closer

func (*UserBucketClient) Delete added in v0.6.0 

func (b *UserBucketClient) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*UserBucketClient) Exists added in v0.6.0 

func (b *UserBucketClient) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists in the bucket.

func (*UserBucketClient) Get added in v0.6.0 

func (b *UserBucketClient) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*UserBucketClient) GetRange added in v0.6.0 

func (b *UserBucketClient) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*UserBucketClient) IsObjNotFoundErr added in v0.6.0 

func (b *UserBucketClient) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*UserBucketClient) Iter added in v0.6.0 

func (b *UserBucketClient) Iter(ctx context.Context, dir string, f func(string) error) error

Iter calls f for each entry in the given directory (not recursive.). The argument to f is the full object name including the prefix of the inspected directory.

func (*UserBucketClient) Name added in v0.6.0 

func (b *UserBucketClient) Name() string

Name returns the bucket name for the provider.

func (*UserBucketClient) ObjectSize added in v0.6.0 

func (b *UserBucketClient) ObjectSize(ctx context.Context, name string) (uint64, error)

ObjectSize returns the size of the specified object.

func (*UserBucketClient) Upload added in v0.6.0 

func (b *UserBucketClient) Upload(ctx context.Context, name string, r io.Reader) error

Upload the contents of the reader as an object into the bucket.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.