Documentation ¶
Overview ¶
Package tsm1 provides a TSDB in the Time Structured Merge tree format.
Index ¶
- Constants
- Variables
- func AddMeasurementToExpr(measurement string, base influxql.Expr) influxql.Expr
- func AppendSeriesFieldKeyBytes(dst, seriesKey, field []byte) []byte
- func BlockCount(block []byte) int
- func BlockType(block []byte) (byte, error)
- func BlockTypeName(typ byte) string
- func BlockTypeToFieldType(typ byte) cursors.FieldType
- func BlockTypeToInfluxQLDataType(typ byte) influxql.DataType
- func BooleanArrayDecodeAll(b []byte, dst []bool) ([]bool, error)
- func BooleanArrayEncodeAll(src []bool, b []byte) ([]byte, error)
- func CollectionToValues(collection *tsdb.SeriesCollection) (map[string][]Value, error)
- func CountTimestamps(b []byte) int
- func DecodeBooleanArrayBlock(block []byte, a *cursors.BooleanArray) error
- func DecodeFloatArrayBlock(block []byte, a *cursors.FloatArray) error
- func DecodeIntegerArrayBlock(block []byte, a *cursors.IntegerArray) error
- func DecodeStringArrayBlock(block []byte, a *cursors.StringArray) error
- func DecodeTimestampArrayBlock(block []byte, a *cursors.TimestampArray) error
- func DecodeUnsignedArrayBlock(block []byte, a *cursors.UnsignedArray) error
- func DefaultFormatFileName(generation, sequence int) string
- func DefaultParseFileName(name string) (int, int, error)
- func EncodeBooleanArrayBlock(a *cursors.BooleanArray, b []byte) ([]byte, error)
- func EncodeFloatArrayBlock(a *cursors.FloatArray, b []byte) ([]byte, error)
- func EncodeIntegerArrayBlock(a *cursors.IntegerArray, b []byte) ([]byte, error)
- func EncodeStringArrayBlock(a *cursors.StringArray, b []byte) ([]byte, error)
- func EncodeUnsignedArrayBlock(a *cursors.UnsignedArray, b []byte) ([]byte, error)
- func ErrCacheMemorySizeLimitExceeded(n, limit uint64) error
- func FloatArrayDecodeAll(b []byte, buf []float64) ([]float64, error)
- func FloatArrayEncodeAll(src []float64, b []byte) ([]byte, error)
- func IntegerArrayDecodeAll(b []byte, dst []int64) ([]int64, error)
- func IntegerArrayEncodeAll(src []int64, b []byte) ([]byte, error)
- func MetricsGroupFromContext(ctx context.Context) *metrics.Group
- func NewBooleanArrayFromValues(v BooleanValues) *cursors.BooleanArray
- func NewContextWithMetricsGroup(ctx context.Context) context.Context
- func NewFloatArrayFromValues(v FloatValues) *cursors.FloatArray
- func NewIndirectIndex() *indirectIndex
- func NewIntegerArrayFromValues(v IntegerValues) *cursors.IntegerArray
- func NewStringArrayFromValues(v StringValues) *cursors.StringArray
- func NewUnsignedArrayFromValues(v UnsignedValues) *cursors.UnsignedArray
- func PrometheusCollectors() []prometheus.Collector
- func SeriesAndFieldFromCompositeKey(key []byte) ([]byte, []byte)
- func SeriesFieldKey(seriesKey, field string) string
- func SeriesFieldKeyBytes(seriesKey, field string) []byte
- func StatsFilename(tsmPath string) string
- func StringArrayDecodeAll(b []byte, dst []string) ([]string, error)
- func StringArrayEncodeAll(src []string, b []byte) ([]byte, error)
- func TimeArrayDecodeAll(b []byte, dst []int64) ([]int64, error)
- func TimeArrayEncodeAll(src []int64, b []byte) ([]byte, error)
- func UnsignedArrayDecodeAll(b []byte, dst []uint64) ([]uint64, error)
- func UnsignedArrayEncodeAll(src []uint64, b []byte) ([]byte, error)
- func ValidateTagPredicate(expr influxql.Expr) (err error)
- func ValuesToPoints(values map[string][]Value) []models.Point
- func ZigZagDecode(v uint64) int64
- func ZigZagEncode(x int64) uint64
- type BatchDeleter
- type BatchDeleters
- type BitReader
- type BlockExporter
- type BlockIterator
- type BooleanDecoder
- type BooleanEncoder
- type BooleanValue
- type BooleanValues
- func (a BooleanValues) Deduplicate() BooleanValues
- func (a BooleanValues) Encode(buf []byte) ([]byte, error)
- func (a BooleanValues) Exclude(min, max int64) BooleanValues
- func (a BooleanValues) FindRange(min, max int64) (int, int)
- func (a BooleanValues) Include(min, max int64) BooleanValues
- func (a BooleanValues) Len() int
- func (a BooleanValues) Less(i, j int) bool
- func (a BooleanValues) MaxTime() int64
- func (a BooleanValues) Merge(b BooleanValues) BooleanValues
- func (a BooleanValues) MinTime() int64
- func (a BooleanValues) Size() int
- func (a BooleanValues) Swap(i, j int)
- type Cache
- func (c *Cache) Age() time.Duration
- func (c *Cache) AppendTimestamps(key []byte, ts []int64) []int64
- func (c *Cache) ApplyEntryFn(f func(key string, entry *entry) error) error
- func (c *Cache) BlockType(key []byte) byte
- func (c *Cache) ClearSnapshot(success bool)
- func (c *Cache) Count() int
- func (c *Cache) Deduplicate()
- func (c *Cache) DeleteBucketRange(ctx context.Context, name string, min, max int64, pred Predicate)
- func (c *Cache) Keys() [][]byte
- func (c *Cache) LastWriteTime() time.Time
- func (c *Cache) MaxSize() uint64
- func (c *Cache) SetMaxSize(size uint64)
- func (c *Cache) Size() uint64
- func (c *Cache) Snapshot() (*Cache, error)
- func (c *Cache) Split(n int) []*Cache
- func (c *Cache) Type(key []byte) (models.FieldType, error)
- func (c *Cache) UpdateAge()
- func (c *Cache) Values(key []byte) Values
- func (c *Cache) Write(key []byte, values []Value) error
- func (c *Cache) WriteMulti(values map[string][]Value) error
- type CacheConfig
- type CacheLoader
- type CacheMemorySizeLimitExceededError
- type CacheStatus
- type CompactionConfig
- type CompactionGroup
- type CompactionPlanner
- type Compactor
- func (c *Compactor) Close()
- func (c *Compactor) CompactFast(tsmFiles []string) ([]string, error)
- func (c *Compactor) CompactFull(tsmFiles []string) ([]string, error)
- func (c *Compactor) DisableCompactions()
- func (c *Compactor) DisableSnapshots()
- func (c *Compactor) EnableCompactions()
- func (c *Compactor) EnableSnapshots()
- func (c *Compactor) Open()
- func (c *Compactor) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)
- func (c *Compactor) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
- func (c *Compactor) WriteSnapshot(ctx context.Context, cache *Cache) ([]string, error)
- type Config
- type DefaultPlanner
- func (c *DefaultPlanner) ForceFull()
- func (c *DefaultPlanner) FullyCompacted() bool
- func (c *DefaultPlanner) ParseFileName(path string) (int, int, error)
- func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup
- func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup
- func (c *DefaultPlanner) PlanOptimize() []CompactionGroup
- func (c *DefaultPlanner) Release(groups []CompactionGroup)
- func (c *DefaultPlanner) SetFileStore(fs *FileStore)
- type Engine
- func (e *Engine) Close() error
- func (e *Engine) CreateCursorIterator(ctx context.Context) (cursors.CursorIterator, error)
- func (e *Engine) DeletePrefixRange(rootCtx context.Context, name []byte, min, max int64, pred Predicate) error
- func (e *Engine) ForEachMeasurementName(fn func(name []byte) error) error
- func (e *Engine) HasTagKey(name, key []byte) (bool, error)
- func (e *Engine) IsIdle() bool
- func (e *Engine) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error)
- func (e *Engine) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor
- func (e *Engine) MeasurementExists(name []byte) (bool, error)
- func (e *Engine) MeasurementFields(ctx context.Context, orgID, bucketID influxdb.ID, measurement string, ...) (cursors.MeasurementFieldsIterator, error)
- func (e *Engine) MeasurementNames(ctx context.Context, orgID, bucketID influxdb.ID, start, end int64) (cursors.StringIterator, error)
- func (e *Engine) MeasurementNamesByRegex(re *regexp.Regexp) ([][]byte, error)
- func (e *Engine) MeasurementStats() (MeasurementStats, error)
- func (e *Engine) MeasurementTagKeys(ctx context.Context, orgID, bucketID influxdb.ID, measurement string, ...) (cursors.StringIterator, error)
- func (e *Engine) MeasurementTagKeysByExpr(name []byte, expr influxql.Expr) (map[string]struct{}, error)
- func (e *Engine) MeasurementTagValues(ctx context.Context, orgID, bucketID influxdb.ID, measurement, tagKey string, ...) (cursors.StringIterator, error)
- func (e *Engine) Open(ctx context.Context) (err error)
- func (e *Engine) Path() string
- func (e *Engine) ScheduleFullCompaction(ctx context.Context) error
- func (e *Engine) SeriesN() int64
- func (e *Engine) SetCompactionsEnabled(enabled bool)
- func (e *Engine) SetDefaultMetricLabels(labels prometheus.Labels)
- func (e *Engine) SetEnabled(enabled bool)
- func (e *Engine) SetFieldName(measurement []byte, name string)
- func (e *Engine) SetSemaphore(s influxdb.Semaphore)
- func (e *Engine) ShouldCompactCache(t time.Time) CacheStatus
- func (e *Engine) TagKeyCardinality(name, key []byte) int
- func (e *Engine) TagKeys(ctx context.Context, orgID, bucketID influxdb.ID, start, end int64, ...) (cursors.StringIterator, error)
- func (e *Engine) TagValues(ctx context.Context, orgID, bucketID influxdb.ID, tagKey string, ...) (cursors.StringIterator, error)
- func (e *Engine) WithCompactionLimiter(limiter limiter.Fixed)
- func (e *Engine) WithCompactionPlanner(planner CompactionPlanner)
- func (e *Engine) WithCurrentGenerationFunc(fn func() int)
- func (e *Engine) WithFileStoreObserver(obs FileStoreObserver)
- func (e *Engine) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)
- func (e *Engine) WithLogger(log *zap.Logger)
- func (e *Engine) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
- func (e *Engine) WritePoints(points []models.Point) error
- func (e *Engine) WriteSnapshot(ctx context.Context, status CacheStatus) error
- func (e *Engine) WriteValues(values map[string][]Value) error
- type EngineOption
- type FileStat
- type FileStore
- func (f *FileStore) Apply(fn func(r TSMFile) error) error
- func (f *FileStore) BlockCount(path string, idx int) int
- func (f *FileStore) Close() error
- func (f *FileStore) Cost(key []byte, min, max int64) query.IteratorCost
- func (f *FileStore) Count() int
- func (f *FileStore) CreateSnapshot(ctx context.Context) (backupID int, backupDirFullPath string, err error)
- func (f *FileStore) CurrentGeneration() int
- func (f *FileStore) Delete(keys [][]byte) error
- func (f *FileStore) DeleteRange(keys [][]byte, min, max int64) error
- func (f *FileStore) DiskSizeBytes() int64
- func (f *FileStore) Files() []TSMFile
- func (f *FileStore) ForEachFile(fn func(f TSMFile) bool)
- func (f *FileStore) InternalBackupPath(backupID int) string
- func (f *FileStore) KeyCursor(ctx context.Context, key []byte, t int64, ascending bool) *KeyCursor
- func (f *FileStore) Keys() map[string]byte
- func (f *FileStore) LastModified() time.Time
- func (f *FileStore) MeasurementStats() (MeasurementStats, error)
- func (f *FileStore) NextGeneration() int
- func (f *FileStore) Open(ctx context.Context) error
- func (f *FileStore) ParseFileName(path string) (int, int, error)
- func (f *FileStore) Read(key []byte, t int64) ([]Value, error)
- func (f *FileStore) Replace(oldFiles, newFiles []string) error
- func (f *FileStore) ReplaceWithCallback(oldFiles, newFiles []string, updatedFn func(r []TSMFile)) error
- func (f *FileStore) SetCurrentGenerationFunc(fn func() int)
- func (f *FileStore) Stats() []FileStat
- func (f *FileStore) TSMReader(path string) *TSMReader
- func (f *FileStore) Type(key []byte) (byte, error)
- func (f *FileStore) WalkKeys(seek []byte, fn func(key []byte, typ byte) error) error
- func (f *FileStore) WithLogger(log *zap.Logger)
- func (f *FileStore) WithObserver(obs FileStoreObserver)
- func (f *FileStore) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
- type FileStoreObserver
- type FileStoreStatistics
- type FloatDecoder
- type FloatEncoder
- type FloatValue
- type FloatValues
- func (a FloatValues) Deduplicate() FloatValues
- func (a FloatValues) Encode(buf []byte) ([]byte, error)
- func (a FloatValues) Exclude(min, max int64) FloatValues
- func (a FloatValues) FindRange(min, max int64) (int, int)
- func (a FloatValues) Include(min, max int64) FloatValues
- func (a FloatValues) Len() int
- func (a FloatValues) Less(i, j int) bool
- func (a FloatValues) MaxTime() int64
- func (a FloatValues) Merge(b FloatValues) FloatValues
- func (a FloatValues) MinTime() int64
- func (a FloatValues) Size() int
- func (a FloatValues) Swap(i, j int)
- type FormatFileNameFunc
- type IndexEntry
- type IndexWriter
- type IntegerDecoder
- type IntegerEncoder
- type IntegerValue
- type IntegerValues
- func (a IntegerValues) Deduplicate() IntegerValues
- func (a IntegerValues) Encode(buf []byte) ([]byte, error)
- func (a IntegerValues) Exclude(min, max int64) IntegerValues
- func (a IntegerValues) FindRange(min, max int64) (int, int)
- func (a IntegerValues) Include(min, max int64) IntegerValues
- func (a IntegerValues) Len() int
- func (a IntegerValues) Less(i, j int) bool
- func (a IntegerValues) MaxTime() int64
- func (a IntegerValues) Merge(b IntegerValues) IntegerValues
- func (a IntegerValues) MinTime() int64
- func (a IntegerValues) Size() int
- func (a IntegerValues) Swap(i, j int)
- type KeyCursor
- func (c *KeyCursor) Close()
- func (c *KeyCursor) Next()
- func (c *KeyCursor) ReadBooleanArrayBlock(values *cursors.BooleanArray) (*cursors.BooleanArray, error)
- func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error)
- func (c *KeyCursor) ReadFloatArrayBlock(values *cursors.FloatArray) (*cursors.FloatArray, error)
- func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error)
- func (c *KeyCursor) ReadIntegerArrayBlock(values *cursors.IntegerArray) (*cursors.IntegerArray, error)
- func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error)
- func (c *KeyCursor) ReadStringArrayBlock(values *cursors.StringArray) (*cursors.StringArray, error)
- func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error)
- func (c *KeyCursor) ReadUnsignedArrayBlock(values *cursors.UnsignedArray) (*cursors.UnsignedArray, error)
- func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, error)
- type KeyIterator
- func NewCacheKeyIterator(cache *Cache, size int, interrupt chan struct{}) KeyIterator
- func NewTSMBatchKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)
- func NewTSMKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)
- type MeasurementStats
- type ParseFileNameFunc
- type Predicate
- type Report
- type ReportSummary
- type SQLBlockExporter
- type Snapshotter
- type StringDecoder
- type StringEncoder
- type StringValue
- type StringValues
- func (a StringValues) Deduplicate() StringValues
- func (a StringValues) Encode(buf []byte) ([]byte, error)
- func (a StringValues) Exclude(min, max int64) StringValues
- func (a StringValues) FindRange(min, max int64) (int, int)
- func (a StringValues) Include(min, max int64) StringValues
- func (a StringValues) Len() int
- func (a StringValues) Less(i, j int) bool
- func (a StringValues) MaxTime() int64
- func (a StringValues) Merge(b StringValues) StringValues
- func (a StringValues) MinTime() int64
- func (a StringValues) Size() int
- func (a StringValues) Swap(i, j int)
- type TSMFile
- type TSMIndex
- type TSMIndexIterator
- func (t *TSMIndexIterator) Entries() []IndexEntry
- func (t *TSMIndexIterator) Err() error
- func (t *TSMIndexIterator) Key() []byte
- func (t *TSMIndexIterator) Next() bool
- func (t *TSMIndexIterator) Peek() []byte
- func (t *TSMIndexIterator) Seek(key []byte) (exact, ok bool)
- func (t *TSMIndexIterator) Type() byte
- type TSMIterator
- type TSMReader
- func (r *TSMReader) BatchDelete() BatchDeleter
- func (t *TSMReader) BlockIterator() *BlockIterator
- func (t *TSMReader) Close() error
- func (t *TSMReader) Contains(key []byte) bool
- func (t *TSMReader) Delete(keys [][]byte) error
- func (t *TSMReader) DeletePrefix(prefix []byte, minTime, maxTime int64, pred Predicate, dead func([]byte)) error
- func (t *TSMReader) DeleteRange(keys [][]byte, minTime, maxTime int64) error
- func (t *TSMReader) Free() error
- func (t *TSMReader) HasTombstones() bool
- func (t *TSMReader) InUse() bool
- func (t *TSMReader) IndexSize() uint32
- func (t *TSMReader) Iterator(key []byte) TSMIterator
- func (t *TSMReader) KeyCount() int
- func (t *TSMReader) KeyRange() ([]byte, []byte)
- func (t *TSMReader) LastModified() int64
- func (t *TSMReader) MaybeContainsValue(key []byte, ts int64) bool
- func (t *TSMReader) MeasurementStats() (MeasurementStats, error)
- func (t *TSMReader) OverlapsKeyPrefixRange(min, max []byte) bool
- func (t *TSMReader) OverlapsKeyRange(min, max []byte) bool
- func (t *TSMReader) OverlapsTimeRange(min, max int64) bool
- func (t *TSMReader) Path() string
- func (t *TSMReader) Read(key []byte, timestamp int64) ([]Value, error)
- func (t *TSMReader) ReadAll(key []byte) ([]Value, error)
- func (t *TSMReader) ReadAt(entry *IndexEntry, vals []Value) ([]Value, error)
- func (t *TSMReader) ReadBooleanArrayBlockAt(entry *IndexEntry, vals *cursors.BooleanArray) error
- func (t *TSMReader) ReadBooleanBlockAt(entry *IndexEntry, vals *[]BooleanValue) ([]BooleanValue, error)
- func (t *TSMReader) ReadBytes(e *IndexEntry, b []byte) (uint32, []byte, error)
- func (t *TSMReader) ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error)
- func (t *TSMReader) ReadFloatArrayBlockAt(entry *IndexEntry, vals *cursors.FloatArray) error
- func (t *TSMReader) ReadFloatBlockAt(entry *IndexEntry, vals *[]FloatValue) ([]FloatValue, error)
- func (t *TSMReader) ReadIntegerArrayBlockAt(entry *IndexEntry, vals *cursors.IntegerArray) error
- func (t *TSMReader) ReadIntegerBlockAt(entry *IndexEntry, vals *[]IntegerValue) ([]IntegerValue, error)
- func (t *TSMReader) ReadStringArrayBlockAt(entry *IndexEntry, vals *cursors.StringArray) error
- func (t *TSMReader) ReadStringBlockAt(entry *IndexEntry, vals *[]StringValue) ([]StringValue, error)
- func (t *TSMReader) ReadUnsignedArrayBlockAt(entry *IndexEntry, vals *cursors.UnsignedArray) error
- func (t *TSMReader) ReadUnsignedBlockAt(entry *IndexEntry, vals *[]UnsignedValue) ([]UnsignedValue, error)
- func (t *TSMReader) Ref()
- func (t *TSMReader) Remove() error
- func (t *TSMReader) Rename(path string) error
- func (t *TSMReader) Size() uint32
- func (t *TSMReader) Stats() FileStat
- func (t *TSMReader) TimeRange() (int64, int64)
- func (t *TSMReader) TimeRangeIterator(key []byte, min, max int64) *TimeRangeIterator
- func (t *TSMReader) TimeRangeMaxTimeIterator(key []byte, min, max int64) *TimeRangeMaxTimeIterator
- func (t *TSMReader) TombstoneFiles() []FileStat
- func (t *TSMReader) TombstoneRange(key []byte, buf []TimeRange) []TimeRange
- func (t *TSMReader) Type(key []byte) (byte, error)
- func (t *TSMReader) Unref()
- func (t *TSMReader) WithObserver(obs FileStoreObserver)
- type TSMWriter
- type TimeDecoder
- type TimeEncoder
- type TimeRange
- type TimeRangeIterator
- func (b *TimeRangeIterator) Err() error
- func (b *TimeRangeIterator) HasData() bool
- func (b *TimeRangeIterator) Key() []byte
- func (b *TimeRangeIterator) Next() bool
- func (b *TimeRangeIterator) Seek(key []byte) (exact, ok bool)
- func (b *TimeRangeIterator) Stats() cursors.CursorStats
- func (b *TimeRangeIterator) Type() byte
- type TimeRangeMaxTimeIterator
- func (b *TimeRangeMaxTimeIterator) Err() error
- func (b *TimeRangeMaxTimeIterator) HasData() bool
- func (b *TimeRangeMaxTimeIterator) Key() []byte
- func (b *TimeRangeMaxTimeIterator) MaxTime() int64
- func (b *TimeRangeMaxTimeIterator) Next() bool
- func (b *TimeRangeMaxTimeIterator) Seek(key []byte) (exact, ok bool)
- func (b *TimeRangeMaxTimeIterator) Stats() cursors.CursorStats
- func (b *TimeRangeMaxTimeIterator) Type() byte
- type Tombstone
- type Tombstoner
- func (t *Tombstoner) Add(keys [][]byte) error
- func (t *Tombstoner) AddPrefixRange(key []byte, min, max int64, predicate []byte) error
- func (t *Tombstoner) AddRange(keys [][]byte, min, max int64) error
- func (t *Tombstoner) Delete() error
- func (t *Tombstoner) Flush() error
- func (t *Tombstoner) HasTombstones() bool
- func (t *Tombstoner) Rollback() error
- func (t *Tombstoner) TombstoneFiles() []FileStat
- func (t *Tombstoner) Walk(fn func(t Tombstone) error) error
- func (t *Tombstoner) WithObserver(obs FileStoreObserver)
- type UnsignedValue
- type UnsignedValues
- func (a UnsignedValues) Deduplicate() UnsignedValues
- func (a UnsignedValues) Encode(buf []byte) ([]byte, error)
- func (a UnsignedValues) Exclude(min, max int64) UnsignedValues
- func (a UnsignedValues) FindRange(min, max int64) (int, int)
- func (a UnsignedValues) Include(min, max int64) UnsignedValues
- func (a UnsignedValues) Len() int
- func (a UnsignedValues) Less(i, j int) bool
- func (a UnsignedValues) MaxTime() int64
- func (a UnsignedValues) Merge(b UnsignedValues) UnsignedValues
- func (a UnsignedValues) MinTime() int64
- func (a UnsignedValues) Size() int
- func (a UnsignedValues) Swap(i, j int)
- type Value
- func DecodeBlock(block []byte, vals []Value) ([]Value, error)
- func NewBooleanValue(t int64, v bool) Value
- func NewFloatValue(t int64, v float64) Value
- func NewIntegerValue(t int64, v int64) Value
- func NewStringValue(t int64, v string) Value
- func NewUnsignedValue(t int64, v uint64) Value
- func NewValue(t int64, v interface{}) Value
- type Values
- func (a Values) BlockType() byte
- func (a Values) Contains(min, max int64) bool
- func (a Values) Deduplicate() Values
- func (a Values) Encode(buf []byte) ([]byte, error)
- func (a Values) Exclude(min, max int64) Values
- func (a Values) FindRange(min, max int64) (int, int)
- func (a Values) Include(min, max int64) Values
- func (a Values) InfluxQLType() (influxql.DataType, error)
- func (a Values) Len() int
- func (a Values) Less(i, j int) bool
- func (a Values) MaxTime() int64
- func (a Values) Merge(b Values) Values
- func (a Values) MinTime() int64
- func (a Values) Size() int
- func (a Values) Swap(i, j int)
- type VerifyTSM
- type WALConfig
Constants ¶
const ( // CompactionTempExtension is the extension used for temporary files created during compaction. CompactionTempExtension = "tmp" // TSMFileExtension is the extension used for TSM files. TSMFileExtension = "tsm" // TSSFileExtension is the extension used for TSM stats files. TSSFileExtension = "tss" )
const ( DefaultMADVWillNeed = false // DefaultLargeSeriesWriteThreshold is the number of series per write // that requires the series index be pregrown before insert. DefaultLargeSeriesWriteThreshold = 10000 )
const ( DefaultCompactFullWriteColdDuration = time.Duration(4 * time.Hour) DefaultCompactThroughput = 48 * 1024 * 1024 DefaultCompactThroughputBurst = 48 * 1024 * 1024 DefaultCompactMaxConcurrent = 0 )
const ( DefaultCacheMaxMemorySize = toml.Size(1024 << 20) // 1GB DefaultCacheSnapshotMemorySize = toml.Size(25 << 20) // 25MB DefaultCacheSnapshotAgeDuration = toml.Duration(0) // Defaults to off. DefaultCacheSnapshotWriteColdDuration = toml.Duration(10 * time.Minute) // Ten minutes )
Default Cache configuration values.
const ( DefaultWALEnabled = true DefaultWALFsyncDelay = time.Duration(0) )
Default WAL configuration values.
const ( // BlockFloat64 designates a block encodes float64 values. BlockFloat64 = byte(0) // BlockInteger designates a block encodes int64 values. BlockInteger = byte(1) // BlockBoolean designates a block encodes boolean values. BlockBoolean = byte(2) // BlockString designates a block encodes string values. BlockString = byte(3) // BlockUnsigned designates a block encodes uint64 values. BlockUnsigned = byte(4) // BlockUndefined represents an undefined block type value. BlockUndefined = BlockUnsigned + 1 )
const ( // The extension used to describe temporary snapshot files. TmpTSMFileExtension = "tmp" // The extension used to describe corrupt snapshot files. BadTSMFileExtension = "bad" )
const ( // MeasurementMagicNumber is written as the first 4 bytes of a data file to // identify the file as a tsm1 stats file. MeasurementStatsMagicNumber string = "TSS1" // MeasurementStatsVersion indicates the version of the TSS1 file format. MeasurementStatsVersion byte = 1 )
const ( // MagicNumber is written as the first 4 bytes of a data file to // identify the file as a tsm1 formatted file MagicNumber uint32 = 0x16D116D1 // Version indicates the version of the TSM file format. Version byte = 1 )
const ( // InvalidMinNanoTime is an invalid nano timestamp that has an ordinal // value lower than models.MinNanoTime, the minimum valid timestamp // that can be represented. InvalidMinNanoTime = models.MinNanoTime - 1 )
const (
// MaxPointsPerBlock is the maximum number of points in an encoded block in a TSM file
MaxPointsPerBlock = 1000
)
Variables ¶
var ( //ErrNoValues is returned when TSMWriter.WriteIndex is called and there are no values to write. ErrNoValues = fmt.Errorf("no values written") // ErrTSMClosed is returned when performing an operation against a closed TSM file. ErrTSMClosed = fmt.Errorf("tsm file closed") // ErrMaxKeyLengthExceeded is returned when attempting to write a key that is too long. ErrMaxKeyLengthExceeded = fmt.Errorf("max key length exceeded") // ErrMaxBlocksExceeded is returned when attempting to write a block past the allowed number. ErrMaxBlocksExceeded = fmt.Errorf("max blocks exceeded") )
var DefaultMaxConcurrentOpens = runtime.GOMAXPROCS(0)
var ErrFileInUse = fmt.Errorf("file still in use")
ErrFileInUse is returned when attempting to remove or close a TSM file that is still being used.
var ( // ErrSnapshotInProgress is returned if a snapshot is attempted while one is already running. ErrSnapshotInProgress = fmt.Errorf("snapshot in progress") )
var ( // ErrStringArrayEncodeTooLarge reports that the encoded length of a slice of strings is too large. ErrStringArrayEncodeTooLarge = errors.New("StringArrayEncodeAll: source length too large") )
var ( // Static objects to prevent small allocs. KeyFieldSeparatorBytes = []byte(keyFieldSeparator) )
var WithMadviseWillNeed = func(willNeed bool) tsmReaderOption { return func(r *TSMReader) { r.madviseWillNeed = willNeed } }
WithMadviseWillNeed is an option for specifying whether to provide a MADV_WILL need hint to the kernel.
var WithTSMReaderLogger = func(logger *zap.Logger) tsmReaderOption { return func(r *TSMReader) { r.logger = logger } }
Functions ¶
func AddMeasurementToExpr ¶
func AppendSeriesFieldKeyBytes ¶
AppendSeriesFieldKeyBytes combines seriesKey and field such that can be used to search a TSM index. The value is appended to dst and the extended buffer returned.
func BlockCount ¶
BlockCount returns the number of timestamps encoded in block.
func BlockType ¶
BlockType returns the type of value encoded in a block or an error if the block type is unknown.
func BlockTypeName ¶
BlockTypeName returns a string name for the block type.
func BlockTypeToFieldType ¶
func BooleanArrayEncodeAll ¶
BooleanArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.
func CollectionToValues ¶
func CollectionToValues(collection *tsdb.SeriesCollection) (map[string][]Value, error)
CollectionToValues takes in a series collection and returns it as a map of series key to values. It returns an error if any of the points could not be converted.
func CountTimestamps ¶
func DecodeBooleanArrayBlock ¶
func DecodeBooleanArrayBlock(block []byte, a *cursors.BooleanArray) error
DecodeBooleanArrayBlock decodes the boolean block from the byte slice and writes the values to a.
func DecodeFloatArrayBlock ¶
func DecodeFloatArrayBlock(block []byte, a *cursors.FloatArray) error
DecodeFloatArrayBlock decodes the float block from the byte slice and writes the values to a.
func DecodeIntegerArrayBlock ¶
func DecodeIntegerArrayBlock(block []byte, a *cursors.IntegerArray) error
DecodeIntegerArrayBlock decodes the integer block from the byte slice and writes the values to a.
func DecodeStringArrayBlock ¶
func DecodeStringArrayBlock(block []byte, a *cursors.StringArray) error
DecodeStringArrayBlock decodes the string block from the byte slice and writes the values to a.
func DecodeTimestampArrayBlock ¶
func DecodeTimestampArrayBlock(block []byte, a *cursors.TimestampArray) error
DecodeTimestampArrayBlock decodes the timestamps from the specified block, ignoring the block type and the values.
func DecodeUnsignedArrayBlock ¶
func DecodeUnsignedArrayBlock(block []byte, a *cursors.UnsignedArray) error
DecodeUnsignedArrayBlock decodes the unsigned integer block from the byte slice and writes the values to a.
func DefaultFormatFileName ¶
DefaultFormatFileName is the default implementation to format TSM filenames.
func DefaultParseFileName ¶
DefaultParseFileName is used to parse the filenames of TSM files.
func EncodeBooleanArrayBlock ¶
func EncodeBooleanArrayBlock(a *cursors.BooleanArray, b []byte) ([]byte, error)
func EncodeFloatArrayBlock ¶
func EncodeFloatArrayBlock(a *cursors.FloatArray, b []byte) ([]byte, error)
func EncodeIntegerArrayBlock ¶
func EncodeIntegerArrayBlock(a *cursors.IntegerArray, b []byte) ([]byte, error)
func EncodeStringArrayBlock ¶
func EncodeStringArrayBlock(a *cursors.StringArray, b []byte) ([]byte, error)
func EncodeUnsignedArrayBlock ¶
func EncodeUnsignedArrayBlock(a *cursors.UnsignedArray, b []byte) ([]byte, error)
func ErrCacheMemorySizeLimitExceeded ¶
ErrCacheMemorySizeLimitExceeded returns an error indicating an operation could not be completed due to exceeding the cache-max-memory-size setting.
func FloatArrayEncodeAll ¶
FloatArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.
Currently only the float compression scheme used in Facebook's Gorilla is supported, so this method implements a batch oriented version of that.
func IntegerArrayEncodeAll ¶
IntegerArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.
IntegerArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.
Important: IntegerArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into IntegerArrayEncodeAll.
func MetricsGroupFromContext ¶
MetricsGroupFromContext returns the tsm1 metrics.Group associated with the context or nil if no group has been assigned.
func NewBooleanArrayFromValues ¶
func NewBooleanArrayFromValues(v BooleanValues) *cursors.BooleanArray
func NewContextWithMetricsGroup ¶
NewContextWithMetricsGroup creates a new context with a tsm1 metrics.Group for tracking various metrics when accessing TSM data.
func NewFloatArrayFromValues ¶
func NewFloatArrayFromValues(v FloatValues) *cursors.FloatArray
func NewIndirectIndex ¶
func NewIndirectIndex() *indirectIndex
NewIndirectIndex returns a new indirect index.
func NewIntegerArrayFromValues ¶
func NewIntegerArrayFromValues(v IntegerValues) *cursors.IntegerArray
func NewStringArrayFromValues ¶
func NewStringArrayFromValues(v StringValues) *cursors.StringArray
func NewUnsignedArrayFromValues ¶
func NewUnsignedArrayFromValues(v UnsignedValues) *cursors.UnsignedArray
func PrometheusCollectors ¶
func PrometheusCollectors() []prometheus.Collector
PrometheusCollectors returns all prometheus metrics for the tsm1 package.
func SeriesAndFieldFromCompositeKey ¶
SeriesAndFieldFromCompositeKey returns the series key and the field key extracted from the composite key.
func SeriesFieldKey ¶
SeriesFieldKey combine a series key and field name for a unique string to be hashed to a numeric ID.
func SeriesFieldKeyBytes ¶
func StatsFilename ¶
StatsFilename returns the path to the stats file for a given TSM file path.
func StringArrayEncodeAll ¶
StringArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.
Currently only the string compression scheme used snappy.
func TimeArrayEncodeAll ¶
TimeArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capacity to b.
TimeArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.
Timestamp values to be encoded should be sorted before encoding. When encoded, the values are first delta-encoded. The first value is the starting timestamp, subsequent values are the difference from the prior value.
Important: TimeArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into TimeArrayEncodeAll.
func UnsignedArrayEncodeAll ¶
UnsignedArrayEncodeAll encodes src into b, returning b and any error encountered. The returned slice may be of a different length and capactity to b.
UnsignedArrayEncodeAll implements batch oriented versions of the three integer encoding types we support: uncompressed, simple8b and RLE.
Important: IntegerArrayEncodeAll modifies the contents of src by using it as scratch space for delta encoded values. It is NOT SAFE to use src after passing it into IntegerArrayEncodeAll.
func ValidateTagPredicate ¶
func ValuesToPoints ¶
ValuesToPoints takes in a map of values and returns a slice of models.Point.
func ZigZagDecode ¶
ZigZagDecode converts a previously zigzag encoded uint64 back to a int64.
func ZigZagEncode ¶
ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3].
Types ¶
type BatchDeleter ¶
type BatchDeleters ¶
type BatchDeleters []BatchDeleter
func (BatchDeleters) Commit ¶
func (a BatchDeleters) Commit() error
func (BatchDeleters) DeleteRange ¶
func (a BatchDeleters) DeleteRange(keys [][]byte, min, max int64) error
func (BatchDeleters) Rollback ¶
func (a BatchDeleters) Rollback() error
type BitReader ¶
type BitReader struct {
// contains filtered or unexported fields
}
BitReader reads bits from an io.Reader.
func NewBitReader ¶
NewBitReader returns a new instance of BitReader that reads from data.
func (*BitReader) CanReadBitFast ¶
CanReadBitFast returns true if calling ReadBitFast() is allowed. Fast bit reads are allowed when at least 2 values are in the buffer. This is because it is not required to refilled the buffer and the caller can inline the calls.
func (*BitReader) ReadBitFast ¶
ReadBitFast is an optimized bit read. IMPORTANT: Only allowed if CanReadFastBit() is true!
type BlockExporter ¶
BlockExporter writes all blocks in a file to a given format.
type BlockIterator ¶
type BlockIterator struct {
// contains filtered or unexported fields
}
BlockIterator allows iterating over each block in a TSM file in order. It provides raw access to the block bytes without decoding them.
func (*BlockIterator) Err ¶
func (b *BlockIterator) Err() error
Err returns any errors encounter during iteration.
func (*BlockIterator) Next ¶
func (b *BlockIterator) Next() bool
Next returns true if there are more blocks to iterate through.
func (*BlockIterator) PeekNext ¶
func (b *BlockIterator) PeekNext() []byte
PeekNext returns the next key to be iterated or an empty string.
type BooleanDecoder ¶
type BooleanDecoder struct {
// contains filtered or unexported fields
}
BooleanDecoder decodes a series of booleans from an in-memory buffer.
func (*BooleanDecoder) Error ¶
func (e *BooleanDecoder) Error() error
Error returns the error encountered during decoding, if one occurred.
func (*BooleanDecoder) Next ¶
func (e *BooleanDecoder) Next() bool
Next returns whether there are any bits remaining in the decoder. It returns false if there was an error decoding. The error is available on the Error method.
func (*BooleanDecoder) Read ¶
func (e *BooleanDecoder) Read() bool
Read returns the next bit from the decoder.
func (*BooleanDecoder) SetBytes ¶
func (e *BooleanDecoder) SetBytes(b []byte)
SetBytes initializes the decoder with a new set of bytes to read from. This must be called before calling any other methods.
type BooleanEncoder ¶
type BooleanEncoder struct {
// contains filtered or unexported fields
}
BooleanEncoder encodes a series of booleans to an in-memory buffer.
func NewBooleanEncoder ¶
func NewBooleanEncoder(sz int) BooleanEncoder
NewBooleanEncoder returns a new instance of BooleanEncoder.
func (*BooleanEncoder) Bytes ¶
func (e *BooleanEncoder) Bytes() ([]byte, error)
Bytes returns a new byte slice containing the encoded booleans from previous calls to Write.
func (*BooleanEncoder) Reset ¶
func (e *BooleanEncoder) Reset()
Reset sets the encoder to its initial state.
func (*BooleanEncoder) Write ¶
func (e *BooleanEncoder) Write(b bool)
Write encodes b to the underlying buffer.
type BooleanValue ¶
type BooleanValue = value.BooleanValue
func DecodeBooleanBlock ¶
func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)
DecodeBooleanBlock decodes the boolean block from the byte slice and appends the boolean values to a.
func NewRawBooleanValue ¶
func NewRawBooleanValue(t int64, v bool) BooleanValue
NewRawBooleanValue returns a new boolean value.
type BooleanValues ¶
type BooleanValues []BooleanValue
BooleanValues represents a slice of Boolean values.
func (BooleanValues) Deduplicate ¶
func (a BooleanValues) Deduplicate() BooleanValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (BooleanValues) Exclude ¶
func (a BooleanValues) Exclude(min, max int64) BooleanValues
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (BooleanValues) FindRange ¶
func (a BooleanValues) FindRange(min, max int64) (int, int)
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (BooleanValues) Include ¶
func (a BooleanValues) Include(min, max int64) BooleanValues
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (BooleanValues) Less ¶
func (a BooleanValues) Less(i, j int) bool
func (BooleanValues) MaxTime ¶
func (a BooleanValues) MaxTime() int64
func (BooleanValues) Merge ¶
func (a BooleanValues) Merge(b BooleanValues) BooleanValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (BooleanValues) MinTime ¶
func (a BooleanValues) MinTime() int64
func (BooleanValues) Size ¶
func (a BooleanValues) Size() int
func (BooleanValues) Swap ¶
func (a BooleanValues) Swap(i, j int)
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache maintains an in-memory store of Values for a set of keys.
func NewCache ¶
NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. Only used for engine caches, never for snapshots.
func (*Cache) Age ¶
Age returns the age of the cache, which is the duration since it was last snapshotted.
func (*Cache) AppendTimestamps ¶
AppendTimestamps appends ts with the timestamps for the specified key. It is the responsibility of the caller to sort and or deduplicate the slice.
func (*Cache) ApplyEntryFn ¶
ApplyEntryFn applies the function f to each entry in the Cache. ApplyEntryFn calls f on each entry in turn, within the same goroutine. It is safe for use by multiple goroutines.
func (*Cache) BlockType ¶
BlockType returns the TSM block type for the specified key or BlockUndefined if the type cannot be determined either because the key does not exist or there are no values for the key.
func (*Cache) ClearSnapshot ¶
ClearSnapshot removes the snapshot cache from the list of flushing caches and adjusts the size.
func (*Cache) Deduplicate ¶
func (c *Cache) Deduplicate()
Deduplicate sorts the snapshot before returning it. The compactor and any queries coming in while it writes will need the values sorted.
func (*Cache) DeleteBucketRange ¶
DeleteBucketRange removes values for all keys containing points with timestamps between min and max contained in the bucket identified by name from the cache.
func (*Cache) LastWriteTime ¶
LastWriteTime returns the time that the cache was last written to.
func (*Cache) SetMaxSize ¶
SetMaxSize updates the memory limit of the cache.
func (*Cache) Snapshot ¶
Snapshot takes a snapshot of the current cache, adds it to the slice of caches that are being flushed, and resets the current cache with new values.
func (*Cache) UpdateAge ¶
func (c *Cache) UpdateAge()
UpdateAge updates the age statistic based on the current time.
func (*Cache) Write ¶
Write writes the set of values for the key to the cache. This function is goroutine-safe. It returns an error if the cache will exceed its max size by adding the new values.
func (*Cache) WriteMulti ¶
WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values. The write attempts to write as many values as possible. If one key fails, the others can still succeed and an error will be returned.
type CacheConfig ¶
type CacheConfig struct { // MaxMemorySize is the maximum size a shard's cache can reach before it starts // rejecting writes. MaxMemorySize toml.Size `toml:"max-memory-size"` // SnapshotMemorySize is the size at which the engine will snapshot the cache and // write it to a TSM file, freeing up memory SnapshotMemorySize toml.Size `toml:"snapshot-memory-size"` // SnapshotAgeDuration, when set, will ensure that the cache is always snapshotted // if it's age is greater than this duration, regardless of the cache's size. SnapshotAgeDuration toml.Duration `toml:"snapshot-age-duration"` // SnapshotWriteColdDuration is the length of time at which the engine will snapshot // the cache and write it to a new TSM file if the shard hasn't received writes or // deletes. // // SnapshotWriteColdDuration should not be larger than SnapshotAgeDuration SnapshotWriteColdDuration toml.Duration `toml:"snapshot-write-cold-duration"` }
CacheConfig holds all of the configuration for the in memory cache of values that are waiting to be snapshot.
func NewCacheConfig ¶
func NewCacheConfig() CacheConfig
NewCacheConfig initialises a new CacheConfig with default values.
type CacheLoader ¶
type CacheLoader struct {
// contains filtered or unexported fields
}
CacheLoader processes a set of WAL segment files, and loads a cache with the data contained within those files.
func NewCacheLoader ¶
func NewCacheLoader(files []string) *CacheLoader
NewCacheLoader returns a new instance of a CacheLoader.
func (*CacheLoader) Load ¶
func (cl *CacheLoader) Load(cache *Cache) error
Load returns a cache loaded with the data contained within the segment files.
func (*CacheLoader) WithLogger ¶
func (cl *CacheLoader) WithLogger(logger *zap.Logger)
WithLogger sets the logger on the CacheLoader.
type CacheMemorySizeLimitExceededError ¶
CacheMemorySizeLimitExceededError is the type of error returned from the cache when a write would place it over its size limit.
func (CacheMemorySizeLimitExceededError) Error ¶
func (c CacheMemorySizeLimitExceededError) Error() string
type CacheStatus ¶
type CacheStatus int
CacheStatus describes the current state of the cache, with respect to whether it is ready to be snapshotted or not.
const ( CacheStatusOkay CacheStatus = iota // Cache is Okay - do not snapshot. CacheStatusSizeExceeded // The cache is large enough to be snapshotted. CacheStatusAgeExceeded // The cache is past the age threshold to be snapshotted. CacheStatusColdNoWrites // The cache has not been written to for long enough that it should be snapshotted. CacheStatusRetention // The cache was snapshotted before running retention. CacheStatusFullCompaction // The cache was snapshotted as part of a full compaction. CacheStatusBackup // The cache was snapshotted before running backup. )
Possible types of Cache status
func (CacheStatus) String ¶
func (i CacheStatus) String() string
type CompactionConfig ¶
type CompactionConfig struct { // FullWriteColdDuration is the duration at which the engine will compact all TSM // files in a shard if it hasn't received a write or delete FullWriteColdDuration toml.Duration `toml:"full-write-cold-duration"` // Throughput is the rate limit in bytes per second that we will allow TSM compactions // to write to disk. Not that short bursts are allowed to happen at a possibly larger // value, set by CompactThroughputBurst. A value of 0 here will disable compaction rate // limiting Throughput toml.Size `toml:"throughput"` // ThroughputBurst is the rate limit in bytes per second that we will allow TSM compactions // to write to disk. If this is not set, the burst value will be set to equal the normal // throughput ThroughputBurst toml.Size `toml:"throughput-burst"` // MaxConcurrent is the maximum number of concurrent full and level compactions that can // run at one time. A value of 0 results in 50% of runtime.GOMAXPROCS(0) used at runtime. MaxConcurrent int `toml:"max-concurrent"` }
CompactionConfing holds all of the configuration for compactions. Eventually we want to move this out of tsm1 so that it can be scheduled more intelligently.
type CompactionGroup ¶
type CompactionGroup []string
CompactionGroup represents a list of files eligible to be compacted together.
type CompactionPlanner ¶
type CompactionPlanner interface { Plan(lastWrite time.Time) []CompactionGroup PlanLevel(level int) []CompactionGroup PlanOptimize() []CompactionGroup Release(group []CompactionGroup) FullyCompacted() bool // ForceFull causes the planner to return a full compaction plan the next // time Plan() is called if there are files that could be compacted. ForceFull() SetFileStore(fs *FileStore) }
CompactionPlanner determines what TSM files and WAL segments to include in a given compaction run.
type Compactor ¶
type Compactor struct { Dir string Size int FileStore interface { SetCurrentGenerationFunc(func() int) NextGeneration() int TSMReader(path string) *TSMReader } // RateLimit is the limit for disk writes for all concurrent compactions. RateLimit limiter.Rate // contains filtered or unexported fields }
Compactor merges multiple TSM files into new files or writes a Cache into 1 or more TSM files.
func NewCompactor ¶
func NewCompactor() *Compactor
NewCompactor returns a new instance of Compactor.
func (*Compactor) CompactFast ¶
CompactFast writes multiple smaller TSM files into 1 or more larger files.
func (*Compactor) CompactFull ¶
CompactFull writes multiple smaller TSM files into 1 or more larger files.
func (*Compactor) DisableCompactions ¶
func (c *Compactor) DisableCompactions()
DisableSnapshots disables the compactor from performing compactions.
func (*Compactor) DisableSnapshots ¶
func (c *Compactor) DisableSnapshots()
DisableSnapshots disables the compactor from performing snapshots.
func (*Compactor) EnableCompactions ¶
func (c *Compactor) EnableCompactions()
EnableCompactions allows the compactor to perform compactions.
func (*Compactor) EnableSnapshots ¶
func (c *Compactor) EnableSnapshots()
EnableSnapshots allows the compactor to perform snapshots.
func (*Compactor) WithFormatFileNameFunc ¶
func (c *Compactor) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)
func (*Compactor) WithParseFileNameFunc ¶
func (c *Compactor) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
type Config ¶
type Config struct { // MacConcurrentOpens controls the concurrency of opening tsm files during // engine opening. MaxConcurrentOpens int `toml:"max-concurrent-opens"` // MADVWillNeed controls whether we hint to the kernel that we intend to page // in mmap'd sections of TSM files. This setting defaults to off, as it has // been found to be problematic in some cases. It may help users who have // slow disks. MADVWillNeed bool `toml:"use-madv-willneed"` // LargeSeriesWriteThreshold is the threshold before a write requires // preallocation to improve throughput. Currently used in the series file. LargeSeriesWriteThreshold int `toml:"large-series-write-threshold"` Compaction CompactionConfig `toml:"compaction"` Cache CacheConfig `toml:"cache"` }
Config contains all of the configuration necessary to run a tsm1 engine.
type DefaultPlanner ¶
type DefaultPlanner struct { FileStore fileStore // contains filtered or unexported fields }
DefaultPlanner implements CompactionPlanner using a strategy to roll up multiple generations of TSM files into larger files in stages. It attempts to minimize the number of TSM files on disk while rolling up a bounder number of files.
func NewDefaultPlanner ¶
func NewDefaultPlanner(fs fileStore, writeColdDuration time.Duration) *DefaultPlanner
func (*DefaultPlanner) ForceFull ¶
func (c *DefaultPlanner) ForceFull()
ForceFull causes the planner to return a full compaction plan the next time a plan is requested. When ForceFull is called, level and optimize plans will not return plans until a full plan is requested and released.
func (*DefaultPlanner) FullyCompacted ¶
func (c *DefaultPlanner) FullyCompacted() bool
FullyCompacted returns true if the shard is fully compacted.
func (*DefaultPlanner) ParseFileName ¶
func (c *DefaultPlanner) ParseFileName(path string) (int, int, error)
func (*DefaultPlanner) Plan ¶
func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup
Plan returns a set of TSM files to rewrite for level 4 or higher. The planning returns multiple groups if possible to allow compactions to run concurrently.
func (*DefaultPlanner) PlanLevel ¶
func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup
PlanLevel returns a set of TSM files to rewrite for a specific level.
func (*DefaultPlanner) PlanOptimize ¶
func (c *DefaultPlanner) PlanOptimize() []CompactionGroup
PlanOptimize returns all TSM files if they are in different generations in order to optimize the index across TSM files. Each returned compaction group can be compacted concurrently.
func (*DefaultPlanner) Release ¶
func (c *DefaultPlanner) Release(groups []CompactionGroup)
Release removes the files reference in each compaction group allowing new plans to be able to use them.
func (*DefaultPlanner) SetFileStore ¶
func (c *DefaultPlanner) SetFileStore(fs *FileStore)
type Engine ¶
type Engine struct { Cache *Cache Compactor *Compactor CompactionPlan CompactionPlanner FileStore *FileStore MaxPointsPerBlock int // CacheFlushMemorySizeThreshold specifies the minimum size threshold for // the cache when the engine should write a snapshot to a TSM file CacheFlushMemorySizeThreshold uint64 // CacheFlushAgeDurationThreshold specified the maximum age a cache can reach // before it is snapshotted, regardless of its size. CacheFlushAgeDurationThreshold time.Duration // CacheFlushWriteColdDuration specifies the length of time after which if // no writes have been committed to the WAL, the engine will write // a snapshot of the cache to a TSM file CacheFlushWriteColdDuration time.Duration // contains filtered or unexported fields }
Engine represents a storage engine with compressed blocks.
func (*Engine) CreateCursorIterator ¶
func (*Engine) DeletePrefixRange ¶
func (e *Engine) DeletePrefixRange(rootCtx context.Context, name []byte, min, max int64, pred Predicate) error
DeletePrefixRange removes all TSM data belonging to a bucket, and removes all index and series file data associated with the bucket. The provided time range ensures that only bucket data for that range is removed.
func (*Engine) ForEachMeasurementName ¶
ForEachMeasurementName iterates over each measurement name in the engine.
func (*Engine) IsIdle ¶
IsIdle returns true if the cache is empty, there are no running compactions and the shard is fully compacted.
func (*Engine) IteratorCost ¶
func (e *Engine) IteratorCost(measurement string, opt query.IteratorOptions) (query.IteratorCost, error)
IteratorCost produces the cost of an iterator.
func (*Engine) MeasurementFields ¶
func (e *Engine) MeasurementFields(ctx context.Context, orgID, bucketID influxdb.ID, measurement string, start, end int64, predicate influxql.Expr) (cursors.MeasurementFieldsIterator, error)
MeasurementFields returns an iterator which enumerates the field schema for the given bucket and measurement, filtered using the optional the predicate and limited to the // time range [start, end].
MeasurementFields will always return a MeasurementFieldsIterator if there is no error.
If the context is canceled before MeasurementFields has finished processing, a non-nil error will be returned along with statistics for the already scanned data.
func (*Engine) MeasurementNames ¶
func (e *Engine) MeasurementNames(ctx context.Context, orgID, bucketID influxdb.ID, start, end int64) (cursors.StringIterator, error)
MeasurementNames returns an iterator which enumerates the measurements for the given bucket and limited to the time range [start, end].
MeasurementNames will always return a StringIterator if there is no error.
If the context is canceled before MeasurementNames has finished processing, a non-nil error will be returned along with statistics for the already scanned data.
func (*Engine) MeasurementNamesByRegex ¶
func (*Engine) MeasurementStats ¶
func (e *Engine) MeasurementStats() (MeasurementStats, error)
MeasurementStats returns the current measurement stats for the engine.
func (*Engine) MeasurementTagKeys ¶
func (e *Engine) MeasurementTagKeys(ctx context.Context, orgID, bucketID influxdb.ID, measurement string, start, end int64, predicate influxql.Expr) (cursors.StringIterator, error)
MeasurementTagKeys returns an iterator which enumerates the tag keys for the given bucket and measurement, filtered using the optional the predicate and limited to the // time range [start, end].
MeasurementTagKeys will always return a StringIterator if there is no error.
If the context is canceled before MeasurementTagKeys has finished processing, a non-nil error will be returned along with statistics for the already scanned data.
func (*Engine) MeasurementTagKeysByExpr ¶
func (*Engine) MeasurementTagValues ¶
func (e *Engine) MeasurementTagValues(ctx context.Context, orgID, bucketID influxdb.ID, measurement, tagKey string, start, end int64, predicate influxql.Expr) (cursors.StringIterator, error)
MeasurementTagValues returns an iterator which enumerates the tag values for the given bucket, measurement and tag key, filtered using the optional the predicate and limited to the time range [start, end].
MeasurementTagValues will always return a StringIterator if there is no error.
If the context is canceled before TagValues has finished processing, a non-nil error will be returned along with statistics for the already scanned data.
func (*Engine) ScheduleFullCompaction ¶
ScheduleFullCompaction will force the engine to fully compact all data stored. This will cancel and running compactions and snapshot any data in the cache to TSM files. This is an expensive operation.
func (*Engine) SetCompactionsEnabled ¶
SetCompactionsEnabled enables compactions on the engine. When disabled all running compactions are aborted and new compactions stop running.
func (*Engine) SetDefaultMetricLabels ¶
func (e *Engine) SetDefaultMetricLabels(labels prometheus.Labels)
SetDefaultMetricLabels sets the default labels for metrics on the engine. It must be called before the Engine is opened.
func (*Engine) SetEnabled ¶
SetEnabled sets whether the engine is enabled.
func (*Engine) SetFieldName ¶
func (*Engine) SetSemaphore ¶
func (e *Engine) SetSemaphore(s influxdb.Semaphore)
SetSemaphore sets the semaphore used to coordinate full compactions across multiple engines.
func (*Engine) ShouldCompactCache ¶
func (e *Engine) ShouldCompactCache(t time.Time) CacheStatus
ShouldCompactCache returns a status indicating if the Cache should be snapshotted. There are three situations when the cache should be snapshotted:
- the Cache size is over its flush size threshold; - the Cache has not been snapshotted for longer than its flush time threshold; or - the Cache has not been written since the write cold threshold.
func (*Engine) TagKeyCardinality ¶
func (*Engine) TagKeys ¶
func (e *Engine) TagKeys(ctx context.Context, orgID, bucketID influxdb.ID, start, end int64, predicate influxql.Expr) (cursors.StringIterator, error)
TagKeys returns an iterator which enumerates the tag keys for the given bucket matching the predicate within the time range [start, end].
TagKeys will always return a StringIterator if there is no error.
If the context is canceled before TagKeys has finished processing, a non-nil error will be returned along with a partial result of the already scanned keys.
func (*Engine) TagValues ¶
func (e *Engine) TagValues(ctx context.Context, orgID, bucketID influxdb.ID, tagKey string, start, end int64, predicate influxql.Expr) (cursors.StringIterator, error)
TagValues returns an iterator which enumerates the values for the specific tagKey in the given bucket matching the predicate within the time range [start, end].
TagValues will always return a StringIterator if there is no error.
If the context is canceled before TagValues has finished processing, a non-nil error will be returned along with a partial result of the already scanned values.
func (*Engine) WithCompactionLimiter ¶
WithCompactionLimiter sets the compaction limiter, which is used to limit the number of concurrent compactions.
func (*Engine) WithCompactionPlanner ¶
func (e *Engine) WithCompactionPlanner(planner CompactionPlanner)
func (*Engine) WithCurrentGenerationFunc ¶
func (*Engine) WithFileStoreObserver ¶
func (e *Engine) WithFileStoreObserver(obs FileStoreObserver)
func (*Engine) WithFormatFileNameFunc ¶
func (e *Engine) WithFormatFileNameFunc(formatFileNameFunc FormatFileNameFunc)
func (*Engine) WithLogger ¶
WithLogger sets the logger for the engine.
func (*Engine) WithParseFileNameFunc ¶
func (e *Engine) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
func (*Engine) WritePoints ¶
WritePoints saves the set of points in the engine.
func (*Engine) WriteSnapshot ¶
func (e *Engine) WriteSnapshot(ctx context.Context, status CacheStatus) error
type EngineOption ¶
type EngineOption func(i *Engine)
An EngineOption is a functional option for changing the configuration of an Engine.
func WithCompactionPlanner ¶
func WithCompactionPlanner(planner CompactionPlanner) EngineOption
WithCompactionPlanner sets the compaction planner for the engine.
func WithSnapshotter ¶
func WithSnapshotter(snapshotter Snapshotter) EngineOption
WithSnapshotter sets the callbacks for the engine to use when creating snapshots.
type FileStat ¶
type FileStat struct { Path string HasTombstone bool Size uint32 LastModified int64 MinTime, MaxTime int64 MinKey, MaxKey []byte }
FileStat holds information about a TSM file on disk.
func (FileStat) MaybeContainsKey ¶
ContainsKey returns true if the min and max keys of the file overlap the arguments min and max.
func (FileStat) OverlapsKeyRange ¶
OverlapsKeyRange returns true if the min and max keys of the file overlap the arguments min and max.
func (FileStat) OverlapsTimeRange ¶
OverlapsTimeRange returns true if the time range of the file intersect min and max.
type FileStore ¶
type FileStore struct {
// contains filtered or unexported fields
}
FileStore is an abstraction around multiple TSM files.
func NewFileStore ¶
NewFileStore returns a new instance of FileStore based on the given directory.
func (*FileStore) Apply ¶
Apply calls fn on each TSMFile in the store concurrently. The level of concurrency is set to GOMAXPROCS.
func (*FileStore) BlockCount ¶
BlockCount returns number of values stored in the block at location idx in the file at path. If path does not match any file in the store, 0 is returned. If idx is out of range for the number of blocks in the file, 0 is returned.
func (*FileStore) CreateSnapshot ¶
func (f *FileStore) CreateSnapshot(ctx context.Context) (backupID int, backupDirFullPath string, err error)
CreateSnapshot creates hardlinks for all tsm and tombstone files in the path provided.
func (*FileStore) CurrentGeneration ¶
CurrentGeneration returns the current generation of the TSM files. Delegates to currentGenerationFunc, if set. Only called by tests.
func (*FileStore) DeleteRange ¶
DeleteRange removes the values for keys between timestamps min and max. This should only be used with smaller batches of series keys.
func (*FileStore) DiskSizeBytes ¶
DiskSizeBytes returns the total number of bytes consumed by the files in the FileStore.
func (*FileStore) Files ¶
Files returns the slice of TSM files currently loaded. This is only used for tests, and the files aren't guaranteed to stay valid in the presence of compactions.
func (*FileStore) ForEachFile ¶
ForEachFile calls fn for all TSM files or until fn returns false. fn is called on the same goroutine as the caller.
func (*FileStore) InternalBackupPath ¶
func (*FileStore) KeyCursor ¶
KeyCursor returns a KeyCursor for key and t across the files in the FileStore.
func (*FileStore) LastModified ¶
LastModified returns the last time the file store was updated with new TSM files or a delete.
func (*FileStore) MeasurementStats ¶
func (f *FileStore) MeasurementStats() (MeasurementStats, error)
MeasurementStats returns the sum of all measurement stats within the store.
func (*FileStore) NextGeneration ¶
NextGeneration increments the max file ID and returns the new value. Delegates to currentGenerationFunc, if set.
func (*FileStore) Read ¶
Read returns the slice of values for the given key and the given timestamp, if any file matches those constraints.
func (*FileStore) ReplaceWithCallback ¶
func (f *FileStore) ReplaceWithCallback(oldFiles, newFiles []string, updatedFn func(r []TSMFile)) error
ReplaceWithCallback replaces oldFiles with newFiles and calls updatedFn with the files to be added the FileStore.
func (*FileStore) SetCurrentGenerationFunc ¶
SetCurrentGenerationFunc must be set before using FileStore.
func (*FileStore) Stats ¶
Stats returns the stats of the underlying files, preferring the cached version if it is still valid.
func (*FileStore) TSMReader ¶
Reader returns a TSMReader for path if one is currently managed by the FileStore. Otherwise it returns nil. If it returns a file, you must call Unref on it when you are done, and never use it after that.
func (*FileStore) WalkKeys ¶
WalkKeys calls fn for every key in every TSM file known to the FileStore. If the key exists in multiple files, it will be invoked for each file.
func (*FileStore) WithLogger ¶
WithLogger sets the logger on the file store.
func (*FileStore) WithObserver ¶
func (f *FileStore) WithObserver(obs FileStoreObserver)
WithObserver sets the observer for the file store.
func (*FileStore) WithParseFileNameFunc ¶
func (f *FileStore) WithParseFileNameFunc(parseFileNameFunc ParseFileNameFunc)
type FileStoreObserver ¶
type FileStoreObserver interface { // FileFinishing is called before a file is renamed to it's final name. FileFinishing(path string) error // FileUnlinking is called before a file is unlinked. FileUnlinking(path string) error }
FileStoreObserver is passed notifications before the file store adds or deletes files. In this way, it can be sure to observe every file that is added or removed even in the presence of process death.
type FileStoreStatistics ¶
FileStoreStatistics keeps statistics about the file store.
type FloatDecoder ¶
type FloatDecoder struct {
// contains filtered or unexported fields
}
FloatDecoder decodes a byte slice into multiple float64 values.
func (*FloatDecoder) Error ¶
func (it *FloatDecoder) Error() error
Error returns the current decoding error.
func (*FloatDecoder) Next ¶
func (it *FloatDecoder) Next() bool
Next returns true if there are remaining values to read.
func (*FloatDecoder) SetBytes ¶
func (it *FloatDecoder) SetBytes(b []byte) error
SetBytes initializes the decoder with b. Must call before calling Next().
func (*FloatDecoder) Values ¶
func (it *FloatDecoder) Values() float64
Values returns the current float64 value.
type FloatEncoder ¶
type FloatEncoder struct {
// contains filtered or unexported fields
}
FloatEncoder encodes multiple float64s into a byte slice.
func NewFloatEncoder ¶
func NewFloatEncoder() *FloatEncoder
NewFloatEncoder returns a new FloatEncoder.
func (*FloatEncoder) Bytes ¶
func (s *FloatEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying byte buffer used in the encoder.
func (*FloatEncoder) Flush ¶
func (s *FloatEncoder) Flush()
Flush indicates there are no more values to encode.
func (*FloatEncoder) Reset ¶
func (s *FloatEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*FloatEncoder) Write ¶
func (s *FloatEncoder) Write(v float64)
Write encodes v to the underlying buffer.
type FloatValue ¶
type FloatValue = value.FloatValue
func DecodeFloatBlock ¶
func DecodeFloatBlock(block []byte, a *[]FloatValue) ([]FloatValue, error)
DecodeFloatBlock decodes the float block from the byte slice and appends the float values to a.
func NewRawFloatValue ¶
func NewRawFloatValue(t int64, v float64) FloatValue
NewRawFloatValue returns a new float value.
type FloatValues ¶
type FloatValues []FloatValue
FloatValues represents a slice of Float values.
func (FloatValues) Deduplicate ¶
func (a FloatValues) Deduplicate() FloatValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (FloatValues) Exclude ¶
func (a FloatValues) Exclude(min, max int64) FloatValues
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (FloatValues) FindRange ¶
func (a FloatValues) FindRange(min, max int64) (int, int)
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (FloatValues) Include ¶
func (a FloatValues) Include(min, max int64) FloatValues
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (FloatValues) Less ¶
func (a FloatValues) Less(i, j int) bool
func (FloatValues) MaxTime ¶
func (a FloatValues) MaxTime() int64
func (FloatValues) Merge ¶
func (a FloatValues) Merge(b FloatValues) FloatValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (FloatValues) MinTime ¶
func (a FloatValues) MinTime() int64
func (FloatValues) Size ¶
func (a FloatValues) Size() int
func (FloatValues) Swap ¶
func (a FloatValues) Swap(i, j int)
type FormatFileNameFunc ¶
FormatFileNameFunc is executed when generating a new TSM filename. Source filenames are provided via src.
type IndexEntry ¶
type IndexEntry struct {
// The min and max time of all points stored in the block.
MinTime, MaxTime int64
// The absolute position in the file where this block is located.
Offset int64
// The size in bytes of the block in the file.
Size uint32
}
IndexEntry is the index information for a given block in a TSM file.
func (*IndexEntry) AppendTo ¶
func (e *IndexEntry) AppendTo(b []byte) []byte
AppendTo writes a binary-encoded version of IndexEntry to b, allocating and returning a new slice, if necessary.
func (*IndexEntry) Contains ¶
func (e *IndexEntry) Contains(t int64) bool
Contains returns true if this IndexEntry may contain values for the given time. The min and max times are inclusive.
func (*IndexEntry) OverlapsTimeRange ¶
func (e *IndexEntry) OverlapsTimeRange(min, max int64) bool
OverlapsTimeRange returns true if the given time ranges are completely within the entry's time bounds.
func (*IndexEntry) String ¶
func (e *IndexEntry) String() string
String returns a string representation of the entry.
func (*IndexEntry) UnmarshalBinary ¶
func (e *IndexEntry) UnmarshalBinary(b []byte) error
UnmarshalBinary decodes an IndexEntry from a byte slice.
type IndexWriter ¶
type IndexWriter interface { // Add records a new block entry for a key in the index. Add(key []byte, blockType byte, minTime, maxTime int64, offset int64, size uint32) // Entries returns all index entries for a key. Entries(key []byte) []IndexEntry // KeyCount returns the count of unique keys in the index. KeyCount() int // Size returns the size of a the current index in bytes. Size() uint32 // MarshalBinary returns a byte slice encoded version of the index. MarshalBinary() ([]byte, error) // WriteTo writes the index contents to a writer. WriteTo(w io.Writer) (int64, error) Close() error Remove() error }
IndexWriter writes a TSMIndex.
func NewDiskIndexWriter ¶
func NewDiskIndexWriter(f *os.File) IndexWriter
NewIndexWriter returns a new IndexWriter.
type IntegerDecoder ¶
type IntegerDecoder struct {
// contains filtered or unexported fields
}
IntegerDecoder decodes a byte slice into int64s.
func (*IntegerDecoder) Error ¶
func (d *IntegerDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*IntegerDecoder) Next ¶
func (d *IntegerDecoder) Next() bool
Next returns true if there are any values remaining to be decoded.
func (*IntegerDecoder) Read ¶
func (d *IntegerDecoder) Read() int64
Read returns the next value from the decoder.
func (*IntegerDecoder) SetBytes ¶
func (d *IntegerDecoder) SetBytes(b []byte)
SetBytes sets the underlying byte slice of the decoder.
type IntegerEncoder ¶
type IntegerEncoder struct {
// contains filtered or unexported fields
}
IntegerEncoder encodes int64s into byte slices.
func NewIntegerEncoder ¶
func NewIntegerEncoder(sz int) IntegerEncoder
NewIntegerEncoder returns a new integer encoder with an initial buffer of values sized at sz.
func (*IntegerEncoder) Bytes ¶
func (e *IntegerEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying buffer.
func (*IntegerEncoder) Reset ¶
func (e *IntegerEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*IntegerEncoder) Write ¶
func (e *IntegerEncoder) Write(v int64)
Write encodes v to the underlying buffers.
type IntegerValue ¶
type IntegerValue = value.IntegerValue
func DecodeIntegerBlock ¶
func DecodeIntegerBlock(block []byte, a *[]IntegerValue) ([]IntegerValue, error)
DecodeIntegerBlock decodes the integer block from the byte slice and appends the integer values to a.
func NewRawIntegerValue ¶
func NewRawIntegerValue(t int64, v int64) IntegerValue
NewRawIntegerValue returns a new integer value.
type IntegerValues ¶
type IntegerValues []IntegerValue
IntegerValues represents a slice of Integer values.
func (IntegerValues) Deduplicate ¶
func (a IntegerValues) Deduplicate() IntegerValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (IntegerValues) Exclude ¶
func (a IntegerValues) Exclude(min, max int64) IntegerValues
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (IntegerValues) FindRange ¶
func (a IntegerValues) FindRange(min, max int64) (int, int)
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (IntegerValues) Include ¶
func (a IntegerValues) Include(min, max int64) IntegerValues
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (IntegerValues) Less ¶
func (a IntegerValues) Less(i, j int) bool
func (IntegerValues) MaxTime ¶
func (a IntegerValues) MaxTime() int64
func (IntegerValues) Merge ¶
func (a IntegerValues) Merge(b IntegerValues) IntegerValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (IntegerValues) MinTime ¶
func (a IntegerValues) MinTime() int64
func (IntegerValues) Size ¶
func (a IntegerValues) Size() int
func (IntegerValues) Swap ¶
func (a IntegerValues) Swap(i, j int)
type KeyCursor ¶
type KeyCursor struct {
// contains filtered or unexported fields
}
KeyCursor allows iteration through keys in a set of files within a FileStore.
func (*KeyCursor) Next ¶
func (c *KeyCursor) Next()
Next moves the cursor to the next position. Data should be read by the ReadBlock functions.
func (*KeyCursor) ReadBooleanArrayBlock ¶
func (c *KeyCursor) ReadBooleanArrayBlock(values *cursors.BooleanArray) (*cursors.BooleanArray, error)
ReadBooleanArrayBlock reads the next block as a set of boolean values.
func (*KeyCursor) ReadBooleanBlock ¶
func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error)
ReadBooleanBlock reads the next block as a set of boolean values.
func (*KeyCursor) ReadFloatArrayBlock ¶
func (c *KeyCursor) ReadFloatArrayBlock(values *cursors.FloatArray) (*cursors.FloatArray, error)
ReadFloatArrayBlock reads the next block as a set of float values.
func (*KeyCursor) ReadFloatBlock ¶
func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error)
ReadFloatBlock reads the next block as a set of float values.
func (*KeyCursor) ReadIntegerArrayBlock ¶
func (c *KeyCursor) ReadIntegerArrayBlock(values *cursors.IntegerArray) (*cursors.IntegerArray, error)
ReadIntegerArrayBlock reads the next block as a set of integer values.
func (*KeyCursor) ReadIntegerBlock ¶
func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error)
ReadIntegerBlock reads the next block as a set of integer values.
func (*KeyCursor) ReadStringArrayBlock ¶
func (c *KeyCursor) ReadStringArrayBlock(values *cursors.StringArray) (*cursors.StringArray, error)
ReadStringArrayBlock reads the next block as a set of string values.
func (*KeyCursor) ReadStringBlock ¶
func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error)
ReadStringBlock reads the next block as a set of string values.
func (*KeyCursor) ReadUnsignedArrayBlock ¶
func (c *KeyCursor) ReadUnsignedArrayBlock(values *cursors.UnsignedArray) (*cursors.UnsignedArray, error)
ReadUnsignedArrayBlock reads the next block as a set of unsigned values.
func (*KeyCursor) ReadUnsignedBlock ¶
func (c *KeyCursor) ReadUnsignedBlock(buf *[]UnsignedValue) ([]UnsignedValue, error)
ReadUnsignedBlock reads the next block as a set of unsigned values.
type KeyIterator ¶
type KeyIterator interface { // Next returns true if there are any values remaining in the iterator. Next() bool // Read returns the key, time range, and raw data for the next block, // or any error that occurred. Read() (key []byte, minTime int64, maxTime int64, data []byte, err error) // Close closes the iterator. Close() error // Err returns any errors encountered during iteration. Err() error // EstimatedIndexSize returns the estimated size of the index that would // be required to store all the series and entries in the KeyIterator. EstimatedIndexSize() int }
KeyIterator allows iteration over set of keys and values in sorted order.
func NewCacheKeyIterator ¶
func NewCacheKeyIterator(cache *Cache, size int, interrupt chan struct{}) KeyIterator
NewCacheKeyIterator returns a new KeyIterator from a Cache.
func NewTSMBatchKeyIterator ¶
func NewTSMBatchKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)
NewTSMBatchKeyIterator returns a new TSM key iterator from readers. size indicates the maximum number of values to encode in a single block.
func NewTSMKeyIterator ¶
func NewTSMKeyIterator(size int, fast bool, interrupt chan struct{}, readers ...*TSMReader) (KeyIterator, error)
NewTSMKeyIterator returns a new TSM key iterator from readers. size indicates the maximum number of values to encode in a single block.
type MeasurementStats ¶
MeasurementStats represents a set of measurement sizes.
func NewMeasurementStats ¶
func NewMeasurementStats() MeasurementStats
NewStats returns a new instance of Stats.
func (MeasurementStats) Add ¶
func (s MeasurementStats) Add(other MeasurementStats)
Add adds the values of all measurements in other to s.
func (MeasurementStats) MeasurementNames ¶
func (s MeasurementStats) MeasurementNames() []string
MeasurementNames returns a list of sorted measurement names.
func (MeasurementStats) ReadFrom ¶
func (s MeasurementStats) ReadFrom(r io.Reader) (n int64, err error)
ReadFrom reads stats from r in a binary format. Reader must also be an io.ByteReader.
func (MeasurementStats) Sub ¶
func (s MeasurementStats) Sub(other MeasurementStats)
Sub subtracts the values of all measurements in other from s.
type ParseFileNameFunc ¶
ParseFileNameFunc is executed when parsing a TSM filename into generation & sequence.
type Predicate ¶
type Predicate interface { Clone() influxdb.Predicate Matches(key []byte) bool Marshal() ([]byte, error) }
Predicate is something that can match on a series key.
func NewProtobufPredicate ¶
NewProtobufPredicate returns a Predicate that matches based on the comparison structure described by the incoming protobuf.
func UnmarshalPredicate ¶
UnmarshalPredicate takes stored predicate bytes from a Marshal call and returns a Predicate.
type Report ¶
type Report struct { Stderr io.Writer Stdout io.Writer Dir string OrgID, BucketID *influxdb.ID // Calculate only results for the provided org or bucket id. Pattern string // Providing "01.tsm" for example would filter for level 1 files. Detailed bool // Detailed will segment cardinality by tag keys. Exact bool // Exact determines if estimation or exact methods are used to determine cardinality. }
Report runs a report over tsm data
type ReportSummary ¶
type ReportSummary struct {
Min, Max int64
Total uint64 //The exact or estimated unique set of series keys across all files.
Organizations map[string]uint64 // The exact or estimated unique set of series keys segmented by org.
Buckets map[string]uint64 // The exact or estimated unique set of series keys segmented by bucket.
// These are calculated when the detailed flag is in use.
Measurements map[string]uint64 // The exact or estimated unique set of series keys segmented by the measurement tag.
FieldKeys map[string]uint64 // The exact or estimated unique set of series keys segmented by the field tag.
TagKeys map[string]uint64 // The exact or estimated unique set of series keys segmented by tag keys.
}
ReportSummary provides a summary of the cardinalities in the processed fileset.
type SQLBlockExporter ¶
type SQLBlockExporter struct { // Write schema, if true. ShowSchema bool // contains filtered or unexported fields }
SQLBlockExporter writes out all blocks for TSM files to SQL.
func NewSQLBlockExporter ¶
func NewSQLBlockExporter(w io.Writer) *SQLBlockExporter
NewSQLBlockExporter returns a new instance of SQLBlockExporter.
func (*SQLBlockExporter) Close ¶
func (e *SQLBlockExporter) Close() error
Close ends the export and writes final output.
func (*SQLBlockExporter) ExportFile ¶
func (e *SQLBlockExporter) ExportFile(filename string) error
ExportFile writes all blocks of the TSM file.
type Snapshotter ¶
type Snapshotter interface { AcquireSegments(context.Context, func(segments []string) error) error CommitSegments(ctx context.Context, segments []string, fn func() error) error }
Snapshotter allows upward signaling of the tsm1 engine to the storage engine. Hopefully it can be removed one day. The weird interface is due to the weird inversion of locking that has to happen.
type StringDecoder ¶
type StringDecoder struct {
// contains filtered or unexported fields
}
StringDecoder decodes a byte slice into strings.
func (*StringDecoder) Error ¶
func (e *StringDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*StringDecoder) Next ¶
func (e *StringDecoder) Next() bool
Next returns true if there are any values remaining to be decoded.
func (*StringDecoder) Read ¶
func (e *StringDecoder) Read() string
Read returns the next value from the decoder.
func (*StringDecoder) SetBytes ¶
func (e *StringDecoder) SetBytes(b []byte) error
SetBytes initializes the decoder with bytes to read from. This must be called before calling any other method.
type StringEncoder ¶
type StringEncoder struct {
// contains filtered or unexported fields
}
StringEncoder encodes multiple strings into a byte slice.
func NewStringEncoder ¶
func NewStringEncoder(sz int) StringEncoder
NewStringEncoder returns a new StringEncoder with an initial buffer ready to hold sz bytes.
func (*StringEncoder) Bytes ¶
func (e *StringEncoder) Bytes() ([]byte, error)
Bytes returns a copy of the underlying buffer.
func (*StringEncoder) Reset ¶
func (e *StringEncoder) Reset()
Reset sets the encoder back to its initial state.
func (*StringEncoder) Write ¶
func (e *StringEncoder) Write(s string)
Write encodes s to the underlying buffer.
type StringValue ¶
type StringValue = value.StringValue
func DecodeStringBlock ¶
func DecodeStringBlock(block []byte, a *[]StringValue) ([]StringValue, error)
DecodeStringBlock decodes the string block from the byte slice and appends the string values to a.
func NewRawStringValue ¶
func NewRawStringValue(t int64, v string) StringValue
NewRawStringValue returns a new string value.
type StringValues ¶
type StringValues []StringValue
StringValues represents a slice of String values.
func (StringValues) Deduplicate ¶
func (a StringValues) Deduplicate() StringValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (StringValues) Exclude ¶
func (a StringValues) Exclude(min, max int64) StringValues
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (StringValues) FindRange ¶
func (a StringValues) FindRange(min, max int64) (int, int)
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (StringValues) Include ¶
func (a StringValues) Include(min, max int64) StringValues
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (StringValues) Less ¶
func (a StringValues) Less(i, j int) bool
func (StringValues) MaxTime ¶
func (a StringValues) MaxTime() int64
func (StringValues) Merge ¶
func (a StringValues) Merge(b StringValues) StringValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (StringValues) MinTime ¶
func (a StringValues) MinTime() int64
func (StringValues) Size ¶
func (a StringValues) Size() int
func (StringValues) Swap ¶
func (a StringValues) Swap(i, j int)
type TSMFile ¶
type TSMFile interface { // Path returns the underlying file path for the TSMFile. If the file // has not be written or loaded from disk, the zero value is returned. Path() string // Read returns all the values in the block where time t resides. Read(key []byte, t int64) ([]Value, error) // ReadAt returns all the values in the block identified by entry. ReadAt(entry *IndexEntry, values []Value) ([]Value, error) ReadFloatBlockAt(entry *IndexEntry, values *[]FloatValue) ([]FloatValue, error) ReadFloatArrayBlockAt(entry *IndexEntry, values *cursors.FloatArray) error ReadIntegerBlockAt(entry *IndexEntry, values *[]IntegerValue) ([]IntegerValue, error) ReadIntegerArrayBlockAt(entry *IndexEntry, values *cursors.IntegerArray) error ReadUnsignedBlockAt(entry *IndexEntry, values *[]UnsignedValue) ([]UnsignedValue, error) ReadUnsignedArrayBlockAt(entry *IndexEntry, values *cursors.UnsignedArray) error ReadStringBlockAt(entry *IndexEntry, values *[]StringValue) ([]StringValue, error) ReadStringArrayBlockAt(entry *IndexEntry, values *cursors.StringArray) error ReadBooleanBlockAt(entry *IndexEntry, values *[]BooleanValue) ([]BooleanValue, error) ReadBooleanArrayBlockAt(entry *IndexEntry, values *cursors.BooleanArray) error // Entries returns the index entries for all blocks for the given key. ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error) // Contains returns true if the file contains any values for the given // key. Contains(key []byte) bool // OverlapsTimeRange returns true if the time range of the file intersect min and max. OverlapsTimeRange(min, max int64) bool // OverlapsKeyRange returns true if the key range of the file intersects min and max. OverlapsKeyRange(min, max []byte) bool // OverlapsKeyPrefixRange returns true if the key range of the file // intersects min and max, evaluating up to the length of min and max // of the key range. OverlapsKeyPrefixRange(min, max []byte) bool // TimeRange returns the min and max time across all keys in the file. TimeRange() (int64, int64) // TombstoneRange returns ranges of time that are deleted for the given key. TombstoneRange(key []byte, buf []TimeRange) []TimeRange // KeyRange returns the min and max keys in the file. KeyRange() ([]byte, []byte) // KeyCount returns the number of distinct keys in the file. KeyCount() int // Iterator returns an iterator over the keys starting at the provided key. You must // call Next before calling any of the accessors. Iterator([]byte) TSMIterator // Type returns the block type of the values stored for the key. Returns one of // BlockFloat64, BlockInt64, BlockBoolean, BlockString. If key does not exist, // an error is returned. Type(key []byte) (byte, error) // BatchDelete return a BatchDeleter that allows for multiple deletes in batches // and group commit or rollback. BatchDelete() BatchDeleter // Delete removes the keys from the set of keys available in this file. Delete(keys [][]byte) error // DeleteRange removes the values for keys between timestamps min and max. DeleteRange(keys [][]byte, min, max int64) error // DeletePrefix removes the values for keys beginning with prefix. It calls dead with // any keys that became dead as a result of this call. DeletePrefix(prefix []byte, min, max int64, pred Predicate, dead func([]byte)) error // HasTombstones returns true if file contains values that have been deleted. HasTombstones() bool // TombstoneFiles returns the tombstone filestats if there are any tombstones // written for this file. TombstoneFiles() []FileStat // Close closes the underlying file resources. Close() error // Size returns the size of the file on disk in bytes. Size() uint32 // Rename renames the existing TSM file to a new name and replaces the mmap backing slice using the new // file name. Index and Reader state are not re-initialized. Rename(path string) error // Remove deletes the file from the filesystem. Remove() error // InUse returns true if the file is currently in use by queries. InUse() bool // Ref records that this file is actively in use. Ref() // Unref records that this file is no longer in use. Unref() // Stats returns summary information about the TSM file. Stats() FileStat // BlockIterator returns an iterator pointing to the first block in the file and // allows sequential iteration to each and every block. BlockIterator() *BlockIterator // TimeRangeIterator returns an iterator over the keys, starting at the provided // key. Calling the HasData accessor will return true if data exists for the // interval [min, max] for the current key. // Next must be called before calling any of the accessors. TimeRangeIterator(key []byte, min, max int64) *TimeRangeIterator // TimeRangeMaxTimeIterator returns an iterator over the keys, starting at the provided // key. Calling the HasData and MaxTime accessors will be restricted to the // interval [min, max] for the current key. // Next must be called before calling any of the accessors. TimeRangeMaxTimeIterator(key []byte, min, max int64) *TimeRangeMaxTimeIterator // Free releases any resources held by the FileStore to free up system resources. Free() error // Stats returns the statistics for the file. MeasurementStats() (MeasurementStats, error) }
TSMFile represents an on-disk TSM file.
type TSMIndex ¶
type TSMIndex interface { // Delete removes the given keys from the index. Returns true if there were any changes. Delete(keys [][]byte) bool // DeleteRange removes the given keys with data between minTime and maxTime from the index. // Returns true if there were any changes. DeleteRange(keys [][]byte, minTime, maxTime int64) bool // DeletePrefix removes keys that begin with the given prefix with data between minTime and // maxTime from the index. Returns true if there were any changes. It calls dead with any // keys that became dead as a result of this call. DeletePrefix(prefix []byte, minTime, maxTime int64, pred Predicate, dead func([]byte)) bool // MaybeContainsKey returns true if the given key may exist in the index. This is faster than // Contains but, may return false positives. MaybeContainsKey(key []byte) bool // Contains return true if the given key exists in the index. Contains(key []byte) bool // MaybeContainsValue returns true if key and time might exist in this file. This function // could return true even though the actual point does not exists. For example, the key may // exist in this file, but not have a point exactly at time t. MaybeContainsValue(key []byte, timestamp int64) bool // ReadEntries reads the index entries for key into entries. ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error) // Entry returns the index entry for the specified key and timestamp. If no entry // matches the key and timestamp, nil is returned. Entry(key []byte, timestamp int64) *IndexEntry // KeyCount returns the count of unique keys in the index. KeyCount() int // Iterator returns an iterator over the keys starting at the provided key. You must // call Next before calling any of the accessors. Iterator([]byte) *TSMIndexIterator // OverlapsTimeRange returns true if the time range of the file intersect min and max. OverlapsTimeRange(min, max int64) bool // OverlapsKeyRange returns true if the min and max keys of the file overlap the arguments min and max. OverlapsKeyRange(min, max []byte) bool // OverlapsKeyPrefixRange returns true if the key range of the file // intersects min and max, evaluating up to the length of min and max // of the key range. OverlapsKeyPrefixRange(min, max []byte) bool // Size returns the size of the current index in bytes. Size() uint32 // TimeRange returns the min and max time across all keys in the file. TimeRange() (int64, int64) // TombstoneRange returns ranges of time that are deleted for the given key. TombstoneRange(key []byte, buf []TimeRange) []TimeRange // KeyRange returns the min and max keys in the file. KeyRange() ([]byte, []byte) // Type returns the block type of the values stored for the key. Returns one of // BlockFloat64, BlockInt64, BlockBool, BlockString. If key does not exist, // an error is returned. Type(key []byte) (byte, error) // UnmarshalBinary populates an index from an encoded byte slice // representation of an index. UnmarshalBinary(b []byte) error // Close closes the index and releases any resources. Close() error }
TSMIndex represent the index section of a TSM file. The index records all blocks, their locations, sizes, min and max times.
type TSMIndexIterator ¶
type TSMIndexIterator struct {
// contains filtered or unexported fields
}
TSMIndexIterator allows one to iterate over the TSM index.
func (*TSMIndexIterator) Entries ¶
func (t *TSMIndexIterator) Entries() []IndexEntry
Entries reports the current list of entries.
func (*TSMIndexIterator) Err ¶
func (t *TSMIndexIterator) Err() error
Err reports if an error stopped the iteration.
func (*TSMIndexIterator) Next ¶
func (t *TSMIndexIterator) Next() bool
Next advances the iterator and reports if it is still valid.
func (*TSMIndexIterator) Peek ¶
func (t *TSMIndexIterator) Peek() []byte
Peek reports the next key or nil if there is not one or an error happened.
func (*TSMIndexIterator) Seek ¶
func (t *TSMIndexIterator) Seek(key []byte) (exact, ok bool)
Seek points the iterator at the smallest key greater than or equal to the given key, returning true if it was an exact match. It returns false for ok if the key does not exist.
func (*TSMIndexIterator) Type ¶
func (t *TSMIndexIterator) Type() byte
Type reports the current type.
type TSMIterator ¶
type TSMReader ¶
type TSMReader struct {
// contains filtered or unexported fields
}
TSMReader is a reader for a TSM file.
func NewTSMReader ¶
NewTSMReader returns a new TSMReader from the given file.
func (*TSMReader) BatchDelete ¶
func (r *TSMReader) BatchDelete() BatchDeleter
BatchDelete returns a BatchDeleter. Only a single goroutine may run a BatchDelete at a time. Callers must either Commit or Rollback the operation.
func (*TSMReader) BlockIterator ¶
func (t *TSMReader) BlockIterator() *BlockIterator
BlockIterator returns a BlockIterator for the underlying TSM file.
func (*TSMReader) DeletePrefix ¶
func (t *TSMReader) DeletePrefix(prefix []byte, minTime, maxTime int64, pred Predicate, dead func([]byte)) error
DeletePrefix removes the given points for keys beginning with prefix. It calls dead with any keys that became dead as a result of this call.
func (*TSMReader) DeleteRange ¶
DeleteRange removes the given points for keys between minTime and maxTime. The series keys passed in must be sorted.
func (*TSMReader) HasTombstones ¶
HasTombstones return true if there are any tombstone entries recorded.
func (*TSMReader) Iterator ¶
func (t *TSMReader) Iterator(key []byte) TSMIterator
Iterator returns an iterator over the keys starting at the provided key. You must call Next before calling any of the accessors.
func (*TSMReader) LastModified ¶
LastModified returns the last time the underlying file was modified.
func (*TSMReader) MaybeContainsValue ¶
MaybeContainsValue returns true if key and time might exists in this file. This function could return true even though the actual point does not exist. For example, the key may exist in this file, but not have a point exactly at time t.
func (*TSMReader) MeasurementStats ¶
func (t *TSMReader) MeasurementStats() (MeasurementStats, error)
MeasurementStats returns the on-disk measurement stats for this file, if available.
func (*TSMReader) OverlapsKeyPrefixRange ¶
OverlapsKeyPrefixRange returns true if the key range of the file intersects min and max, evaluating up to the length of min and max of the key range.
func (*TSMReader) OverlapsKeyRange ¶
OverlapsKeyRange returns true if the key range of the file intersect min and max.
func (*TSMReader) OverlapsTimeRange ¶
OverlapsTimeRange returns true if the time range of the file intersect min and max.
func (*TSMReader) Read ¶
Read returns the values corresponding to the block at the given key and timestamp.
func (*TSMReader) ReadAt ¶
func (t *TSMReader) ReadAt(entry *IndexEntry, vals []Value) ([]Value, error)
ReadAt returns the values corresponding to the given index entry.
func (*TSMReader) ReadBooleanArrayBlockAt ¶
func (t *TSMReader) ReadBooleanArrayBlockAt(entry *IndexEntry, vals *cursors.BooleanArray) error
ReadBooleanArrayBlockAt fills vals with the boolean values corresponding to the given index entry.
func (*TSMReader) ReadBooleanBlockAt ¶
func (t *TSMReader) ReadBooleanBlockAt(entry *IndexEntry, vals *[]BooleanValue) ([]BooleanValue, error)
ReadBooleanBlockAt returns the boolean values corresponding to the given index entry.
func (*TSMReader) ReadEntries ¶
func (t *TSMReader) ReadEntries(key []byte, entries []IndexEntry) ([]IndexEntry, error)
ReadEntries reads the index entries for key into entries.
func (*TSMReader) ReadFloatArrayBlockAt ¶
func (t *TSMReader) ReadFloatArrayBlockAt(entry *IndexEntry, vals *cursors.FloatArray) error
ReadFloatArrayBlockAt fills vals with the float values corresponding to the given index entry.
func (*TSMReader) ReadFloatBlockAt ¶
func (t *TSMReader) ReadFloatBlockAt(entry *IndexEntry, vals *[]FloatValue) ([]FloatValue, error)
ReadFloatBlockAt returns the float values corresponding to the given index entry.
func (*TSMReader) ReadIntegerArrayBlockAt ¶
func (t *TSMReader) ReadIntegerArrayBlockAt(entry *IndexEntry, vals *cursors.IntegerArray) error
ReadIntegerArrayBlockAt fills vals with the integer values corresponding to the given index entry.
func (*TSMReader) ReadIntegerBlockAt ¶
func (t *TSMReader) ReadIntegerBlockAt(entry *IndexEntry, vals *[]IntegerValue) ([]IntegerValue, error)
ReadIntegerBlockAt returns the integer values corresponding to the given index entry.
func (*TSMReader) ReadStringArrayBlockAt ¶
func (t *TSMReader) ReadStringArrayBlockAt(entry *IndexEntry, vals *cursors.StringArray) error
ReadStringArrayBlockAt fills vals with the string values corresponding to the given index entry.
func (*TSMReader) ReadStringBlockAt ¶
func (t *TSMReader) ReadStringBlockAt(entry *IndexEntry, vals *[]StringValue) ([]StringValue, error)
ReadStringBlockAt returns the string values corresponding to the given index entry.
func (*TSMReader) ReadUnsignedArrayBlockAt ¶
func (t *TSMReader) ReadUnsignedArrayBlockAt(entry *IndexEntry, vals *cursors.UnsignedArray) error
ReadUnsignedArrayBlockAt fills vals with the unsigned values corresponding to the given index entry.
func (*TSMReader) ReadUnsignedBlockAt ¶
func (t *TSMReader) ReadUnsignedBlockAt(entry *IndexEntry, vals *[]UnsignedValue) ([]UnsignedValue, error)
ReadUnsignedBlockAt returns the unsigned values corresponding to the given index entry.
func (*TSMReader) Ref ¶
func (t *TSMReader) Ref()
Ref records a usage of this TSMReader. If there are active references when the reader is closed or removed, the reader will remain open until there are no more references.
func (*TSMReader) TimeRangeIterator ¶
func (t *TSMReader) TimeRangeIterator(key []byte, min, max int64) *TimeRangeIterator
TimeRangeIterator returns an iterator over the keys, starting at the provided key. Calling the HasData accessor will return true if data exists for the interval [min, max] for the current key. Next must be called before calling any of the accessors.
func (*TSMReader) TimeRangeMaxTimeIterator ¶
func (t *TSMReader) TimeRangeMaxTimeIterator(key []byte, min, max int64) *TimeRangeMaxTimeIterator
TimeRangeMaxTimeIterator returns an iterator over the keys, starting at the provided key. Calling the HasData and MaxTime accessors will be restricted to the interval [min, max] for the current key and MaxTime ≤ max. Next must be called before calling any of the accessors.
func (*TSMReader) TombstoneFiles ¶
TombstoneFiles returns any tombstone files associated with this TSM file.
func (*TSMReader) TombstoneRange ¶
TombstoneRange returns ranges of time that are deleted for the given key.
func (*TSMReader) Unref ¶
func (t *TSMReader) Unref()
Unref removes a usage record of this TSMReader. If the Reader was closed by another goroutine while there were active references, the file will be closed and remove
func (*TSMReader) WithObserver ¶
func (t *TSMReader) WithObserver(obs FileStoreObserver)
WithObserver sets the observer for the TSM reader.
type TSMWriter ¶
type TSMWriter interface { // Write writes a new block for key containing and values. Writes append // blocks in the order that the Write function is called. The caller is // responsible for ensuring keys and blocks are sorted appropriately. // Values are encoded as a full block. The caller is responsible for // ensuring a fixed number of values are encoded in each block as well as // ensuring the Values are sorted. The first and last timestamp values are // used as the minimum and maximum values for the index entry. Write(key []byte, values Values) error // WriteBlock writes a new block for key containing the bytes in block. WriteBlock appends // blocks in the order that the WriteBlock function is called. The caller is // responsible for ensuring keys and blocks are sorted appropriately, and that the // block and index information is correct for the block. The minTime and maxTime // timestamp values are used as the minimum and maximum values for the index entry. WriteBlock(key []byte, minTime, maxTime int64, block []byte) error // WriteIndex finishes the TSM write streams and writes the index. WriteIndex() error // Flushes flushes all pending changes to the underlying file resources. Flush() error // Close closes any underlying file resources. Close() error // Size returns the current size in bytes of the file. Size() uint32 // Stats returns the statistics generated by the writer. MeasurementStats() MeasurementStats Remove() error }
TSMWriter writes TSM formatted key and values.
func NewTSMWriter ¶
NewTSMWriter returns a new TSMWriter writing to w.
type TimeDecoder ¶
type TimeDecoder struct {
// contains filtered or unexported fields
}
TimeDecoder decodes a byte slice into timestamps.
func (*TimeDecoder) Error ¶
func (d *TimeDecoder) Error() error
Error returns the last error encountered by the decoder.
func (*TimeDecoder) Init ¶
func (d *TimeDecoder) Init(b []byte)
Init initializes the decoder with bytes to read from.
func (*TimeDecoder) Next ¶
func (d *TimeDecoder) Next() bool
Next returns true if there are any timestamps remaining to be decoded.
func (*TimeDecoder) Read ¶
func (d *TimeDecoder) Read() int64
Read returns the next timestamp from the decoder.
type TimeEncoder ¶
TimeEncoder encodes time.Time to byte slices.
func NewTimeEncoder ¶
func NewTimeEncoder(sz int) TimeEncoder
NewTimeEncoder returns a TimeEncoder with an initial buffer ready to hold sz bytes.
type TimeRangeIterator ¶
type TimeRangeIterator struct {
// contains filtered or unexported fields
}
TimeRangeIterator will iterate over the keys of a TSM file, starting at the provided key. It is used to determine if each key has data which exists within a specified time interval.
func (*TimeRangeIterator) HasData ¶
func (b *TimeRangeIterator) HasData() bool
HasData reports true if the current key has data for the time range.
func (*TimeRangeIterator) Key ¶
func (b *TimeRangeIterator) Key() []byte
Key reports the current key.
func (*TimeRangeIterator) Next ¶
func (b *TimeRangeIterator) Next() bool
Next advances the iterator and reports if it is still valid.
func (*TimeRangeIterator) Seek ¶
func (b *TimeRangeIterator) Seek(key []byte) (exact, ok bool)
Seek points the iterator at the smallest key greater than or equal to the given key, returning true if it was an exact match. It returns false for ok if the key does not exist.
func (*TimeRangeIterator) Stats ¶
func (b *TimeRangeIterator) Stats() cursors.CursorStats
Stats returns statistics accumulated by the iterator for any block reads.
type TimeRangeMaxTimeIterator ¶
type TimeRangeMaxTimeIterator struct {
// contains filtered or unexported fields
}
TimeRangeMaxTimeIterator will iterate over the keys of a TSM file, starting at the provided key. It is used to determine if each key has data which exists within a specified time interval.
func (*TimeRangeMaxTimeIterator) HasData ¶
func (b *TimeRangeMaxTimeIterator) HasData() bool
HasData reports true if the current key has data for the time range.
func (*TimeRangeMaxTimeIterator) Key ¶
func (b *TimeRangeMaxTimeIterator) Key() []byte
Key reports the current key.
func (*TimeRangeMaxTimeIterator) MaxTime ¶
func (b *TimeRangeMaxTimeIterator) MaxTime() int64
MaxTime returns the maximum timestamp for the current key within the requested time range. If an error occurred or there is no data, InvalidMinTimeStamp will be returned, which is less than models.MinTimeStamp. This property can be leveraged when enumerating keys to find the maximum timestamp, as this value will always be lower than any valid timestamp returned.
NOTE: If MaxTime is equal to the upper bounds of the queried time range, it means data was found equal to or beyond the requested time range and does not mean that data exists at that specific timestamp.
func (*TimeRangeMaxTimeIterator) Next ¶
func (b *TimeRangeMaxTimeIterator) Next() bool
Next advances the iterator and reports if it is still valid.
func (*TimeRangeMaxTimeIterator) Seek ¶
func (b *TimeRangeMaxTimeIterator) Seek(key []byte) (exact, ok bool)
Seek points the iterator at the smallest key greater than or equal to the given key, returning true if it was an exact match. It returns false for ok if the key does not exist.
func (*TimeRangeMaxTimeIterator) Stats ¶
func (b *TimeRangeMaxTimeIterator) Stats() cursors.CursorStats
Stats returns statistics accumulated by the iterator for any block reads.
type Tombstone ¶
type Tombstone struct { // Key is the tombstoned series key. Key []byte // Prefix indicates if this tombstone entry is a prefix key, meaning all // keys with a prefix matching Key should be removed for the [Min, Max] range. Prefix bool // Min and Max are the min and max unix nanosecond time ranges of Key that are deleted. Min, Max int64 // Predicate stores the marshaled form of some predicate for matching keys. Predicate []byte }
Tombstone represents an individual deletion.
type Tombstoner ¶
type Tombstoner struct { // Path is the location of the file to record tombstone. This should be the // full path to a TSM file. Path string FilterFn func(k []byte) bool // contains filtered or unexported fields }
Tombstoner records tombstones when entries are deleted.
func NewTombstoner ¶
func NewTombstoner(path string, filterFn func(k []byte) bool) *Tombstoner
NewTombstoner constructs a Tombstoner for the given path. FilterFn can be nil.
func (*Tombstoner) Add ¶
func (t *Tombstoner) Add(keys [][]byte) error
Add adds the all keys, across all timestamps, to the tombstone.
func (*Tombstoner) AddPrefixRange ¶
func (t *Tombstoner) AddPrefixRange(key []byte, min, max int64, predicate []byte) error
AddPrefixRange adds a prefix-based tombstone key with an explicit range.
func (*Tombstoner) AddRange ¶
func (t *Tombstoner) AddRange(keys [][]byte, min, max int64) error
AddRange adds all keys to the tombstone specifying only the data between min and max to be removed.
func (*Tombstoner) Delete ¶
func (t *Tombstoner) Delete() error
Delete removes all the tombstone files from disk.
func (*Tombstoner) Flush ¶
func (t *Tombstoner) Flush() error
func (*Tombstoner) HasTombstones ¶
func (t *Tombstoner) HasTombstones() bool
HasTombstones return true if there are any tombstone entries recorded.
func (*Tombstoner) Rollback ¶
func (t *Tombstoner) Rollback() error
func (*Tombstoner) TombstoneFiles ¶
func (t *Tombstoner) TombstoneFiles() []FileStat
TombstoneFiles returns any tombstone files associated with Tombstoner's TSM file.
func (*Tombstoner) Walk ¶
func (t *Tombstoner) Walk(fn func(t Tombstone) error) error
Walk calls fn for every Tombstone under the Tombstoner.
func (*Tombstoner) WithObserver ¶
func (t *Tombstoner) WithObserver(obs FileStoreObserver)
WithObserver sets a FileStoreObserver for when the tombstone file is written.
type UnsignedValue ¶
type UnsignedValue = value.UnsignedValue
func DecodeUnsignedBlock ¶
func DecodeUnsignedBlock(block []byte, a *[]UnsignedValue) ([]UnsignedValue, error)
DecodeUnsignedBlock decodes the unsigned integer block from the byte slice and appends the unsigned integer values to a.
func NewRawUnsignedValue ¶
func NewRawUnsignedValue(t int64, v uint64) UnsignedValue
NewRawUnsignedValue returns a new unsigned integer value.
type UnsignedValues ¶
type UnsignedValues []UnsignedValue
UnsignedValues represents a slice of Unsigned values.
func (UnsignedValues) Deduplicate ¶
func (a UnsignedValues) Deduplicate() UnsignedValues
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (UnsignedValues) Exclude ¶
func (a UnsignedValues) Exclude(min, max int64) UnsignedValues
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (UnsignedValues) FindRange ¶
func (a UnsignedValues) FindRange(min, max int64) (int, int)
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (UnsignedValues) Include ¶
func (a UnsignedValues) Include(min, max int64) UnsignedValues
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (UnsignedValues) Less ¶
func (a UnsignedValues) Less(i, j int) bool
func (UnsignedValues) MaxTime ¶
func (a UnsignedValues) MaxTime() int64
func (UnsignedValues) Merge ¶
func (a UnsignedValues) Merge(b UnsignedValues) UnsignedValues
Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.
func (UnsignedValues) MinTime ¶
func (a UnsignedValues) MinTime() int64
func (UnsignedValues) Size ¶
func (a UnsignedValues) Size() int
func (UnsignedValues) Swap ¶
func (a UnsignedValues) Swap(i, j int)
type Value ¶
func DecodeBlock ¶
DecodeBlock takes a byte slice and decodes it into values of the appropriate type based on the block.
func NewBooleanValue ¶
NewBooleanValue returns a new boolean value.
func NewFloatValue ¶
NewFloatValue returns a new float value.
func NewIntegerValue ¶
NewIntegerValue returns a new integer value.
func NewStringValue ¶
NewStringValue returns a new string value.
func NewUnsignedValue ¶
NewUnsignedValue returns a new unsigned integer value.
type Values ¶
type Values []Value
Values represents a slice of values.
func (Values) Contains ¶
Contains returns true if values exist for the time interval [min, max] inclusive. The values must be sorted before calling Contains or the results are undefined.
func (Values) Deduplicate ¶
Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept. The returned Values are sorted if necessary.
func (Values) Encode ¶
Encode converts the values to a byte slice. If there are no values, this function panics.
func (Values) Exclude ¶
Exclude returns the subset of values not in [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (Values) FindRange ¶
FindRange returns the positions where min and max would be inserted into the array. If a[0].UnixNano() > max or a[len-1].UnixNano() < min then FindRange returns (-1, -1) indicating the array is outside the [min, max]. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (Values) Include ¶
Include returns the subset values between min and max inclusive. The values must be deduplicated and sorted before calling Exclude or the results are undefined.
func (Values) InfluxQLType ¶
InfluxQLType returns the influxql.DataType the values map to.
type WALConfig ¶
type WALConfig struct { // Enabled controls if the WAL is enabled. Enabled bool `toml:"enabled"` // WALFsyncDelay is the amount of time that a write will wait before fsyncing. A // duration greater than 0 can be used to batch up multiple fsync calls. This is // useful for slower disks or when WAL write contention is seen. A value of 0 fsyncs // every write to the WAL. FsyncDelay toml.Duration `toml:"fsync-delay"` }
WALConfig holds all of the configuration about the WAL.
func NewWALConfig ¶
func NewWALConfig() WALConfig
Source Files ¶
- array_cursor.gen.go
- array_cursor_iterator.gen.go
- array_cursor_iterator.go
- array_encoding.go
- batch_boolean.go
- batch_float.go
- batch_integer.go
- batch_string.go
- batch_timestamp.go
- bit_reader.go
- block_exporter.go
- bool.go
- cache.go
- cache_entry.go
- cachestatus_string.go
- compact.gen.go
- compact.go
- config.go
- encoding.gen.go
- encoding.go
- engine.go
- engine_cursor.go
- engine_delete_prefix.go
- engine_measurement_schema.go
- engine_schema.go
- errors.go
- file_store.gen.go
- file_store.go
- file_store_array.gen.go
- file_store_key_iterator.go
- file_store_observer.go
- float.go
- int.go
- metrics.go
- mmap_unix.go
- predicate.go
- reader.gen.go
- reader.go
- reader_block_iterator.go
- reader_fault_buffer.go
- reader_index.go
- reader_index_iterator.go
- reader_mmap.go
- reader_offsets.go
- reader_prefix_tree.go
- reader_range_iterator.go
- reader_range_maxtime_iterator.go
- reader_time_range.go
- report.go
- ring.go
- scheduler.go
- stats.go
- string.go
- timestamp.go
- tombstone.go
- value.go
- verify_tsm.go
- writer.go