tsm1

package
v1.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 6, 2016 License: MIT Imports: 31 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CompactionTempExtension = "tmp"
	TSMFileExtension        = "tsm"
)
View Source 
const (
	// BlockFloat64 designates a block encodes float64 values
	BlockFloat64 = byte(0)

	// BlockInteger designates a block encodes int64 values
	BlockInteger = byte(1)

	// BlockBoolean designates a block encodes boolean values
	BlockBoolean = byte(2)

	// BlockString designates a block encodes string values
	BlockString = byte(3)
)
View Source 
const (
	// DefaultSegmentSize of 10MB is the size at which segment files will be rolled over
	DefaultSegmentSize = 10 * 1024 * 1024

	// FileExtension is the file extension we expect for wal segments
	WALFileExtension = "wal"

	WALFilePrefix = "_"
)
View Source 
const (
	// MagicNumber is written as the first 4 bytes of a data file to
	// identify the file as a tsm1 formatted file
	MagicNumber uint32 = 0x16D116D1

	Version byte = 1
)

Variables

View Source 
var (
	ErrCacheInvalidCheckpoint = fmt.Errorf("invalid checkpoint")
	ErrSnapshotInProgress     = fmt.Errorf("snapshot in progress")
)
View Source 
var (
	MaxTime = time.Unix(0, math.MaxInt64)
	MinTime = time.Unix(0, 0)
)
View Source 
var (
	ErrWALClosed  = fmt.Errorf("WAL closed")
	ErrWALCorrupt = fmt.Errorf("corrupted WAL entry")
)
View Source 
var (
	ErrNoValues             = fmt.Errorf("no values written")
	ErrTSMClosed            = fmt.Errorf("tsm file closed")
	ErrMaxKeyLengthExceeded = fmt.Errorf("max key length exceeded")
	ErrMaxBlocksExceeded    = fmt.Errorf("max blocks exceeded")
)
View Source 
var ErrFileInUse = fmt.Errorf("file still in use")

Functions

func BlockCount added in v0.10.0 

func BlockCount(block []byte) int

func BlockType added in v0.10.0 

func BlockType(block []byte) (byte, error)

BlockType returns the type of value encoded in a block or an error if the block type is unknown.

func CountTimestamps added in v0.10.0 

func CountTimestamps(b []byte) int

func ErrCacheMemorySizeLimitExceeded added in v1.1.1 

func ErrCacheMemorySizeLimitExceeded(n, limit uint64) error

func NewEngine 

func NewEngine(id uint64, path string, walPath string, opt tsdb.EngineOptions) tsdb.Engine

NewEngine returns a new instance of Engine.

func NewIndirectIndex 

func NewIndirectIndex() *indirectIndex

func NewMultiFieldCursor 

func NewMultiFieldCursor(fields []string, cursors []tsdb.Cursor, ascending bool) tsdb.Cursor

NewMultiFieldCursor returns an instance of Cursor that joins the results of cursors.

func ParseTSMFileName added in v0.10.0 

func ParseTSMFileName(name string) (int, int, error)

ParseTSMFileName parses the generation and sequence from a TSM file name.

func SeriesAndFieldFromCompositeKey added in v1.0.0 

func SeriesAndFieldFromCompositeKey(key []byte) ([]byte, string)

func SeriesFieldKey 

func SeriesFieldKey(seriesKey, field string) string

SeriesFieldKey combine a series key and field name for a unique string to be hashed to a numeric ID

func ZigZagDecode 

func ZigZagDecode(v uint64) int64

ZigZagDecode converts a previously zigzag encoded uint64 back to a int64

func ZigZagEncode 

func ZigZagEncode(x int64) uint64

ZigZagEncode converts a int64 to a uint64 by zig zagging negative and positive values across even and odd numbers. Eg. [0,-1,1,-2] becomes [0, 1, 2, 3]

Types

type BitReader added in v1.0.0 

type BitReader struct {
	// contains filtered or unexported fields
}

BitReader reads bits from an io.Reader.

func NewBitReader added in v1.0.0 

func NewBitReader(data []byte) *BitReader

NewBitReader returns a new instance of BitReader that reads from data.

func (*BitReader) CanReadBitFast added in v1.0.0 

func (r *BitReader) CanReadBitFast() bool

CanReadBitFast returns true if calling ReadBitFast() is allowed. Fast bit reads are allowed when at least 2 values are in the buffer. This is because it is not required to refilled the buffer and the caller can inline the calls.

func (*BitReader) ReadBit added in v1.0.0 

func (r *BitReader) ReadBit() (bool, error)

ReadBit returns the next bit from the underlying data.

func (*BitReader) ReadBitFast added in v1.0.0 

func (r *BitReader) ReadBitFast() bool

ReadBitFast is an optimized bit read. IMPORTANT: Only allowed if CanReadFastBit() is true!

func (*BitReader) ReadBits added in v1.0.0 

func (r *BitReader) ReadBits(nbits uint) (uint64, error)

ReadBits reads nbits from the underlying data.

func (*BitReader) Reset added in v1.0.0 

func (r *BitReader) Reset(data []byte)

Reset sets the underlying reader on b and reinitializes.

type BlockIterator added in v0.10.0 

type BlockIterator struct {
	// contains filtered or unexported fields
}

BlockIterator allows iterating over each block in a TSM file in order. It provides raw access to the block bytes without decoding them.

func (*BlockIterator) Next added in v0.10.0 

func (b *BlockIterator) Next() bool

func (*BlockIterator) PeekNext added in v0.10.0 

func (b *BlockIterator) PeekNext() string

func (*BlockIterator) Read added in v0.10.0 

func (b *BlockIterator) Read() (string, int64, int64, uint32, []byte, error)

type BooleanDecoder added in v0.12.0 

type BooleanDecoder struct {
	// contains filtered or unexported fields
}

BooleanDecoder decodes a series of booleans from an in-memory buffer.

func (*BooleanDecoder) Error added in v0.12.0 

func (e *BooleanDecoder) Error() error

func (*BooleanDecoder) Next added in v0.12.0 

func (e *BooleanDecoder) Next() bool

func (*BooleanDecoder) Read added in v0.12.0 

func (e *BooleanDecoder) Read() bool

func (*BooleanDecoder) SetBytes added in v1.0.0 

func (e *BooleanDecoder) SetBytes(b []byte)

SetBytes initializes the decoder with a new set of bytes to read from. This must be called before calling any other methods.

type BooleanEncoder added in v0.12.0 

type BooleanEncoder struct {
	// contains filtered or unexported fields
}

BooleanEncoder encodes a series of booleans to an in-memory buffer.

func NewBooleanEncoder added in v0.12.0 

func NewBooleanEncoder(sz int) BooleanEncoder

NewBooleanEncoder returns a new instance of BooleanEncoder.

func (*BooleanEncoder) Bytes added in v0.12.0 

func (e *BooleanEncoder) Bytes() ([]byte, error)

func (*BooleanEncoder) Reset added in v1.1.1 

func (e *BooleanEncoder) Reset()

func (*BooleanEncoder) Write added in v0.12.0 

func (e *BooleanEncoder) Write(b bool)

type BooleanValue added in v0.12.0 

type BooleanValue struct {
	// contains filtered or unexported fields
}

func DecodeBooleanBlock added in v0.12.0 

func DecodeBooleanBlock(block []byte, a *[]BooleanValue) ([]BooleanValue, error)

func (*BooleanValue) Size added in v0.12.0 

func (b *BooleanValue) Size() int

func (*BooleanValue) String added in v0.12.0 

func (f *BooleanValue) String() string

func (*BooleanValue) UnixNano added in v0.12.0 

func (b *BooleanValue) UnixNano() int64

func (*BooleanValue) Value added in v0.12.0 

func (b *BooleanValue) Value() interface{}

type BooleanValues added in v0.12.0 

type BooleanValues []BooleanValue

BooleanValues represents a slice of Boolean values.

func (BooleanValues) Deduplicate added in v0.12.0 

func (a BooleanValues) Deduplicate() BooleanValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.

func (BooleanValues) Exclude added in v1.0.0 

func (a BooleanValues) Exclude(min, max int64) BooleanValues

Exclude returns the subset of values not in [min, max]

func (BooleanValues) Include added in v1.0.0 

func (a BooleanValues) Include(min, max int64) BooleanValues

Include returns the subset values between min and max inclusive.

func (BooleanValues) Len added in v0.12.0 

func (a BooleanValues) Len() int

Sort methods

func (BooleanValues) Less added in v0.12.0 

func (a BooleanValues) Less(i, j int) bool

func (BooleanValues) MaxTime added in v1.0.0 

func (a BooleanValues) MaxTime() int64

func (BooleanValues) Merge added in v1.0.0 

func (a BooleanValues) Merge(b BooleanValues) BooleanValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (BooleanValues) MinTime added in v1.0.0 

func (a BooleanValues) MinTime() int64

func (BooleanValues) Size added in v1.0.0 

func (a BooleanValues) Size() int

func (BooleanValues) Swap added in v0.12.0 

func (a BooleanValues) Swap(i, j int)

type Cache added in v0.10.0 

type Cache struct {
	// contains filtered or unexported fields
}

Cache maintains an in-memory store of Values for a set of keys.

func NewCache added in v0.10.0 

func NewCache(maxSize uint64, path string) *Cache

NewCache returns an instance of a cache which will use a maximum of maxSize bytes of memory. Only used for engine caches, never for snapshots

func (*Cache) ClearSnapshot added in v0.10.0 

func (c *Cache) ClearSnapshot(success bool)

ClearSnapshot will remove the snapshot cache from the list of flushing caches and adjust the size

func (*Cache) Deduplicate added in v0.10.0 

func (c *Cache) Deduplicate()

Deduplicate sorts the snapshot before returning it. The compactor and any queries coming in while it writes will need the values sorted

func (*Cache) Delete added in v0.10.0 

func (c *Cache) Delete(keys []string)

Delete will remove the keys from the cache

func (*Cache) DeleteRange added in v1.0.0 

func (c *Cache) DeleteRange(keys []string, min, max int64)

DeleteRange will remove the values for all keys containing points between min and max from the cache.

func (*Cache) Keys added in v0.10.0 

func (c *Cache) Keys() []string

Keys returns a sorted slice of all keys under management by the cache.

func (*Cache) MaxSize added in v0.10.0 

func (c *Cache) MaxSize() uint64

MaxSize returns the maximum number of bytes the cache may consume.

func (*Cache) RLock added in v0.12.0 

func (c *Cache) RLock()

func (*Cache) RUnlock added in v0.12.0 

func (c *Cache) RUnlock()

func (*Cache) SetMaxSize added in v1.0.0 

func (c *Cache) SetMaxSize(size uint64)

func (*Cache) Size added in v0.10.0 

func (c *Cache) Size() uint64

Size returns the number of point-calcuated bytes the cache currently uses.

func (*Cache) Snapshot added in v0.10.0 

func (c *Cache) Snapshot() (*Cache, error)

Snapshot will take a snapshot of the current cache, add it to the slice of caches that are being flushed, and reset the current cache with new values

func (*Cache) Statistics added in v1.0.0 

func (c *Cache) Statistics(tags map[string]string) []models.Statistic

Statistics returns statistics for periodic monitoring.

func (*Cache) Store added in v0.10.0 

func (c *Cache) Store() map[string]*entry

Store returns the underlying cache store. This is not goroutine safe! Protect access by using the Lock and Unlock functions on Cache.

func (*Cache) UpdateAge added in v0.12.0 

func (c *Cache) UpdateAge()

Updates the age statistic

func (*Cache) UpdateCompactTime added in v0.12.0 

func (c *Cache) UpdateCompactTime(d time.Duration)

Updates WAL compaction time statistic

func (*Cache) Values added in v0.10.0 

func (c *Cache) Values(key string) Values

Values returns a copy of all values, deduped and sorted, for the given key.

func (*Cache) Write added in v0.10.0 

func (c *Cache) Write(key string, values []Value) error

Write writes the set of values for the key to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values.

func (*Cache) WriteMulti added in v0.10.0 

func (c *Cache) WriteMulti(values map[string][]Value) error

WriteMulti writes the map of keys and associated values to the cache. This function is goroutine-safe. It returns an error if the cache will exceeded its max size by adding the new values. The write attempts to write as many values as possible. If one key fails, the others can still succeed and an error will be returned.

type CacheLoader added in v0.10.0 

type CacheLoader struct {
	Logger *log.Logger
	// contains filtered or unexported fields
}

CacheLoader processes a set of WAL segment files, and loads a cache with the data contained within those files. Processing of the supplied files take place in the order they exist in the files slice.

func NewCacheLoader added in v0.10.0 

func NewCacheLoader(files []string) *CacheLoader

NewCacheLoader returns a new instance of a CacheLoader.

func (*CacheLoader) Load added in v0.10.0 

func (cl *CacheLoader) Load(cache *Cache) error

Load returns a cache loaded with the data contained within the segment files. If, during reading of a segment file, corruption is encountered, that segment file is truncated up to and including the last valid byte, and processing continues with the next segment file.

func (*CacheLoader) SetLogOutput added in v1.0.0 

func (cl *CacheLoader) SetLogOutput(w io.Writer)

SetLogOutput sets the logger used for all messages. It must not be called after the Open method has been called.

type CacheStatistics added in v1.0.0 

type CacheStatistics struct {
	MemSizeBytes        int64
	DiskSizeBytes       int64
	SnapshotCount       int64
	CacheAgeMs          int64
	CachedBytes         int64
	WALCompactionTimeMs int64
	WriteOK             int64
	WriteErr            int64
	WriteDropped        int64
}

CacheStatistics hold statistics related to the cache.

type CompactionGroup added in v0.10.0 

type CompactionGroup []string

type CompactionPlanner added in v0.10.0 

type CompactionPlanner interface {
	Plan(lastWrite time.Time) []CompactionGroup
	PlanLevel(level int) []CompactionGroup
	PlanOptimize() []CompactionGroup
}

CompactionPlanner determines what TSM files and WAL segments to include in a given compaction run.

type Compactor added in v0.10.0 

type Compactor struct {
	Dir  string
	Size int

	FileStore interface {
		NextGeneration() int
	}
	// contains filtered or unexported fields
}

Compactor merges multiple TSM files into new files or writes a Cache into 1 or more TSM files

func (*Compactor) Close added in v1.0.0 

func (c *Compactor) Close()

func (*Compactor) CompactFast added in v0.10.0 

func (c *Compactor) CompactFast(tsmFiles []string) ([]string, error)

Compact will write multiple smaller TSM files into 1 or more larger files

func (*Compactor) CompactFull added in v0.10.0 

func (c *Compactor) CompactFull(tsmFiles []string) ([]string, error)

Compact will write multiple smaller TSM files into 1 or more larger files

func (*Compactor) DisableCompactions added in v1.1.1 

func (c *Compactor) DisableCompactions()

func (*Compactor) DisableSnapshots added in v1.1.1 

func (c *Compactor) DisableSnapshots()

func (*Compactor) EnableCompactions added in v1.1.1 

func (c *Compactor) EnableCompactions()

func (*Compactor) EnableSnapshots added in v1.1.1 

func (c *Compactor) EnableSnapshots()

func (*Compactor) Open added in v1.0.0 

func (c *Compactor) Open()

func (*Compactor) WriteSnapshot added in v0.10.0 

func (c *Compactor) WriteSnapshot(cache *Cache) ([]string, error)

WriteSnapshot will write a Cache snapshot to a new TSM files.

type DefaultPlanner added in v0.10.0 

type DefaultPlanner struct {
	FileStore interface {
		Stats() []FileStat
		LastModified() time.Time
		BlockCount(path string, idx int) int
	}

	// CompactFullWriteColdDuration specifies the length of time after
	// which if no writes have been committed to the WAL, the engine will
	// do a full compaction of the TSM files in this shard. This duration
	// should always be greater than the CacheFlushWriteColdDuraion
	CompactFullWriteColdDuration time.Duration
	// contains filtered or unexported fields
}

DefaultPlanner implements CompactionPlanner using a strategy to roll up multiple generations of TSM files into larger files in stages. It attempts to minimize the number of TSM files on disk while rolling up a bounder number of files.

func (*DefaultPlanner) Plan added in v0.10.0 

func (c *DefaultPlanner) Plan(lastWrite time.Time) []CompactionGroup

Plan returns a set of TSM files to rewrite for level 4 or higher. The planning returns multiple groups if possible to allow compactions to run concurrently.

func (*DefaultPlanner) PlanLevel added in v0.10.0 

func (c *DefaultPlanner) PlanLevel(level int) []CompactionGroup

PlanLevel returns a set of TSM files to rewrite for a specific level

func (*DefaultPlanner) PlanOptimize added in v1.0.0 

func (c *DefaultPlanner) PlanOptimize() []CompactionGroup

PlanOptimize returns all TSM files if they are in different generations in order to optimize the index across TSM files. Each returned compaction group can be compacted concurrently.

type DeleteRangeWALEntry added in v1.0.0 

type DeleteRangeWALEntry struct {
	Keys     []string
	Min, Max int64
}

DeleteRangeWALEntry represents the deletion of multiple series.

func (*DeleteRangeWALEntry) Encode added in v1.0.0 

func (w *DeleteRangeWALEntry) Encode(b []byte) ([]byte, error)

func (*DeleteRangeWALEntry) MarshalBinary added in v1.0.0 

func (w *DeleteRangeWALEntry) MarshalBinary() ([]byte, error)

func (*DeleteRangeWALEntry) Type added in v1.0.0 

func (w *DeleteRangeWALEntry) Type() WalEntryType

func (*DeleteRangeWALEntry) UnmarshalBinary added in v1.0.0 

func (w *DeleteRangeWALEntry) UnmarshalBinary(b []byte) error

type DeleteWALEntry added in v0.10.0 

type DeleteWALEntry struct {
	Keys []string
}

DeleteWALEntry represents the deletion of multiple series.

func (*DeleteWALEntry) Encode added in v0.10.0 

func (w *DeleteWALEntry) Encode(dst []byte) ([]byte, error)

func (*DeleteWALEntry) MarshalBinary added in v0.10.0 

func (w *DeleteWALEntry) MarshalBinary() ([]byte, error)

func (*DeleteWALEntry) Type added in v0.10.0 

func (w *DeleteWALEntry) Type() WalEntryType

func (*DeleteWALEntry) UnmarshalBinary added in v0.10.0 

func (w *DeleteWALEntry) UnmarshalBinary(b []byte) error

type EmptyValue 

type EmptyValue struct{}

func (EmptyValue) Size 

func (e EmptyValue) Size() int

func (EmptyValue) String added in v0.10.0 

func (e EmptyValue) String() string

func (EmptyValue) UnixNano 

func (e EmptyValue) UnixNano() int64

func (EmptyValue) Value 

func (e EmptyValue) Value() interface{}

type Engine 

type Engine struct {
	WAL            *WAL
	Cache          *Cache
	Compactor      *Compactor
	CompactionPlan CompactionPlanner
	FileStore      *FileStore

	MaxPointsPerBlock int

	// CacheFlushMemorySizeThreshold specifies the minimum size threshodl for
	// the cache when the engine should write a snapshot to a TSM file
	CacheFlushMemorySizeThreshold uint64

	// CacheFlushWriteColdDuration specifies the length of time after which if
	// no writes have been committed to the WAL, the engine will write
	// a snapshot of the cache to a TSM file
	CacheFlushWriteColdDuration time.Duration
	// contains filtered or unexported fields
}

Engine represents a storage engine with compressed blocks.

func (*Engine) Backup added in v0.10.0 

func (e *Engine) Backup(w io.Writer, basePath string, since time.Time) error

Backup will write a tar archive of any TSM files modified since the passed in time to the passed in writer. The basePath will be prepended to the names of the files in the archive. It will force a snapshot of the WAL first then perform the backup with a read lock against the file store. This means that new TSM files will not be able to be created in this shard while the backup is running. For shards that are still acively getting writes, this could cause the WAL to backup, increasing memory usage and evenutally rejecting writes.

func (*Engine) Close 

func (e *Engine) Close() error

Close closes the engine. Subsequent calls to Close are a nop.

func (*Engine) ContainsSeries added in v1.0.0 

func (e *Engine) ContainsSeries(keys []string) (map[string]bool, error)

ContainsSeries returns a map of keys indicating whether the key exists and has values or not.

func (*Engine) CreateIterator added in v0.12.0 

func (e *Engine) CreateIterator(opt influxql.IteratorOptions) (influxql.Iterator, error)

func (*Engine) CreateSnapshot added in v1.0.0 

func (e *Engine) CreateSnapshot() (string, error)

CreateSnapshot will create a temp directory that holds temporary hardlinks to the underylyng shard files

func (*Engine) DeleteMeasurement 

func (e *Engine) DeleteMeasurement(name string, seriesKeys []string) error

DeleteMeasurement deletes a measurement and all related series.

func (*Engine) DeleteSeries 

func (e *Engine) DeleteSeries(seriesKeys []string) error

DeleteSeries removes all series keys from the engine.

func (*Engine) DeleteSeriesRange added in v1.0.0 

func (e *Engine) DeleteSeriesRange(seriesKeys []string, min, max int64) error

DeleteSeriesRange removes the values between min and max (inclusive) from all series.

func (*Engine) Format 

func (e *Engine) Format() tsdb.EngineFormat

Format returns the format type of this engine

func (*Engine) Index added in v0.12.0 

func (e *Engine) Index() *tsdb.DatabaseIndex

Index returns the database index.

func (*Engine) KeyCursor added in v0.10.0 

func (e *Engine) KeyCursor(key string, t int64, ascending bool) *KeyCursor

func (*Engine) LoadMetadataIndex 

func (e *Engine) LoadMetadataIndex(shardID uint64, index *tsdb.DatabaseIndex) error

LoadMetadataIndex loads the shard metadata into memory.

func (*Engine) MeasurementFields added in v0.12.0 

func (e *Engine) MeasurementFields(measurement string) *tsdb.MeasurementFields

MeasurementFields returns the measurement fields for a measurement.

func (*Engine) Open 

func (e *Engine) Open() error

Open opens and initializes the engine.

func (*Engine) Path 

func (e *Engine) Path() string

Path returns the path the engine was opened with.

func (*Engine) Restore added in v1.0.0 

func (e *Engine) Restore(r io.Reader, basePath string) error

Restore will read a tar archive generated by Backup(). Only files that match basePath will be copied into the directory. This obtains a write lock so no operations can be performed while restoring.

func (*Engine) SeriesCount 

func (e *Engine) SeriesCount() (n int, err error)

SeriesCount returns the number of series buckets on the shard.

func (*Engine) SetCompactionsEnabled added in v1.0.0 

func (e *Engine) SetCompactionsEnabled(enabled bool)

SetCompactionsEnabled enables compactions on the engine. When disabled all running compactions are aborted and new compactions stop running.

func (*Engine) SetEnabled added in v1.0.0 

func (e *Engine) SetEnabled(enabled bool)

func (*Engine) SetLogOutput 

func (e *Engine) SetLogOutput(w io.Writer)

SetLogOutput sets the logger used for all messages. It is safe for concurrent use.

func (*Engine) ShouldCompactCache added in v0.10.0 

func (e *Engine) ShouldCompactCache(lastWriteTime time.Time) bool

ShouldCompactCache returns true if the Cache is over its flush threshold or if the passed in lastWriteTime is older than the write cold threshold

func (*Engine) Statistics added in v1.0.0 

func (e *Engine) Statistics(tags map[string]string) []models.Statistic

Statistics returns statistics for periodic monitoring.

func (*Engine) WritePoints 

func (e *Engine) WritePoints(points []models.Point) error

WritePoints writes metadata and point data into the engine. Returns an error if new points are added to an existing key.

func (*Engine) WriteSnapshot added in v0.10.0 

func (e *Engine) WriteSnapshot() error

WriteSnapshot will snapshot the cache and write a new TSM file with its contents, releasing the snapshot when done.

func (*Engine) WriteTo 

func (e *Engine) WriteTo(w io.Writer) (n int64, err error)

type EngineStatistics added in v1.0.0 

type EngineStatistics struct {
	CacheCompactions        int64 // Counter of cache compactions that have ever run.
	CacheCompactionsActive  int64 // Gauge of cache compactions currently running.
	CacheCompactionErrors   int64 // Counter of cache compactions that have failed due to error.
	CacheCompactionDuration int64 // Counter of number of wall nanoseconds spent in cache compactions.

	TSMCompactions        [3]int64 // Counter of TSM compactions (by level) that have ever run.
	TSMCompactionsActive  [3]int64 // Gauge of TSM compactions (by level) currently running.
	TSMCompactionErrors   [3]int64 // Counter of TSM compcations (by level) that have failed due to error.
	TSMCompactionDuration [3]int64 // Counter of number of wall nanoseconds spent in TSM compactions (by level).

	TSMOptimizeCompactions        int64 // Counter of optimize compactions that have ever run.
	TSMOptimizeCompactionsActive  int64 // Gauge of optimize compactions currently running.
	TSMOptimizeCompactionErrors   int64 // Counter of optimize compactions that have failed due to error.
	TSMOptimizeCompactionDuration int64 // Counter of number of wall nanoseconds spent in optimize compactions.

	TSMFullCompactions        int64 // Counter of full compactions that have ever run.
	TSMFullCompactionsActive  int64 // Gauge of full compactions currently running.
	TSMFullCompactionErrors   int64 // Counter of full compactions that have failed due to error.
	TSMFullCompactionDuration int64 // Counter of number of wall nanoseconds spent in full compactions.
}

EngineStatistics maintains statistics for the engine.

type FileStat added in v0.10.0 

type FileStat struct {
	Path             string
	HasTombstone     bool
	Size             uint32
	LastModified     int64
	MinTime, MaxTime int64
	MinKey, MaxKey   string
}

func (FileStat) ContainsKey added in v0.10.0 

func (f FileStat) ContainsKey(key string) bool

func (FileStat) OverlapsKeyRange added in v0.10.0 

func (f FileStat) OverlapsKeyRange(min, max string) bool

func (FileStat) OverlapsTimeRange added in v0.10.0 

func (f FileStat) OverlapsTimeRange(min, max int64) bool

type FileStore added in v0.10.0 

type FileStore struct {
	// contains filtered or unexported fields
}

func NewFileStore added in v0.10.0 

func NewFileStore(dir string) *FileStore

func (*FileStore) Add added in v0.10.0 

func (f *FileStore) Add(files ...TSMFile)

func (*FileStore) BlockCount added in v0.10.0 

func (f *FileStore) BlockCount(path string, idx int) int

BlockCount returns number of values stored in the block at location idx in the file at path. If path does not match any file in the store, 0 is returned. If idx is out of range for the number of blocks in the file, 0 is returned.

func (*FileStore) Close added in v0.10.0 

func (f *FileStore) Close() error

func (*FileStore) Count added in v0.10.0 

func (f *FileStore) Count() int

Returns the number of TSM files currently loaded

func (*FileStore) CreateSnapshot added in v1.0.0 

func (f *FileStore) CreateSnapshot() (string, error)

CreateSnapshot will create hardlinks for all tsm and tombstone files in the path provided

func (*FileStore) CurrentGeneration added in v0.10.0 

func (f *FileStore) CurrentGeneration() int

CurrentGeneration returns the current generation of the TSM files

func (*FileStore) Delete added in v0.10.0 

func (f *FileStore) Delete(keys []string) error

func (*FileStore) DeleteRange added in v1.0.0 

func (f *FileStore) DeleteRange(keys []string, min, max int64) error

DeleteRange removes the values for keys between min and max.

func (*FileStore) Files added in v0.12.0 

func (f *FileStore) Files() []TSMFile

Files returns TSM files currently loaded.

func (*FileStore) KeyCursor added in v0.10.0 

func (f *FileStore) KeyCursor(key string, t int64, ascending bool) *KeyCursor

func (*FileStore) Keys added in v0.10.0 

func (f *FileStore) Keys() map[string]byte

Keys returns all keys and types for all files

func (*FileStore) LastModified added in v0.10.0 

func (f *FileStore) LastModified() time.Time

LastModified returns the last time the file store was updated with new TSM files or a delete

func (*FileStore) NextGeneration added in v0.10.0 

func (f *FileStore) NextGeneration() int

NextGeneration returns the max file ID + 1

func (*FileStore) Open added in v0.10.0 

func (f *FileStore) Open() error

func (*FileStore) Read added in v0.10.0 

func (f *FileStore) Read(key string, t int64) ([]Value, error)

func (*FileStore) Remove added in v0.10.0 

func (f *FileStore) Remove(paths ...string)

Remove removes the files with matching paths from the set of active files. It does not remove the paths from disk.

func (*FileStore) Replace added in v0.10.0 

func (f *FileStore) Replace(oldFiles, newFiles []string) error

func (*FileStore) SetLogOutput added in v1.0.0 

func (f *FileStore) SetLogOutput(w io.Writer)

SetLogOutput sets the logger used for all messages. It is safe for concurrent use.

func (*FileStore) Statistics added in v1.0.0 

func (f *FileStore) Statistics(tags map[string]string) []models.Statistic

Statistics returns statistics for periodic monitoring.

func (*FileStore) Stats added in v0.10.0 

func (f *FileStore) Stats() []FileStat

func (*FileStore) Type added in v0.10.0 

func (f *FileStore) Type(key string) (byte, error)

func (*FileStore) WalkKeys added in v0.12.0 

func (f *FileStore) WalkKeys(fn func(key []byte, typ byte) error) error

WalkKeys calls fn for every key in every TSM file known to the FileStore. If the key exists in multiple files, it will be invoked for each file.

type FileStoreStatistics added in v1.0.0 

type FileStoreStatistics struct {
	DiskBytes int64
	FileCount int64
}

FileStoreStatistics keeps statistics about the file store.

type FloatDecoder 

type FloatDecoder struct {
	// contains filtered or unexported fields
}

FloatDecoder decodes a byte slice into multipe float64 values

func (*FloatDecoder) Error 

func (it *FloatDecoder) Error() error

func (*FloatDecoder) Next 

func (it *FloatDecoder) Next() bool

func (*FloatDecoder) SetBytes added in v1.0.0 

func (it *FloatDecoder) SetBytes(b []byte) error

SetBytes initializes the decoder with b. Must call before calling Next().

func (*FloatDecoder) Values 

func (it *FloatDecoder) Values() float64

type FloatEncoder 

type FloatEncoder struct {
	// contains filtered or unexported fields
}

FloatEncoder encodes multiple float64s into a byte slice

func NewFloatEncoder 

func NewFloatEncoder() *FloatEncoder

func (*FloatEncoder) Bytes 

func (s *FloatEncoder) Bytes() ([]byte, error)

func (*FloatEncoder) Finish 

func (s *FloatEncoder) Finish()

func (*FloatEncoder) Push 

func (s *FloatEncoder) Push(v float64)

func (*FloatEncoder) Reset added in v1.1.1 

func (s *FloatEncoder) Reset()

type FloatValue 

type FloatValue struct {
	// contains filtered or unexported fields
}

func DecodeFloatBlock added in v0.10.0 

func DecodeFloatBlock(block []byte, a *[]FloatValue) ([]FloatValue, error)

func (*FloatValue) Size 

func (f *FloatValue) Size() int

func (*FloatValue) String added in v0.10.0 

func (f *FloatValue) String() string

func (*FloatValue) UnixNano 

func (f *FloatValue) UnixNano() int64

func (*FloatValue) Value 

func (f *FloatValue) Value() interface{}

type FloatValues added in v0.12.0 

type FloatValues []FloatValue

FloatValues represents a slice of Float values.

func (FloatValues) Deduplicate added in v0.12.0 

func (a FloatValues) Deduplicate() FloatValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.

func (FloatValues) Exclude added in v1.0.0 

func (a FloatValues) Exclude(min, max int64) FloatValues

Exclude returns the subset of values not in [min, max]

func (FloatValues) Include added in v1.0.0 

func (a FloatValues) Include(min, max int64) FloatValues

Include returns the subset values between min and max inclusive.

func (FloatValues) Len added in v0.12.0 

func (a FloatValues) Len() int

Sort methods

func (FloatValues) Less added in v0.12.0 

func (a FloatValues) Less(i, j int) bool

func (FloatValues) MaxTime added in v1.0.0 

func (a FloatValues) MaxTime() int64

func (FloatValues) Merge added in v1.0.0 

func (a FloatValues) Merge(b FloatValues) FloatValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (FloatValues) MinTime added in v1.0.0 

func (a FloatValues) MinTime() int64

func (FloatValues) Size added in v1.0.0 

func (a FloatValues) Size() int

func (FloatValues) Swap added in v0.12.0 

func (a FloatValues) Swap(i, j int)

type IndexEntry 

type IndexEntry struct {

	// The min and max time of all points stored in the block.
	MinTime, MaxTime int64

	// The absolute position in the file where this block is located.
	Offset int64

	// The size in bytes of the block in the file.
	Size uint32
}

IndexEntry is the index information for a given block in a TSM file.

func (*IndexEntry) AppendTo added in v0.12.0 

func (e *IndexEntry) AppendTo(b []byte) []byte

AppendTo will write a binary-encoded version of IndexEntry to b, allocating and returning a new slice, if necessary

func (*IndexEntry) Contains 

func (e *IndexEntry) Contains(t int64) bool

Returns true if this IndexEntry may contain values for the given time. The min and max times are inclusive.

func (*IndexEntry) OverlapsTimeRange added in v0.10.0 

func (e *IndexEntry) OverlapsTimeRange(min, max int64) bool

func (*IndexEntry) String added in v0.10.0 

func (e *IndexEntry) String() string

func (*IndexEntry) UnmarshalBinary 

func (e *IndexEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary decodes an IndexEntry from a byte slice

type IndexWriter 

type IndexWriter interface {
	// Add records a new block entry for a key in the index.
	Add(key string, blockType byte, minTime, maxTime int64, offset int64, size uint32)

	// Entries returns all index entries for a key.
	Entries(key string) []IndexEntry

	// Keys returns the unique set of keys in the index.
	Keys() []string

	// KeyCount returns the count of unique keys in the index.
	KeyCount() int

	// Size returns the size of a the current index in bytes
	Size() uint32

	// MarshalBinary returns a byte slice encoded version of the index.
	MarshalBinary() ([]byte, error)

	// WriteTo writes the index contents to a writer
	WriteTo(w io.Writer) (int64, error)
}

IndexWriter writes a TSMIndex

func NewIndexWriter added in v1.0.0 

func NewIndexWriter() IndexWriter

type IntegerDecoder added in v0.12.0 

type IntegerDecoder struct {
	// contains filtered or unexported fields
}

IntegerDecoder decodes a byte slice into int64s.

func (*IntegerDecoder) Error added in v0.12.0 

func (d *IntegerDecoder) Error() error

func (*IntegerDecoder) Next added in v0.12.0 

func (d *IntegerDecoder) Next() bool

func (*IntegerDecoder) Read added in v0.12.0 

func (d *IntegerDecoder) Read() int64

func (*IntegerDecoder) SetBytes added in v0.12.0 

func (d *IntegerDecoder) SetBytes(b []byte)

type IntegerEncoder added in v0.12.0 

type IntegerEncoder struct {
	// contains filtered or unexported fields
}

IntegerEncoder encoders int64 into byte slices

func NewIntegerEncoder added in v0.12.0 

func NewIntegerEncoder(sz int) IntegerEncoder

func (*IntegerEncoder) Bytes added in v0.12.0 

func (e *IntegerEncoder) Bytes() ([]byte, error)

func (*IntegerEncoder) Reset added in v1.1.1 

func (e *IntegerEncoder) Reset()

func (*IntegerEncoder) Write added in v0.12.0 

func (e *IntegerEncoder) Write(v int64)

type IntegerValue added in v0.12.0 

type IntegerValue struct {
	// contains filtered or unexported fields
}

func DecodeIntegerBlock added in v0.12.0 

func DecodeIntegerBlock(block []byte, a *[]IntegerValue) ([]IntegerValue, error)

func (*IntegerValue) Size added in v0.12.0 

func (v *IntegerValue) Size() int

func (*IntegerValue) String added in v0.12.0 

func (f *IntegerValue) String() string

func (*IntegerValue) UnixNano added in v0.12.0 

func (v *IntegerValue) UnixNano() int64

func (*IntegerValue) Value added in v0.12.0 

func (v *IntegerValue) Value() interface{}

type IntegerValues added in v0.12.0 

type IntegerValues []IntegerValue

IntegerValues represents a slice of Integer values.

func (IntegerValues) Deduplicate added in v0.12.0 

func (a IntegerValues) Deduplicate() IntegerValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.

func (IntegerValues) Exclude added in v1.0.0 

func (a IntegerValues) Exclude(min, max int64) IntegerValues

Exclude returns the subset of values not in [min, max]

func (IntegerValues) Include added in v1.0.0 

func (a IntegerValues) Include(min, max int64) IntegerValues

Include returns the subset values between min and max inclusive.

func (IntegerValues) Len added in v0.12.0 

func (a IntegerValues) Len() int

Sort methods

func (IntegerValues) Less added in v0.12.0 

func (a IntegerValues) Less(i, j int) bool

func (IntegerValues) MaxTime added in v1.0.0 

func (a IntegerValues) MaxTime() int64

func (IntegerValues) Merge added in v1.0.0 

func (a IntegerValues) Merge(b IntegerValues) IntegerValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (IntegerValues) MinTime added in v1.0.0 

func (a IntegerValues) MinTime() int64

func (IntegerValues) Size added in v1.0.0 

func (a IntegerValues) Size() int

func (IntegerValues) Swap added in v0.12.0 

func (a IntegerValues) Swap(i, j int)

type KeyCursor added in v0.10.0 

type KeyCursor struct {
	// contains filtered or unexported fields
}

func (*KeyCursor) Close added in v0.10.0 

func (c *KeyCursor) Close()

Close removes all references on the cursor.

func (*KeyCursor) Next added in v0.10.0 

func (c *KeyCursor) Next()

Next moves the cursor to the next position. Data should be read by the ReadBlock functions.

func (*KeyCursor) ReadBooleanBlock added in v0.12.0 

func (c *KeyCursor) ReadBooleanBlock(buf *[]BooleanValue) ([]BooleanValue, error)

ReadBooleanBlock reads the next block as a set of boolean values.

func (*KeyCursor) ReadFloatBlock added in v0.12.0 

func (c *KeyCursor) ReadFloatBlock(buf *[]FloatValue) ([]FloatValue, error)

ReadFloatBlock reads the next block as a set of float values.

func (*KeyCursor) ReadIntegerBlock added in v0.12.0 

func (c *KeyCursor) ReadIntegerBlock(buf *[]IntegerValue) ([]IntegerValue, error)

ReadIntegerBlock reads the next block as a set of integer values.

func (*KeyCursor) ReadStringBlock added in v0.12.0 

func (c *KeyCursor) ReadStringBlock(buf *[]StringValue) ([]StringValue, error)

ReadStringBlock reads the next block as a set of string values.

type KeyIterator added in v0.10.0 

type KeyIterator interface {
	Next() bool
	Read() (string, int64, int64, []byte, error)
	Close() error
}

KeyIterator allows iteration over set of keys and values in sorted order.

func NewCacheKeyIterator added in v0.10.0 

func NewCacheKeyIterator(cache *Cache, size int) KeyIterator

func NewTSMKeyIterator added in v0.10.0 

func NewTSMKeyIterator(size int, fast bool, readers ...*TSMReader) (KeyIterator, error)

type SegmentInfo added in v0.10.0 

type SegmentInfo struct {
	// contains filtered or unexported fields
}

SegmentInfo represents metadata about a segment.

type StringDecoder 

type StringDecoder struct {
	// contains filtered or unexported fields
}

func (*StringDecoder) Error 

func (e *StringDecoder) Error() error

func (*StringDecoder) Next 

func (e *StringDecoder) Next() bool

func (*StringDecoder) Read 

func (e *StringDecoder) Read() string

func (*StringDecoder) SetBytes added in v1.0.0 

func (e *StringDecoder) SetBytes(b []byte) error

SetBytes initializes the decoder with bytes to read from. This must be called before calling any other method.

type StringEncoder 

type StringEncoder struct {
	// contains filtered or unexported fields
}

func NewStringEncoder 

func NewStringEncoder(sz int) StringEncoder

func (*StringEncoder) Bytes 

func (e *StringEncoder) Bytes() ([]byte, error)

func (*StringEncoder) Reset added in v1.1.1 

func (e *StringEncoder) Reset()

func (*StringEncoder) Write 

func (e *StringEncoder) Write(s string)

type StringValue 

type StringValue struct {
	// contains filtered or unexported fields
}

func DecodeStringBlock added in v0.10.0 

func DecodeStringBlock(block []byte, a *[]StringValue) ([]StringValue, error)

func (*StringValue) Size 

func (v *StringValue) Size() int

func (*StringValue) String 

func (f *StringValue) String() string

func (*StringValue) UnixNano 

func (v *StringValue) UnixNano() int64

func (*StringValue) Value 

func (v *StringValue) Value() interface{}

type StringValues added in v0.12.0 

type StringValues []StringValue

StringValues represents a slice of String values.

func (StringValues) Deduplicate added in v0.12.0 

func (a StringValues) Deduplicate() StringValues

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.

func (StringValues) Exclude added in v1.0.0 

func (a StringValues) Exclude(min, max int64) StringValues

Exclude returns the subset of values not in [min, max]

func (StringValues) Include added in v1.0.0 

func (a StringValues) Include(min, max int64) StringValues

Include returns the subset values between min and max inclusive.

func (StringValues) Len added in v0.12.0 

func (a StringValues) Len() int

Sort methods

func (StringValues) Less added in v0.12.0 

func (a StringValues) Less(i, j int) bool

func (StringValues) MaxTime added in v1.0.0 

func (a StringValues) MaxTime() int64

func (StringValues) Merge added in v1.0.0 

func (a StringValues) Merge(b StringValues) StringValues

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (StringValues) MinTime added in v1.0.0 

func (a StringValues) MinTime() int64

func (StringValues) Size added in v1.0.0 

func (a StringValues) Size() int

func (StringValues) Swap added in v0.12.0 

func (a StringValues) Swap(i, j int)

type TSMFile added in v0.10.0 

type TSMFile interface {
	// Path returns the underlying file path for the TSMFile.  If the file
	// has not be written or loaded from disk, the zero value is returned.
	Path() string

	// Read returns all the values in the block where time t resides
	Read(key string, t int64) ([]Value, error)

	// ReadAt returns all the values in the block identified by entry.
	ReadAt(entry *IndexEntry, values []Value) ([]Value, error)
	ReadFloatBlockAt(entry *IndexEntry, values *[]FloatValue) ([]FloatValue, error)
	ReadIntegerBlockAt(entry *IndexEntry, values *[]IntegerValue) ([]IntegerValue, error)
	ReadStringBlockAt(entry *IndexEntry, values *[]StringValue) ([]StringValue, error)
	ReadBooleanBlockAt(entry *IndexEntry, values *[]BooleanValue) ([]BooleanValue, error)

	// Entries returns the index entries for all blocks for the given key.
	Entries(key string) []IndexEntry
	ReadEntries(key string, entries *[]IndexEntry)

	// Returns true if the TSMFile may contain a value with the specified
	// key and time
	ContainsValue(key string, t int64) bool

	// Contains returns true if the file contains any values for the given
	// key.
	Contains(key string) bool

	// TimeRange returns the min and max time across all keys in the file.
	TimeRange() (int64, int64)

	// TombstoneRange returns ranges of time that are deleted for the given key.
	TombstoneRange(key string) []TimeRange

	// KeyRange returns the min and max keys in the file.
	KeyRange() (string, string)

	// KeyCount returns the number of distict keys in the file.
	KeyCount() int

	// KeyAt returns the key located at index position idx
	KeyAt(idx int) ([]byte, byte)

	// Type returns the block type of the values stored for the key.  Returns one of
	// BlockFloat64, BlockInt64, BlockBoolean, BlockString.  If key does not exist,
	// an error is returned.
	Type(key string) (byte, error)

	// Delete removes the keys from the set of keys available in this file.
	Delete(keys []string) error

	// DeleteRange removes the values for keys between min and max.
	DeleteRange(keys []string, min, max int64) error

	// HasTombstones returns true if file contains values that have been deleted.
	HasTombstones() bool

	// TombstoneFiles returns the tombstone filestats if there are any tombstones
	// written for this file.
	TombstoneFiles() []FileStat

	// Close the underlying file resources
	Close() error

	// Size returns the size of the file on disk in bytes.
	Size() uint32

	// Rename renames the existing TSM file to a new name and replaces the mmap backing slice using the new
	// file name.  Index and Reader state are not re-initialized.
	Rename(path string) error

	// Remove deletes the file from the filesystem
	Remove() error

	// Returns true if the file is currently in use by queries
	InUse() bool

	// Ref records that this file is actively in use
	Ref()

	// Unref records that this file is no longer in user
	Unref()

	// Stats returns summary information about the TSM file.
	Stats() FileStat

	// BlockIterator returns an iterator pointing to the first block in the file and
	// allows sequential iteration to each every block.
	BlockIterator() *BlockIterator
	// contains filtered or unexported methods
}

type TSMIndex 

type TSMIndex interface {

	// Delete removes the given keys from the index.
	Delete(keys []string)

	// DeleteRange removes the given keys with data between minTime and maxTime from the index.
	DeleteRange(keys []string, minTime, maxTime int64)

	// Contains return true if the given key exists in the index.
	Contains(key string) bool

	// ContainsValue returns true if key and time might exists in this file.  This function could
	// return true even though the actual point does not exists.  For example, the key may
	// exists in this file, but not have point exactly at time t.
	ContainsValue(key string, timestamp int64) bool

	// Entries returns all index entries for a key.
	Entries(key string) []IndexEntry

	// ReadEntries reads the index entries for key into entries.
	ReadEntries(key string, entries *[]IndexEntry)

	// Entry returns the index entry for the specified key and timestamp.  If no entry
	// matches the key and timestamp, nil is returned.
	Entry(key string, timestamp int64) *IndexEntry

	// Key returns the key in the index at the given postion.
	Key(index int) (string, []IndexEntry)

	// KeyAt returns the key in the index at the given postion.
	KeyAt(index int) ([]byte, byte)

	// KeyCount returns the count of unique keys in the index.
	KeyCount() int

	// Size returns the size of a the current index in bytes
	Size() uint32

	// TimeRange returns the min and max time across all keys in the file.
	TimeRange() (int64, int64)

	// TombstoneRange returns ranges of time that are deleted for the given key.
	TombstoneRange(key string) []TimeRange

	// KeyRange returns the min and max keys in the file.
	KeyRange() (string, string)

	// Type returns the block type of the values stored for the key.  Returns one of
	// BlockFloat64, BlockInt64, BlockBool, BlockString.  If key does not exist,
	// an error is returned.
	Type(key string) (byte, error)

	// UnmarshalBinary populates an index from an encoded byte slice
	// representation of an index.
	UnmarshalBinary(b []byte) error
}

TSMIndex represent the index section of a TSM file. The index records all blocks, their locations, sizes, min and max times.

type TSMReader added in v0.10.0 

type TSMReader struct {
	// contains filtered or unexported fields
}

func NewTSMReader 

func NewTSMReader(f *os.File) (*TSMReader, error)

func (*TSMReader) BlockIterator added in v0.10.0 

func (t *TSMReader) BlockIterator() *BlockIterator

func (*TSMReader) Close added in v0.10.0 

func (t *TSMReader) Close() error

func (*TSMReader) Contains added in v0.10.0 

func (t *TSMReader) Contains(key string) bool

func (*TSMReader) ContainsValue added in v0.10.0 

func (t *TSMReader) ContainsValue(key string, ts int64) bool

ContainsValue returns true if key and time might exists in this file. This function could return true even though the actual point does not exists. For example, the key may exists in this file, but not have point exactly at time t.

func (*TSMReader) Delete added in v0.10.0 

func (t *TSMReader) Delete(keys []string) error

func (*TSMReader) DeleteRange added in v1.0.0 

func (t *TSMReader) DeleteRange(keys []string, minTime, maxTime int64) error

DeleteRange removes the given points for keys between minTime and maxTime

func (*TSMReader) Entries added in v0.10.0 

func (t *TSMReader) Entries(key string) []IndexEntry

func (*TSMReader) HasTombstones added in v0.10.0 

func (t *TSMReader) HasTombstones() bool

HasTombstones return true if there are any tombstone entries recorded.

func (*TSMReader) InUse added in v1.0.0 

func (t *TSMReader) InUse() bool

func (*TSMReader) IndexSize added in v0.10.0 

func (t *TSMReader) IndexSize() uint32

func (*TSMReader) Key added in v0.10.0 

func (t *TSMReader) Key(index int) (string, []IndexEntry)

func (*TSMReader) KeyAt added in v0.10.0 

func (t *TSMReader) KeyAt(idx int) ([]byte, byte)

KeyAt returns the key and key type at position idx in the index.

func (*TSMReader) KeyCount added in v0.10.0 

func (t *TSMReader) KeyCount() int

func (*TSMReader) KeyRange added in v0.10.0 

func (t *TSMReader) KeyRange() (string, string)

KeyRange returns the min and max key across all keys in the file.

func (*TSMReader) LastModified added in v0.10.0 

func (t *TSMReader) LastModified() int64

func (*TSMReader) Path added in v0.10.0 

func (t *TSMReader) Path() string

func (*TSMReader) Read added in v0.10.0 

func (t *TSMReader) Read(key string, timestamp int64) ([]Value, error)

func (*TSMReader) ReadAll added in v0.10.0 

func (t *TSMReader) ReadAll(key string) ([]Value, error)

ReadAll returns all values for a key in all blocks.

func (*TSMReader) ReadAt added in v0.10.0 

func (t *TSMReader) ReadAt(entry *IndexEntry, vals []Value) ([]Value, error)

func (*TSMReader) ReadBooleanBlockAt added in v0.12.0 

func (t *TSMReader) ReadBooleanBlockAt(entry *IndexEntry, vals *[]BooleanValue) ([]BooleanValue, error)

func (*TSMReader) ReadEntries added in v0.12.0 

func (t *TSMReader) ReadEntries(key string, entries *[]IndexEntry)

func (*TSMReader) ReadFloatBlockAt added in v0.12.0 

func (t *TSMReader) ReadFloatBlockAt(entry *IndexEntry, vals *[]FloatValue) ([]FloatValue, error)

func (*TSMReader) ReadIntegerBlockAt added in v0.12.0 

func (t *TSMReader) ReadIntegerBlockAt(entry *IndexEntry, vals *[]IntegerValue) ([]IntegerValue, error)

func (*TSMReader) ReadStringBlockAt added in v0.12.0 

func (t *TSMReader) ReadStringBlockAt(entry *IndexEntry, vals *[]StringValue) ([]StringValue, error)

func (*TSMReader) Ref added in v1.0.0 

func (t *TSMReader) Ref()

Ref records a usage of this TSMReader. If there are active references when the reader is closed or removed, the reader will remain open until there are no more references.

func (*TSMReader) Remove added in v0.10.0 

func (t *TSMReader) Remove() error

Remove removes any underlying files stored on disk for this reader.

func (*TSMReader) Rename added in v1.0.0 

func (t *TSMReader) Rename(path string) error

func (*TSMReader) Size added in v0.10.0 

func (t *TSMReader) Size() uint32

func (*TSMReader) Stats added in v0.10.0 

func (t *TSMReader) Stats() FileStat

func (*TSMReader) TimeRange added in v0.10.0 

func (t *TSMReader) TimeRange() (int64, int64)

TimeRange returns the min and max time across all keys in the file.

func (*TSMReader) TombstoneFiles added in v0.10.0 

func (t *TSMReader) TombstoneFiles() []FileStat

TombstoneFiles returns any tombstone files associated with this TSM file.

func (*TSMReader) TombstoneRange added in v1.0.0 

func (t *TSMReader) TombstoneRange(key string) []TimeRange

TombstoneRange returns ranges of time that are deleted for the given key.

func (*TSMReader) Type added in v0.10.0 

func (t *TSMReader) Type(key string) (byte, error)

func (*TSMReader) Unref added in v1.0.0 

func (t *TSMReader) Unref()

Unref removes a usage record of this TSMReader. If the Reader was closed by another goroutine while there were active references, the file will be closed and remove

type TSMWriter 

type TSMWriter interface {
	// Write writes a new block for key containing and values.  Writes append
	// blocks in the order that the Write function is called.  The caller is
	// responsible for ensuring keys and blocks or sorted appropriately.
	// Values are encoded as a full block.  The caller is responsible for
	// ensuring a fixed number of values are encoded in each block as wells as
	// ensuring the Values are sorted. The first and last timestamp values are
	// used as the minimum and maximum values for the index entry.
	Write(key string, values Values) error

	// WriteBlock writes a new block for key containing the bytes in block.  WriteBlock appends
	// blocks in the order that the WriteBlock function is called.  The caller is
	// responsible for ensuring keys and blocks are sorted appropriately, and that the
	// block and index information is correct for the block.  The minTime and maxTime
	// timestamp values are used as the minimum and maximum values for the index entry.
	WriteBlock(key string, minTime, maxTime int64, block []byte) error

	// WriteIndex finishes the TSM write streams and writes the index.
	WriteIndex() error

	// Closes any underlying file resources.
	Close() error

	// Size returns the current size in bytes of the file
	Size() uint32
}

TSMWriter writes TSM formatted key and values.

func NewTSMWriter 

func NewTSMWriter(w io.Writer) (TSMWriter, error)

type TimeDecoder 

type TimeDecoder struct {
	// contains filtered or unexported fields
}

func (*TimeDecoder) Error 

func (d *TimeDecoder) Error() error

func (*TimeDecoder) Init added in v1.0.0 

func (d *TimeDecoder) Init(b []byte)

func (*TimeDecoder) Next 

func (d *TimeDecoder) Next() bool

func (*TimeDecoder) Read 

func (d *TimeDecoder) Read() int64

type TimeEncoder 

type TimeEncoder interface {
	Write(t int64)
	Bytes() ([]byte, error)
	Reset()
}

TimeEncoder encodes time.Time to byte slices.

func NewTimeEncoder 

func NewTimeEncoder(sz int) TimeEncoder

NewTimeEncoder returns a TimeEncoder

type TimeRange added in v1.0.0 

type TimeRange struct {
	Min, Max int64
}

type Tombstone added in v1.0.0 

type Tombstone struct {
	// Key is the tombstoned series key
	Key string

	// Min and Max are the min and max unix nanosecond time ranges of Key that are deleted.  If
	// the full range is deleted, both values are -1
	Min, Max int64
}

type Tombstoner added in v0.10.0 

type Tombstoner struct {

	// Path is the location of the file to record tombstone. This should be the
	// full path to a TSM file.
	Path string
	// contains filtered or unexported fields
}

func (*Tombstoner) Add added in v0.10.0 

func (t *Tombstoner) Add(keys []string) error

Add add the all keys to the tombstone

func (*Tombstoner) AddRange added in v1.0.0 

func (t *Tombstoner) AddRange(keys []string, min, max int64) error

AddRange adds all keys to the tombstone specifying only the data between min and max to be removed.

func (*Tombstoner) Delete added in v0.10.0 

func (t *Tombstoner) Delete() error

func (*Tombstoner) HasTombstones added in v0.10.0 

func (t *Tombstoner) HasTombstones() bool

HasTombstones return true if there are any tombstone entries recorded.

func (*Tombstoner) ReadAll added in v0.10.0 

func (t *Tombstoner) ReadAll() ([]Tombstone, error)

func (*Tombstoner) TombstoneFiles added in v0.10.0 

func (t *Tombstoner) TombstoneFiles() []FileStat

TombstoneFiles returns any tombstone files associated with this TSM file.

func (*Tombstoner) Walk added in v1.0.0 

func (t *Tombstoner) Walk(fn func(t Tombstone) error) error

type Value 

type Value interface {
	UnixNano() int64
	Value() interface{}
	Size() int
	String() string
	// contains filtered or unexported methods
}

func DecodeBlock 

func DecodeBlock(block []byte, vals []Value) ([]Value, error)

DecodeBlock takes a byte array and will decode into values of the appropriate type based on the block.

func NewBooleanValue added in v1.1.1 

func NewBooleanValue(t int64, v bool) Value

func NewFloatValue added in v1.1.1 

func NewFloatValue(t int64, v float64) Value

func NewIntegerValue added in v1.1.1 

func NewIntegerValue(t int64, v int64) Value

func NewStringValue added in v1.1.1 

func NewStringValue(t int64, v string) Value

func NewValue 

func NewValue(t int64, value interface{}) Value

type Values 

type Values []Value

Values represents a slice of values.

func (Values) Deduplicate 

func (a Values) Deduplicate() Values

Deduplicate returns a new slice with any values that have the same timestamp removed. The Value that appears last in the slice is the one that is kept.

func (Values) Encode 

func (a Values) Encode(buf []byte) ([]byte, error)

Encode converts the values to a byte slice. If there are no values, this function panics.

func (Values) Exclude added in v1.0.0 

func (a Values) Exclude(min, max int64) Values

Exclude returns the subset of values not in [min, max]

func (Values) Include added in v1.0.0 

func (a Values) Include(min, max int64) Values

Include returns the subset values between min and max inclusive.

func (Values) InfluxQLType added in v0.10.0 

func (a Values) InfluxQLType() (influxql.DataType, error)

InfluxQLType returns the influxql.DataType the values map to.

func (Values) Len 

func (a Values) Len() int

Sort methods

func (Values) Less 

func (a Values) Less(i, j int) bool

func (Values) MaxTime 

func (a Values) MaxTime() int64

func (Values) Merge added in v1.0.0 

func (a Values) Merge(b Values) Values

Merge overlays b to top of a. If two values conflict with the same timestamp, b is used. Both a and b must be sorted in ascending order.

func (Values) MinTime 

func (a Values) MinTime() int64

func (Values) Size added in v0.10.0 

func (a Values) Size() int

func (Values) Swap 

func (a Values) Swap(i, j int)

type WAL added in v0.10.0 

type WAL struct {

	// SegmentSize is the file size at which a segment file will be rotated
	SegmentSize int
	// contains filtered or unexported fields
}

func NewWAL added in v0.10.0 

func NewWAL(path string) *WAL

func (*WAL) Close added in v0.10.0 

func (l *WAL) Close() error

Close will finish any flush that is currently in process and close file handles

func (*WAL) CloseSegment added in v0.10.0 

func (l *WAL) CloseSegment() error

CloseSegment closes the current segment if it is non-empty and opens a new one.

func (*WAL) ClosedSegments added in v0.10.0 

func (l *WAL) ClosedSegments() ([]string, error)

func (*WAL) Delete added in v0.10.0 

func (l *WAL) Delete(keys []string) (int, error)

Delete deletes the given keys, returning the segment ID for the operation.

func (*WAL) DeleteRange added in v1.0.0 

func (l *WAL) DeleteRange(keys []string, min, max int64) (int, error)

Delete deletes the given keys, returning the segment ID for the operation.

func (*WAL) LastWriteTime added in v0.10.0 

func (l *WAL) LastWriteTime() time.Time

LastWriteTime is the last time anything was written to the WAL

func (*WAL) Open added in v0.10.0 

func (l *WAL) Open() error

Open opens and initializes the Log. Will recover from previous unclosed shutdowns

func (*WAL) Path added in v0.10.0 

func (l *WAL) Path() string

Path returns the path the log was initialized with.

func (*WAL) Remove added in v0.10.0 

func (l *WAL) Remove(files []string) error

func (*WAL) SetLogOutput added in v1.0.0 

func (l *WAL) SetLogOutput(w io.Writer)

SetLogOutput sets the location that logs are written to. It is safe for concurrent use.

func (*WAL) Statistics added in v1.0.0 

func (l *WAL) Statistics(tags map[string]string) []models.Statistic

Statistics returns statistics for periodic monitoring.

func (*WAL) WritePoints added in v0.10.0 

func (l *WAL) WritePoints(values map[string][]Value) (int, error)

WritePoints writes the given points to the WAL. Returns the WAL segment ID to which the points were written. If an error is returned the segment ID should be ignored.

type WALEntry added in v0.10.0 

type WALEntry interface {
	Type() WalEntryType
	Encode(dst []byte) ([]byte, error)
	MarshalBinary() ([]byte, error)
	UnmarshalBinary(b []byte) error
}

WALEntry is record stored in each WAL segment. Each entry has a type and an opaque, type dependent byte slice data attribute.

type WALSegmentReader added in v0.10.0 

type WALSegmentReader struct {
	// contains filtered or unexported fields
}

WALSegmentReader reads WAL segments.

func NewWALSegmentReader added in v0.10.0 

func NewWALSegmentReader(r io.ReadCloser) *WALSegmentReader

func (*WALSegmentReader) Close added in v0.10.0 

func (r *WALSegmentReader) Close() error

func (*WALSegmentReader) Count added in v0.10.0 

func (r *WALSegmentReader) Count() int64

Count returns the total number of bytes read successfully from the segment, as of the last call to Read(). The segment is guaranteed to be valid up to and including this number of bytes.

func (*WALSegmentReader) Error added in v0.10.0 

func (r *WALSegmentReader) Error() error

func (*WALSegmentReader) Next added in v0.10.0 

func (r *WALSegmentReader) Next() bool

Next indicates if there is a value to read

func (*WALSegmentReader) Read added in v0.10.0 

func (r *WALSegmentReader) Read() (WALEntry, error)

type WALSegmentWriter added in v0.10.0 

type WALSegmentWriter struct {
	// contains filtered or unexported fields
}

WALSegmentWriter writes WAL segments.

func NewWALSegmentWriter added in v0.10.0 

func NewWALSegmentWriter(w io.WriteCloser) *WALSegmentWriter

func (*WALSegmentWriter) Write added in v0.10.0 

func (w *WALSegmentWriter) Write(entryType WalEntryType, compressed []byte) error

type WALStatistics added in v1.0.0 

type WALStatistics struct {
	OldBytes     int64
	CurrentBytes int64
	WriteOK      int64
	WriteErr     int64
}

WALStatistics maintains statistics about the WAL.

type WalEntryType added in v0.10.0 

type WalEntryType byte

WalEntryType is a byte written to a wal segment file that indicates what the following compressed block contains 

const (
	WriteWALEntryType       WalEntryType = 0x01
	DeleteWALEntryType      WalEntryType = 0x02
	DeleteRangeWALEntryType WalEntryType = 0x03
)

type WriteWALEntry added in v0.10.0 

type WriteWALEntry struct {
	Values map[string][]Value
}

WriteWALEntry represents a write of points.

func (*WriteWALEntry) Encode added in v0.10.0 

func (w *WriteWALEntry) Encode(dst []byte) ([]byte, error)

Encode converts the WriteWALEntry into a byte stream using dst if it is large enough. If dst is too small, the slice will be grown to fit the encoded entry.

func (*WriteWALEntry) MarshalBinary added in v0.10.0 

func (w *WriteWALEntry) MarshalBinary() ([]byte, error)

func (*WriteWALEntry) Type added in v0.10.0 

func (w *WriteWALEntry) Type() WalEntryType

func (*WriteWALEntry) UnmarshalBinary added in v0.10.0 

func (w *WriteWALEntry) UnmarshalBinary(b []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.