tsdb

package
v2.0.0-beta.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 13, 2020 License: MIT Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SeriesIndexVersion = 1
	SeriesIndexMagic   = "SIDX"
)
View Source 
const (
	// SeriesIDSize is the size in bytes of a series key ID.
	SeriesIDSize        = 8
	SeriesOffsetSize    = 8
	SeriesIndexElemSize = SeriesOffsetSize + SeriesIDSize

	SeriesIndexLoadFactor = 90 // rhh load factor

	SeriesIndexHeaderSize = 0 +
		4 + 1 +
		8 + 8 +
		8 + 8 +
		8 + 8 +
		8 + 8 +
		0
)
View Source 
const (
	SeriesSegmentVersion = 1
	SeriesSegmentMagic   = "SSEG"

	SeriesSegmentHeaderSize = 4 + 1 // magic + version
)
View Source 
const (
	SeriesEntryFlagSize   = 1
	SeriesEntryHeaderSize = 1 + 8 // flag + id

	SeriesEntryInsertFlag    = 0x01
	SeriesEntryTombstoneFlag = 0x02
)

Series entry constants.

View Source 
const (
	// DefaultLargeSeriesWriteThreshold is the number of series per write
	// that requires the series index be pregrown before insert.
	DefaultLargeSeriesWriteThreshold = 10000
)
View Source 
const DefaultSeriesPartitionCompactThreshold = 1 << 17 // 128K

DefaultSeriesPartitionCompactThreshold is the number of series IDs to hold in the in-memory series map before compacting and rebuilding the on-disk representation.

View Source 
const EOF = query.ZeroTime

EOF represents a "not found" key returned by a Cursor.

View Source 
const (
	// SeriesFilePartitionN is the number of partitions a series file is split into.
	SeriesFilePartitionN = 8
)

Variables

View Source 
var (
	// ErrFieldTypeConflict is returned when a new field already exists with a different type.
	ErrFieldTypeConflict = errors.New("field type conflict")

	// ErrUnknownFieldType is returned when the type of a field cannot be determined.
	ErrUnknownFieldType = errors.New("unknown field type")
)
View Source 
var (
	ErrSeriesFileClosed         = errors.New("tsdb: series file closed")
	ErrInvalidSeriesPartitionID = errors.New("tsdb: invalid series partition id")
)
View Source 
var (
	ErrSeriesPartitionClosed              = errors.New("tsdb: series partition closed")
	ErrSeriesPartitionCompactionCancelled = errors.New("tsdb: series partition compaction cancelled")
)
View Source 
var (
	ErrInvalidSeriesSegment        = errors.New("invalid series segment")
	ErrInvalidSeriesSegmentVersion = errors.New("invalid series segment version")
	ErrSeriesSegmentNotWritable    = errors.New("series segment not writable")
)
View Source 
var ErrInvalidSeriesIndex = errors.New("invalid series index")

Functions

func AppendSeriesEntry 

func AppendSeriesEntry(dst []byte, flag uint8, id SeriesIDTyped, key []byte) []byte

func AppendSeriesKey 

func AppendSeriesKey(dst []byte, name []byte, tags models.Tags) []byte

AppendSeriesKey serializes name and tags to a byte slice. The total length is prepended as a uvarint.

func CompareSeriesKeys 

func CompareSeriesKeys(a, b []byte) int

func DecodeName 

func DecodeName(name [16]byte) (org, bucket platform.ID)

DecodeName converts tsdb internal serialization back to organization and bucket IDs.

func DecodeNameSlice 

func DecodeNameSlice(name []byte) (org, bucket platform.ID)

DecodeNameSlice converts tsdb internal serialization back to organization and bucket IDs.

func EncodeName 

func EncodeName(org, bucket platform.ID) [16]byte

EncodeName converts org/bucket pairs to the tsdb internal serialization

func EncodeNameSlice 

func EncodeNameSlice(org, bucket platform.ID) []byte

EncodeNameSlice converts org/bucket pairs to the tsdb internal serialization but returns a byte slice.

func EncodeNameString 

func EncodeNameString(org, bucket platform.ID) string

EncodeNameString converts org/bucket pairs to the tsdb internal serialization

func EncodeOrgName 

func EncodeOrgName(org platform.ID) [8]byte

EncodeOrgName converts org to the tsdb internal serialization that may be used as a prefix when searching for keys matching a specific organization.

func ExplodePoints 

func ExplodePoints(org, bucket platform.ID, points []models.Point) ([]models.Point, error)

ExplodePoints creates a list of points that only contains one field per point. It also moves the measurement to a tag, and changes the measurement to be the provided argument.

func GenerateSeriesKeys 

func GenerateSeriesKeys(names [][]byte, tagsSlice []models.Tags) [][]byte

GenerateSeriesKeys generates series keys for a list of names & tags using a single large memory block.

func IsValidSeriesEntryFlag 

func IsValidSeriesEntryFlag(flag byte) bool

IsValidSeriesEntryFlag returns true if flag is valid.

func IsValidSeriesSegmentFilename 

func IsValidSeriesSegmentFilename(filename string) bool

IsValidSeriesSegmentFilename returns true if filename is a 4-character lowercase hexadecimal number.

func JoinSeriesOffset 

func JoinSeriesOffset(segmentID uint16, pos uint32) int64

JoinSeriesOffset returns an offset that combines the 2-byte segmentID and 4-byte pos.

func MakeTagsKey 

func MakeTagsKey(keys []string, tags models.Tags) []byte

MakeTagsKey converts a tag set to bytes for use as a lookup key.

func MarshalTags 

func MarshalTags(tags map[string]string) []byte

MarshalTags converts a tag set to bytes for use as a lookup key.

func NewMeasurementSliceIterator 

func NewMeasurementSliceIterator(names [][]byte) *measurementSliceIterator

NewMeasurementSliceIterator returns an iterator over a slice of in-memory measurement names.

func NewTagKeySliceIterator 

func NewTagKeySliceIterator(keys [][]byte) *tagKeySliceIterator

NewTagKeySliceIterator returns a TagKeyIterator that iterates over a slice.

func NewTagValueSliceIterator 

func NewTagValueSliceIterator(values [][]byte) *tagValueSliceIterator

NewTagValueSliceIterator returns a TagValueIterator that iterates over a slice.

func ParseSeriesKey 

func ParseSeriesKey(data []byte) (name []byte, tags models.Tags)

ParseSeriesKey extracts the name & tags from a series key.

func ParseSeriesKeyInto 

func ParseSeriesKeyInto(data []byte, dstTags models.Tags) ([]byte, models.Tags)

ParseSeriesKeyInto extracts the name and tags for data, parsing the tags into dstTags, which is then returened.

The returned dstTags may have a different length and capacity.

func ParseSeriesSegmentFilename 

func ParseSeriesSegmentFilename(filename string) (uint16, error)

ParseSeriesSegmentFilename returns the id represented by the hexadecimal filename.

func PrometheusCollectors 

func PrometheusCollectors() []prometheus.Collector

PrometheusCollectors returns all the metrics associated with the tsdb package.

func ReadSeriesKey 

func ReadSeriesKey(data []byte) (key, remainder []byte)

ReadSeriesKey returns the series key from the beginning of the buffer.

func ReadSeriesKeyFromSegments 

func ReadSeriesKeyFromSegments(a []*SeriesSegment, offset int64) []byte

ReadSeriesKeyFromSegments returns a series key from an offset within a set of segments.

func ReadSeriesKeyLen 

func ReadSeriesKeyLen(data []byte) (sz int, remainder []byte)

func ReadSeriesKeyMeasurement 

func ReadSeriesKeyMeasurement(data []byte) (name, remainder []byte)

func ReadSeriesKeyTag 

func ReadSeriesKeyTag(data []byte) (key, value, remainder []byte)

func ReadSeriesKeyTagN 

func ReadSeriesKeyTagN(data []byte) (n int, remainder []byte)

func SeriesKeySize 

func SeriesKeySize(name []byte, tags models.Tags) int

SeriesKeySize returns the number of bytes required to encode a series key.

func SeriesKeysSize 

func SeriesKeysSize(names [][]byte, tagsSlice []models.Tags) int

SeriesKeysSize returns the number of bytes required to encode a list of name/tags.

func SeriesSegmentSize 

func SeriesSegmentSize(id uint16) uint32

SeriesSegmentSize returns the maximum size of the segment. The size goes up by powers of 2 starting from 4MB and reaching 256MB.

func SplitSeriesOffset 

func SplitSeriesOffset(offset int64) (segmentID uint16, pos uint32)

SplitSeriesOffset splits a offset into its 2-byte segmentID and 4-byte pos parts.

Types

type BooleanArray 

type BooleanArray = cursors.BooleanArray

func NewBooleanArrayLen 

func NewBooleanArrayLen(sz int) *BooleanArray

type BooleanArrayCursor 

type BooleanArrayCursor = cursors.BooleanArrayCursor

type ByTagKey 

type ByTagKey []*query.TagSet

func (ByTagKey) Len 

func (t ByTagKey) Len() int

func (ByTagKey) Less 

func (t ByTagKey) Less(i, j int) bool

func (ByTagKey) Swap 

func (t ByTagKey) Swap(i, j int)

type Config 

type Config struct {
	// LargeSeriesWriteThreshold is the threshold before a write requires
	// preallocation to improve throughput. Currently used in the series file.
	LargeSeriesWriteThreshold int `toml:"large-series-write-threshold"`
}

Config contains all of the configuration related to tsdb.

func NewConfig 

func NewConfig() Config

NewConfig return a new instance of config with default settings.

type Cursor 

type Cursor = cursors.Cursor

type CursorIterator 

type CursorIterator = cursors.CursorIterator

type CursorIterators 

type CursorIterators = cursors.CursorIterators

type CursorRequest 

type CursorRequest = cursors.CursorRequest

type FloatArray 

type FloatArray = cursors.FloatArray

func NewFloatArrayLen 

func NewFloatArrayLen(sz int) *FloatArray

type FloatArrayCursor 

type FloatArrayCursor = cursors.FloatArrayCursor

type IDData 

type IDData struct {
	Offset  int64
	Key     []byte
	Deleted bool
}

IDData keeps track of data about a series ID.

type IntegerArray 

type IntegerArray = cursors.IntegerArray

func NewIntegerArrayLen 

func NewIntegerArrayLen(sz int) *IntegerArray

type IntegerArrayCursor 

type IntegerArrayCursor = cursors.IntegerArrayCursor

type MeasurementIterator 

type MeasurementIterator interface {
	Close() error
	Next() ([]byte, error)
}

MeasurementIterator represents a iterator over a list of measurements.

func MergeMeasurementIterators 

func MergeMeasurementIterators(itrs ...MeasurementIterator) MeasurementIterator

MergeMeasurementIterators returns an iterator that merges a set of iterators. Iterators that are first in the list take precedence and a deletion by those early iterators will invalidate elements by later iterators.

type MeasurementIterators 

type MeasurementIterators []MeasurementIterator

func (MeasurementIterators) Close 

func (a MeasurementIterators) Close() (err error)

type PartialWriteError 

type PartialWriteError struct {
	Reason  string
	Dropped int

	// A sorted slice of series keys that were dropped.
	DroppedKeys [][]byte
}

PartialWriteError indicates a write request could only write a portion of the requested values.

func (PartialWriteError) Error 

func (e PartialWriteError) Error() string

type SeriesCollection 

type SeriesCollection struct {
	Points     []models.Point
	Keys       [][]byte
	SeriesKeys [][]byte
	Names      [][]byte
	Tags       []models.Tags
	Types      []models.FieldType
	SeriesIDs  []SeriesID

	// Keeps track of invalid entries.
	Dropped     uint64
	DroppedKeys [][]byte
	Reason      string
	// contains filtered or unexported fields
}

SeriesCollection is a struct of arrays representation of a collection of series that allows for efficient filtering.

func NewSeriesCollection 

func NewSeriesCollection(points []models.Point) *SeriesCollection

NewSeriesCollection builds a SeriesCollection from a slice of points. It does some filtering of invalid points.

func (*SeriesCollection) Advance 

func (s *SeriesCollection) Advance(length int)

Advance will advance all of the slices that can length elements: x = x[length:].

func (*SeriesCollection) ApplyConcurrentDrops 

func (s *SeriesCollection) ApplyConcurrentDrops()

ApplyConcurrentDrops will remove all of the dropped values during concurrent iteration. It should not be called concurrently with any calls to Invalid.

func (*SeriesCollection) Copy 

func (s *SeriesCollection) Copy(dst, src int)

Copy will copy the element at src into dst in all slices that can: x[dst] = x[src].

func (SeriesCollection) Duplicate 

func (s SeriesCollection) Duplicate() *SeriesCollection

Duplicate returns a copy of the SeriesCollection. The slices are shared. Appending to any of them may or may not be reflected.

func (*SeriesCollection) InvalidateAll 

func (s *SeriesCollection) InvalidateAll(reason string)

InvalidateAll causes all of the entries to become invalid.

func (*SeriesCollection) Iterator 

func (s *SeriesCollection) Iterator() SeriesCollectionIterator

Iterator returns a new iterator over the entries in the collection. Multiple iterators can exist at the same time. Marking entries as invalid/skipped is more expensive, but thread safe. You must call ApplyConcurrentDrops after all of the iterators are finished.

func (*SeriesCollection) Length 

func (s *SeriesCollection) Length() int

Length returns the length of the first non-nil slice in the collection, or 0 if there is no non-nil slice.

func (*SeriesCollection) PartialWriteError 

func (s *SeriesCollection) PartialWriteError() error

PartialWriteError returns a PartialWriteError if any entries have been marked as invalid. It returns an error to avoid `return collection.PartialWriteError()` always being non-nil.

func (*SeriesCollection) Swap 

func (s *SeriesCollection) Swap(i, j int)

Swap will swap the elements at i and j in all slices that can: x[i], x[j] = x[j], x[i].

func (*SeriesCollection) Truncate 

func (s *SeriesCollection) Truncate(length int)

Truncate will truncate all of the slices that can down to length: x = x[:length].

type SeriesCollectionIterator 

type SeriesCollectionIterator struct {
	// contains filtered or unexported fields
}

SeriesCollectionIterator is an iterator over the collection of series.

func (SeriesCollectionIterator) Index 

func (i SeriesCollectionIterator) Index() int

func (*SeriesCollectionIterator) Invalid 

func (i *SeriesCollectionIterator) Invalid(reason string)

Invalid flags the current entry as invalid, including it in the set of dropped keys and recording a reason. Only the first reason is kept. This is safe for concurrent callers, but ApplyConcurrentDrops must be called after all iterators are finished.

func (SeriesCollectionIterator) Key 

func (i SeriesCollectionIterator) Key() []byte

func (SeriesCollectionIterator) Length 

func (i SeriesCollectionIterator) Length() int

func (SeriesCollectionIterator) Name 

func (i SeriesCollectionIterator) Name() []byte

func (*SeriesCollectionIterator) Next 

func (i *SeriesCollectionIterator) Next() bool

Next advances the iterator and returns false if it's done.

func (SeriesCollectionIterator) Point 

func (i SeriesCollectionIterator) Point() models.Point

func (SeriesCollectionIterator) SeriesID 

func (i SeriesCollectionIterator) SeriesID() SeriesID

func (SeriesCollectionIterator) SeriesKey 

func (i SeriesCollectionIterator) SeriesKey() []byte

func (SeriesCollectionIterator) Tags 

func (i SeriesCollectionIterator) Tags() models.Tags

func (SeriesCollectionIterator) Type 

func (i SeriesCollectionIterator) Type() models.FieldType

type SeriesFile 

type SeriesFile struct {
	LargeWriteThreshold int

	Logger *zap.Logger
	// contains filtered or unexported fields
}

SeriesFile represents the section of the index that holds series data.

func NewSeriesFile 

func NewSeriesFile(path string) *SeriesFile

NewSeriesFile returns a new instance of SeriesFile.

func (*SeriesFile) Acquire 

func (f *SeriesFile) Acquire() (*lifecycle.Reference, error)

Acquire ensures that the series file won't be closed until after the reference has been released.

func (*SeriesFile) Close 

func (f *SeriesFile) Close() error

Close unmaps the data file.

func (*SeriesFile) CreateSeriesListIfNotExists 

func (f *SeriesFile) CreateSeriesListIfNotExists(collection *SeriesCollection) error

CreateSeriesListIfNotExists creates a list of series in bulk if they don't exist. It overwrites the collection's Keys and SeriesIDs fields. The collection's SeriesIDs slice will have IDs for every name+tags, creating new series IDs as needed. If any SeriesID is zero, then a type conflict has occurred for that series.

func (*SeriesFile) DeleteSeriesID 

func (f *SeriesFile) DeleteSeriesID(id SeriesID) error

DeleteSeriesID flags a series as permanently deleted. If the series is reintroduced later then it must create a new id.

func (*SeriesFile) DisableCompactions 

func (f *SeriesFile) DisableCompactions()

DisableCompactions prevents new compactions from running.

func (*SeriesFile) DisableMetrics 

func (f *SeriesFile) DisableMetrics()

DisableMetrics ensures that activity is not collected via the prometheus metrics. DisableMetrics must be called before Open.

func (*SeriesFile) EnableCompactions 

func (f *SeriesFile) EnableCompactions()

EnableCompactions allows compactions to run.

func (*SeriesFile) HasSeries 

func (f *SeriesFile) HasSeries(name []byte, tags models.Tags, buf []byte) bool

HasSeries return true if the series exists.

func (*SeriesFile) IsDeleted 

func (f *SeriesFile) IsDeleted(id SeriesID) bool

IsDeleted returns true if the ID has been deleted before.

func (*SeriesFile) Open 

func (f *SeriesFile) Open(ctx context.Context) error

Open memory maps the data file at the file's path.

func (*SeriesFile) Partitions 

func (f *SeriesFile) Partitions() []*SeriesPartition

Partitions returns all partitions.

func (*SeriesFile) Path 

func (f *SeriesFile) Path() string

Path returns the path to the file.

func (*SeriesFile) Series 

func (f *SeriesFile) Series(id SeriesID) ([]byte, models.Tags)

Series returns the parsed series name and tags for an offset.

func (*SeriesFile) SeriesCount 

func (f *SeriesFile) SeriesCount() uint64

SeriesCount returns the number of series.

func (*SeriesFile) SeriesID 

func (f *SeriesFile) SeriesID(name []byte, tags models.Tags, buf []byte) SeriesID

SeriesID returns the series id for the series.

func (*SeriesFile) SeriesIDIterator 

func (f *SeriesFile) SeriesIDIterator() SeriesIDIterator

SeriesIterator returns an iterator over all the series.

func (*SeriesFile) SeriesIDPartition 

func (f *SeriesFile) SeriesIDPartition(id SeriesID) *SeriesPartition

func (*SeriesFile) SeriesIDPartitionID 

func (f *SeriesFile) SeriesIDPartitionID(id SeriesID) int

func (*SeriesFile) SeriesIDTyped 

func (f *SeriesFile) SeriesIDTyped(name []byte, tags models.Tags, buf []byte) SeriesIDTyped

SeriesIDTyped returns the typed series id for the series.

func (*SeriesFile) SeriesIDTypedBySeriesKey 

func (f *SeriesFile) SeriesIDTypedBySeriesKey(key []byte) SeriesIDTyped

SeriesIDTypedBySeriesKey returns the typed series id for the series.

func (*SeriesFile) SeriesKey 

func (f *SeriesFile) SeriesKey(id SeriesID) []byte

SeriesKey returns the series key for a given id.

func (*SeriesFile) SeriesKeyName 

func (f *SeriesFile) SeriesKeyName(id SeriesID) []byte

SeriesKeyName returns the measurement name for a series id.

func (*SeriesFile) SeriesKeyPartition 

func (f *SeriesFile) SeriesKeyPartition(key []byte) *SeriesPartition

func (*SeriesFile) SeriesKeyPartitionID 

func (f *SeriesFile) SeriesKeyPartitionID(key []byte) int

func (*SeriesFile) SeriesKeys 

func (f *SeriesFile) SeriesKeys(ids []SeriesID) [][]byte

SeriesKeys returns a list of series keys from a list of ids.

func (*SeriesFile) SeriesKeysPartitionIDs 

func (f *SeriesFile) SeriesKeysPartitionIDs(keys [][]byte) []int

func (*SeriesFile) SeriesPartitionPath 

func (f *SeriesFile) SeriesPartitionPath(i int) string

SeriesPartitionPath returns the path to a given partition.

func (*SeriesFile) SetDefaultMetricLabels 

func (f *SeriesFile) SetDefaultMetricLabels(labels prometheus.Labels)

SetDefaultMetricLabels sets the default labels for metrics on the Series File. It must be called before the SeriesFile is opened.

func (*SeriesFile) WithLogger 

func (f *SeriesFile) WithLogger(log *zap.Logger)

WithLogger sets the logger on the SeriesFile and all underlying partitions. It must be called before Open.

type SeriesID 

type SeriesID struct{ ID uint64 }

SeriesID is the type of a series id. It is logically a uint64, but encoded as a struct so that we gain more type checking when changing operations on it. The field is exported only so that tests that use reflection based comparisons still work; no one should use the field directly.

func NewSeriesID 

func NewSeriesID(id uint64) SeriesID

NewSeriesID constructs a series id from the raw value. It discards any type information.

func ReadAllSeriesIDIterator 

func ReadAllSeriesIDIterator(itr SeriesIDIterator) ([]SeriesID, error)

ReadAllSeriesIDIterator returns all ids from the iterator.

func (SeriesID) Greater 

func (s SeriesID) Greater(o SeriesID) bool

Greater returns if the SeriesID is greater than the passed in value.

func (SeriesID) IsZero 

func (s SeriesID) IsZero() bool

IsZero returns if the SeriesID is zero.

func (SeriesID) Less 

func (s SeriesID) Less(o SeriesID) bool

Less returns if the SeriesID is less than the passed in value.

func (SeriesID) RawID 

func (s SeriesID) RawID() uint64

ID returns the raw id for the SeriesID.

func (SeriesID) WithType 

func (s SeriesID) WithType(typ models.FieldType) SeriesIDTyped

WithType constructs a SeriesIDTyped with the given type.

type SeriesIDElem 

type SeriesIDElem struct {
	SeriesID SeriesID
	Expr     influxql.Expr
}

SeriesIDElem represents a single series and optional expression.

type SeriesIDElems 

type SeriesIDElems []SeriesIDElem

SeriesIDElems represents a list of series id elements.

func (SeriesIDElems) Len 

func (a SeriesIDElems) Len() int

func (SeriesIDElems) Less 

func (a SeriesIDElems) Less(i, j int) bool

func (SeriesIDElems) Swap 

func (a SeriesIDElems) Swap(i, j int)

type SeriesIDExprIterator 

type SeriesIDExprIterator struct {
	// contains filtered or unexported fields
}

seriesIDExprIterator is an iterator that attaches an associated expression.

func (*SeriesIDExprIterator) Close 

func (itr *SeriesIDExprIterator) Close() error

func (*SeriesIDExprIterator) Next 

func (itr *SeriesIDExprIterator) Next() (SeriesIDElem, error)

Next returns the next element in the iterator.

type SeriesIDIterator 

type SeriesIDIterator interface {
	Next() (SeriesIDElem, error)
	Close() error
}

SeriesIDIterator represents a iterator over a list of series ids.

func DifferenceSeriesIDIterators 

func DifferenceSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

DifferenceSeriesIDIterators returns an iterator that only returns series which occur the first iterator but not the second iterator.

func FilterUndeletedSeriesIDIterator 

func FilterUndeletedSeriesIDIterator(sfile *SeriesFile, itr SeriesIDIterator) (SeriesIDIterator, error)

FilterUndeletedSeriesIDIterator returns an iterator which filters all deleted series.

func IntersectSeriesIDIterators 

func IntersectSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

IntersectSeriesIDIterators returns an iterator that only returns series which occur in both iterators. If both series have associated expressions then they are combined together.

func MergeSeriesIDIterators 

func MergeSeriesIDIterators(itrs ...SeriesIDIterator) SeriesIDIterator

MergeSeriesIDIterators returns an iterator that merges a set of iterators. Iterators that are first in the list take precedence and a deletion by those early iterators will invalidate elements by later iterators.

func NewSeriesIDExprIterator 

func NewSeriesIDExprIterator(itr SeriesIDIterator, expr influxql.Expr) SeriesIDIterator

newSeriesIDExprIterator returns a new instance of seriesIDExprIterator.

func UnionSeriesIDIterators 

func UnionSeriesIDIterators(itr0, itr1 SeriesIDIterator) SeriesIDIterator

UnionSeriesIDIterators returns an iterator that returns series from both both iterators. If both series have associated expressions then they are combined together.

type SeriesIDIterators 

type SeriesIDIterators []SeriesIDIterator

func (SeriesIDIterators) Close 

func (a SeriesIDIterators) Close() (err error)

type SeriesIDSet 

type SeriesIDSet struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

SeriesIDSet represents a lockable bitmap of series ids.

func NewSeriesIDSet 

func NewSeriesIDSet(a ...SeriesID) *SeriesIDSet

NewSeriesIDSet returns a new instance of SeriesIDSet.

func NewSeriesIDSetNegate 

func NewSeriesIDSetNegate(a, b *SeriesIDSet) *SeriesIDSet

NewSeriesIDSetNegate returns a new SeriesIDSet containing all the elements in a that are not present in b. That is, the set difference between a and b.

func (*SeriesIDSet) Add 

func (s *SeriesIDSet) Add(id SeriesID)

Add adds the series id to the set.

func (*SeriesIDSet) AddMany 

func (s *SeriesIDSet) AddMany(ids ...SeriesID)

AddMany adds multiple ids to the SeriesIDSet. AddMany takes a lock, so may not be optimal to call many times with few ids.

func (*SeriesIDSet) AddNoLock 

func (s *SeriesIDSet) AddNoLock(id SeriesID)

AddNoLock adds the series id to the set. Add is not safe for use from multiple goroutines. Callers must manage synchronization.

func (*SeriesIDSet) And 

func (s *SeriesIDSet) And(other *SeriesIDSet) *SeriesIDSet

And returns a new SeriesIDSet containing elements that were present in s and other.

func (*SeriesIDSet) Bytes 

func (s *SeriesIDSet) Bytes() int

Bytes estimates the memory footprint of this SeriesIDSet, in bytes.

func (*SeriesIDSet) Cardinality 

func (s *SeriesIDSet) Cardinality() uint64

Cardinality returns the cardinality of the SeriesIDSet.

func (*SeriesIDSet) Clear 

func (s *SeriesIDSet) Clear()

Clear clears the underlying bitmap for re-use. Clear is safe for use by multiple goroutines.

func (*SeriesIDSet) ClearNoLock 

func (s *SeriesIDSet) ClearNoLock()

ClearNoLock clears the underlying bitmap for re-use without taking a lock.

func (*SeriesIDSet) Clone 

func (s *SeriesIDSet) Clone() *SeriesIDSet

Clone returns a new SeriesIDSet with a deep copy of the underlying bitmap.

func (*SeriesIDSet) CloneNoLock 

func (s *SeriesIDSet) CloneNoLock() *SeriesIDSet

CloneNoLock calls Clone without taking a lock.

func (*SeriesIDSet) Contains 

func (s *SeriesIDSet) Contains(id SeriesID) bool

Contains returns true if the id exists in the set.

func (*SeriesIDSet) ContainsNoLock 

func (s *SeriesIDSet) ContainsNoLock(id SeriesID) bool

ContainsNoLock returns true if the id exists in the set. ContainsNoLock is not safe for use from multiple goroutines. The caller must manage synchronization.

func (*SeriesIDSet) Diff 

func (s *SeriesIDSet) Diff(other *SeriesIDSet)

Diff removes from s any elements also present in other.

func (*SeriesIDSet) Equals 

func (s *SeriesIDSet) Equals(other *SeriesIDSet) bool

Equals returns true if other and s are the same set of ids.

func (*SeriesIDSet) ForEach 

func (s *SeriesIDSet) ForEach(f func(id SeriesID))

ForEach calls f for each id in the set. The function is applied to the IDs in ascending order.

func (*SeriesIDSet) ForEachNoLock 

func (s *SeriesIDSet) ForEachNoLock(f func(id SeriesID))

ForEachNoLock calls f for each id in the set without taking a lock.

func (*SeriesIDSet) Iterator 

func (s *SeriesIDSet) Iterator() SeriesIDSetIterable

Iterator returns an iterator to the underlying bitmap. This iterator is not protected by a lock.

func (*SeriesIDSet) Merge 

func (s *SeriesIDSet) Merge(others ...*SeriesIDSet)

Merge merged the contents of others into s. The caller does not need to provide s as an argument, and the contents of s will always be present in s after Merge returns.

func (*SeriesIDSet) MergeInPlace 

func (s *SeriesIDSet) MergeInPlace(other *SeriesIDSet)

MergeInPlace merges other into s, modifying s in the process.

func (*SeriesIDSet) Remove 

func (s *SeriesIDSet) Remove(id SeriesID)

Remove removes the id from the set.

func (*SeriesIDSet) RemoveNoLock 

func (s *SeriesIDSet) RemoveNoLock(id SeriesID)

RemoveNoLock removes the id from the set. RemoveNoLock is not safe for use from multiple goroutines. The caller must manage synchronization.

func (*SeriesIDSet) RemoveSet 

func (s *SeriesIDSet) RemoveSet(other *SeriesIDSet)

RemoveSet removes all values in other from s, if they exist.

func (*SeriesIDSet) Slice 

func (s *SeriesIDSet) Slice() []uint64

Slice returns a slice of series ids.

func (*SeriesIDSet) String 

func (s *SeriesIDSet) String() string

func (*SeriesIDSet) UnmarshalBinary 

func (s *SeriesIDSet) UnmarshalBinary(data []byte) error

UnmarshalBinary unmarshals data into the set.

func (*SeriesIDSet) UnmarshalBinaryUnsafe 

func (s *SeriesIDSet) UnmarshalBinaryUnsafe(data []byte) error

UnmarshalBinaryUnsafe unmarshals data into the set. References to the underlying data are used so data should not be reused by caller.

func (*SeriesIDSet) WriteTo 

func (s *SeriesIDSet) WriteTo(w io.Writer) (int64, error)

WriteTo writes the set to w.

type SeriesIDSetIterable 

type SeriesIDSetIterable interface {
	HasNext() bool
	Next() uint32
}

type SeriesIDSetIterator 

type SeriesIDSetIterator interface {
	SeriesIDIterator
	SeriesIDSet() *SeriesIDSet
}

SeriesIDSetIterator represents an iterator that can produce a SeriesIDSet.

func NewSeriesIDSetIterator 

func NewSeriesIDSetIterator(ss *SeriesIDSet) SeriesIDSetIterator

func NewSeriesIDSetIterators 

func NewSeriesIDSetIterators(itrs []SeriesIDIterator) []SeriesIDSetIterator

NewSeriesIDSetIterators returns a slice of SeriesIDSetIterator if all itrs can be type casted. Otherwise returns nil.

type SeriesIDSliceIterator 

type SeriesIDSliceIterator struct {
	// contains filtered or unexported fields
}

SeriesIDSliceIterator iterates over a slice of series ids.

func NewSeriesIDSliceIterator 

func NewSeriesIDSliceIterator(ids []SeriesID) *SeriesIDSliceIterator

NewSeriesIDSliceIterator returns a SeriesIDIterator that iterates over a slice.

func (*SeriesIDSliceIterator) Close 

func (itr *SeriesIDSliceIterator) Close() error

func (*SeriesIDSliceIterator) Next 

func (itr *SeriesIDSliceIterator) Next() (SeriesIDElem, error)

Next returns the next series id in the slice.

func (*SeriesIDSliceIterator) SeriesIDSet 

func (itr *SeriesIDSliceIterator) SeriesIDSet() *SeriesIDSet

SeriesIDSet returns a set of all remaining ids.

type SeriesIDTyped 

type SeriesIDTyped struct{ ID uint64 }

SeriesIDType represents a series id with a type. It is logically a uint64, but encoded as a struct so that we gain more type checking when changing operations on it. The field is exported only so that tests that use reflection based comparisons still work; no one should use the field directly.

func NewSeriesIDTyped 

func NewSeriesIDTyped(id uint64) SeriesIDTyped

NewSeriesIDTyped constructs a typed series id from the raw values.

func ReadSeriesEntry 

func ReadSeriesEntry(data []byte) (flag uint8, id SeriesIDTyped, key []byte, sz int64)

func (SeriesIDTyped) HasType 

func (s SeriesIDTyped) HasType() bool

HasType returns if the id actually contains a type.

func (SeriesIDTyped) IsZero 

func (s SeriesIDTyped) IsZero() bool

IsZero returns if the SeriesIDTyped is zero. It ignores any type information.

func (SeriesIDTyped) RawID 

func (s SeriesIDTyped) RawID() uint64

ID returns the raw id for the SeriesIDTyped.

func (SeriesIDTyped) SeriesID 

func (s SeriesIDTyped) SeriesID() SeriesID

SeriesID constructs a SeriesID, discarding any type information.

func (SeriesIDTyped) Type 

func (s SeriesIDTyped) Type() models.FieldType

Type returns the associated type.

type SeriesIndex 

type SeriesIndex struct {
	// contains filtered or unexported fields
}

SeriesIndex represents an index of key-to-id & id-to-offset mappings.

func NewSeriesIndex 

func NewSeriesIndex(path string) *SeriesIndex

func (*SeriesIndex) Clone 

func (idx *SeriesIndex) Clone() *SeriesIndex

Clone returns a copy of idx for use during compaction. In-memory maps are not cloned.

func (*SeriesIndex) Close 

func (idx *SeriesIndex) Close() (err error)

Close unmaps the index file.

func (*SeriesIndex) Count 

func (idx *SeriesIndex) Count() uint64

Count returns the number of series in the index.

func (*SeriesIndex) Delete 

func (idx *SeriesIndex) Delete(id SeriesID)

Delete marks the series id as deleted.

func (*SeriesIndex) FindIDByNameTags 

func (idx *SeriesIndex) FindIDByNameTags(segments []*SeriesSegment, name []byte, tags models.Tags, buf []byte) SeriesIDTyped

func (*SeriesIndex) FindIDBySeriesKey 

func (idx *SeriesIndex) FindIDBySeriesKey(segments []*SeriesSegment, key []byte) SeriesIDTyped

func (*SeriesIndex) FindIDListByNameTags 

func (idx *SeriesIndex) FindIDListByNameTags(segments []*SeriesSegment, names [][]byte, tagsSlice []models.Tags, buf []byte) (ids []SeriesIDTyped, ok bool)

func (*SeriesIndex) FindOffsetByID 

func (idx *SeriesIndex) FindOffsetByID(id SeriesID) int64

func (*SeriesIndex) GrowBy 

func (idx *SeriesIndex) GrowBy(delta int)

GrowBy preallocates the in-memory hashmap to a larger size.

func (*SeriesIndex) InMemCount 

func (idx *SeriesIndex) InMemCount() uint64

InMemCount returns the number of series in the in-memory index.

func (*SeriesIndex) InMemSize 

func (idx *SeriesIndex) InMemSize() uint64

InMemSize returns the heap size of the index in bytes. The returned value is an estimation and does not include include all allocated memory.

func (*SeriesIndex) Insert 

func (idx *SeriesIndex) Insert(key []byte, id SeriesIDTyped, offset int64)

func (*SeriesIndex) IsDeleted 

func (idx *SeriesIndex) IsDeleted(id SeriesID) bool

IsDeleted returns true if series id has been deleted.

func (*SeriesIndex) OnDiskCount 

func (idx *SeriesIndex) OnDiskCount() uint64

OnDiskCount returns the number of series in the on-disk index.

func (*SeriesIndex) OnDiskSize 

func (idx *SeriesIndex) OnDiskSize() uint64

OnDiskSize returns the on-disk size of the index in bytes.

func (*SeriesIndex) Open 

func (idx *SeriesIndex) Open() (err error)

Open memory-maps the index file.

func (*SeriesIndex) Recover 

func (idx *SeriesIndex) Recover(segments []*SeriesSegment) error

Recover rebuilds the in-memory index for all new entries.

type SeriesIndexHeader 

type SeriesIndexHeader struct {
	Version uint8

	MaxSeriesID SeriesID
	MaxOffset   int64

	Count    uint64
	Capacity int64

	KeyIDMap struct {
		Offset int64
		Size   int64
	}

	IDOffsetMap struct {
		Offset int64
		Size   int64
	}
}

SeriesIndexHeader represents the header of a series index.

func NewSeriesIndexHeader 

func NewSeriesIndexHeader() SeriesIndexHeader

NewSeriesIndexHeader returns a new instance of SeriesIndexHeader.

func ReadSeriesIndexHeader 

func ReadSeriesIndexHeader(data []byte) (hdr SeriesIndexHeader, err error)

ReadSeriesIndexHeader returns the header from data.

func (*SeriesIndexHeader) WriteTo 

func (hdr *SeriesIndexHeader) WriteTo(w io.Writer) (n int64, err error)

WriteTo writes the header to w.

type SeriesPartition 

type SeriesPartition struct {
	CompactThreshold    int
	LargeWriteThreshold int

	Logger *zap.Logger
	// contains filtered or unexported fields
}

SeriesPartition represents a subset of series file data.

func NewSeriesPartition 

func NewSeriesPartition(id int, path string) *SeriesPartition

NewSeriesPartition returns a new instance of SeriesPartition.

func (*SeriesPartition) AppendSeriesIDs 

func (p *SeriesPartition) AppendSeriesIDs(a []SeriesID) []SeriesID

AppendSeriesIDs returns a list of all series ids.

func (*SeriesPartition) Close 

func (p *SeriesPartition) Close() (err error)

Close unmaps the data files.

func (*SeriesPartition) Compacting 

func (p *SeriesPartition) Compacting() bool

Compacting returns if the SeriesPartition is currently compacting.

func (*SeriesPartition) CreateSeriesListIfNotExists 

func (p *SeriesPartition) CreateSeriesListIfNotExists(collection *SeriesCollection, keyPartitionIDs []int) error

CreateSeriesListIfNotExists creates a list of series in bulk if they don't exist. The ids parameter is modified to contain series IDs for all keys belonging to this partition. If the type does not match the existing type for the key, a zero id is stored.

func (*SeriesPartition) DeleteSeriesID 

func (p *SeriesPartition) DeleteSeriesID(id SeriesID) error

DeleteSeriesID flags a series as permanently deleted. If the series is reintroduced later then it must create a new id.

func (*SeriesPartition) DisableCompactions 

func (p *SeriesPartition) DisableCompactions()

func (*SeriesPartition) DiskSize 

func (p *SeriesPartition) DiskSize() uint64

DiskSize returns the number of bytes taken up on disk by the partition.

func (*SeriesPartition) EnableCompactions 

func (p *SeriesPartition) EnableCompactions()

func (*SeriesPartition) FindIDBySeriesKey 

func (p *SeriesPartition) FindIDBySeriesKey(key []byte) SeriesID

FindIDBySeriesKey return the series id for the series key.

func (*SeriesPartition) FindIDTypedBySeriesKey 

func (p *SeriesPartition) FindIDTypedBySeriesKey(key []byte) SeriesIDTyped

FindIDTypedBySeriesKey return the typed series id for the series key.

func (*SeriesPartition) ID 

func (p *SeriesPartition) ID() int

ID returns the partition id.

func (*SeriesPartition) IndexPath 

func (p *SeriesPartition) IndexPath() string

IndexPath returns the path to the series index.

func (*SeriesPartition) IsDeleted 

func (p *SeriesPartition) IsDeleted(id SeriesID) bool

IsDeleted returns true if the ID has been deleted before.

func (*SeriesPartition) Open 

func (p *SeriesPartition) Open() error

Open memory maps the data file at the partition's path.

func (*SeriesPartition) Path 

func (p *SeriesPartition) Path() string

Path returns the path to the partition.

func (*SeriesPartition) Series 

func (p *SeriesPartition) Series(id SeriesID) ([]byte, models.Tags)

Series returns the parsed series name and tags for an offset.

func (*SeriesPartition) SeriesCount 

func (p *SeriesPartition) SeriesCount() uint64

SeriesCount returns the number of series.

func (*SeriesPartition) SeriesKey 

func (p *SeriesPartition) SeriesKey(id SeriesID) []byte

SeriesKey returns the series key for a given id.

type SeriesPartitionCompactor 

type SeriesPartitionCompactor struct {
	// contains filtered or unexported fields
}

SeriesPartitionCompactor represents an object reindexes a series partition and optionally compacts segments.

func NewSeriesPartitionCompactor 

func NewSeriesPartitionCompactor() *SeriesPartitionCompactor

NewSeriesPartitionCompactor returns a new instance of SeriesPartitionCompactor.

func (*SeriesPartitionCompactor) Compact 

func (c *SeriesPartitionCompactor) Compact(p *SeriesPartition) (time.Duration, error)

Compact rebuilds the series partition index.

type SeriesSegment 

type SeriesSegment struct {
	// contains filtered or unexported fields
}

SeriesSegment represents a log of series entries.

func CloneSeriesSegments 

func CloneSeriesSegments(a []*SeriesSegment) []*SeriesSegment

CloneSeriesSegments returns a copy of a slice of segments.

func CreateSeriesSegment 

func CreateSeriesSegment(id uint16, path string) (*SeriesSegment, error)

CreateSeriesSegment generates an empty segment at path.

func FindSegment 

func FindSegment(a []*SeriesSegment, id uint16) *SeriesSegment

FindSegment returns a segment by id.

func NewSeriesSegment 

func NewSeriesSegment(id uint16, path string) *SeriesSegment

NewSeriesSegment returns a new instance of SeriesSegment.

func (*SeriesSegment) AppendSeriesIDs 

func (s *SeriesSegment) AppendSeriesIDs(a []SeriesID) []SeriesID

AppendSeriesIDs appends all the segments ids to a slice. Returns the new slice.

func (*SeriesSegment) CanWrite 

func (s *SeriesSegment) CanWrite(data []byte) bool

CanWrite returns true if segment has space to write entry data.

func (*SeriesSegment) Clone 

func (s *SeriesSegment) Clone() *SeriesSegment

Clone returns a copy of the segment. Excludes the write handler, if set.

func (*SeriesSegment) Close 

func (s *SeriesSegment) Close() (err error)

Close unmaps the segment.

func (*SeriesSegment) CloseForWrite 

func (s *SeriesSegment) CloseForWrite() (err error)

func (*SeriesSegment) Data 

func (s *SeriesSegment) Data() []byte

Data returns the raw data.

func (*SeriesSegment) Flush 

func (s *SeriesSegment) Flush() error

Flush flushes the buffer to disk.

func (*SeriesSegment) ForEachEntry 

func (s *SeriesSegment) ForEachEntry(fn func(flag uint8, id SeriesIDTyped, offset int64, key []byte) error) error

ForEachEntry executes fn for every entry in the segment.

func (*SeriesSegment) ID 

func (s *SeriesSegment) ID() uint16

ID returns the id the segment was initialized with.

func (*SeriesSegment) InitForWrite 

func (s *SeriesSegment) InitForWrite() (err error)

InitForWrite initializes a write handle for the segment. This is only used for the last segment in the series file.

func (*SeriesSegment) MaxSeriesID 

func (s *SeriesSegment) MaxSeriesID() SeriesID

MaxSeriesID returns the highest series id in the segment.

func (*SeriesSegment) Open 

func (s *SeriesSegment) Open() error

Open memory maps the data file at the file's path.

func (*SeriesSegment) Size 

func (s *SeriesSegment) Size() int64

Size returns the size of the data in the segment. This is only populated once InitForWrite() is called.

func (*SeriesSegment) Slice 

func (s *SeriesSegment) Slice(pos uint32) []byte

Slice returns a byte slice starting at pos.

func (*SeriesSegment) WriteLogEntry 

func (s *SeriesSegment) WriteLogEntry(data []byte) (offset int64, err error)

WriteLogEntry writes entry data into the segment. Returns the offset of the beginning of the entry.

type SeriesSegmentHeader 

type SeriesSegmentHeader struct {
	Version uint8
}

SeriesSegmentHeader represents the header of a series segment.

func NewSeriesSegmentHeader 

func NewSeriesSegmentHeader() SeriesSegmentHeader

NewSeriesSegmentHeader returns a new instance of SeriesSegmentHeader.

func ReadSeriesSegmentHeader 

func ReadSeriesSegmentHeader(data []byte) (hdr SeriesSegmentHeader, err error)

ReadSeriesSegmentHeader returns the header from data.

func (*SeriesSegmentHeader) WriteTo 

func (hdr *SeriesSegmentHeader) WriteTo(w io.Writer) (n int64, err error)

WriteTo writes the header to w.

type StringArray 

type StringArray = cursors.StringArray

func NewStringArrayLen 

func NewStringArrayLen(sz int) *StringArray

type StringArrayCursor 

type StringArrayCursor = cursors.StringArrayCursor

type TagKeyIterator 

type TagKeyIterator interface {
	Close() error
	Next() ([]byte, error)
}

TagKeyIterator represents a iterator over a list of tag keys.

func MergeTagKeyIterators 

func MergeTagKeyIterators(itrs ...TagKeyIterator) TagKeyIterator

MergeTagKeyIterators returns an iterator that merges a set of iterators.

type TagKeyIterators 

type TagKeyIterators []TagKeyIterator

func (TagKeyIterators) Close 

func (a TagKeyIterators) Close() (err error)

type TagValueIterator 

type TagValueIterator interface {
	Close() error
	Next() ([]byte, error)
}

TagValueIterator represents a iterator over a list of tag values.

func MergeTagValueIterators 

func MergeTagValueIterators(itrs ...TagValueIterator) TagValueIterator

MergeTagValueIterators returns an iterator that merges a set of iterators.

type TagValueIterators 

type TagValueIterators []TagValueIterator

func (TagValueIterators) Close 

func (a TagValueIterators) Close() (err error)

type TimestampArray 

type TimestampArray = cursors.TimestampArray

type UnsignedArray 

type UnsignedArray = cursors.UnsignedArray

func NewUnsignedArrayLen 

func NewUnsignedArrayLen(sz int) *UnsignedArray

type UnsignedArrayCursor 

type UnsignedArrayCursor = cursors.UnsignedArrayCursor

type Verify 

type Verify struct {
	Concurrent int
	Logger     *zap.Logger
	// contains filtered or unexported fields
}

Verify contains configuration for running verification of series files.

func NewVerify 

func NewVerify() Verify

NewVerify constructs a Verify with good defaults.

func (Verify) VerifyIndex 

func (v Verify) VerifyIndex(indexPath string, segments []*SeriesSegment,
	ids map[uint64]IDData) (valid bool, err error)

VerifyIndex performs verification on an index in a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition. The ids map must be built from verifying the passed in segments.

func (Verify) VerifyPartition 

func (v Verify) VerifyPartition(partitionPath string) (valid bool, err error)

VerifyPartition performs verifications on a partition of a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition.

func (Verify) VerifySegment 

func (v Verify) VerifySegment(segmentPath string, ids map[uint64]IDData) (valid bool, err error)

VerifySegment performs verifications on a segment of a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the partition. The ids map is populated with information about the ids stored in the segment.

func (Verify) VerifySeriesFile 

func (v Verify) VerifySeriesFile(filePath string) (valid bool, err error)

VerifySeriesFile performs verifications on a series file. The error is only returned if there was some fatal problem with operating, not if there was a problem with the series file.

Source Files

View all Source files

Directories

Path Synopsis
Package tsi1 provides a memory-mapped index implementation that supports high cardinality series.
 Package tsi1 provides a memory-mapped index implementation that supports high cardinality series.
Package tsm1 provides a TSDB in the Time Structured Merge tree format.
 Package tsm1 provides a TSDB in the Time Structured Merge tree format.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.