chunk

Documentation

Index

• Constants
• type Batch
  • func (b *Batch) At() (int64, float64)
  • func (b *Batch) AtFloatHistogram() (int64, unsafe.Pointer)
  • func (b *Batch) AtHistogram() (int64, unsafe.Pointer)
  • func (b *Batch) AtTime() int64
  • func (b *Batch) GetIteratorID() int
  • func (b *Batch) HasNext() chunkenc.ValueType
  • func (b *Batch) Next()
  • func (b *Batch) SetIteratorID(id int)
• type Chunk
  • func NewChunk(metric labels.Labels, c EncodedChunk, from, through model.Time) Chunk
  • func (c *Chunk) Samples(from, through model.Time) ([]model.SamplePair, []mimirpb.Histogram, error)
• type EncodedChunk
  • func NewForEncoding(encoding Encoding) (EncodedChunk, error)
• type Encoding
  • func (e Encoding) String() string
• type ErrorIterator
  • func (e ErrorIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram)
  • func (e ErrorIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram)
  • func (e ErrorIterator) Batch(_ int, _ chunkenc.ValueType, _ *zeropool.Pool[*histogram.Histogram], ...) Batch
  • func (e ErrorIterator) Err() error
  • func (e ErrorIterator) FindAtOrAfter(_ model.Time) chunkenc.ValueType
  • func (e ErrorIterator) Scan() chunkenc.ValueType
  • func (e ErrorIterator) Timestamp() int64
  • func (e ErrorIterator) Value() model.SamplePair
• type Iterator

Constants

const BatchSize = 12

BatchSize is samples per batch; this was choose by benchmarking all sizes from 1 to 128.

const (
	// ChunkLen is the length of a chunk in bytes.
	ChunkLen = 1024
)

Types

type Batch

type Batch struct {
	Timestamps [BatchSize]int64
	// Values stores float values related to this batch if ValueType is chunkenc.ValFloat.
	// If ValueType is chunkenc.ValHistogram or chunkenc.ValFloatHistogram, it is used to store the iteratorID the
	// pointer value at the same index comes from. The iteratorID is required to ensure the counter reset is calculated
	// properly - if we switch between different iterators, we cannot trust the previously calculated counter reset for
	// the next sample is calculated correctly. See https://github.com/prometheus/prometheus/issues/15346 for more
	// information. Values is reused in two different scenarios to save space.
	Values [BatchSize]float64
	// PointerValues store pointers to non-float complex values like histograms, float histograms or future additions.
	// Since Batch is expected to be passed by value, the array needs to be constant sized,
	// however increasing the size of the Batch also adds memory management overhead. Using the unsafe.Pointer
	// combined with the ValueType implements a kind of "union" type to keep the memory use down.
	PointerValues [BatchSize]unsafe.Pointer
	ValueType     chunkenc.ValueType
	Index         int
	Length        int
}

Batches are sorted sets of (timestamp, value) pairs, where all values are of the same type (i.e. floats/histograms).

Batch is intended to be small, and passed by value!

func (*Batch) At

func (b *Batch) At() (int64, float64)

func (*Batch) AtFloatHistogram

func (b *Batch) AtFloatHistogram() (int64, unsafe.Pointer)

func (*Batch) AtHistogram

func (b *Batch) AtHistogram() (int64, unsafe.Pointer)

func (*Batch) AtTime

func (b *Batch) AtTime() int64

func (*Batch) GetIteratorID

func (b *Batch) GetIteratorID() int

GetIteratorID retrieves the non overlapping iterator id. Only call when type is chunkenc.ValHistogram or chunkenc.ValFloatHistogram.

func (*Batch) HasNext

func (b *Batch) HasNext() chunkenc.ValueType

func (*Batch) Next

func (b *Batch) Next()

func (*Batch) SetIteratorID

func (b *Batch) SetIteratorID(id int)

SetIteratorID stores the non overlapping iterator id. Only call when type is chunkenc.ValHistogram or chunkenc.ValFloatHistogram.

type Chunk

type Chunk struct {
	From    model.Time    `json:"from"`
	Through model.Time    `json:"through"`
	Metric  labels.Labels `json:"metric"`
	Data    EncodedChunk  `json:"-"`
}

Chunk contains encoded timeseries data

func NewChunk

func NewChunk(metric labels.Labels, c EncodedChunk, from, through model.Time) Chunk

NewChunk creates a new chunk

func (*Chunk) Samples

func (c *Chunk) Samples(from, through model.Time) ([]model.SamplePair, []mimirpb.Histogram, error)

Samples returns all SamplePairs and Histograms for the chunk.

type EncodedChunk

type EncodedChunk interface {
	// Add adds a SamplePair to the chunks, performs any necessary
	// re-encoding, and creates any necessary overflow chunk.
	// The returned EncodedChunk is the overflow chunk if it was created.
	// The returned EncodedChunk is nil if the sample got appended to the same chunk.
	Add(sample model.SamplePair) (EncodedChunk, error)
	// AddHistogram adds a histogram to the chunks, performs any necessary
	// re-encoding, and creates any necessary overflow chunk.
	// The returned EncodedChunk is the overflow chunk if it was created.
	// The returned EncodedChunk is nil if the histogram got appended to the same chunk.
	AddHistogram(timestamp int64, histogram *histogram.Histogram) (EncodedChunk, error)

	// AddFloatHistogram adds a float histogram to the chunks, performs any necessary
	// re-encoding, and creates any necessary overflow chunk.
	// The returned EncodedChunk is the overflow chunk if it was created.
	// The returned EncodedChunk is nil if the histogram got appended to the same chunk.
	AddFloatHistogram(timestamp int64, histogram *histogram.FloatHistogram) (EncodedChunk, error)

	// NewIterator returns an iterator for the chunks.
	// The iterator passed as argument is for re-use. Depending on implementation,
	// the iterator can be re-used or a new iterator can be allocated.
	NewIterator(Iterator) Iterator
	Marshal(io.Writer) error
	UnmarshalFromBuf([]byte) error
	Encoding() Encoding

	// Len returns the number of samples in the chunk.  Implementations may be
	// expensive.
	Len() int
}

EncodedChunk is the interface for encoded chunks. Chunks are generally not goroutine-safe.

func NewForEncoding

func NewForEncoding(encoding Encoding) (EncodedChunk, error)

NewForEncoding allows configuring what chunk type you want

type Encoding

type Encoding byte

Encoding defines which encoding we are using, delta, doubledelta, or varbit

const (
	// PrometheusXorChunk is a wrapper around Prometheus XOR-encoded chunk.
	// IMPORTANT: for backward compatibility reasons we need to keep the value hardcoded.
	PrometheusXorChunk Encoding = 4
	// PrometheusHistogramChunk is a wrapper around Prometheus histogram-encoded chunk.
	// IMPORTANT: for backward compatibility reasons we need to keep the value hardcoded.
	PrometheusHistogramChunk Encoding = 5
	// PrometheusFloatHistogramChunk is a wrapper around Prometheus histogram-encoded chunk.
	// IMPORTANT: for backward compatibility reasons we need to keep the value hardcoded.
	PrometheusFloatHistogramChunk Encoding = 6
)

func (Encoding) String

func (e Encoding) String() string

String implements flag.Value.

type ErrorIterator

type ErrorIterator string

func (ErrorIterator) AtFloatHistogram

func (e ErrorIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram)

func (ErrorIterator) AtHistogram

func (e ErrorIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram)

func (ErrorIterator) Batch

func (e ErrorIterator) Batch(_ int, _ chunkenc.ValueType, _ *zeropool.Pool[*histogram.Histogram], _ *zeropool.Pool[*histogram.FloatHistogram]) Batch

func (ErrorIterator) Err

func (e ErrorIterator) Err() error

func (ErrorIterator) FindAtOrAfter

func (e ErrorIterator) FindAtOrAfter(_ model.Time) chunkenc.ValueType

func (ErrorIterator) Scan

func (e ErrorIterator) Scan() chunkenc.ValueType

func (ErrorIterator) Timestamp

func (e ErrorIterator) Timestamp() int64

func (ErrorIterator) Value

func (e ErrorIterator) Value() model.SamplePair

type Iterator

type Iterator interface {
	// Scans the next value in the chunk. Directly after the iterator has
	// been created, the next value is the first value in the
	// chunk. Otherwise, it is the value following the last value scanned or
	// found (by one of the Find... methods). Returns chunkenc.ValNone if either the
	// end of the chunk is reached or an error has occurred.
	Scan() chunkenc.ValueType
	// Finds the oldest value at or after the provided time. Returns chunkenc.ValNone
	// if either the chunk contains no value at or after the provided time,
	// or an error has occurred.
	FindAtOrAfter(model.Time) chunkenc.ValueType
	// Returns the last value scanned (by the scan method) or found (by one
	// of the find... methods). It returns model.ZeroSamplePair before any of
	// those methods were called.
	Value() model.SamplePair
	AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram)
	AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram)
	Timestamp() int64
	// Batch returns a batch of the provisded size; NB not idempotent!  Should only be called
	// once per Scan.
	// When the optional *zeropool.Pool arguments hPool and fhPool are passed, they will be
	// used to optimize memory allocations for histogram.Histogram and histogram.FloatHistogram
	// objects.
	// For example, when creating a batch of chunkenc.ValHistogram or chunkenc.ValFloatHistogram
	// objects, the histogram.Histogram or histogram.FloatHistograms objects already present in
	// the hPool or fhPool pool will be used instead of creating new ones.
	Batch(size int, valueType chunkenc.ValueType, hPool *zeropool.Pool[*histogram.Histogram], fhPool *zeropool.Pool[*histogram.FloatHistogram]) Batch
	// Returns the last error encountered. In general, an error signals data
	// corruption in the chunk and requires quarantining.
	Err() error
}

Iterator enables efficient access to the content of a chunk. It is generally not safe to use an Iterator concurrently with or after chunk mutation.