chunks

Documentation

Index

• Constants
• Variables
• type ByteSlice
• type ChunkDiskMapper
  • func NewChunkDiskMapper(dir string, pool chunkenc.Pool, writeBufferSize int) (*ChunkDiskMapper, error)
  • func (cdm *ChunkDiskMapper) Chunk(ref uint64) (chunkenc.Chunk, error)
  • func (cdm *ChunkDiskMapper) Close() error
  • func (cdm *ChunkDiskMapper) CutNewFile() (returnErr error)
  • func (cdm *ChunkDiskMapper) DeleteCorrupted(originalErr error) error
  • func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef, chunkRef uint64, mint, maxt int64, numSamples uint16) error) (err error)
  • func (cdm *ChunkDiskMapper) Size() (int64, error)
  • func (cdm *ChunkDiskMapper) Truncate(mint int64) error
  • func (cdm *ChunkDiskMapper) WriteChunk(seriesRef uint64, mint, maxt int64, chk chunkenc.Chunk) (chkRef uint64, err error)
• type CorruptionErr
  • func (e *CorruptionErr) Error() string
• type Iterator
• type Meta
  • func (cm *Meta) OverlapsClosedInterval(mint, maxt int64) bool
• type Reader
  • func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error)
  • func (s *Reader) Chunk(ref uint64) (chunkenc.Chunk, error)
  • func (s *Reader) Close() error
  • func (s *Reader) Size() int64
• type Writer
  • func NewWriter(dir string) (*Writer, error)
  • func NewWriterWithSegSize(dir string, segmentSize int64) (*Writer, error)
  • func (w *Writer) Close() error
  • func (w *Writer) WriteChunks(chks ...Meta) error

Constants

const (
	// MagicChunks is 4 bytes at the head of a series file.
	MagicChunks = 0x85BD40DD
	// MagicChunksSize is the size in bytes of MagicChunks.
	MagicChunksSize = 4

	ChunksFormatVersionSize = 1

	// SegmentHeaderSize defines the total size of the header part.
	SegmentHeaderSize = MagicChunksSize + ChunksFormatVersionSize + segmentHeaderPaddingSize
)

Segment header fields constants.

const (
	// MaxChunkLengthFieldSize defines the maximum size of the data length part.
	MaxChunkLengthFieldSize = binary.MaxVarintLen32
	// ChunkEncodingSize defines the size of the chunk encoding part.
	ChunkEncodingSize = 1
)

Chunk fields constants.

const (
	// MintMaxtSize is the size of the mint/maxt for head chunk file and chunks.
	MintMaxtSize = 8
	// SeriesRefSize is the size of series reference on disk.
	SeriesRefSize = 8
	// HeadChunkFileHeaderSize is the total size of the header for the head chunk file.
	HeadChunkFileHeaderSize = SegmentHeaderSize
	// MaxHeadChunkFileSize is the max size of a head chunk file.
	MaxHeadChunkFileSize = 128 * 1024 * 1024 // 128 MiB.
	// CRCSize is the size of crc32 sum on disk.
	CRCSize = 4
	// MaxHeadChunkMetaSize is the max size of an mmapped chunks minus the chunks data.
	// Max because the uvarint size can be smaller.
	MaxHeadChunkMetaSize = SeriesRefSize + 2*MintMaxtSize + ChunksFormatVersionSize + MaxChunkLengthFieldSize + CRCSize
	// MinWriteBufferSize is the minimum write buffer size allowed.
	MinWriteBufferSize = 64 * 1024 // 64KB.
	// MaxWriteBufferSize is the maximum write buffer size allowed.
	MaxWriteBufferSize = 8 * 1024 * 1024 // 8 MiB.
	// DefaultWriteBufferSize is the default write buffer size.
	DefaultWriteBufferSize = 4 * 1024 * 1024 // 4 MiB.
)

const (
	// DefaultChunkSegmentSize is the default chunks segment size.
	DefaultChunkSegmentSize = 512 * 1024 * 1024
)

const (
	// MagicHeadChunks is 4 bytes at the beginning of a head chunk file.
	MagicHeadChunks = 0x0130BC91
)

Head chunk file header fields constants.

Variables

var (
	// ErrChunkDiskMapperClosed returned by any method indicates
	// that the ChunkDiskMapper was closed.
	ErrChunkDiskMapperClosed = errors.New("ChunkDiskMapper closed")
)

var (
	// HeadChunkFilePreallocationSize is the size to which the m-map file should be preallocated when a new file is cut.
	// Windows needs pre-allocations while the other OS does not.
	HeadChunkFilePreallocationSize int64
)

Types

type ByteSlice

type ByteSlice interface {
	Len() int
	Range(start, end int) []byte
}

ByteSlice abstracts a byte slice.

type ChunkDiskMapper

type ChunkDiskMapper struct {
	// contains filtered or unexported fields
}

ChunkDiskMapper is for writing the Head block chunks to the disk and access chunks via mmapped file.

func NewChunkDiskMapper

func NewChunkDiskMapper(dir string, pool chunkenc.Pool, writeBufferSize int) (*ChunkDiskMapper, error)

NewChunkDiskMapper returns a new writer against the given directory using the default head chunk file duration. NOTE: 'IterateAllChunks' method needs to be called at least once after creating ChunkDiskMapper to set the maxt of all the file.

func (*ChunkDiskMapper) Chunk

func (cdm *ChunkDiskMapper) Chunk(ref uint64) (chunkenc.Chunk, error)

Chunk returns a chunk from a given reference.

func (*ChunkDiskMapper) Close

func (cdm *ChunkDiskMapper) Close() error

Close closes all the open files in ChunkDiskMapper. It is not longer safe to access chunks from this struct after calling Close.

func (*ChunkDiskMapper) CutNewFile

func (cdm *ChunkDiskMapper) CutNewFile() (returnErr error)

CutNewFile creates a new m-mapped file.

func (*ChunkDiskMapper) DeleteCorrupted

func (cdm *ChunkDiskMapper) DeleteCorrupted(originalErr error) error

DeleteCorrupted deletes all the head chunk files after the one which had the corruption (including the corrupt file).

func (*ChunkDiskMapper) IterateAllChunks

func (cdm *ChunkDiskMapper) IterateAllChunks(f func(seriesRef, chunkRef uint64, mint, maxt int64, numSamples uint16) error) (err error)

IterateAllChunks iterates on all the chunks in its byte slices in the order of the head chunk file sequence and runs the provided function on each chunk. It returns on the first error encountered. NOTE: This method needs to be called at least once after creating ChunkDiskMapper to set the maxt of all the file.

func (*ChunkDiskMapper) Size

func (cdm *ChunkDiskMapper) Size() (int64, error)

Size returns the size of the chunk files.

func (*ChunkDiskMapper) Truncate

func (cdm *ChunkDiskMapper) Truncate(mint int64) error

Truncate deletes the head chunk files which are strictly below the mint. mint should be in milliseconds.

func (*ChunkDiskMapper) WriteChunk

func (cdm *ChunkDiskMapper) WriteChunk(seriesRef uint64, mint, maxt int64, chk chunkenc.Chunk) (chkRef uint64, err error)

WriteChunk writes the chunk to the disk. The returned chunk ref is the reference from where the chunk encoding starts for the chunk.

type CorruptionErr

type CorruptionErr struct {
	Dir       string
	FileIndex int
	Err       error
}

CorruptionErr is an error that's returned when corruption is encountered.

func (*CorruptionErr) Error

func (e *CorruptionErr) Error() string

type Iterator

type Iterator interface {
	// At returns the current meta.
	// It depends on implementation if the chunk is populated or not.
	At() Meta
	// Next advances the iterator by one.
	Next() bool
	// Err returns optional error if Next is false.
	Err() error
}

Iterator iterates over the chunk of a time series.

type Meta

type Meta struct {
	// Ref and Chunk hold either a reference that can be used to retrieve
	// chunk data or the data itself.
	// When it is a reference it is the segment offset at which the chunk bytes start.
	// Generally, only one of them is set.
	Ref   uint64
	Chunk chunkenc.Chunk

	// Time range the data covers.
	// When MaxTime == math.MaxInt64 the chunk is still open and being appended to.
	MinTime, MaxTime int64
}

Meta holds information about a chunk of data.

func (*Meta) OverlapsClosedInterval

func (cm *Meta) OverlapsClosedInterval(mint, maxt int64) bool

OverlapsClosedInterval Returns true if the chunk overlaps [mint, maxt].

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader implements a ChunkReader for a serialized byte stream of series data.

func NewDirReader

func NewDirReader(dir string, pool chunkenc.Pool) (*Reader, error)

NewDirReader returns a new Reader against sequentially numbered files in the given directory.

func (*Reader) Chunk

func (s *Reader) Chunk(ref uint64) (chunkenc.Chunk, error)

Chunk returns a chunk from a given reference.

func (*Reader) Close

func (s *Reader) Close() error

func (*Reader) Size

func (s *Reader) Size() int64

Size returns the size of the chunks.

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer implements the ChunkWriter interface for the standard serialization format.

func NewWriter

func NewWriter(dir string) (*Writer, error)

NewWriter returns a new writer against the given directory using the default segment size.

func NewWriterWithSegSize

func NewWriterWithSegSize(dir string, segmentSize int64) (*Writer, error)

NewWriterWithSegSize returns a new writer against the given directory and allows setting a custom size for the segments.

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) WriteChunks

func (w *Writer) WriteChunks(chks ...Meta) error

WriteChunks writes as many chunks as possible to the current segment, cuts a new segment when the current segment is full and writes the rest of the chunks in the new segment.