block

Documentation

Overview

Package block contains common functionality for interacting with TSDB blocks in the context of Thanos.

Index

• Constants
• func Delete(ctx context.Context, bucket objstore.Bucket, id ulid.ULID) error
• func Download(ctx context.Context, logger log.Logger, bucket objstore.Bucket, id ulid.ULID, ...) error
• func IgnoreCompleteOutsideChunk(mint int64, maxt int64, _ *chunks.Meta, curr *chunks.Meta) (bool, error)
• func IgnoreDuplicateOutsideChunk(_ int64, _ int64, last *chunks.Meta, curr *chunks.Meta) (bool, error)
• func IgnoreIssue347OutsideChunk(_ int64, maxt int64, _ *chunks.Meta, curr *chunks.Meta) (bool, error)
• func IsBlockDir(path string) (id ulid.ULID, ok bool)
• func ReadIndexCache(logger log.Logger, fn string) (version int, symbols map[uint32]string, lvals map[string][]string, ...)
• func Repair(logger log.Logger, dir string, id ulid.ULID, source SourceType, ...) (resid ulid.ULID, err error)
• func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir string) error
• func VerifyIndex(logger log.Logger, fn string, minTime int64, maxTime int64) error
• func WriteIndexCache(logger log.Logger, fn string, r *index.Reader) error
• func WriteMetaFile(logger log.Logger, dir string, meta *Meta) error
• type Meta
  • func DownloadMeta(ctx context.Context, logger log.Logger, bkt objstore.Bucket, id ulid.ULID) (Meta, error)
  • func InjectThanosMeta(logger log.Logger, bdir string, meta ThanosMeta, ...) (*Meta, error)
  • func ReadMetaFile(dir string) (*Meta, error)
• type SourceType
• type Stats
  • func GatherIndexIssueStats(logger log.Logger, fn string, minTime int64, maxTime int64) (stats Stats, err error)
  • func (i Stats) AnyErr() error
  • func (i Stats) CriticalErr() error
  • func (i Stats) Issue347OutsideChunksErr() error
• type ThanosDownsampleMeta
• type ThanosMeta

Constants

const (
	// MetaFilename is the known JSON filename for meta information.
	MetaFilename = "meta.json"
	// IndexFilename is the known index file for block index.
	IndexFilename = "index"
	// ChunksDirname is the known dir name for chunks with compressed samples.
	ChunksDirname = "chunks"

	// DebugMetas is a directory for debug meta files that happen in the past. Useful for debugging.
	DebugMetas = "debug/metas"
)

const IndexCacheFilename = "index.cache.json"

IndexCacheFilename is the canonical name for index cache files.

Functions

func Delete

func Delete(ctx context.Context, bucket objstore.Bucket, id ulid.ULID) error

Delete removes directory that is mean to be block directory. NOTE: Prefer this method instead of objstore.Delete to avoid deleting empty dir (whole bucket) by mistake.

func Download

func Download(ctx context.Context, logger log.Logger, bucket objstore.Bucket, id ulid.ULID, dst string) error

Download downloads directory that is mean to be block directory.

func IgnoreCompleteOutsideChunk

func IgnoreCompleteOutsideChunk(mint int64, maxt int64, _ *chunks.Meta, curr *chunks.Meta) (bool, error)

func IgnoreDuplicateOutsideChunk

func IgnoreDuplicateOutsideChunk(_ int64, _ int64, last *chunks.Meta, curr *chunks.Meta) (bool, error)

func IgnoreIssue347OutsideChunk

func IgnoreIssue347OutsideChunk(_ int64, maxt int64, _ *chunks.Meta, curr *chunks.Meta) (bool, error)

func IsBlockDir

func IsBlockDir(path string) (id ulid.ULID, ok bool)

func ReadIndexCache

func ReadIndexCache(logger log.Logger, fn string) (
	version int,
	symbols map[uint32]string,
	lvals map[string][]string,
	postings map[labels.Label]index.Range,
	err error,
)

ReadIndexCache reads an index cache file.

func Repair

func Repair(logger log.Logger, dir string, id ulid.ULID, source SourceType, ignoreChkFns ...ignoreFnType) (resid ulid.ULID, err error)

Repair open the block with given id in dir and creates a new one with fixed data. It: - removes out of order duplicates - all "complete" outsiders (they will not accessed anyway) - removes all near "complete" outside chunks introduced by https://github.com/prometheus/tsdb/issues/347. Fixable inconsistencies are resolved in the new block. TODO(bplotka): https://github.com/improbable-eng/thanos/issues/378

func Upload

func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir string) error

Upload uploads block from given block dir that ends with block id. It makes sure cleanup is done on error to avoid partial block uploads. It also verifies basic features of Thanos block. TODO(bplotka): Ensure bucket operations have reasonable backoff retries.

func VerifyIndex

func VerifyIndex(logger log.Logger, fn string, minTime int64, maxTime int64) error

VerifyIndex does a full run over a block index and verifies that it fulfills the order invariants.

func WriteIndexCache

func WriteIndexCache(logger log.Logger, fn string, r *index.Reader) error

WriteIndexCache writes a cache file containing the first lookup stages for an index file.

func WriteMetaFile

func WriteMetaFile(logger log.Logger, dir string, meta *Meta) error

WriteMetaFile writes the given meta into <dir>/meta.json.

Types

type Meta

type Meta struct {
	Version int `json:"version"`

	tsdb.BlockMeta

	Thanos ThanosMeta `json:"thanos"`
}

Meta describes the a block's meta. It wraps the known TSDB meta structure and extends it by Thanos-specific fields.

func DownloadMeta

func DownloadMeta(ctx context.Context, logger log.Logger, bkt objstore.Bucket, id ulid.ULID) (Meta, error)

DownloadMeta downloads only meta file from bucket by block ID.

func InjectThanosMeta

func InjectThanosMeta(logger log.Logger, bdir string, meta ThanosMeta, downsampledMeta *tsdb.BlockMeta) (*Meta, error)

InjectThanosMeta sets Thanos meta to the block meta JSON and saves it to the disk. NOTE: It should be used after writing any block by any Thanos component, otherwise we will miss crucial metadata.

func ReadMetaFile

func ReadMetaFile(dir string) (*Meta, error)

ReadMetaFile reads the given meta from <dir>/meta.json.

type SourceType

type SourceType string

const (
	UnknownSource         SourceType = ""
	SidecarSource         SourceType = "sidecar"
	CompactorSource       SourceType = "compactor"
	CompactorRepairSource SourceType = "compactor.repair"
	RulerSource           SourceType = "ruler"
	BucketRepairSource    SourceType = "bucket.repair"
	TestSource            SourceType = "test"
)

type Stats

type Stats struct {
	// TotalSeries represents total number of series in block.
	TotalSeries int
	// OutOfOrderSeries represents number of series that have out of order chunks.
	OutOfOrderSeries int

	// OutOfOrderChunks represents number of chunks that are out of order (older time range is after younger one)
	OutOfOrderChunks int
	// DuplicatedChunks represents number of chunks with same time ranges within same series, potential duplicates.
	DuplicatedChunks int
	// OutsideChunks represents number of all chunks that are before or after time range specified in block meta.
	OutsideChunks int
	// CompleteOutsideChunks is subset of OutsideChunks that will be never accessed. They are completely out of time range specified in block meta.
	CompleteOutsideChunks int
	// Issue347OutsideChunks represents subset of OutsideChunks that are outsiders caused by https://github.com/prometheus/tsdb/issues/347
	// and is something that Thanos handle.
	//
	// Specifically we mean here chunks with minTime == block.maxTime and maxTime > block.MaxTime. These are
	// are segregated into separate counters. These chunks are safe to be deleted, since they are duplicated across 2 blocks.
	Issue347OutsideChunks int
}

func GatherIndexIssueStats

func GatherIndexIssueStats(logger log.Logger, fn string, minTime int64, maxTime int64) (stats Stats, err error)

GatherIndexIssueStats returns useful counters as well as outsider chunks (chunks outside of block time range) that helps to assess index health. It considers https://github.com/prometheus/tsdb/issues/347 as something that Thanos can handle. See Stats.Issue347OutsideChunks for details.

func (Stats) AnyErr

func (i Stats) AnyErr() error

AnyErr returns error if stats indicates any block issue.

func (Stats) CriticalErr

func (i Stats) CriticalErr() error

CriticalErr returns error if stats indicates critical block issue, that might solved only by manual repair procedure.

func (Stats) Issue347OutsideChunksErr

func (i Stats) Issue347OutsideChunksErr() error

Issue347OutsideChunksErr returns error if stats indicates issue347 block issue, that is repaired explicitly before compaction (on plan block).

type ThanosDownsampleMeta

type ThanosDownsampleMeta struct {
	Resolution int64 `json:"resolution"`
}

type ThanosMeta

type ThanosMeta struct {
	Labels     map[string]string    `json:"labels"`
	Downsample ThanosDownsampleMeta `json:"downsample"`

	// Source is a real upload source of the block.
	Source SourceType `json:"source"`
}

ThanosMeta holds block meta information specific to Thanos.