metadata

Documentation

Index

• Constants
• Variables
• func ReadMarker(ctx context.Context, logger log.Logger, bkt objstore.InstrumentedBucketReader, ...) error
• type DeletionMark
• type DeletionRequest
• type File
• type HashFunc
• type Marker
• type Matchers
  • func (m *Matchers) UnmarshalYAML(value *yaml.Node) (err error)
• type Meta
  • func InjectThanos(logger log.Logger, bdir string, meta Thanos, downsampledMeta *tsdb.BlockMeta) (*Meta, error)
  • func Read(rc io.ReadCloser) (_ *Meta, err error)
  • func ReadFromDir(dir string) (*Meta, error)
  • func (m *Meta) String() string
  • func (m Meta) Write(w io.Writer) error
  • func (m Meta) WriteToDir(logger log.Logger, dir string) error
• type NoCompactMark
• type NoCompactReason
• type ObjectHash
  • func CalculateHash(p string, hf HashFunc, logger log.Logger) (ObjectHash, error)
  • func (oh *ObjectHash) Equal(other *ObjectHash) bool
• type Rewrite
• type SourceType
• type Thanos
  • func (m *Thanos) GroupKey() string
• type ThanosDownsample

Constants

const (
	// DeletionMarkFilename is the known json filename for optional file storing details about when block is marked for deletion.
	// If such file is present in block dir, it means the block is meant to be deleted after certain delay.
	DeletionMarkFilename = "deletion-mark.json"
	// NoCompactMarkFilename is the known json filename for optional file storing details about why block has to be excluded from compaction.
	// If such file is present in block dir, it means the block has to excluded from compaction (both vertical and horizontal) or rewrite (e.g deletions).
	NoCompactMarkFilename = "no-compact-mark.json"

	// DeletionMarkVersion1 is the version of deletion-mark file supported by Thanos.
	DeletionMarkVersion1 = 1
	// NoCompactMarkVersion1 is the version of no-compact-mark file supported by Thanos.
	NoCompactMarkVersion1 = 1
)

const (
	// ManualNoCompactReason is a custom reason of excluding from compaction that should be added when no-compact mark is added for unknown/user specified reason.
	ManualNoCompactReason NoCompactReason = "manual"
	// IndexSizeExceedingNoCompactReason is a reason of index being too big (for example exceeding 64GB limit: https://github.com/thanos-io/thanos/issues/1424)
	// This reason can be ignored when vertical block sharding will be implemented.
	IndexSizeExceedingNoCompactReason = "index-size-exceeding"
	// OutOfOrderChunksNoCompactReason is a reason of to no compact block with index contains out of order chunk so that the compaction is not blocked.
	OutOfOrderChunksNoCompactReason = "block-index-out-of-order-chunk"
)

const (
	// MetaFilename is the known JSON filename for meta information.
	MetaFilename = "meta.json"
	// TSDBVersion1 is a enumeration of TSDB meta versions supported by Thanos.
	TSDBVersion1 = 1
	// ThanosVersion1 is a enumeration of Thanos section of TSDB meta supported by Thanos.
	ThanosVersion1 = 1
)

Variables

var (
	// ErrorMarkerNotFound is the error when marker file is not found.
	ErrorMarkerNotFound = errors.New("marker not found")
	// ErrorUnmarshalMarker is the error when unmarshalling marker JSON file.
	// This error can occur because marker has been partially uploaded to block storage
	// or the marker file is not a valid json file.
	ErrorUnmarshalMarker = errors.New("unmarshal marker JSON")
)

Functions

func ReadMarker

func ReadMarker(ctx context.Context, logger log.Logger, bkt objstore.InstrumentedBucketReader, dir string, marker Marker) error

ReadMarker reads the given mark file from <dir>/<marker filename>.json in bucket.

Types

type DeletionMark

type DeletionMark struct {
	// ID of the tsdb block.
	ID ulid.ULID `json:"id"`
	// Version of the file.
	Version int `json:"version"`
	// Details is a human readable string giving details of reason.
	Details string `json:"details,omitempty"`

	// DeletionTime is a unix timestamp of when the block was marked to be deleted.
	DeletionTime int64 `json:"deletion_time"`
}

DeletionMark stores block id and when block was marked for deletion.

type DeletionRequest

type DeletionRequest struct {
	Matchers  Matchers             `json:"matchers" yaml:"matchers"`
	Intervals tombstones.Intervals `json:"intervals,omitempty" yaml:"intervals,omitempty"`
	RequestID string               `json:"request_id,omitempty" yaml:"request_id,omitempty"`
}

type File

type File struct {
	RelPath string `json:"rel_path"`
	// SizeBytes is optional (e.g meta.json does not show size).
	SizeBytes int64 `json:"size_bytes,omitempty"`

	// Hash is an optional hash of this file. Used for potentially avoiding an extra download.
	Hash *ObjectHash `json:"hash,omitempty"`
}

type HashFunc

type HashFunc string

HashFunc indicates what type of hash it is.

const (
	// SHA256Func shows that SHA256 has been used to generate the hash.
	SHA256Func HashFunc = "SHA256"
	// NoneFunc shows that hashes should not be added. Used internally.
	NoneFunc HashFunc = ""
)

type Marker

type Marker interface {
	// contains filtered or unexported methods
}

type Matchers

type Matchers []*labels.Matcher

func (*Matchers) UnmarshalYAML

func (m *Matchers) UnmarshalYAML(value *yaml.Node) (err error)

type Meta

type Meta struct {
	tsdb.BlockMeta

	Thanos Thanos `json:"thanos"`
}

Meta describes the a block's meta. It wraps the known TSDB meta structure and extends it by Thanos-specific fields.

func InjectThanos

func InjectThanos(logger log.Logger, bdir string, meta Thanos, downsampledMeta *tsdb.BlockMeta) (*Meta, error)

InjectThanos sets Thanos meta to the block meta JSON and saves it to the disk. NOTE: It should be used after writing any block by any Thanos component, otherwise we will miss crucial metadata.

func Read

func Read(rc io.ReadCloser) (_ *Meta, err error)

Read the block meta from the given reader.

func ReadFromDir

func ReadFromDir(dir string) (*Meta, error)

ReadFromDir reads the given meta from <dir>/meta.json.

func (*Meta) String

func (m *Meta) String() string

func (Meta) Write

func (m Meta) Write(w io.Writer) error

Write writes the given encoded meta to writer.

func (Meta) WriteToDir

func (m Meta) WriteToDir(logger log.Logger, dir string) error

WriteToDir writes the encoded meta into <dir>/meta.json.

type NoCompactMark

type NoCompactMark struct {
	// ID of the tsdb block.
	ID ulid.ULID `json:"id"`
	// Version of the file.
	Version int `json:"version"`
	// Details is a human readable string giving details of reason.
	Details string `json:"details,omitempty"`

	// NoCompactTime is a unix timestamp of when the block was marked for no compact.
	NoCompactTime int64           `json:"no_compact_time"`
	Reason        NoCompactReason `json:"reason"`
}

NoCompactMark marker stores reason of block being excluded from compaction if needed.

type NoCompactReason

type NoCompactReason string

NoCompactReason is a reason for a block to be excluded from compaction.

type ObjectHash

type ObjectHash struct {
	Func  HashFunc `json:"hashFunc"`
	Value string   `json:"value"`
}

ObjectHash stores the hash of an object in the object storage.

func CalculateHash

func CalculateHash(p string, hf HashFunc, logger log.Logger) (ObjectHash, error)

CalculateHash calculates the hash of the given type.

func (*ObjectHash) Equal

func (oh *ObjectHash) Equal(other *ObjectHash) bool

Equal returns true if two hashes are equal.

type Rewrite

type Rewrite struct {
	// ULIDs of all source head blocks that went into the block.
	Sources []ulid.ULID `json:"sources,omitempty"`
	// Deletions if applied (in order).
	DeletionsApplied []DeletionRequest `json:"deletions_applied,omitempty"`
	// Relabels if applied.
	RelabelsApplied []*relabel.Config `json:"relabels_applied,omitempty"`
}

type SourceType

type SourceType string

const (
	// TODO(bwplotka): Merge with pkg/component package.
	UnknownSource         SourceType = ""
	SidecarSource         SourceType = "sidecar"
	ReceiveSource         SourceType = "receive"
	CompactorSource       SourceType = "compactor"
	CompactorRepairSource SourceType = "compactor.repair"
	RulerSource           SourceType = "ruler"
	BucketRepairSource    SourceType = "bucket.repair"
	BucketRewriteSource   SourceType = "bucket.rewrite"
	TestSource            SourceType = "test"
)

type Thanos

type Thanos struct {
	// Version of Thanos meta file. If none specified, 1 is assumed (since first version did not have explicit version specified).
	Version int `json:"version,omitempty"`

	// Labels are the external labels identifying the producer as well as tenant.
	// See https://thanos.io/tip/thanos/storage.md#external-labels for details.
	Labels     map[string]string `json:"labels"`
	Downsample ThanosDownsample  `json:"downsample"`

	// Source is a real upload source of the block.
	Source SourceType `json:"source"`

	// List of segment files (in chunks directory), in sorted order. Optional.
	// Deprecated. Use Files instead.
	SegmentFiles []string `json:"segment_files,omitempty"`

	// File is a sorted (by rel path) list of all files in block directory of this block known to TSDB.
	// Sorted by relative path.
	// Useful to avoid API call to get size of each file, as well as for debugging purposes.
	// Optional, added in v0.17.0.
	Files []File `json:"files,omitempty"`

	// Rewrites is present when any rewrite (deletion, relabel etc) were applied to this block. Optional.
	Rewrites []Rewrite `json:"rewrites,omitempty"`
}

Thanos holds block meta information specific to Thanos.

func (*Thanos) GroupKey

func (m *Thanos) GroupKey() string

Returns a unique identifier for the compaction group the block belongs to. It considers the downsampling resolution and the block's labels.

type ThanosDownsample

type ThanosDownsample struct {
	Resolution int64 `json:"resolution"`
}