metadata

package
v0.32.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 2, 2023 License: Apache-2.0 Imports: 21 Imported by: 42

Documentation

Index

Constants

View Source 
const (
	// DeletionMarkFilename is the known json filename for optional file storing details about when block is marked for deletion.
	// If such file is present in block dir, it means the block is meant to be deleted after certain delay.
	DeletionMarkFilename = "deletion-mark.json"
	// NoCompactMarkFilename is the known json filename for optional file storing details about why block has to be excluded from compaction.
	// If such file is present in block dir, it means the block has to excluded from compaction (both vertical and horizontal) or rewrite (e.g deletions).
	NoCompactMarkFilename = "no-compact-mark.json"
	// NoDownsampleMarkFilename is the known json filenanme for optional file storing details about why block has to be excluded from downsampling.
	// If such file is present in block dir, it means the block has to be excluded from downsampling.
	NoDownsampleMarkFilename = "no-downsample-mark.json"
	// DeletionMarkVersion1 is the version of deletion-mark file supported by Thanos.
	DeletionMarkVersion1 = 1
	// NoCompactMarkVersion1 is the version of no-compact-mark file supported by Thanos.
	NoCompactMarkVersion1 = 1
	// NoDownsampleVersion1 is the version of no-downsample-mark file supported by Thanos.
	NoDownsampleMarkVersion1 = 1
)
View Source 
const (
	// ManualNoCompactReason is a custom reason of excluding from compaction that should be added when no-compact mark is added for unknown/user specified reason.
	ManualNoCompactReason NoCompactReason = "manual"
	// ManualNoDownsampleReason is a custom reason of excluding from downsample that should be added when no-downsample mark is added for unknown/user specified reason.
	ManualNoDownsampleReason NoDownsampleReason = "manual"
	// IndexSizeExceedingNoCompactReason is a reason of index being too big (for example exceeding 64GB limit: https://github.com/thanos-io/thanos/issues/1424)
	// This reason can be ignored when vertical block sharding will be implemented.
	IndexSizeExceedingNoCompactReason = "index-size-exceeding"
	// OutOfOrderChunksNoCompactReason is a reason of to no compact block with index contains out of order chunk so that the compaction is not blocked.
	OutOfOrderChunksNoCompactReason = "block-index-out-of-order-chunk"
)
View Source 
const (
	// MetaFilename is the known JSON filename for meta information.
	MetaFilename = "meta.json"
	// TSDBVersion1 is a enumeration of TSDB meta versions supported by Thanos.
	TSDBVersion1 = 1
	// ThanosVersion1 is a enumeration of Thanos section of TSDB meta supported by Thanos.
	ThanosVersion1 = 1
)

Variables

View Source 
var (
	// ErrorMarkerNotFound is the error when marker file is not found.
	ErrorMarkerNotFound = errors.New("marker not found")
	// ErrorUnmarshalMarker is the error when unmarshalling marker JSON file.
	// This error can occur because marker has been partially uploaded to block storage
	// or the marker file is not a valid json file.
	ErrorUnmarshalMarker = errors.New("unmarshal marker JSON")
)

Functions

func ConvertExtensions added in v0.32.0 

func ConvertExtensions(extensions any, v any) (any, error)

ConvertExtensions converts extensions with `any` type into specific type `v` that the caller expects.

func ReadMarker added in v0.17.0 

func ReadMarker(ctx context.Context, logger log.Logger, bkt objstore.InstrumentedBucketReader, dir string, marker Marker) error

ReadMarker reads the given mark file from <dir>/<marker filename>.json in bucket.

Types

type DeletionMark added in v0.12.0 

type DeletionMark struct {
	// ID of the tsdb block.
	ID ulid.ULID `json:"id"`
	// Version of the file.
	Version int `json:"version"`
	// Details is a human readable string giving details of reason.
	Details string `json:"details,omitempty"`

	// DeletionTime is a unix timestamp of when the block was marked to be deleted.
	DeletionTime int64 `json:"deletion_time"`
}

DeletionMark stores block id and when block was marked for deletion.

type DeletionRequest added in v0.18.0 

type DeletionRequest struct {
	Matchers  Matchers             `json:"matchers" yaml:"matchers"`
	Intervals tombstones.Intervals `json:"intervals,omitempty" yaml:"intervals,omitempty"`
	RequestID string               `json:"request_id,omitempty" yaml:"request_id,omitempty"`
}

type File added in v0.17.0 

type File struct {
	RelPath string `json:"rel_path"`
	// SizeBytes is optional (e.g meta.json does not show size).
	SizeBytes int64 `json:"size_bytes,omitempty"`

	// Hash is an optional hash of this file. Used for potentially avoiding an extra download.
	Hash *ObjectHash `json:"hash,omitempty"`
}

type HashFunc added in v0.19.0 

type HashFunc string

HashFunc indicates what type of hash it is. 

const (
	// SHA256Func shows that SHA256 has been used to generate the hash.
	SHA256Func HashFunc = "SHA256"
	// NoneFunc shows that hashes should not be added. Used internally.
	NoneFunc HashFunc = ""
)

type IndexStats added in v0.32.0 

type IndexStats struct {
	SeriesMaxSize int64 `json:"series_max_size,omitempty"`
	ChunkMaxSize  int64 `json:"chunk_max_size,omitempty"`
}

type Marker added in v0.17.0 

type Marker interface {
	// contains filtered or unexported methods
}

type Matchers added in v0.18.0 

type Matchers []*labels.Matcher

func (*Matchers) UnmarshalYAML added in v0.18.0 

func (m *Matchers) UnmarshalYAML(value *yaml.Node) (err error)

type Meta 

type Meta struct {
	tsdb.BlockMeta

	Thanos Thanos `json:"thanos"`
}

Meta describes the a block's meta. It wraps the known TSDB meta structure and extends it by Thanos-specific fields.

func InjectThanos 

func InjectThanos(logger log.Logger, bdir string, meta Thanos, downsampledMeta *tsdb.BlockMeta) (*Meta, error)

InjectThanos sets Thanos meta to the block meta JSON and saves it to the disk. NOTE: It should be used after writing any block by any Thanos component, otherwise we will miss crucial metadata.

func Read 

func Read(rc io.ReadCloser) (_ *Meta, err error)

Read the block meta from the given reader.

func ReadFromDir added in v0.18.0 

func ReadFromDir(dir string) (*Meta, error)

ReadFromDir reads the given meta from <dir>/meta.json.

func (*Meta) String added in v0.18.0 

func (m *Meta) String() string

func (Meta) Write added in v0.17.0 

func (m Meta) Write(w io.Writer) error

Write writes the given encoded meta to writer.

func (Meta) WriteToDir added in v0.17.0 

func (m Meta) WriteToDir(logger log.Logger, dir string) error

WriteToDir writes the encoded meta into <dir>/meta.json.

type NoCompactMark added in v0.17.0 

type NoCompactMark struct {
	// ID of the tsdb block.
	ID ulid.ULID `json:"id"`
	// Version of the file.
	Version int `json:"version"`
	// Details is a human readable string giving details of reason.
	Details string `json:"details,omitempty"`

	// NoCompactTime is a unix timestamp of when the block was marked for no compact.
	NoCompactTime int64           `json:"no_compact_time"`
	Reason        NoCompactReason `json:"reason"`
}

NoCompactMark marker stores reason of block being excluded from compaction if needed.

type NoCompactReason added in v0.17.0 

type NoCompactReason string

NoCompactReason is a reason for a block to be excluded from compaction.

type NoDownsampleMark added in v0.30.0 

type NoDownsampleMark struct {
	// ID of the tsdb block.
	ID ulid.ULID `json:"id"`
	// Version of the file.
	Version int `json:"version"`
	// Details is a human readable string giving details of reason.
	Details string `json:"details,omitempty"`

	// NoDownsampleTime is a unix timestamp of when the block was marked for no downsample.
	NoDownsampleTime int64              `json:"no_downsample_time"`
	Reason           NoDownsampleReason `json:"reason"`
}

NoDownsampleMark marker stores reason of block being excluded from downsample if needed.

type NoDownsampleReason added in v0.30.0 

type NoDownsampleReason string

NoDownsampleReason is a reason for a block to be excluded from downsample.

type ObjectHash added in v0.19.0 

type ObjectHash struct {
	Func  HashFunc `json:"hashFunc"`
	Value string   `json:"value"`
}

ObjectHash stores the hash of an object in the object storage.

func CalculateHash added in v0.19.0 

func CalculateHash(p string, hf HashFunc, logger log.Logger) (ObjectHash, error)

CalculateHash calculates the hash of the given type.

func (*ObjectHash) Equal added in v0.19.0 

func (oh *ObjectHash) Equal(other *ObjectHash) bool

Equal returns true if two hashes are equal.

type Rewrite added in v0.18.0 

type Rewrite struct {
	// ULIDs of all source head blocks that went into the block.
	Sources []ulid.ULID `json:"sources,omitempty"`
	// Deletions if applied (in order).
	DeletionsApplied []DeletionRequest `json:"deletions_applied,omitempty"`
	// Relabels if applied.
	RelabelsApplied []*relabel.Config `json:"relabels_applied,omitempty"`
}

type SourceType 

type SourceType string
const (
	// TODO(bwplotka): Merge with pkg/component package.
	UnknownSource         SourceType = ""
	SidecarSource         SourceType = "sidecar"
	ReceiveSource         SourceType = "receive"
	CompactorSource       SourceType = "compactor"
	CompactorRepairSource SourceType = "compactor.repair"
	RulerSource           SourceType = "ruler"
	BucketRepairSource    SourceType = "bucket.repair"
	BucketRewriteSource   SourceType = "bucket.rewrite"
	TestSource            SourceType = "test"
)

type Thanos 

type Thanos struct {
	// Version of Thanos meta file. If none specified, 1 is assumed (since first version did not have explicit version specified).
	Version int `json:"version,omitempty"`

	// Labels are the external labels identifying the producer as well as tenant.
	// See https://thanos.io/tip/thanos/storage.md#external-labels for details.
	Labels     map[string]string `json:"labels"`
	Downsample ThanosDownsample  `json:"downsample"`

	// Source is a real upload source of the block.
	Source SourceType `json:"source"`

	// List of segment files (in chunks directory), in sorted order. Optional.
	// Deprecated. Use Files instead.
	SegmentFiles []string `json:"segment_files,omitempty"`

	// File is a sorted (by rel path) list of all files in block directory of this block known to TSDB.
	// Sorted by relative path.
	// Useful to avoid API call to get size of each file, as well as for debugging purposes.
	// Optional, added in v0.17.0.
	Files []File `json:"files,omitempty"`

	// Rewrites is present when any rewrite (deletion, relabel etc) were applied to this block. Optional.
	Rewrites []Rewrite `json:"rewrites,omitempty"`

	// IndexStats contains stats info related to block index.
	IndexStats IndexStats `json:"index_stats,omitempty"`

	// Extensions are used for plugin any arbitrary additional information for block. Optional.
	Extensions any `json:"extensions,omitempty"`
}

Thanos holds block meta information specific to Thanos.

func (*Thanos) GroupKey added in v0.25.0 

func (m *Thanos) GroupKey() string

GroupKey returns a unique identifier for the compaction group the block belongs to. It considers the downsampling resolution and the block's labels.

func (*Thanos) ParseExtensions added in v0.32.0 

func (m *Thanos) ParseExtensions(v any) (any, error)

func (*Thanos) ResolutionString added in v0.32.0 

func (m *Thanos) ResolutionString() string

ResolutionString returns a the block's resolution as a string.

type ThanosDownsample 

type ThanosDownsample struct {
	Resolution int64 `json:"resolution"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.