compact

Documentation

Index

• Constants
• func ApplyRetentionPolicyByResolution(ctx context.Context, logger log.Logger, bkt objstore.Bucket, ...) error
• func BestEffortCleanAbortedPartialUploads(ctx context.Context, logger log.Logger, partial map[ulid.ULID]error, ...)
• func GroupKey(meta metadata.Thanos) string
• func IsHaltError(err error) bool
• func IsIssue347Error(err error) bool
• func IsRetryError(err error) bool
• func RepairIssue347(ctx context.Context, logger log.Logger, bkt objstore.Bucket, ...) error
• func UntilNextDownsampling(m *metadata.Meta) (time.Duration, error)
• type BlocksCleaner
  • func NewBlocksCleaner(logger log.Logger, bkt objstore.Bucket, ...) *BlocksCleaner
  • func (s *BlocksCleaner) DeleteMarkedBlocks(ctx context.Context) error
• type BucketCompactor
  • func NewBucketCompactor(logger log.Logger, sy *Syncer, comp tsdb.Compactor, compactDir string, ...) (*BucketCompactor, error)
  • func (c *BucketCompactor) Compact(ctx context.Context) error
• type Group
  • func (cg *Group) Add(meta *metadata.Meta) error
  • func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) (bool, ulid.ULID, error)
  • func (cg *Group) IDs() (ids []ulid.ULID)
  • func (cg *Group) Key() string
  • func (cg *Group) Labels() labels.Labels
  • func (cg *Group) Resolution() int64
• type HaltError
  • func (e HaltError) Error() string
• type Issue347Error
  • func (e Issue347Error) Error() string
• type ResolutionLevel
• type RetryError
  • func (e RetryError) Error() string
• type Syncer
  • func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, ...) (*Syncer, error)
  • func (s *Syncer) GarbageCollect(ctx context.Context) error
  • func (s *Syncer) Groups() (res []*Group, err error)
  • func (s *Syncer) Metas() map[ulid.ULID]*metadata.Meta
  • func (s *Syncer) Partial() map[ulid.ULID]error
  • func (s *Syncer) SyncMetas(ctx context.Context) error

Constants

const (
	ResolutionLevelRaw = ResolutionLevel(downsample.ResLevel0)
	ResolutionLevel5m  = ResolutionLevel(downsample.ResLevel1)
	ResolutionLevel1h  = ResolutionLevel(downsample.ResLevel2)
)

const (
	// PartialUploadThresholdAge is a time after partial block is assumed aborted and ready to be cleaned.
	// Keep it long as it is based on block creation time not upload start time.
	PartialUploadThresholdAge = 2 * 24 * time.Hour
)

Functions

func ApplyRetentionPolicyByResolution

func ApplyRetentionPolicyByResolution(
	ctx context.Context,
	logger log.Logger,
	bkt objstore.Bucket,
	metas map[ulid.ULID]*metadata.Meta,
	retentionByResolution map[ResolutionLevel]time.Duration,
	blocksMarkedForDeletion prometheus.Counter,
) error

ApplyRetentionPolicyByResolution removes blocks depending on the specified retentionByResolution based on blocks MaxTime. A value of 0 disables the retention for its resolution.

func BestEffortCleanAbortedPartialUploads

func BestEffortCleanAbortedPartialUploads(
	ctx context.Context,
	logger log.Logger,
	partial map[ulid.ULID]error,
	bkt objstore.Bucket,
	deleteAttempts prometheus.Counter,
	blockCleanups prometheus.Counter,
	blockCleanupFailures prometheus.Counter,
)

func GroupKey

func GroupKey(meta metadata.Thanos) string

GroupKey returns a unique identifier for the group the block belongs to. It considers the downsampling resolution and the block's labels.

func IsHaltError

func IsHaltError(err error) bool

IsHaltError returns true if the base error is a HaltError. If a multierror is passed, any halt error will return true.

func IsIssue347Error

func IsIssue347Error(err error) bool

IsIssue347Error returns true if the base error is a Issue347Error.

func IsRetryError

func IsRetryError(err error) bool

IsRetryError returns true if the base error is a RetryError. If a multierror is passed, all errors must be retriable.

func RepairIssue347

func RepairIssue347(ctx context.Context, logger log.Logger, bkt objstore.Bucket, blocksMarkedForDeletion prometheus.Counter, issue347Err error) error

RepairIssue347 repairs the https://github.com/prometheus/tsdb/issues/347 issue when having issue347Error.

func UntilNextDownsampling

func UntilNextDownsampling(m *metadata.Meta) (time.Duration, error)

UntilNextDownsampling calculates how long it will take until the next downsampling operation. Returns an error if there will be no downsampling.

Types

type BlocksCleaner

type BlocksCleaner struct {
	// contains filtered or unexported fields
}

BlocksCleaner is a struct that deletes blocks from bucket which are marked for deletion.

func NewBlocksCleaner

func NewBlocksCleaner(logger log.Logger, bkt objstore.Bucket, ignoreDeletionMarkFilter *block.IgnoreDeletionMarkFilter, deleteDelay time.Duration, blocksCleaned prometheus.Counter, blockCleanupFailures prometheus.Counter) *BlocksCleaner

NewBlocksCleaner creates a new BlocksCleaner.

func (*BlocksCleaner) DeleteMarkedBlocks

func (s *BlocksCleaner) DeleteMarkedBlocks(ctx context.Context) error

DeleteMarkedBlocks uses ignoreDeletionMarkFilter to gather the blocks that are marked for deletion and deletes those if older than given deleteDelay.

type BucketCompactor

type BucketCompactor struct {
	// contains filtered or unexported fields
}

BucketCompactor compacts blocks in a bucket.

func NewBucketCompactor

func NewBucketCompactor(
	logger log.Logger,
	sy *Syncer,
	comp tsdb.Compactor,
	compactDir string,
	bkt objstore.Bucket,
	concurrency int,
) (*BucketCompactor, error)

NewBucketCompactor creates a new bucket compactor.

func (*BucketCompactor) Compact

func (c *BucketCompactor) Compact(ctx context.Context) error

Compact runs compaction over bucket.

type Group

type Group struct {
	// contains filtered or unexported fields
}

Group captures a set of blocks that have the same origin labels and downsampling resolution. Those blocks generally contain the same series and can thus efficiently be compacted.

func (*Group) Add

func (cg *Group) Add(meta *metadata.Meta) error

Add the block with the given meta to the group.

func (*Group) Compact

func (cg *Group) Compact(ctx context.Context, dir string, comp tsdb.Compactor) (bool, ulid.ULID, error)

Compact plans and runs a single compaction against the group. The compacted result is uploaded into the bucket the blocks were retrieved from.

func (*Group) IDs

func (cg *Group) IDs() (ids []ulid.ULID)

IDs returns all sorted IDs of blocks in the group.

func (*Group) Key

func (cg *Group) Key() string

Key returns an identifier for the group.

func (*Group) Labels

func (cg *Group) Labels() labels.Labels

Labels returns the labels that all blocks in the group share.

func (*Group) Resolution

func (cg *Group) Resolution() int64

Resolution returns the common downsampling resolution of blocks in the group.

type HaltError

type HaltError struct {
	// contains filtered or unexported fields
}

HaltError is a type wrapper for errors that should halt any further progress on compactions.

func (HaltError) Error

func (e HaltError) Error() string

type Issue347Error

type Issue347Error struct {
	// contains filtered or unexported fields
}

Issue347Error is a type wrapper for errors that should invoke repair process for broken block.

func (Issue347Error) Error

func (e Issue347Error) Error() string

type ResolutionLevel

type ResolutionLevel int64

type RetryError

type RetryError struct {
	// contains filtered or unexported fields
}

RetryError is a type wrapper for errors that should trigger warning log and retry whole compaction loop, but aborting current compaction further progress.

func (RetryError) Error

func (e RetryError) Error() string

type Syncer

type Syncer struct {
	// contains filtered or unexported fields
}

Syncer synchronizes block metas from a bucket into a local directory. It sorts them into compaction groups based on equal label sets.

func NewSyncer

func NewSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, fetcher block.MetadataFetcher, duplicateBlocksFilter *block.DeduplicateFilter, ignoreDeletionMarkFilter *block.IgnoreDeletionMarkFilter, blocksMarkedForDeletion prometheus.Counter, blockSyncConcurrency int, acceptMalformedIndex bool, enableVerticalCompaction bool) (*Syncer, error)

NewMetaSyncer returns a new Syncer for the given Bucket and directory. Blocks must be at least as old as the sync delay for being considered.

func (*Syncer) GarbageCollect

func (s *Syncer) GarbageCollect(ctx context.Context) error

GarbageCollect marks blocks for deletion from bucket if their data is available as part of a block with a higher compaction level. Call to SyncMetas function is required to populate duplicateIDs in duplicateBlocksFilter.

func (*Syncer) Groups

func (s *Syncer) Groups() (res []*Group, err error)

Groups returns the compaction groups for all blocks currently known to the syncer. It creates all groups from the scratch on every call.

func (*Syncer) Metas

func (s *Syncer) Metas() map[ulid.ULID]*metadata.Meta

Metas returns loaded metadata blocks since last sync.

func (*Syncer) Partial

func (s *Syncer) Partial() map[ulid.ULID]error

Partial returns partial blocks since last sync.

func (*Syncer) SyncMetas

func (s *Syncer) SyncMetas(ctx context.Context) error

SyncMetas synchronises local state of block metas with what we have in the bucket.