Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

compactor

package

Go to main page

Versions in this module

v1

v1.10.0

Nov 9, 2024

v1.9.1

Oct 17, 2024

v1.9.0

Oct 3, 2024

v1.8.0

Sep 12, 2024

v1.7.1

Jul 31, 2024

v1.7.0

Jul 31, 2024

v1.6.1

Jun 21, 2024

v1.6.0

May 30, 2024

v1.4.0

Feb 6, 2024

v1.3.0

Jan 18, 2024

Changes in this version

+ type BlocksPlannerFactory func(cfg Config) Planner

type Config

+ BlocksPlannerFactory BlocksPlannerFactory

+ DownsamplerEnabled bool

type ConfigProvider

+ CompactorDownsamplerEnabled func(userId string) bool

v1.2.1

Dec 15, 2023

v1.2.0

Nov 16, 2023

Changes in this version

+ const CompactionOrderNewestFirst

+ const CompactionOrderOldestFirst

+ const CompactionSplitByFingerprint

+ const CompactionSplitByStacktracePartition

+ var CompactionOrders = []string

+ var CompactionSplitBys = []string

+ var RingOp = ring.NewOp([]ring.InstanceState{ ... }, nil)

+ func DefaultGroupKey(meta block.Meta) string

+ type BlockCompactor struct

+ func (c *BlockCompactor) CompactWithSplitting(ctx context.Context, dest string, dirs []string, shardCount, stageSize uint64) ([]ulid.ULID, error)

+ type BlocksCleaner struct

+ func NewBlocksCleaner(cfg BlocksCleanerConfig, bucketClient objstore.Bucket, ...) *BlocksCleaner

+ type BlocksCleanerConfig struct

+ CleanupConcurrency int

+ CleanupInterval time.Duration

+ DeleteBlocksConcurrency int

+ DeletionDelay time.Duration

+ NoBlocksFileCleanupEnabled bool

+ TenantCleanupDelay time.Duration

+ type BlocksCompactorFactory func(ctx context.Context, cfg Config, logger log.Logger, reg prometheus.Registerer) (Compactor, Planner, error)

+ type BlocksGrouperFactory func(ctx context.Context, cfg Config, cfgProvider ConfigProvider, userID string, ...) Grouper

+ type BucketCompactor struct

+ func NewBucketCompactor(logger log.Logger, sy *Syncer, grouper Grouper, planner Planner, ...) (*BucketCompactor, error)

+ func (c *BucketCompactor) Compact(ctx context.Context, maxCompactionTime time.Duration) (rerr error)

+ type BucketCompactorMetrics struct

+ func NewBucketCompactorMetrics(blocksMarkedForDeletion prometheus.Counter, reg prometheus.Registerer) *BucketCompactorMetrics

+ type Compactor interface

+ CompactWithSplitting func(ctx context.Context, dst string, dirs []string, shardCount, stageSize uint64) (result []ulid.ULID, _ error)

+ type CompactorMetrics struct

+ Duration *prometheus.HistogramVec

+ InProgress *prometheus.GaugeVec

+ OverlappingBlocks prometheus.Counter

+ Ran *prometheus.CounterVec

+ Range *prometheus.HistogramVec

+ Samples *prometheus.HistogramVec

+ Size *prometheus.HistogramVec

+ Split *prometheus.HistogramVec

+ type Config struct

+ BlockRanges DurationList

+ BlockSyncConcurrency int

+ BlocksCompactorFactory BlocksCompactorFactory

+ BlocksGrouperFactory BlocksGrouperFactory

+ CleanupConcurrency int

+ CleanupInterval time.Duration

+ CompactionConcurrency int

+ CompactionInterval time.Duration

+ CompactionJobsOrder string

+ CompactionRetries int

+ CompactionSplitBy string

+ CompactionWaitPeriod time.Duration

+ DataDir string

+ DeletionDelay time.Duration

+ DisabledTenants flagext.StringSliceCSV

+ EnabledTenants flagext.StringSliceCSV

+ MaxCompactionTime time.Duration

+ MaxOpeningBlocksConcurrency int

+ MetaSyncConcurrency int

+ NoBlocksFileCleanupEnabled bool

+ ShardingRing RingConfig

+ TenantCleanupDelay time.Duration

+ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger)

+ func (cfg *Config) Validate(maxBlockDuration time.Duration) error

+ type ConfigProvider interface

+ CompactorBlocksRetentionPeriod func(user string) time.Duration

+ CompactorPartialBlockDeletionDelay func(userID string) (delay time.Duration, valid bool)

+ CompactorSplitAndMergeShards func(userID string) int

+ CompactorSplitAndMergeStageSize func(userID string) int

+ CompactorSplitGroups func(userID string) int

+ CompactorTenantShardSize func(userID string) int

+ type DeduplicateFilter interface

+ DuplicateIDs func() []ulid.ULID

+ type DurationList []time.Duration

+ func (d *DurationList) Set(s string) error

+ func (d *DurationList) String() string

+ func (d *DurationList) ToMilliseconds() []int64

+ type Grouper interface

+ Groups func(blocks map[ulid.ULID]*block.Meta) (res []*Job, err error)

+ type Job struct

+ func NewJob(userID string, key string, lset labels.Labels, resolution int64, ...) *Job

+ func (job *Job) AppendMeta(meta *block.Meta) error

+ func (job *Job) IDs() (ids []ulid.ULID)

+ func (job *Job) Key() string

+ func (job *Job) Labels() labels.Labels

+ func (job *Job) MaxTime() int64

+ func (job *Job) Metas() []*block.Meta

+ func (job *Job) MinCompactionLevel() int

+ func (job *Job) MinTime() int64

+ func (job *Job) Resolution() int64

+ func (job *Job) ShardingKey() string

+ func (job *Job) SplitStageSize() uint32

+ func (job *Job) SplittingShards() uint32

+ func (job *Job) String() string

+ func (job *Job) UseSplitting() bool

+ func (job *Job) UserID() string

+ type JobsOrderFunc func(jobs []*Job) []*Job

+ func GetJobsOrderFunction(name string) JobsOrderFunc

+ type LabelRemoverFilter struct

+ func NewLabelRemoverFilter(labels []string) *LabelRemoverFilter

+ func (f *LabelRemoverFilter) Filter(_ context.Context, metas map[ulid.ULID]*block.Meta, _ block.GaugeVec) error

+ type MultitenantCompactor struct

+ func NewMultitenantCompactor(compactorCfg Config, bucketClient objstore.Bucket, cfgProvider ConfigProvider, ...) (*MultitenantCompactor, error)

+ func (c *MultitenantCompactor) RingHandler(w http.ResponseWriter, req *http.Request)

+ type NoCompactionMarkFilter struct

+ func NewNoCompactionMarkFilter(bkt objstore.BucketReader, removeNoCompactBlocks bool) *NoCompactionMarkFilter

+ func (f *NoCompactionMarkFilter) Filter(ctx context.Context, metas map[ulid.ULID]*block.Meta, synced block.GaugeVec) error

+ func (f *NoCompactionMarkFilter) NoCompactMarkedBlocks() map[ulid.ULID]struct{}

+ type Planner interface

+ Plan func(ctx context.Context, metasByMinTime []*block.Meta) ([]*block.Meta, error)

+ type RingConfig struct

+ Common util.CommonRingConfig

+ ObservePeriod time.Duration

+ WaitActiveInstanceTimeout time.Duration

+ WaitStabilityMaxDuration time.Duration

+ WaitStabilityMinDuration time.Duration

+ func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger)

+ func (cfg *RingConfig) ToBasicLifecyclerConfig(logger log.Logger) (ring.BasicLifecyclerConfig, error)

+ type ShardAwareDeduplicateFilter struct

+ func NewShardAwareDeduplicateFilter() *ShardAwareDeduplicateFilter

+ func (f *ShardAwareDeduplicateFilter) DuplicateIDs() []ulid.ULID

+ func (f *ShardAwareDeduplicateFilter) Filter(ctx context.Context, metas map[ulid.ULID]*block.Meta, synced block.GaugeVec) error

+ type SplitAndMergeGrouper struct

+ func NewSplitAndMergeGrouper(userID string, ranges []int64, shardCount uint32, splitStageSize uint32, ...) *SplitAndMergeGrouper

+ func (g *SplitAndMergeGrouper) Groups(blocks map[ulid.ULID]*block.Meta) (res []*Job, err error)

+ type SplitAndMergePlanner struct

+ func NewSplitAndMergePlanner(ranges []int64) *SplitAndMergePlanner

+ func (c *SplitAndMergePlanner) Plan(_ context.Context, metasByMinTime []*block.Meta) ([]*block.Meta, error)

+ type Syncer struct

+ func NewMetaSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, ...) (*Syncer, error)

+ func (s *Syncer) GarbageCollect(ctx context.Context) error

+ func (s *Syncer) Metas() map[ulid.ULID]*block.Meta

+ func (s *Syncer) SyncMetas(ctx context.Context) error