compaction

Documentation

Index

• Variables
• type ByMinSize
  • func (a ByMinSize) Len() int
  • func (a ByMinSize) Less(i, j int) bool
  • func (a ByMinSize) Swap(i, j int)
• type CompactResult
• type Compactor
  • func NewCompactor(metadataPool doc.MetadataArrayPool, docsMaxBatch int, ...) (*Compactor, error)
  • func (c *Compactor) Close() error
  • func (c *Compactor) Compact(segs []segment.Segment, filter segment.DocumentsFilter, ...) (CompactResult, error)
  • func (c *Compactor) CompactUsingBuilder(builder segment.DocumentsBuilder, segs []segment.Segment, ...) (fst.Segment, error)
• type CompactorOptions
• type Level
• type Plan
  • func NewPlan(compactableSegments []Segment, opts PlannerOptions) (*Plan, error)
  • func (p *Plan) Len() int
  • func (p *Plan) Less(i, j int) bool
  • func (p *Plan) Swap(i, j int)
• type PlannerOptions
  • func (o PlannerOptions) Validate() error
• type Segment
• type Task
  • func (t Task) Summary() TaskSummary
• type TaskSummary
• type TasksOrderBy

Variables

var (
	// DefaultLevels are the default Level(s) used for compaction options.
	DefaultLevels = []Level{
		{
			MinSizeInclusive: 0,
			MaxSizeExclusive: 1 << 18,
		},
	}

	// DefaultOptions are the default compaction PlannerOptions.
	DefaultOptions = PlannerOptions{
		MutableSegmentSizeThreshold:   0,
		MutableCompactionAgeThreshold: 0,
		Levels:                        DefaultLevels,
		OrderBy:                       TasksOrderedByOldestMutableAndSize,
	}
)

var (
	// ErrCompactorBuilderEmpty is returned when the compaction
	// would result in an empty segment.
	ErrCompactorBuilderEmpty = errors.New("builder has no documents")
)

Types

type ByMinSize

type ByMinSize []Level

ByMinSize orders a []Level by MinSize in ascending order.

func (ByMinSize) Len

func (a ByMinSize) Len() int

func (ByMinSize) Less

func (a ByMinSize) Less(i, j int) bool

func (ByMinSize) Swap

func (a ByMinSize) Swap(i, j int)

type CompactResult

type CompactResult struct {
	Compacted        fst.Segment
	SegmentMetadatas []segment.SegmentsBuilderSegmentMetadata
}

CompactResult is the result of a call to compact.

type Compactor

type Compactor struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Compactor is a compactor.

func NewCompactor

func NewCompactor(
	metadataPool doc.MetadataArrayPool,
	docsMaxBatch int,
	builderOpts builder.Options,
	fstOpts fst.Options,
	opts CompactorOptions,
) (*Compactor, error)

NewCompactor returns a new compactor which reuses buffers to avoid allocating intermediate buffers when compacting.

func (*Compactor) Close

func (c *Compactor) Close() error

Close closes the compactor and frees buffered resources.

func (*Compactor) Compact

func (c *Compactor) Compact(
	segs []segment.Segment,
	filter segment.DocumentsFilter,
	filterCounter tally.Counter,
	reporterOptions mmap.ReporterOptions,
) (CompactResult, error)

Compact will take a set of segments and compact them into an immutable FST segment, if there is a single mutable segment it can directly be converted into an FST segment, otherwise an intermediary mutable segment (reused by the compactor between runs) is used to combine all the segments together first before compacting into an FST segment. Note: This is not thread safe and only a single compaction may happen at a time.

func (*Compactor) CompactUsingBuilder

func (c *Compactor) CompactUsingBuilder(
	builder segment.DocumentsBuilder,
	segs []segment.Segment,
	reporterOptions mmap.ReporterOptions,
) (fst.Segment, error)

CompactUsingBuilder compacts segments together using a provided segment builder.

type CompactorOptions

type CompactorOptions struct {
	// FSTWriterOptions if not nil are the options used to
	// construct the FST writer.
	FSTWriterOptions *fst.WriterOptions

	// MmapDocsData when enabled will encode and mmmap the
	// documents data, rather than keeping the original
	// documents with references to substrings in the metric
	// IDs (done for memory savings).
	MmapDocsData bool
}

CompactorOptions is a set of compactor options.

type Level

type Level struct {
	MinSizeInclusive int64
	MaxSizeExclusive int64
}

Level defines a range of (min, max) sizes such that any segments within the Level are allowed to be compacted together.

type Plan

type Plan struct {
	Tasks          []Task
	UnusedSegments []Segment
	OrderBy        TasksOrderBy
}

Plan is a logical collection of compaction Tasks. The tasks do not depened on each other, and maybe performed sequentially or in parallel.

func NewPlan

func NewPlan(compactableSegments []Segment, opts PlannerOptions) (*Plan, error)

NewPlan returns a new compaction.Plan per the rules above and the knobs provided.

func (*Plan) Len

func (p *Plan) Len() int

func (*Plan) Less

func (p *Plan) Less(i, j int) bool

func (*Plan) Swap

func (p *Plan) Swap(i, j int)

type PlannerOptions

type PlannerOptions struct {
	// MutableSegmentSizeThreshold is the maximum size a mutable segment is
	// allowed to grow before it's rotated out for compactions.
	MutableSegmentSizeThreshold int64
	// MutableCompactionAgeThreshold is minimum age required of a mutable segment
	// before it would be considered for compaction in steady state.
	MutableCompactionAgeThreshold time.Duration
	// Levels define the levels for compactions.
	Levels []Level
	// OrderBy defines the order of tasks in the compaction plan returned.
	OrderBy TasksOrderBy
}

PlannerOptions are the knobs to tweak planning behaviour.

func (PlannerOptions) Validate

func (o PlannerOptions) Validate() error

Validate ensures the receiver PlannerOptions specify valid values for each of the knobs.

type Segment

type Segment struct {
	Age  time.Duration
	Size int64
	Type segments.Type

	// Either builder or segment should be set, not both.
	Builder segment.Builder
	Segment segment.Segment
}

Segment identifies a candidate for compaction.

type Task

type Task struct {
	Segments []Segment
}

Task identifies a collection of segments to compact.

func (Task) Summary

func (t Task) Summary() TaskSummary

Summary returns the TaskSummary for the given task.

type TaskSummary

type TaskSummary struct {
	NumMutable           int
	NumFST               int
	CumulativeMutableAge time.Duration
	CumulativeSize       int64
}

TaskSummary is a collection of statistics about a compaction task.

type TasksOrderBy

type TasksOrderBy byte

TasksOrderBy controls the order of tasks returned in the plan.

const (
	// TasksOrderedByOldestMutableAndSize orders tasks with oldest mutable segment age (cumulative), and then by size.
	TasksOrderedByOldestMutableAndSize TasksOrderBy = iota
)