compaction

package
v0.10.3-0...-304cdc7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 11, 2024 License: Apache-2.0 Imports: 45 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetBM25FieldIDs 

func GetBM25FieldIDs(coll *schemapb.CollectionSchema) []int64

func LoadBM25Stats 

func LoadBM25Stats(ctx context.Context, chunkManager storage.ChunkManager, schema *schemapb.CollectionSchema, segmentID int64, statsBinlogs []*datapb.FieldBinlog) (map[int64]*storage.BM25Stats, error)

func LoadStats 

func LoadStats(ctx context.Context, chunkManager storage.ChunkManager, schema *schemapb.CollectionSchema, segmentID int64, statsBinlogs []*datapb.FieldBinlog) ([]*storage.PkStatistics, error)

func NewClusteringCompactionTask 

func NewClusteringCompactionTask(
	ctx context.Context,
	binlogIO io.BinlogIO,
	plan *datapb.CompactionPlan,
) *clusteringCompactionTask

func NewCompactionAllocator 

func NewCompactionAllocator(segmentAlloc, logIDAlloc allocator.Interface) *compactionAlloactor

func NewExecutor 

func NewExecutor() *executor

func NewMixCompactionTask 

func NewMixCompactionTask(
	ctx context.Context,
	binlogIO io.BinlogIO,
	plan *datapb.CompactionPlan,
) *mixCompactionTask

Types

type BatchApplyRet 

type BatchApplyRet = struct {
	StartIdx     int
	Segment2Hits map[int64][]bool
}

type ClusterBuffer 

type ClusterBuffer struct {
	// contains filtered or unexported fields
}

type Compactor 

type Compactor interface {
	Complete()
	Compact() (*datapb.CompactionPlanResult, error)
	Stop()
	GetPlanID() typeutil.UniqueID
	GetCollection() typeutil.UniqueID
	GetChannelName() string
	GetCompactionType() datapb.CompactionType
	GetSlotUsage() int64
}

type Executor 

type Executor interface {
	Start(ctx context.Context)
	Execute(task Compactor) (bool, error)
	Slots() int64
	RemoveTask(planID int64)
	GetResults(planID int64) []*datapb.CompactionPlanResult
	DiscardByDroppedChannel(channel string)
	DiscardPlan(channel string)
}

type FlushSignal 

type FlushSignal struct {
	// contains filtered or unexported fields
}

type LevelZeroCompactionTask 

type LevelZeroCompactionTask struct {
	io.BinlogIO
	// contains filtered or unexported fields
}

func NewLevelZeroCompactionTask 

func NewLevelZeroCompactionTask(
	ctx context.Context,
	binlogIO io.BinlogIO,
	cm storage.ChunkManager,
	plan *datapb.CompactionPlan,
) *LevelZeroCompactionTask

func (*LevelZeroCompactionTask) Compact 

func (t *LevelZeroCompactionTask) Compact() (*datapb.CompactionPlanResult, error)

func (*LevelZeroCompactionTask) Complete 

func (t *LevelZeroCompactionTask) Complete()

func (*LevelZeroCompactionTask) GetChannelName 

func (t *LevelZeroCompactionTask) GetChannelName() string

func (*LevelZeroCompactionTask) GetCollection 

func (t *LevelZeroCompactionTask) GetCollection() int64

func (*LevelZeroCompactionTask) GetCompactionType 

func (t *LevelZeroCompactionTask) GetCompactionType() datapb.CompactionType

func (*LevelZeroCompactionTask) GetPlanID 

func (t *LevelZeroCompactionTask) GetPlanID() typeutil.UniqueID

func (*LevelZeroCompactionTask) GetSlotUsage 

func (t *LevelZeroCompactionTask) GetSlotUsage() int64

func (*LevelZeroCompactionTask) Stop 

func (t *LevelZeroCompactionTask) Stop()

type MockCompactor 

type MockCompactor struct {
	mock.Mock
}

MockCompactor is an autogenerated mock type for the Compactor type

func NewMockCompactor 

func NewMockCompactor(t interface {
	mock.TestingT
	Cleanup(func())
}) *MockCompactor

NewMockCompactor creates a new instance of MockCompactor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockCompactor) Compact 

func (_m *MockCompactor) Compact() (*datapb.CompactionPlanResult, error)

Compact provides a mock function with given fields:

func (*MockCompactor) Complete 

func (_m *MockCompactor) Complete()

Complete provides a mock function with given fields:

func (*MockCompactor) EXPECT 

func (_m *MockCompactor) EXPECT() *MockCompactor_Expecter

func (*MockCompactor) GetChannelName 

func (_m *MockCompactor) GetChannelName() string

GetChannelName provides a mock function with given fields:

func (*MockCompactor) GetCollection 

func (_m *MockCompactor) GetCollection() int64

GetCollection provides a mock function with given fields:

func (*MockCompactor) GetCompactionType 

func (_m *MockCompactor) GetCompactionType() datapb.CompactionType

GetCompactionType provides a mock function with given fields:

func (*MockCompactor) GetPlanID 

func (_m *MockCompactor) GetPlanID() int64

GetPlanID provides a mock function with given fields:

func (*MockCompactor) GetSlotUsage 

func (_m *MockCompactor) GetSlotUsage() int64

GetSlotUsage provides a mock function with given fields:

func (*MockCompactor) Stop 

func (_m *MockCompactor) Stop()

Stop provides a mock function with given fields:

type MockCompactor_Compact_Call 

type MockCompactor_Compact_Call struct {
	*mock.Call
}

MockCompactor_Compact_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Compact'

func (*MockCompactor_Compact_Call) Return 

func (_c *MockCompactor_Compact_Call) Return(_a0 *datapb.CompactionPlanResult, _a1 error) *MockCompactor_Compact_Call

func (*MockCompactor_Compact_Call) Run 

func (_c *MockCompactor_Compact_Call) Run(run func()) *MockCompactor_Compact_Call

func (*MockCompactor_Compact_Call) RunAndReturn 

func (_c *MockCompactor_Compact_Call) RunAndReturn(run func() (*datapb.CompactionPlanResult, error)) *MockCompactor_Compact_Call

type MockCompactor_Complete_Call 

type MockCompactor_Complete_Call struct {
	*mock.Call
}

MockCompactor_Complete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Complete'

func (*MockCompactor_Complete_Call) Return 

func (_c *MockCompactor_Complete_Call) Return() *MockCompactor_Complete_Call

func (*MockCompactor_Complete_Call) Run 

func (_c *MockCompactor_Complete_Call) Run(run func()) *MockCompactor_Complete_Call

func (*MockCompactor_Complete_Call) RunAndReturn 

func (_c *MockCompactor_Complete_Call) RunAndReturn(run func()) *MockCompactor_Complete_Call

type MockCompactor_Expecter 

type MockCompactor_Expecter struct {
	// contains filtered or unexported fields
}

func (*MockCompactor_Expecter) Compact 

func (_e *MockCompactor_Expecter) Compact() *MockCompactor_Compact_Call

Compact is a helper method to define mock.On call

func (*MockCompactor_Expecter) Complete 

func (_e *MockCompactor_Expecter) Complete() *MockCompactor_Complete_Call

Complete is a helper method to define mock.On call

func (*MockCompactor_Expecter) GetChannelName 

func (_e *MockCompactor_Expecter) GetChannelName() *MockCompactor_GetChannelName_Call

GetChannelName is a helper method to define mock.On call

func (*MockCompactor_Expecter) GetCollection 

func (_e *MockCompactor_Expecter) GetCollection() *MockCompactor_GetCollection_Call

GetCollection is a helper method to define mock.On call

func (*MockCompactor_Expecter) GetCompactionType 

func (_e *MockCompactor_Expecter) GetCompactionType() *MockCompactor_GetCompactionType_Call

GetCompactionType is a helper method to define mock.On call

func (*MockCompactor_Expecter) GetPlanID 

func (_e *MockCompactor_Expecter) GetPlanID() *MockCompactor_GetPlanID_Call

GetPlanID is a helper method to define mock.On call

func (*MockCompactor_Expecter) GetSlotUsage 

func (_e *MockCompactor_Expecter) GetSlotUsage() *MockCompactor_GetSlotUsage_Call

GetSlotUsage is a helper method to define mock.On call

func (*MockCompactor_Expecter) Stop 

func (_e *MockCompactor_Expecter) Stop() *MockCompactor_Stop_Call

Stop is a helper method to define mock.On call

type MockCompactor_GetChannelName_Call 

type MockCompactor_GetChannelName_Call struct {
	*mock.Call
}

MockCompactor_GetChannelName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetChannelName'

func (*MockCompactor_GetChannelName_Call) Return 

func (_c *MockCompactor_GetChannelName_Call) Return(_a0 string) *MockCompactor_GetChannelName_Call

func (*MockCompactor_GetChannelName_Call) Run 

func (_c *MockCompactor_GetChannelName_Call) Run(run func()) *MockCompactor_GetChannelName_Call

func (*MockCompactor_GetChannelName_Call) RunAndReturn 

func (_c *MockCompactor_GetChannelName_Call) RunAndReturn(run func() string) *MockCompactor_GetChannelName_Call

type MockCompactor_GetCollection_Call 

type MockCompactor_GetCollection_Call struct {
	*mock.Call
}

MockCompactor_GetCollection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCollection'

func (*MockCompactor_GetCollection_Call) Return 

func (_c *MockCompactor_GetCollection_Call) Return(_a0 int64) *MockCompactor_GetCollection_Call

func (*MockCompactor_GetCollection_Call) Run 

func (_c *MockCompactor_GetCollection_Call) Run(run func()) *MockCompactor_GetCollection_Call

func (*MockCompactor_GetCollection_Call) RunAndReturn 

func (_c *MockCompactor_GetCollection_Call) RunAndReturn(run func() int64) *MockCompactor_GetCollection_Call

type MockCompactor_GetCompactionType_Call 

type MockCompactor_GetCompactionType_Call struct {
	*mock.Call
}

MockCompactor_GetCompactionType_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCompactionType'

func (*MockCompactor_GetCompactionType_Call) Return 

func (_c *MockCompactor_GetCompactionType_Call) Return(_a0 datapb.CompactionType) *MockCompactor_GetCompactionType_Call

func (*MockCompactor_GetCompactionType_Call) Run 

func (_c *MockCompactor_GetCompactionType_Call) Run(run func()) *MockCompactor_GetCompactionType_Call

func (*MockCompactor_GetCompactionType_Call) RunAndReturn 

func (_c *MockCompactor_GetCompactionType_Call) RunAndReturn(run func() datapb.CompactionType) *MockCompactor_GetCompactionType_Call

type MockCompactor_GetPlanID_Call 

type MockCompactor_GetPlanID_Call struct {
	*mock.Call
}

MockCompactor_GetPlanID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPlanID'

func (*MockCompactor_GetPlanID_Call) Return 

func (_c *MockCompactor_GetPlanID_Call) Return(_a0 int64) *MockCompactor_GetPlanID_Call

func (*MockCompactor_GetPlanID_Call) Run 

func (_c *MockCompactor_GetPlanID_Call) Run(run func()) *MockCompactor_GetPlanID_Call

func (*MockCompactor_GetPlanID_Call) RunAndReturn 

func (_c *MockCompactor_GetPlanID_Call) RunAndReturn(run func() int64) *MockCompactor_GetPlanID_Call

type MockCompactor_GetSlotUsage_Call 

type MockCompactor_GetSlotUsage_Call struct {
	*mock.Call
}

MockCompactor_GetSlotUsage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSlotUsage'

func (*MockCompactor_GetSlotUsage_Call) Return 

func (_c *MockCompactor_GetSlotUsage_Call) Return(_a0 int64) *MockCompactor_GetSlotUsage_Call

func (*MockCompactor_GetSlotUsage_Call) Run 

func (_c *MockCompactor_GetSlotUsage_Call) Run(run func()) *MockCompactor_GetSlotUsage_Call

func (*MockCompactor_GetSlotUsage_Call) RunAndReturn 

func (_c *MockCompactor_GetSlotUsage_Call) RunAndReturn(run func() int64) *MockCompactor_GetSlotUsage_Call

type MockCompactor_Stop_Call 

type MockCompactor_Stop_Call struct {
	*mock.Call
}

MockCompactor_Stop_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stop'

func (*MockCompactor_Stop_Call) Return 

func (_c *MockCompactor_Stop_Call) Return() *MockCompactor_Stop_Call

func (*MockCompactor_Stop_Call) Run 

func (_c *MockCompactor_Stop_Call) Run(run func()) *MockCompactor_Stop_Call

func (*MockCompactor_Stop_Call) RunAndReturn 

func (_c *MockCompactor_Stop_Call) RunAndReturn(run func()) *MockCompactor_Stop_Call

type MultiSegmentWriter 

type MultiSegmentWriter struct {
	// contains filtered or unexported fields
}

Not concurrent safe.

func NewMultiSegmentWriter 

func NewMultiSegmentWriter(binlogIO io.BinlogIO, allocator *compactionAlloactor, plan *datapb.CompactionPlan, maxRows int64, partitionID, collectionID int64, bm25Fields []int64) *MultiSegmentWriter

func (*MultiSegmentWriter) Finish 

func (w *MultiSegmentWriter) Finish() ([]*datapb.CompactionSegment, error)

DONOT return an empty list if every insert of the segment is deleted, append an empty segment instead

func (*MultiSegmentWriter) Write 

func (w *MultiSegmentWriter) Write(v *storage.Value) error

func (*MultiSegmentWriter) WriteRecord 

func (w *MultiSegmentWriter) WriteRecord(r storage.Record) error

type PQItem 

type PQItem struct {
	Value *storage.Value
	Index int
	Pos   int
}

type PriorityQueue 

type PriorityQueue []*PQItem

func (PriorityQueue) Len 

func (pq PriorityQueue) Len() int

func (PriorityQueue) Less 

func (pq PriorityQueue) Less(i, j int) bool

func (*PriorityQueue) Pop 

func (pq *PriorityQueue) Pop() interface{}

func (*PriorityQueue) Push 

func (pq *PriorityQueue) Push(x interface{})

func (PriorityQueue) Swap 

func (pq PriorityQueue) Swap(i, j int)

type SegmentDeltaWriter 

type SegmentDeltaWriter struct {
	// contains filtered or unexported fields
}

func NewSegmentDeltaWriter 

func NewSegmentDeltaWriter(segmentID, partitionID, collectionID int64) *SegmentDeltaWriter

func (*SegmentDeltaWriter) Finish 

func (w *SegmentDeltaWriter) Finish() (*storage.Blob, *writebuffer.TimeRange, error)

func (*SegmentDeltaWriter) GetCollectionID 

func (w *SegmentDeltaWriter) GetCollectionID() int64

func (*SegmentDeltaWriter) GetPartitionID 

func (w *SegmentDeltaWriter) GetPartitionID() int64

func (*SegmentDeltaWriter) GetRowNum 

func (w *SegmentDeltaWriter) GetRowNum() int64

func (*SegmentDeltaWriter) GetSegmentID 

func (w *SegmentDeltaWriter) GetSegmentID() int64

func (*SegmentDeltaWriter) GetTimeRange 

func (w *SegmentDeltaWriter) GetTimeRange() *writebuffer.TimeRange

func (*SegmentDeltaWriter) Write 

func (w *SegmentDeltaWriter) Write(pk storage.PrimaryKey, ts typeutil.Timestamp)

func (*SegmentDeltaWriter) WriteBatch 

func (w *SegmentDeltaWriter) WriteBatch(pks []storage.PrimaryKey, tss []typeutil.Timestamp)

type SegmentDeserializeReader 

type SegmentDeserializeReader struct {
	PKFieldID int64
	// contains filtered or unexported fields
}

func NewSegmentDeserializeReader 

func NewSegmentDeserializeReader(ctx context.Context, binlogPaths [][]string, binlogIO binlogIO.BinlogIO, PKFieldID int64, bm25FieldIDs []int64) *SegmentDeserializeReader

func (*SegmentDeserializeReader) Close 

func (r *SegmentDeserializeReader) Close()

func (*SegmentDeserializeReader) Next 

func (r *SegmentDeserializeReader) Next() (*storage.Value, error)

type SegmentWriter 

type SegmentWriter struct {
	// contains filtered or unexported fields
}

func NewSegmentWriter 

func NewSegmentWriter(sch *schemapb.CollectionSchema, maxCount int64, batchSize int, segID, partID, collID int64, Bm25Fields []int64) (*SegmentWriter, error)

func (*SegmentWriter) Finish 

func (w *SegmentWriter) Finish() (*storage.Blob, error)

func (*SegmentWriter) FlushAndIsEmpty 

func (w *SegmentWriter) FlushAndIsEmpty() bool

func (*SegmentWriter) FlushAndIsFull 

func (w *SegmentWriter) FlushAndIsFull() bool

func (*SegmentWriter) GetBm25Stats 

func (w *SegmentWriter) GetBm25Stats() map[int64]*storage.BM25Stats

func (*SegmentWriter) GetBm25StatsBlob 

func (w *SegmentWriter) GetBm25StatsBlob() (map[int64]*storage.Blob, error)

func (*SegmentWriter) GetCollectionID 

func (w *SegmentWriter) GetCollectionID() int64

func (*SegmentWriter) GetPartitionID 

func (w *SegmentWriter) GetPartitionID() int64

func (*SegmentWriter) GetPkID 

func (w *SegmentWriter) GetPkID() int64

func (*SegmentWriter) GetRowNum 

func (w *SegmentWriter) GetRowNum() int64

func (*SegmentWriter) GetSegmentID 

func (w *SegmentWriter) GetSegmentID() int64

func (*SegmentWriter) GetTimeRange 

func (w *SegmentWriter) GetTimeRange() *writebuffer.TimeRange

func (*SegmentWriter) GetTotalSize 

func (w *SegmentWriter) GetTotalSize() int64

func (*SegmentWriter) IsEmpty 

func (w *SegmentWriter) IsEmpty() bool

func (*SegmentWriter) IsFull 

func (w *SegmentWriter) IsFull() bool

func (*SegmentWriter) IsFullWithBinlogMaxSize 

func (w *SegmentWriter) IsFullWithBinlogMaxSize(binLogMaxSize uint64) bool

func (*SegmentWriter) SerializeYield 

func (w *SegmentWriter) SerializeYield() ([]*storage.Blob, *writebuffer.TimeRange, error)

func (*SegmentWriter) Write 

func (w *SegmentWriter) Write(v *storage.Value) error

func (*SegmentWriter) WriteRecord 

func (w *SegmentWriter) WriteRecord(r storage.Record) error

func (*SegmentWriter) WrittenMemorySize 

func (w *SegmentWriter) WrittenMemorySize() uint64

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.