metacache

package
v0.10.3-0...-7418609 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 18, 2024 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source
const (
	// NullSegment means the segment id to discard
	// happens when segment compacted to 0 lines and target segment is dropped directly
	NullSegment = int64(-1)
)

Variables

This section is empty.

Functions

This section is empty.

Types

type BloomFilterSet

type BloomFilterSet struct {
	// contains filtered or unexported fields
}

BloomFilterSet is a struct with multiple `storage.PkStatstics`. it maintains bloom filter generated from segment primary keys. it may be updated with new insert FieldData when serving growing segments.

func NewBloomFilterSet

func NewBloomFilterSet(historyEntries ...*storage.PkStatistics) *BloomFilterSet

NewBloomFilterSet returns a BloomFilterSet with provided historyEntries. Shall serve Flushed segments only. For growing segments, use `NewBloomFilterSetWithBatchSize` instead.

func NewBloomFilterSetWithBatchSize

func NewBloomFilterSetWithBatchSize(batchSize uint, historyEntries ...*storage.PkStatistics) *BloomFilterSet

NewBloomFilterSetWithBatchSize returns a BloomFilterSet. The batchSize parameter is used to initialize new bloom filter. It shall be the estimated row count per batch for segment to sync with.

func (*BloomFilterSet) BatchPkExist

func (bfs *BloomFilterSet) BatchPkExist(lc *storage.BatchLocationsCache) []bool

func (*BloomFilterSet) BatchPkExistWithHits

func (bfs *BloomFilterSet) BatchPkExistWithHits(lc *storage.BatchLocationsCache, hits []bool) []bool

func (*BloomFilterSet) GetHistory

func (bfs *BloomFilterSet) GetHistory() []*storage.PkStatistics

func (*BloomFilterSet) PkExists

func (bfs *BloomFilterSet) PkExists(lc *storage.LocationsCache) bool

func (*BloomFilterSet) Roll

func (bfs *BloomFilterSet) Roll(newStats ...*storage.PrimaryKeyStats)

func (*BloomFilterSet) UpdatePKRange

func (bfs *BloomFilterSet) UpdatePKRange(ids storage.FieldData) error

type MetaCache

type MetaCache interface {
	// Collection returns collection id of metacache.
	Collection() int64
	// Schema returns collection schema.
	Schema() *schemapb.CollectionSchema
	// AddSegment adds a segment from segment info.
	AddSegment(segInfo *datapb.SegmentInfo, factory PkStatsFactory, actions ...SegmentAction)
	// UpdateSegments applies action to segment(s) satisfy the provided filters.
	UpdateSegments(action SegmentAction, filters ...SegmentFilter)
	// RemoveSegments removes segments matches the provided filter.
	RemoveSegments(filters ...SegmentFilter) []int64
	// GetSegmentsBy returns segments statify the provided filters.
	GetSegmentsBy(filters ...SegmentFilter) []*SegmentInfo
	// GetSegmentByID returns segment with provided segment id if exists.
	GetSegmentByID(id int64, filters ...SegmentFilter) (*SegmentInfo, bool)
	// GetSegmentIDs returns ids of segments which satifiy the provided filters.
	GetSegmentIDsBy(filters ...SegmentFilter) []int64
	// PredictSegments returns the segment ids which may contain the provided primary key.
	PredictSegments(pk storage.PrimaryKey, filters ...SegmentFilter) ([]int64, bool)
	// DetectMissingSegments returns the segment ids which is missing in datanode.
	DetectMissingSegments(segments map[int64]struct{}) []int64
	// UpdateSegmentView updates the segments BF from datacoord view.
	UpdateSegmentView(partitionID int64, newSegments []*datapb.SyncSegmentInfo, newSegmentsBF []*BloomFilterSet, allSegments map[int64]struct{})
}

func NewMetaCache

func NewMetaCache(info *datapb.ChannelWatchInfo, factory PkStatsFactory) MetaCache

type MockMetaCache

type MockMetaCache struct {
	mock.Mock
}

MockMetaCache is an autogenerated mock type for the MetaCache type

func NewMockMetaCache

func NewMockMetaCache(t interface {
	mock.TestingT
	Cleanup(func())
}) *MockMetaCache

NewMockMetaCache creates a new instance of MockMetaCache. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockMetaCache) AddSegment

func (_m *MockMetaCache) AddSegment(segInfo *datapb.SegmentInfo, factory PkStatsFactory, actions ...SegmentAction)

AddSegment provides a mock function with given fields: segInfo, factory, actions

func (*MockMetaCache) Collection

func (_m *MockMetaCache) Collection() int64

Collection provides a mock function with given fields:

func (*MockMetaCache) DetectMissingSegments

func (_m *MockMetaCache) DetectMissingSegments(segments map[int64]struct{}) []int64

DetectMissingSegments provides a mock function with given fields: segments

func (*MockMetaCache) EXPECT

func (_m *MockMetaCache) EXPECT() *MockMetaCache_Expecter

func (*MockMetaCache) GetSegmentByID

func (_m *MockMetaCache) GetSegmentByID(id int64, filters ...SegmentFilter) (*SegmentInfo, bool)

GetSegmentByID provides a mock function with given fields: id, filters

func (*MockMetaCache) GetSegmentIDsBy

func (_m *MockMetaCache) GetSegmentIDsBy(filters ...SegmentFilter) []int64

GetSegmentIDsBy provides a mock function with given fields: filters

func (*MockMetaCache) GetSegmentsBy

func (_m *MockMetaCache) GetSegmentsBy(filters ...SegmentFilter) []*SegmentInfo

GetSegmentsBy provides a mock function with given fields: filters

func (*MockMetaCache) PredictSegments

func (_m *MockMetaCache) PredictSegments(pk storage.PrimaryKey, filters ...SegmentFilter) ([]int64, bool)

PredictSegments provides a mock function with given fields: pk, filters

func (*MockMetaCache) RemoveSegments

func (_m *MockMetaCache) RemoveSegments(filters ...SegmentFilter) []int64

RemoveSegments provides a mock function with given fields: filters

func (*MockMetaCache) Schema

func (_m *MockMetaCache) Schema() *schemapb.CollectionSchema

Schema provides a mock function with given fields:

func (*MockMetaCache) UpdateSegmentView

func (_m *MockMetaCache) UpdateSegmentView(partitionID int64, newSegments []*datapb.SyncSegmentInfo, newSegmentsBF []*BloomFilterSet, allSegments map[int64]struct{})

UpdateSegmentView provides a mock function with given fields: partitionID, newSegments, newSegmentsBF, allSegments

func (*MockMetaCache) UpdateSegments

func (_m *MockMetaCache) UpdateSegments(action SegmentAction, filters ...SegmentFilter)

UpdateSegments provides a mock function with given fields: action, filters

type MockMetaCache_AddSegment_Call

type MockMetaCache_AddSegment_Call struct {
	*mock.Call
}

MockMetaCache_AddSegment_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddSegment'

func (*MockMetaCache_AddSegment_Call) Return

func (*MockMetaCache_AddSegment_Call) Run

func (*MockMetaCache_AddSegment_Call) RunAndReturn

type MockMetaCache_Collection_Call

type MockMetaCache_Collection_Call struct {
	*mock.Call
}

MockMetaCache_Collection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Collection'

func (*MockMetaCache_Collection_Call) Return

func (*MockMetaCache_Collection_Call) Run

func (*MockMetaCache_Collection_Call) RunAndReturn

type MockMetaCache_DetectMissingSegments_Call

type MockMetaCache_DetectMissingSegments_Call struct {
	*mock.Call
}

MockMetaCache_DetectMissingSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DetectMissingSegments'

func (*MockMetaCache_DetectMissingSegments_Call) Return

func (*MockMetaCache_DetectMissingSegments_Call) Run

func (*MockMetaCache_DetectMissingSegments_Call) RunAndReturn

type MockMetaCache_Expecter

type MockMetaCache_Expecter struct {
	// contains filtered or unexported fields
}

func (*MockMetaCache_Expecter) AddSegment

func (_e *MockMetaCache_Expecter) AddSegment(segInfo interface{}, factory interface{}, actions ...interface{}) *MockMetaCache_AddSegment_Call

AddSegment is a helper method to define mock.On call

  • segInfo *datapb.SegmentInfo
  • factory PkStatsFactory
  • actions ...SegmentAction

func (*MockMetaCache_Expecter) Collection

Collection is a helper method to define mock.On call

func (*MockMetaCache_Expecter) DetectMissingSegments

func (_e *MockMetaCache_Expecter) DetectMissingSegments(segments interface{}) *MockMetaCache_DetectMissingSegments_Call

DetectMissingSegments is a helper method to define mock.On call

  • segments map[int64]struct{}

func (*MockMetaCache_Expecter) GetSegmentByID

func (_e *MockMetaCache_Expecter) GetSegmentByID(id interface{}, filters ...interface{}) *MockMetaCache_GetSegmentByID_Call

GetSegmentByID is a helper method to define mock.On call

  • id int64
  • filters ...SegmentFilter

func (*MockMetaCache_Expecter) GetSegmentIDsBy

func (_e *MockMetaCache_Expecter) GetSegmentIDsBy(filters ...interface{}) *MockMetaCache_GetSegmentIDsBy_Call

GetSegmentIDsBy is a helper method to define mock.On call

  • filters ...SegmentFilter

func (*MockMetaCache_Expecter) GetSegmentsBy

func (_e *MockMetaCache_Expecter) GetSegmentsBy(filters ...interface{}) *MockMetaCache_GetSegmentsBy_Call

GetSegmentsBy is a helper method to define mock.On call

  • filters ...SegmentFilter

func (*MockMetaCache_Expecter) PredictSegments

func (_e *MockMetaCache_Expecter) PredictSegments(pk interface{}, filters ...interface{}) *MockMetaCache_PredictSegments_Call

PredictSegments is a helper method to define mock.On call

  • pk storage.PrimaryKey
  • filters ...SegmentFilter

func (*MockMetaCache_Expecter) RemoveSegments

func (_e *MockMetaCache_Expecter) RemoveSegments(filters ...interface{}) *MockMetaCache_RemoveSegments_Call

RemoveSegments is a helper method to define mock.On call

  • filters ...SegmentFilter

func (*MockMetaCache_Expecter) Schema

Schema is a helper method to define mock.On call

func (*MockMetaCache_Expecter) UpdateSegmentView

func (_e *MockMetaCache_Expecter) UpdateSegmentView(partitionID interface{}, newSegments interface{}, newSegmentsBF interface{}, allSegments interface{}) *MockMetaCache_UpdateSegmentView_Call

UpdateSegmentView is a helper method to define mock.On call

  • partitionID int64
  • newSegments []*datapb.SyncSegmentInfo
  • newSegmentsBF []*BloomFilterSet
  • allSegments map[int64]struct{}

func (*MockMetaCache_Expecter) UpdateSegments

func (_e *MockMetaCache_Expecter) UpdateSegments(action interface{}, filters ...interface{}) *MockMetaCache_UpdateSegments_Call

UpdateSegments is a helper method to define mock.On call

  • action SegmentAction
  • filters ...SegmentFilter

type MockMetaCache_GetSegmentByID_Call

type MockMetaCache_GetSegmentByID_Call struct {
	*mock.Call
}

MockMetaCache_GetSegmentByID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentByID'

func (*MockMetaCache_GetSegmentByID_Call) Return

func (*MockMetaCache_GetSegmentByID_Call) Run

func (*MockMetaCache_GetSegmentByID_Call) RunAndReturn

type MockMetaCache_GetSegmentIDsBy_Call

type MockMetaCache_GetSegmentIDsBy_Call struct {
	*mock.Call
}

MockMetaCache_GetSegmentIDsBy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentIDsBy'

func (*MockMetaCache_GetSegmentIDsBy_Call) Return

func (*MockMetaCache_GetSegmentIDsBy_Call) Run

func (*MockMetaCache_GetSegmentIDsBy_Call) RunAndReturn

type MockMetaCache_GetSegmentsBy_Call

type MockMetaCache_GetSegmentsBy_Call struct {
	*mock.Call
}

MockMetaCache_GetSegmentsBy_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetSegmentsBy'

func (*MockMetaCache_GetSegmentsBy_Call) Return

func (*MockMetaCache_GetSegmentsBy_Call) Run

func (*MockMetaCache_GetSegmentsBy_Call) RunAndReturn

type MockMetaCache_PredictSegments_Call

type MockMetaCache_PredictSegments_Call struct {
	*mock.Call
}

MockMetaCache_PredictSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PredictSegments'

func (*MockMetaCache_PredictSegments_Call) Return

func (*MockMetaCache_PredictSegments_Call) Run

func (*MockMetaCache_PredictSegments_Call) RunAndReturn

type MockMetaCache_RemoveSegments_Call

type MockMetaCache_RemoveSegments_Call struct {
	*mock.Call
}

MockMetaCache_RemoveSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RemoveSegments'

func (*MockMetaCache_RemoveSegments_Call) Return

func (*MockMetaCache_RemoveSegments_Call) Run

func (*MockMetaCache_RemoveSegments_Call) RunAndReturn

type MockMetaCache_Schema_Call

type MockMetaCache_Schema_Call struct {
	*mock.Call
}

MockMetaCache_Schema_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Schema'

func (*MockMetaCache_Schema_Call) Return

func (*MockMetaCache_Schema_Call) Run

func (*MockMetaCache_Schema_Call) RunAndReturn

type MockMetaCache_UpdateSegmentView_Call

type MockMetaCache_UpdateSegmentView_Call struct {
	*mock.Call
}

MockMetaCache_UpdateSegmentView_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateSegmentView'

func (*MockMetaCache_UpdateSegmentView_Call) Return

func (*MockMetaCache_UpdateSegmentView_Call) Run

func (_c *MockMetaCache_UpdateSegmentView_Call) Run(run func(partitionID int64, newSegments []*datapb.SyncSegmentInfo, newSegmentsBF []*BloomFilterSet, allSegments map[int64]struct{})) *MockMetaCache_UpdateSegmentView_Call

func (*MockMetaCache_UpdateSegmentView_Call) RunAndReturn

type MockMetaCache_UpdateSegments_Call

type MockMetaCache_UpdateSegments_Call struct {
	*mock.Call
}

MockMetaCache_UpdateSegments_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateSegments'

func (*MockMetaCache_UpdateSegments_Call) Return

func (*MockMetaCache_UpdateSegments_Call) Run

func (*MockMetaCache_UpdateSegments_Call) RunAndReturn

type PkStatsFactory

type PkStatsFactory func(vchannel *datapb.SegmentInfo) *BloomFilterSet

type SegmentAction

type SegmentAction func(info *SegmentInfo)

func CompactTo

func CompactTo(compactTo int64) SegmentAction

func FinishSyncing

func FinishSyncing(batchSize int64) SegmentAction

func MergeSegmentAction

func MergeSegmentAction(actions ...SegmentAction) SegmentAction

MergeSegmentAction is the util function to merge multiple SegmentActions into one.

func RollStats

func RollStats(newStats ...*storage.PrimaryKeyStats) SegmentAction

func SetStartPosRecorded

func SetStartPosRecorded(flag bool) SegmentAction

func StartSyncing

func StartSyncing(batchSize int64) SegmentAction

func UpdateBufferedRows

func UpdateBufferedRows(bufferedRows int64) SegmentAction

func UpdateCheckpoint

func UpdateCheckpoint(checkpoint *msgpb.MsgPosition) SegmentAction

func UpdateNumOfRows

func UpdateNumOfRows(numOfRows int64) SegmentAction

func UpdateState

func UpdateState(state commonpb.SegmentState) SegmentAction

type SegmentFilter

type SegmentFilter interface {
	Filter(info *SegmentInfo) bool
	AddFilter(*segmentCriterion)
}

func WithCompacted

func WithCompacted() SegmentFilter

func WithLevel

func WithLevel(level datapb.SegmentLevel) SegmentFilter

func WithNoSyncingTask

func WithNoSyncingTask() SegmentFilter

func WithPartitionID

func WithPartitionID(partitionID int64) SegmentFilter

func WithPartitionIDs

func WithPartitionIDs(partitionIDs []int64) SegmentFilter

func WithSegmentIDs

func WithSegmentIDs(segmentIDs ...int64) SegmentFilter

func WithSegmentState

func WithSegmentState(states ...commonpb.SegmentState) SegmentFilter

func WithStartPosNotRecorded

func WithStartPosNotRecorded() SegmentFilter

type SegmentFilterFunc

type SegmentFilterFunc func(info *SegmentInfo) bool

SegmentFilterFunc implements segment filter with other filters logic.

func (SegmentFilterFunc) AddFilter

func (f SegmentFilterFunc) AddFilter(criterion *segmentCriterion)

func (SegmentFilterFunc) Filter

func (f SegmentFilterFunc) Filter(info *SegmentInfo) bool

type SegmentIDFilter

type SegmentIDFilter struct {
	// contains filtered or unexported fields
}

SegmentIDFilter segment filter with segment ids.

func (*SegmentIDFilter) AddFilter

func (f *SegmentIDFilter) AddFilter(criterion *segmentCriterion)

func (*SegmentIDFilter) Filter

func (f *SegmentIDFilter) Filter(info *SegmentInfo) bool

type SegmentInfo

type SegmentInfo struct {
	// contains filtered or unexported fields
}

func NewSegmentInfo

func NewSegmentInfo(info *datapb.SegmentInfo, bfs *BloomFilterSet) *SegmentInfo

func (*SegmentInfo) Checkpoint

func (s *SegmentInfo) Checkpoint() *msgpb.MsgPosition

func (*SegmentInfo) Clone

func (s *SegmentInfo) Clone() *SegmentInfo

func (*SegmentInfo) CompactTo

func (s *SegmentInfo) CompactTo() int64

func (*SegmentInfo) FlushedRows

func (s *SegmentInfo) FlushedRows() int64

FlushedRows return flushed rows number.

func (*SegmentInfo) GetBloomFilterSet

func (s *SegmentInfo) GetBloomFilterSet() *BloomFilterSet

func (*SegmentInfo) GetHistory

func (s *SegmentInfo) GetHistory() []*storage.PkStatistics

func (*SegmentInfo) Level

func (s *SegmentInfo) Level() datapb.SegmentLevel

func (*SegmentInfo) NumOfRows

func (s *SegmentInfo) NumOfRows() int64

NumOfRows returns sum of number of rows, including flushed, syncing and buffered

func (*SegmentInfo) PartitionID

func (s *SegmentInfo) PartitionID() int64

func (*SegmentInfo) SegmentID

func (s *SegmentInfo) SegmentID() int64

func (*SegmentInfo) StartPosition

func (s *SegmentInfo) StartPosition() *msgpb.MsgPosition

func (*SegmentInfo) State

func (s *SegmentInfo) State() commonpb.SegmentState

type SegmentStateFilter

type SegmentStateFilter struct {
	// contains filtered or unexported fields
}

SegmentStateFilter segment filter with segment states.

func (*SegmentStateFilter) AddFilter

func (f *SegmentStateFilter) AddFilter(criterion *segmentCriterion)

func (*SegmentStateFilter) Filter

func (f *SegmentStateFilter) Filter(info *SegmentInfo) bool

type StorageV2Cache

type StorageV2Cache struct {
	// contains filtered or unexported fields
}

func NewStorageV2Cache

func NewStorageV2Cache(schema *schemapb.CollectionSchema) (*StorageV2Cache, error)

func (*StorageV2Cache) ArrowSchema

func (s *StorageV2Cache) ArrowSchema() *arrow.Schema

func (*StorageV2Cache) GetOrCreateSpace

func (s *StorageV2Cache) GetOrCreateSpace(segmentID int64, creator func() (*milvus_storage.Space, error)) (*milvus_storage.Space, error)

func (*StorageV2Cache) SetSpace

func (s *StorageV2Cache) SetSpace(segmentID int64, space *milvus_storage.Space)

only for unit test

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL