metadata

Documentation

Index

• Constants
• Variables
• func BulkAddMetricsSegment(allMMetadata []*MetricsSegmentMetadata)
• func BulkAddSegmentMicroIndex(allMetadata []*SegmentMicroIndex)
• func CheckAndGetColsForSegKey(segKey string, vtable string) (map[string]bool, bool)
• func CheckMicroIndicesForUnrotated(currQuery *structs.SearchQuery, lookupTimeRange *dtu.TimeRange, ...) (map[string]*structs.SegmentSearchRequest, uint64, uint64, error)
• func DeleteMetricsSegmentKey(dirPath string) error
• func DeleteSegmentKey(segKey string)
• func DeleteVirtualTable(vTable string, orgid uint64)
• func DoesColumnExistForTable(cName string, indices []string) bool
• func ExtractUnrotatedSSRFromSearchNode(node *structs.SearchNode, timeRange *dtu.TimeRange, indexNames []string, ...) map[string]*structs.SegmentSearchRequest
• func FilterSegmentsByTime(timeRange *dtu.TimeRange, indexNames []string, orgid uint64) (map[string]map[string]*dtu.TimeRange, uint64, uint64)
• func GetAllColNames(indices []string) []string
• func GetBlockSearchInfoForKey(key string) (map[uint16]*structs.BlockMetadataHolder, error)
• func GetBlockSummariesForKey(key string) ([]*structs.BlockSummary, error)
• func GetMetricSegmentsOverTheTimeRange(tRange *dtu.MetricsTimeRange, orgid uint64) map[string]*structs.MetricsMeta
• func GetMetricsSegmentRequests(tRange *dtu.MetricsTimeRange, querySummary *summary.QuerySummary, orgid uint64) (map[string][]*structs.MetricsSearchRequest, error)
• func GetNumOfSearchedRecordsRotated(segKey string) uint64
• func GetSearchInfoForPQSQuery(key string, spqmr *pqmr.SegmentPQMRResults) (map[uint16]*structs.BlockMetadataHolder, []*structs.BlockSummary, error)
• func GetTSRangeForMissingBlocks(segKey string, tRange *dtu.TimeRange, spqmr *pqmr.SegmentPQMRResults) *dtu.TimeRange
• func GetTotalSMICount() int64
• func GetUniqueTagKeysForRotated(tRange *dtu.MetricsTimeRange, myid uint64) (map[string]struct{}, error)
• func InitBlockMetaCheckLimiter(unloadedBlockLimit int64)
• func InitMockColumnarMetadataStore(dir string, count int, numBlocks int, entryCount int)
• func InitMockMetricsMetadataStore(entryCount int) error
• func IsUnrotatedQueryNeeded(timeRange *dtu.TimeRange, indexNames []string) bool
• func RebalanceInMemoryCmi(metadataSizeBytes uint64)
• func RebalanceInMemorySsm(ssmSizeBytes uint64)
• func ResetMetricsMetadata_TestOnly()
• func RunCmiCheck(segkey string, tableName string, timeRange *dtu.TimeRange, ...) (*structs.SegmentSearchRequest, uint64, uint64, error)
• type MetricsSegmentMetadata
  • func InitMetricsMicroIndex(mMeta *structs.MetricsMeta) *MetricsSegmentMetadata
  • func (mm *MetricsSegmentMetadata) LoadSearchMetadata() error
  • func (mm *MetricsSegmentMetadata) ReadMetricNamesBloom(fileName string) error
• type MetricsSegmentSearchMetadata
• type SegmentMicroIndex
  • func InitSegmentMicroIndex(segMetaInfo *structs.SegMeta) *SegmentMicroIndex
  • func (sm *SegmentMicroIndex) LoadSearchMetadata(rbuf []byte) ([]byte, error)
• type SegmentMicroIndices
  • func (smi *SegmentMicroIndices) GetCMIForBlockAndColumn(blkNum uint16, cname string) (*structs.CmiContainer, error)
  • func (smi *SegmentMicroIndices) GetCMIsForBlock(blkNum uint16) (map[string]*structs.CmiContainer, error)
• type SegmentSearchMetadata

Constants

const INITIAL_NUM_BLOCKS = 1000

Variables

var GlobalBlockMicroIndexCheckLimiter *semaphore.WeightedSemaphore

var GlobalSegStoreSummary = &structs.AllSegStoreSummary{}

Functions

func BulkAddMetricsSegment

func BulkAddMetricsSegment(allMMetadata []*MetricsSegmentMetadata)

func BulkAddSegmentMicroIndex

func BulkAddSegmentMicroIndex(allMetadata []*SegmentMicroIndex)

func CheckAndGetColsForSegKey

func CheckAndGetColsForSegKey(segKey string, vtable string) (map[string]bool, bool)

returns the a map with columns as keys and returns a bool if the segkey/table was found

func CheckMicroIndicesForUnrotated

func CheckMicroIndicesForUnrotated(currQuery *structs.SearchQuery, lookupTimeRange *dtu.TimeRange, indexNames []string,
	allBlocksToSearch map[string]map[string]*structs.BlockTracker, bloomWords map[string]bool, bloomOp utils.LogicalOperator, rangeFilter map[string]string,
	rangeOp utils.FilterOperator, isRange bool, wildcardValue bool, qid uint64) (map[string]*structs.SegmentSearchRequest, uint64, uint64, error)

filters unrotated blocks based on search conditions returns the final search request, total blocks, sum of filtered blocks, and any errors

func DeleteMetricsSegmentKey

func DeleteMetricsSegmentKey(dirPath string) error

func DeleteSegmentKey

func DeleteSegmentKey(segKey string)

func DeleteVirtualTable

func DeleteVirtualTable(vTable string, orgid uint64)

func DoesColumnExistForTable

func DoesColumnExistForTable(cName string, indices []string) bool

func ExtractUnrotatedSSRFromSearchNode

func ExtractUnrotatedSSRFromSearchNode(node *structs.SearchNode, timeRange *dtu.TimeRange, indexNames []string,
	rawSearchKeys map[string]map[string]*structs.BlockTracker, querySummary *summary.QuerySummary, qid uint64) map[string]*structs.SegmentSearchRequest

func FilterSegmentsByTime

func FilterSegmentsByTime(timeRange *dtu.TimeRange, indexNames []string, orgid uint64) (map[string]map[string]*dtu.TimeRange, uint64, uint64)

 Returns:
	 1. map of tableName -> map segKey -> segKey time range
	 2. []string of segKeys that are in the time range. that exist in the map
	 2. final matched count
	 3. total possible count

func GetAllColNames

func GetAllColNames(indices []string) []string

func GetBlockSearchInfoForKey

func GetBlockSearchInfoForKey(key string) (map[uint16]*structs.BlockMetadataHolder, error)

func GetBlockSummariesForKey

func GetBlockSummariesForKey(key string) ([]*structs.BlockSummary, error)

func GetMetricSegmentsOverTheTimeRange

func GetMetricSegmentsOverTheTimeRange(tRange *dtu.MetricsTimeRange, orgid uint64) map[string]*structs.MetricsMeta

func GetMetricsSegmentRequests

func GetMetricsSegmentRequests(tRange *dtu.MetricsTimeRange, querySummary *summary.QuerySummary, orgid uint64) (map[string][]*structs.MetricsSearchRequest, error)

Returns all tagTrees that we need to search and what MetricsSegments & MetricsBlocks pass time filtering.

Returns map[string][]*structs.MetricSearchRequest, mapping a tagsTree to all MetricSearchRequest that pass time filtering

func GetNumOfSearchedRecordsRotated

func GetNumOfSearchedRecordsRotated(segKey string) uint64

func GetSearchInfoForPQSQuery

func GetSearchInfoForPQSQuery(key string, spqmr *pqmr.SegmentPQMRResults) (map[uint16]*structs.BlockMetadataHolder,
	[]*structs.BlockSummary, error)

returns block search info, block summaries, and any errors encountered block search info will be loaded for all possible columns

func GetTSRangeForMissingBlocks

func GetTSRangeForMissingBlocks(segKey string, tRange *dtu.TimeRange, spqmr *pqmr.SegmentPQMRResults) *dtu.TimeRange

returns the time range of the blocks in the segment that do not exist in spqmr if the timeRange is nil, no blocks were found in metadata that do not exist in spqmr

func GetTotalSMICount

func GetTotalSMICount() int64

func GetUniqueTagKeysForRotated

func GetUniqueTagKeysForRotated(tRange *dtu.MetricsTimeRange, myid uint64) (map[string]struct{}, error)

func InitBlockMetaCheckLimiter

func InitBlockMetaCheckLimiter(unloadedBlockLimit int64)

func InitMockColumnarMetadataStore

func InitMockColumnarMetadataStore(dir string, count int, numBlocks int, entryCount int)

function to init mock server in memory. Should only be called by tests

func InitMockMetricsMetadataStore

func InitMockMetricsMetadataStore(entryCount int) error

func IsUnrotatedQueryNeeded

func IsUnrotatedQueryNeeded(timeRange *dtu.TimeRange, indexNames []string) bool

func RebalanceInMemoryCmi

func RebalanceInMemoryCmi(metadataSizeBytes uint64)

func RebalanceInMemorySsm

func RebalanceInMemorySsm(ssmSizeBytes uint64)

Internally, this will allocate 30% of the SSM size to metrics and the remaining to logs

func ResetMetricsMetadata_TestOnly

func ResetMetricsMetadata_TestOnly()

func RunCmiCheck

func RunCmiCheck(segkey string, tableName string, timeRange *dtu.TimeRange,
	blockTracker *structs.BlockTracker, bloomKeys map[string]bool, bloomOp utils.LogicalOperator,
	rangeFilter map[string]string, rangeOp utils.FilterOperator, isRange bool, wildCardValue bool,
	currQuery *structs.SearchQuery, colsToCheck map[string]bool, wildcardCol bool,
	qid uint64, isQueryPersistent bool, pqid string) (*structs.SegmentSearchRequest, uint64, uint64, error)

TODO: function is getting to big and has many args, needs to be refactored Returns all search requests, number of blocks checked, number of blocks passed, error

Types

type MetricsSegmentMetadata

type MetricsSegmentMetadata struct {
	structs.MetricsMeta // original read metrics meta
	MetricsSegmentSearchMetadata
}

func InitMetricsMicroIndex

func InitMetricsMicroIndex(mMeta *structs.MetricsMeta) *MetricsSegmentMetadata

func (*MetricsSegmentMetadata) LoadSearchMetadata

func (mm *MetricsSegmentMetadata) LoadSearchMetadata() error

func (*MetricsSegmentMetadata) ReadMetricNamesBloom

func (mm *MetricsSegmentMetadata) ReadMetricNamesBloom(fileName string) error

type MetricsSegmentSearchMetadata

type MetricsSegmentSearchMetadata struct {
	// contains filtered or unexported fields
}

type SegmentMicroIndex

type SegmentMicroIndex struct {
	structs.SegMeta
	SegmentMicroIndices
	SegmentSearchMetadata
}

Top level segment metadata for access of cmis/search metadata

func InitSegmentMicroIndex

func InitSegmentMicroIndex(segMetaInfo *structs.SegMeta) *SegmentMicroIndex

func (*SegmentMicroIndex) LoadSearchMetadata

func (sm *SegmentMicroIndex) LoadSearchMetadata(rbuf []byte) ([]byte, error)

type SegmentMicroIndices

type SegmentMicroIndices struct {
	MicroIndexSize uint64
	// contains filtered or unexported fields
}

Holder structure for just the segment microindices

func (*SegmentMicroIndices) GetCMIForBlockAndColumn

func (smi *SegmentMicroIndices) GetCMIForBlockAndColumn(blkNum uint16, cname string) (*structs.CmiContainer, error)

Returns the cmi for a given block & column, or any errors encountered

func (*SegmentMicroIndices) GetCMIsForBlock

func (smi *SegmentMicroIndices) GetCMIsForBlock(blkNum uint16) (map[string]*structs.CmiContainer, error)

Returns all columnar cmis for a given block or any errors encountered

type SegmentSearchMetadata

type SegmentSearchMetadata struct {
	BlockSummaries     []*structs.BlockSummary
	BlockSearchInfo    map[uint16]*structs.BlockMetadataHolder
	SearchMetadataSize uint64
	// contains filtered or unexported fields
}

Holder structure for just the segment search metadata (blk summaries & blockSearchInfo)