utils

Documentation

Index

• Constants
• Variables
• func ConvertFloatBytesToMB(bytes float64) float64
• func ConvertGroupByKey(rec []byte) ([]string, error)
• func ConvertIntBytesToMB(bytes int64) int64
• func ConvertOperatorToString(op LogicalOperator) string
• func ConvertUintBytesToMB(bytes uint64) uint64
• func GetCurrentTimeMillis() uint64
• func GetLiteralFromString(identifier string) (v interface{})
• func IsNumeric(exp interface{}) bool
• func MaxInt64(a1 int64, b1 int64) int64
• func MaxUint16(a1 uint16, b1 uint16) uint16
• func MaxUint64(a1 uint64, b1 uint64) uint64
• func MinInt64(a1 int64, b1 int64) int64
• func MinUint64(a1 uint64, b1 uint64) uint64
• type AggColUsageMode
• type AggregateFunctions
  • func (e AggregateFunctions) String() string
• type ArithmeticOperator
• type BlkRecIdxContainer
• type CValueDictEnclosure
  • func (e *CValueDictEnclosure) GetValue() (interface{}, error)
• type CValueEnclosure
  • func Reduce(e1 CValueEnclosure, e2 CValueEnclosure, fun AggregateFunctions) (CValueEnclosure, error)
  • func (e *CValueEnclosure) ConvertValue(val interface{}) error
  • func (e *CValueEnclosure) GetFloatValue() (float64, error)
  • func (e *CValueEnclosure) GetString() (string, error)
  • func (e *CValueEnclosure) GetUIntValue() (uint64, error)
  • func (e *CValueEnclosure) GetValue() (interface{}, error)
• type DtypeEnclosure
  • func CreateDtypeEnclosure(inVal interface{}, qid uint64) (*DtypeEnclosure, error)
  • func (dte *DtypeEnclosure) AddStringAsByteSlice()
  • func (dte *DtypeEnclosure) GetRegexp() *regexp.Regexp
  • func (dte *DtypeEnclosure) GetValue() (interface{}, error)
  • func (dte *DtypeEnclosure) GobDecode(data []byte) error
  • func (dte *DtypeEnclosure) GobEncode() ([]byte, error)
  • func (dte *DtypeEnclosure) IsFullWildcard() bool
  • func (dte *DtypeEnclosure) IsNumeric() bool
  • func (dte *DtypeEnclosure) IsRegex() bool
  • func (dte *DtypeEnclosure) IsString() bool
  • func (dte *DtypeEnclosure) Reset()
  • func (dte *DtypeEnclosure) SetRegexp(exp *regexp.Regexp)
• type FilterOperator
  • func (e FilterOperator) ToString() string
• type LogicalAndArithmeticOperator
• type LogicalOperator
• type MathFunctions
• type NumTypeEnclosure
  • func (self *NumTypeEnclosure) ReduceFast(e2Dtype SS_DTYPE, e2int64 int64, e2float64 float64, fun AggregateFunctions) error
  • func (nte *NumTypeEnclosure) ToCValueEnclosure() (*CValueEnclosure, error)
• type RR_ENC_TYPE
• type RangeFunctions
• type RangeNumType
• type RecordResultContainer
• type SIGNAL_TYPE
• type SS_DTYPE
• type SS_IntUintFloatTypes
  • func GetNumberTypeAndVal(numstr string) (SS_IntUintFloatTypes, int64, uint64, float64)
• type SegKeyInfo
• type TagOperator
  • func (e TagOperator) String() string
• type TimeFunctions
• type TimeUnit
  • func ConvertSubseconds(subsecond string) (TimeUnit, error)

Constants

const (
	SIGNAL_METRICS_OTSDB      = 1
	SIGNAL_EVENTS             = 2
	SIGNAL_JAEGER_TRACES      = 3
	SIGNAL_METRICS_INFLUX     = 4
	SIGNAL_METRICS_PROMETHEUS = 5
)

const (
	RR_ENC_UINT64 = 1
	RR_ENC_BITSET = 2
)

const BLOCK_BLOOM_SEPARATOR = ":"

const BLOCK_BLOOM_SIZE = 100 // the default should be on the smaller side. Let dynamic bloom sizing fix the optimal one

const BLOCK_MICRO_CHECK_MULTINODE_MEM_PERCENT = 15

const BLOCK_MICRO_MULTINODE_MEM_PERCENT = 80

const BLOCK_RI_MAP_SIZE = 100

const BLOCK_SUMMARY_LEN_SIZE = 4

const BLOCK_SUMMARY_REC_CNT_SIZE = 2

const BLOCK_SUMMARY_SIZE = 50_000

const BLOCK_SUMMARY_TS_SIZE = 8

const BLOOM_COLL_PROBABILITY = 0.001

const BLOOM_SIZE_HISTORY = 5 // number of entries to analyze to get next block's bloom size

const BUFFER_MEM_PERCENT = 5

const COL_BLK_OFF_START = NUMCOLS_OFF_END

const COL_OFF_BYTE_SIZE = 2

const DEFAULT_TIME_SLICE_SIZE = 10_000

const FILE_READ_BUFFER_SIZE = 100_000

const LEN_BLKNUM_CMI_SIZE = 2

const LEN_BLOCK_CMI_SIZE = 4

const LEN_PQMR_BLK_SIZE = 2

const MAX_RAW_DATAPOINTS_IN_RESULT = 5_000_000

const MAX_RECORD_SIZE = 63_000

leave some room for column name/value meta since we use 2 bytes for record len, columnname-len, we can accommodate 65535

const MAX_RECS_PER_WIP = 65_534

const METRICS_MEMORY_MEM_PERCENT = 20

const MICRO_IDX_CHECK_MEM_PERCENT = 5 // percent allocated for runtime checking & loading of cmis

const MICRO_IDX_MEM_PERCENT = 35 // percent allocated for both rotated & unrotated metadata (cmi/searchmetadata)

GLOBAL Defs proportion of available to allocate for specific uses

const MS_IN_DAY = 86_400_000 // 24 * 60 * 60 * 1000

const MS_IN_HOUR = 3_600_000 // 60 * 60 * 1000

const MS_IN_MIN = 60_000 // 60 * 1000

const MULTINODE_SSM_MEM_PERCENT = 20

const NUMCOLS_OFF_END = NUMCOLS_OFF_START + 2

const NUMCOLS_OFF_START = RECORDLEN_BYTE_SIZE

const PQMR_SIZE uint = 4000 // init size of pqs bitset

const RAW_SEARCH_MEM_PERCENT = 15 // minimum percent allocated for segsearch

const RAW_SEARCH_MULTINODE_MEM_PERCENT = 95

const RECORDLEN_BYTE_SIZE = 2

const RI_BLK_LEN_SIZE = 4

const RI_SIZE = 2_000_000

const SEGMENT_ROTATE_DURATION_SECONDS = 15 * 60 // 15 mins

const SEGMENT_ROTATE_SLEEP_DURATION_SECONDS = 60 // 1 min

const SSM_MEM_PERCENT = 20

const STALE_RECENTLY_ROTATED_ENTRY_MS = 60_000 // one minute

const WIP_NUM_RECS = 4000

const WIP_SIZE = 2_000_000

if you change this size, adjust the block bloom size

Variables

var BYTE_EMPTY_STRING = []byte("")

var BYTE_SPACE = []byte(" ")

var BYTE_SPACE_LEN = len(BYTE_SPACE)

var CMI_BLOOM_INDEX = []byte{0x01}

var CMI_INVERTED_INDEX = []byte{0x03}

var CMI_RANGE_INDEX = []byte{0x02}

var MAX_ACTIVE_SERIES_PER_SEGMENT = 10_000_000

var MAX_BYTES_METRICS_BLOCK uint64 = 1e+8 // 100MB

var MAX_BYTES_METRICS_SEGMENT uint64 = 1e+10 // 10GB

var METRICS_SEARCH_ALLOCATE_BLOCK uint64 = 1.5e+8 // 150MB

var QUERY_EARLY_EXIT_LIMIT = uint64(10_000)

var QUERY_MAX_BUCKETS = uint64(10_000)

var ReflectFilterOperator = map[FilterOperator]FilterOperator{
	Equals:               Equals,
	NotEquals:            NotEquals,
	LessThan:             GreaterThan,
	LessThanOrEqualTo:    GreaterThanOrEqualTo,
	GreaterThan:          LessThan,
	GreaterThanOrEqualTo: LessThanOrEqualTo,
	IsNull:               IsNull,
	IsNotNull:            IsNotNull,
}

Maps a filter to its equivalent if left and right were swapped If a range filter is given as left op right, what is the right op if it swaps to right op* left

var STAR_BYTE = []byte("*")

var STAR_TREE_BLOCK = []byte{3}

var TIMESTAMP_TOPDIFF_VARENC = []byte{2}

var UPLOAD_INGESTNODE_DIR = time.Duration(1 * time.Minute) // one minute

var VALTYPE_DICT_ARRAY = []byte{0x14}

var VALTYPE_ENC_BACKFILL = []byte{0x13}

var VALTYPE_ENC_BOOL = []byte{0x01}

var VALTYPE_ENC_FLOAT64 = []byte{0x11}

var VALTYPE_ENC_INT16 = []byte{0x08}

var VALTYPE_ENC_INT32 = []byte{0x09}

var VALTYPE_ENC_INT64 = []byte{0x10}

var VALTYPE_ENC_INT8 = []byte{0x07}

var VALTYPE_ENC_LARGE_STRING = []byte{0x12}

var VALTYPE_ENC_RNT_FLOAT64 = []byte{0x02}

var VALTYPE_ENC_RNT_SIGNED_INT = []byte{0x01}

var VALTYPE_ENC_RNT_UNSIGNED_INT = []byte{0x00}

var VALTYPE_ENC_SMALL_STRING = []byte{0x02}

var VALTYPE_ENC_UINT16 = []byte{0x04}

var VALTYPE_ENC_UINT32 = []byte{0x05}

var VALTYPE_ENC_UINT64 = []byte{0x06}

var VALTYPE_ENC_UINT8 = []byte{0x03}

var VALTYPE_RAW_JSON = []byte{0x15}

var VERSION_MBLOCKSUMMARY = []byte{0x01}

var VERSION_TAGSTREE = []byte{0x01}

var VERSION_TSGFILE = []byte{0x01}

var VERSION_TSOFILE = []byte{0x01}

var ZSTD_COMLUNAR_BLOCK = []byte{0}

var ZSTD_DICTIONARY_BLOCK = []byte{1}

Functions

func ConvertFloatBytesToMB

func ConvertFloatBytesToMB(bytes float64) float64

func ConvertGroupByKey

func ConvertGroupByKey(rec []byte) ([]string, error)

converts the input byte slice to a string representation of all read values returns array of strings with groupBy values

func ConvertIntBytesToMB

func ConvertIntBytesToMB(bytes int64) int64

func ConvertOperatorToString

func ConvertOperatorToString(op LogicalOperator) string

func ConvertUintBytesToMB

func ConvertUintBytesToMB(bytes uint64) uint64

func GetCurrentTimeMillis

func GetCurrentTimeMillis() uint64

func GetLiteralFromString

func GetLiteralFromString(identifier string) (v interface{})

func IsNumeric

func IsNumeric(exp interface{}) bool

func MaxInt64

func MaxInt64(a1 int64, b1 int64) int64

func MaxUint16

func MaxUint16(a1 uint16, b1 uint16) uint16

func MaxUint64

func MaxUint64(a1 uint64, b1 uint64) uint64

func MinInt64

func MinInt64(a1 int64, b1 int64) int64

func MinUint64

func MinUint64(a1 uint64, b1 uint64) uint64

Types

type AggColUsageMode

type AggColUsageMode int

For columns used by aggs with eval statements, we should keep their raw values because we need to evaluate them For columns only used by aggs without eval statements, we should not keep their raw values because it is a waste of performance If we only use two modes. Later occurring aggs will overwrite earlier occurring aggs' usage status. E.g. stats dc(eval(lower(state))), dc(state)

const (
	NoEvalUsage   AggColUsageMode = iota // NoEvalUsage indicates that the column will be used by an aggregator without an eval function
	WithEvalUsage                        // WithEvalUsage indicates that the column will be used by an aggregator with an eval function
	BothUsage                            // BothUsage indicates that the column will be used by both types of aggregators simultaneously
)

type AggregateFunctions

type AggregateFunctions int

const (
	Count AggregateFunctions = iota + 1
	Avg
	Min
	Max
	Range
	Sum
	Cardinality
	Quantile
	TopK
	BottomK
	Stddev
	Stdvar
	Group
	Values
	List
	Estdc
	EstdcError
	ExactPerc
	Median
	Mode
	Perc
	UpperPerc
	Stdev
	Stdevp
	Sumsq
	Var
	Varp
	First
	Last
	Earliest
	EarliestTime
	Latest
	LatestTime
	StatsRate
)

func (AggregateFunctions) String

func (e AggregateFunctions) String() string

type ArithmeticOperator

type ArithmeticOperator int

const (
	Add ArithmeticOperator = iota
	Subtract
	Divide
	Multiply
	Modulo
	BitwiseAnd
	BitwiseOr
	BitwiseExclusiveOr
)

type BlkRecIdxContainer

type BlkRecIdxContainer struct {
	BlkRecIndexes    map[uint16]map[uint16]uint64
	VirtualTableName string
}

type CValueDictEnclosure

type CValueDictEnclosure struct {
	Dtype       SS_DTYPE
	CValString  string
	CValBool    bool
	CValUInt64  uint64
	CValInt64   int64
	CValFloat64 float64
	CValUInt32  uint32
	CValInt32   int32
	CValUInt16  uint16
	CValInt16   int16
	CValUInt    uint8
	CValInt     int8
}

func (*CValueDictEnclosure) GetValue

func (e *CValueDictEnclosure) GetValue() (interface{}, error)

type CValueEnclosure

type CValueEnclosure struct {
	Dtype SS_DTYPE
	CVal  interface{}
}

func Reduce

func Reduce(e1 CValueEnclosure, e2 CValueEnclosure, fun AggregateFunctions) (CValueEnclosure, error)

func (*CValueEnclosure) ConvertValue

func (e *CValueEnclosure) ConvertValue(val interface{}) error

resets the CValueEnclosure to the given value

func (*CValueEnclosure) GetFloatValue

func (e *CValueEnclosure) GetFloatValue() (float64, error)

func (*CValueEnclosure) GetString

func (e *CValueEnclosure) GetString() (string, error)

func (*CValueEnclosure) GetUIntValue

func (e *CValueEnclosure) GetUIntValue() (uint64, error)

Returns a uint64 representation of the value

if its a number, casts to uint64 if its a string, xxhashed and returns it

func (*CValueEnclosure) GetValue

func (e *CValueEnclosure) GetValue() (interface{}, error)

type DtypeEnclosure

type DtypeEnclosure struct {
	Dtype          SS_DTYPE
	BoolVal        uint8
	UnsignedVal    uint64
	SignedVal      int64
	FloatVal       float64
	StringVal      string
	StringValBytes []byte // byte slice representation of StringVal
	// contains filtered or unexported fields
}

every time you change this struct remember to adjust CreateDtypeEnclosure and ResetDtypeEnclosure

func CreateDtypeEnclosure

func CreateDtypeEnclosure(inVal interface{}, qid uint64) (*DtypeEnclosure, error)

func (*DtypeEnclosure) AddStringAsByteSlice

func (dte *DtypeEnclosure) AddStringAsByteSlice()

func (*DtypeEnclosure) GetRegexp

func (dte *DtypeEnclosure) GetRegexp() *regexp.Regexp

func (*DtypeEnclosure) GetValue

func (dte *DtypeEnclosure) GetValue() (interface{}, error)

func (*DtypeEnclosure) GobDecode

func (dte *DtypeEnclosure) GobDecode(data []byte) error

func (*DtypeEnclosure) GobEncode

func (dte *DtypeEnclosure) GobEncode() ([]byte, error)

func (*DtypeEnclosure) IsFullWildcard

func (dte *DtypeEnclosure) IsFullWildcard() bool

func (*DtypeEnclosure) IsNumeric

func (dte *DtypeEnclosure) IsNumeric() bool

func (*DtypeEnclosure) IsRegex

func (dte *DtypeEnclosure) IsRegex() bool

func (*DtypeEnclosure) IsString

func (dte *DtypeEnclosure) IsString() bool

func (*DtypeEnclosure) Reset

func (dte *DtypeEnclosure) Reset()

func (*DtypeEnclosure) SetRegexp

func (dte *DtypeEnclosure) SetRegexp(exp *regexp.Regexp)

type FilterOperator

type FilterOperator int

const (
	Equals FilterOperator = iota
	NotEquals
	LessThan
	LessThanOrEqualTo
	GreaterThan
	GreaterThanOrEqualTo
	// Between - on the query parser to break down
	// In - on the query parser to break down
	IsNull
	IsNotNull
)

func (FilterOperator) ToString

func (e FilterOperator) ToString() string

type LogicalAndArithmeticOperator

type LogicalAndArithmeticOperator int

const (
	LetAdd LogicalAndArithmeticOperator = iota
	LetSubtract
	LetDivide
	LetMultiply
	LetModulo
	LetPower
	LetEquals
	LetNotEquals
	LetLessThan
	LetLessThanOrEqualTo
	LetGreaterThan
	LetGreaterThanOrEqualTo
	LetAnd
	LetOr
	LetUnless
)

type LogicalOperator

type LogicalOperator int

const (
	Or LogicalOperator = iota
	And
	Exclusion
)

type MathFunctions

type MathFunctions int

const (
	Round MathFunctions = iota + 1
	Ceil
	Floor
	Abs
	Sqrt
	Exp
	Ln
	Log2
	Log10
	Sgn
	Deg
	Rad
	Clamp
	Clamp_Max
	Clamp_Min
	Timestamp
	Acos
	Acosh
	Asin
	Asinh
	Atan
	Atanh
	Cos
	Cosh
	Sin
	Sinh
	Tan
	Tanh
)

type NumTypeEnclosure

type NumTypeEnclosure struct {
	Ntype    SS_DTYPE `json:"ntype,omitempty"`
	IntgrVal int64    `json:"intgrVal,omitempty"`
	FloatVal float64  `json:"floatVal,omitempty"`
}

used for numeric calcs and promotions

func (*NumTypeEnclosure) ReduceFast

func (self *NumTypeEnclosure) ReduceFast(e2Dtype SS_DTYPE, e2int64 int64,
	e2float64 float64, fun AggregateFunctions) error

func (*NumTypeEnclosure) ToCValueEnclosure

func (nte *NumTypeEnclosure) ToCValueEnclosure() (*CValueEnclosure, error)

type RR_ENC_TYPE

type RR_ENC_TYPE uint8

type RangeFunctions

type RangeFunctions int

const (
	Derivative RangeFunctions = iota + 1
	Predict_Linear
	Rate
	IRate
	Increase
	Delta
	IDelta
	Avg_Over_Time
	Min_Over_Time
	Max_Over_Time
	Sum_Over_Time
	Count_Over_Time
	Stdvar_Over_Time
	Stddev_Over_Time
	Last_Over_Time
	Present_Over_Time
	Quantile_Over_Time
	Changes
	Resets
)

type RangeNumType

type RangeNumType uint8

const (
	RNT_UNSIGNED_INT RangeNumType = iota
	RNT_SIGNED_INT
	RNT_FLOAT64
)

If you add new datatype under RangeNumType please add corresponding encoding VALTYPE_ENC_RNT_* in the following block

type RecordResultContainer

type RecordResultContainer struct {
	SegKeyInfo       SegKeyInfo // Information about the segment key for a record (remote or not)
	BlockNum         uint16     // Block number of the record
	RecordNum        uint16     // Record number of the record
	SortColumnValue  float64    // Sort column value of the record
	TimeStamp        uint64     // Timestamp of the record
	VirtualTableName string     // Table name of the record
}

type SIGNAL_TYPE

type SIGNAL_TYPE uint8

type SS_DTYPE

type SS_DTYPE uint8

const (
	SS_INVALID SS_DTYPE = iota
	SS_DT_BOOL
	SS_DT_SIGNED_NUM
	SS_DT_UNSIGNED_NUM
	SS_DT_FLOAT
	SS_DT_STRING
	SS_DT_STRING_SET
	SS_DT_BACKFILL
	SS_DT_SIGNED_32_NUM
	SS_DT_USIGNED_32_NUM
	SS_DT_SIGNED_16_NUM
	SS_DT_USIGNED_16_NUM
	SS_DT_SIGNED_8_NUM
	SS_DT_USIGNED_8_NUM
	SS_DT_ARRAY_DICT
	SS_DT_RAW_JSON
)

type SS_IntUintFloatTypes

type SS_IntUintFloatTypes int

const (
	SS_UINT8 SS_IntUintFloatTypes = iota
	SS_UINT16
	SS_UINT32
	SS_UINT64
	SS_INT8
	SS_INT16
	SS_INT32
	SS_INT64
	SS_FLOAT64
)

func GetNumberTypeAndVal

func GetNumberTypeAndVal(numstr string) (SS_IntUintFloatTypes, int64, uint64, float64)

type SegKeyInfo

type SegKeyInfo struct {
	// Encoded segment key
	SegKeyEnc uint16
	// If the RRC came from a remote node
	IsRemote bool
	// if IsRemote, Record will be initialized to a string of the form <<node_id>>-<<segkey>>-<<block_num>>-<<record_num>>
	RecordId string
}

Information about the segment key for a record Stores if the RRC came from a remote node

type TagOperator

type TagOperator int

const (
	Equal TagOperator = iota
	NotEqual
	Regex
	NegRegex
)

func (TagOperator) String

func (e TagOperator) String() string

type TimeFunctions

type TimeFunctions float64

const (
	Hour TimeFunctions = iota + 1
	Minute
	Month
	Year
	DayOfMonth
	DayOfWeek
	DayOfYear
	DaysInMonth
)

type TimeUnit

type TimeUnit uint8

const (
	TMMicrosecond TimeUnit = iota
	TMMillisecond
	TMCentisecond
	TMDecisecond
	TMSecond
	TMMinute
	TMHour
	TMDay
	TMWeek
	TMMonth
	TMQuarter
)

func ConvertSubseconds

func ConvertSubseconds(subsecond string) (TimeUnit, error)

Convert subseconds