format

package

v1.0.0-beta1 Latest Latest Go to latest Published: Nov 24, 2022 License: MPL-2.0 Imports: 9 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/vkcom/statshouse

Documentation ¶

Index ¶

Constants
Variables
func AddRawValuePrefix(s string) string
func AllowedResolution(r int) int
func AppendHexStringValue(dst []byte, src []byte) []byte
func AppendValidStringValue(dst []byte, src []byte) ([]byte, error)
func AppendValidStringValueLegacy(dst []byte, src []byte) ([]byte, error)
func CodeTagValue(code int32) string
func ContainsRawTagValue(s mem.RO) (int32, bool)
func FilterBuildArch(buildArch int32) int32
func ForceValidStringValue(src string) []byte
func GetBuildArchKey(arch string) int32
func HasRawValuePrefix(s string) bool
func ISO8601Date2BuildDateKey(str string) int32
func IsValueCodeZero(s string) bool
func NewTagID(i int) string
func ParseCodeTagValue(s string) (int32, error)
func ParseTagIDForAPI(tagID string) int
func TagID(i int) string
func TagIDTagToTagID(tagIDTag int32) string
func ValidDashboardName(s string) bool
func ValidFloatValue(f float64) bool
func ValidGroupName(s string) bool
func ValidMetricKind(kind string) bool
func ValidMetricName(s mem.RO) bool
func ValidRawKind(s string) bool
func ValidStringValue(s mem.RO) bool
func ValidStringValueLegacy(s mem.RO) bool
func ValidTagName(s string) bool
func ValidTagValueForAPI(s string) bool
type CreateMappingExtra
type DashboardMeta
- func (m DashboardMeta) MarshalBinary() ([]byte, error)
- func (m *DashboardMeta) UnmarshalBinary(data []byte) error
type MetaStorageInterface
type MetricKind
type MetricMetaTag
type MetricMetaValue
- func (m MetricMetaValue) MarshalBinary() ([]byte, error)
- func (m *MetricMetaValue) RestoreCachedInfo() error
- func (m *MetricMetaValue) UnmarshalBinary(data []byte) error
type MetricMetaValueOld
type MetricsGroup
- func (m *MetricsGroup) RestoreCachedInfo() error

Constants ¶

View Source

const (
	TagIDShift       = 100 // "0" is set to 100, "1" to 101, "_s" to 99, "_h" to 98, leaving space in both directions
	TagIDShiftLegacy = 2   // in the past we used this shift, so it got into DB

	BuiltinGroupIDDefault = -1 // for all metrics with group not known. We want to edit it in the future, so not 0
	BuiltinGroupIDBuiltin = -2 // for all built in metrics

	BuiltinMetricIDAgentSamplingFactor        = -1
	BuiltinMetricIDAggBucketReceiveDelaySec   = -2 // Also approximates insert delay, interesting for historic buckets
	BuiltinMetricIDAggInsertSize              = -3 // If all contributors come on time, count will be 1 (per shard). If some come through historic conveyor, can be larger.
	BuiltinMetricIDTLByteSizePerInflightType  = -4
	BuiltinMetricIDAggKeepAlive               = -5 // How many keep-alive were among contributors
	BuiltinMetricIDAggSizeCompressed          = -6
	BuiltinMetricIDAggSizeUncompressed        = -7
	BuiltinMetricIDAggAdditionsToEstimator    = -8 // How many new tags were inserted into bucket
	BuiltinMetricIDAggHourCardinality         = -9
	BuiltinMetricIDAggSamplingFactor          = -10
	BuiltinMetricIDIngestionStatus            = -11
	BuiltinMetricIDAggInsertTime              = -12
	BuiltinMetricIDAggHistoricBucketsWaiting  = -13
	BuiltinMetricIDAggBucketAggregateTimeSec  = -14
	BuiltinMetricIDAggActiveSenders           = -15
	BuiltinMetricIDAgentDiskCacheErrors       = -18
	BuiltinMetricIDTimingErrors               = -20
	BuiltinMetricIDAgentReceivedBatchSize     = -21
	BuiltinMetricIDAggMapping                 = -23
	BuiltinMetricIDAggInsertTimeReal          = -24
	BuiltinMetricIDAgentHistoricQueueSize     = -25
	BuiltinMetricIDAggHistoricSecondsWaiting  = -26
	BuiltinMetricIDAggInsertSizeReal          = -27
	BuiltinMetricIDAgentPerMetricSampleBudget = -29 // Not informative - TODO improve or remove
	BuiltinMetricIDAgentMapping               = -30
	BuiltinMetricIDAgentReceivedPacketSize    = -31
	BuiltinMetricIDAggMappingCreated          = -33
	BuiltinMetricIDVersions                   = -34
	BuiltinMetricIDBadges                     = -35 // copy of some other metrics for efficient show of errors and warnings
	BuiltinMetricIDAutoConfig                 = -36
	BuiltinMetricIDJournalVersions            = -37 // has smart custom sending logic
	BuiltinMetricIDPromScrapeTime             = -38
	BuiltinMetricIDAgentHeartbeatVersion      = -41 // TODO - remove
	BuiltinMetricIDAgentHeartbeatArgs         = -42 // TODO - remove, this metric was writing larger than allowed strings to DB in the past
	BuiltinMetricIDUsageMemory                = -43
	BuiltinMetricIDUsageCPU                   = -44
	BuiltinMetricIDGeneratorConstCounter      = -45
	BuiltinMetricIDGeneratorSinCounter        = -46
	BuiltinMetricIDHeartbeatVersion           = -47
	BuiltinMetricIDHeartbeatArgs              = -48 // this metric was writing larger than allowed strings to DB in the past
	BuiltinMetricIDAPIRPCServiceTime          = -49 // TODO - harmonize 3 timing API metrics into single new one with protocol
	BuiltinMetricIDAPIBRS                     = -50
	BuiltinMetricIDAPIEndpointResponseTime    = -51
	BuiltinMetricIDAPIEndpointServiceTime     = -52
	BuiltinMetricIDBudgetHost                 = -53 // these 2 metrics are invisible, but host mapping is flood-protected by their names
	BuiltinMetricIDBudgetAggregatorHost       = -54 // we want to see limits properly credited in flood meta metric tags
	BuiltinMetricIDAPIActiveQueries           = -55
	BuiltinMetricIDRPCRequests                = -56
	BuiltinMetricIDBudgetUnknownMetric        = -57
	BuiltinMetricIDHeartbeatArgs2             = -58 // if args do not fit into BuiltinMetricIDHeartbeatArgs, we put tail here.
	BuiltinMetricIDHeartbeatArgs3             = -59 // ...
	BuiltinMetricIDHeartbeatArgs4             = -60 // OK, enough!
	BuiltinMetricIDContributorsLog            = -61
	BuiltinMetricIDContributorsLogRev         = -62
	BuiltinMetricIDGeneratorGapsCounter       = -63
	BuiltinMetricIDGroupSizeBeforeSampling    = -64
	BuiltinMetricIDGroupSizeAfterSampling     = -65

	// metric names used in code directly
	BuiltinMetricNameAggBucketReceiveDelaySec = "__agg_bucket_receive_delay_sec"
	BuiltinMetricNameAgentSamplingFactor      = "__src_sampling_factor"
	BuiltinMetricNameAggSamplingFactor        = "__agg_sampling_factor"
	BuiltinMetricNameIngestionStatus          = "__src_ingestion_status"
	BuiltinMetricNameAggMappingCreated        = "__agg_mapping_created"
	BuiltinMetricNameBadges                   = "__badges"
	BuiltinMetricNamePromScrapeTime           = "__prom_scrape_time"
	BuiltinMetricNameAPIRPCServiceTime        = "__api_rpc_service_time"
	BuiltinMetricNameUsageMemory              = "__usage_mem"
	BuiltinMetricNameUsageCPU                 = "__usage_cpu"
	BuiltinMetricNameAPIBRS                   = "__api_big_response_storage_size"
	BuiltinMetricNameAPIEndpointResponseTime  = "__api_endpoint_response_time"
	BuiltinMetricNameAPIEndpointServiceTime   = "__api_endpoint_service_time"
	BuiltinMetricNameBudgetHost               = "__budget_host"
	BuiltinMetricNameBudgetAggregatorHost     = "__budget_aggregator_host"
	BuiltinMetricNameAPIActiveQueries         = "__api_active_queries"
	BuiltinMetricNameBudgetUnknownMetric      = "__budget_unknown_metric"

	TagValueIDBadgeIngestionErrorsOld  = -11 // remove from API, then stop writing
	TagValueIDBadgeAggMappingErrorsOld = -33 // remove from API, then stop writing
	TagValueIDBadgeAgentSamplingFactor = -1
	TagValueIDBadgeAggSamplingFactor   = -10
	TagValueIDBadgeIngestionErrors     = 1
	TagValueIDBadgeAggMappingErrors    = 2
	TagValueIDBadgeContributors        = 3 // # of agents who sent this second. Hyper important to distinguish between holes in your data and problems with agents (no connectivity, etc.).

	TagValueIDRPCRequestsStatusOK          = 1
	TagValueIDRPCRequestsStatusErrLocal    = 2
	TagValueIDRPCRequestsStatusErrUpstream = 3
	TagValueIDRPCRequestsStatusHijack      = 4

	TagValueIDProduction = 1
	TagValueIDStaging    = 2

	TagValueIDAPILaneFast = 1
	TagValueIDAPILaneSlow = 2

	TagValueIDConveyorRecent   = 1
	TagValueIDConveyorHistoric = 2

	TagValueIDAggregatorOriginal = 1
	TagValueIDAggregatorSpare    = 2

	TagValueIDTimingFutureBucketRecent         = 1
	TagValueIDTimingFutureBucketHistoric       = 2
	TagValueIDTimingLateRecent                 = 3
	TagValueIDTimingLongWindowThrownAgent      = 4
	TagValueIDTimingLongWindowThrownAggregator = 5
	TagValueIDTimingMissedSeconds              = 6

	TagValueIDRouteDirect       = 1
	TagValueIDRouteIngressProxy = 2

	TagValueIDSecondReal    = 1
	TagValueIDSecondPhantom = 2

	TagValueIDInsertTimeOK    = 1
	TagValueIDInsertTimeError = 2

	TagValueIDHistoricQueueMemory = 1
	TagValueIDHistoricQueueDisk   = 2

	TagValueIDDiskCacheErrorWrite             = 1
	TagValueIDDiskCacheErrorRead              = 2
	TagValueIDDiskCacheErrorDelete            = 3
	TagValueIDDiskCacheErrorReadNotConfigured = 5
	TagValueIDDiskCacheErrorCompressFailed    = 6

	TagValueIDSrcIngestionStatusOKCached                     = 10
	TagValueIDSrcIngestionStatusOKUncached                   = 11
	TagValueIDSrcIngestionStatusErrMetricNotFound            = 21
	TagValueIDSrcIngestionStatusErrNanInfValue               = 23
	TagValueIDSrcIngestionStatusErrNanInfCounter             = 24
	TagValueIDSrcIngestionStatusErrNegativeCounter           = 25
	TagValueIDSrcIngestionStatusErrMapOther                  = 30 // never written, for historic data
	TagValueIDSrcIngestionStatusWarnMapTagNameNotFound       = 33
	TagValueIDSrcIngestionStatusErrMapInvalidRawTagValue     = 34
	TagValueIDSrcIngestionStatusErrMapTagValueCached         = 35
	TagValueIDSrcIngestionStatusErrMapTagValue               = 36
	TagValueIDSrcIngestionStatusErrMapGlobalQueueOverload    = 37
	TagValueIDSrcIngestionStatusErrMapPerMetricQueueOverload = 38
	TagValueIDSrcIngestionStatusErrMapTagValueEncoding       = 39
	TagValueIDSrcIngestionStatusOKLegacy                     = 40 // never written, for historic data
	TagValueIDSrcIngestionStatusErrMetricNonCanonical        = 41 // never written, for historic data
	TagValueIDSrcIngestionStatusErrMetricInvisible           = 42
	TagValueIDSrcIngestionStatusErrLegacyProtocol            = 43 // we stopped adding metrics to conveyor via legacy protocol
	TagValueIDSrcIngestionStatusWarnDeprecatedT              = 44
	TagValueIDSrcIngestionStatusWarnDeprecatedStop           = 45
	TagValueIDSrcIngestionStatusWarnMapTagSetTwice           = 46
	TagValueIDSrcIngestionStatusWarnDeprecatedKeyName        = 47
	TagValueIDSrcIngestionStatusErrMetricNameEncoding        = 48
	TagValueIDSrcIngestionStatusErrMapTagNameEncoding        = 49
	TagValueIDSrcIngestionStatusErrValueUniqueBothSet        = 50
	TagValueIDSrcIngestionStatusWarnOldCounterSemantic       = 51

	TagValueIDPacketFormatLegacy   = 1
	TagValueIDPacketFormatTL       = 2
	TagValueIDPacketFormatMsgPack  = 3
	TagValueIDPacketFormatJSON     = 4
	TagValueIDPacketFormatProtobuf = 5
	TagValueIDPacketFormatRPC      = 6
	TagValueIDPacketFormatEmpty    = 7

	TagValueIDAgentReceiveStatusOK    = 1
	TagValueIDAgentReceiveStatusError = 2

	TagValueIDAggMappingDolphinLegacy = 1 // unused, remains for historic purpose
	TagValueIDAggMappingTags          = 2
	TagValueIDAggMappingMetaMetrics   = 3
	TagValueIDAggMappingJournalUpdate = 4

	TagValueIDAggSamplingFactorReasonInsertSize = 2 // we have no other reasons yet

	TagValueIDAggMappingStatusOKCached     = 1
	TagValueIDAggMappingStatusOKUncached   = 2
	TagValueIDAggMappingStatusErrUncached  = 3
	TagValueIDAggMappingStatusNotFound     = 4
	TagValueIDAggMappingStatusImmediateOK  = 5
	TagValueIDAggMappingStatusImmediateErr = 6
	TagValueIDAggMappingStatusEnqueued     = 7
	TagValueIDAggMappingStatusDelayedOK    = 8
	TagValueIDAggMappingStatusDelayedErr   = 9

	TagValueIDAgentFirstSampledMetricBudgetPerMetric = 1
	TagValueIDAgentFirstSampledMetricBudgetUnused    = 2

	TagValueIDGroupSizeSamplingFit     = 1
	TagValueIDGroupSizeSamplingSampled = 2

	TagValueIDAgentMappingStatusAllDead   = 1
	TagValueIDAgentMappingStatusOKFirst   = 2
	TagValueIDAgentMappingStatusOKSecond  = 3
	TagValueIDAgentMappingStatusErrSingle = 4
	TagValueIDAgentMappingStatusErrBoth   = 5

	TagValueIDAggMappingCreatedStatusOK                    = 1
	TagValueIDAggMappingCreatedStatusCreated               = 2
	TagValueIDAggMappingCreatedStatusFlood                 = 3
	TagValueIDAggMappingCreatedStatusErrorPMC              = 4
	TagValueIDAggMappingCreatedStatusErrorInvariant        = 5
	TagValueIDAggMappingCreatedStatusErrorNotAskedToCreate = 6
	TagValueIDAggMappingCreatedStatusErrorInvalidValue     = 7

	TagValueIDComponentAgent        = 1
	TagValueIDComponentAggregator   = 2
	TagValueIDComponentIngressProxy = 3
	TagValueIDComponentAPI          = 4

	TagValueIDAutoConfigOK             = 1
	TagValueIDAutoConfigErrorSend      = 2
	TagValueIDAutoConfigErrorKeepAlive = 3
	TagValueIDAutoConfigWrongCluster   = 4

	TagValueIDSizeCounter           = 1
	TagValueIDSizeValue             = 2
	TagValueIDSizeSingleValue       = 3
	TagValueIDSizePercentiles       = 4
	TagValueIDSizeUnique            = 5
	TagValueIDSizeStringTop         = 6
	TagValueIDSizeSampleFactors     = 7
	TagValueIDSizeIngestionStatusOK = 8
	TagValueIDSizeBuiltIn           = 9

	TagValueIDScrapeError = 1

	TagValueIDCPUUsageUser = 1
	TagValueIDCPUUsageSys  = 2

	TagValueIDHeartbeatEventStart     = 1
	TagValueIDHeartbeatEventHeartbeat = 2

	TagValueIDBuildArchAMD64 = 1
	TagValueIDBuildArch386   = 2
	TagValueIDBuildArchARM64 = 3
	TagValueIDBuildArchARM   = 4
)

View Source

const (
	MaxTags      = 16
	MaxStringLen = 128 // both for normal tags and _s, _h tags (string tops, hostnames)

	TagValueIDUnspecified  = 0
	TagValueIDMappingFlood = -1

	EffectiveWeightOne = 128                      // metric.Weight is multiplied by this and rounded. Do not make too big or metric with weight set to 0 will disappear completely.
	MaxEffectiveWeight = 100 * EffectiveWeightOne // do not make too high, we multiply this by sum of metric serialized length during sampling

	// we deprecated legacy canonical names "key0".."key15", "skey"
	// and made shorter new names "0".."15", "_s", "_h"
	PreKeyTagID    = "prekey"
	StringTopTagID = "skey"

	EnvTagID   = tagIDPrefix + "0"
	EnvTagName = "env" // this is legacy name. We want to get rid of all different names for tag 0.
	LETagName  = "le"

	NewStringTopTagID = "_s"
	NewHostTagID      = "_h"

	StringTopTagIndex = -1 // used as flag during mapping
	HostTagIndex      = -2 // used as flag during mapping

	// added for lots of built-in metrics automatically
	BuildArchTag  = 10
	AggHostTag    = 11
	AggShardTag   = 12
	AggReplicaTag = 13
	RouteTag      = 14
	AgentEnvTag   = 15

	// legacy values, used only by API to process data from version1 database
	TagValueIDProductionLegacy   = 22676293   // stlogs_s_production, actual value matters only for legacy conveyor
	TagValueNullLegacy           = "null"     // see TagValueIDNull
	TagValueIDMappingFloodLegacy = 22136242   // STATLOGS_KEY_KEYOVERFLOW_INT
	TagValueIDRawDeltaLegacy     = 10_000_000 // STATLOGS_INT_NOCONVERT
)

View Source

const (
	MetricKindCounter          = "counter"
	MetricKindValue            = "value"
	MetricKindValuePercentiles = "value_p"
	MetricKindUnique           = "unique"
	MetricKindStringTopLegacy  = "stop" // Legacy, not valid. Converted into counter during RestoreMetricInfo
	MetricKindMixed            = "mixed"
	MetricKindMixedPercentiles = "mixed_p"
)

Do not change values, they are stored in DB

View Source

const (
	MetricEvent       int32 = 0
	DashboardEvent    int32 = 1
	MetricsGroupEvent int32 = 2
	PromConfigEvent   int32 = 3
)

Variables ¶

View Source

var (
	BuiltinMetrics = map[int32]*MetricMetaValue{
		BuiltinMetricIDAgentSamplingFactor: {
			Name: BuiltinMetricNameAgentSamplingFactor,
			Kind: MetricKindValue,
			Description: `Sample factor selected by agent.
Calculated by agent from scratch every second to fit all collected data into network budget.
Count of this metric is proportional to # of clients who set it in particular second. 
Set only if greater than 1.`,
			Tags: []MetricMetaTag{{
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
			PreKeyTagID: "1",
		},
		BuiltinMetricIDAggBucketReceiveDelaySec: {
			Name: BuiltinMetricNameAggBucketReceiveDelaySec,
			Kind: MetricKindValue,
			Description: `Difference between timestamp of received bucket and aggregator wall clock.
Count of this metric is # of agents who sent this second (per replica*shard), and they do it every second to keep this metric stable.
Set by aggregator.`,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDSecondReal:    "real",
					TagValueIDSecondPhantom: "phantom",
				}),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggInsertSize: {
			Name:        "__agg_insert_size",
			Kind:        MetricKindValue,
			Description: "Size of aggregated bucket inserted into clickhouse. Written when second is inserted, which can be much later.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "type",
				ValueComments: convertToValueComments(insertKindToValue),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDTLByteSizePerInflightType: {
			Name:        "__src_tl_byte_size_per_inflight_type",
			Kind:        MetricKindValue,
			Description: "Approximate uncompressed byte size of various parts of TL representation of time bucket.\nSet by agent.",
			Tags: []MetricMetaTag{{
				Description:   "inflight_type",
				ValueComments: convertToValueComments(insertKindToValue),
			}},
		},
		BuiltinMetricIDAggKeepAlive: {
			Name:        "__agg_keep_alive",
			Kind:        MetricKindCounter,
			Description: "Number of keep-alive empty inserts (which follow normal insert conveyor) in aggregated bucket.\nSet by aggregator.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggSizeCompressed: {
			Name:        "__agg_size_compressed",
			Kind:        MetricKindValue,
			Description: "Compressed size of bucket received from agent (size of raw TL request).\nSet by aggregator.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggSizeUncompressed: {
			Name:        "__agg_size_uncompressed",
			Kind:        MetricKindValue,
			Description: "Uncompressed size of bucket received from agent.\nSet by aggregator.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggAdditionsToEstimator: {
			Name: "__agg_additions_to_estimator",
			Kind: MetricKindValue,
			Description: `How many unique metric-tag combinations were inserted into aggregation bucket.
Set by aggregator. Max(value)@host shows host responsible for most combinations, and is very order-dependent.`,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggHourCardinality: {
			Name: "__agg_hour_cardinality",
			Kind: MetricKindValue,
			Description: `Estimated unique metric-tag combinations per hour.
Linear interpolation between previous hour and value collected so far for this hour.
Steps of interpolation can be visible on graph.
Each aggregator writes value on every insert to particular second, multiplied by # of aggregator shards.
So avg() of this metric shows estimated full cardinality with or without grouping by aggregator.`,
			Resolution: 60,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "metric",
				IsMetric:    true,
			}},
			PreKeyTagID: "4",
		},
		BuiltinMetricIDAggSamplingFactor: {
			Name: BuiltinMetricNameAggSamplingFactor,
			Kind: MetricKindValue,
			Description: `Sample factor selected by aggregator.
Calculated by aggregator from scratch every second to fit all collected data into clickhouse insert budget.
Set only if greater than 1.`,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "-",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAggSamplingFactorReasonInsertSize: "insert_size",
				}),
			}},
			PreKeyTagID: "4",
		},
		BuiltinMetricIDIngestionStatus: {
			Name: BuiltinMetricNameIngestionStatus,
			Kind: MetricKindCounter,
			Description: `Status of receiving metrics by agent.
Most errors are due to various data format violation.
Some, like 'err_map_per_metric_queue_overload', 'err_map_tag_value', 'err_map_tag_value_cached' indicate tag mapping subsystem slowdowns or errors.
This metric uses sampling budgets of metric it refers to, so flooding by errors cannot affect other metrics.
'err_*_utf8'' statuses store original string value in hex.`,
			StringTopDescription: "string_value",
			Tags: []MetricMetaTag{{
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDSrcIngestionStatusOKCached:                     "ok_cached",
					TagValueIDSrcIngestionStatusOKUncached:                   "ok_uncached",
					TagValueIDSrcIngestionStatusErrMetricNotFound:            "err_metric_not_found",
					TagValueIDSrcIngestionStatusErrNanInfValue:               "err_nan_inf_value",
					TagValueIDSrcIngestionStatusErrNanInfCounter:             "err_nan_inf_counter",
					TagValueIDSrcIngestionStatusErrNegativeCounter:           "err_negative_counter",
					TagValueIDSrcIngestionStatusErrMapOther:                  "err_map_other",
					TagValueIDSrcIngestionStatusWarnMapTagNameNotFound:       "warn_tag_not_found",
					TagValueIDSrcIngestionStatusErrMapInvalidRawTagValue:     "err_map_invalid_raw_tag_value",
					TagValueIDSrcIngestionStatusErrMapTagValueCached:         "err_map_tag_value_cached",
					TagValueIDSrcIngestionStatusErrMapTagValue:               "err_map_tag_value",
					TagValueIDSrcIngestionStatusErrMapGlobalQueueOverload:    "err_map_global_queue_overload",
					TagValueIDSrcIngestionStatusErrMapPerMetricQueueOverload: "err_map_per_metric_queue_overload",
					TagValueIDSrcIngestionStatusErrMapTagValueEncoding:       "err_validate_tag_value_utf8",
					TagValueIDSrcIngestionStatusOKLegacy:                     "ok_legacy_protocol",
					TagValueIDSrcIngestionStatusErrMetricNonCanonical:        "non_canonical_name",
					TagValueIDSrcIngestionStatusErrMetricInvisible:           "err_metric_disabled",
					TagValueIDSrcIngestionStatusErrLegacyProtocol:            "err_legacy_protocol",
					TagValueIDSrcIngestionStatusWarnDeprecatedT:              "warn_deprecated_field_t",
					TagValueIDSrcIngestionStatusWarnDeprecatedStop:           "warn_deprecated_field_stop",
					TagValueIDSrcIngestionStatusWarnMapTagSetTwice:           "warn_map_tag_set_twice",
					TagValueIDSrcIngestionStatusWarnDeprecatedKeyName:        "warn_deprecated_tag_name",
					TagValueIDSrcIngestionStatusErrMetricNameEncoding:        "err_validate_metric_utf8",
					TagValueIDSrcIngestionStatusErrMapTagNameEncoding:        "err_validate_tag_name_utf8",
					TagValueIDSrcIngestionStatusErrValueUniqueBothSet:        "err_value_unique_both_set",
					TagValueIDSrcIngestionStatusWarnOldCounterSemantic:       "warn_deprecated_counter_semantic",
				}),
			}, {
				Description: "tag_id",
			}},
			PreKeyTagID: "1",
		},
		BuiltinMetricIDAggInsertTime: {
			Name:        "__agg_insert_time",
			Kind:        MetricKindValue,
			Description: "Time inserting this second into clickhouse took. Written when second is inserted, which can be much later.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDInsertTimeOK:    "ok",
					TagValueIDInsertTimeError: "error",
				}),
			}},
		},
		BuiltinMetricIDAggHistoricBucketsWaiting: {
			Name: "__agg_historic_buckets_waiting",
			Kind: MetricKindValue,
			Description: `Time difference of historic seconds (several per contributor) waiting to be inserted via historic conveyor.
Count is number of such seconds waiting.`,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}},
		},
		BuiltinMetricIDAggBucketAggregateTimeSec: {
			Name: "__agg_bucket_aggregate_time_sec",
			Kind: MetricKindValue,
			Description: `Time between agent bucket is received and fully aggregated into aggregator bucket.
Set by aggregator. Max(value)@host shows agent responsible for longest aggregation.`,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggActiveSenders: {
			Name:        "__agg_active_senders",
			Kind:        MetricKindValue,
			Description: "Number of insert lines between aggregator and clickhouse busy with insertion.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAgentDiskCacheErrors: {
			Name:        "__src_disc_cache_errors",
			Kind:        MetricKindCounter,
			Description: "Disk cache errors. Written by agent.",
			Tags: []MetricMetaTag{{
				Description: "kind",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDDiskCacheErrorWrite:             "err_write",
					TagValueIDDiskCacheErrorRead:              "err_read",
					TagValueIDDiskCacheErrorDelete:            "err_delete",
					TagValueIDDiskCacheErrorReadNotConfigured: "err_read_not_configured",
					TagValueIDDiskCacheErrorCompressFailed:    "err_compress",
				}),
			}},
		},
		BuiltinMetricIDTimingErrors: {
			Name: "__timing_errors",
			Kind: MetricKindValue,
			Description: `Timing errors - sending data too early or too late.
Set by either agent or aggregator, depending on status.`,
			Tags: []MetricMetaTag{{
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDTimingFutureBucketRecent:         "clock_future_recent",
					TagValueIDTimingFutureBucketHistoric:       "clock_future_historic",
					TagValueIDTimingLateRecent:                 "late_recent",
					TagValueIDTimingLongWindowThrownAgent:      "out_of_window_agent",
					TagValueIDTimingLongWindowThrownAggregator: "out_of_window_aggregator",
					TagValueIDTimingMissedSeconds:              "missed_seconds",
				}),
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAgentReceivedBatchSize: {
			Name:        "__src_ingested_metric_batch_size",
			Kind:        MetricKindValue,
			Description: "Size in bytes of metric batches received by agent.\nCount is # of such batches.",
			Tags: []MetricMetaTag{{
				Description:   "format",
				ValueComments: convertToValueComments(packetFormatToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAgentReceiveStatusOK:    "ok",
					TagValueIDAgentReceiveStatusError: "error",
				}),
			}},
		},
		BuiltinMetricIDAggMapping: {
			Name:        "__agg_mapping_status",
			Kind:        MetricKindCounter,
			Description: "Status of mapping on aggregator side.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "mapper",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAggMappingDolphinLegacy: "dolphin_legacy",
					TagValueIDAggMappingTags:          "client_pmc",
					TagValueIDAggMappingMetaMetrics:   "client_meta_metric",
					TagValueIDAggMappingJournalUpdate: "journal_update",
				}),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAggMappingStatusOKCached:     "ok_cached",
					TagValueIDAggMappingStatusOKUncached:   "ok_uncached",
					TagValueIDAggMappingStatusErrUncached:  "err_uncached",
					TagValueIDAggMappingStatusNotFound:     "not_found",
					TagValueIDAggMappingStatusImmediateOK:  "ok_immediate",
					TagValueIDAggMappingStatusImmediateErr: "err_immediate",
					TagValueIDAggMappingStatusEnqueued:     "enqueued",
					TagValueIDAggMappingStatusDelayedOK:    "ok_delayed",
					TagValueIDAggMappingStatusDelayedErr:   "err_delayed",
				}),
			}},
		},
		BuiltinMetricIDAggInsertTimeReal: {
			Name:        "__agg_insert_time_real",
			Kind:        MetricKindValue,
			Description: "Time of aggregated bucket inserting into clickhouse took in this second.\nactual seconds inserted can be from the past.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDInsertTimeOK:    "ok",
					TagValueIDInsertTimeError: "error",
				}),
			}},
		},
		BuiltinMetricIDAgentHistoricQueueSize: {
			Name:        "__src_historic_queue_size_bytes",
			Kind:        MetricKindValue,
			Description: "Historic queue size in memory and on disk.\nDisk size increases when second is written, decreases when file is deleted.",
			Tags: []MetricMetaTag{{
				Description: "storage",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDHistoricQueueMemory: "memory",
					TagValueIDHistoricQueueDisk:   "disk",
				}),
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDAggHistoricSecondsWaiting: {
			Name:        "__agg_historic_seconds_waiting",
			Kind:        MetricKindValue,
			Description: "Time difference of aggregated historic seconds waiting to be inserted via historic conveyor. Count is number of unique seconds waiting.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "aggregator_role",
				ValueComments: convertToValueComments(aggregatorRoleToValue),
			}},
		},
		BuiltinMetricIDAggInsertSizeReal: {
			Name:        "__agg_insert_size_real",
			Kind:        MetricKindValue,
			Description: "Size of aggregated bucket inserted into clickhouse in this second (actual seconds inserted can be from the past).",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description:   "conveyor",
				ValueComments: convertToValueComments(conveyorToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDInsertTimeOK:    "ok",
					TagValueIDInsertTimeError: "error",
				}),
			}},
		},
		BuiltinMetricIDAgentPerMetricSampleBudget: {
			Name:        "__src_per_metric_sample_budget_bytes",
			Kind:        MetricKindValue,
			Description: "Agent sampling budget per sampled metric, or remaining budget if none were sampled.",
			Tags: []MetricMetaTag{{
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAgentFirstSampledMetricBudgetPerMetric: "per_metric",
					TagValueIDAgentFirstSampledMetricBudgetUnused:    "remaining",
				}),
			}},
		},
		BuiltinMetricIDAgentMapping: {
			Name:        "__src_mapping_time",
			Kind:        MetricKindValue,
			Description: "Time and status of mapping request.\nWritten by agent.",
			Tags: []MetricMetaTag{{
				Description: "mapper",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAggMappingDolphinLegacy: "dolphin_legacy",
					TagValueIDAggMappingTags:          "pmc",
					TagValueIDAggMappingMetaMetrics:   "meta_metric",
				}),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAgentMappingStatusAllDead:   "error_all_dead",
					TagValueIDAgentMappingStatusOKFirst:   "ok_first",
					TagValueIDAgentMappingStatusOKSecond:  "ok_second",
					TagValueIDAgentMappingStatusErrSingle: "error_single_alive",
					TagValueIDAgentMappingStatusErrBoth:   "error_both_alive",
				}),
			}},
		},
		BuiltinMetricIDAgentReceivedPacketSize: {
			Name:        "__src_ingested_packet_size",
			Kind:        MetricKindValue,
			Description: "Size in bytes of packets received by agent. Also count is # of received packets.",
			Tags: []MetricMetaTag{{
				Description:   "format",
				ValueComments: convertToValueComments(packetFormatToValue),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAgentReceiveStatusOK:    "ok",
					TagValueIDAgentReceiveStatusError: "error",
				}),
			}},
		},
		BuiltinMetricIDAggMappingCreated: {
			Name: BuiltinMetricNameAggMappingCreated,
			Kind: MetricKindValue,
			Description: `Status of mapping string tags to integer values.
Value is actual integer value created (by incrementing global counter).
Set by aggregator.`,
			StringTopDescription: "Tag Values",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAggMappingCreatedStatusOK:                    "ok",
					TagValueIDAggMappingCreatedStatusCreated:               "created",
					TagValueIDAggMappingCreatedStatusFlood:                 "mapping_flood",
					TagValueIDAggMappingCreatedStatusErrorPMC:              "err_pmc",
					TagValueIDAggMappingCreatedStatusErrorInvariant:        "err_pmc_invariant",
					TagValueIDAggMappingCreatedStatusErrorNotAskedToCreate: "err_not_asked_to_create",
					TagValueIDAggMappingCreatedStatusErrorInvalidValue:     "err_invalid_value",
				}),
			}, {
				Description: "tag_id",
			}},
			PreKeyTagID: "4",
		},
		BuiltinMetricIDVersions: {
			Name:                 "__build_version",
			Kind:                 MetricKindCounter,
			Description:          "Build Version (commit) of statshouse components.",
			StringTopDescription: "Build Commit",
			Resolution:           60,
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "commit_date",
				Raw:         true,
			}, {
				Description: "commit_timestamp",
				RawKind:     "timestamp",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}},
		},
		BuiltinMetricIDBadges: {
			Name:        BuiltinMetricNameBadges,
			Kind:        MetricKindValue,
			Description: "System metric used to display UI badges above plot. Stores stripped copy of some other builtin metrics.",
			Resolution:  5,
			Tags: []MetricMetaTag{{
				Description: "badge",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDBadgeIngestionErrorsOld:  "ingestion_errors_legacy",
					TagValueIDBadgeAggMappingErrorsOld: "mapping_errors_legacy",
					TagValueIDBadgeAgentSamplingFactor: "agent_sampling_actor",
					TagValueIDBadgeAggSamplingFactor:   "aggregator_sampling_actor",
					TagValueIDBadgeIngestionErrors:     "ingestion_errors",
					TagValueIDBadgeAggMappingErrors:    "mapping_errors",
					TagValueIDBadgeContributors:        "contributors",
				}),
			}, {
				Description: "metric",
				IsMetric:    true,
			}},
			PreKeyTagID: "2",
		},
		BuiltinMetricIDAutoConfig: {
			Name:        "__autoconfig",
			Kind:        MetricKindCounter,
			Description: "Status of agent get config message, used to configure sharding on agents.\nSet by aggregator.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAutoConfigOK:             "ok_config",
					TagValueIDAutoConfigErrorSend:      "err_send",
					TagValueIDAutoConfigErrorKeepAlive: "err_keep_alive",
					TagValueIDAutoConfigWrongCluster:   "err_config_cluster",
				}),
			}, {
				Description: "shard_replica",
				Raw:         true,
			}, {
				Description: "total_shard_replicas",
				Raw:         true,
			}},
		},
		BuiltinMetricIDJournalVersions: {
			Name:                 "__metric_journal_version",
			Kind:                 MetricKindCounter,
			Description:          "Metadata journal version plus stable hash of journal state.",
			StringTopDescription: "Journal Hash",
			Resolution:           60,
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "version",
				Raw:         true,
			}},
		},
		BuiltinMetricIDPromScrapeTime: {
			Name:        BuiltinMetricNamePromScrapeTime,
			Kind:        MetricKindValue,
			Description: "Time of scraping prom metrics",
			Tags: []MetricMetaTag{
				{
					Description: "-",
				}, {
					Description: "job",
				}, {
					Description: "instance",
				}, {
					Description: "err",
					ValueComments: convertToValueComments(map[int32]string{
						TagValueIDScrapeError: "scrape",
					}),
				},
			},
		},
		BuiltinMetricIDUsageMemory: {
			Name:        BuiltinMetricNameUsageMemory,
			Kind:        MetricKindValue,
			Description: "Memory usage of statshouse components.",
			Resolution:  60,
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}},
		},
		BuiltinMetricIDUsageCPU: {
			Name:        BuiltinMetricNameUsageCPU,
			Kind:        MetricKindValue,
			Description: "CPU usage of statshouse components, CPU seconds per second.",
			Resolution:  60,
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "sys/user",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDCPUUsageSys:  "sys",
					TagValueIDCPUUsageUser: "user"}),
			}},
		},
		BuiltinMetricIDHeartbeatVersion: {
			Name:                 "__heartbeat_version",
			Kind:                 MetricKindValue,
			Description:          "Heartbeat value is uptime",
			StringTopDescription: "Build Commit",
			Resolution:           60,
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "event_type",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDHeartbeatEventStart:     "start",
					TagValueIDHeartbeatEventHeartbeat: "heartbeat"}),
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "commit_date",
				Raw:         true,
			}, {
				Description: "commit_timestamp",
				RawKind:     "timestamp",
			}, {
				Description: "host",
			}, {
				Description: "remote_ip",
				RawKind:     "ip",
			}},
		},
		BuiltinMetricIDHeartbeatArgs: createBuiltinMetricIDHeartbeatArgs("__heartbeat_args",
			"Commandline of statshouse components.\nIf too long, continued in __heartbeat_args2, __heartbeat_args3, __heartbeat_args4.\nHeartbeat value is uptime."),
		BuiltinMetricIDHeartbeatArgs2: createBuiltinMetricIDHeartbeatArgs("__heartbeat_args2",
			"Commandline of statshouse components.\nContinuation of __heartbeat_args.\nHeartbeat value is uptime."),
		BuiltinMetricIDHeartbeatArgs3: createBuiltinMetricIDHeartbeatArgs("__heartbeat_args3",
			"Commandline of statshouse components.\nContinuation of __heartbeat_args.\nHeartbeat value is uptime."),
		BuiltinMetricIDHeartbeatArgs4: createBuiltinMetricIDHeartbeatArgs("__heartbeat_args4",
			"Commandline of statshouse components.\nContinuation of __heartbeat_args.\nHeartbeat value is uptime."),
		BuiltinMetricIDGeneratorConstCounter: {
			Name:        "__fn_const_counter",
			Kind:        MetricKindCounter,
			Description: "Counter generated on the fly by constant function",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDGeneratorSinCounter: {
			Name:        "__fn_sin_counter",
			Kind:        MetricKindCounter,
			Description: "Test counter generated on the fly by sine function",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDAPIRPCServiceTime: {
			Name:        BuiltinMetricNameAPIRPCServiceTime,
			Kind:        MetricKindValue,
			Description: "Time to handle RPC query by API.",
			Tags: []MetricMetaTag{{
				Description: "methods",
			}, {
				Description: "error_code",
				Raw:         true,
			}, {
				Description: "token_name",
			}, {
				Description: "host",
			}},
		},
		BuiltinMetricIDAPIBRS: {
			Name:        BuiltinMetricNameAPIBRS,
			Kind:        MetricKindValue,
			Description: "Size of storage inside API of big response chunks",
			Tags: []MetricMetaTag{{
				Description: "host",
			}},
		},
		BuiltinMetricIDAPIEndpointResponseTime: {
			Name:        BuiltinMetricNameAPIEndpointResponseTime,
			Kind:        MetricKindValue,
			Description: "Time to handle and respond to HTTP query by API",
			Tags: []MetricMetaTag{{
				Description: "endpoint",
			}, {
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "http_code",
			}, {
				Description: "token_name",
			}, {
				Description: "data_format",
			}},
		},
		BuiltinMetricIDAPIEndpointServiceTime: {
			Name:        BuiltinMetricNameAPIEndpointServiceTime,
			Kind:        MetricKindValue,
			Description: "Time to handle HTTP query by API",
			Tags: []MetricMetaTag{{
				Description: "endpoint",
			}, {
				Description: "metric",
				IsMetric:    true,
			}, {
				Description: "http_code",
			}, {
				Description: "token_name",
			}, {
				Description: "data_format",
			}},
		},
		BuiltinMetricIDBudgetUnknownMetric: {
			Name:        BuiltinMetricNameBudgetUnknownMetric,
			Kind:        MetricKindCounter,
			Description: "Invisible metric used only for accounting budget to create mappings with metric not found",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDBudgetHost: {
			Name:        BuiltinMetricNameBudgetHost,
			Kind:        MetricKindCounter,
			Description: "Invisible metric used only for accounting budget to create host mappings",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDBudgetAggregatorHost: {
			Name:        BuiltinMetricNameBudgetAggregatorHost,
			Kind:        MetricKindCounter,
			Description: "Invisible metric used only for accounting budget to create host mappings of aggregators themselves",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDAPIActiveQueries: {
			Name:        BuiltinMetricNameAPIActiveQueries,
			Kind:        MetricKindValue,
			Description: "Active queries to clickhouse by API.\nRequests are assigned to lanes by estimated processing time.",
			Tags: []MetricMetaTag{{
				Description: "-",
			}, {
				Description: "version",
			}, {
				Description: "lane",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDAPILaneFast: "fast",
					TagValueIDAPILaneSlow: "slow"}),
			}, {
				Description: "host",
			}},
		},
		BuiltinMetricIDRPCRequests: {
			Name:        "__rpc_request_size",
			Kind:        MetricKindValue,
			Description: "Size of RPC request bodies.\nFor ingress proxy, key_id can be used to identify senders.",
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "tag",
				RawKind:     "hex",
				ValueComments: convertToValueComments(map[int32]string{
					0x3285ff57: "getConfig",
					0x4285ff57: "getConfig2",
					0x3285ff54: "getMetrics",
					0x4285ff54: "getMetrics2",
					0x42855554: "getMetrics3",
					0x3285ff56: "getTagMapping",
					0x4285ff56: "getTagMapping2",
					0x75a7f68e: "getTagMappingBootstrap",
					0x51df72a3: "getTargets",
					0x41df72a3: "getTargets2",
					0x3285ff53: "sendKeepAlive",
					0x4285ff53: "sendKeepAlive2",

					0x44575940: "sendSourceBucket2",
					0x3285ff58: "testConnection",
					0x4285ff58: "testConnection2",
					0x28bea524: "autoCreate",
				}),
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDRPCRequestsStatusOK:          "ok",
					TagValueIDRPCRequestsStatusErrLocal:    "err_local",
					TagValueIDRPCRequestsStatusErrUpstream: "err_upstream",
					TagValueIDRPCRequestsStatusHijack:      "hijacked"}),
			}, {
				Description: "-",
			}, {
				Description: "-",
			}, {
				Description: "key_id",
				RawKind:     "hex",
			}, {
				Description: "host",
			}},
		},
		BuiltinMetricIDContributorsLog: {
			Name: "__contributors_log",
			Kind: MetricKindCounter,
			Description: `Used to invalidate API caches.
Timestamps of all inserted seconds per second are recorded here.
To see which seconds change when, use __contributors_log_rev`,
			Tags: []MetricMetaTag{{
				Description: "timestamp",
				RawKind:     "timestamp",
			}},
		},
		BuiltinMetricIDContributorsLogRev: {
			Name:        "__contributors_log_rev",
			Kind:        MetricKindValue,
			Description: "Reverse index of __contributors_log, used to invalidate API caches.\nValue (and key1) is UNIX timestamp of second when this second was changed.",
			Tags: []MetricMetaTag{{
				Description: "insert_timestamp",
				RawKind:     "timestamp",
			}},
		},
		BuiltinMetricIDGeneratorGapsCounter: {
			Name:        "__fn_gaps_counter",
			Kind:        MetricKindCounter,
			Description: "Test counter with constant value, but with multiple gaps",
			Tags:        []MetricMetaTag{},
		},
		BuiltinMetricIDGroupSizeBeforeSampling: {
			Name:        "__group_size_before_sampling",
			Kind:        MetricKindValue,
			Description: "Group size before sampling, bytes.",
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "group",
				Raw:         true,
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDGroupSizeSamplingFit:     "fit",
					TagValueIDGroupSizeSamplingSampled: "sampled",
				}),
			}},
		},
		BuiltinMetricIDGroupSizeAfterSampling: {
			Name:        "__group_size_after_sampling",
			Kind:        MetricKindValue,
			Description: "Group size after sampling, bytes.",
			Tags: []MetricMetaTag{{
				Description:   "component",
				ValueComments: convertToValueComments(componentToValue),
			}, {
				Description: "group",
				Raw:         true,
			}, {
				Description: "status",
				ValueComments: convertToValueComments(map[int32]string{
					TagValueIDGroupSizeSamplingFit:     "fit",
					TagValueIDGroupSizeSamplingSampled: "sampled",
				}),
			}},
		},
	}

	BuiltinMetricByName           map[string]*MetricMetaValue
	BuiltinMetricAllowedToReceive map[string]*MetricMetaValue
)

Functions ¶

func AddRawValuePrefix ¶

func AddRawValuePrefix(s string) string

func AllowedResolution ¶

func AllowedResolution(r int) int

func AppendHexStringValue ¶

func AppendHexStringValue(dst []byte, src []byte) []byte

func AppendValidStringValue ¶

func AppendValidStringValue(dst []byte, src []byte) ([]byte, error)

func AppendValidStringValueLegacy ¶

func AppendValidStringValueLegacy(dst []byte, src []byte) ([]byte, error)

func CodeTagValue ¶

func CodeTagValue(code int32) string

func ContainsRawTagValue ¶

func ContainsRawTagValue(s mem.RO) (int32, bool)

We allow both signed and unsigned 32-bit values, however values outside both ranges are prohibited

func FilterBuildArch ¶

func FilterBuildArch(buildArch int32) int32

Limit build arch for built in-metrics collected by source We do not sample built-in metrics, hence should protect from cardinality explosion

func ForceValidStringValue ¶

func ForceValidStringValue(src string) []byte

We use this function only to validate host names. Can be slow, this is OK.

func HasRawValuePrefix ¶

func HasRawValuePrefix(s string) bool

func ISO8601Date2BuildDateKey ¶

func ISO8601Date2BuildDateKey(str string) int32

func IsValueCodeZero ¶

func IsValueCodeZero(s string) bool

func NewTagID ¶

func NewTagID(i int) string

func ParseCodeTagValue ¶

func ParseCodeTagValue(s string) (int32, error)

func ParseTagIDForAPI ¶

func ParseTagIDForAPI(tagID string) int

func TagID ¶

func TagID(i int) string

func TagIDTagToTagID ¶

func TagIDTagToTagID(tagIDTag int32) string

func ValidDashboardName ¶

func ValidDashboardName(s string) bool

func ValidFloatValue ¶

func ValidFloatValue(f float64) bool

func ValidGroupName ¶

func ValidGroupName(s string) bool

func ValidMetricKind ¶

func ValidMetricKind(kind string) bool

func ValidMetricName ¶

func ValidMetricName(s mem.RO) bool

func ValidRawKind ¶

func ValidRawKind(s string) bool

func ValidStringValue ¶

func ValidStringValue(s mem.RO) bool

Motivation - we want our tag values be 'good' and displayable in UI, copyable to edit boxes, etc. Our rules: tag must be UTF-8 string tag must not be longer than MaxStringLen tag contains no whitespaces to the left or to the right (trimmed) all whitespaces inside are ASCII spaces, and there must not be more than 1 consecutive space.

Often, scripts insert tags with extra spaces, new line, some trash, etc. We help people by replacing slightly invalid tag values with valid tag values. 1. trim 2. replace all consecutive whitespaces inside with single ASCII space 3. trim to MaxStringLen (if las utf symbol fits only partly, we remove it completely) 4. non-printable characters are replaced by roadsign (invalid rune) but invalid UTF-8 is still error

TODO - replace invalid UTF-8 with roadsigns as well to simplify rules

func ValidStringValueLegacy ¶

func ValidStringValueLegacy(s mem.RO) bool

Legacy rules replaced non-printables including whitespaces (except ASCII space) into roadsigns This was found to be not ideal set of rules, so they were changed

func ValidTagName ¶

func ValidTagName(s string) bool

func ValidTagValueForAPI ¶

func ValidTagValueForAPI(s string) bool

Types ¶

type CreateMappingExtra ¶

type CreateMappingExtra struct {
	Create    bool
	Metric    string
	TagIDKey  int32
	ClientEnv int32
	AgentEnv  int32
	Route     int32
	BuildArch int32
	HostName  string
	Host      int32
}

TODO - better place?

type DashboardMeta ¶

type DashboardMeta struct {
	DashboardID int32  `json:"dashboard_id"` // I'm sure day will come when we will be forced to make this int32
	Name        string `json:"name"`
	Version     int64  `json:"version,omitempty"`
	UpdateTime  uint32 `json:"update_time"`

	DeleteTime uint32                 `json:"delete_time"` // TODO - do not store event-specific information in journal event.
	JSONData   map[string]interface{} `json:"data"`        // TODO - there must be a better way?
}

func (DashboardMeta) MarshalBinary ¶

func (m DashboardMeta) MarshalBinary() ([]byte, error)

func (*DashboardMeta) UnmarshalBinary ¶

func (m *DashboardMeta) UnmarshalBinary(data []byte) error

type MetaStorageInterface ¶

type MetaStorageInterface interface {
	Version() int64
	StateHash() string
	GetMetaMetric(metricID int32) *MetricMetaValue
	GetMetaMetricByName(metricName string) *MetricMetaValue
	GetGroup(id int32) *MetricsGroup
}

type MetricKind ¶

type MetricKind int

type MetricMetaTag ¶

type MetricMetaTag struct {
	Name          string            `json:"name,omitempty"`
	Description   string            `json:"description,omitempty"`
	Raw           bool              `json:"raw,omitempty"`
	RawKind       string            `json:"raw_kind,omitempty"` // UI can show some raw values beautifully - timestamps, hex values, etc.
	ID2Value      map[int32]string  `json:"id2value,omitempty"`
	ValueComments map[string]string `json:"value_comments,omitempty"`

	Comment2Value map[string]string `json:"-"` // Should be restored from ValueComments after reading
	IsMetric      bool              `json:"-"` // Only for built-in metrics so never saved or parsed
	Index         int               `json:"-"` // Should be restored from position in MetricMetaValue.Tags
	LegacyName    bool              `json:"-"` // Set for "key0".."key15", "env", "skey" to generate ingestion warning
}

type MetricMetaValue ¶

type MetricMetaValue struct {
	MetricID   int32  `json:"metric_id"`
	Name       string `json:"name"`
	Version    int64  `json:"version,omitempty"`
	UpdateTime uint32 `json:"update_time"`

	Description          string          `json:"description,omitempty"`
	Tags                 []MetricMetaTag `json:"tags,omitempty"`
	Visible              bool            `json:"visible,omitempty"`
	Kind                 string          `json:"kind"`
	Weight               float64         `json:"weight,omitempty"`
	Resolution           int             `json:"resolution,omitempty"`             // no invariants
	StringTopName        string          `json:"string_top_name,omitempty"`        // no invariants
	StringTopDescription string          `json:"string_top_description,omitempty"` // no invariants
	PreKeyTagID          string          `json:"pre_key_tag_id,omitempty"`
	PreKeyFrom           uint32          `json:"pre_key_from,omitempty"`
	GroupID              int32           `json:"group_id"`

	RawTagMask          uint32                   `json:"-"` // Should be restored from Tags after reading
	Name2Tag            map[string]MetricMetaTag `json:"-"` // Should be restored from Tags after reading
	EffectiveResolution int                      `json:"-"` // Should be restored from Tags after reading
	PreKeyIndex         int                      `json:"-"` // index of tag which goes to 'prekey' column, or <0 if no tag goes
	EffectiveWeight     int64                    `json:"-"`
	HasPercentiles      bool                     `json:"-"`
	RoundSampleFactors  bool                     `json:"-"` // Experimental, set if magic word in description is found
}

func (MetricMetaValue) MarshalBinary ¶

func (m MetricMetaValue) MarshalBinary() ([]byte, error)

func (*MetricMetaValue) RestoreCachedInfo ¶

func (m *MetricMetaValue) RestoreCachedInfo() error

Always restores maximum info, if error is returned, metric is non-canonical and should not be saved

func (*MetricMetaValue) UnmarshalBinary ¶

func (m *MetricMetaValue) UnmarshalBinary(data []byte) error

type MetricMetaValueOld ¶

type MetricMetaValueOld struct {
	MetricID             int32           `json:"metric_id"`
	Name                 string          `json:"name"`
	Description          string          `json:"description,omitempty"`
	Tags                 []MetricMetaTag `json:"tags,omitempty"`
	Visible              bool            `json:"visible,omitempty"`
	Kind                 string          `json:"kind"`
	Weight               int64           `json:"weight,omitempty"`
	Resolution           int             `json:"resolution,omitempty"`             // no invariants
	StringTopName        string          `json:"string_top_name,omitempty"`        // no invariants
	StringTopDescription string          `json:"string_top_description,omitempty"` // no invariants
	PreKeyTagID          string          `json:"pre_key_tag_id,omitempty"`
	PreKeyFrom           uint32          `json:"pre_key_from,omitempty"`
	UpdateTime           uint32          `json:"update_time"`
	Version              int64           `json:"version,omitempty"`
}

type MetricsGroup ¶

type MetricsGroup struct {
	ID         int32  `json:"group_id"`
	Name       string `json:"name"`
	Version    int64  `json:"version,omitempty"`
	UpdateTime uint32 `json:"update_time"`
	DeleteTime uint32 `json:"delete_time"` // TODO - remove, make Visible flag, as in metrics

	Weight float64 `json:"weight"`

	EffectiveWeight int64 `json:"-"`
}

func (*MetricsGroup) RestoreCachedInfo ¶

func (m *MetricsGroup) RestoreCachedInfo() error

Always restores maximum info, if error is returned, metric is non-canonical and should not be saved

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL

Documentation ¶

Index ¶

Constants ¶

Variables ¶

Functions ¶

func AddRawValuePrefix ¶

func AllowedResolution ¶

func AppendHexStringValue ¶

func AppendValidStringValue ¶

func AppendValidStringValueLegacy ¶

func CodeTagValue ¶

func ContainsRawTagValue ¶

func FilterBuildArch ¶

func ForceValidStringValue ¶

func GetBuildArchKey ¶

func HasRawValuePrefix ¶

func ISO8601Date2BuildDateKey ¶

func IsValueCodeZero ¶

func NewTagID ¶

func ParseCodeTagValue ¶

func ParseTagIDForAPI ¶

func TagID ¶

func TagIDTagToTagID ¶

func ValidDashboardName ¶

func ValidFloatValue ¶

func ValidGroupName ¶

func ValidMetricKind ¶

func ValidMetricName ¶

func ValidRawKind ¶

func ValidStringValue ¶

func ValidStringValueLegacy ¶

func ValidTagName ¶

func ValidTagValueForAPI ¶

Types ¶

type CreateMappingExtra ¶

type DashboardMeta ¶

func (DashboardMeta) MarshalBinary ¶

func (*DashboardMeta) UnmarshalBinary ¶

type MetaStorageInterface ¶

type MetricKind ¶

type MetricMetaTag ¶

type MetricMetaValue ¶

func (MetricMetaValue) MarshalBinary ¶

func (*MetricMetaValue) RestoreCachedInfo ¶

func (*MetricMetaValue) UnmarshalBinary ¶

type MetricMetaValueOld ¶

type MetricsGroup ¶

func (*MetricsGroup) RestoreCachedInfo ¶

Source Files ¶