validation

package
v2.1.1-0...-c609277 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 30, 2021 License: AGPL-3.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// Local ingestion rate strategy
	LocalIngestionRateStrategy = "local"

	// Global ingestion rate strategy
	GlobalIngestionRateStrategy = "global"
)
View Source 
const (
	ReasonLabel = "reason"
	// InvalidLabels is a reason for discarding log lines which have labels that cannot be parsed.
	InvalidLabels = "invalid_labels"
	MissingLabels = "missing_labels"

	MissingLabelsErrorMsg = "error at least one label pair is required per stream"
	InvalidLabelsErrorMsg = "Error parsing labels '%s' with error: %s"
	// RateLimited is one of the values for the reason to discard samples.
	// Declared here to avoid duplication in ingester and distributor.
	RateLimited         = "rate_limited"
	RateLimitedErrorMsg = "" /* 200-byte string literal not displayed */
	// LineTooLong is a reason for discarding too long log lines.
	LineTooLong         = "line_too_long"
	LineTooLongErrorMsg = "Max entry size '%d' bytes exceeded for stream '%s' while adding an entry with length '%d' bytes"
	// StreamLimit is a reason for discarding lines when we can't create a new stream
	// because the limit of active streams has been reached.
	StreamLimit         = "stream_limit"
	StreamLimitErrorMsg = "" /* 185-byte string literal not displayed */
	// StreamRateLimit is a reason for discarding lines when the streams own rate limit is hit
	// rather than the overall ingestion rate limit.
	StreamRateLimit = "per_stream_rate_limit"
	OutOfOrder      = "out_of_order"
	// GreaterThanMaxSampleAge is a reason for discarding log lines which are older than the current time - `reject_old_samples_max_age`
	GreaterThanMaxSampleAge         = "greater_than_max_sample_age"
	GreaterThanMaxSampleAgeErrorMsg = "entry for stream '%s' has timestamp too old: %v"
	// TooFarInFuture is a reason for discarding log lines which are newer than the current time + `creation_grace_period`
	TooFarInFuture         = "too_far_in_future"
	TooFarInFutureErrorMsg = "entry for stream '%s' has timestamp too new: %v"
	// MaxLabelNamesPerSeries is a reason for discarding a log line which has too many label names
	MaxLabelNamesPerSeries         = "max_label_names_per_series"
	MaxLabelNamesPerSeriesErrorMsg = "entry for stream '%s' has %d label names; limit %d"
	// LabelNameTooLong is a reason for discarding a log line which has a label name too long
	LabelNameTooLong         = "label_name_too_long"
	LabelNameTooLongErrorMsg = "stream '%s' has label name too long: '%s'"
	// LabelValueTooLong is a reason for discarding a log line which has a lable value too long
	LabelValueTooLong         = "label_value_too_long"
	LabelValueTooLongErrorMsg = "stream '%s' has label value too long: '%s'"
	// DuplicateLabelNames is a reason for discarding a log line which has duplicate label names
	DuplicateLabelNames         = "duplicate_label_names"
	DuplicateLabelNamesErrorMsg = "stream '%s' has duplicate label name: '%s'"
)

Variables

View Source 
var DiscardedBytes = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "discarded_bytes_total",
		Help:      "The total number of bytes that were discarded.",
	},
	[]string{ReasonLabel, "tenant"},
)

DiscardedBytes is a metric of the total discarded bytes, by reason.

View Source 
var DiscardedSamples = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "discarded_samples_total",
		Help:      "The total number of samples that were discarded.",
	},
	[]string{ReasonLabel, "tenant"},
)

DiscardedSamples is a metric of the number of discarded samples, by reason.

View Source 
var MutatedBytes = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "mutated_bytes_total",
		Help:      "The total number of bytes that have been mutated.",
	},
	[]string{ReasonLabel, "truncated"},
)

MutatedBytes is a metric of the total mutated bytes, by reason.

View Source 
var MutatedSamples = prometheus.NewCounterVec(
	prometheus.CounterOpts{
		Namespace: "loki",
		Name:      "mutated_samples_total",
		Help:      "The total number of samples that have been mutated.",
	},
	[]string{ReasonLabel, "truncated"},
)

MutatedSamples is a metric of the total number of lines mutated, by reason.

View Source 
var Unlimited = RateLimit{
	Limit: rate.Inf,
	Burst: 0,
}

Functions

func SetDefaultLimitsForYAMLUnmarshalling 

func SetDefaultLimitsForYAMLUnmarshalling(defaults Limits)

SetDefaultLimitsForYAMLUnmarshalling sets global default limits, used when loading Limits from YAML files. This is used to ensure per-tenant limits are defaulted to those values.

Types

type ForEachTenantLimitCallback 

type ForEachTenantLimitCallback func(userID string, limit *Limits)

type Limits 

type Limits struct {
	// Distributor enforced limits.
	IngestionRateStrategy  string           `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"`
	IngestionRateMB        float64          `yaml:"ingestion_rate_mb" json:"ingestion_rate_mb"`
	IngestionBurstSizeMB   float64          `yaml:"ingestion_burst_size_mb" json:"ingestion_burst_size_mb"`
	MaxLabelNameLength     int              `yaml:"max_label_name_length" json:"max_label_name_length"`
	MaxLabelValueLength    int              `yaml:"max_label_value_length" json:"max_label_value_length"`
	MaxLabelNamesPerSeries int              `yaml:"max_label_names_per_series" json:"max_label_names_per_series"`
	RejectOldSamples       bool             `yaml:"reject_old_samples" json:"reject_old_samples"`
	RejectOldSamplesMaxAge model.Duration   `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"`
	CreationGracePeriod    model.Duration   `yaml:"creation_grace_period" json:"creation_grace_period"`
	EnforceMetricName      bool             `yaml:"enforce_metric_name" json:"enforce_metric_name"`
	MaxLineSize            flagext.ByteSize `yaml:"max_line_size" json:"max_line_size"`
	MaxLineSizeTruncate    bool             `yaml:"max_line_size_truncate" json:"max_line_size_truncate"`

	// Ingester enforced limits.
	MaxLocalStreamsPerUser       int              `yaml:"max_streams_per_user" json:"max_streams_per_user"`
	MaxGlobalStreamsPerUser      int              `yaml:"max_global_streams_per_user" json:"max_global_streams_per_user"`
	UnorderedWrites              bool             `yaml:"unordered_writes" json:"unordered_writes"`
	MaxLocalStreamRateBytes      flagext.ByteSize `yaml:"max_stream_rate_bytes" json:"max_stream_rate_bytes"`
	MaxLocalStreamBurstRateBytes flagext.ByteSize `yaml:"max_stream_burst_rate_bytes" json:"max_stream_burst_rate_bytes"`

	// Querier enforced limits.
	MaxChunksPerQuery          int            `yaml:"max_chunks_per_query" json:"max_chunks_per_query"`
	MaxQuerySeries             int            `yaml:"max_query_series" json:"max_query_series"`
	MaxQueryLookback           model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"`
	MaxQueryLength             model.Duration `yaml:"max_query_length" json:"max_query_length"`
	MaxQueryParallelism        int            `yaml:"max_query_parallelism" json:"max_query_parallelism"`
	CardinalityLimit           int            `yaml:"cardinality_limit" json:"cardinality_limit"`
	MaxStreamsMatchersPerQuery int            `yaml:"max_streams_matchers_per_query" json:"max_streams_matchers_per_query"`
	MaxConcurrentTailRequests  int            `yaml:"max_concurrent_tail_requests" json:"max_concurrent_tail_requests"`
	MaxEntriesLimitPerQuery    int            `yaml:"max_entries_limit_per_query" json:"max_entries_limit_per_query"`
	MaxCacheFreshness          model.Duration `yaml:"max_cache_freshness_per_query" json:"max_cache_freshness_per_query"`
	MaxQueriersPerTenant       int            `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"`

	// Query frontend enforced limits. The default is actually parameterized by the queryrange config.
	QuerySplitDuration  model.Duration `yaml:"split_queries_by_interval" json:"split_queries_by_interval"`
	MinShardingLookback model.Duration `yaml:"min_sharding_lookback" json:"min_sharding_lookback"`

	// Ruler defaults and limits.
	RulerEvaluationDelay          model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
	RulerMaxRulesPerRuleGroup     int            `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"`
	RulerMaxRuleGroupsPerTenant   int            `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"`
	RulerRemoteWriteQueueCapacity int            `yaml:"ruler_remote_write_queue_capacity" json:"ruler_remote_write_queue_capacity"`

	// Global and per tenant retention
	RetentionPeriod model.Duration    `yaml:"retention_period" json:"retention_period"`
	StreamRetention []StreamRetention `yaml:"retention_stream" json:"retention_stream"`

	// Config for overrides, convenient if it goes here.
	PerTenantOverrideConfig string         `yaml:"per_tenant_override_config" json:"per_tenant_override_config"`
	PerTenantOverridePeriod model.Duration `yaml:"per_tenant_override_period" json:"per_tenant_override_period"`
}

Limits describe all the limits for users; can be used to describe global default limits via flags, or per-user limits via yaml config. NOTE: we use custom `model.Duration` instead of standard `time.Duration` because, to support user-friendly duration format (e.g: "1h30m45s") in JSON value.

func (*Limits) RegisterFlags 

func (l *Limits) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*Limits) UnmarshalYAML 

func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*Limits) Validate 

func (l *Limits) Validate() error

Validate validates that this limits config is valid.

type Overrides 

type Overrides struct {
	// contains filtered or unexported fields
}

Overrides periodically fetch a set of per-user overrides, and provides convenience functions for fetching the correct value.

func NewOverrides 

func NewOverrides(defaults Limits, tenantLimits TenantLimits) (*Overrides, error)

NewOverrides makes a new Overrides.

func (*Overrides) CardinalityLimit 

func (o *Overrides) CardinalityLimit(userID string) int

CardinalityLimit whether to enforce the presence of a metric name.

func (*Overrides) CreationGracePeriod 

func (o *Overrides) CreationGracePeriod(userID string) time.Duration

CreationGracePeriod is misnamed, and actually returns how far into the future we should accept samples.

func (*Overrides) DefaultLimits 

func (o *Overrides) DefaultLimits() *Limits

func (*Overrides) EnforceMetricName 

func (o *Overrides) EnforceMetricName(userID string) bool

EnforceMetricName whether to enforce the presence of a metric name.

func (*Overrides) EvaluationDelay 

func (o *Overrides) EvaluationDelay(userID string) time.Duration

EvaluationDelay returns the rules evaluation delay for a given user.

func (*Overrides) ForEachTenantLimit 

func (o *Overrides) ForEachTenantLimit(callback ForEachTenantLimitCallback)

func (*Overrides) IngestionBurstSizeBytes 

func (o *Overrides) IngestionBurstSizeBytes(userID string) int

IngestionBurstSizeBytes returns the burst size for ingestion rate.

func (*Overrides) IngestionRateBytes 

func (o *Overrides) IngestionRateBytes(userID string) float64

IngestionRateBytes returns the limit on ingester rate (MBs per second).

func (*Overrides) IngestionRateStrategy 

func (o *Overrides) IngestionRateStrategy() string

IngestionRateStrategy returns whether the ingestion rate limit should be individually applied to each distributor instance (local) or evenly shared across the cluster (global).

func (*Overrides) MaxCacheFreshness 

func (o *Overrides) MaxCacheFreshness(userID string) time.Duration

func (*Overrides) MaxChunksPerQuery 

func (o *Overrides) MaxChunksPerQuery(userID string) int

MaxChunksPerQuery returns the maximum number of chunks allowed per query.

func (*Overrides) MaxChunksPerQueryFromStore 

func (o *Overrides) MaxChunksPerQueryFromStore(userID string) int

Compatibility with Cortex interface, this method is set to be removed in 1.12, so nooping in Loki until then.

func (*Overrides) MaxConcurrentTailRequests 

func (o *Overrides) MaxConcurrentTailRequests(userID string) int

MaxConcurrentTailRequests returns the limit to number of concurrent tail requests.

func (*Overrides) MaxEntriesLimitPerQuery 

func (o *Overrides) MaxEntriesLimitPerQuery(userID string) int

MaxEntriesLimitPerQuery returns the limit to number of entries the querier should return per query.

func (*Overrides) MaxGlobalStreamsPerUser 

func (o *Overrides) MaxGlobalStreamsPerUser(userID string) int

MaxGlobalStreamsPerUser returns the maximum number of streams a user is allowed to store across the cluster.

func (*Overrides) MaxLabelNameLength 

func (o *Overrides) MaxLabelNameLength(userID string) int

MaxLabelNameLength returns maximum length a label name can be.

func (*Overrides) MaxLabelNamesPerSeries 

func (o *Overrides) MaxLabelNamesPerSeries(userID string) int

MaxLabelNamesPerSeries returns maximum number of label/value pairs timeseries.

func (*Overrides) MaxLabelValueLength 

func (o *Overrides) MaxLabelValueLength(userID string) int

MaxLabelValueLength returns maximum length a label value can be. This also is the maximum length of a metric name.

func (*Overrides) MaxLineSize 

func (o *Overrides) MaxLineSize(userID string) int

MaxLineSize returns the maximum size in bytes the distributor should allow.

func (*Overrides) MaxLineSizeTruncate 

func (o *Overrides) MaxLineSizeTruncate(userID string) bool

MaxLineSizeShouldTruncate returns whether lines longer than max should be truncated.

func (*Overrides) MaxLocalStreamBurstRateBytes 

func (o *Overrides) MaxLocalStreamBurstRateBytes(userID string) int

func (*Overrides) MaxLocalStreamRateBytes 

func (o *Overrides) MaxLocalStreamRateBytes(userID string) int

func (*Overrides) MaxLocalStreamsPerUser 

func (o *Overrides) MaxLocalStreamsPerUser(userID string) int

MaxLocalStreamsPerUser returns the maximum number of streams a user is allowed to store in a single ingester.

func (*Overrides) MaxQueriersPerUser 

func (o *Overrides) MaxQueriersPerUser(userID string) int

MaxQueriersPerUser returns the maximum number of queriers that can handle requests for this user.

func (*Overrides) MaxQueryLength 

func (o *Overrides) MaxQueryLength(userID string) time.Duration

MaxQueryLength returns the limit of the length (in time) of a query.

func (*Overrides) MaxQueryLookback 

func (o *Overrides) MaxQueryLookback(userID string) time.Duration

MaxQueryLookback returns the max lookback period of queries.

func (*Overrides) MaxQueryParallelism 

func (o *Overrides) MaxQueryParallelism(userID string) int

MaxQueryParallelism returns the limit to the number of sub-queries the frontend will process in parallel.

func (*Overrides) MaxQuerySeries 

func (o *Overrides) MaxQuerySeries(userID string) int

MaxQueryLength returns the limit of the series of metric queries.

func (*Overrides) MaxStreamsMatchersPerQuery 

func (o *Overrides) MaxStreamsMatchersPerQuery(userID string) int

MaxStreamsMatchersPerQuery returns the limit to number of streams matchers per query.

func (*Overrides) MinShardingLookback 

func (o *Overrides) MinShardingLookback(userID string) time.Duration

MinShardingLookback returns the tenant specific min sharding lookback (e.g from when we should start sharding).

func (*Overrides) PerStreamRateLimit 

func (o *Overrides) PerStreamRateLimit(userID string) RateLimit

func (*Overrides) QuerySplitDuration 

func (o *Overrides) QuerySplitDuration(userID string) time.Duration

QuerySplitDuration returns the tenant specific splitby interval applied in the query frontend.

func (*Overrides) RejectOldSamples 

func (o *Overrides) RejectOldSamples(userID string) bool

RejectOldSamples returns true when we should reject samples older than certain age.

func (*Overrides) RejectOldSamplesMaxAge 

func (o *Overrides) RejectOldSamplesMaxAge(userID string) time.Duration

RejectOldSamplesMaxAge returns the age at which samples should be rejected.

func (*Overrides) RetentionPeriod 

func (o *Overrides) RetentionPeriod(userID string) time.Duration

RetentionPeriod returns the retention period for a given user.

func (*Overrides) RulerMaxRuleGroupsPerTenant 

func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int

RulerMaxRuleGroupsPerTenant returns the maximum number of rule groups for a given user.

func (*Overrides) RulerMaxRulesPerRuleGroup 

func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int

RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user.

func (*Overrides) RulerRemoteWriteQueueCapacity 

func (o *Overrides) RulerRemoteWriteQueueCapacity(userID string) int

RulerRemoteWriteQueueCapacity returns the remote-write queue capacity for a given user.

func (*Overrides) RulerTenantShardSize 

func (o *Overrides) RulerTenantShardSize(userID string) int

RulerTenantShardSize returns shard size (number of rulers) used by this tenant when using shuffle-sharding strategy. Not used in Loki.

func (*Overrides) StreamRetention 

func (o *Overrides) StreamRetention(userID string) []StreamRetention

StreamRetention returns the retention period for a given user.

func (*Overrides) UnorderedWrites 

func (o *Overrides) UnorderedWrites(userID string) bool

type RateLimit 

type RateLimit struct {
	Limit rate.Limit
	Burst int
}

RateLimit is a colocated limit & burst config. It largely exists to eliminate accidental misconfigurations due to race conditions when requesting the limit & burst config sequentially, between which the Limits configuration may have updated.

type StreamRetention 

type StreamRetention struct {
	Period   model.Duration    `yaml:"period" json:"period"`
	Priority int               `yaml:"priority" json:"priority"`
	Selector string            `yaml:"selector" json:"selector"`
	Matchers []*labels.Matcher `yaml:"-" json:"-"` // populated during validation.
}

type TenantLimits 

type TenantLimits interface {
	// TenantLimits is a function that returns limits for given tenant, or
	// nil, if there are no tenant-specific limits.
	TenantLimits(userID string) *Limits
	ForEachTenantLimit(ForEachTenantLimitCallback)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.