logql

package
v3.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 8, 2024 License: AGPL-3.0 Imports: 60 Imported by: 0

Documentation

Overview

Fork of https://raw.githubusercontent.com/cockroachdb/cockroach/065aa74206c9ec9bcd8b9ea2a6c62ddde8aab0a6/pkg/util/treeprinter/tree_printer.go

Index

Constants

View Source
const (
	StreamsKey = "streams"
	MetricsKey = "metrics"
)

expression type used in metrics

View Source
const (
	SuccessKey = "success"
	FailureKey = "failure"
	NoopKey    = "noop"
)

parsing evaluation result used in metrics

View Source
const (
	QueryTypeMetric  = "metric"
	QueryTypeFilter  = "filter"
	QueryTypeLimited = "limited"
	QueryTypeLabels  = "labels"
	QueryTypeSeries  = "series"
	QueryTypeStats   = "stats"
	QueryTypeShards  = "shards"
	QueryTypeVolume  = "volume"
)
View Source
const (
	ShardLastOverTime     = "last_over_time"
	ShardFirstOverTime    = "first_over_time"
	ShardQuantileOverTime = "quantile_over_time"
)
View Source
const (
	DefaultBlockedQueryMessage = "blocked by policy"
)
View Source
const MaxChildrenDisplay = 3

MaxChildrenDisplay defines the maximum number of children that should be shown by explain.

View Source
const (
	QuantileSketchMatrixType = "QuantileSketchMatrix"
)

Variables

View Source
var (
	QueryTime = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "logql",
		Name:      "query_duration_seconds",
		Help:      "LogQL query timings",
		Buckets:   prometheus.DefBuckets,
	}, []string{"query_type"})

	QueriesBlocked = promauto.NewCounterVec(prometheus.CounterOpts{
		Namespace: constants.Loki,
		Name:      "blocked_queries",
		Help:      "Count of queries blocked by per-tenant policy",
	}, []string{"user"})
)
View Source
var (
	NoLimits = &fakeLimits{maxSeries: math.MaxInt32}
)

Functions

func EvaluatorUnsupportedType

func EvaluatorUnsupportedType(expr syntax.Expr, ev EvaluatorFactory) error

EvaluatorUnsupportedType is a helper for signaling that an evaluator does not support an Expr type

func MatchForSeriesRequest

func MatchForSeriesRequest(xs []string) ([][]*labels.Matcher, error)

MatchForSeriesRequest extracts and parses multiple matcher groups from a slice of strings. Does not perform validation as it's used for series queries which allow empty matchers

func MergeQuantileSketchVector

func MergeQuantileSketchVector(next bool, r StepResult, stepEvaluator StepEvaluator, params Params) (promql_parser.Value, error)

MergeQuantileSketchVector joins the results from stepEvaluator into a ProbabilisticQuantileMatrix.

func ParseShard

func ParseShard(s string) (Shard, ShardVersion, error)

func ParseShards

func ParseShards(strs []string) (Shards, ShardVersion, error)

ParseShards parses a list of string encoded shards

func PopulateMatrixFromScalar

func PopulateMatrixFromScalar(data promql.Scalar, params Params) promql.Matrix

func PrintMatches

func PrintMatches(matches []string) string

func Quantile

func Quantile(q float64, values vector.HeapByMaxValue) float64

Quantile calculates the given Quantile of a vector of samples.

The Vector will be sorted. If 'values' has zero elements, NaN is returned. If q<0, -Inf is returned. If q>1, +Inf is returned.

func QueryType

func QueryType(expr syntax.Expr) (string, error)

func RecordDetectedFieldsQueryMetrics

func RecordDetectedFieldsQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, stats logql_stats.Result)

func RecordDetectedLabelsQueryMetrics

func RecordDetectedLabelsQueryMetrics(ctx context.Context, log log.Logger, start time.Time, end time.Time, query string, status string, stats logql_stats.Result)

func RecordLabelQueryMetrics

func RecordLabelQueryMetrics(
	ctx context.Context,
	log log.Logger,
	start, end time.Time,
	label, query, status string,
	stats logql_stats.Result,
)

func RecordRangeAndInstantQueryMetrics

func RecordRangeAndInstantQueryMetrics(
	ctx context.Context,
	log log.Logger,
	p Params,
	status string,
	stats logql_stats.Result,
	result promql_parser.Value,
)

func RecordSeriesQueryMetrics

func RecordSeriesQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, match []string, status string, shards []string, stats logql_stats.Result)

func RecordShardsQueryMetrics

func RecordShardsQueryMetrics(
	ctx context.Context,
	log log.Logger,
	start,
	end time.Time,
	query string,
	targetBytesPerShard uint64,
	status string,
	shards int,
	stats logql_stats.Result,
)

func RecordStatsQueryMetrics

func RecordStatsQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, status string, stats logql_stats.Result)

func RecordVolumeQueryMetrics

func RecordVolumeQueryMetrics(ctx context.Context, log log.Logger, start, end time.Time, query string, limit uint32, step time.Duration, status string, stats logql_stats.Result)

func ResultIterator

func ResultIterator(res logqlmodel.Result, direction logproto.Direction) (iter.EntryIterator, error)

ResultIterator coerces a downstream streams result into an iter.EntryIterator

func Sortable

func Sortable(q Params) (bool, error)

Sortable logql contain sort or sort_desc.

Types

type AbsentRangeVectorEvaluator

type AbsentRangeVectorEvaluator struct {
	// contains filtered or unexported fields
}

func (AbsentRangeVectorEvaluator) Close

func (AbsentRangeVectorEvaluator) Error

func (*AbsentRangeVectorEvaluator) Explain

func (e *AbsentRangeVectorEvaluator) Explain(parent Node)

func (*AbsentRangeVectorEvaluator) Next

type AccumulatedStreams

type AccumulatedStreams struct {
	// contains filtered or unexported fields
}

heap impl for keeping only the top n results across m streams importantly, AccumulatedStreams is _bounded_, so it will only store the top `limit` results across all streams. To implement this, we use a min-heap when looking for the max values (logproto.FORWARD) and vice versa for logproto.BACKWARD. This allows us to easily find the 'worst' value and replace it with a better one. Once we've fully processed all log lines, we return the heap in opposite order and then reverse it to get the correct order. Heap implements container/heap.Interface solely to use heap.Interface as a library. It is not intended for the heap pkg functions to otherwise call this type.

func NewStreamAccumulator

func NewStreamAccumulator(params Params) *AccumulatedStreams

NewStreamAccumulator returns an accumulator for limited log queries. Log queries, sharded thousands of times and each returning <limit> results, can be _considerably_ larger. In this case, we eagerly accumulate the results into a logsAccumulator, discarding values over the limit to keep memory pressure down while other subqueries are executing.

func (*AccumulatedStreams) Accumulate

func (acc *AccumulatedStreams) Accumulate(_ context.Context, x logqlmodel.Result, _ int) error

func (*AccumulatedStreams) Find

func (acc *AccumulatedStreams) Find(labels string) (int, bool)

func (*AccumulatedStreams) Len

func (acc *AccumulatedStreams) Len() int

number of streams

func (*AccumulatedStreams) Less

func (acc *AccumulatedStreams) Less(i, j int) bool

first order by timestamp, then by labels

func (*AccumulatedStreams) Pop

func (acc *AccumulatedStreams) Pop() any

Pop returns a stream with one entry. It pops the first entry of the first stream

func (*AccumulatedStreams) Push

func (acc *AccumulatedStreams) Push(x any)

func (*AccumulatedStreams) Result

func (acc *AccumulatedStreams) Result() []logqlmodel.Result

Note: can only be called once as it will alter stream ordreing.

func (*AccumulatedStreams) Swap

func (acc *AccumulatedStreams) Swap(i, j int)

type Accumulator

type Accumulator interface {
	Accumulate(context.Context, logqlmodel.Result, int) error
	Result() []logqlmodel.Result
}

Accumulator is an interface for accumulating query results.

type AvgOverTime

type AvgOverTime struct {
	// contains filtered or unexported fields
}

type BatchRangeVectorAggregator

type BatchRangeVectorAggregator func([]promql.FPoint) float64

BatchRangeVectorAggregator aggregates samples for a given range of samples. It receives the current milliseconds timestamp and the list of point within the range.

type BinOpStepEvaluator

type BinOpStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*BinOpStepEvaluator) Close

func (e *BinOpStepEvaluator) Close() (lastError error)

func (*BinOpStepEvaluator) Error

func (e *BinOpStepEvaluator) Error() error

func (*BinOpStepEvaluator) Explain

func (e *BinOpStepEvaluator) Explain(parent Node)

func (*BinOpStepEvaluator) Next

func (e *BinOpStepEvaluator) Next() (bool, int64, StepResult)

type BufferedAccumulator

type BufferedAccumulator struct {
	// contains filtered or unexported fields
}

func NewBufferedAccumulator

func NewBufferedAccumulator(n int) *BufferedAccumulator

NewBufferedAccumulator returns an accumulator which aggregates all query results in a slice. This is useful for metric queries, which are generally small payloads and the memory overhead for buffering is negligible.

func (*BufferedAccumulator) Accumulate

func (a *BufferedAccumulator) Accumulate(_ context.Context, acc logqlmodel.Result, i int) error

func (*BufferedAccumulator) Result

func (a *BufferedAccumulator) Result() []logqlmodel.Result

type ConcatLogSelectorExpr

type ConcatLogSelectorExpr struct {
	DownstreamLogSelectorExpr
	// contains filtered or unexported fields
}

ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr

func (*ConcatLogSelectorExpr) Pretty

func (c *ConcatLogSelectorExpr) Pretty(level int) string

ConcatLogSelectorExpr has no representation in LogQL. Its prettified version is e.g. `concat(downstream<{foo="bar"} |= "error", shard=1_of_3>)`

func (*ConcatLogSelectorExpr) String

func (c *ConcatLogSelectorExpr) String() string

type ConcatSampleExpr

type ConcatSampleExpr struct {
	DownstreamSampleExpr
	// contains filtered or unexported fields
}

ConcatSampleExpr is an expr for concatenating multiple SampleExpr Contract: The embedded SampleExprs within a linked list of ConcatSampleExprs must be of the same structure. This makes special implementations of SampleExpr.Associative() unnecessary.

func (*ConcatSampleExpr) Pretty

func (c *ConcatSampleExpr) Pretty(level int) string

ConcatSampleExpr has no LogQL repretenstation. It is expressed in in the prettified version as e.g. `concat(downstream<count_over_time({foo="bar"}), shard=...> ++ )`

func (*ConcatSampleExpr) String

func (c *ConcatSampleExpr) String() string

func (*ConcatSampleExpr) Walk

func (c *ConcatSampleExpr) Walk(f syntax.WalkFn)

type ConcatStepEvaluator

type ConcatStepEvaluator struct {
	// contains filtered or unexported fields
}

func NewConcatStepEvaluator

func NewConcatStepEvaluator(evaluators []StepEvaluator) *ConcatStepEvaluator

NewConcatStepEvaluator joins multiple StepEvaluators. Contract: They must be of identical start, end, and step values.

func (*ConcatStepEvaluator) Close

func (e *ConcatStepEvaluator) Close() (lastErr error)

func (*ConcatStepEvaluator) Error

func (e *ConcatStepEvaluator) Error() error

func (*ConcatStepEvaluator) Explain

func (e *ConcatStepEvaluator) Explain(parent Node)

func (*ConcatStepEvaluator) Next

func (e *ConcatStepEvaluator) Next() (bool, int64, StepResult)

type ConstantShards

type ConstantShards int

func (ConstantShards) GetStats

func (s ConstantShards) GetStats(_ syntax.Expr) (stats.Stats, error)

func (ConstantShards) ShardingRanges

func (s ConstantShards) ShardingRanges(_ syntax.Expr, _ uint64) ([]logproto.Shard, []logproto.ChunkRefGroup, error)

func (ConstantShards) Shards

func (s ConstantShards) Shards(_ syntax.Expr) (int, uint64, error)

type CountOverTime

type CountOverTime struct {
	// contains filtered or unexported fields
}

type DefaultEvaluator

type DefaultEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultEvaluator

func NewDefaultEvaluator(querier Querier, maxLookBackPeriod time.Duration) *DefaultEvaluator

NewDefaultEvaluator constructs a DefaultEvaluator

func (*DefaultEvaluator) NewIterator

func (*DefaultEvaluator) NewStepEvaluator

func (ev *DefaultEvaluator) NewStepEvaluator(
	ctx context.Context,
	nextEvFactory SampleEvaluatorFactory,
	expr syntax.SampleExpr,
	q Params,
) (StepEvaluator, error)

type DownstreamEngine

type DownstreamEngine struct {
	// contains filtered or unexported fields
}

DownstreamEngine is an Engine implementation that can split queries into more parallelizable forms via querying the underlying backend shards individually and re-aggregating them.

func NewDownstreamEngine

func NewDownstreamEngine(opts EngineOpts, downstreamable Downstreamable, limits Limits, logger log.Logger) *DownstreamEngine

NewDownstreamEngine constructs a *DownstreamEngine

func (*DownstreamEngine) Opts

func (ng *DownstreamEngine) Opts() EngineOpts

func (*DownstreamEngine) Query

func (ng *DownstreamEngine) Query(ctx context.Context, p Params) Query

Query constructs a Query

type DownstreamEvaluator

type DownstreamEvaluator struct {
	Downstreamer
	// contains filtered or unexported fields
}

DownstreamEvaluator is an evaluator which handles shard aware AST nodes

func NewDownstreamEvaluator

func NewDownstreamEvaluator(downstreamer Downstreamer) *DownstreamEvaluator

func (DownstreamEvaluator) Downstream

func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery, acc Accumulator) ([]logqlmodel.Result, error)

Downstream runs queries and collects stats from the embedded Downstreamer

func (*DownstreamEvaluator) NewIterator

func (ev *DownstreamEvaluator) NewIterator(
	ctx context.Context,
	expr syntax.LogSelectorExpr,
	params Params,
) (iter.EntryIterator, error)

NewIterator returns the iter.EntryIterator for a given LogSelectorExpr

func (*DownstreamEvaluator) NewStepEvaluator

func (ev *DownstreamEvaluator) NewStepEvaluator(
	ctx context.Context,
	nextEvFactory SampleEvaluatorFactory,
	expr syntax.SampleExpr,
	params Params,
) (StepEvaluator, error)

NewStepEvaluator returns a NewStepEvaluator for a given SampleExpr

type DownstreamLogSelectorExpr

type DownstreamLogSelectorExpr struct {
	syntax.LogSelectorExpr
	// contains filtered or unexported fields
}

DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation

func (DownstreamLogSelectorExpr) Pretty

func (d DownstreamLogSelectorExpr) Pretty(level int) string

The DownstreamLogSelectorExpr is not part of LogQL. In the prettified version it's represented as e.g. `downstream<{foo="bar"} |= "error", shard=1_of_3>`

func (DownstreamLogSelectorExpr) String

func (d DownstreamLogSelectorExpr) String() string

type DownstreamQuery

type DownstreamQuery struct {
	Params Params
}

type DownstreamSampleExpr

type DownstreamSampleExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

DownstreamSampleExpr is a SampleExpr which signals downstream computation

func (DownstreamSampleExpr) Pretty

func (d DownstreamSampleExpr) Pretty(level int) string

The DownstreamSampleExpr is not part of LogQL. In the prettified version it's represented as e.g. `downstream<count_over_time({foo="bar"} |= "error"), shard=1_of_3>`

func (DownstreamSampleExpr) String

func (d DownstreamSampleExpr) String() string

func (DownstreamSampleExpr) Walk

type Downstreamable

type Downstreamable interface {
	Downstreamer(context.Context) Downstreamer
}

type Downstreamer

type Downstreamer interface {
	Downstream(context.Context, []DownstreamQuery, Accumulator) ([]logqlmodel.Result, error)
}

Downstreamer is an interface for deferring responsibility for query execution. It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs.

type DynamicBoundsStrategy

type DynamicBoundsStrategy struct {
	// contains filtered or unexported fields
}

func NewDynamicBoundsStrategy

func NewDynamicBoundsStrategy(resolver ShardResolver, targetBytesPerShard uint64) DynamicBoundsStrategy

func (DynamicBoundsStrategy) Resolver

func (s DynamicBoundsStrategy) Resolver() ShardResolver

func (DynamicBoundsStrategy) Shards

type EmptyEvaluator added in v3.1.0

type EmptyEvaluator[R StepResult] struct {
	// contains filtered or unexported fields
}

func (EmptyEvaluator[_]) Close added in v3.1.0

func (EmptyEvaluator[_]) Close() error

Close implements StepEvaluator.

func (EmptyEvaluator[_]) Error added in v3.1.0

func (EmptyEvaluator[_]) Error() error

Error implements StepEvaluator.

func (EmptyEvaluator[SampleVector]) Explain added in v3.1.0

func (EmptyEvaluator[SampleVector]) Explain(parent Node)

func (EmptyEvaluator[_]) Next added in v3.1.0

func (e EmptyEvaluator[_]) Next() (ok bool, ts int64, r StepResult)

Next implements StepEvaluator.

type Engine

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the LogQL engine.

func NewEngine

func NewEngine(opts EngineOpts, q Querier, l Limits, logger log.Logger) *Engine

NewEngine creates a new LogQL Engine.

func (*Engine) Query

func (ng *Engine) Query(params Params) Query

Query creates a new LogQL query. Instant/Range type is derived from the parameters.

type EngineOpts

type EngineOpts struct {
	// MaxLookBackPeriod is the maximum amount of time to look back for log lines.
	// only used for instant log queries.
	MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"`

	// LogExecutingQuery will control if we log the query when Exec is called.
	LogExecutingQuery bool `yaml:"-"`
}

EngineOpts is the list of options to use with the LogQL query engine.

func (*EngineOpts) RegisterFlagsWithPrefix

func (opts *EngineOpts) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type EntryEvaluatorFactory

type EntryEvaluatorFactory interface {
	// NewIterator returns the iter.EntryIterator for a given LogSelectorExpr
	NewIterator(context.Context, syntax.LogSelectorExpr, Params) (iter.EntryIterator, error)
}

type EvaluatorFactory

type EvaluatorFactory interface {
	SampleEvaluatorFactory
	EntryEvaluatorFactory
}

EvaluatorFactory is an interface for iterating over data at different nodes in the AST

type FirstOverTime

type FirstOverTime struct {
	// contains filtered or unexported fields
}

type LabelReplaceEvaluator

type LabelReplaceEvaluator struct {
	// contains filtered or unexported fields
}

func (*LabelReplaceEvaluator) Close

func (e *LabelReplaceEvaluator) Close() error

func (*LabelReplaceEvaluator) Error

func (e *LabelReplaceEvaluator) Error() error

func (*LabelReplaceEvaluator) Explain

func (e *LabelReplaceEvaluator) Explain(parent Node)

func (*LabelReplaceEvaluator) Next

type LastOverTime

type LastOverTime struct {
	// contains filtered or unexported fields
}

type Limits

type Limits interface {
	MaxQuerySeries(context.Context, string) int
	MaxQueryRange(ctx context.Context, userID string) time.Duration
	QueryTimeout(context.Context, string) time.Duration
	BlockedQueries(context.Context, string) []*validation.BlockedQuery
}

Limits allow the engine to fetch limits for a given users.

type LiteralParams

type LiteralParams struct {
	// contains filtered or unexported fields
}

LiteralParams impls Params

func NewLiteralParams

func NewLiteralParams(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
	storeChunks *logproto.ChunkRefGroup,
) (LiteralParams, error)

func NewLiteralParamsWithCaching added in v3.1.0

func NewLiteralParamsWithCaching(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
	storeChunks *logproto.ChunkRefGroup,
	cachingOptions resultscache.CachingOptions,
) (LiteralParams, error)

func (LiteralParams) CachingOptions added in v3.1.0

func (p LiteralParams) CachingOptions() resultscache.CachingOptions

CachingOptions returns whether Loki query created from this params should be cached.

func (LiteralParams) Copy

func (p LiteralParams) Copy() LiteralParams

func (LiteralParams) Direction

func (p LiteralParams) Direction() logproto.Direction

Direction impls Params

func (LiteralParams) End

func (p LiteralParams) End() time.Time

End impls Params

func (LiteralParams) GetExpression

func (p LiteralParams) GetExpression() syntax.Expr

GetExpression impls Params

func (LiteralParams) GetStoreChunks added in v3.1.0

func (p LiteralParams) GetStoreChunks() *logproto.ChunkRefGroup

StoreChunks impls Params

func (LiteralParams) Interval

func (p LiteralParams) Interval() time.Duration

Interval impls Params

func (LiteralParams) Limit

func (p LiteralParams) Limit() uint32

Limit impls Params

func (LiteralParams) QueryString

func (p LiteralParams) QueryString() string

String impls Params

func (LiteralParams) Shards

func (p LiteralParams) Shards() []string

Shards impls Params

func (LiteralParams) Start

func (p LiteralParams) Start() time.Time

Start impls Params

func (LiteralParams) Step

func (p LiteralParams) Step() time.Duration

Step impls Params

type LiteralStepEvaluator

type LiteralStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*LiteralStepEvaluator) Close

func (e *LiteralStepEvaluator) Close() error

func (*LiteralStepEvaluator) Error

func (e *LiteralStepEvaluator) Error() error

func (*LiteralStepEvaluator) Explain

func (e *LiteralStepEvaluator) Explain(parent Node)

func (*LiteralStepEvaluator) Next

func (e *LiteralStepEvaluator) Next() (bool, int64, StepResult)

type MapperMetrics

type MapperMetrics struct {
	DownstreamQueries *prometheus.CounterVec // downstream queries total, partitioned by streams/metrics
	ParsedQueries     *prometheus.CounterVec // parsed ASTs total, partitioned by success/failure/noop
	DownstreamFactor  prometheus.Histogram   // per request downstream factor
}

MapperMetrics is the metrics wrapper used in logql mapping (shard and range)

func NewRangeMapperMetrics

func NewRangeMapperMetrics(registerer prometheus.Registerer) *MapperMetrics

func NewShardMapperMetrics

func NewShardMapperMetrics(registerer prometheus.Registerer) *MapperMetrics

type MapperStats

type MapperStats struct {
	// contains filtered or unexported fields
}

func NewMapperStats

func NewMapperStats() *MapperStats

func (*MapperStats) AddSplitQueries

func (s *MapperStats) AddSplitQueries(num int)

AddSplitQueries add num split queries to the counter

func (*MapperStats) GetSplitQueries

func (s *MapperStats) GetSplitQueries() int

GetSplitQueries returns the number of split queries

type MatrixStepEvaluator

type MatrixStepEvaluator struct {
	// contains filtered or unexported fields
}

MatrixStepEvaluator exposes a promql.Matrix as a StepEvaluator. Ensure that the resulting StepEvaluator maintains the same shape that the parameters expect. For example, it's possible that a downstream query returns matches no log streams and thus returns an empty matrix. However, we still need to ensure that it can be merged effectively with another leg that may match series. Therefore, we determine our steps from the parameters and not the underlying Matrix.

func NewMatrixStepEvaluator

func NewMatrixStepEvaluator(start, end time.Time, step time.Duration, m promql.Matrix) *MatrixStepEvaluator

func (*MatrixStepEvaluator) Close

func (m *MatrixStepEvaluator) Close() error

func (*MatrixStepEvaluator) Error

func (m *MatrixStepEvaluator) Error() error

func (*MatrixStepEvaluator) Explain

func (m *MatrixStepEvaluator) Explain(parent Node)

func (*MatrixStepEvaluator) Next

func (m *MatrixStepEvaluator) Next() (bool, int64, StepResult)

type MaxOverTime

type MaxOverTime struct {
	// contains filtered or unexported fields
}

type MergeFirstOverTimeExpr added in v3.1.0

type MergeFirstOverTimeExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

func (MergeFirstOverTimeExpr) String added in v3.1.0

func (e MergeFirstOverTimeExpr) String() string

func (*MergeFirstOverTimeExpr) Walk added in v3.1.0

type MergeLastOverTimeExpr added in v3.1.0

type MergeLastOverTimeExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

func (MergeLastOverTimeExpr) String added in v3.1.0

func (e MergeLastOverTimeExpr) String() string

func (*MergeLastOverTimeExpr) Walk added in v3.1.0

func (e *MergeLastOverTimeExpr) Walk(f syntax.WalkFn)

type MinOverTime

type MinOverTime struct {
	// contains filtered or unexported fields
}

type MockDownstreamer

type MockDownstreamer struct {
	*Engine
}

func (MockDownstreamer) Downstream

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery, _ Accumulator) ([]logqlmodel.Result, error)

func (MockDownstreamer) Downstreamer

func (m MockDownstreamer) Downstreamer(_ context.Context) Downstreamer

type MockQuerier

type MockQuerier struct {
	// contains filtered or unexported fields
}

Shard aware mock querier

func NewMockQuerier

func NewMockQuerier(shards int, streams []logproto.Stream) MockQuerier

func (MockQuerier) SelectLogs

func (MockQuerier) SelectSamples

type Node

type Node struct {
	// contains filtered or unexported fields
}

Node is a handle associated with a specific depth in a tree. See below for sample usage.

func NewTree

func NewTree() Node

NewTree creates a tree printer and returns a sentinel node reference which should be used to add the root. Sample usage:

tp := NewTree()
root := n.Child("root")
root.Child("child-1")
root.Child("child-2").Child("grandchild\ngrandchild-more-info")
root.Child("child-3")

fmt.Print(tp.String())

Output:

root
 ├── child-1
 ├── child-2
 │    └── grandchild
 │        grandchild-more-info
 └── child-3

Note that the Child calls can't be rearranged arbitrarily; they have to be in the order they need to be displayed (depth-first pre-order).

func (Node) AddEmptyLine

func (n Node) AddEmptyLine()

AddEmptyLine adds an empty line to the output; used to introduce vertical spacing as needed.

func (Node) AddLine

func (n Node) AddLine(v string)

AddLine adds a new line to a child node without an edge.

func (Node) Child

func (n Node) Child(text string) Node

Child adds a node as a child of the given node. Multi-line strings are supported with appropriate indentation.

func (Node) Childf

func (n Node) Childf(format string, args ...interface{}) Node

Childf adds a node as a child of the given node.

func (Node) FormattedRows

func (n Node) FormattedRows() []string

FormattedRows returns the formatted rows. Can only be called on the result of treeprinter.New.

func (Node) String

func (n Node) String() string

type OneOverTime

type OneOverTime struct {
}

type Params

type Params interface {
	QueryString() string
	Start() time.Time
	End() time.Time
	Step() time.Duration
	Interval() time.Duration
	Limit() uint32
	Direction() logproto.Direction
	Shards() []string
	GetExpression() syntax.Expr
	GetStoreChunks() *logproto.ChunkRefGroup
	CachingOptions() resultscache.CachingOptions
}

Params details the parameters associated with a loki request

func ParamOverridesFromShard added in v3.1.0

func ParamOverridesFromShard(base Params, shard *ShardWithChunkRefs) (result Params)

type ParamsWithChunkOverrides added in v3.1.0

type ParamsWithChunkOverrides struct {
	Params
	StoreChunksOverride *logproto.ChunkRefGroup
}

func (ParamsWithChunkOverrides) GetStoreChunks added in v3.1.0

func (p ParamsWithChunkOverrides) GetStoreChunks() *logproto.ChunkRefGroup

type ParamsWithExpressionOverride

type ParamsWithExpressionOverride struct {
	Params
	ExpressionOverride syntax.Expr
}

ParamsWithExpressionOverride overrides the query expression so that the query string and the expression can differ. This is useful for for query planning when plan my not match externally available logql syntax

func (ParamsWithExpressionOverride) GetExpression

func (p ParamsWithExpressionOverride) GetExpression() syntax.Expr

GetExpression returns the parsed expression of the query.

type ParamsWithShardsOverride

type ParamsWithShardsOverride struct {
	Params
	ShardsOverride []string
}

ParamsWithExpressionOverride overrides the shards. Since the backing implementation of the Params interface is unknown they are embedded and the original shards are shadowed.

func (ParamsWithShardsOverride) Shards

func (p ParamsWithShardsOverride) Shards() []string

Shards returns this overwriting shards.

type PowerOfTwoStrategy

type PowerOfTwoStrategy struct {
	// contains filtered or unexported fields
}

func NewPowerOfTwoStrategy

func NewPowerOfTwoStrategy(resolver ShardResolver) PowerOfTwoStrategy

func (PowerOfTwoStrategy) Resolver

func (s PowerOfTwoStrategy) Resolver() ShardResolver

func (PowerOfTwoStrategy) Shards

PowerOfTwo strategy does not support precomputed chunk refs

type ProbabilisticQuantileMatrix

type ProbabilisticQuantileMatrix []ProbabilisticQuantileVector

func (ProbabilisticQuantileMatrix) Merge

func (ProbabilisticQuantileMatrix) Release

func (m ProbabilisticQuantileMatrix) Release()

func (ProbabilisticQuantileMatrix) String

func (ProbabilisticQuantileMatrix) ToProto

func (ProbabilisticQuantileMatrix) Type

type ProbabilisticQuantileSample

type ProbabilisticQuantileSample struct {
	T int64
	F sketch.QuantileSketch

	Metric labels.Labels
}

func (ProbabilisticQuantileSample) ToProto

type ProbabilisticQuantileVector

type ProbabilisticQuantileVector []ProbabilisticQuantileSample

func (ProbabilisticQuantileVector) Merge

func (ProbabilisticQuantileVector) QuantileSketchVec

func (ProbabilisticQuantileVector) Release

func (q ProbabilisticQuantileVector) Release()

func (ProbabilisticQuantileVector) SampleVector

func (ProbabilisticQuantileVector) SampleVector() promql.Vector

func (ProbabilisticQuantileVector) ToProto

type QuantileOverTime

type QuantileOverTime struct {
	// contains filtered or unexported fields
}

type QuantileSketchAccumulator

type QuantileSketchAccumulator struct {
	// contains filtered or unexported fields
}

func (*QuantileSketchAccumulator) Accumulate

func (*QuantileSketchAccumulator) Result

type QuantileSketchEvalExpr

type QuantileSketchEvalExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

QuantileSketchEvalExpr evaluates a quantile sketch to the actual quantile.

func (QuantileSketchEvalExpr) String

func (e QuantileSketchEvalExpr) String() string

func (*QuantileSketchEvalExpr) Walk

type QuantileSketchMatrixStepEvaluator

type QuantileSketchMatrixStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchMatrixStepEvaluator steps through a matrix of quantile sketch vectors, ie t-digest or DDSketch structures per time step.

func (*QuantileSketchMatrixStepEvaluator) Close

func (*QuantileSketchMatrixStepEvaluator) Error

func (*QuantileSketchMatrixStepEvaluator) Explain

func (*QuantileSketchMatrixStepEvaluator) Explain(parent Node)

func (*QuantileSketchMatrixStepEvaluator) Next

type QuantileSketchMergeExpr

type QuantileSketchMergeExpr struct {
	syntax.SampleExpr
	// contains filtered or unexported fields
}

func (QuantileSketchMergeExpr) String

func (e QuantileSketchMergeExpr) String() string

func (*QuantileSketchMergeExpr) Walk

type QuantileSketchMergeStepEvaluator

type QuantileSketchMergeStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchMergeStepEvaluator merges multiple quantile sketches into one for each step.

func NewQuantileSketchMergeStepEvaluator

func NewQuantileSketchMergeStepEvaluator(evaluators []StepEvaluator) *QuantileSketchMergeStepEvaluator

func (*QuantileSketchMergeStepEvaluator) Close

func (*QuantileSketchMergeStepEvaluator) Error

func (*QuantileSketchMergeStepEvaluator) Explain

func (e *QuantileSketchMergeStepEvaluator) Explain(parent Node)

func (*QuantileSketchMergeStepEvaluator) Next

type QuantileSketchStepEvaluator

type QuantileSketchStepEvaluator struct {
	// contains filtered or unexported fields
}

func (*QuantileSketchStepEvaluator) Close

func (*QuantileSketchStepEvaluator) Error

func (*QuantileSketchStepEvaluator) Explain

func (e *QuantileSketchStepEvaluator) Explain(parent Node)

func (*QuantileSketchStepEvaluator) Next

type QuantileSketchVectorStepEvaluator

type QuantileSketchVectorStepEvaluator struct {
	// contains filtered or unexported fields
}

QuantileSketchVectorStepEvaluator evaluates a quantile sketch into a promql.Vector.

func NewQuantileSketchVectorStepEvaluator

func NewQuantileSketchVectorStepEvaluator(inner StepEvaluator, quantile float64) *QuantileSketchVectorStepEvaluator

func (*QuantileSketchVectorStepEvaluator) Close

func (*QuantileSketchVectorStepEvaluator) Error

func (*QuantileSketchVectorStepEvaluator) Explain

func (e *QuantileSketchVectorStepEvaluator) Explain(parent Node)

func (*QuantileSketchVectorStepEvaluator) Next

type Querier

type Querier interface {
	SelectLogs(context.Context, SelectLogParams) (iter.EntryIterator, error)
	SelectSamples(context.Context, SelectSampleParams) (iter.SampleIterator, error)
}

Querier allows a LogQL expression to fetch an EntryIterator for a set of matchers and filters

type Query

type Query interface {
	// Exec processes the query.
	Exec(ctx context.Context) (logqlmodel.Result, error)
}

Query is a LogQL query to be executed.

type QueryParams

type QueryParams interface {
	LogSelector() (syntax.LogSelectorExpr, error)
	GetStart() time.Time
	GetEnd() time.Time
	GetShards() []string
}

type QueryRangeType

type QueryRangeType string
var (
	InstantType QueryRangeType = "instant"
	RangeType   QueryRangeType = "range"
)

func GetRangeType

func GetRangeType(q Params) QueryRangeType

GetRangeType returns whether a query is an instant query or range query

type RangeMapper

type RangeMapper struct {
	// contains filtered or unexported fields
}

RangeMapper is used to rewrite LogQL sample expressions into multiple downstream sample expressions with a smaller time range that can be executed using the downstream engine.

A rewrite is performed using the following rules:

  1. Check if query is splittable based on the range.
  2. Check if the query is splittable based on the query AST
  3. Range aggregations are split into multiple downstream range aggregation expressions that are concatenated with an appropriate vector aggregator with a grouping operator. If the range aggregation has a grouping, the grouping is also applied to the resultant vector aggregator expression. If the range aggregation has no grouping, a grouping operator using "without" is applied to the resultant vector aggregator expression to preserve the stream labels.
  4. Vector aggregations are split into multiple downstream vector aggregations that are merged with vector aggregation using "without" and then aggregated using the vector aggregation with the same operator, either with or without grouping.
  5. Left and right-hand side of binary operations are split individually using the same rules as above.

func NewRangeMapper

func NewRangeMapper(interval time.Duration, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error)

NewRangeMapper creates a new RangeMapper instance with the given duration as split interval. The interval must be greater than 0.

func NewRangeMapperWithSplitAlign

func NewRangeMapperWithSplitAlign(interval time.Duration, splitAlign time.Time, metrics *MapperMetrics, stats *MapperStats) (RangeMapper, error)

NewRangeMapperWithSplitAlign is similar to `NewRangeMapper` except it accepts additional `splitAlign` argument and used to align the subqueries generated according to that. Look at `rangeSplitAlign` method for more information.

func (RangeMapper) Map

func (m RangeMapper) Map(expr syntax.SampleExpr, vectorAggrPushdown *syntax.VectorAggregationExpr, recorder *downstreamRecorder) (syntax.SampleExpr, error)

Map rewrites sample expression expr and returns the resultant sample expression to be executed by the downstream engine It is called recursively on the expression tree. The function takes an optional vector aggregation as second argument, that is pushed down to the downstream expression.

func (RangeMapper) Parse

func (m RangeMapper) Parse(expr syntax.Expr) (bool, syntax.Expr, error)

Parse parses the given LogQL query string into a sample expression and applies the rewrite rules for splitting it into a sample expression that can be executed by the downstream engine. It returns a boolean indicating whether a rewrite was possible, the rewritten sample expression, and an error in case the rewrite failed.

type RangeStreamingAgg

type RangeStreamingAgg interface {
	// contains filtered or unexported methods
}

RangeStreamingAgg streaming aggregates sample for each sample

type RangeVectorEvaluator

type RangeVectorEvaluator struct {
	// contains filtered or unexported fields
}

func (*RangeVectorEvaluator) Close

func (r *RangeVectorEvaluator) Close() error

func (*RangeVectorEvaluator) Error

func (r *RangeVectorEvaluator) Error() error

func (*RangeVectorEvaluator) Explain

func (r *RangeVectorEvaluator) Explain(parent Node)

func (*RangeVectorEvaluator) Next

func (r *RangeVectorEvaluator) Next() (bool, int64, StepResult)

type RangeVectorIterator

type RangeVectorIterator interface {
	Next() bool
	At() (int64, StepResult)
	Close() error
	Error() error
}

RangeVectorIterator iterates through a range of samples. To fetch the current vector use `At` with a `BatchRangeVectorAggregator` or `RangeStreamingAgg`.

type RateCounterOverTime

type RateCounterOverTime struct {
	// contains filtered or unexported fields
}

rateCounter calculates the per-second rate of values extracted from log lines and treat them like a "counter" metric.

type RateLogBytesOverTime

type RateLogBytesOverTime struct {
	// contains filtered or unexported fields
}

rateLogBytes calculates the per-second rate of log bytes.

type RateLogsOverTime

type RateLogsOverTime struct {
	// contains filtered or unexported fields
}

rateLogs calculates the per-second rate of log lines or values extracted from log lines

type Resp

type Resp struct {
	I   int
	Res logqlmodel.Result
	Err error
}

type SampleEvaluatorFactory

type SampleEvaluatorFactory interface {
	// NewStepEvaluator returns a NewStepEvaluator for a given SampleExpr. It's explicitly passed another NewStepEvaluator// in order to enable arbitrary computation of embedded expressions. This allows more modular & extensible
	// NewStepEvaluator implementations which can be composed.
	NewStepEvaluator(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)
}

type SampleEvaluatorFunc

type SampleEvaluatorFunc func(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

func (SampleEvaluatorFunc) NewStepEvaluator

func (s SampleEvaluatorFunc) NewStepEvaluator(ctx context.Context, nextEvaluatorFactory SampleEvaluatorFactory, expr syntax.SampleExpr, p Params) (StepEvaluator, error)

type SampleVector

type SampleVector promql.Vector

func (SampleVector) QuantileSketchVec

func (p SampleVector) QuantileSketchVec() ProbabilisticQuantileVector

func (SampleVector) SampleVector

func (p SampleVector) SampleVector() promql.Vector

type SelectLogParams

type SelectLogParams struct {
	*logproto.QueryRequest
}

SelectParams specifies parameters passed to data selections.

func (SelectLogParams) LogSelector

func (s SelectLogParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectLogParams) String

func (s SelectLogParams) String() string

func (SelectLogParams) WithStoreChunks added in v3.1.0

func (s SelectLogParams) WithStoreChunks(chunkRefGroup *logproto.ChunkRefGroup) SelectLogParams

type SelectSampleParams

type SelectSampleParams struct {
	*logproto.SampleQueryRequest
}

func (SelectSampleParams) Expr

Expr returns the SampleExpr from the SelectSampleParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) LogSelector

func (s SelectSampleParams) LogSelector() (syntax.LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) WithStoreChunks added in v3.1.0

func (s SelectSampleParams) WithStoreChunks(chunkRefGroup *logproto.ChunkRefGroup) SelectSampleParams

type Shard

type Shard struct {
	PowerOfTwo *index.ShardAnnotation
	Bounded    *logproto.Shard
}

Shard represents a shard annotation It holds either a power of two shard (legacy) or a bounded shard

func NewBoundedShard

func NewBoundedShard(shard logproto.Shard) Shard

func NewPowerOfTwoShard

func NewPowerOfTwoShard(shard index.ShardAnnotation) Shard

func (Shard) Bind added in v3.1.0

func (*Shard) GetFromThrough

func (s *Shard) GetFromThrough() (model.Fingerprint, model.Fingerprint)

func (*Shard) Match

func (s *Shard) Match(fp model.Fingerprint) bool

implement FingerprintFilter

func (Shard) Ptr

func (s Shard) Ptr() *Shard

convenience method for unaddressability concerns using constructors in literals (tests)

func (Shard) String

func (s Shard) String() string

func (*Shard) Variant

func (s *Shard) Variant() ShardVersion

type ShardMapper

type ShardMapper struct {
	// contains filtered or unexported fields
}

func NewShardMapper

func NewShardMapper(strategy ShardingStrategy, metrics *MapperMetrics, shardAggregation []string) ShardMapper

func (ShardMapper) Map

func (m ShardMapper) Map(expr syntax.Expr, r *downstreamRecorder, topLevel bool) (syntax.Expr, uint64, error)

func (ShardMapper) Parse

func (m ShardMapper) Parse(parsed syntax.Expr) (noop bool, bytesPerShard uint64, expr syntax.Expr, err error)

type ShardResolver

type ShardResolver interface {
	Shards(expr syntax.Expr) (int, uint64, error)
	// ShardingRanges returns shards and optionally a set of precomputed chunk refs for each group. If present,
	// they will be used in lieu of resolving chunk refs from the index durin evaluation.
	// If chunks are present, the number of shards returned must match the number of chunk ref groups.
	ShardingRanges(expr syntax.Expr, targetBytesPerShard uint64) ([]logproto.Shard, []logproto.ChunkRefGroup, error)
	GetStats(e syntax.Expr) (stats.Stats, error)
}

type ShardVersion

type ShardVersion uint8
const (
	PowerOfTwoVersion ShardVersion = iota
	BoundedVersion
)

TODO(owen-d): refactor this file. There's too many layers (sharding strategies, sharding resolvers). Eventually we should have a single strategy (bounded) and a single resolver (dynamic). It's likely this could be refactored anyway -- I was in a rush writing it the first time around.

func ParseShardVersion

func ParseShardVersion(s string) (ShardVersion, error)

func (ShardVersion) Strategy

func (v ShardVersion) Strategy(resolver ShardResolver, defaultTargetShardBytes uint64) ShardingStrategy

func (ShardVersion) String

func (v ShardVersion) String() string

type ShardWithChunkRefs added in v3.1.0

type ShardWithChunkRefs struct {
	Shard
	// contains filtered or unexported fields
}

ShardWithChunkRefs is a convenience type for passing around shards with associated chunk refs. The chunk refs are optional as determined by their contents (zero chunks means no precomputed refs) and are used to precompute chunk refs for each group

type ShardingStrategy

type ShardingStrategy interface {
	// The chunks for each shard are optional and are used to precompute chunk refs for each group
	Shards(expr syntax.Expr) (shards []ShardWithChunkRefs, maxBytesPerShard uint64, err error)
	Resolver() ShardResolver
}

type Shards

type Shards []Shard

func (Shards) Encode

func (xs Shards) Encode() (encoded []string)

type StddevOverTime

type StddevOverTime struct {
	// contains filtered or unexported fields
}

type StdvarOverTime

type StdvarOverTime struct {
	// contains filtered or unexported fields
}

type StepEvaluator

type StepEvaluator interface {
	// while Next returns a promql.Value, the only acceptable types are Scalar and Vector.
	Next() (ok bool, ts int64, r StepResult)
	// Close all resources used.
	Close() error
	// Reports any error
	Error() error
	// Explain returns a print of the step evaluation tree
	Explain(Node)
}

StepEvaluator evaluate a single step of a query.

func NewMergeFirstOverTimeStepEvaluator added in v3.1.0

func NewMergeFirstOverTimeStepEvaluator(params Params, m []promql.Matrix) StepEvaluator

func NewMergeLastOverTimeStepEvaluator added in v3.1.0

func NewMergeLastOverTimeStepEvaluator(params Params, m []promql.Matrix) StepEvaluator

func NewResultStepEvaluator

func NewResultStepEvaluator(res logqlmodel.Result, params Params) (StepEvaluator, error)

NewResultStepEvaluator coerces a downstream vector or matrix into a StepEvaluator

type StepResult

type StepResult interface {
	SampleVector() promql.Vector
	QuantileSketchVec() ProbabilisticQuantileVector
}

type SumOverTime

type SumOverTime struct {
	// contains filtered or unexported fields
}

type VectorAggEvaluator

type VectorAggEvaluator struct {
	// contains filtered or unexported fields
}

func (*VectorAggEvaluator) Close

func (e *VectorAggEvaluator) Close() error

func (*VectorAggEvaluator) Error

func (e *VectorAggEvaluator) Error() error

func (*VectorAggEvaluator) Explain

func (e *VectorAggEvaluator) Explain(parent Node)

func (*VectorAggEvaluator) Next

func (e *VectorAggEvaluator) Next() (bool, int64, StepResult)

type VectorIterator

type VectorIterator struct {
	// contains filtered or unexported fields
}

VectorIterator return simple vector like (1).

func (*VectorIterator) Close

func (r *VectorIterator) Close() error

func (*VectorIterator) Error

func (r *VectorIterator) Error() error

func (*VectorIterator) Explain

func (i *VectorIterator) Explain(parent Node)

func (*VectorIterator) Next

func (r *VectorIterator) Next() (bool, int64, StepResult)

type VectorStepEvaluator

type VectorStepEvaluator struct {
	// contains filtered or unexported fields
}

func NewVectorStepEvaluator

func NewVectorStepEvaluator(start time.Time, data promql.Vector) *VectorStepEvaluator

func (*VectorStepEvaluator) Close

func (e *VectorStepEvaluator) Close() error

func (*VectorStepEvaluator) Error

func (e *VectorStepEvaluator) Error() error

func (*VectorStepEvaluator) Explain

func (e *VectorStepEvaluator) Explain(parent Node)

func (*VectorStepEvaluator) Next

func (e *VectorStepEvaluator) Next() (bool, int64, StepResult)

Directories

Path Synopsis
log
logfmt
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
Adapted from https://github.com/go-logfmt/logfmt/ but []byte as parameter instead Original license is MIT.
This file is taken from the golang text/scanner package so `bufLen` can be set to `maxInputSize`.
This file is taken from the golang text/scanner package so `bufLen` can be set to `maxInputSize`.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL