logql

package
v1.6.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 24, 2020 License: Apache-2.0 Imports: 32 Imported by: 91

Documentation

Index

Constants

View Source 
const (
	// vector ops
	OpTypeSum     = "sum"
	OpTypeAvg     = "avg"
	OpTypeMax     = "max"
	OpTypeMin     = "min"
	OpTypeCount   = "count"
	OpTypeStddev  = "stddev"
	OpTypeStdvar  = "stdvar"
	OpTypeBottomK = "bottomk"
	OpTypeTopK    = "topk"

	// range vector ops
	OpRangeTypeCount     = "count_over_time"
	OpRangeTypeRate      = "rate"
	OpRangeTypeBytes     = "bytes_over_time"
	OpRangeTypeBytesRate = "bytes_rate"

	// binops - logical/set
	OpTypeOr     = "or"
	OpTypeAnd    = "and"
	OpTypeUnless = "unless"

	// binops - operations
	OpTypeAdd = "+"
	OpTypeSub = "-"
	OpTypeMul = "*"
	OpTypeDiv = "/"
	OpTypeMod = "%"
	OpTypePow = "^"

	// binops - comparison
	OpTypeCmpEQ = "=="
	OpTypeNEQ   = "!="
	OpTypeGT    = ">"
	OpTypeGTE   = ">="
	OpTypeLT    = "<"
	OpTypeLTE   = "<="
)
View Source 
const (
	QueryTypeMetric  = "metric"
	QueryTypeFilter  = "filter"
	QueryTypeLimited = "limited"
)
View Source 
const (
	StreamsKey = "streams"
	MetricsKey = "metrics"
	SuccessKey = "success"
	FailureKey = "failure"
	NoopKey    = "noop"
)

keys used in metrics

View Source 
const ADD = 57390
View Source 
const AND = 57382
View Source 
const AVG = 57370
View Source 
const BOOL = 57380
View Source 
const BOTTOMK = 57376
View Source 
const BY = 57365
View Source 
const BYTES_OVER_TIME = 57378
View Source 
const BYTES_RATE = 57379
View Source 
const CLOSE_BRACE = 57356
View Source 
const CLOSE_BRACKET = 57358
View Source 
const CLOSE_PARENTHESIS = 57364
View Source 
const CMP_EQ = 57384
View Source 
const COMMA = 57359
View Source 
const COUNT = 57373
View Source 
const COUNT_OVER_TIME = 57367
View Source 
const DIV = 57393
View Source 
const DOT = 57360
View Source 
const DURATION = 57349
View Source 
const EQ = 57352
View Source 
const GT = 57388
View Source 
const GTE = 57389
View Source 
const IDENTIFIER = 57346
View Source 
const LABELS = 57351
View Source 
const LT = 57386
View Source 
const LTE = 57387
View Source 
const MATCHERS = 57350
View Source 
const MAX = 57371
View Source 
const MIN = 57372
View Source 
const MOD = 57394
View Source 
const MUL = 57392
View Source 
const NEQ = 57385
View Source 
const NRE = 57354
View Source 
const NUMBER = 57348
View Source 
const OPEN_BRACE = 57355
View Source 
const OPEN_BRACKET = 57357
View Source 
const OPEN_PARENTHESIS = 57363
View Source 
const OR = 57381
View Source 
const PIPE_EXACT = 57362
View Source 
const PIPE_MATCH = 57361
View Source 
const POW = 57395
View Source 
const RATE = 57368
View Source 
const RE = 57353
View Source 
const STDDEV = 57374
View Source 
const STDVAR = 57375
View Source 
const STRING = 57347
View Source 
const SUB = 57391
View Source 
const SUM = 57369
View Source 
const TOPK = 57377
View Source 
const UNLESS = 57383
View Source 
const ValueTypeStreams = "streams"

ValueTypeStreams promql.ValueType for log streams

View Source 
const WITHOUT = 57366

Variables

View Source 
var (
	ExtractBytes = bytesSampleExtractor{}
	ExtractCount = countSampleExtractor{}
)
View Source 
var TrueFilter = trueFilter{}

TrueFilter is a filter that returns and matches all log lines whatever their content.

Functions

func EvaluatorUnsupportedType added in v1.5.0 

func EvaluatorUnsupportedType(expr Expr, ev Evaluator) error

EvaluatorUnsupportedType is a helper for signaling that an evaluator does not support an Expr type

func IsComparisonOperator added in v1.6.0 

func IsComparisonOperator(op string) bool

func IsLogicalBinOp added in v1.4.0 

func IsLogicalBinOp(op string) bool

IsLogicalBinOp tests whether an operation is a logical/set binary operation

func IsParseError added in v1.4.0 

func IsParseError(err error) bool

IsParseError returns true if the err is a ast parsing error.

func ParseMatchers 

func ParseMatchers(input string) ([]*labels.Matcher, error)

ParseMatchers parses a string and returns labels matchers, if the expression contains anything else it will return an error.

func PopulateMatrixFromScalar added in v1.4.0 

func PopulateMatrixFromScalar(data promql.Scalar, params Params) promql.Matrix

func QueryType added in v1.4.0 

func QueryType(query string) (string, error)

func RecordMetrics added in v1.4.0 

func RecordMetrics(ctx context.Context, p Params, status string, stats stats.Result)

func ResultIterator added in v1.6.0 

func ResultIterator(res Result, params Params) (iter.EntryIterator, error)

ResultIterator coerces a downstream streams result into an iter.EntryIterator

Types

type BinOpOptions added in v1.6.0 

type BinOpOptions struct {
	ReturnBool bool
}

type ConcatLogSelectorExpr added in v1.5.0 

type ConcatLogSelectorExpr struct {
	DownstreamLogSelectorExpr
	// contains filtered or unexported fields
}

ConcatLogSelectorExpr is an expr for concatenating multiple LogSelectorExpr

func (ConcatLogSelectorExpr) String added in v1.5.0 

func (c ConcatLogSelectorExpr) String() string

type ConcatSampleExpr added in v1.5.0 

type ConcatSampleExpr struct {
	DownstreamSampleExpr
	// contains filtered or unexported fields
}

ConcatSampleExpr is an expr for concatenating multiple SampleExpr Contract: The embedded SampleExprs within a linked list of ConcatSampleExprs must be of the same structure. This makes special implementations of SampleExpr.Associative() unnecessary.

func (ConcatSampleExpr) String added in v1.5.0 

func (c ConcatSampleExpr) String() string

type DefaultEvaluator added in v1.6.0 

type DefaultEvaluator struct {
	// contains filtered or unexported fields
}

func NewDefaultEvaluator added in v1.6.0 

func NewDefaultEvaluator(querier Querier, maxLookBackPeriod time.Duration) *DefaultEvaluator

NewDefaultEvaluator constructs a DefaultEvaluator

func (*DefaultEvaluator) Iterator added in v1.6.0 

func (ev *DefaultEvaluator) Iterator(ctx context.Context, expr LogSelectorExpr, q Params) (iter.EntryIterator, error)

func (*DefaultEvaluator) StepEvaluator added in v1.6.0 

func (ev *DefaultEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv Evaluator,
	expr SampleExpr,
	q Params,
) (StepEvaluator, error)

type DownstreamEvaluator added in v1.6.0 

type DownstreamEvaluator struct {
	Downstreamer
	// contains filtered or unexported fields
}

DownstreamEvaluator is an evaluator which handles shard aware AST nodes

func NewDownstreamEvaluator added in v1.6.0 

func NewDownstreamEvaluator(downstreamer Downstreamer) *DownstreamEvaluator

func (DownstreamEvaluator) Downstream added in v1.6.0 

func (ev DownstreamEvaluator) Downstream(ctx context.Context, queries []DownstreamQuery) ([]Result, error)

Downstream runs queries and collects stats from the embedded Downstreamer

func (*DownstreamEvaluator) Iterator added in v1.6.0 

func (ev *DownstreamEvaluator) Iterator(
	ctx context.Context,
	expr LogSelectorExpr,
	params Params,
) (iter.EntryIterator, error)

Iterator returns the iter.EntryIterator for a given LogSelectorExpr

func (*DownstreamEvaluator) StepEvaluator added in v1.6.0 

func (ev *DownstreamEvaluator) StepEvaluator(
	ctx context.Context,
	nextEv Evaluator,
	expr SampleExpr,
	params Params,
) (StepEvaluator, error)

Evaluator returns a StepEvaluator for a given SampleExpr

type DownstreamLogSelectorExpr added in v1.5.0 

type DownstreamLogSelectorExpr struct {
	LogSelectorExpr
	// contains filtered or unexported fields
}

DownstreamLogSelectorExpr is a LogSelectorExpr which signals downstream computation

func (DownstreamLogSelectorExpr) String added in v1.5.0 

func (d DownstreamLogSelectorExpr) String() string

type DownstreamQuery added in v1.6.0 

type DownstreamQuery struct {
	Expr   Expr
	Params Params
	Shards Shards
}

type DownstreamSampleExpr added in v1.5.0 

type DownstreamSampleExpr struct {
	SampleExpr
	// contains filtered or unexported fields
}

DownstreamSampleExpr is a SampleExpr which signals downstream computation

func (DownstreamSampleExpr) String added in v1.5.0 

func (d DownstreamSampleExpr) String() string

type Downstreamable added in v1.6.0 

type Downstreamable interface {
	Downstreamer() Downstreamer
}

type Downstreamer added in v1.6.0 

type Downstreamer interface {
	Downstream(context.Context, []DownstreamQuery) ([]Result, error)
}

Downstreamer is an interface for deferring responsibility for query execution. It is decoupled from but consumed by a downStreamEvaluator to dispatch ASTs.

type Engine added in v0.4.0 

type Engine struct {
	// contains filtered or unexported fields
}

Engine is the LogQL engine.

func NewEngine added in v0.4.0 

func NewEngine(opts EngineOpts, q Querier) *Engine

NewEngine creates a new LogQL Engine.

func (*Engine) Query added in v1.6.0 

func (ng *Engine) Query(params Params) Query

Query creates a new LogQL query. Instant/Range type is derived from the parameters.

type EngineOpts added in v0.4.0 

type EngineOpts struct {
	// Timeout for queries execution
	Timeout time.Duration `yaml:"timeout"`
	// MaxLookBackPeriod is the maximum amount of time to look back for log lines.
	// only used for instant log queries.
	MaxLookBackPeriod time.Duration `yaml:"max_look_back_period"`
}

EngineOpts is the list of options to use with the LogQL query engine.

func (*EngineOpts) RegisterFlagsWithPrefix added in v1.6.0 

func (opts *EngineOpts) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

type Evaluator added in v1.4.0 

type Evaluator interface {
	// StepEvaluator returns a StepEvaluator for a given SampleExpr. It's explicitly passed another StepEvaluator// in order to enable arbitrary computation of embedded expressions. This allows more modular & extensible
	// StepEvaluator implementations which can be composed.
	StepEvaluator(ctx context.Context, nextEvaluator Evaluator, expr SampleExpr, p Params) (StepEvaluator, error)
	// Iterator returns the iter.EntryIterator for a given LogSelectorExpr
	Iterator(context.Context, LogSelectorExpr, Params) (iter.EntryIterator, error)
}

Evaluator is an interface for iterating over data at different nodes in the AST

type Expr 

type Expr interface {
	fmt.Stringer
	// contains filtered or unexported methods
}

Expr is the root expression which can be a SampleExpr or LogSelectorExpr

func ParseExpr 

func ParseExpr(input string) (expr Expr, err error)

ParseExpr parses a string and returns an Expr.

type LineFilter added in v1.4.0 

type LineFilter interface {
	Filter(line []byte) bool
}

LineFilter is a interface to filter log lines.

type LineFilterFunc added in v1.4.0 

type LineFilterFunc func(line []byte) bool

LineFilterFunc is a syntax sugar for creating line filter from a function

func (LineFilterFunc) Filter added in v1.4.0 

func (f LineFilterFunc) Filter(line []byte) bool

type LiteralParams added in v1.4.0 

type LiteralParams struct {
	// contains filtered or unexported fields
}

LiteralParams impls Params

func NewLiteralParams added in v1.6.0 

func NewLiteralParams(
	qs string,
	start, end time.Time,
	step, interval time.Duration,
	direction logproto.Direction,
	limit uint32,
	shards []string,
) LiteralParams

func (LiteralParams) Copy added in v1.6.0 

func (p LiteralParams) Copy() LiteralParams

func (LiteralParams) Direction added in v1.4.0 

func (p LiteralParams) Direction() logproto.Direction

Direction impls Params

func (LiteralParams) End added in v1.4.0 

func (p LiteralParams) End() time.Time

End impls Params

func (LiteralParams) Interval added in v1.6.0 

func (p LiteralParams) Interval() time.Duration

Interval impls Params

func (LiteralParams) Limit added in v1.4.0 

func (p LiteralParams) Limit() uint32

Limit impls Params

func (LiteralParams) Query added in v1.4.0 

func (p LiteralParams) Query() string

String impls Params

func (LiteralParams) Shards added in v1.6.0 

func (p LiteralParams) Shards() []string

Shards impls Params

func (LiteralParams) Start added in v1.4.0 

func (p LiteralParams) Start() time.Time

Start impls Params

func (LiteralParams) Step added in v1.4.0 

func (p LiteralParams) Step() time.Duration

Step impls Params

type LogSelectorExpr added in v0.4.0 

type LogSelectorExpr interface {
	Filter() (LineFilter, error)
	Matchers() []*labels.Matcher
	Expr
}

LogSelectorExpr is a LogQL expression filtering and returning logs.

func NewFilterExpr 

func NewFilterExpr(left LogSelectorExpr, ty labels.MatchType, match string) LogSelectorExpr

NewFilterExpr wraps an existing Expr with a next filter expression.

func ParseLogSelector added in v0.4.0 

func ParseLogSelector(input string) (LogSelectorExpr, error)

ParseLogSelector parses a log selector expression `{app="foo"} |= "filter"`

type MatrixStepper added in v1.6.0 

type MatrixStepper struct {
	// contains filtered or unexported fields
}

MatrixStepper exposes a promql.Matrix as a StepEvaluator. Ensure that the resulting StepEvaluator maintains the same shape that the parameters expect. For example, it's possible that a downstream query returns matches no log streams and thus returns an empty matrix. However, we still need to ensure that it can be merged effectively with another leg that may match series. Therefore, we determine our steps from the parameters and not the underlying Matrix.

func NewMatrixStepper added in v1.6.0 

func NewMatrixStepper(start, end time.Time, step time.Duration, m promql.Matrix) *MatrixStepper

func (*MatrixStepper) Close added in v1.6.0 

func (m *MatrixStepper) Close() error

func (*MatrixStepper) Error added in v1.6.0 

func (m *MatrixStepper) Error() error

func (*MatrixStepper) Next added in v1.6.0 

func (m *MatrixStepper) Next() (bool, int64, promql.Vector)

type MockDownstreamer added in v1.6.0 

type MockDownstreamer struct {
	*Engine
}

func (MockDownstreamer) Downstream added in v1.6.0 

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery) ([]Result, error)

func (MockDownstreamer) Downstreamer added in v1.6.0 

func (m MockDownstreamer) Downstreamer() Downstreamer

type MockQuerier added in v1.6.0 

type MockQuerier struct {
	// contains filtered or unexported fields
}

Shard aware mock querier

func NewMockQuerier added in v1.6.0 

func NewMockQuerier(shards int, streams []logproto.Stream) MockQuerier

func (MockQuerier) SelectLogs added in v1.6.0 

func (q MockQuerier) SelectLogs(ctx context.Context, req SelectLogParams) (iter.EntryIterator, error)

func (MockQuerier) SelectSamples added in v1.6.0 

func (q MockQuerier) SelectSamples(ctx context.Context, req SelectSampleParams) (iter.SampleIterator, error)

type Params added in v1.4.0 

type Params interface {
	Query() string
	Start() time.Time
	End() time.Time
	Step() time.Duration
	Interval() time.Duration
	Limit() uint32
	Direction() logproto.Direction
	Shards() []string
}

Params details the parameters associated with a loki request

type ParseError 

type ParseError struct {
	// contains filtered or unexported fields
}

ParseError is what is returned when we failed to parse.

func (ParseError) Error 

func (p ParseError) Error() string

type Querier 

type Querier interface {
	SelectLogs(context.Context, SelectLogParams) (iter.EntryIterator, error)
	SelectSamples(context.Context, SelectSampleParams) (iter.SampleIterator, error)
}

Querier allows a LogQL expression to fetch an EntryIterator for a set of matchers and filters

type Query added in v0.4.0 

type Query interface {
	// Exec processes the query.
	Exec(ctx context.Context) (Result, error)
}

Query is a LogQL query to be executed.

type QueryParams added in v1.6.0 

type QueryParams interface {
	LogSelector() (LogSelectorExpr, error)
	GetStart() time.Time
	GetEnd() time.Time
	GetShards() []string
}

type QueryRangeType added in v1.4.0 

type QueryRangeType string
var (
	InstantType QueryRangeType = "instant"
	RangeType   QueryRangeType = "range"
)

func GetRangeType added in v1.4.0 

func GetRangeType(q Params) QueryRangeType

GetRangeType returns whether a query is an instant query or range query

type RangeVectorAggregator added in v0.4.0 

type RangeVectorAggregator func([]promql.Point) float64

RangeVectorAggregator aggregates samples for a given range of samples. It receives the current milliseconds timestamp and the list of point within the range.

type RangeVectorIterator added in v0.4.0 

type RangeVectorIterator interface {
	Next() bool
	At(aggregator RangeVectorAggregator) (int64, promql.Vector)
	Close() error
	Error() error
}

RangeVectorIterator iterates through a range of samples. To fetch the current vector use `At` with a `RangeVectorAggregator`.

type Result added in v1.4.0 

type Result struct {
	Data       parser.Value
	Statistics stats.Result
}

Result is the result of a query execution.

type SampleExpr added in v0.4.0 

type SampleExpr interface {
	// Selector is the LogQL selector to apply when retrieving logs.
	Selector() LogSelectorExpr
	Extractor() (SampleExtractor, error)
	// Operations returns the list of operations used in this SampleExpr
	Operations() []string
	Expr
}

SampleExpr is a LogQL expression filtering logs and returning metric samples.

func ParseSampleExpr added in v1.6.0 

func ParseSampleExpr(input string) (SampleExpr, error)

ParseSampleExpr parses a string and returns the sampleExpr

type SampleExtractor added in v1.6.0 

type SampleExtractor interface {
	Extract(line []byte) (float64, bool)
}

SampleExtractor transforms a log entry into a sample. In case of failure the second return value will be false.

type SelectLogParams added in v1.6.0 

type SelectLogParams struct {
	*logproto.QueryRequest
}

SelectParams specifies parameters passed to data selections.

func (SelectLogParams) LogSelector added in v1.6.0 

func (s SelectLogParams) LogSelector() (LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type SelectSampleParams added in v1.6.0 

type SelectSampleParams struct {
	*logproto.SampleQueryRequest
}

func (SelectSampleParams) Expr added in v1.6.0 

func (s SelectSampleParams) Expr() (SampleExpr, error)

Expr returns the SampleExpr from the SelectSampleParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

func (SelectSampleParams) LogSelector added in v1.6.0 

func (s SelectSampleParams) LogSelector() (LogSelectorExpr, error)

LogSelector returns the LogSelectorExpr from the SelectParams. The `LogSelectorExpr` can then returns all matchers and filters to use for that request.

type ShardMapper added in v1.5.0 

type ShardMapper struct {
	// contains filtered or unexported fields
}

func NewShardMapper added in v1.5.0 

func NewShardMapper(shards int, metrics *ShardingMetrics) (ShardMapper, error)

func (ShardMapper) Map added in v1.5.0 

func (m ShardMapper) Map(expr Expr, r *shardRecorder) (Expr, error)

func (ShardMapper) Parse added in v1.6.0 

func (m ShardMapper) Parse(query string) (noop bool, expr Expr, err error)

type ShardedEngine added in v1.6.0 

type ShardedEngine struct {
	// contains filtered or unexported fields
}

ShardedEngine is an Engine implementation that can split queries into more parallelizable forms via querying the underlying backend shards individually and reaggregating them.

func NewShardedEngine added in v1.6.0 

func NewShardedEngine(opts EngineOpts, downstreamable Downstreamable, metrics *ShardingMetrics) *ShardedEngine

NewShardedEngine constructs a *ShardedEngine

func (*ShardedEngine) Query added in v1.6.0 

func (ng *ShardedEngine) Query(p Params, shards int) Query

Query constructs a Query

type ShardingMetrics added in v1.6.0 

type ShardingMetrics struct {
	// contains filtered or unexported fields
}

ShardingMetrics is the metrics wrapper used in shard mapping

func NewShardingMetrics added in v1.6.0 

func NewShardingMetrics(registerer prometheus.Registerer) *ShardingMetrics

type Shards added in v1.6.0 

type Shards []astmapper.ShardAnnotation

func ParseShards added in v1.6.0 

func ParseShards(strs []string) (Shards, error)

ParseShards parses a list of string encoded shards

func (Shards) Encode added in v1.6.0 

func (xs Shards) Encode() (encoded []string)

type StepEvaluator added in v0.4.0 

type StepEvaluator interface {
	// while Next returns a promql.Value, the only acceptable types are Scalar and Vector.
	Next() (bool, int64, promql.Vector)
	// Close all resources used.
	Close() error
	// Reports any error
	Error() error
}

StepEvaluator evaluate a single step of a query.

func ConcatEvaluator added in v1.6.0 

func ConcatEvaluator(evaluators []StepEvaluator) (StepEvaluator, error)

ConcatEvaluator joins multiple StepEvaluators. Contract: They must be of identical start, end, and step values.

func ResultStepEvaluator added in v1.6.0 

func ResultStepEvaluator(res Result, params Params) (StepEvaluator, error)

ResultStepEvaluator coerces a downstream vector or matrix into a StepEvaluator

type Streams added in v0.4.0 

type Streams []logproto.Stream

Streams is promql.Value

func (Streams) Len added in v1.6.0 

func (streams Streams) Len() int

func (Streams) Less added in v1.6.0 

func (streams Streams) Less(i, j int) bool

func (Streams) String added in v0.4.0 

func (Streams) String() string

String implements `promql.Value`

func (Streams) Swap added in v1.6.0 

func (streams Streams) Swap(i, j int)

func (Streams) Type added in v0.4.0 

func (Streams) Type() parser.ValueType

Type implements `promql.Value`

Source Files

View all Source files

Directories

Path Synopsis
log
logfmt Module
Package marshal converts internal objects to loghttp model objects.
 Package marshal converts internal objects to loghttp model objects.
legacy
Package marshal converts internal objects to loghttp model objects.
 Package marshal converts internal objects to loghttp model objects.
Package stats provides primitives for recording metrics across the query path.
 Package stats provides primitives for recording metrics across the query path.
syntax module
legacy

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.