drain

Documentation

Index

• Constants
• func DetectLogFormat(line string) string
• type Chunk
  • func (c Chunk) ForRange(start, end, step model.Time) []logproto.PatternSample
• type Chunks
  • func (c *Chunks) Add(ts model.Time)
  • func (c Chunks) Iterator(pattern string, from, through, step model.Time) iter.Iterator
• type Config
  • func DefaultConfig() *Config
• type DedupingTokenizer
  • func (d DedupingTokenizer) Join(tokens []string, state interface{}) string
• type Drain
  • func New(tenantID string, config *Config, limits Limits, format string, ...) *Drain
  • func (d *Drain) Clusters() []*LogCluster
  • func (d *Drain) Delete(cluster *LogCluster)
  • func (d *Drain) Prune()
  • func (d *Drain) Train(content string, ts int64) *LogCluster
  • func (d *Drain) TrainPattern(content string, samples []*logproto.PatternSample) *LogCluster
  • func (d *Drain) TrainTokens(tokens []string, stringer func([]string) string, ts int64) *LogCluster
• type Limits
• type LineTokenizer
• type LogCluster
  • func (c *LogCluster) Iterator(from, through, step model.Time) iter.Iterator
  • func (c *LogCluster) Prune(olderThan time.Duration)
  • func (c *LogCluster) Samples() []*logproto.PatternSample
  • func (c *LogCluster) String() string
• type LogClusterCache
  • func (c *LogClusterCache) Get(key int) *LogCluster
  • func (c *LogClusterCache) Set(key int, cluster *LogCluster)
  • func (c *LogClusterCache) Values() []*LogCluster
• type Metrics
• type Node

Constants

const (
	FormatLogfmt  = "logfmt"
	FormatJSON    = "json"
	FormatUnknown = "unknown"
)

const (
	TimeResolution = model.Time(int64(time.Second*10) / 1e6)
)

Functions

func DetectLogFormat

func DetectLogFormat(line string) string

DetectLogFormat guesses at how the logs are encoded based on some simple heuristics. It only runs on the first log line when a new stream is created, so it could do some more complex parsing or regex.

Types

type Chunk

type Chunk struct {
	Samples []logproto.PatternSample
}

func (Chunk) ForRange

func (c Chunk) ForRange(start, end, step model.Time) []logproto.PatternSample

ForRange returns samples with only the values in the given range [start:end) and aggregates them by step duration. start and end are in milliseconds since epoch. step is a duration in milliseconds.

type Chunks

type Chunks []Chunk

func (*Chunks) Add

func (c *Chunks) Add(ts model.Time)

func (Chunks) Iterator

func (c Chunks) Iterator(pattern string, from, through, step model.Time) iter.Iterator

type Config

type Config struct {
	LogClusterDepth      int
	SimTh                float64
	MaxChildren          int
	ExtraDelimiters      []string
	MaxClusters          int
	ParamString          string
	MaxEvictionRatio     float64
	MaxAllowedLineLength int
	// contains filtered or unexported fields
}

func DefaultConfig

func DefaultConfig() *Config

type DedupingTokenizer

type DedupingTokenizer struct {
	LineTokenizer
	// contains filtered or unexported fields
}

func (DedupingTokenizer) Join

func (d DedupingTokenizer) Join(tokens []string, state interface{}) string

type Drain

type Drain struct {
	// contains filtered or unexported fields
}

func New

func New(tenantID string, config *Config, limits Limits, format string, metrics *Metrics) *Drain

func (*Drain) Clusters

func (d *Drain) Clusters() []*LogCluster

func (*Drain) Delete

func (d *Drain) Delete(cluster *LogCluster)

func (*Drain) Prune

func (d *Drain) Prune()

func (*Drain) Train

func (d *Drain) Train(content string, ts int64) *LogCluster

func (*Drain) TrainPattern

func (d *Drain) TrainPattern(content string, samples []*logproto.PatternSample) *LogCluster

func (*Drain) TrainTokens

func (d *Drain) TrainTokens(tokens []string, stringer func([]string) string, ts int64) *LogCluster

type Limits

type Limits interface {
	PatternIngesterTokenizableJSONFields(userID string) []string
}

type LineTokenizer

type LineTokenizer interface {
	Tokenize(line string, tokens []string, state interface{}) ([]string, interface{})
	Join(tokens []string, state interface{}) string
	Clone(tokens []string, state interface{}) ([]string, interface{})
}

type LogCluster

type LogCluster struct {
	Size       int
	Tokens     []string
	TokenState interface{}
	Stringer   func([]string, interface{}) string

	Chunks Chunks
	// contains filtered or unexported fields
}

func (*LogCluster) Iterator

func (c *LogCluster) Iterator(from, through, step model.Time) iter.Iterator

func (*LogCluster) Prune

func (c *LogCluster) Prune(olderThan time.Duration)

func (*LogCluster) Samples

func (c *LogCluster) Samples() []*logproto.PatternSample

func (*LogCluster) String

func (c *LogCluster) String() string

type LogClusterCache

type LogClusterCache struct {
	// contains filtered or unexported fields
}

func (*LogClusterCache) Get

func (c *LogClusterCache) Get(key int) *LogCluster

func (*LogClusterCache) Set

func (c *LogClusterCache) Set(key int, cluster *LogCluster)

func (*LogClusterCache) Values

func (c *LogClusterCache) Values() []*LogCluster

type Metrics

type Metrics struct {
	PatternsEvictedTotal  prometheus.Counter
	PatternsPrunedTotal   prometheus.Counter
	PatternsDetectedTotal prometheus.Counter
	TokensPerLine         prometheus.Observer
	StatePerLine          prometheus.Observer
}

type Node

type Node struct {
	// contains filtered or unexported fields
}