parse

Documentation

Index

• Constants
• Variables
• func DefaultTimeParser(in interface{}) (time.Time, error)
• func NoopTimeParser(in interface{}) (time.Time, error)
• func StrictUnixTimeParser(in interface{}) (time.Time, error)
• type DefaultOutputFormatter
  • func (f *DefaultOutputFormatter) FormatField(s *State, k string, v interface{}, w *bytes.Buffer)
  • func (f *DefaultOutputFormatter) FormatLevel(s *State, level Level, w *bytes.Buffer)
  • func (f *DefaultOutputFormatter) FormatMessage(s *State, msg string, highlight bool, w *bytes.Buffer)
  • func (f *DefaultOutputFormatter) FormatTime(s *State, t time.Time, w *bytes.Buffer)
• type FilterScheme
  • func (f *FilterScheme) AddJQ(p string) error
  • func (f *FilterScheme) AddMatchRegex(rx string) error
  • func (f *FilterScheme) AddNoMatchRegex(rx string) error
  • func (f *FilterScheme) Run(l *line) (bool, error)
• type InputSchema
  • func (s *InputSchema) ReadLine(l *line) error
• type Level
  • func BunyanV0LevelParser(in interface{}) (Level, error)
  • func DefaultLevelParser(in interface{}) (Level, error)
  • func LagerLevelParser(in interface{}) (Level, error)
  • func NoopLevelParser(in interface{}) (Level, error)
• type LevelParser
• type OutputFormatter
• type OutputSchema
  • func (s *OutputSchema) Emit(l *line, w *bytes.Buffer)
  • func (s *OutputSchema) EmitError(msg string)
• type State
• type Summary
  • func ReadLog(r io.Reader, w io.Writer, ins *InputSchema, outs *OutputSchema, ...) (Summary, error)
  • func (s Summary) String() string
• type TimeParser

Constants

const LineBufferSize = 1 * 1024 * 1024 // 1 MiB

LineBufferSize is the longest we're willing to look for a newline in the input.

Variables

var (
	ErrAlreadyAdded = errors.New("regex already added")
	ErrConflict     = errors.New("attempt to add regex when a conflicting regex has already been added")
)

var DefaultVariables = []string{
	"$TS",
	"$RAW", "$MSG",
	"$LVL", "$UNKNOWN", "$TRACE", "$DEBUG", "$INFO", "$WARN", "$ERROR", "$PANIC", "$DPANIC", "$FATAL",
}

DefaultVariables are variables available to JQ programs.

Functions

func DefaultTimeParser

func DefaultTimeParser(in interface{}) (time.Time, error)

DefaultTimeParser treats numbers as seconds since the Unix epoch and strings as RFC3339 timestamps.

func NoopTimeParser

func NoopTimeParser(in interface{}) (time.Time, error)

NoopTimeParser panics if called.

func StrictUnixTimeParser

func StrictUnixTimeParser(in interface{}) (time.Time, error)

StrictUnixTimeParser always treats the incoming data as a float64 number of seconds since the Unix epoch.

Types

type DefaultOutputFormatter

type DefaultOutputFormatter struct {
	Aurora aurora.Aurora // Controls the use of color.

	// If true, print ↑ for fields that have an identical value as the previous line.
	ElideDuplicateFields bool

	// The time.Format string to show times in, like time.RFC3339.  If empty, show relative
	// times since the time the program started.  (A minus sign indicates the past; positive
	// values are in the future, good for when you are following a log file.)
	AbsoluteTimeFormat string

	// If non-empty, print only the fractional seconds for log lines that occurred on the same
	// second as the previous line.  For example, if SecondsOnlyFormat is set to ".000":
	//
	// INFO 2020-01-02T03:04:05.123Z first event
	// INFO                    .456  next event
	// INFO                    .789  another event
	// INFO 2020-01-02T03:04:05.000Z a brand new second
	//
	// Decimals are only aligned by careful selection of AbsoluteTimeFormat and
	// SecondsOnlyFormat strings.  The algorithm does nothing smart.
	SubSecondsOnlyFormat string

	Zone            *time.Location      // Zone is the time zone to display the output in.
	HighlightFields map[string]struct{} // HighlightFields visually distinguishes the named fields.
}

func (*DefaultOutputFormatter) FormatField

func (f *DefaultOutputFormatter) FormatField(s *State, k string, v interface{}, w *bytes.Buffer)

func (*DefaultOutputFormatter) FormatLevel

func (f *DefaultOutputFormatter) FormatLevel(s *State, level Level, w *bytes.Buffer)

func (*DefaultOutputFormatter) FormatMessage

func (f *DefaultOutputFormatter) FormatMessage(s *State, msg string, highlight bool, w *bytes.Buffer)

func (*DefaultOutputFormatter) FormatTime

func (f *DefaultOutputFormatter) FormatTime(s *State, t time.Time, w *bytes.Buffer)

type FilterScheme

type FilterScheme struct {
	JQ           *gojq.Code
	MatchRegex   *regexp.Regexp
	NoMatchRegex *regexp.Regexp
}

FilterScheme controls how lines are filtered.

func (*FilterScheme) AddJQ

func (f *FilterScheme) AddJQ(p string) error

AddJQ compiles the provided jq program and adds it to the filter.

func (*FilterScheme) AddMatchRegex

func (f *FilterScheme) AddMatchRegex(rx string) error

Add a MatchRegex to this filter scheme. A MatchRegex filters out all lines that do not match it. An empty string is a no-op. This method may only be called with a non-empty string once, and returns an ErrConflict if a NoMatchRegex is set.

func (*FilterScheme) AddNoMatchRegex

func (f *FilterScheme) AddNoMatchRegex(rx string) error

Add a NoMatchRegex to this filter scheme. A NoMatchRegex filters out all lines that match it. An empty string is a no-op. This method may only be called with a non-empty string once, and returns an ErrConflict if a MatchRegex is set.

func (*FilterScheme) Run

func (f *FilterScheme) Run(l *line) (bool, error)

Run runs all the filters defined in this FilterScheme against the provided line. The return value is true if the line should be removed from the output ("filtered").

type InputSchema

type InputSchema struct {
	TimeKey     string      // The name of the key that holds the timestamp.
	TimeFormat  TimeParser  // How to turn the value of the time key into a time.Time.
	LevelKey    string      // The name of the key that holds the log level.
	LevelFormat LevelParser // How to turn the value of the level key into a Level.
	MessageKey  string      // The name of the key that holds the main log message.

	NoTimeKey    bool // If set, suppress any time handling.
	NoLevelKey   bool // If set, suppress any level handling.
	NoMessageKey bool // If set, suppress any message handling.

	// If true, print an error when non-JSON lines appear in the input.  If false, treat them
	// as normal messages with as much information extracted as possible.
	Strict bool

	// DeleteKeys is a list of keys to delete; used when the log lines contain version
	// information that is used for guessing the schema.
	DeleteKeys []string

	// UpgradeKeys is a list of keys to merge into the raw data.  For example, lager puts
	// everything in the "data" key.
	UpgradeKeys []string
}

InputSchema controls the interpretation of incoming log lines.

func (*InputSchema) ReadLine

func (s *InputSchema) ReadLine(l *line) error

ReadLine parses a log line into the provided line object.

type Level

type Level uint8

Level is a log level. This exists so that you can write jq expressions like "select($LVL<$WARN)". Whatever logger you're using probably has totally different levels because nobody can agree on them. Feel free to add them here in the right place.

const (
	LevelUnknown Level = iota
	LevelTrace
	LevelDebug
	LevelInfo
	LevelWarn
	LevelError
	LevelPanic
	LevelDPanic
	LevelFatal
)

func BunyanV0LevelParser

func BunyanV0LevelParser(in interface{}) (Level, error)

BunyanV0LLevelParser maps bunyan's float64 levels to log levels.

func DefaultLevelParser

func DefaultLevelParser(in interface{}) (Level, error)

DefaultLevelParser uses common strings to determine the log level. Case does not matter; info is the same log level as INFO.

func LagerLevelParser

func LagerLevelParser(in interface{}) (Level, error)

LagerLevelParser maps lager's float64 levels to log levels.

func NoopLevelParser

func NoopLevelParser(in interface{}) (Level, error)

NoopLevelParser panics if called.

type LevelParser

type LevelParser func(interface{}) (Level, error)

LevelParser is a function that parses log levels in log messages.

type OutputFormatter

type OutputFormatter interface {
	// FormatTime is a function that formats a time.Time and outputs it to an io.Writer.
	FormatTime(s *State, t time.Time, w *bytes.Buffer)

	// FormatLevel is a function that formats a log level and outputs it to an io.Writer.
	FormatLevel(s *State, lvl Level, w *bytes.Buffer)

	// FormatMessage is a function that formats a log message and outputs it to an io.Writer.
	FormatMessage(s *State, msg string, highlight bool, w *bytes.Buffer)

	// FormatField is a function that formats a (key, value) pair and outputs it to an io.Writer.
	FormatField(s *State, k string, v interface{}, w *bytes.Buffer)
}

OutputFormatter describes an object that actually does the output formatting. Methods take a bytes.Buffer so they can output incrementally as with an io.Writer, but without worrying about write errors or short writes.

type OutputSchema

type OutputSchema struct {
	PriorityFields []string         // PriorityFields controls which fields are printed first.
	Formatter      OutputFormatter  // Actually does the formatting.
	EmitErrorFn    func(msg string) // A function that sees all errors.
	BeforeContext  int              // Context lines to print before a match.
	AfterContext   int              // Context lines to print after a match.
	// contains filtered or unexported fields
}

OutputSchema controls how output lines are formatted.

func (*OutputSchema) Emit

func (s *OutputSchema) Emit(l *line, w *bytes.Buffer)

Emit emits a formatted line to the provided buffer. Emit must not mutate line.

func (*OutputSchema) EmitError

func (s *OutputSchema) EmitError(msg string)

EmitError prints any internal errors, so that log lines are not silently ignored if they are unparseable.

type State

type State struct {
	// contains filtered or unexported fields
}

State keeps state between log lines.

type Summary

type Summary struct {
	Lines    int
	Errors   int
	Filtered int
}

func ReadLog

func ReadLog(r io.Reader, w io.Writer, ins *InputSchema, outs *OutputSchema, filter *FilterScheme) (Summary, error)

ReadLog reads a stream of JSON-formatted log lines from the provided reader according to the input schema, reformatting it and writing to the provided writer according to the output schema. Parse errors are handled according to the input schema. Any other errors, not including io.EOF on the reader, are returned.

func (Summary) String

func (s Summary) String() string

type TimeParser

type TimeParser func(interface{}) (time.Time, error)

TimeParser is a function that parses timestamps in log messages.