tokenize

Documentation

Index

• func NewLineEndSplitFunc(re *regexp.Regexp, flushAtEOF bool, trimFunc trimFunc) bufio.SplitFunc
• func NewLineStartSplitFunc(re *regexp.Regexp, flushAtEOF bool, trimFunc trimFunc) bufio.SplitFunc
• func NewNewlineSplitFunc(enc encoding.Encoding, flushAtEOF bool, trimFunc trimFunc) (bufio.SplitFunc, error)
• func SplitNone(maxLogSize int) bufio.SplitFunc
• type Flusher
  • func (f *Flusher) Flushed()
  • func (f *Flusher) ShouldFlush() bool
  • func (f *Flusher) SplitFunc(splitFunc bufio.SplitFunc) bufio.SplitFunc
  • func (f *Flusher) UpdateDataChangeTime(length int)
• type FlusherConfig
  • func NewFlusherConfig() FlusherConfig
  • func (c *FlusherConfig) Build() *Flusher
• type Multiline
• type MultilineConfig
  • func NewMultilineConfig() MultilineConfig
  • func (c MultilineConfig) Build(enc encoding.Encoding, ...) (bufio.SplitFunc, error)
• type Splitter
• type SplitterConfig
  • func NewSplitterConfig() SplitterConfig
  • func (c *SplitterConfig) Build(flushAtEOF bool, maxLogSize int) (*Splitter, error)

Functions

func NewLineEndSplitFunc

func NewLineEndSplitFunc(re *regexp.Regexp, flushAtEOF bool, trimFunc trimFunc) bufio.SplitFunc

NewLineEndSplitFunc creates a bufio.SplitFunc that splits an incoming stream into tokens that end with a match to the regex pattern provided

func NewLineStartSplitFunc

func NewLineStartSplitFunc(re *regexp.Regexp, flushAtEOF bool, trimFunc trimFunc) bufio.SplitFunc

NewLineStartSplitFunc creates a bufio.SplitFunc that splits an incoming stream into tokens that start with a match to the regex pattern provided

func NewNewlineSplitFunc

func NewNewlineSplitFunc(enc encoding.Encoding, flushAtEOF bool, trimFunc trimFunc) (bufio.SplitFunc, error)

NewNewlineSplitFunc splits log lines by newline, just as bufio.ScanLines, but never returning an token using EOF as a terminator

func SplitNone

func SplitNone(maxLogSize int) bufio.SplitFunc

SplitNone doesn't split any of the bytes, it reads in all of the bytes and returns it all at once. This is for when the encoding is nop

Types

type Flusher

type Flusher struct {
	// contains filtered or unexported fields
}

Flusher keeps information about flush state

func (*Flusher) Flushed

func (f *Flusher) Flushed()

Flushed reset data length

func (*Flusher) ShouldFlush

func (f *Flusher) ShouldFlush() bool

ShouldFlush returns true if data should be forcefully flushed

func (*Flusher) SplitFunc

func (f *Flusher) SplitFunc(splitFunc bufio.SplitFunc) bufio.SplitFunc

func (*Flusher) UpdateDataChangeTime

func (f *Flusher) UpdateDataChangeTime(length int)

type FlusherConfig

type FlusherConfig struct {
	Period time.Duration `mapstructure:"force_flush_period"`
}

FlusherConfig is a configuration of Flusher helper

func NewFlusherConfig

func NewFlusherConfig() FlusherConfig

NewFlusherConfig creates a default Flusher config

func (*FlusherConfig) Build

func (c *FlusherConfig) Build() *Flusher

Build creates Flusher from configuration

type Multiline

type Multiline struct {
	SplitFunc bufio.SplitFunc
	Force     *Flusher
}

Multiline consists of splitFunc and variables needed to perform force flush

type MultilineConfig

type MultilineConfig struct {
	LineStartPattern string `mapstructure:"line_start_pattern"`
	LineEndPattern   string `mapstructure:"line_end_pattern"`
}

MultilineConfig is the configuration of a multiline helper

func NewMultilineConfig

func NewMultilineConfig() MultilineConfig

NewMultilineConfig creates a new Multiline config

func (MultilineConfig) Build

func (c MultilineConfig) Build(enc encoding.Encoding, flushAtEOF, preserveLeadingWhitespaces, preserveTrailingWhitespaces bool, force *Flusher, maxLogSize int) (bufio.SplitFunc, error)

Build will build a Multiline operator.

type Splitter

type Splitter struct {
	Decoder   *decoder.Decoder
	SplitFunc bufio.SplitFunc
	Flusher   *Flusher
}

Splitter consolidates Flusher and dependent splitFunc

type SplitterConfig

type SplitterConfig struct {
	Encoding                    string          `mapstructure:"encoding,omitempty"`
	Flusher                     FlusherConfig   `mapstructure:",squash,omitempty"`
	Multiline                   MultilineConfig `mapstructure:"multiline,omitempty"`
	PreserveLeadingWhitespaces  bool            `mapstructure:"preserve_leading_whitespaces,omitempty"`
	PreserveTrailingWhitespaces bool            `mapstructure:"preserve_trailing_whitespaces,omitempty"`
}

SplitterConfig consolidates MultilineConfig and FlusherConfig

func NewSplitterConfig

func NewSplitterConfig() SplitterConfig

NewSplitterConfig returns default SplitterConfig

func (*SplitterConfig) Build

func (c *SplitterConfig) Build(flushAtEOF bool, maxLogSize int) (*Splitter, error)

Build builds Splitter struct