harvester

Documentation

Overview

The harvester package harvest different inputs for new information. Currently two harvester types exist:

• log
• stdin

The log harvester reads a file line by line. In case the end of a file is found with an incomplete line, the line pointer stays at the beginning of the incomplete line. As soon as the line is completed, it is read and returned.

The stdin harvesters reads data from stdin.

Index

• func MatchAnyRegexps(regexps []*regexp.Regexp, text string) bool
• type FileSource
• type Harvester
  • func NewHarvester(cfg *HarvesterConfig, path string, state input.FileState, ...) (*Harvester, error)
  • func (h *Harvester) GetState() input.FileState
  • func (h *Harvester) Harvest()
  • func (h *Harvester) SendStateUpdate() bool
  • func (h *Harvester) SetOffset(offset int64)
• type HarvesterConfig
• type LogSource

Functions

func MatchAnyRegexps

func MatchAnyRegexps(regexps []*regexp.Regexp, text string) bool

MatchAnyRegexps checks if the text matches any of the regular expressions

Types

type FileSource

type FileSource interface {
	LogSource
	Stat() (os.FileInfo, error)
	Continuable() bool // can we continue processing after EOF?
}

type Harvester

type Harvester struct {
	Path   string /* the file path to harvest */
	Config *HarvesterConfig

	State input.FileState

	SpoolerChan chan *input.FileEvent

	ExcludeLinesRegexp []*regexp.Regexp
	IncludeLinesRegexp []*regexp.Regexp
	// contains filtered or unexported fields
}

func NewHarvester

func NewHarvester(
	cfg *HarvesterConfig,
	path string,
	state input.FileState,
	spooler chan *input.FileEvent,
	offset int64,
	done chan struct{},
) (*Harvester, error)

func (*Harvester) GetState

func (h *Harvester) GetState() input.FileState

func (*Harvester) Harvest

func (h *Harvester) Harvest()

Log harvester reads files line by line and sends events to the defined output

func (*Harvester) SendStateUpdate

func (h *Harvester) SendStateUpdate() bool

SendStateUpdate send an empty event with the current state to update the registry

func (*Harvester) SetOffset

func (h *Harvester) SetOffset(offset int64)

type HarvesterConfig

type HarvesterConfig struct {
	common.EventMetadata `config:",inline"` // Fields and tags to add to events.

	BufferSize         int    `config:"harvester_buffer_size"`
	DocumentType       string `config:"document_type"`
	Encoding           string `config:"encoding"`
	InputType          string `config:"input_type"`
	TailFiles          bool   `config:"tail_files"`
	Backoff            string `config:"backoff"`
	BackoffDuration    time.Duration
	BackoffFactor      int    `config:"backoff_factor"`
	MaxBackoff         string `config:"max_backoff"`
	MaxBackoffDuration time.Duration
	CloseOlder         string `config:"close_older"`
	CloseOlderDuration time.Duration
	ForceCloseFiles    bool                   `config:"force_close_files"`
	ExcludeLines       []*regexp.Regexp       `config:"exclude_lines"`
	IncludeLines       []*regexp.Regexp       `config:"include_lines"`
	MaxBytes           int                    `config:"max_bytes"`
	Multiline          *input.MultilineConfig `config:"multiline"`
	JSON               *input.JSONConfig      `config:"json"`
}

type LogSource

type LogSource interface {
	io.ReadCloser
	Name() string
}