harvester

Documentation

Overview

The harvester package harvest different inputs for new information. Currently two harvester types exist:

• log
• stdin

The log harvester reads a file line by line. In case the end of a file is found with an incomplete line, the line pointer stays at the beginning of the incomplete line. As soon as the line is completed, it is read and returned.

The stdin harvesters reads data from stdin.

Index

• func InitRegexps(exprs []string) ([]*regexp.Regexp, error)
• func MatchAnyRegexps(regexps []*regexp.Regexp, text string) bool
• type FileSource
• type FileStat
  • func NewFileStat(fi os.FileInfo, lastIteration uint32) *FileStat
  • func (fs *FileStat) Continue(old *FileStat)
  • func (fs *FileStat) Finished() bool
  • func (fs *FileStat) Ignore()
  • func (fs *FileStat) Skip(returnOffset int64)
• type Harvester
  • func NewHarvester(prospectorCfg config.ProspectorConfig, cfg *config.HarvesterConfig, ...) (*Harvester, error)
  • func (h *Harvester) GetOffset() int64
  • func (h *Harvester) GetState() *input.FileState
  • func (h *Harvester) Harvest()
  • func (h *Harvester) SetOffset(offset int64)
  • func (h *Harvester) Start()
  • func (h *Harvester) Stop()
• type LogSource
• type Typer

Functions

func InitRegexps

func InitRegexps(exprs []string) ([]*regexp.Regexp, error)

InitRegexps initializes a list of compiled regular expressions.

func MatchAnyRegexps

func MatchAnyRegexps(regexps []*regexp.Regexp, text string) bool

MatchAnyRegexps checks if the text matches any of the regular expressions

Types

type FileSource

type FileSource interface {
	LogSource
	Stat() (os.FileInfo, error)
	Continuable() bool // can we continue processing after EOF?
}

type FileStat

type FileStat struct {
	Fileinfo      os.FileInfo /* the file info */
	Return        chan int64  /* the harvester will send an event with its offset when it closes */
	LastIteration uint32      /* int number of the last iterations in which we saw this file */
}

Contains statistic about file when it was last seend by the prospector

func NewFileStat

func NewFileStat(fi os.FileInfo, lastIteration uint32) *FileStat

func (*FileStat) Continue

func (fs *FileStat) Continue(old *FileStat)

func (*FileStat) Finished

func (fs *FileStat) Finished() bool

func (*FileStat) Ignore

func (fs *FileStat) Ignore()

Ignore forgets about the previous harvester results and let it continue on the old file - start a new channel to use with the new harvester.

func (*FileStat) Skip

func (fs *FileStat) Skip(returnOffset int64)

type Harvester

type Harvester struct {
	Path             string /* the file path to harvest */
	ProspectorConfig config.ProspectorConfig
	Config           *config.HarvesterConfig

	Stat        *FileStat
	SpoolerChan chan *input.FileEvent

	ExcludeLinesRegexp []*regexp.Regexp
	IncludeLinesRegexp []*regexp.Regexp
	// contains filtered or unexported fields
}

func NewHarvester

func NewHarvester(
	prospectorCfg config.ProspectorConfig,
	cfg *config.HarvesterConfig,
	path string,
	stat *FileStat,
	spooler chan *input.FileEvent,
) (*Harvester, error)

func (*Harvester) GetOffset

func (h *Harvester) GetOffset() int64

func (*Harvester) GetState

func (h *Harvester) GetState() *input.FileState

GetState returns current state of harvester

func (*Harvester) Harvest

func (h *Harvester) Harvest()

Log harvester reads files line by line and sends events to the defined output

func (*Harvester) SetOffset

func (h *Harvester) SetOffset(offset int64)

func (*Harvester) Start

func (h *Harvester) Start()

func (*Harvester) Stop

func (h *Harvester) Stop()

type LogSource

type LogSource interface {
	io.ReadCloser
	Name() string
}

type Typer

type Typer interface {
	// contains filtered or unexported methods
}

Interface for the different harvester types