file

Documentation

Index

• type Finder
  • func (f Finder) FindFiles() []string
• type Fingerprint
  • func (f Fingerprint) Copy() *Fingerprint
  • func (f Fingerprint) StartsWith(old *Fingerprint) bool
• type InputConfig
  • func NewInputConfig(operatorID string) *InputConfig
  • func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, error)
• type InputOperator
  • func (f *InputOperator) NewFingerprint(file *os.File) (*Fingerprint, error)
  • func (f *InputOperator) NewReader(path string, file *os.File, fp *Fingerprint) (*Reader, error)
  • func (f *InputOperator) Start() error
  • func (f *InputOperator) Stop() error
• type PositionalScanner
  • func NewPositionalScanner(r io.Reader, maxLogSize int, startOffset int64, splitFunc bufio.SplitFunc) *PositionalScanner
  • func (ps *PositionalScanner) Pos() int64
• type Reader
  • func (f *Reader) Close()
  • func (f *Reader) Copy(file *os.File) (*Reader, error)
  • func (f *Reader) InitializeOffset(startAtBeginning bool) error
  • func (f *Reader) Read(dst []byte) (int, error)
  • func (f *Reader) ReadHeaders(ctx context.Context)
  • func (f *Reader) ReadToEnd(ctx context.Context)

Types

type Finder

type Finder struct {
	Include []string `mapstructure:"include,omitempty" json:"include,omitempty" yaml:"include,omitempty"`
	Exclude []string `mapstructure:"exclude,omitempty" json:"exclude,omitempty" yaml:"exclude,omitempty"`
}

Finder is responsible for find files according to the include and exclude rules

func (Finder) FindFiles

func (f Finder) FindFiles() []string

FindFiles gets a list of paths given an array of glob patterns to include and exclude

type Fingerprint

type Fingerprint struct {
	// FirstBytes represents the first N bytes of a file
	FirstBytes []byte
}

Fingerprint is used to identify a file A file's fingerprint is the first N bytes of the file, where N is the fingerprintSize on the file_input operator

func (Fingerprint) Copy

func (f Fingerprint) Copy() *Fingerprint

Copy creates a new copy of the fingerprint

func (Fingerprint) StartsWith

func (f Fingerprint) StartsWith(old *Fingerprint) bool

StartsWith returns true if the fingerprints are the same or if the new fingerprint starts with the old one This is important functionality for tracking new files, since their initial size is typically less than that of a fingerprint. As the file grows, its fingerprint is updated until it reaches a maximum size, as configured on the operator

type InputConfig

type InputConfig struct {
	helper.InputConfig `yaml:",inline"`
	Finder             `mapstructure:",squash" yaml:",inline"`

	PollInterval            helper.Duration        `json:"poll_interval,omitempty"               yaml:"poll_interval,omitempty"`
	Multiline               helper.MultilineConfig `json:"multiline,omitempty"                   yaml:"multiline,omitempty"`
	IncludeFileName         bool                   `json:"include_file_name,omitempty"           yaml:"include_file_name,omitempty"`
	IncludeFilePath         bool                   `json:"include_file_path,omitempty"           yaml:"include_file_path,omitempty"`
	IncludeFileNameResolved bool                   `json:"include_file_name_resolved,omitempty"  yaml:"include_file_name_resolved,omitempty"`
	IncludeFilePathResolved bool                   `json:"include_file_path_resolved,omitempty"  yaml:"include_file_path_resolved,omitempty"`
	StartAt                 string                 `json:"start_at,omitempty"                    yaml:"start_at,omitempty"`
	FingerprintSize         helper.ByteSize        `json:"fingerprint_size,omitempty"            yaml:"fingerprint_size,omitempty"`
	MaxLogSize              helper.ByteSize        `json:"max_log_size,omitempty"                yaml:"max_log_size,omitempty"`
	MaxConcurrentFiles      int                    `json:"max_concurrent_files,omitempty"        yaml:"max_concurrent_files,omitempty"`
	DeleteAfterRead         bool                   `json:"delete_after_read,omitempty"           yaml:"delete_after_read,omitempty"`
	LabelRegex              string                 `json:"label_regex,omitempty"                 yaml:"label_regex,omitempty"`
	Encoding                helper.EncodingConfig  `json:",inline,omitempty"                     yaml:",inline,omitempty"`
	FilenameRecallPeriod    helper.Duration        `json:"filename_recall_period,omitempty"      yaml:"filename_recall_period,omitempty"`
}

InputConfig is the configuration of a file input operator

func NewInputConfig

func NewInputConfig(operatorID string) *InputConfig

NewInputConfig creates a new input config with default values

func (InputConfig) Build

func (c InputConfig) Build(context operator.BuildContext) ([]operator.Operator, error)

Build will build a file input operator from the supplied configuration

type InputOperator

type InputOperator struct {
	helper.InputOperator

	FilePathField         entry.Field
	FileNameField         entry.Field
	FilePathResolvedField entry.Field
	FileNameResolvedField entry.Field
	PollInterval          time.Duration
	SplitFunc             bufio.SplitFunc
	MaxLogSize            int
	MaxConcurrentFiles    int
	SeenPaths             map[string]time.Time
	// contains filtered or unexported fields
}

InputOperator is an operator that monitors files for entries

func (*InputOperator) NewFingerprint

func (f *InputOperator) NewFingerprint(file *os.File) (*Fingerprint, error)

NewFingerprint creates a new fingerprint from an open file

func (*InputOperator) NewReader

func (f *InputOperator) NewReader(path string, file *os.File, fp *Fingerprint) (*Reader, error)

NewReader creates a new file reader

func (*InputOperator) Start

func (f *InputOperator) Start() error

Start will start the file monitoring process

func (*InputOperator) Stop

func (f *InputOperator) Stop() error

Stop will stop the file monitoring process

type PositionalScanner

type PositionalScanner struct {
	*bufio.Scanner
	// contains filtered or unexported fields
}

PositionalScanner is a scanner that maintains position

func NewPositionalScanner

func NewPositionalScanner(r io.Reader, maxLogSize int, startOffset int64, splitFunc bufio.SplitFunc) *PositionalScanner

NewPositionalScanner creates a new positional scanner

func (*PositionalScanner) Pos

func (ps *PositionalScanner) Pos() int64

Pos returns the current position of the scanner

type Reader

type Reader struct {
	Fingerprint *Fingerprint
	Offset      int64

	// HeaderLabels is an optional map that contains entry labels
	// derived from a log files' headers, added to every record
	HeaderLabels map[string]string

	*zap.SugaredLogger `json:"-"`
	// contains filtered or unexported fields
}

Reader manages a single file

func (*Reader) Close

func (f *Reader) Close()

Close will close the file

func (*Reader) Copy

func (f *Reader) Copy(file *os.File) (*Reader, error)

Copy creates a deep copy of a Reader

func (*Reader) InitializeOffset

func (f *Reader) InitializeOffset(startAtBeginning bool) error

InitializeOffset sets the starting offset

func (*Reader) Read

func (f *Reader) Read(dst []byte) (int, error)

func (*Reader) ReadHeaders

func (f *Reader) ReadHeaders(ctx context.Context)

ReadHeaders will read a files headers

func (*Reader) ReadToEnd

func (f *Reader) ReadToEnd(ctx context.Context)

ReadToEnd will read until the end of the file