fileconsumer

Documentation

Index

• Variables
• type AlphabeticalSortRule
• type BaseSortRule
• type Config
  • func NewConfig() *Config
  • func (c Config) Build(logger *zap.SugaredLogger, emit emit.Callback) (*Manager, error)
  • func (c Config) BuildWithSplitFunc(logger *zap.SugaredLogger, emit emit.Callback, splitFunc bufio.SplitFunc) (*Manager, error)
• type EmitFunc
• type FileAttributes
  • func (f *FileAttributes) HeaderAttributesCopy() map[string]any
• type Finder
  • func (f Finder) FindCurrent(files []string) ([]string, error)
  • func (f Finder) FindFiles() ([]string, error)
• type HeaderConfig
• type Manager
  • func (m *Manager) Start(persister operator.Persister) error
  • func (m *Manager) Stop() error
• type MatchingCriteria
• type NumericSortRule
• type OrderingCriteria
• type Reader
  • func (r *Reader) Close()
  • func (r *Reader) Read(dst []byte) (int, error)
  • func (r *Reader) ReadToEnd(ctx context.Context)
• type SortRuleImpl
  • func (sr *SortRuleImpl) Unmarshal(component *confmap.Conf) error
• type TimestampSortRule

Variables

var AllowHeaderMetadataParsing = featuregate.GlobalRegistry().MustRegister(
	"filelog.allowHeaderMetadataParsing",
	featuregate.StageAlpha,
	featuregate.WithRegisterDescription("When enabled, allows usage of the `header` setting."),
	featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/18198"),
)

Types

type AlphabeticalSortRule

type AlphabeticalSortRule struct {
	BaseSortRule `mapstructure:",squash"`
}

type BaseSortRule

type BaseSortRule struct {
	RegexKey  string `mapstructure:"regex_key,omitempty"`
	Ascending bool   `mapstructure:"ascending,omitempty"`
	SortType  string `mapstructure:"sort_type,omitempty"`
}

Deprecated: [v0.82.0] This will be made internal in a future release, tentatively v0.83.0.

type Config

type Config struct {
	MatchingCriteria        `mapstructure:",squash"`
	IncludeFileName         bool                  `mapstructure:"include_file_name,omitempty"`
	IncludeFilePath         bool                  `mapstructure:"include_file_path,omitempty"`
	IncludeFileNameResolved bool                  `mapstructure:"include_file_name_resolved,omitempty"`
	IncludeFilePathResolved bool                  `mapstructure:"include_file_path_resolved,omitempty"`
	PollInterval            time.Duration         `mapstructure:"poll_interval,omitempty"`
	StartAt                 string                `mapstructure:"start_at,omitempty"`
	FingerprintSize         helper.ByteSize       `mapstructure:"fingerprint_size,omitempty"`
	MaxLogSize              helper.ByteSize       `mapstructure:"max_log_size,omitempty"`
	MaxConcurrentFiles      int                   `mapstructure:"max_concurrent_files,omitempty"`
	MaxBatches              int                   `mapstructure:"max_batches,omitempty"`
	DeleteAfterRead         bool                  `mapstructure:"delete_after_read,omitempty"`
	Splitter                helper.SplitterConfig `mapstructure:",squash,omitempty"`
	Header                  *HeaderConfig         `mapstructure:"header,omitempty"`
}

Config is the configuration of a file input operator

func NewConfig

func NewConfig() *Config

NewConfig creates a new input config with default values

func (Config) Build

func (c Config) Build(logger *zap.SugaredLogger, emit emit.Callback) (*Manager, error)

Build will build a file input operator from the supplied configuration

func (Config) BuildWithSplitFunc

func (c Config) BuildWithSplitFunc(logger *zap.SugaredLogger, emit emit.Callback, splitFunc bufio.SplitFunc) (*Manager, error)

BuildWithSplitFunc will build a file input operator with customized splitFunc function

type EmitFunc

type EmitFunc func(ctx context.Context, attrs *FileAttributes, token []byte)

Deprecated: [v0.82.0] Use emit.Callback instead. This will be removed in a future release, tentatively v0.84.0.

type FileAttributes

type FileAttributes struct {
	Name             string `json:"-"`
	Path             string `json:"-"`
	NameResolved     string `json:"-"`
	PathResolved     string `json:"-"`
	HeaderAttributes map[string]any
}

Deprecated: [v0.82.0] This will be removed in a future release, tentatively v0.84.0.

func (*FileAttributes) HeaderAttributesCopy

func (f *FileAttributes) HeaderAttributesCopy() map[string]any

HeaderAttributesCopy gives a copy of the HeaderAttributes, in order to restrict mutation of the HeaderAttributes.

Deprecated: [v0.82.0] This will be removed in a future release, tentatively v0.84.0.

type Finder

type Finder = MatchingCriteria

Deprecated: [v0.82.0] Use MatchingCriteria instead. This will be removed in v0.83.0.

func (Finder) FindCurrent

func (f Finder) FindCurrent(files []string) ([]string, error)

FindCurrent gets the current file to read from a list of files if ordering_criteria is configured otherwise it returns the list of files.

Deprecated: [v0.82.0] This will be made internal in a future release, tentatively v0.83.0.

func (Finder) FindFiles

func (f Finder) FindFiles() ([]string, error)

FindFiles gets a list of paths given an array of glob patterns to include and exclude

Deprecated: [v0.80.0] This will be made internal in a future release, tentatively v0.83.0.

type HeaderConfig

type HeaderConfig struct {
	Pattern           string            `mapstructure:"pattern"`
	MetadataOperators []operator.Config `mapstructure:"metadata_operators"`
}

type Manager

type Manager struct {
	*zap.SugaredLogger
	// contains filtered or unexported fields
}

func (*Manager) Start

func (m *Manager) Start(persister operator.Persister) error

func (*Manager) Stop

func (m *Manager) Stop() error

Stop will stop the file monitoring process

type MatchingCriteria

type MatchingCriteria struct {
	Include          []string         `mapstructure:"include,omitempty"`
	Exclude          []string         `mapstructure:"exclude,omitempty"`
	OrderingCriteria OrderingCriteria `mapstructure:"ordering_criteria,omitempty"`
}

type NumericSortRule

type NumericSortRule struct {
	BaseSortRule `mapstructure:",squash"`
}

type OrderingCriteria

type OrderingCriteria struct {
	Regex  string         `mapstructure:"regex,omitempty"`
	SortBy []SortRuleImpl `mapstructure:"sort_by,omitempty"`
}

type Reader

type Reader struct {
	*zap.SugaredLogger `json:"-"` // json tag excludes embedded fields from storage

	Fingerprint *fingerprint.Fingerprint
	Offset      int64

	FileAttributes map[string]any

	HeaderFinalized bool
	// contains filtered or unexported fields
}

Reader manages a single file

Deprecated: [v0.80.0] This will be made internal in a future release, tentatively v0.82.0.

func (*Reader) Close

func (r *Reader) Close()

Close will close the file

func (*Reader) Read

func (r *Reader) Read(dst []byte) (int, error)

Read from the file and update the fingerprint if necessary

func (*Reader) ReadToEnd

func (r *Reader) ReadToEnd(ctx context.Context)

ReadToEnd will read until the end of the file

type SortRuleImpl

type SortRuleImpl struct {
	// contains filtered or unexported fields
}

Deprecated: [v0.82.0] This will be made internal in a future release, tentatively v0.83.0.

func (*SortRuleImpl) Unmarshal

func (sr *SortRuleImpl) Unmarshal(component *confmap.Conf) error

type TimestampSortRule

type TimestampSortRule struct {
	BaseSortRule `mapstructure:",squash"`
	Layout       string `mapstructure:"layout,omitempty"`
	Location     string `mapstructure:"location,omitempty"`
}