processors

package
v3.3.12 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 13, 2020 License: BSD-2-Clause Imports: 25 Imported by: 12

Documentation

Overview

Package processors implements preprocessors for ingesters. The intended usage is to create a ProcessorSet and call ProcessorSet.Process(). Calls to ProcessorSet.Process() are thread-safe while Process() calls on specific processors is not.

Index

Constants

View Source 
const (
	JsonExtractProcessor    string = `jsonextract`
	JsonArraySplitProcessor string = `jsonarraysplit`
)
View Source 
const (
	ForwarderProcessor string = `forwarder`
)
View Source 
const (
	GzipProcessor string = `gzip`
)
View Source 
const (
	JsonFilterProcessor string = `jsonfilter`
)
View Source 
const (
	RegexExtractProcessor = `regexextract`
)
View Source 
const (
	RegexRouterProcessor = `regexrouter`
)
View Source 
const (
	RegexTimestampProcessor string = `regextimestamp`
)

Variables

View Source 
var (
	ErrUnknownType   = errors.New("Unknown entry encoder type")
	ErrInvalidWriter = errors.New("Writer is nil")
)
View Source 
var (
	ErrNoUnixOnWindows = errors.New("Unix transport not available on Windows")
	ErrMissingTarget   = errors.New("Target IP:Port or Unix path required")
	ErrUnknownProtocol = errors.New("Unknown protocol")
	ErrUnknownFormat   = errors.New("Unknown format")
	ErrClosed          = errors.New("Closed")
	ErrNilTagger       = errors.New("invalid parameter, missing tagger")
)
View Source 
var (
	ErrMissStrictConflict   = errors.New("Strict-Extraction and Passthrough-Misses are mutually exclusive")
	ErrMissingExtractions   = errors.New("Extractions specifications missing")
	ErrInvalidExtractions   = errors.New("Invalid Extractions")
	ErrInvalidKeyname       = errors.New("Invalid keyname")
	ErrDuplicateKey         = errors.New("Duplicate extraction key")
	ErrDuplicateKeyname     = errors.New("Duplicate keys")
	ErrSingleArraySplitOnly = errors.New("jsonarraysplit only supports a single extraction")
)
View Source 
var (
	ErrMatchAction = errors.New("Match-Action must be either 'pass' or 'drop' (default pass)")
	ErrMatchLogic  = errors.New("Match-Logic must be either 'and' or 'or' (default and)")
)
View Source 
var (
	ErrUnknownProcessor = errors.New("Unknown preprocessor")
	ErrNilConfig        = errors.New("Nil configuration")
	ErrNotFound         = errors.New("Processor not found")
	ErrNotReady         = errors.New("ProcessorSet not ready")
	ErrInvalidEntry     = errors.New("ErrInvalidEntry")
)
View Source 
var (
	ErrMissingRegex           = errors.New("Missing regular expression")
	ErrMissingRouteExtraction = errors.New("Missing route extraction name")
	ErrMissingRoutes          = errors.New("Missing route specifications")
	ErrMissingExtractNames    = errors.New("Regular expression does not extract any names")
)
View Source 
var (
	ErrEmptyRegex = errors.New("Empty regular expression")
	ErrEmptyMatch = errors.New("Empty TS-Match-Name")
	ErrNoSubexps  = errors.New("Must specify at least one subexpression")
)
View Source 
var (
	ErrNotGzipped = errors.New("Input is not a gzipped stream")
)

Functions

func CheckProcessor added in v3.3.0 

func CheckProcessor(id string) error

func PopSet added in v3.3.8 

func PopSet(cnt int) []*entry.Entry

func ProcessorLoadConfig added in v3.3.0 

func ProcessorLoadConfig(vc *config.VariableConfig) (cfg interface{}, err error)

Types

type EntryEncoder added in v3.3.8 

type EntryEncoder interface {
	Encode(*entry.Entry) error
	Reset(io.Writer)
}

type Forwarder added in v3.3.8 

type Forwarder struct {
	ForwarderConfig
	sync.Mutex
	// contains filtered or unexported fields
}

func NewForwarder added in v3.3.8 

func NewForwarder(cfg ForwarderConfig, tgr Tagger) (nf *Forwarder, err error)

func (*Forwarder) Close added in v3.3.8 

func (nf *Forwarder) Close() (err error)

func (*Forwarder) Process added in v3.3.8 

func (nf *Forwarder) Process(ent *entry.Entry) (r []*entry.Entry, err error)

type ForwarderConfig added in v3.3.8 

type ForwarderConfig struct {
	Target                   string
	Protocol                 string
	Delimiter                string
	Format                   string
	Tag                      []string
	Regex                    []string
	Source                   []string
	Timeout                  uint //timeout in seconds for a write
	Buffer                   uint //number of entries in flight (basically channel buffer size)
	Non_Blocking             bool
	Insecure_Skip_TLS_Verify bool
}

func ForwarderLoadConfig added in v3.3.8 

func ForwarderLoadConfig(vc *config.VariableConfig) (c ForwarderConfig, err error)

func (*ForwarderConfig) Validate added in v3.3.8 

func (nfc *ForwarderConfig) Validate() (err error)

type GzipDecompressor 

type GzipDecompressor struct {
	GzipDecompressorConfig
	// contains filtered or unexported fields
}

GzipDecompressor does not have any state

func NewGzipDecompressor 

func NewGzipDecompressor(cfg GzipDecompressorConfig) (*GzipDecompressor, error)

func (GzipDecompressor) Close added in v3.3.8 

func (n GzipDecompressor) Close() error

func (*GzipDecompressor) Config 

func (gd *GzipDecompressor) Config(v interface{}) (err error)

func (*GzipDecompressor) Process 

func (gd *GzipDecompressor) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type GzipDecompressorConfig 

type GzipDecompressorConfig struct {
	Passthrough_Non_Gzip bool
}

func GzipLoadConfig 

func GzipLoadConfig(vc *config.VariableConfig) (c GzipDecompressorConfig, err error)

type JsonArraySplitConfig added in v3.3.0 

type JsonArraySplitConfig struct {
	Passthrough_Misses bool
	Extraction         string
	Force_JSON_Object  bool
}

func JsonArraySplitLoadConfig added in v3.3.0 

func JsonArraySplitLoadConfig(vc *config.VariableConfig) (c JsonArraySplitConfig, err error)

type JsonArraySplitter added in v3.3.0 

type JsonArraySplitter struct {
	JsonArraySplitConfig
	// contains filtered or unexported fields
}

func NewJsonArraySplitter added in v3.3.0 

func NewJsonArraySplitter(cfg JsonArraySplitConfig) (*JsonArraySplitter, error)

func (JsonArraySplitter) Close added in v3.3.8 

func (n JsonArraySplitter) Close() error

func (*JsonArraySplitter) Config added in v3.3.0 

func (j *JsonArraySplitter) Config(v interface{}) (err error)

func (*JsonArraySplitter) Process added in v3.3.0 

func (je *JsonArraySplitter) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type JsonExtractConfig added in v3.3.0 

type JsonExtractConfig struct {
	Passthrough_Misses bool
	Strict_Extraction  bool
	Force_JSON_Object  bool
	Extractions        string
}

func JsonExtractLoadConfig added in v3.3.0 

func JsonExtractLoadConfig(vc *config.VariableConfig) (c JsonExtractConfig, err error)

type JsonExtractor added in v3.3.0 

type JsonExtractor struct {
	JsonExtractConfig
	// contains filtered or unexported fields
}

JsonExtractor

func NewJsonExtractor added in v3.3.0 

func NewJsonExtractor(cfg JsonExtractConfig) (*JsonExtractor, error)

func (JsonExtractor) Close added in v3.3.8 

func (n JsonExtractor) Close() error

func (*JsonExtractor) Config added in v3.3.0 

func (j *JsonExtractor) Config(v interface{}) (err error)

func (*JsonExtractor) Process added in v3.3.0 

func (je *JsonExtractor) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type JsonFilter added in v3.3.7 

type JsonFilter struct {
	JsonFilterConfig
	// contains filtered or unexported fields
}

func NewJsonFilter added in v3.3.7 

func NewJsonFilter(cfg JsonFilterConfig) (*JsonFilter, error)

NewJsonFilter instantiates a JsonFilter preprocessor. It will attempt to open and read the files specified in the configuration; nonexistent files or permissions problems will return an error.

func (JsonFilter) Close added in v3.3.8 

func (n JsonFilter) Close() error

func (*JsonFilter) Config added in v3.3.7 

func (j *JsonFilter) Config(v interface{}) (err error)

func (*JsonFilter) Process added in v3.3.7 

func (j *JsonFilter) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type JsonFilterConfig added in v3.3.7 

type JsonFilterConfig struct {
	// what to do when an entry matches: "pass" or "drop"
	Match_Action string

	// "and" or "or", specifying that either *all* fields must match or that *any* field will be sufficient
	Match_Logic string

	// each Field-Filter consists of the field to match, a comma, and the path to the file containing possible values, e.g. "foo.bar,/tmp/values"
	Field_Filter []string
}

func JsonFilterLoadConfig added in v3.3.7 

func JsonFilterLoadConfig(vc *config.VariableConfig) (c JsonFilterConfig, err error)

type Processor added in v3.3.0 

type Processor interface {
	Process(*entry.Entry) ([]*entry.Entry, error) //process an data item potentially setting a tag
	Close() error                                 //give the processor a chance to tide up
}

Processor is an interface that takes a block of entries and processes them, returning a new block

type ProcessorConfig added in v3.3.0 

type ProcessorConfig map[string]*config.VariableConfig

func (ProcessorConfig) CheckConfig added in v3.3.0 

func (pc ProcessorConfig) CheckConfig(name string) (err error)

func (ProcessorConfig) CheckProcessors added in v3.3.0 

func (pc ProcessorConfig) CheckProcessors(set []string) (err error)

func (ProcessorConfig) ProcessorSet added in v3.3.0 

func (pc ProcessorConfig) ProcessorSet(t tagWriter, names []string) (pr *ProcessorSet, err error)

func (ProcessorConfig) Validate added in v3.3.0 

func (pc ProcessorConfig) Validate() (err error)

type ProcessorSet added in v3.3.0 

type ProcessorSet struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewProcessorSet added in v3.3.0 

func NewProcessorSet(wtr entWriter) *ProcessorSet

func (*ProcessorSet) AddProcessor added in v3.3.0 

func (pr *ProcessorSet) AddProcessor(p Processor)

func (*ProcessorSet) Close added in v3.3.8 

func (pr *ProcessorSet) Close() (err error)

Close will close the underlying preprocessors within the set. This function DOES NOT close the ingest muxer handle. It is ONLY for shutting down preprocessors

func (*ProcessorSet) Enabled added in v3.3.0 

func (pr *ProcessorSet) Enabled() bool

func (*ProcessorSet) Process added in v3.3.0 

func (pr *ProcessorSet) Process(ent *entry.Entry) error

func (*ProcessorSet) ProcessContext added in v3.3.2 

func (pr *ProcessorSet) ProcessContext(ent *entry.Entry, ctx context.Context) error

type RegexExtractConfig added in v3.3.0 

type RegexExtractConfig struct {
	Passthrough_Misses bool
	Regex              string
	Template           string
}

func RegexExtractLoadConfig added in v3.3.0 

func RegexExtractLoadConfig(vc *config.VariableConfig) (c RegexExtractConfig, err error)

type RegexExtractor added in v3.3.0 

type RegexExtractor struct {
	RegexExtractConfig
	// contains filtered or unexported fields
}

func NewRegexExtractor added in v3.3.0 

func NewRegexExtractor(cfg RegexExtractConfig) (*RegexExtractor, error)

func (RegexExtractor) Close added in v3.3.8 

func (n RegexExtractor) Close() error

func (*RegexExtractor) Config added in v3.3.0 

func (re *RegexExtractor) Config(v interface{}) (err error)

func (*RegexExtractor) Process added in v3.3.0 

func (re *RegexExtractor) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type RegexRouteConfig added in v3.3.0 

type RegexRouteConfig struct {
	Regex            string
	Route_Extraction string
	Route            []string
	Drop_Misses      bool
}

func RegexRouteLoadConfig added in v3.3.0 

func RegexRouteLoadConfig(vc *config.VariableConfig) (c RegexRouteConfig, err error)

type RegexRouter added in v3.3.0 

type RegexRouter struct {
	RegexRouteConfig
	// contains filtered or unexported fields
}

func NewRegexRouter added in v3.3.0 

func NewRegexRouter(cfg RegexRouteConfig, tagger Tagger) (*RegexRouter, error)

func (RegexRouter) Close added in v3.3.8 

func (n RegexRouter) Close() error

func (*RegexRouter) Config added in v3.3.0 

func (rr *RegexRouter) Config(v interface{}, tagger Tagger) (err error)

func (*RegexRouter) Process added in v3.3.0 

func (rr *RegexRouter) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type RegexTimestamp added in v3.3.0 

type RegexTimestamp struct {
	RegexTimestampConfig
	// contains filtered or unexported fields
}

func NewRegexTimestampProcessor added in v3.3.0 

func NewRegexTimestampProcessor(cfg RegexTimestampConfig) (*RegexTimestamp, error)

func (RegexTimestamp) Close added in v3.3.8 

func (n RegexTimestamp) Close() error

func (*RegexTimestamp) Config added in v3.3.0 

func (rt *RegexTimestamp) Config(v interface{}) (err error)

func (*RegexTimestamp) Process added in v3.3.0 

func (rt *RegexTimestamp) Process(ent *entry.Entry) (rset []*entry.Entry, err error)

type RegexTimestampConfig added in v3.3.0 

type RegexTimestampConfig struct {
	Regex                     string // the regular expression to apply to the data
	TS_Match_Name             string // the submatch which contains the timestamp
	Timestamp_Format_Override string
	Timezone_Override         string
	Assume_Local_Timezone     bool
}

func RegexTimestampLoadConfig added in v3.3.0 

func RegexTimestampLoadConfig(vc *config.VariableConfig) (c RegexTimestampConfig, err error)

type SetAllocator added in v3.3.8 

type SetAllocator struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewSetAllocator added in v3.3.8 

func NewSetAllocator(allocSize, reallocSize int) (sa *SetAllocator, err error)

func (*SetAllocator) Get added in v3.3.8 

func (sa *SetAllocator) Get(cnt int) (r []*entry.Entry)

type Tagger added in v3.3.0 

type Tagger interface {
	NegotiateTag(name string) (entry.EntryTag, error)
	LookupTag(entry.EntryTag) (string, bool)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.