config

package
v0.1.11 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 15, 2024 License: Apache-2.0 Imports: 8 Imported by: 3

Documentation

Index

Constants

View Source 
const PresetIngesterStage = "preset-ingester"

Variables

This section is empty.

Functions

func JsonUnmarshalStrict added in v0.1.3 

func JsonUnmarshalStrict(data []byte, v interface{}) error

JsonUnmarshalStrict is like Unmarshal except that any fields that are found in the data that do not have corresponding struct members, or mapping keys that are duplicates, will result in an error.

Types

type ConfigFileStruct added in v0.1.3 

type ConfigFileStruct struct {
	LogLevel        string          `yaml:"log-level,omitempty" json:"log-level,omitempty"`
	MetricsSettings MetricsSettings `yaml:"metricsSettings,omitempty" json:"metricsSettings,omitempty"`
	Pipeline        []Stage         `yaml:"pipeline,omitempty" json:"pipeline,omitempty"`
	Parameters      []StageParam    `yaml:"parameters,omitempty" json:"parameters,omitempty"`
	PerfSettings    PerfSettings    `yaml:"perfSettings,omitempty" json:"perfSettings,omitempty"`
}

func ParseConfig 

func ParseConfig(opts Options) (ConfigFileStruct, error)

ParseConfig creates the internal unmarshalled representation from the Pipeline and Parameters json

type Encode 

type Encode struct {
	Type        string                 `yaml:"type" json:"type"`
	Prom        *api.PromEncode        `yaml:"prom,omitempty" json:"prom,omitempty"`
	Kafka       *api.EncodeKafka       `yaml:"kafka,omitempty" json:"kafka,omitempty"`
	S3          *api.EncodeS3          `yaml:"s3,omitempty" json:"s3,omitempty"`
	OtlpLogs    *api.EncodeOtlpLogs    `yaml:"otlplogs,omitempty" json:"otlplogs,omitempty"`
	OtlpMetrics *api.EncodeOtlpMetrics `yaml:"otlpmetrics,omitempty" json:"otlpmetrics,omitempty"`
	OtlpTraces  *api.EncodeOtlpTraces  `yaml:"otlptraces,omitempty" json:"otlptraces,omitempty"`
}

type Extract 

type Extract struct {
	Type       string                `yaml:"type" json:"type"`
	Aggregates *api.Aggregates       `yaml:"aggregates,omitempty" json:"aggregates,omitempty"`
	ConnTrack  *api.ConnTrack        `yaml:"conntrack,omitempty" json:"conntrack,omitempty"`
	Timebased  *api.ExtractTimebased `yaml:"timebased,omitempty" json:"timebased,omitempty"`
}

type File 

type File struct {
	Filename string      `yaml:"filename" json:"filename"`
	Decoder  api.Decoder `yaml:"decoder" json:"decoder"`
	Loop     bool        `yaml:"loop" json:"loop"`
	Chunks   int         `yaml:"chunks" json:"chunks"`
}

type GenericMap 

type GenericMap map[string]interface{}

func (GenericMap) Copy 

func (m GenericMap) Copy() GenericMap

Copy will create a flat copy of GenericMap

func (GenericMap) IsDuplicate added in v0.1.9 

func (m GenericMap) IsDuplicate() bool

func (GenericMap) IsTransportProtocol added in v0.1.10 

func (m GenericMap) IsTransportProtocol() bool

func (GenericMap) IsValidProtocol added in v0.1.10 

func (m GenericMap) IsValidProtocol() bool

type Health 

type Health struct {
	Address string
	Port    string
}

type Ingest 

type Ingest struct {
	Type      string               `yaml:"type" json:"type"`
	File      *File                `yaml:"file,omitempty" json:"file,omitempty"`
	Collector *api.IngestCollector `yaml:"collector,omitempty" json:"collector,omitempty"`
	Kafka     *api.IngestKafka     `yaml:"kafka,omitempty" json:"kafka,omitempty"`
	GRPC      *api.IngestGRPCProto `yaml:"grpc,omitempty" json:"grpc,omitempty"`
	Synthetic *api.IngestSynthetic `yaml:"synthetic,omitempty" json:"synthetic,omitempty"`
	Stdin     *api.IngestStdin     `yaml:"stdin,omitempty" json:"stdin,omitempty"`
}

type MetricsSettings added in v0.1.4 

type MetricsSettings struct {
	api.PromConnectionInfo
	DisableGlobalServer bool   `` /* 255-byte string literal not displayed */
	Prefix              string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix for names of the operational metrics"`
	NoPanic             bool   `yaml:"noPanic,omitempty" json:"noPanic,omitempty"`
	SuppressGoMetrics   bool   `yaml:"suppressGoMetrics,omitempty" json:"suppressGoMetrics,omitempty" doc:"filter out Go and process metrics"`
}

MetricsSettings is similar to api.PromEncode, but is global to the application, ie. it also works with operational metrics. Also, currently FLP doesn't support defining more than one PromEncode stage. If this feature is added later, these global settings will help configuring common setting for all PromEncode stages - PromEncode settings would then act as overrides.

type Options 

type Options struct {
	PipeLine        string
	Parameters      string
	MetricsSettings string
	Health          Health
	Profile         Profile
}

type PerfSettings added in v0.1.4 

type PerfSettings struct {
	BatcherMaxLen  int           `yaml:"batcherMaxLen,omitempty" json:"batcherMaxLen,omitempty"`
	BatcherTimeout time.Duration `yaml:"batcherMaxTimeout,omitempty" json:"batcherMaxTimeout,omitempty"`
	NodeBufferLen  int           `yaml:"nodeBufferLen,omitempty" json:"nodeBufferLen,omitempty"`
}

PerfSettings allows setting some internal configuration parameters

type PipelineBuilderStage 

type PipelineBuilderStage struct {
	// contains filtered or unexported fields
}

PipelineBuilderStage holds information about a created pipeline stage. This stage can be used to chain a following stage, or several of them (resulting in a fork). Example: 

firstStage := NewCollectorPipeline("first stage", ...)
secondStage := firstStage.DecodeJSON("second stage")
thirdStage := secondStage.WriteLoki("third stage", ...)
forkedStage := secondStage.WriteStdout("fork following second stage", ...)

All created stages hold a pointer to the whole pipeline, so that the resulting pipeline can be retrieve from any of the stages: 

forkedStage.GetStages()
forkedStage.GetStageParams()
// is equivalent to:
firstStage.GetStages()
firstStage.GetStageParams()

func NewCollectorPipeline 

func NewCollectorPipeline(name string, ingest api.IngestCollector) PipelineBuilderStage

NewCollectorPipeline creates a new pipeline from an `IngestCollector` initial stage (listening for NetFlows / IPFIX)

func NewGRPCPipeline 

func NewGRPCPipeline(name string, ingest api.IngestGRPCProto) PipelineBuilderStage

NewGRPCPipeline creates a new pipeline from an `IngestGRPCProto` initial stage (listening for NetObserv's eBPF agent protobuf)

func NewKafkaPipeline 

func NewKafkaPipeline(name string, ingest api.IngestKafka) PipelineBuilderStage

NewKafkaPipeline creates a new pipeline from an `IngestKafka` initial stage (listening for flow events on Kafka)

func NewPipeline added in v0.1.3 

func NewPipeline(name string, ingest *Ingest) (PipelineBuilderStage, error)

NewPipeline creates a new pipeline from an existing ingest

func NewPresetIngesterPipeline added in v0.1.11 

func NewPresetIngesterPipeline() PipelineBuilderStage

NewPresetIngesterPipeline creates a new partial pipeline without ingest stage

func (*PipelineBuilderStage) Aggregate 

func (b *PipelineBuilderStage) Aggregate(name string, aggs api.Aggregates) PipelineBuilderStage

Aggregate chains the current stage with an aggregate stage and returns that new stage

func (*PipelineBuilderStage) ConnTrack added in v0.1.3 

func (b *PipelineBuilderStage) ConnTrack(name string, ct api.ConnTrack) PipelineBuilderStage

ConnTrack chains the current stage with a ConnTrack stage and returns that new stage

func (*PipelineBuilderStage) EncodeKafka 

func (b *PipelineBuilderStage) EncodeKafka(name string, kafka api.EncodeKafka) PipelineBuilderStage

EncodeKafka chains the current stage with an EncodeKafka stage (writing to a Kafka topic) and returns that new stage

func (*PipelineBuilderStage) EncodePrometheus 

func (b *PipelineBuilderStage) EncodePrometheus(name string, prom api.PromEncode) PipelineBuilderStage

EncodePrometheus chains the current stage with a PromEncode stage (to expose metrics in Prometheus format) and returns that new stage

func (*PipelineBuilderStage) EncodeS3 added in v0.1.10 

func (b *PipelineBuilderStage) EncodeS3(name string, s3 api.EncodeS3) PipelineBuilderStage

EncodeS3 chains the current stage with an EncodeS3 stage (writing to s3 bucket) and returns that new stage

func (*PipelineBuilderStage) ExtractTimebased added in v0.1.4 

func (b *PipelineBuilderStage) ExtractTimebased(name string, tb api.ExtractTimebased) PipelineBuilderStage

ExtractTimebased chains the current stage with a ExtractTimebased stage and returns that new stage

func (*PipelineBuilderStage) GetStageParams 

func (b *PipelineBuilderStage) GetStageParams() []StageParam

GetStageParams returns the current pipeline stage params. It can be called from any of the stages, they share the same pipeline reference.

func (*PipelineBuilderStage) GetStages 

func (b *PipelineBuilderStage) GetStages() []Stage

GetStages returns the current pipeline stages. It can be called from any of the stages, they share the same pipeline reference.

func (*PipelineBuilderStage) IntoConfigFileStruct added in v0.1.11 

func (b *PipelineBuilderStage) IntoConfigFileStruct(cfs *ConfigFileStruct) *ConfigFileStruct

IntoConfigFileStruct injects the current pipeline and params in the provided ConfigFileStruct object.

func (*PipelineBuilderStage) ToConfigFileStruct added in v0.1.11 

func (b *PipelineBuilderStage) ToConfigFileStruct() *ConfigFileStruct

ToConfigFileStruct returns the current pipeline and params as a new ConfigFileStruct object.

func (*PipelineBuilderStage) TransformFilter 

func (b *PipelineBuilderStage) TransformFilter(name string, filter api.TransformFilter) PipelineBuilderStage

TransformFilter chains the current stage with a TransformFilter stage and returns that new stage

func (*PipelineBuilderStage) TransformGeneric 

func (b *PipelineBuilderStage) TransformGeneric(name string, gen api.TransformGeneric) PipelineBuilderStage

TransformGeneric chains the current stage with a TransformGeneric stage and returns that new stage

func (*PipelineBuilderStage) TransformNetwork 

func (b *PipelineBuilderStage) TransformNetwork(name string, nw api.TransformNetwork) PipelineBuilderStage

TransformNetwork chains the current stage with a TransformNetwork stage and returns that new stage

func (*PipelineBuilderStage) WriteIpfix added in v0.1.9 

func (b *PipelineBuilderStage) WriteIpfix(name string, ipfix api.WriteIpfix) PipelineBuilderStage

WriteIpfix chains the current stage with a WriteIpfix stage and returns that new stage

func (*PipelineBuilderStage) WriteLoki 

func (b *PipelineBuilderStage) WriteLoki(name string, loki api.WriteLoki) PipelineBuilderStage

WriteLoki chains the current stage with a WriteLoki stage and returns that new stage

func (*PipelineBuilderStage) WriteStdout 

func (b *PipelineBuilderStage) WriteStdout(name string, stdout api.WriteStdout) PipelineBuilderStage

WriteStdout chains the current stage with a WriteStdout stage and returns that new stage

type Profile added in v0.1.4 

type Profile struct {
	Port int
}

type Stage 

type Stage struct {
	Name    string `yaml:"name" json:"name"`
	Follows string `yaml:"follows,omitempty" json:"follows,omitempty"`
}

type StageParam 

type StageParam struct {
	Name      string     `yaml:"name" json:"name"`
	Ingest    *Ingest    `yaml:"ingest,omitempty" json:"ingest,omitempty"`
	Transform *Transform `yaml:"transform,omitempty" json:"transform,omitempty"`
	Extract   *Extract   `yaml:"extract,omitempty" json:"extract,omitempty"`
	Encode    *Encode    `yaml:"encode,omitempty" json:"encode,omitempty"`
	Write     *Write     `yaml:"write,omitempty" json:"write,omitempty"`
}

func NewAggregateParams added in v0.1.3 

func NewAggregateParams(name string, aggs api.Aggregates) StageParam

func NewCollectorParams added in v0.1.3 

func NewCollectorParams(name string, ingest api.IngestCollector) StageParam

func NewConnTrackParams added in v0.1.3 

func NewConnTrackParams(name string, ct api.ConnTrack) StageParam

func NewEncodeKafkaParams added in v0.1.3 

func NewEncodeKafkaParams(name string, kafka api.EncodeKafka) StageParam

func NewEncodePrometheusParams added in v0.1.3 

func NewEncodePrometheusParams(name string, prom api.PromEncode) StageParam

func NewEncodeS3Params added in v0.1.10 

func NewEncodeS3Params(name string, s3 api.EncodeS3) StageParam

func NewGRPCParams added in v0.1.3 

func NewGRPCParams(name string, ingest api.IngestGRPCProto) StageParam

func NewKafkaParams added in v0.1.3 

func NewKafkaParams(name string, ingest api.IngestKafka) StageParam

func NewTimbasedParams added in v0.1.4 

func NewTimbasedParams(name string, ct api.ExtractTimebased) StageParam

func NewTransformFilterParams added in v0.1.3 

func NewTransformFilterParams(name string, filter api.TransformFilter) StageParam

func NewTransformGenericParams added in v0.1.3 

func NewTransformGenericParams(name string, gen api.TransformGeneric) StageParam

func NewTransformNetworkParams added in v0.1.3 

func NewTransformNetworkParams(name string, nw api.TransformNetwork) StageParam

func NewWriteIpfixParams added in v0.1.9 

func NewWriteIpfixParams(name string, ipfix api.WriteIpfix) StageParam

func NewWriteLokiParams added in v0.1.3 

func NewWriteLokiParams(name string, loki api.WriteLoki) StageParam

func NewWriteStdoutParams added in v0.1.3 

func NewWriteStdoutParams(name string, stdout api.WriteStdout) StageParam

type Transform 

type Transform struct {
	Type    string                `yaml:"type" json:"type"`
	Generic *api.TransformGeneric `yaml:"generic,omitempty" json:"generic,omitempty"`
	Filter  *api.TransformFilter  `yaml:"filter,omitempty" json:"filter,omitempty"`
	Network *api.TransformNetwork `yaml:"network,omitempty" json:"network,omitempty"`
}

type Write 

type Write struct {
	Type   string           `yaml:"type" json:"type"`
	Loki   *api.WriteLoki   `yaml:"loki,omitempty" json:"loki,omitempty"`
	Stdout *api.WriteStdout `yaml:"stdout,omitempty" json:"stdout,omitempty"`
	Ipfix  *api.WriteIpfix  `yaml:"ipfix,omitempty" json:"ipfix,omitempty"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.