Documentation ¶
Index ¶
- Constants
- func ConnTrackOperationName(operation string) string
- func ConnTrackOutputRecordTypeName(operation string) string
- func DecoderName(decoder string) string
- func FilterOperationName(operation string) string
- func GetEnumName(enum interface{}, operation string) string
- func GetEnumReflectionTypeByFieldName(enumName string) reflect.Type
- func KafkaEncodeBalancerName(operation string) string
- func PromEncodeOperationName(operation string) string
- func TransformFilterOperationName(operation string) string
- func TransformGenericOperationName(operation string) string
- func TransformNetworkOperationName(operation string) string
- type API
- type AggregateBy
- type AggregateDefinition
- type AggregateOperation
- type ClientTLS
- type ConnTrack
- type ConnTrackHash
- type ConnTrackOperationEnum
- type ConnTrackOutputRecordTypeEnum
- type Decoder
- type DecoderEnum
- type Duration
- type EncodeKafka
- type EncodeS3
- type ExtractTimebased
- type FieldGroup
- type FilterOperationEnum
- type GenericTransform
- type GenericTransformRule
- type IngestCollector
- type IngestGRPCProto
- type IngestKafka
- type KafkaEncodeBalancerEnum
- type KeyDefinition
- type NetworkTransformRule
- type NetworkTransformRules
- type OutputField
- type PromEncode
- type PromEncodeOperationEnum
- type PromMetricsFilter
- type PromMetricsItem
- type PromMetricsItems
- type PromTLSConf
- type TimebasedFilterRule
- type TransformFilter
- type TransformFilterOperationEnum
- type TransformFilterRule
- type TransformGeneric
- type TransformGenericOperationEnum
- type TransformNetwork
- type TransformNetworkOperationEnum
- type WriteLoki
- type WriteStdout
Constants ¶
const ( FileType = "file" FileLoopType = "file_loop" FileChunksType = "file_chunks" CollectorType = "collector" GRPCType = "grpc" FakeType = "fake" KafkaType = "kafka" S3Type = "s3" StdoutType = "stdout" LokiType = "loki" AggregateType = "aggregates" TimebasedType = "timebased" PromType = "prom" GenericType = "generic" NetworkType = "network" FilterType = "filter" ConnTrackType = "conntrack" NoneType = "none" AddRegExIfRuleType = "add_regex_if" AddIfRuleType = "add_if" AddSubnetRuleType = "add_subnet" AddLocationRuleType = "add_location" AddServiceRuleType = "add_service" AddKubernetesRuleType = "add_kubernetes" TagYaml = "yaml" TagDoc = "doc" TagEnum = "enum" )
const ( HashIdFieldName = "_HashId" RecordTypeFieldName = "_RecordType" IsFirstFieldName = "_IsFirst" )
const ( OpAddRegexIf = "add_regex_if" OpAddIf = "add_if" OpAddSubnet = "add_subnet" OpAddLocation = "add_location" OpAddService = "add_service" OpAddKubernetes = "add_kubernetes" )
Variables ¶
This section is empty.
Functions ¶
func ConnTrackOperationName ¶
func DecoderName ¶
func FilterOperationName ¶ added in v0.1.4
func GetEnumName ¶
GetEnumName gets the name of an enum value from the representing enum struct based on `TagYaml` tag.
func GetEnumReflectionTypeByFieldName ¶
GetEnumReflectionTypeByFieldName gets the enum struct `reflection Type` from the name of the struct (using fields from `enums{}` struct).
func KafkaEncodeBalancerName ¶
func PromEncodeOperationName ¶
Types ¶
type API ¶
type API struct { PromEncode PromEncode `yaml:"prom" doc:"## Prometheus encode API\nFollowing is the supported API format for prometheus encode:\n"` KafkaEncode EncodeKafka `yaml:"kafka" doc:"## Kafka encode API\nFollowing is the supported API format for kafka encode:\n"` S3Encode EncodeS3 `yaml:"s3" doc:"## S3 encode API\nFollowing is the supported API format for S3 encode:\n"` IngestCollector IngestCollector `yaml:"collector" doc:"## Ingest collector API\nFollowing is the supported API format for the NetFlow / IPFIX collector:\n"` IngestKafka IngestKafka `yaml:"kafka" doc:"## Ingest Kafka API\nFollowing is the supported API format for the kafka ingest:\n"` IngestGRPCProto IngestGRPCProto `` /* 154-byte string literal not displayed */ TransformGeneric TransformGeneric `yaml:"generic" doc:"## Transform Generic API\nFollowing is the supported API format for generic transformations:\n"` TransformFilter TransformFilter `yaml:"filter" doc:"## Transform Filter API\nFollowing is the supported API format for filter transformations:\n"` TransformNetwork TransformNetwork `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"` WriteLoki WriteLoki `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"` WriteStdout WriteStdout `yaml:"stdout" doc:"## Write Standard Output\nFollowing is the supported API format for writing to standard output:\n"` ExtractAggregate AggregateDefinition `` /* 126-byte string literal not displayed */ ConnectionTracking ConnTrack `` /* 126-byte string literal not displayed */ ExtractTimebased ExtractTimebased `` /* 132-byte string literal not displayed */ }
type AggregateBy ¶
type AggregateBy []string
type AggregateDefinition ¶
type AggregateDefinition struct { Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"description of aggregation result"` GroupByKeys AggregateBy `yaml:"groupByKeys,omitempty" json:"groupByKeys,omitempty" doc:"list of fields on which to aggregate"` OperationType AggregateOperation `yaml:"operationType,omitempty" json:"operationType,omitempty" doc:"sum, min, max, avg or raw_values"` OperationKey string `yaml:"operationKey,omitempty" json:"operationKey,omitempty" doc:"internal field on which to perform the operation"` }
type AggregateOperation ¶
type AggregateOperation string
type ClientTLS ¶ added in v0.1.3
type ClientTLS struct { InsecureSkipVerify bool `` /* 144-byte string literal not displayed */ CACertPath string `yaml:"caCertPath,omitempty" json:"caCertPath,omitempty" doc:"path to the CA certificate"` UserCertPath string `yaml:"userCertPath,omitempty" json:"userCertPath,omitempty" doc:"path to the user certificate"` UserKeyPath string `yaml:"userKeyPath,omitempty" json:"userKeyPath,omitempty" doc:"path to the user private key"` }
type ConnTrack ¶
type ConnTrack struct { // TODO: should by a pointer instead? KeyDefinition KeyDefinition `yaml:"keyDefinition,omitempty" doc:"fields that are used to identify the connection"` OutputRecordTypes []string `yaml:"outputRecordTypes,omitempty" enum:"ConnTrackOutputRecordTypeEnum" doc:"output record types to emit"` OutputFields []OutputField `yaml:"outputFields,omitempty" doc:"list of output fields"` EndConnectionTimeout Duration `yaml:"endConnectionTimeout,omitempty" doc:"duration of time to wait from the last flow log to end a connection"` UpdateConnectionInterval Duration `yaml:"updateConnectionInterval,omitempty" doc:"duration of time to wait between update reports of a connection"` }
type ConnTrackHash ¶
type ConnTrackHash struct { FieldGroupRefs []string `yaml:"fieldGroupRefs,omitempty" doc:"list of field group names to build the hash"` FieldGroupARef string `yaml:"fieldGroupARef,omitempty" doc:"field group name of endpoint A"` FieldGroupBRef string `yaml:"fieldGroupBRef,omitempty" doc:"field group name of endpoint B"` }
ConnTrackHash determines how to compute the connection hash. A and B are treated as the endpoints of the connection. When FieldGroupARef and FieldGroupBRef are set, the hash is computed in a way that flow logs from A to B will have the same hash as flow logs from B to A. When they are not set, a different hash will be computed for A->B and B->A, and they are tracked as different connections.
type ConnTrackOperationEnum ¶
type Decoder ¶
type Decoder struct {
Type string `yaml:"type" json:"type" enum:"DecoderEnum" doc:"one of the following:"`
}
type DecoderEnum ¶
type Duration ¶ added in v0.1.3
Duration is a wrapper of time.Duration that allows json marshaling. https://stackoverflow.com/a/48051946/2749989
func (Duration) MarshalJSON ¶ added in v0.1.3
func (Duration) MarshalYAML ¶ added in v0.1.3
func (*Duration) UnmarshalJSON ¶ added in v0.1.3
func (*Duration) UnmarshalYAML ¶ added in v0.1.3
type EncodeKafka ¶
type EncodeKafka struct { Address string `yaml:"address" json:"address" doc:"address of kafka server"` Topic string `yaml:"topic" json:"topic" doc:"kafka topic to write to"` Balancer string `yaml:"balancer,omitempty" json:"balancer,omitempty" enum:"KafkaEncodeBalancerEnum" doc:"one of the following:"` WriteTimeout int64 `` /* 130-byte string literal not displayed */ ReadTimeout int64 `` /* 127-byte string literal not displayed */ BatchBytes int64 `` /* 139-byte string literal not displayed */ BatchSize int `` /* 136-byte string literal not displayed */ TLS *ClientTLS `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"` }
type EncodeS3 ¶ added in v0.1.5
type EncodeS3 struct { Account string `yaml:"account" json:"account" doc:"tenant id for this flow collector"` Endpoint string `yaml:"endpoint" json:"endpoint" doc:"address of s3 server"` AccessKeyId string `yaml:"accessKeyId" json:"accessKeyId" doc:"username to connect to server"` SecretAccessKey string `yaml:"secretAccessKey" json:"secretAccessKey" doc:"password to connect to server"` Bucket string `yaml:"bucket" json:"bucket" doc:"bucket into which to store objects"` WriteTimeout Duration `yaml:"writeTimeout,omitempty" json:"writeTimeout,omitempty" doc:"timeout (in seconds) for write operation"` BatchSize int `` /* 131-byte string literal not displayed */ ObjectHeaderParameters map[string]interface{} `` /* 142-byte string literal not displayed */ }
type ExtractTimebased ¶ added in v0.1.4
type ExtractTimebased struct {
Rules []TimebasedFilterRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of filter rules, each includes:"`
}
type FieldGroup ¶
type FilterOperationEnum ¶ added in v0.1.4
type FilterOperationEnum struct { FilterOperationSum string `yaml:"sum" json:"sum" doc:"set output field to sum of parameters fields in the time window"` FilterOperationAvg string `yaml:"avg" json:"avg" doc:"set output field to average of parameters fields in the time window"` FilterOperationMin string `yaml:"min" json:"min" doc:"set output field to minimum of parameters fields in the time window"` FilterOperationMax string `yaml:"max" json:"max" doc:"set output field to maximum of parameters fields in the time window"` FilterOperationLast string `yaml:"last" json:"last" doc:"set output field to last of parameters fields in the time window"` FilterOperationDiff string `yaml:"diff" json:"diff" doc:"set output field to the difference of the first and last parameters fields in the time window"` }
type GenericTransform ¶
type GenericTransform []GenericTransformRule
type GenericTransformRule ¶
type IngestCollector ¶
type IngestCollector struct { HostName string `yaml:"hostName,omitempty" json:"hostName,omitempty" doc:"the hostname to listen on"` Port int `` /* 156-byte string literal not displayed */ PortLegacy int `` /* 163-byte string literal not displayed */ BatchMaxLen int `` /* 133-byte string literal not displayed */ }
type IngestGRPCProto ¶
type IngestKafka ¶
type IngestKafka struct { Brokers []string `yaml:"brokers,omitempty" json:"brokers,omitempty" doc:"list of kafka broker addresses"` Topic string `yaml:"topic,omitempty" json:"topic,omitempty" doc:"kafka topic to listen on"` GroupId string `yaml:"groupid,omitempty" json:"groupid,omitempty" doc:"separate groupid for each consumer on specified topic"` GroupBalancers []string `` /* 132-byte string literal not displayed */ StartOffset string `` /* 161-byte string literal not displayed */ BatchReadTimeout int64 `yaml:"batchReadTimeout,omitempty" json:"batchReadTimeout,omitempty" doc:"how often (in milliseconds) to process input"` Decoder Decoder `yaml:"decoder,omitempty" json:"decoder" doc:"decoder to use (E.g. json or protobuf)"` BatchMaxLen int `` /* 133-byte string literal not displayed */ PullQueueCapacity int `` /* 127-byte string literal not displayed */ PullMaxBytes int `yaml:"pullMaxBytes,omitempty" json:"pullMaxBytes,omitempty" doc:"the maximum number of bytes being pulled from kafka"` CommitInterval int64 `` /* 192-byte string literal not displayed */ TLS *ClientTLS `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"` }
type KafkaEncodeBalancerEnum ¶
type KafkaEncodeBalancerEnum struct { RoundRobin string `yaml:"roundRobin" json:"roundRobin" doc:"RoundRobin balancer"` LeastBytes string `yaml:"leastBytes" json:"leastBytes" doc:"LeastBytes balancer"` Hash string `yaml:"hash" json:"hash" doc:"Hash balancer"` Crc32 string `yaml:"crc32" json:"crc32" doc:"Crc32 balancer"` Murmur2 string `yaml:"murmur2" json:"murmur2" doc:"Murmur2 balancer"` }
type KeyDefinition ¶
type KeyDefinition struct { FieldGroups []FieldGroup `yaml:"fieldGroups,omitempty" doc:"list of field group definitions"` Hash ConnTrackHash `yaml:"hash,omitempty" doc:"how to build the connection hash"` }
type NetworkTransformRule ¶
type NetworkTransformRule struct { Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"` Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"` Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"` Assignee string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"` }
type NetworkTransformRules ¶
type NetworkTransformRules []NetworkTransformRule
type OutputField ¶
type OutputField struct { Name string `yaml:"name,omitempty" doc:"output field name"` Operation string `yaml:"operation,omitempty" enum:"ConnTrackOperationEnum" doc:"aggregate operation on the field value"` SplitAB bool `yaml:"splitAB,omitempty" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."` Input string `yaml:"input,omitempty" doc:"The input field to base the operation on. When omitted, 'name' is used"` }
type PromEncode ¶
type PromEncode struct { Metrics PromMetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of prometheus metric definitions, each includes:"` Port int `yaml:"port,omitempty" json:"port,omitempty" doc:"port number to expose \"/metrics\" endpoint"` Prefix string `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"` ExpiryTime int `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"seconds of no-flow to wait before deleting prometheus data item"` TLS *PromTLSConf `yaml:"tls,omitempty" json:"tls,omitempty" doc:"TLS configuration for the prometheus endpoint"` }
type PromEncodeOperationEnum ¶
type PromEncodeOperationEnum struct { Gauge string `yaml:"gauge" json:"gauge" doc:"single numerical value that can arbitrarily go up and down"` Counter string `yaml:"counter" json:"counter" doc:"monotonically increasing counter whose value can only increase"` Histogram string `yaml:"histogram" json:"histogram" doc:"counts samples in configurable buckets"` AggHistogram string `yaml:"agg_histogram" json:"agg_histogram" doc:"counts samples in configurable buckets, pre-aggregated via an Aggregate stage"` }
type PromMetricsFilter ¶
type PromMetricsItem ¶
type PromMetricsItem struct { Name string `yaml:"name" json:"name" doc:"the metric name"` Type string `yaml:"type" json:"type" enum:"PromEncodeOperationEnum" doc:"one of the following:"` Filter PromMetricsFilter `yaml:"filter" json:"filter" doc:"an optional criterion to filter entries by"` ValueKey string `yaml:"valueKey" json:"valueKey" doc:"entry key from which to resolve metric value"` Labels []string `yaml:"labels" json:"labels" doc:"labels to be associated with the metric"` Buckets []float64 `yaml:"buckets" json:"buckets" doc:"histogram buckets"` }
type PromMetricsItems ¶
type PromMetricsItems []PromMetricsItem
type PromTLSConf ¶ added in v0.1.4
type TimebasedFilterRule ¶ added in v0.1.4
type TimebasedFilterRule struct { Name string `yaml:"name,omitempty" json:"name,omitempty" doc:"description of filter result"` IndexKey string `yaml:"indexKey,omitempty" json:"indexKey,omitempty" doc:"internal field to index TopK"` OperationType string `` /* 127-byte string literal not displayed */ OperationKey string `yaml:"operationKey,omitempty" json:"operationKey,omitempty" doc:"internal field on which to perform the operation"` TopK int `yaml:"topK,omitempty" json:"topK,omitempty" doc:"number of highest incidence to report (default - report all)"` Reversed bool `yaml:"reversed,omitempty" json:"reversed,omitempty" doc:"report lowest incidence instead of highest (default - false)"` TimeInterval Duration `yaml:"timeInterval,omitempty" json:"timeInterval,omitempty" doc:"time duration of data to use to compute the metric"` }
type TransformFilter ¶
type TransformFilter struct {
Rules []TransformFilterRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of filter rules, each includes:"`
}
type TransformFilterOperationEnum ¶
type TransformFilterOperationEnum struct { RemoveField string `yaml:"remove_field" json:"remove_field" doc:"removes the field from the entry"` RemoveEntryIfExists string `yaml:"remove_entry_if_exists" json:"remove_entry_if_exists" doc:"removes the entry if the field exists"` RemoveEntryIfDoesntExist string `yaml:"remove_entry_if_doesnt_exist" json:"remove_entry_if_doesnt_exist" doc:"removes the entry if the field doesnt exist"` RemoveEntryIfEqual string `yaml:"remove_entry_if_equal" json:"remove_entry_if_equal" doc:"removes the entry if the field value equals specified value"` RemoveEntryIfNotEqual string `` /* 139-byte string literal not displayed */ }
type TransformFilterRule ¶
type TransformFilterRule struct { Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"` Type string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformFilterOperationEnum" doc:"one of the following:"` Value interface{} `yaml:"value,omitempty" json:"value,omitempty" doc:"specified value of input field:"` }
type TransformGeneric ¶
type TransformGeneric struct { Policy string `` /* 143-byte string literal not displayed */ Rules []GenericTransformRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of transform rules, each includes:"` }
type TransformGenericOperationEnum ¶
type TransformGenericOperationEnum struct { PreserveOriginalKeys string `yaml:"preserve_original_keys" json:"preserve_original_keys" doc:"adds new keys in addition to existing keys (default)"` ReplaceKeys string `yaml:"replace_keys" json:"replace_keys" doc:"removes all old keys and uses only the new keys"` }
type TransformNetwork ¶
type TransformNetwork struct { Rules NetworkTransformRules `yaml:"rules" json:"rules" doc:"list of transform rules, each includes:"` KubeConfigPath string `yaml:"kubeConfigPath,omitempty" json:"kubeConfigPath,omitempty" doc:"path to kubeconfig file (optional)"` ServicesFile string `yaml:"servicesFile,omitempty" json:"servicesFile,omitempty" doc:"path to services file (optional, default: /etc/services)"` ProtocolsFile string `` /* 126-byte string literal not displayed */ }
func (*TransformNetwork) GetServiceFiles ¶ added in v0.1.4
func (tn *TransformNetwork) GetServiceFiles() (string, string)
type TransformNetworkOperationEnum ¶
type TransformNetworkOperationEnum struct { AddRegExIf string `yaml:"add_regex_if" json:"add_regex_if" doc:"add output field if input field satisfies regex pattern from parameters field"` AddIf string `yaml:"add_if" json:"add_if" doc:"add output field if input field satisfies criteria from parameters field"` AddSubnet string `yaml:"add_subnet" json:"add_subnet" doc:"add output subnet field from input field and prefix length from parameters field"` AddLocation string `yaml:"add_location" json:"add_location" doc:"add output location fields from input"` AddService string `yaml:"add_service" json:"add_service" doc:"add output network service field from input port and parameters protocol field"` AddKubernetes string `yaml:"add_kubernetes" json:"add_kubernetes" doc:"add output kubernetes fields from input"` }
type WriteLoki ¶
type WriteLoki struct { URL string `yaml:"url,omitempty" json:"url,omitempty" doc:"the address of an existing Loki service to push the flows to"` TenantID string `yaml:"tenantID,omitempty" json:"tenantID,omitempty" doc:"identifies the tenant for the request"` BatchWait string `yaml:"batchWait,omitempty" json:"batchWait,omitempty" doc:"maximum amount of time to wait before sending a batch"` BatchSize int `` /* 126-byte string literal not displayed */ Timeout string `yaml:"timeout,omitempty" json:"timeout,omitempty" doc:"maximum time to wait for a server to respond to a request"` MinBackoff string `yaml:"minBackoff,omitempty" json:"minBackoff,omitempty" doc:"initial backoff time for client connection between retries"` MaxBackoff string `yaml:"maxBackoff,omitempty" json:"maxBackoff,omitempty" doc:"maximum backoff time for client connection between retries"` MaxRetries int `yaml:"maxRetries,omitempty" json:"maxRetries,omitempty" doc:"maximum number of retries for client connections"` Labels []string `yaml:"labels,omitempty" json:"labels,omitempty" doc:"map of record fields to be used as labels"` StaticLabels model.LabelSet `yaml:"staticLabels,omitempty" json:"staticLabels,omitempty" doc:"map of common labels to set on each flow"` IgnoreList []string `yaml:"ignoreList,omitempty" json:"ignoreList,omitempty" doc:"map of record fields to be removed from the record"` ClientConfig *promConfig.HTTPClientConfig `yaml:"clientConfig,omitempty" json:"clientConfig,omitempty" doc:"clientConfig"` TimestampLabel model.LabelName `yaml:"timestampLabel,omitempty" json:"timestampLabel,omitempty" doc:"label to use for time indexing"` // TimestampScale provides the scale in time of the units from the timestamp // E.g. UNIX timescale is '1s' (one second) while other clock sources might have // scales of '1ms' (one millisecond) or just '1' (one nanosecond) // Default value is '1s' TimestampScale string `yaml:"timestampScale,omitempty" json:"timestampScale,omitempty" doc:"timestamp units scale (e.g. for UNIX = 1s)"` }
func (*WriteLoki) SetDefaults ¶ added in v0.1.3
func (w *WriteLoki) SetDefaults()
type WriteStdout ¶
type WriteStdout struct {
Format string `` /* 198-byte string literal not displayed */
}