api

package
v0.1.3-rc3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 2, 2022 License: Apache-2.0 Imports: 11 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FileType              = "file"
	FileLoopType          = "file_loop"
	FileChunksType        = "file_chunks"
	CollectorType         = "collector"
	GRPCType              = "grpc"
	FakeType              = "fake"
	KafkaType             = "kafka"
	StdoutType            = "stdout"
	LokiType              = "loki"
	AggregateType         = "aggregates"
	PromType              = "prom"
	GenericType           = "generic"
	NetworkType           = "network"
	FilterType            = "filter"
	ConnTrackType         = "conntrack"
	NoneType              = "none"
	ConnTrackingRuleType  = "conn_tracking"
	AddRegExIfRuleType    = "add_regex_if"
	AddIfRuleType         = "add_if"
	AddSubnetRuleType     = "add_subnet"
	AddLocationRuleType   = "add_location"
	AddServiceRuleType    = "add_service"
	AddKubernetesRuleType = "add_kubernetes"

	TagYaml = "yaml"
	TagDoc  = "doc"
	TagEnum = "enum"
)
View Source 
const (
	HashIdFieldName     = "_HashId"
	RecordTypeFieldName = "_RecordType"
)

Variables

This section is empty.

Functions

func ConnTrackOperationName 

func ConnTrackOperationName(operation string) string

func ConnTrackOutputRecordTypeName 

func ConnTrackOutputRecordTypeName(operation string) string

func DecoderName 

func DecoderName(decoder string) string

func GetEnumName 

func GetEnumName(enum interface{}, operation string) string

GetEnumName gets the name of an enum value from the representing enum struct based on `TagYaml` tag.

func GetEnumReflectionTypeByFieldName 

func GetEnumReflectionTypeByFieldName(enumName string) reflect.Type

GetEnumReflectionTypeByFieldName gets the enum struct `reflection Type` from the name of the struct (using fields from `enums{}` struct).

func KafkaEncodeBalancerName 

func KafkaEncodeBalancerName(operation string) string

func PromEncodeOperationName 

func PromEncodeOperationName(operation string) string

func TransformFilterOperationName 

func TransformFilterOperationName(operation string) string

func TransformGenericOperationName 

func TransformGenericOperationName(operation string) string

func TransformNetworkOperationName 

func TransformNetworkOperationName(operation string) string

Types

type API 

type API struct {
	PromEncode         PromEncode          `yaml:"prom" doc:"## Prometheus encode API\nFollowing is the supported API format for prometheus encode:\n"`
	KafkaEncode        EncodeKafka         `yaml:"kafka" doc:"## Kafka encode API\nFollowing is the supported API format for kafka encode:\n"`
	IngestCollector    IngestCollector     `yaml:"collector" doc:"## Ingest collector API\nFollowing is the supported API format for the NetFlow / IPFIX collector:\n"`
	IngestKafka        IngestKafka         `yaml:"kafka" doc:"## Ingest Kafka API\nFollowing is the supported API format for the kafka ingest:\n"`
	IngestGRPCProto    IngestGRPCProto     `` /* 154-byte string literal not displayed */
	TransformGeneric   TransformGeneric    `yaml:"generic" doc:"## Transform Generic API\nFollowing is the supported API format for generic transformations:\n"`
	TransformFilter    TransformFilter     `yaml:"filter" doc:"## Transform Filter API\nFollowing is the supported API format for filter transformations:\n"`
	TransformNetwork   TransformNetwork    `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"`
	WriteLoki          WriteLoki           `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"`
	WriteStdout        WriteStdout         `yaml:"stdout" doc:"## Write Standard Output\nFollowing is the supported API format for writing to standard output:\n"`
	ExtractAggregate   AggregateDefinition `` /* 126-byte string literal not displayed */
	ConnectionTracking ConnTrack           `` /* 126-byte string literal not displayed */
}

type AggregateBy 

type AggregateBy []string

type AggregateDefinition 

type AggregateDefinition struct {
	Name      string             `yaml:"name,omitempty" json:"name,omitempty" doc:"description of aggregation result"`
	By        AggregateBy        `yaml:"by,omitempty" json:"by,omitempty" doc:"list of fields on which to aggregate"`
	Operation AggregateOperation `yaml:"operation,omitempty" json:"operation,omitempty" doc:"sum, min, max, avg or raw_values"`
	RecordKey string             `yaml:"recordKey,omitempty" json:"recordKey,omitempty" doc:"internal field on which to perform the operation"`
	TopK      int                `yaml:"topK,omitempty" json:"topK,omitempty" doc:"number of highest incidence to report (default - report all)"`
}

type AggregateOperation 

type AggregateOperation string

type ClientTLS added in v0.1.3 

type ClientTLS struct {
	InsecureSkipVerify bool   `` /* 144-byte string literal not displayed */
	CACertPath         string `yaml:"caCertPath,omitempty" json:"caCertPath,omitempty" doc:"path to the CA certificate"`
	UserCertPath       string `yaml:"userCertPath,omitempty" json:"userCertPath,omitempty" doc:"path to the user certificate"`
	UserKeyPath        string `yaml:"userKeyPath,omitempty" json:"userKeyPath,omitempty" doc:"path to the user private key"`
}

func (*ClientTLS) Build added in v0.1.3 

func (c *ClientTLS) Build() (*tls.Config, error)

type ConnTrack 

type ConnTrack struct {
	// TODO: should by a pointer instead?
	KeyDefinition        KeyDefinition `yaml:"keyDefinition,omitempty" doc:"fields that are used to identify the connection"`
	OutputRecordTypes    []string      `yaml:"outputRecordTypes,omitempty" enum:"ConnTrackOutputRecordTypeEnum" doc:"output record types to emit"`
	OutputFields         []OutputField `yaml:"outputFields,omitempty" doc:"list of output fields"`
	EndConnectionTimeout Duration      `yaml:"endConnectionTimeout,omitempty" doc:"duration of time to wait from the last flow log to end a connection"`
}

type ConnTrackHash 

type ConnTrackHash struct {
	FieldGroupRefs []string `yaml:"fieldGroupRefs,omitempty" doc:"list of field group names to build the hash"`
	FieldGroupARef string   `yaml:"fieldGroupARef,omitempty" doc:"field group name of endpoint A"`
	FieldGroupBRef string   `yaml:"fieldGroupBRef,omitempty" doc:"field group name of endpoint B"`
}

ConnTrackHash determines how to compute the connection hash. A and B are treated as the endpoints of the connection. When FieldGroupARef and FieldGroupBRef are set, the hash is computed in a way that flow logs from A to B will have the same hash as flow logs from B to A. When they are not set, a different hash will be computed for A->B and B->A, and they are tracked as different connections.

type ConnTrackOperationEnum 

type ConnTrackOperationEnum struct {
	Sum   string `yaml:"sum" doc:"sum"`
	Count string `yaml:"count" doc:"count"`
	Min   string `yaml:"min" doc:"min"`
	Max   string `yaml:"max" doc:"max"`
}

type ConnTrackOutputRecordTypeEnum 

type ConnTrackOutputRecordTypeEnum struct {
	NewConnection string `yaml:"newConnection" doc:"New connection"`
	EndConnection string `yaml:"endConnection" doc:"End connection"`
	FlowLog       string `yaml:"flowLog" doc:"Flow log"`
}

type Decoder 

type Decoder struct {
	Type string `yaml:"type" json:"type" enum:"DecoderEnum" doc:"one of the following:"`
}

type DecoderEnum 

type DecoderEnum struct {
	JSON     string `yaml:"json" json:"json" doc:"JSON decoder"`
	Protobuf string `yaml:"protobuf" json:"protobuf" doc:"Protobuf decoder"`
}

type Duration added in v0.1.3 

type Duration struct {
	time.Duration
}

Duration is a wrapper of time.Duration that allows json marshaling. https://stackoverflow.com/a/48051946/2749989

func (Duration) MarshalJSON added in v0.1.3 

func (d Duration) MarshalJSON() ([]byte, error)

func (Duration) MarshalYAML added in v0.1.3 

func (d Duration) MarshalYAML() (interface{}, error)

func (*Duration) UnmarshalJSON added in v0.1.3 

func (d *Duration) UnmarshalJSON(b []byte) error

func (*Duration) UnmarshalYAML added in v0.1.3 

func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error

type EncodeKafka 

type EncodeKafka struct {
	Address      string     `yaml:"address" json:"address" doc:"address of kafka server"`
	Topic        string     `yaml:"topic" json:"topic" doc:"kafka topic to write to"`
	Balancer     string     `yaml:"balancer,omitempty" json:"balancer,omitempty" enum:"KafkaEncodeBalancerEnum" doc:"one of the following:"`
	WriteTimeout int64      `` /* 130-byte string literal not displayed */
	ReadTimeout  int64      `` /* 127-byte string literal not displayed */
	BatchBytes   int64      `` /* 139-byte string literal not displayed */
	BatchSize    int        `` /* 136-byte string literal not displayed */
	TLS          *ClientTLS `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"`
}

type FieldGroup 

type FieldGroup struct {
	Name   string   `yaml:"name,omitempty" doc:"field group name"`
	Fields []string `yaml:"fields" doc:"list of fields in the group"`
}

type GenericTransform 

type GenericTransform []GenericTransformRule

type GenericTransformRule 

type GenericTransformRule struct {
	Input  string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"`
	Output string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
}

type IngestCollector 

type IngestCollector struct {
	HostName    string `yaml:"hostName,omitempty" json:"hostName,omitempty" doc:"the hostname to listen on"`
	Port        int    `` /* 156-byte string literal not displayed */
	PortLegacy  int    `` /* 163-byte string literal not displayed */
	BatchMaxLen int    `` /* 133-byte string literal not displayed */
}

type IngestGRPCProto 

type IngestGRPCProto struct {
	Port      int `yaml:"port,omitempty" json:"port,omitempty" doc:"the port number to listen on"`
	BufferLen int `` /* 181-byte string literal not displayed */
}

type IngestKafka 

type IngestKafka struct {
	Brokers          []string   `yaml:"brokers,omitempty" json:"brokers,omitempty" doc:"list of kafka broker addresses"`
	Topic            string     `yaml:"topic,omitempty" json:"topic,omitempty" doc:"kafka topic to listen on"`
	GroupId          string     `yaml:"groupid,omitempty" json:"groupid,omitempty" doc:"separate groupid for each consumer on specified topic"`
	GroupBalancers   []string   `` /* 132-byte string literal not displayed */
	StartOffset      string     `` /* 161-byte string literal not displayed */
	BatchReadTimeout int64      `yaml:"batchReadTimeout,omitempty" json:"batchReadTimeout,omitempty" doc:"how often (in milliseconds) to process input"`
	Decoder          Decoder    `yaml:"decoder,omitempty" json:"decoder" doc:"decoder to use (E.g. json or protobuf)"`
	BatchMaxLen      int        `` /* 133-byte string literal not displayed */
	CommitInterval   int64      `` /* 192-byte string literal not displayed */
	TLS              *ClientTLS `yaml:"tls" json:"tls" doc:"TLS client configuration (optional)"`
}

type KafkaEncodeBalancerEnum 

type KafkaEncodeBalancerEnum struct {
	RoundRobin string `yaml:"roundRobin" json:"roundRobin" doc:"RoundRobin balancer"`
	LeastBytes string `yaml:"leastBytes" json:"leastBytes" doc:"LeastBytes balancer"`
	Hash       string `yaml:"hash" json:"hash" doc:"Hash balancer"`
	Crc32      string `yaml:"crc32" json:"crc32" doc:"Crc32 balancer"`
	Murmur2    string `yaml:"murmur2" json:"murmur2" doc:"Murmur2 balancer"`
}

type KeyDefinition 

type KeyDefinition struct {
	FieldGroups []FieldGroup  `yaml:"fieldGroups,omitempty" doc:"list of field group definitions"`
	Hash        ConnTrackHash `yaml:"hash,omitempty" doc:"how to build the connection hash"`
}

type NetworkTransformRule 

type NetworkTransformRule struct {
	Input      string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"`
	Output     string `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
	Type       string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformNetworkOperationEnum" doc:"one of the following:"`
	Parameters string `yaml:"parameters,omitempty" json:"parameters,omitempty" doc:"parameters specific to type"`
	Assignee   string `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
}

type NetworkTransformRules 

type NetworkTransformRules []NetworkTransformRule

type OutputField 

type OutputField struct {
	Name      string `yaml:"name,omitempty" doc:"output field name"`
	Operation string `yaml:"operation,omitempty" enum:"ConnTrackOperationEnum" doc:"aggregate operation on the field value"`
	SplitAB   bool   `yaml:"splitAB,omitempty" doc:"When true, 2 output fields will be created. One for A->B and one for B->A flows."`
	Input     string `yaml:"input,omitempty" doc:"The input field to base the operation on. When omitted, 'name' is used"`
}

type PromEncode 

type PromEncode struct {
	Metrics    PromMetricsItems `yaml:"metrics,omitempty" json:"metrics,omitempty" doc:"list of prometheus metric definitions, each includes:"`
	Port       int              `yaml:"port,omitempty" json:"port,omitempty" doc:"port number to expose \"/metrics\" endpoint"`
	Prefix     string           `yaml:"prefix,omitempty" json:"prefix,omitempty" doc:"prefix added to each metric name"`
	ExpiryTime int              `yaml:"expiryTime,omitempty" json:"expiryTime,omitempty" doc:"seconds of no-flow to wait before deleting prometheus data item"`
}

type PromEncodeOperationEnum 

type PromEncodeOperationEnum struct {
	Gauge     string `yaml:"gauge" json:"gauge" doc:"single numerical value that can arbitrarily go up and down"`
	Counter   string `yaml:"counter" json:"counter" doc:"monotonically increasing counter whose value can only increase"`
	Histogram string `yaml:"histogram" json:"histogram" doc:"counts samples in configurable buckets"`
}

type PromMetricsFilter 

type PromMetricsFilter struct {
	Key   string `yaml:"key" json:"key" doc:"the key to match and filter by"`
	Value string `yaml:"value" json:"value" doc:"the value to match and filter by"`
}

type PromMetricsItem 

type PromMetricsItem struct {
	Name     string            `yaml:"name" json:"name" doc:"the metric name"`
	Type     string            `yaml:"type" json:"type" enum:"PromEncodeOperationEnum" doc:"one of the following:"`
	Filter   PromMetricsFilter `yaml:"filter" json:"filter" doc:"the criterion to filter entries by"`
	ValueKey string            `yaml:"valueKey" json:"valueKey" doc:"entry key from which to resolve metric value"`
	Labels   []string          `yaml:"labels" json:"labels" doc:"labels to be associated with the metric"`
	Buckets  []float64         `yaml:"buckets" json:"buckets" doc:"histogram buckets"`
}

type PromMetricsItems 

type PromMetricsItems []PromMetricsItem

type TransformFilter 

type TransformFilter struct {
	Rules []TransformFilterRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of filter rules, each includes:"`
}

type TransformFilterOperationEnum 

type TransformFilterOperationEnum struct {
	RemoveField              string `yaml:"remove_field" json:"remove_field" doc:"removes the field from the entry"`
	RemoveEntryIfExists      string `yaml:"remove_entry_if_exists" json:"remove_entry_if_exists" doc:"removes the entry if the field exists"`
	RemoveEntryIfDoesntExist string `yaml:"remove_entry_if_doesnt_exist" json:"remove_entry_if_doesnt_exist" doc:"removes the entry if the field doesnt exist"`
}

type TransformFilterRule 

type TransformFilterRule struct {
	Input string `yaml:"input,omitempty" json:"input,omitempty" doc:"entry input field"`
	Type  string `yaml:"type,omitempty" json:"type,omitempty" enum:"TransformFilterOperationEnum" doc:"one of the following:"`
}

type TransformGeneric 

type TransformGeneric struct {
	Policy string                 `` /* 143-byte string literal not displayed */
	Rules  []GenericTransformRule `yaml:"rules,omitempty" json:"rules,omitempty" doc:"list of transform rules, each includes:"`
}

type TransformGenericOperationEnum 

type TransformGenericOperationEnum struct {
	PreserveOriginalKeys string `yaml:"preserve_original_keys" json:"preserve_original_keys" doc:"adds new keys in addition to existing keys (default)"`
	ReplaceKeys          string `yaml:"replace_keys" json:"replace_keys" doc:"removes all old keys and uses only the new keys"`
}

type TransformNetwork 

type TransformNetwork struct {
	Rules          NetworkTransformRules `yaml:"rules" json:"rules" doc:"list of transform rules, each includes:"`
	KubeConfigPath string                `yaml:"kubeConfigPath,omitempty" json:"kubeConfigPath,omitempty" doc:"path to kubeconfig file (optional)"`
	ServicesFile   string                `yaml:"servicesFile,omitempty" json:"servicesFile,omitempty" doc:"path to services file (optional, default: /etc/services)"`
	ProtocolsFile  string                `` /* 126-byte string literal not displayed */
}

type TransformNetworkOperationEnum 

type TransformNetworkOperationEnum struct {
	ConnTracking  string `` /* 148-byte string literal not displayed */
	AddRegExIf    string `yaml:"add_regex_if" json:"add_regex_if" doc:"add output field if input field satisfies regex pattern from parameters field"`
	AddIf         string `yaml:"add_if" json:"add_if" doc:"add output field if input field satisfies criteria from parameters field"`
	AddSubnet     string `yaml:"add_subnet" json:"add_subnet" doc:"add output subnet field from input field and prefix length from parameters field"`
	AddLocation   string `yaml:"add_location" json:"add_location" doc:"add output location fields from input"`
	AddService    string `yaml:"add_service" json:"add_service" doc:"add output network service field from input port and parameters protocol field"`
	AddKubernetes string `yaml:"add_kubernetes" json:"add_kubernetes" doc:"add output kubernetes fields from input"`
}

type WriteLoki 

type WriteLoki struct {
	URL            string                       `yaml:"url,omitempty" json:"url,omitempty" doc:"the address of an existing Loki service to push the flows to"`
	TenantID       string                       `yaml:"tenantID,omitempty" json:"tenantID,omitempty" doc:"identifies the tenant for the request"`
	BatchWait      string                       `yaml:"batchWait,omitempty" json:"batchWait,omitempty" doc:"maximum amount of time to wait before sending a batch"`
	BatchSize      int                          `` /* 126-byte string literal not displayed */
	Timeout        string                       `yaml:"timeout,omitempty" json:"timeout,omitempty" doc:"maximum time to wait for a server to respond to a request"`
	MinBackoff     string                       `yaml:"minBackoff,omitempty" json:"minBackoff,omitempty" doc:"initial backoff time for client connection between retries"`
	MaxBackoff     string                       `yaml:"maxBackoff,omitempty" json:"maxBackoff,omitempty" doc:"maximum backoff time for client connection between retries"`
	MaxRetries     int                          `yaml:"maxRetries,omitempty" json:"maxRetries,omitempty" doc:"maximum number of retries for client connections"`
	Labels         []string                     `yaml:"labels,omitempty" json:"labels,omitempty" doc:"map of record fields to be used as labels"`
	StaticLabels   model.LabelSet               `yaml:"staticLabels,omitempty" json:"staticLabels,omitempty" doc:"map of common labels to set on each flow"`
	IgnoreList     []string                     `yaml:"ignoreList,omitempty" json:"ignoreList,omitempty" doc:"map of record fields to be removed from the record"`
	ClientConfig   *promConfig.HTTPClientConfig `yaml:"clientConfig,omitempty" json:"clientConfig,omitempty" doc:"clientConfig"`
	TimestampLabel model.LabelName              `yaml:"timestampLabel,omitempty" json:"timestampLabel,omitempty" doc:"label to use for time indexing"`
	// TimestampScale provides the scale in time of the units from the timestamp
	// E.g. UNIX timescale is '1s' (one second) while other clock sources might have
	// scales of '1ms' (one millisecond) or just '1' (one nanosecond)
	// Default value is '1s'
	TimestampScale string `yaml:"timestampScale,omitempty" json:"timestampScale,omitempty" doc:"timestamp units scale (e.g. for UNIX = 1s)"`
}

func (*WriteLoki) SetDefaults added in v0.1.3 

func (w *WriteLoki) SetDefaults()

func (*WriteLoki) Validate 

func (wl *WriteLoki) Validate() error

type WriteStdout 

type WriteStdout struct {
	Format string `` /* 198-byte string literal not displayed */
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.