splunk

README

splunk HTTP Event Collector output

It sends events to splunk.

By default it only stores original event under the "event" key according to the Splunk output format.

If other fields are required it is possible to copy fields values from the original event to the other fields relative to the output json. Copies are not allowed directly to the root of output event or "event" field and any of its subfields.

For example, timestamps and service name can be copied to provide additional meta data to the Splunk:

copy_fields:
  - from: ts
  	to: time
  - from: service
  	to: fields.service_name

Here the plugin will lookup for "ts" and "service" fields in the original event and if they are present they will be copied to the output json starting on the same level as the "event" key. If the field is not found in the original event plugin will not populate new field in output json.

In:

{
  "ts":"1723651045",
  "service":"some-service",
  "message":"something happened"
}

Out:

{
  "event": {
    "ts":"1723651045",
    "service":"some-service",
    "message":"something happened"
  },
  "time": "1723651045",
  "fields": {
    "service_name": "some-service"
  }
}

Config params

endpoint string required

A full URI address of splunk HEC endpoint. Format: http://127.0.0.1:8088/services/collector.

use_gzip bool default=false

If set, the plugin will use gzip encoding.

gzip_compression_level string default=default options=default|no|best-speed|best-compression|huffman-only

Gzip compression level. Used if use_gzip=true.

token string required

Token for an authentication for a HEC endpoint.

keep_alive KeepAliveConfig

Keep-alive config.

KeepAliveConfig params:

• max_idle_conn_duration - idle keep-alive connections are closed after this duration. By default idle connections are closed after 10s.
• max_conn_duration - keep-alive connections are closed after this duration. If set to 0 - connection duration is unlimited. By default connection duration is unlimited.

workers_count cfg.Expression default=gomaxprocs*4

How many workers will be instantiated to send batches.

request_timeout cfg.Duration default=1s

Client timeout when sends requests to HTTP Event Collector.

batch_size cfg.Expression default=capacity/4

A maximum quantity of events to pack into one batch.

batch_size_bytes cfg.Expression default=0

A minimum size of events in a batch to send. If both batch_size and batch_size_bytes are set, they will work together.

batch_flush_timeout cfg.Duration default=200ms

After this timeout the batch will be sent even if batch isn't completed.

retry int default=10

Retries of insertion. If File.d cannot insert for this number of attempts, File.d will fall with non-zero exit code or skip message (see fatal_on_failed_insert).

fatal_on_failed_insert bool default=false

After an insert error, fall with a non-zero exit code or not Experimental feature

retention cfg.Duration default=1s

Retention milliseconds for retry to DB.

retention_exponentially_multiplier int default=2

Multiplier for exponential increase of retention between retries

copy_fields []CopyField

List of field paths copy from field in original event to field in output json. To fields paths are relative to output json - one level higher since original event is stored under the "event" key. Supports nested fields in both from and to. Supports copying whole original event, but does not allow to copy directly to the output root or the "event" key with any of its subkeys.

Generated using insane-doc

Documentation

Overview

Package splunk is an output plugin that sends events to splunk database.

Index

• func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig)
• type Config
• type CopyField
• type KeepAliveConfig
• type Plugin
  • func (p *Plugin) Out(event *pipeline.Event)
  • func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams)
  • func (p *Plugin) Stop()

Functions

func Factory

func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig)

Types

type Config

type Config struct {
	// > @3@4@5@6
	// >
	// > A full URI address of splunk HEC endpoint. Format: `http://127.0.0.1:8088/services/collector`.
	Endpoint string `json:"endpoint" required:"true"` // *

	// > @3@4@5@6
	// >
	// > If set, the plugin will use gzip encoding.
	UseGzip bool `json:"use_gzip" default:"false"` // *

	// > @3@4@5@6
	// >
	// > Gzip compression level. Used if `use_gzip=true`.
	GzipCompressionLevel string `json:"gzip_compression_level" default:"default" options:"default|no|best-speed|best-compression|huffman-only"` // *

	// > @3@4@5@6
	// >
	// > Token for an authentication for a HEC endpoint.
	Token string `json:"token" required:"true"` // *

	// > @3@4@5@6
	// >
	// > Keep-alive config.
	// >
	// > `KeepAliveConfig` params:
	// > * `max_idle_conn_duration` - idle keep-alive connections are closed after this duration.
	// > By default idle connections are closed after `10s`.
	// > * `max_conn_duration` - keep-alive connections are closed after this duration.
	// > If set to `0` - connection duration is unlimited.
	// > By default connection duration is unlimited.
	KeepAlive KeepAliveConfig `json:"keep_alive" child:"true"` // *

	// > @3@4@5@6
	// >
	// > How many workers will be instantiated to send batches.
	WorkersCount  cfg.Expression `json:"workers_count" default:"gomaxprocs*4" parse:"expression"` // *
	WorkersCount_ int

	// > @3@4@5@6
	// >
	// > Client timeout when sends requests to HTTP Event Collector.
	RequestTimeout  cfg.Duration `json:"request_timeout" default:"1s" parse:"duration"` // *
	RequestTimeout_ time.Duration

	// > @3@4@5@6
	// >
	// > A maximum quantity of events to pack into one batch.
	BatchSize  cfg.Expression `json:"batch_size" default:"capacity/4" parse:"expression"` // *
	BatchSize_ int

	// > @3@4@5@6
	// >
	// > A minimum size of events in a batch to send.
	// > If both batch_size and batch_size_bytes are set, they will work together.
	BatchSizeBytes  cfg.Expression `json:"batch_size_bytes" default:"0" parse:"expression"` // *
	BatchSizeBytes_ int

	// > @3@4@5@6
	// >
	// > After this timeout the batch will be sent even if batch isn't completed.
	BatchFlushTimeout  cfg.Duration `json:"batch_flush_timeout" default:"200ms" parse:"duration"` // *
	BatchFlushTimeout_ time.Duration

	// > @3@4@5@6
	// >
	// > Retries of insertion. If File.d cannot insert for this number of attempts,
	// > File.d will fall with non-zero exit code or skip message (see fatal_on_failed_insert).
	Retry int `json:"retry" default:"10"` // *

	// > @3@4@5@6
	// >
	// > After an insert error, fall with a non-zero exit code or not
	// > **Experimental feature**
	FatalOnFailedInsert bool `json:"fatal_on_failed_insert" default:"false"` // *

	// > @3@4@5@6
	// >
	// > Retention milliseconds for retry to DB.
	Retention  cfg.Duration `json:"retention" default:"1s" parse:"duration"` // *
	Retention_ time.Duration

	// > @3@4@5@6
	// >
	// > Multiplier for exponential increase of retention between retries
	RetentionExponentMultiplier int `json:"retention_exponentially_multiplier" default:"2"` // *

	// > @3@4@5@6
	// >
	// > List of field paths copy `from` field in original event `to` field in output json.
	// > To fields paths are relative to output json - one level higher since original
	// > event is stored under the "event" key. Supports nested fields in both from and to.
	// > Supports copying whole original event, but does not allow to copy directly to the output root
	// > or the "event" key with any of its subkeys.
	CopyFields []CopyField `json:"copy_fields" slice:"true"` // *
}

! config-params ^ config-params

type CopyField

type CopyField struct {
	From string `json:"from"`
	To   string `json:"to"`
}

type KeepAliveConfig

type KeepAliveConfig struct {
	// Idle keep-alive connections are closed after this duration.
	MaxIdleConnDuration  cfg.Duration `json:"max_idle_conn_duration" parse:"duration" default:"10s"`
	MaxIdleConnDuration_ time.Duration

	// Keep-alive connections are closed after this duration.
	MaxConnDuration  cfg.Duration `json:"max_conn_duration" parse:"duration" default:"0"`
	MaxConnDuration_ time.Duration
}

type Plugin

type Plugin struct {
	// contains filtered or unexported fields
}

func (*Plugin) Out

func (p *Plugin) Out(event *pipeline.Event)

func (*Plugin) Start

func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.OutputPluginParams)

func (*Plugin) Stop

func (p *Plugin) Stop()