http

README

HTTP plugin

Reads events from HTTP requests with the body delimited by a new line.

Also, it emulates some protocols to allow receiving events from a wide range of software that use HTTP to transmit data. E.g. file.d may pretend to be Elasticsearch allows clients to send events using Elasticsearch protocol. So you can use Elasticsearch filebeat output plugin to send data to file.d.

⚠ Currently event commitment mechanism isn't implemented for this plugin. Plugin answers with HTTP code OK 200 right after it has read all the request body. It doesn't wait until events are committed.

Example: Emulating elastic through http:

pipelines:
  example_k8s_pipeline:
    settings:
      capacity: 1024
    input:
      # define input type.
      type: http
      # pretend elastic search, emulate it's protocol.
      emulate_mode: "elasticsearch"
      # define http port.
      address: ":9200"
    actions:
      # parse elastic search query.
      - type: parse_es
      # decode elastic search json.
      - type: json_decode
        # field is required.
        field: message
    output:
      # Let's write to kafka example.
      type: kafka
      brokers: [kafka-local:9092, kafka-local:9091]
      default_topic: yourtopic-k8s-data
      use_topic_field: true
      topic_field: pipeline_kafka_topic

      # Or we can write to file:
      # type: file
      # target_file: "./output.txt"

Setup:

# run server.
# config.yaml should contains yaml config above.
go run ./cmd/file.d --config=config.yaml

# now do requests.
curl "localhost:9200/_bulk" -H 'Content-Type: application/json' -d \
'{"index":{"_index":"index-main","_type":"span"}}
{"message": "hello", "kind": "normal"}
'

Config params

address string default=:9200

An address to listen to. Omit ip/host to listen all network interfaces. E.g. :88

emulate_mode string default=no options=no|elasticsearch

Which protocol to emulate.

ca_cert string

CA certificate in PEM encoding. This can be a path or the content of the certificate. If both ca_cert and private_key are set, the server starts accepting connections in TLS mode.

private_key string

CA private key in PEM encoding. This can be a path or the content of the key. If both ca_cert and private_key are set, the server starts accepting connections in TLS mode.

auth AuthConfig

Auth config. Disabled by default. See AuthConfig for details. You can use 'warn' log level for logging authorizations.

meta cfg.MetaTemplates

Meta params

Add meta information to an event (look at Meta params) Use go-template syntax

Example: user_agent: '{{ index (index .request.Header "User-Agent") 0}}'

cors CORSConfig

CORS config. See CORSConfig for details.

header string default=Authorization

Override default Authorization header

strategy string default=disabled options=disabled|basic|bearer

AuthStrategy.Strategy describes strategy to use.

secrets map[string]string

AuthStrategy.Secrets describes secrets in key-value format. If the strategy is basic, then the key is the login, the value is the password. If the strategy is bearer, then the key is the name, the value is the Bearer token. Key uses in the http_input_total metric.

Meta params

login

remote_addr net.IP

request http.Request

params url.Values

Generated using insane-doc

Documentation

Index

• func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig)
• type AuthConfig
• type AuthStrategy
• type CORSConfig
• type Config
• type EmulateMode
• type Plugin
  • func (p *Plugin) Commit(_ *pipeline.Event)
  • func (p *Plugin) PassEvent(_ *pipeline.Event) bool
  • func (p *Plugin) ServeHTTP(w http.ResponseWriter, r *http.Request)
  • func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.InputPluginParams)
  • func (p *Plugin) Stop()

Functions

func Factory

func Factory() (pipeline.AnyPlugin, pipeline.AnyConfig)

Types

type AuthConfig

type AuthConfig struct {
	// > @3@4@5@6
	// >
	// > Override default Authorization header
	Header string `json:"header" default:"Authorization"` // *

	// > @3@4@5@6
	// >
	// > AuthStrategy.Strategy describes strategy to use.
	Strategy  string `json:"strategy" default:"disabled" options:"disabled|basic|bearer"` // *
	Strategy_ AuthStrategy
	// > @3@4@5@6
	// >
	// > AuthStrategy.Secrets describes secrets in key-value format.
	// > If the `strategy` is basic, then the key is the login, the value is the password.
	// > If the `strategy` is bearer, then the key is the name, the value is the Bearer token.
	// > Key uses in the http_input_total metric.
	Secrets map[string]string `json:"secrets"` // *
}

! config-params ^ config-params

type AuthStrategy

type AuthStrategy byte

const (
	StrategyDisabled AuthStrategy = iota
	StrategyBasic
	StrategyBearer
)

type CORSConfig

type CORSConfig struct {
	AllowedOrigins []string `json:"allowed_origins"`
	DefaultOrigin  string   `json:"default_origin"  default:"*"`
	AllowedHeaders []string `json:"allowed_headers"`
	ExposedHeaders []string `json:"exposed_headers"`
}

type Config

type Config struct {
	// > @3@4@5@6
	// >
	// > An address to listen to. Omit ip/host to listen all network interfaces. E.g. `:88`
	Address string `json:"address" default:":9200"` // *
	// > @3@4@5@6
	// >
	// > Which protocol to emulate.
	EmulateMode  string `json:"emulate_mode" default:"no" options:"no|elasticsearch"` // *
	EmulateMode_ EmulateMode
	// > @3@4@5@6
	// >
	// > CA certificate in PEM encoding. This can be a path or the content of the certificate.
	// > If both ca_cert and private_key are set, the server starts accepting connections in TLS mode.
	CACert string `json:"ca_cert" default:""` // *
	// > @3@4@5@6
	// >
	// > CA private key in PEM encoding. This can be a path or the content of the key.
	// > If both ca_cert and private_key are set, the server starts accepting connections in TLS mode.
	PrivateKey string `json:"private_key" default:""` // *

	// > @3@4@5@6
	// >
	// > Auth config.
	// > Disabled by default.
	// > See AuthConfig for details.
	// > You can use 'warn' log level for logging authorizations.
	Auth AuthConfig `json:"auth" child:"true"` // *

	// > @3@4@5@6
	// >
	// > Meta params
	// >
	// > Add meta information to an event (look at Meta params)
	// > Use [go-template](https://pkg.go.dev/text/template) syntax
	// >
	// > Example: “`user_agent: '{{ index (index .request.Header "User-Agent") 0}}'“`
	Meta cfg.MetaTemplates `json:"meta"` // *

	// > @3@4@5@6
	// >
	// > CORS config.
	// > See CORSConfig for details.
	CORS CORSConfig `json:"cors" child:"true"` // *
}

! config-params ^ config-params

type EmulateMode

type EmulateMode byte

const (
	EmulateModeNo EmulateMode = iota
	EmulateModeElasticSearch
)

type Plugin

type Plugin struct {
	// contains filtered or unexported fields
}

func (*Plugin) Commit

func (p *Plugin) Commit(_ *pipeline.Event)

func (*Plugin) PassEvent

func (p *Plugin) PassEvent(_ *pipeline.Event) bool

PassEvent decides pass or discard event.

func (*Plugin) ServeHTTP

func (p *Plugin) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (*Plugin) Start

func (p *Plugin) Start(config pipeline.AnyConfig, params *pipeline.InputPluginParams)

func (*Plugin) Stop

func (p *Plugin) Stop()