logparser

package
v0.0.0-...-dbd657b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 12, 2019 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// S3ALBLogParser S3 ALB logs parser
	S3ALBLogParser = NewCustomLogParser("timestamp", regexp.MustCompile(`^(?P<type>[^ ]*) (?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) ((?P<target_ip>[^ ]+)[:-](?P<target_port>[0-9]+)|-) (?P<request_processing_time>[-.0-9]*) (?P<target_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<target_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^ ]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*) (?P<target_group_arn>[^ ]*) \"(?P<trace_id>[^\"]*)\"`)).
		WithKindMap(map[string]string{
			"timestamp":                "timeISO8601",
			"client_port":              "uint16",
			"target_port":              "uint16",
			"request_processing_time":  "float64",
			"target_processing_time":   "float64",
			"response_processing_time": "float64",
			"request_url":              "urlencoded",
			"received_bytes":           "int64",
			"sent_bytes":               "int64",
			"elb_status_code":          "int16",
			"target_status_code":       "int16",
		}).
		WithEmptyValues(map[string]string{
			"user_agent":               "-",
			"ssl_cipher":               "-",
			"ssl_protocol":             "-",
			"request_processing_time":  "-1",
			"target_processing_time":   "-1",
			"response_processing_time": "-1",
			"target_status_code":       "-",
		})
)
View Source 
var (
	// S3CloudFrontWebLogParser parser for CloudFront Web logs
	S3CloudFrontWebLogParser = NewCustomLogParser("timestamp", regexp.MustCompile(`^(?P<timestamp>[^\t]*\t[^\t]*)\t(?P<x_edge_location>[^\t]*)\t(?P<sc_bytes>[^\t]*)\t(?P<c_ip>[^\t]*)\t(?P<cs_method>[^\t]*)\t(?P<cs_host>[^\t]*)\t(?P<cs_uri_stem>[^\t]*)\t(?P<sc_status>[^\t]*)\t(?P<cs_referer>[^\t]*)\t(?P<cs_user_agent>[^\t]*)\t(?P<cs_uri_query>[^\t]*)\t(?P<cs_cookie>[^\t]*)\t(?P<x_edge_result_type>[^\t]*)\t(?P<x_edge_request_id>[^\t]*)\t(?P<x_host_header>[^\t]*)\t(?P<cs_protocol>[^\t]*)\t(?P<cs_bytes>[^\t]*)\t(?P<time_taken>[^\t]*)\t(?P<x_forwarded_for>[^\t]*)\t(?P<ssl_protocol>[^\t]*)\t(?P<ssl_cipher>[^\t]*)\t(?P<x_edge_response_result_type>[^\t]*)\t(?P<cs_protocol_version>[^\t]*)\t(?P<fle_status>[^\t]*)\t(?P<fle_encrypted_fields>[^\s]*)`)).
		WithKindMap(map[string]string{
			"timestamp":       "time:2006-01-02\t15:04:05",
			"x_edge_location": "deepurlencoded",
			"cs_bytes":        "uint64",
			"sc_bytes":        "uint64",
			"cs_host":         "deepurlencoded",
			"cs_uri_stem":     "deepurlencoded",
			"sc_status":       "int16",
			"cs_referer":      "deepurlencoded",
			"cs_user_agent":   "deepurlencoded",
			"cs_uri_query":    "deepurlencoded",
			"cs_cookie":       "deepurlencoded",
			"time_taken":      "float64",
		}).
		WithReIgnore(regexp.MustCompile(`^#`)).
		WithEmptyValues(map[string]string{
			"cs_uri_query":         "-",
			"cs_bytes":             "-",
			"x_forwarded_for":      "-",
			"ssl_protocol":         "-",
			"ssl_cipher":           "-",
			"fle_status":           "-",
			"fle_encrypted_fields": "-",
		})
)
View Source 
var (
	// S3ELBLogParser S3 ELB logs parser
	S3ELBLogParser = NewCustomLogParser("timestamp", regexp.MustCompile(`^(?P<timestamp>[^ ]*) (?P<elb>[^ ]*) (?P<client_ip>[^ ]*):(?P<client_port>[0-9]*) ((?P<backend_ip>[^ ]+)[:-](?P<backend_port>[0-9]+)|-) (?P<request_processing_time>[-.0-9]*) (?P<backend_processing_time>[-.0-9]*) (?P<response_processing_time>[-.0-9]*) (?P<elb_status_code>|[-0-9]*) (?P<backend_status_code>-|[-0-9]*) (?P<received_bytes>[-0-9]*) (?P<sent_bytes>[-0-9]*) \"(?P<request_verb>[^ ]*) (?P<request_url>[^ ]*) (?P<request_proto>- |[^ ]*)\" \"(?P<user_agent>[^\"]*)\" (?P<ssl_cipher>[A-Z0-9-]+) (?P<ssl_protocol>[A-Za-z0-9.-]*)`)).
		WithKindMap(map[string]string{
			"timestamp":                "timeISO8601",
			"client_port":              "uint16",
			"backend_port":             "uint16",
			"request_processing_time":  "float64",
			"backend_processing_time":  "float64",
			"response_processing_time": "float64",
			"request_url":              "urlencoded",
			"received_bytes":           "int64",
			"sent_bytes":               "int64",
			"elb_status_code":          "int16",
			"backend_status_code":      "int16",
		}).
		WithEmptyValues(map[string]string{
			"user_agent":               "-",
			"ssl_cipher":               "-",
			"ssl_protocol":             "-",
			"elb_status_code":          "-",
			"request_processing_time":  "-1",
			"backend_processing_time":  "-1",
			"response_processing_time": "-1",
			"backend_status_code":      "-",
		})
)
View Source 
var (
	// S3WAFLogParser S3 WAF logs parser
	S3WAFLogParser = NewJSONLogParser("timestamp", mustKindFromString("timeUnixMilliseconds"))
)

Functions

func CreateEvent 

func CreateEvent(line *string, timestamp time.Time, fields common.MapStr) *beat.Event

CreateEvent creates an event to be passed to elastic output

Types

type CustomLogParser 

type CustomLogParser struct {
	// contains filtered or unexported fields
}

CustomLogParser contains information of S3 objects (sqsMessage not null implies that this object is extracted from an SQS message)

func NewCustomLogParser 

func NewCustomLogParser(timestampField string, re *regexp.Regexp) *CustomLogParser

NewCustomLogParser creates a new custom log parser based on regular expression to detect fields in a log line (re)

func (*CustomLogParser) Copy 

func (c *CustomLogParser) Copy() *CustomLogParser

Copy generates a new CustomLogParser from current one

func (*CustomLogParser) Parse 

func (c *CustomLogParser) Parse(reader io.Reader, mh func(*beat.Event), eh func(string, error)) error

Parse parses a reader and sends errors and parsed elements to handlers

func (*CustomLogParser) SetKindMap 

func (c *CustomLogParser) SetKindMap(reKindMap map[string]string) error

SetKindMap configures current log parser to map types passed on reKindMap and returns error (if present)

func (*CustomLogParser) WithEmptyValues 

func (c *CustomLogParser) WithEmptyValues(emptyValues map[string]string) *CustomLogParser

WithEmptyValues configures current log parser to take into account emptyValues

func (*CustomLogParser) WithKindMap 

func (c *CustomLogParser) WithKindMap(reKindMap map[string]string) *CustomLogParser

WithKindMap configures current log parser to map types passed on reKindMap

func (*CustomLogParser) WithReIgnore 

func (c *CustomLogParser) WithReIgnore(reIgnore *regexp.Regexp) *CustomLogParser

WithReIgnore configures current log parser to ignore lines that match reIgnore

type JSONLogParser 

type JSONLogParser struct {
	// contains filtered or unexported fields
}

JSONLogParser JSON log parser

func NewJSONLogParser 

func NewJSONLogParser(timestampField string, timestampKind kindElement) *JSONLogParser

NewJSONLogParser creates a new JSON log parser

func NewJSONLogParserConfig 

func NewJSONLogParserConfig(cfg *common.Config) (*JSONLogParser, error)

NewJSONLogParserConfig creates a new JSON log parser based on a map os strins

func (*JSONLogParser) Parse 

func (j *JSONLogParser) Parse(reader io.Reader, mh func(*beat.Event), eh func(string, error)) error

Parse parses a reader and sends errors and parsed elements to handlers

type JSONLogParserConfig 

type JSONLogParserConfig struct {
	TimestampField  string `config:"timestamp_field" validate:"required"`
	TimestampFormat string `config:"timestamp_format" validate:"required"`
}

JSONLogParserConfig JSONLogParser configuration

type LogParser 

type LogParser interface {
	Parse(io.Reader, func(*beat.Event), func(string, error)) error
}

LogParser interface to inherit on each type of log parsers

func GetPredefinedParser 

func GetPredefinedParser(n string, config *common.Config) (LogParser, error)

GetPredefinedParser gets a predefined parser based on its name

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.