parser

package
v1.4.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 4, 2018 License: Apache-2.0 Imports: 20 Imported by: 44

Documentation

Index

Constants

View Source 
const (
	KeyCSVSchema   = "csv_schema"      // csv 每个列的列名和类型 long/string/float/date
	KeyCSVSplitter = "csv_splitter"    // csv 的分隔符
	KeyCSVLabels   = "csv_labels"      // csv 额外增加的标签信息，比如机器信息等
	KeyAutoRename  = "csv_auto_rename" // 是否将不合法的字段名称重命名一下, 比如 header-host 重命名为 header_host
)
View Source 
const (
	KeyGrokMode               = "grok_mode"     //是否替换\n以匹配多行
	KeyGrokPatterns           = "grok_patterns" // grok 模式串名
	KeyGrokCustomPatternFiles = "grok_custom_pattern_files"
	KeyGrokCustomPatterns     = "grok_custom_patterns"

	KeyTimeZoneOffset = "timezone_offset"
)
View Source 
const (
	LONG   = "long"
	FLOAT  = "float"
	STRING = "string"
	DATE   = "date"
	DROP   = "drop"
)
View Source 
const (
	KEY_SRC_IP   = "source_ip"
	KEY_METHOD   = "method"
	KEY_TOPIC    = "topic"
	KEY_CODE     = "code"
	KEY_RESP_LEN = "resp_len"
	KEY_DURATION = "duration"
	KEY_LOG_TIME = "log_time"
	KEY_ERROR    = "error"
	KEY_WARN     = "warn"
	EMPTY_STRING = ""
)
View Source 
const (
	NginxSchema      string = "nginx_schema"
	NginxConfPath           = "nginx_log_format_path"
	NginxLogFormat          = "nginx_log_format_name"
	NginxFormatRegex        = "nginx_log_format_regex"
)
View Source 
const (
	KeyParserName = utils.GlobalKeyName
	KeyParserType = "type"
	KeyRunnerName = "runner_name"
	KeyLabels     = "labels" // 额外增加的标签信息，比如机器信息等
)

conf 字段

View Source 
const (
	TypeCSV        = "csv"
	TypeLogv1      = "qiniulog"
	TypeKafkaRest  = "kafkarest"
	TypeRaw        = "raw"
	TypeEmpty      = "empty"
	TypeGrok       = "grok"
	TypeInnerSQL   = "_sql"
	TypeInnerMysql = "_mysql"
	TypeJson       = "json"
	TypeNginx      = "nginx"
	TypeSyslog     = "syslog"
)

parser 的类型

View Source 
const (
	LogHeadPrefix string = "prefix"
	LogHeadDate   string = "date"
	LogHeadTime   string = "time"
	LogHeadReqid  string = "reqid"
	LogHeadLevel  string = "level"
	LogHeadFile   string = "file"
	LogHeadLog    string = "log" //默认在最后，不能改变顺序
)
View Source 
const (
	KeyQiniulogPrefix = "qiniulog_prefix" //qiniulog的日志前缀
	KeyLogHeaders     = "qiniulog_log_headers"
)

conf 字段

View Source 
const (
	KeyRaw       = "raw"
	KeyTimestamp = "timestamp"
)
View Source 
const (
	KeyRFCType = "syslog_rfc"

	SyslogEofLine = "!@#pandora-EOF-line#@!"
)
View Source 
const DEFAULT_PATTERNS = `` /* 41293-byte string literal not displayed */
View Source 
const MaxGrokMultiLineBuffer = 64 * 1024 * 1024 // 64MB
View Source 
const MaxParserSchemaErrOutput = 5
View Source 
const (
	ModeMulti = "multi"
)
View Source 
const SECOND_PER_5MIN = 5 * 60
View Source 
const SECOND_PER_DAY = 24 * 60 * 60

Variables

View Source 
var (
	HeaderPattern = map[string]string{
		LogHeadDate:  "^[1-9]\\d{3}/[0-1]\\d/[0-3]\\d$",
		LogHeadTime:  "^[0-2]\\d:[0-6]\\d:[0-6]\\d(\\.\\d{6})?$",
		LogHeadReqid: "^\\[\\w+\\]\\[\\w+\\]$",
		LogHeadLevel: "^\\[[A-Z]+\\]$",
		LogHeadFile:  ":\\d+:$",
	}
	CompliedPatterns map[string]*regexp.Regexp
)
View Source 
var ModeKeyOptions = map[string][]utils.Option{
	TypeJson: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeNginx: {
		{
			KeyName:      NginxConfPath,
			ChooseOnly:   false,
			Default:      "/opt/nginx/conf/nginx.conf",
			DefaultNoUse: true,
			Description:  "nginx配置路径(nginx_log_format_path)",
		},
		{
			KeyName:      NginxLogFormat,
			ChooseOnly:   false,
			Default:      "main",
			DefaultNoUse: true,
			Description:  "nginx日志格式名称(nginx_log_format_name)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      NginxSchema,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "指定nginx字段类型(nginx_schema)",
		},
		{
			KeyName:      NginxFormatRegex,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "直接通过正则表达式解析(nginx_log_format_regex)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeGrok: {
		{
			KeyName:      KeyGrokPatterns,
			ChooseOnly:   false,
			Default:      "%{COMMON_LOG_FORMAT}",
			DefaultNoUse: true,
			Description:  "匹配日志的grok表达式(grok_patterns)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyGrokMode,
			ChooseOnly:    true,
			ChooseOptions: []interface{}{"oneline", ModeMulti},
			Default:       "oneline",
			DefaultNoUse:  false,
			Description:   "grok单行多行模式(grok_mode)",
		},
		{
			KeyName:      KeyGrokCustomPatternFiles,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern 文件路径(grok_custom_pattern_files)",
		},
		{
			KeyName:      KeyGrokCustomPatterns,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "自定义 pattern (grok_custom_patterns)",
		},
		OptionTimezoneOffset,
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},

	TypeCSV: {
		{
			KeyName:      KeyCSVSchema,
			ChooseOnly:   false,
			Default:      "abc string,xyz long,data1 string,data2 float",
			DefaultNoUse: true,
			Description:  "csv格式的字段类型(csv_schema)",
		},
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyCSVSplitter,
			ChooseOnly:   false,
			Default:      ",",
			DefaultNoUse: false,
			Description:  "csv分隔符(csv_splitter)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		OptionTimezoneOffset,
		{
			KeyName:       KeyAutoRename,
			ChooseOnly:    true,
			ChooseOptions: []interface{}{"true", "false"},
			Default:       "true",
			DefaultNoUse:  false,
			Description:   "自动将字段名称中的'-'更改为'_'",
		},
	},
	TypeRaw: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyTimestamp,
			ChooseOnly:    true,
			ChooseOptions: []interface{}{"true", "false"},
			Default:       "true",
			DefaultNoUse:  false,
			Description:   "数据附带时间戳(timestamp)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeLogv1: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
		{
			KeyName:      KeyQiniulogPrefix,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "日志前缀(qiniulog_prefix)",
		},
		{
			KeyName:      KeyLogHeaders,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "七牛日志格式顺序(qiniulog_log_headers)",
		},
	},
	TypeSyslog: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:       KeyRFCType,
			ChooseOnly:    true,
			Default:       "automic",
			ChooseOptions: []interface{}{"automic", "rfc3164", "rfc5424", "rfc6587"},
			DefaultNoUse:  false,
			Description:   "syslog使用的rfc协议(syslog_rfc)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeKafkaRest: {
		{
			KeyName:      KeyParserName,
			ChooseOnly:   false,
			Default:      "parser",
			DefaultNoUse: false,
			Description:  "parser名称(name)",
		},
		{
			KeyName:      KeyLabels,
			ChooseOnly:   false,
			Default:      "",
			DefaultNoUse: false,
			Description:  "额外的标签信息(labels)",
		},
	},
	TypeEmpty: {},
}
View Source 
var ModeUsages = []utils.KeyValue{
	{TypeJson, "json 格式解析"},
	{TypeNginx, "nginx 日志解析"},
	{TypeGrok, "grok 方式解析"},
	{TypeCSV, "csv 格式日志解析"},
	{TypeRaw, "raw 原始日志按行解析"},
	{TypeSyslog, "syslog 日志解析"},
	{TypeLogv1, "qiniulog 七牛日志库解析"},
	{TypeKafkaRest, "kafkarest 日志格式解析"},
	{TypeEmpty, "empty 通过解析清空数据"},
}

ModeUsages 用途说明

View Source 
var (
	OptionTimezoneOffset = utils.Option{
		KeyName:    KeyTimeZoneOffset,
		ChooseOnly: true,
		Default:    "0",
		ChooseOptions: []interface{}{"0", "-1", "-2", "-3", "-4",
			"-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12",
			"1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"},
		DefaultNoUse: false,
		Description:  "时区偏移量(timezone_offset)",
	}
)
View Source 
var SampleLogs = map[string]string{
	TypeNginx: `110.110.101.101 - - [21/Mar/2017:18:14:17 +0800] "GET /files/yyyysx HTTP/1.1" 206 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com llEAAFgmnoIa3q0U "0.040" 0.040 760 "-" "-" - - QCloud
1.11.1.1 - - [25/Mar/2017:18:14:17 +0800] "GET /files HTTP/1.1" 200 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com sfvfv123axs "0.040" 0.040 760 "-" "-" - - QCloud`,
	TypeGrok: `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
123.45.12.1 user-identifier bob [10/Oct/2013:13:55:36 -0700] "GET /hello.gif HTTP/1.0" 200 2326`,
	TypeJson: `{"a":"b","c":1,"d":1.1}`,
	TypeCSV: `a,123,bcd,1.2
xsxs,456,asv,5.12`,
	TypeRaw: `raw log1[05-May-2017 13:44:39]  [pool log] pid 4109
script_filename = /data/html/
[0x00007fec119d1720] curl_exec() /data/html/xyframework/base.go:123
[05-May-2017 13:45:39]  [pool log] pid 4108`,
	TypeSyslog:    `<38>Feb 05 01:02:03 abc system[253]: Listening at 0.0.0.0:3000`,
	TypeLogv1:     `2016/10/20 17:30:21.433423 [GE2owHck-Y4IWJHS][WARN] github.com/qiniu/http/rpcutil.v1/rpc_util.go:203: E18102: The specified repo does not exist under the provided appid ~`,
	TypeKafkaRest: `[2016-12-05 03:35:20,682] INFO 172.16.16.191 - - [05/Dec/2016:03:35:20 +0000] "POST /topics/VIP_VvBVy0tuMPPspm1A_0000000000 HTTP/1.1" 200 101640  46 (io.confluent.rest-utils.requests)`,
	TypeEmpty:     "empty 通过解析清空数据",
}

SampleLogs 样例日志，用于前端界面试玩解析器

Functions

func DetectType added in v1.3.6 

func DetectType(data []byte) (detected int)

func Time1Day 

func Time1Day(t int64) string

func Time1DayInt 

func Time1DayInt(t int64) int64

func Time5Min 

func Time5Min(t int64) string

func Time5MinInt 

func Time5MinInt(t int64) int64

Types

type Automatic added in v1.3.6 

type Automatic struct{}

func (*Automatic) GetParser added in v1.3.6 

func (f *Automatic) GetParser(line []byte) SysLogParser

func (*Automatic) IsNewLine added in v1.3.6 

func (f *Automatic) IsNewLine(data []byte) bool

type CsvParser 

type CsvParser struct {
	// contains filtered or unexported fields
}

func (*CsvParser) Name 

func (p *CsvParser) Name() string

func (*CsvParser) Parse 

func (p *CsvParser) Parse(lines []string) ([]sender.Data, error)

func (*CsvParser) Rename added in v1.4.1 

func (p *CsvParser) Rename(datas []sender.Data) []sender.Data

func (*CsvParser) Type added in v1.3.6 

func (p *CsvParser) Type() string

type CsvType 

type CsvType string

Type 类型常量 

const (
	TypeFloat   CsvType = "float"
	TypeLong    CsvType = "long"
	TypeString  CsvType = "string"
	TypeDate    CsvType = "date"
	TypeJsonMap CsvType = "jsonmap"
)

type EmptyParser 

type EmptyParser struct {
	// contains filtered or unexported fields
}

func (*EmptyParser) Name 

func (p *EmptyParser) Name() string

func (*EmptyParser) Parse 

func (p *EmptyParser) Parse(lines []string) (datas []sender.Data, err error)

type Format added in v1.3.6 

type Format interface {
	GetParser([]byte) SysLogParser
	IsNewLine(data []byte) bool
}

func GetFormt added in v1.3.6 

func GetFormt(format string) Format

type GrokParser 

type GrokParser struct {
	Patterns []string // 正式的pattern名称

	CustomPatterns     string
	CustomPatternFiles []string
	// contains filtered or unexported fields
}

func (*GrokParser) Name 

func (gp *GrokParser) Name() string

func (*GrokParser) Parse 

func (gp *GrokParser) Parse(lines []string) ([]sender.Data, error)

func (*GrokParser) Type added in v1.3.6 

func (gp *GrokParser) Type() string

type JsonParser 

type JsonParser struct {
	// contains filtered or unexported fields
}

func (*JsonParser) Name 

func (im *JsonParser) Name() string

func (*JsonParser) Parse 

func (im *JsonParser) Parse(lines []string) ([]sender.Data, error)

func (*JsonParser) Type added in v1.3.6 

func (im *JsonParser) Type() string

type KafaRestlogParser 

type KafaRestlogParser struct {
	// contains filtered or unexported fields
}

func (*KafaRestlogParser) Name 

func (krp *KafaRestlogParser) Name() string

func (*KafaRestlogParser) Parse 

func (krp *KafaRestlogParser) Parse(lines []string) ([]sender.Data, error)

func (*KafaRestlogParser) ParseCode 

func (krp *KafaRestlogParser) ParseCode(fields []string) int

func (*KafaRestlogParser) ParseDuration 

func (krp *KafaRestlogParser) ParseDuration(fields []string) int

func (*KafaRestlogParser) ParseIp 

func (krp *KafaRestlogParser) ParseIp(fields []string) string

func (*KafaRestlogParser) ParseLogTime 

func (krp *KafaRestlogParser) ParseLogTime(fields []string) int64

func (*KafaRestlogParser) ParseMethod 

func (krp *KafaRestlogParser) ParseMethod(fields []string) string

func (*KafaRestlogParser) ParseRespCL 

func (krp *KafaRestlogParser) ParseRespCL(fields []string) int

func (*KafaRestlogParser) ParseTopic 

func (krp *KafaRestlogParser) ParseTopic(fields []string) string

func (*KafaRestlogParser) Type added in v1.3.6 

func (krp *KafaRestlogParser) Type() string

type Label added in v1.2.1 

type Label struct {
	Name  string
	Value string
}

func GetLabels added in v1.2.1 

func GetLabels(labelList []string, nameMap map[string]struct{}) (labels []Label)

type LogParser 

type LogParser interface {
	Name() string
	// parse lines into structured datas
	Parse(lines []string) (datas []sender.Data, err error)
}

func NewCsvParser 

func NewCsvParser(c conf.MapConf) (LogParser, error)

func NewEmptyParser 

func NewEmptyParser(c conf.MapConf) (LogParser, error)

func NewGrokParser 

func NewGrokParser(c conf.MapConf) (LogParser, error)

func NewJsonParser 

func NewJsonParser(c conf.MapConf) (LogParser, error)

func NewKafaRestlogParser 

func NewKafaRestlogParser(c conf.MapConf) (LogParser, error)

func NewNginxParser added in v1.2.1 

func NewNginxParser(c conf.MapConf) (LogParser, error)

func NewQiniulogParser 

func NewQiniulogParser(c conf.MapConf) (LogParser, error)

func NewRawlogParser 

func NewRawlogParser(c conf.MapConf) (LogParser, error)

func NewSyslogParser added in v1.3.6 

func NewSyslogParser(c conf.MapConf) (LogParser, error)

type LogParts added in v1.3.6 

type LogParts map[string]interface{}

type NginxParser added in v1.2.1 

type NginxParser struct {
	// contains filtered or unexported fields
}

func NewNginxAccParser added in v1.2.1 

func NewNginxAccParser(c conf.MapConf) (p *NginxParser, err error)

func (*NginxParser) Name added in v1.2.1 

func (p *NginxParser) Name() string

func (*NginxParser) Parse added in v1.2.1 

func (p *NginxParser) Parse(lines []string) ([]sender.Data, error)

func (*NginxParser) Type added in v1.3.6 

func (p *NginxParser) Type() string

type ParserRegistry 

type ParserRegistry struct {
	// contains filtered or unexported fields
}

func NewParserRegistry 

func NewParserRegistry() *ParserRegistry

func (*ParserRegistry) NewLogParser 

func (ps *ParserRegistry) NewLogParser(conf conf.MapConf) (p LogParser, err error)

func (*ParserRegistry) RegisterParser 

func (ps *ParserRegistry) RegisterParser(parserType string, constructor func(conf.MapConf) (LogParser, error)) error

type ParserType added in v1.3.6 

type ParserType interface {
	Type() string
}

type QiniulogParser 

type QiniulogParser struct {
	// contains filtered or unexported fields
}

func (*QiniulogParser) GetParser 

func (p *QiniulogParser) GetParser(head string) (func(string) (string, string, error), error)

func (*QiniulogParser) Name 

func (p *QiniulogParser) Name() string

func (*QiniulogParser) Parse 

func (p *QiniulogParser) Parse(lines []string) ([]sender.Data, error)

func (*QiniulogParser) Type added in v1.3.6 

func (p *QiniulogParser) Type() string

type RFC3164 added in v1.3.6 

type RFC3164 struct{}

func (*RFC3164) GetParser added in v1.3.6 

func (f *RFC3164) GetParser(line []byte) SysLogParser

func (*RFC3164) IsNewLine added in v1.3.6 

func (f *RFC3164) IsNewLine(data []byte) bool

type RFC5424 added in v1.3.6 

type RFC5424 struct{}

func (*RFC5424) GetParser added in v1.3.6 

func (f *RFC5424) GetParser(line []byte) SysLogParser

func (*RFC5424) IsNewLine added in v1.3.6 

func (f *RFC5424) IsNewLine(data []byte) bool

type RFC6587 added in v1.3.6 

type RFC6587 struct{}

func (*RFC6587) GetParser added in v1.3.6 

func (f *RFC6587) GetParser(line []byte) SysLogParser

func (*RFC6587) IsNewLine added in v1.3.6 

func (f *RFC6587) IsNewLine(data []byte) bool

type RawlogParser 

type RawlogParser struct {
	// contains filtered or unexported fields
}

func (*RawlogParser) Name 

func (p *RawlogParser) Name() string

func (*RawlogParser) Parse 

func (p *RawlogParser) Parse(lines []string) ([]sender.Data, error)

func (*RawlogParser) Type added in v1.3.6 

func (p *RawlogParser) Type() string

type SysLogParser added in v1.3.6 

type SysLogParser interface {
	Parse() error
	Dump() LogParts
	Location(*time.Location)
}

type SyslogParser added in v1.3.6 

type SyslogParser struct {
	// contains filtered or unexported fields
}

func (*SyslogParser) Name added in v1.3.6 

func (p *SyslogParser) Name() string

func (*SyslogParser) Parse added in v1.3.6 

func (p *SyslogParser) Parse(lines []string) ([]sender.Data, error)

func (*SyslogParser) Type added in v1.3.6 

func (p *SyslogParser) Type() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.