Documentation ¶
Index ¶
- Constants
- Variables
- func ConvertWebParserConfig(conf conf.MapConf) conf.MapConf
- func DetectType(data []byte) (detected int)
- func HasSpace(spliter string) bool
- func Time1Day(t int64) string
- func Time1DayInt(t int64) int64
- func Time5Min(t int64) string
- func Time5MinInt(t int64) int64
- type Automatic
- type CsvParser
- type CsvType
- type EmptyParser
- type Flushable
- type Format
- type GrokParser
- type JsonParser
- type KafaRestlogParser
- func (krp *KafaRestlogParser) Name() string
- func (krp *KafaRestlogParser) Parse(lines []string) ([]Data, error)
- func (krp *KafaRestlogParser) ParseCode(fields []string) int
- func (krp *KafaRestlogParser) ParseDuration(fields []string) int
- func (krp *KafaRestlogParser) ParseIp(fields []string) string
- func (krp *KafaRestlogParser) ParseLogTime(fields []string) int64
- func (krp *KafaRestlogParser) ParseMethod(fields []string) string
- func (krp *KafaRestlogParser) ParseRespCL(fields []string) int
- func (krp *KafaRestlogParser) ParseTopic(fields []string) string
- func (krp *KafaRestlogParser) Type() string
- type Label
- type LogParser
- func NewCsvParser(c conf.MapConf) (LogParser, error)
- func NewEmptyParser(c conf.MapConf) (LogParser, error)
- func NewGrokParser(c conf.MapConf) (LogParser, error)
- func NewJsonParser(c conf.MapConf) (LogParser, error)
- func NewKafaRestlogParser(c conf.MapConf) (LogParser, error)
- func NewMysqllogParser(c conf.MapConf) (LogParser, error)
- func NewNginxParser(c conf.MapConf) (LogParser, error)
- func NewQiniulogParser(c conf.MapConf) (LogParser, error)
- func NewRawlogParser(c conf.MapConf) (LogParser, error)
- func NewSyslogParser(c conf.MapConf) (LogParser, error)
- type LogParts
- type MysqllogParser
- type NginxParser
- type ParserRegistry
- type ParserType
- type QiniulogParser
- type RFC3164
- type RFC5424
- type RFC6587
- type RawlogParser
- type SysLogParser
- type SyslogParser
Constants ¶
View Source
const ( KeyCSVSchema = "csv_schema" // csv 每个列的列名和类型 long/string/float/date KeyCSVSplitter = "csv_splitter" // csv 的分隔符 KeyCSVLabels = "csv_labels" // csv 额外增加的标签信息,比如机器信息等 KeyAutoRename = "csv_auto_rename" // 是否将不合法的字段名称重命名一下, 比如 header-host 重命名为 header_host KeyCSVAllowNoMatch = "csv_allow_no_match" // 允许实际分隔的数据和schema不相等,不相等时按顺序赋值 KeyCSVAllowMore = "csv_allow_more" // 允许实际字段比schema多 KeyCSVAllowMoreStartNum = "csv_more_start_number" // 允许实际字段比schema多,名称开始的数字 KeyCSVIgnoreInvalidField = "csv_ignore_invalid" // 忽略解析错误的字段 )
View Source
const ( KeyGrokMode = "grok_mode" //是否替换\n以匹配多行 KeyGrokPatterns = "grok_patterns" // grok 模式串名 KeyGrokCustomPatternFiles = "grok_custom_pattern_files" KeyGrokCustomPatterns = "grok_custom_patterns" KeyTimeZoneOffset = "timezone_offset" )
View Source
const ( LONG = "long" FLOAT = "float" STRING = "string" DATE = "date" DROP = "drop" )
View Source
const ( KEY_SRC_IP = "source_ip" KEY_METHOD = "method" KEY_TOPIC = "topic" KEY_CODE = "code" KEY_RESP_LEN = "resp_len" KEY_DURATION = "duration" KEY_LOG_TIME = "log_time" KEY_ERROR = "error" KEY_WARN = "warn" EMPTY_STRING = "" )
View Source
const ( NginxSchema string = "nginx_schema" NginxConfPath = "nginx_log_format_path" NginxLogFormat = "nginx_log_format_name" NginxFormatRegex = "nginx_log_format_regex" )
View Source
const ( KeyParserName = GlobalKeyName KeyParserType = "type" KeyLabels = "labels" // 额外增加的标签信息,比如机器信息等 KeyDisableRecordErrData = "disable_record_errdata" )
conf 字段
View Source
const ( TypeCSV = "csv" TypeLogv1 = "qiniulog" TypeKafkaRest = "kafkarest" TypeRaw = "raw" TypeEmpty = "empty" TypeGrok = "grok" TypeInnerSQL = "_sql" TypeInnerMysql = "_mysql" TypeJson = "json" TypeNginx = "nginx" TypeSyslog = "syslog" TypeMysqlLog = "mysqllog" )
parser 的类型
View Source
const ( LogHeadPrefix string = "prefix" LogHeadDate string = "date" LogHeadTime string = "time" LogHeadReqid string = "reqid" LogHeadLevel string = "level" LogHeadFile string = "file" LogHeadLog string = "log" //默认在最后,不能改变顺序 )
View Source
const ( KeyQiniulogPrefix = "qiniulog_prefix" //qiniulog的日志前缀 KeyLogHeaders = "qiniulog_log_headers" )
conf 字段
View Source
const ( KeyRaw = "raw" KeyTimestamp = "timestamp" )
View Source
const ( KeyRFCType = "syslog_rfc" PandoraParseFlushSignal = "!@#pandora-EOF-line#@!" )
View Source
const DEFAULT_PATTERNS = `` /* 41430-byte string literal not displayed */
View Source
const MaxGrokMultiLineBuffer = 64 * 1024 * 1024 // 64MB
View Source
const MaxParserSchemaErrOutput = 5
View Source
const (
ModeMulti = "multi"
)
View Source
const SECOND_PER_5MIN = 5 * 60
View Source
const SECOND_PER_DAY = 24 * 60 * 60
Variables ¶
View Source
var ( HeaderPattern = map[string]string{ LogHeadDate: "^[1-9]\\d{3}/[0-1]\\d/[0-3]\\d$", LogHeadTime: "^[0-2]\\d:[0-6]\\d:[0-6]\\d(\\.\\d{6})?$", LogHeadReqid: "^\\[\\w+\\]\\[\\w+\\]$", LogHeadLevel: "^\\[[A-Z]+\\]$", LogHeadFile: ":\\d+:$", } CompliedPatterns map[string]*regexp.Regexp )
View Source
var ( OptionTimezoneOffset = Option{ KeyName: KeyTimeZoneOffset, ChooseOnly: true, Default: "0", ChooseOptions: []interface{}{"0", "-1", "-2", "-3", "-4", "-5", "-6", "-7", "-8", "-9", "-10", "-11", "-12", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12"}, DefaultNoUse: false, Description: "时区偏移量(timezone_offset)", Advance: true, ToolTip: `若实际为东八区时间,读取为UTC时间,则实际多读取了8小时,选择"-8",修正回CST中国北京时间。`, } OptionLabels = Option{ KeyName: KeyLabels, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "额外的标签信息(labels)", Advance: true, ToolTip: `额外的标签信息,同样逗号分隔,如 "app logkit, user pandora"`, ToolTipActive: true, } OptionDisableRecordErrData = Option{ KeyName: KeyDisableRecordErrData, Element: Radio, ChooseOnly: true, ChooseOptions: []interface{}{"false", "true"}, Default: "false", DefaultNoUse: false, Description: "禁止记录解析失败数据(disable_record_errdata)", Advance: true, ToolTip: `解析失败的数据会默认出现在"pandora_stash"字段,该选项可以禁止记录解析失败的数据`, } OptionParserName = Option{ KeyName: KeyParserName, ChooseOnly: false, Default: "parser", DefaultNoUse: false, Description: "指定名称(name)", Advance: true, } )
View Source
var ModeKeyOptions = map[string][]Option{ TypeJson: { OptionParserName, OptionLabels, OptionDisableRecordErrData, }, TypeNginx: { { KeyName: NginxConfPath, ChooseOnly: false, Default: "", Required: true, Placeholder: "/opt/nginx/conf/nginx.conf", DefaultNoUse: true, Description: "nginx配置路径(nginx_log_format_path)", ToolTip: `nginx配置文件,确认配置文件包含 log_format 中填写的格式`, }, { KeyName: NginxLogFormat, ChooseOnly: false, Default: "main", Placeholder: "main", Required: true, DefaultNoUse: true, Description: "nginx日志格式名称(nginx_log_format_name)", }, { KeyName: NginxSchema, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "手动指定字段类型(nginx_schema)", Advance: true, ToolTip: `nginx日志都被解析为string,指定该格式可以设置为float、long、date三种类型。如 time_local date,bytes_sent long,request_time float`, ToolTipActive: true, }, { KeyName: NginxFormatRegex, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "手动指定正则表达式解析(nginx_log_format_regex)", Advance: true, ToolTip: "若根据配置文件自动生成的正则表达式无效,可通过此配置手动填写", }, OptionParserName, OptionLabels, OptionDisableRecordErrData, }, TypeGrok: { { KeyName: KeyGrokPatterns, ChooseOnly: false, Default: "%{COMMON_LOG_FORMAT}", Placeholder: "%{COMMON_LOG_FORMAT}", Required: true, DefaultNoUse: true, Description: "匹配日志的grok表达式(grok_patterns)", ToolTip: `用于匹配日志的grok表达式,多个用逗号分隔,如 "%{COMMON_LOG_FORMAT},%{QINIU_LOG_FORMAT}"`, ToolTipActive: true, }, { KeyName: KeyGrokCustomPatterns, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "自定义grok表达式(grok_custom_patterns)", }, { KeyName: KeyGrokCustomPatternFiles, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "自定义grok表达式文件路径(grok_custom_pattern_files)", Advance: true, ToolTip: `从机器获得自定义grok表达式文件`, }, OptionParserName, OptionTimezoneOffset, OptionLabels, OptionDisableRecordErrData, }, TypeCSV: { { KeyName: KeyCSVSplitter, ChooseOnly: false, Default: ",", Placeholder: ",", Required: true, DefaultNoUse: false, Description: "分隔符(csv_splitter)", ToolTip: `csv_splitter csv文件的分隔符定义,默认为','`, }, { KeyName: KeyCSVSchema, ChooseOnly: false, Default: "", Required: true, Placeholder: "abc string,xyz long,data1 string,data2 float", DefaultNoUse: true, Description: "指定字段类型(csv_schema)", ToolTip: `按照逗号分隔的字符串,如"abc string",字段类型现在支持string, long, jsonmap, float,date`, }, { KeyName: KeyCSVAllowNoMatch, Element: Radio, ChooseOnly: true, Advance: true, ChooseOptions: []interface{}{"false", "true"}, Default: "false", DefaultNoUse: false, Description: "允许日志数量可变(csv_allow_no_match)", ToolTip: `允许schema定义的字段数量和实际日志分隔出来的数量不匹配,默认不允许`, }, { KeyName: KeyCSVAllowMore, Advance: true, Default: "unknown", DefaultNoUse: false, Description: "多余字段命名(csv_allow_more)", ToolTip: `允许日志数量比定义的schema多且命名为该字段后跟数字,如"unknown0, unknown1"`, }, { KeyName: KeyCSVAllowMoreStartNum, Advance: true, Default: "0", DefaultNoUse: false, Description: "多余字段开始数字(csv_more_start_number)", ToolTip: `多余字段命名数字后缀开始数字,默认如"unknown0", 填1则为"unknown1"`, }, { KeyName: KeyCSVIgnoreInvalidField, Element: Radio, ChooseOnly: true, Advance: true, ChooseOptions: []interface{}{"false", "true"}, Default: "false", DefaultNoUse: false, Description: "忽略解析错误的字段(csv_ignore_invalid)", ToolTip: `忽略解析错误的部分,剩余部分继续发送`, }, OptionParserName, OptionLabels, OptionTimezoneOffset, { KeyName: KeyAutoRename, Element: Radio, ChooseOnly: true, ChooseOptions: []interface{}{"true", "false"}, Default: "true", DefaultNoUse: false, Description: "将字段名称中的'-'更改为'_'", Advance: true, }, OptionDisableRecordErrData, }, TypeRaw: { { KeyName: KeyTimestamp, Element: Radio, ChooseOnly: true, ChooseOptions: []interface{}{"true", "false"}, Default: "true", DefaultNoUse: false, Description: "添加系统时间戳(timestamp)", }, OptionParserName, OptionLabels, OptionDisableRecordErrData, }, TypeLogv1: { OptionLabels, { KeyName: KeyQiniulogPrefix, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "日志前缀(qiniulog_prefix)", Advance: true, }, { KeyName: KeyLogHeaders, ChooseOnly: false, Default: "", DefaultNoUse: false, Description: "日志格式顺序(qiniulog_log_headers)", Advance: true, }, OptionParserName, OptionDisableRecordErrData, }, TypeSyslog: { { KeyName: KeyRFCType, ChooseOnly: true, Default: "automic", ChooseOptions: []interface{}{"automic", "rfc3164", "rfc5424", "rfc6587"}, DefaultNoUse: false, Description: "rfc协议(syslog_rfc)", }, OptionParserName, OptionLabels, OptionDisableRecordErrData, }, TypeKafkaRest: { OptionParserName, OptionLabels, OptionDisableRecordErrData, }, TypeEmpty: {}, TypeMysqlLog: { OptionParserName, OptionLabels, OptionDisableRecordErrData, }, }
View Source
var ModeUsages = []KeyValue{ {TypeRaw, "按原始日志逐行发送"}, {TypeJson, "按 json 格式解析"}, {TypeNginx, "按 nginx 日志解析"}, {TypeGrok, "按 grok 格式解析"}, {TypeCSV, "按 csv 格式解析"}, {TypeSyslog, "按 syslog 格式解析"}, {TypeLogv1, "按七牛日志库格式解析"}, {TypeKafkaRest, "按 kafkarest 日志解析"}, {TypeEmpty, "通过解析清空数据"}, {TypeMysqlLog, "按 mysql 慢请求日志解析"}, }
ModeUsages 用途说明
View Source
var SampleLogs = map[string]string{
TypeNginx: `110.110.101.101 - - [21/Mar/2017:18:14:17 +0800] "GET /files/yyyysx HTTP/1.1" 206 607 1 "-" "Apache-HttpClient/4.4.1 (Java/1.7.0_80)" "-" "122.121.111.222, 122.121.111.333, 192.168.90.61" "192.168.42.54:5000" www.qiniu.com llEAAFgmnoIa3q0U "0.040" 0.040 760 "-" "-" - - QCloud`,
TypeGrok: `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
123.45.12.1 user-identifier bob [10/Oct/2013:13:55:36 -0700] "GET /hello.gif HTTP/1.0" 200 2326`,
TypeJson: `{"a":"b","c":1,"d":1.1}`,
TypeCSV: `a,123,bcd,1.2`,
TypeRaw: `raw log1[05-May-2017 13:44:39] [pool log] pid 4109`,
TypeSyslog: `<38>Feb 05 01:02:03 abc system[253]: Listening at 0.0.0.0:3000`,
TypeLogv1: `2016/10/20 17:30:21.433423 [GE2owHck-Y4IWJHS][WARN] github.com/qiniu/http/rpcutil.v1/rpc_util.go:203: E18102: The specified repo does not exist under the provided appid ~`,
TypeKafkaRest: `[2016-12-05 03:35:20,682] INFO 172.16.16.191 - - [05/Dec/2016:03:35:20 +0000] "POST /topics/VIP_VvBVy0tuMPPspm1A_0000000000 HTTP/1.1" 200 101640 46 (io.confluent.rest-utils.requests)`,
TypeEmpty: "empty 通过解析清空数据",
TypeMysqlLog: `# Time: 2017-12-24T02:42:00.126000Z
# User@Host: rdsadmin[rdsadmin] @ localhost [127.0.0.1] Id: 3
# Query_time: 0.020363 Lock_time: 0.018450 Rows_sent: 0 Rows_examined: 1
SET timestamp=1514083320;
use foo;
SELECT count(*) from mysql.rds_replication_status WHERE master_host IS NOT NULL and master_port IS NOT NULL GROUP BY action_timestamp,called_by_user,action,mysql_version,master_host,master_port ORDER BY action_timestamp LIMIT 1;
#`,
}
SampleLogs 样例日志,用于前端界面试玩解析器
Functions ¶
func ConvertWebParserConfig ¶ added in v1.4.6
func DetectType ¶ added in v1.3.6
func Time1DayInt ¶
func Time5MinInt ¶
Types ¶
type Automatic ¶ added in v1.3.6
type Automatic struct{}
func (*Automatic) GetParser ¶ added in v1.3.6
func (f *Automatic) GetParser(line []byte) SysLogParser
type EmptyParser ¶
type EmptyParser struct {
// contains filtered or unexported fields
}
func (*EmptyParser) Name ¶
func (p *EmptyParser) Name() string
func (*EmptyParser) Parse ¶
func (p *EmptyParser) Parse(lines []string) (datas []Data, err error)
type Format ¶ added in v1.3.6
type Format interface { GetParser([]byte) SysLogParser IsNewLine(data []byte) bool }
type GrokParser ¶
type GrokParser struct { Patterns []string // 正式的pattern名称 CustomPatterns string CustomPatternFiles []string // contains filtered or unexported fields }
func (*GrokParser) Name ¶
func (gp *GrokParser) Name() string
func (*GrokParser) Parse ¶
func (gp *GrokParser) Parse(lines []string) ([]Data, error)
func (*GrokParser) Type ¶ added in v1.3.6
func (gp *GrokParser) Type() string
type JsonParser ¶
type JsonParser struct {
// contains filtered or unexported fields
}
func (*JsonParser) Name ¶
func (im *JsonParser) Name() string
func (*JsonParser) Parse ¶
func (im *JsonParser) Parse(lines []string) ([]Data, error)
func (*JsonParser) Type ¶ added in v1.3.6
func (im *JsonParser) Type() string
type KafaRestlogParser ¶
type KafaRestlogParser struct {
// contains filtered or unexported fields
}
func (*KafaRestlogParser) Name ¶
func (krp *KafaRestlogParser) Name() string
func (*KafaRestlogParser) Parse ¶
func (krp *KafaRestlogParser) Parse(lines []string) ([]Data, error)
func (*KafaRestlogParser) ParseCode ¶
func (krp *KafaRestlogParser) ParseCode(fields []string) int
func (*KafaRestlogParser) ParseDuration ¶
func (krp *KafaRestlogParser) ParseDuration(fields []string) int
func (*KafaRestlogParser) ParseIp ¶
func (krp *KafaRestlogParser) ParseIp(fields []string) string
func (*KafaRestlogParser) ParseLogTime ¶
func (krp *KafaRestlogParser) ParseLogTime(fields []string) int64
func (*KafaRestlogParser) ParseMethod ¶
func (krp *KafaRestlogParser) ParseMethod(fields []string) string
func (*KafaRestlogParser) ParseRespCL ¶
func (krp *KafaRestlogParser) ParseRespCL(fields []string) int
func (*KafaRestlogParser) ParseTopic ¶
func (krp *KafaRestlogParser) ParseTopic(fields []string) string
func (*KafaRestlogParser) Type ¶ added in v1.3.6
func (krp *KafaRestlogParser) Type() string
type LogParser ¶
type MysqllogParser ¶ added in v1.4.5
type MysqllogParser struct {
// contains filtered or unexported fields
}
func (*MysqllogParser) Flush ¶ added in v1.4.5
func (p *MysqllogParser) Flush() (data Data, err error)
func (*MysqllogParser) Name ¶ added in v1.4.5
func (p *MysqllogParser) Name() string
func (*MysqllogParser) Parse ¶ added in v1.4.5
func (p *MysqllogParser) Parse(lines []string) ([]Data, error)
func (*MysqllogParser) Type ¶ added in v1.4.5
func (p *MysqllogParser) Type() string
type NginxParser ¶ added in v1.2.1
type NginxParser struct {
// contains filtered or unexported fields
}
func NewNginxAccParser ¶ added in v1.2.1
func NewNginxAccParser(c conf.MapConf) (p *NginxParser, err error)
func (*NginxParser) Name ¶ added in v1.2.1
func (p *NginxParser) Name() string
func (*NginxParser) Parse ¶ added in v1.2.1
func (p *NginxParser) Parse(lines []string) ([]Data, error)
func (*NginxParser) Type ¶ added in v1.3.6
func (p *NginxParser) Type() string
type ParserRegistry ¶
type ParserRegistry struct {
// contains filtered or unexported fields
}
func NewParserRegistry ¶
func NewParserRegistry() *ParserRegistry
func (*ParserRegistry) NewLogParser ¶
func (ps *ParserRegistry) NewLogParser(conf conf.MapConf) (p LogParser, err error)
func (*ParserRegistry) RegisterParser ¶
type ParserType ¶ added in v1.3.6
type ParserType interface {
Type() string
}
type QiniulogParser ¶
type QiniulogParser struct {
// contains filtered or unexported fields
}
func (*QiniulogParser) Name ¶
func (p *QiniulogParser) Name() string
func (*QiniulogParser) Parse ¶
func (p *QiniulogParser) Parse(lines []string) ([]Data, error)
func (*QiniulogParser) Type ¶ added in v1.3.6
func (p *QiniulogParser) Type() string
type RFC3164 ¶ added in v1.3.6
type RFC3164 struct{}
func (*RFC3164) GetParser ¶ added in v1.3.6
func (f *RFC3164) GetParser(line []byte) SysLogParser
type RFC5424 ¶ added in v1.3.6
type RFC5424 struct{}
func (*RFC5424) GetParser ¶ added in v1.3.6
func (f *RFC5424) GetParser(line []byte) SysLogParser
type RFC6587 ¶ added in v1.3.6
type RFC6587 struct{}
func (*RFC6587) GetParser ¶ added in v1.3.6
func (f *RFC6587) GetParser(line []byte) SysLogParser
type RawlogParser ¶
type RawlogParser struct {
// contains filtered or unexported fields
}
func (*RawlogParser) Name ¶
func (p *RawlogParser) Name() string
func (*RawlogParser) Parse ¶
func (p *RawlogParser) Parse(lines []string) ([]Data, error)
func (*RawlogParser) Type ¶ added in v1.3.6
func (p *RawlogParser) Type() string
type SysLogParser ¶ added in v1.3.6
type SyslogParser ¶ added in v1.3.6
type SyslogParser struct {
// contains filtered or unexported fields
}
func (*SyslogParser) Flush ¶ added in v1.4.5
func (p *SyslogParser) Flush() (data Data, err error)
func (*SyslogParser) Name ¶ added in v1.3.6
func (p *SyslogParser) Name() string
func (*SyslogParser) Parse ¶ added in v1.3.6
func (p *SyslogParser) Parse(lines []string) ([]Data, error)
func (*SyslogParser) Type ¶ added in v1.3.6
func (p *SyslogParser) Type() string
Click to show internal directories.
Click to hide internal directories.