sieve

README

Sieve

Sieve is a logyard client that parses the log stream and streams back the extracted structured data.

Documentation

Overview

provides fast matching algorithms TODO: aho-corasic on substring matching

Index

• type Event
  • func (event *Event) MustPublish(pub *zmqpubsub.Publisher)
• type EventHandler
• type EventParser
• type EventParserGroup
• type EventParserSpec
  • func (p *EventParserSpec) ToEventParser() *EventParser
• type JsonEventHandler
  • func NewJsonEventHandler(severity string, descTmpl string) JsonEventHandler
  • func (handler JsonEventHandler) HandleEvent(results []string, event *Event) error
• type MultiRegexpMatcher
  • func NewMultiRegexpMatcher() *MultiRegexpMatcher
  • func (m *MultiRegexpMatcher) Build()
  • func (m *MultiRegexpMatcher) Match(text string) (string, []string)
  • func (m *MultiRegexpMatcher) MustAdd(name string, substring string, re string)
• type Parser
  • func NewParser(tree map[string]EventParserGroup) Parser
  • func NewStackatoParser(spec map[string]map[string]EventParserSpec) Parser
  • func (parser Parser) Build()
  • func (parser Parser) DeleteSamples()
  • func (parser Parser) Parse(group_name string, text string) (*Event, error)
• type SimpleEventHandler
  • func NewSimpleEventHandler(severity string, descTmpl string) SimpleEventHandler
  • func (handler SimpleEventHandler) HandleEvent(results []string, event *Event) error

Types

type Event

type Event struct {
	Type     string                 `json:"type"`     // Event identifier.
	Desc     string                 `json:"desc"`     // Event description
	Severity string                 `json:"severity"` // Event severity (INFO, WARN, ERROR)
	Info     map[string]interface{} `json:"info"`     // Aribtrary data specific to this event
	Process  string                 `json:"process"`  // The process that generated this event
	common.MessageCommon
}

func (*Event) MustPublish

func (event *Event) MustPublish(pub *zmqpubsub.Publisher)

type EventHandler

type EventHandler interface {
	HandleEvent(results []string, event *Event) error
}

type EventParser

type EventParser struct {
	Substring string       // substring unique to this log record for efficient matching
	Re        string       // regex to use for matching
	Sample    string       // sample log record
	Handler   EventHandler // event handler
}

type EventParserGroup

type EventParserGroup map[string]*EventParser

EventParserGroup is a group of event parsers, which group is matched in a single attempt, independent of other groups.

type EventParserSpec

type EventParserSpec struct {
	Substring   string `json:"substring"`
	Re          string `json:"regex"`
	Sample      string `json:"sample"`
	Format      string `json:"format"`
	Severity    string `json:"severity"`
	HandlerType string `json:"handlertype"`
}

TODO: somehow merge this redundant struct with EventParser.

func (*EventParserSpec) ToEventParser

func (p *EventParserSpec) ToEventParser() *EventParser

type JsonEventHandler

type JsonEventHandler struct {
	*template.Template
	Severity string
}

SimpleEventHandler assigns event description formatted based on fields of JSON extracted from the first and only regex match group

func NewJsonEventHandler

func NewJsonEventHandler(severity string, descTmpl string) JsonEventHandler

func (JsonEventHandler) HandleEvent

func (handler JsonEventHandler) HandleEvent(results []string, event *Event) error

type MultiRegexpMatcher

type MultiRegexpMatcher struct {
	// contains filtered or unexported fields
}

MultiRegexpMatch allows matching a string against multiple regular expressions along with substrings for a fast fail-early matching.

func NewMultiRegexpMatcher

func NewMultiRegexpMatcher() *MultiRegexpMatcher

func (*MultiRegexpMatcher) Build

func (m *MultiRegexpMatcher) Build()

func (*MultiRegexpMatcher) Match

func (m *MultiRegexpMatcher) Match(text string) (string, []string)

Match tries to match the text against one of the substring/regexp as efficiently as possible.

func (*MultiRegexpMatcher) MustAdd

func (m *MultiRegexpMatcher) MustAdd(name string, substring string, re string)

type Parser

type Parser struct {
	// contains filtered or unexported fields
}

func NewParser

func NewParser(tree map[string]EventParserGroup) Parser

func NewStackatoParser

func NewStackatoParser(spec map[string]map[string]EventParserSpec) Parser

func (Parser) Build

func (parser Parser) Build()

func (Parser) DeleteSamples

func (parser Parser) DeleteSamples()

DeleteSamples deletes the samples (EventParser.Sample) to free up some memory.

func (Parser) Parse

func (parser Parser) Parse(group_name string, text string) (*Event, error)

Parser parses the given message under the given group and returns the matching event.

type SimpleEventHandler

type SimpleEventHandler struct {
	*template.Template
	Severity string
}

SimpleEventHandler assigns event description formatted based on regex groups

func NewSimpleEventHandler

func NewSimpleEventHandler(severity string, descTmpl string) SimpleEventHandler

func (SimpleEventHandler) HandleEvent

func (handler SimpleEventHandler) HandleEvent(results []string, event *Event) error