log

README

Log validation service

To get log validation,

1. register log listener, to dynamically detect any log changes (log shrinking/rotation is supported), any new log records are queued to be validated.
2. run log validation. Log validation verifies actual and expected log records, shifting record from actual logs pending queue.
3. reset - optionally reset log queues, to discard pending validation logs.

Service Id	Action	Description	Request	Response
validator/log	listen	start listening for log file changes on specified location	ListenRequest	ListenResponse
validator/log	reset	discard logs detected by listener	ResetRequest	ResetResponse
validator/log	assert	perform validation on provided expected log records against actual log file records.	AssertRequest	AssertResponse

Documentation

Index

• Constants
• func New() endly.Service
• type AssertRequest
• type AssertResponse
  • func (r *AssertResponse) Assertion() []*assertly.Validation
• type ExpectedRecord
• type File
  • func (f *File) HasPendingLogs() bool
  • func (f *File) PushLogRecord(record *Record)
  • func (f *File) Reset(object storage.Object)
  • func (f *File) ShiftLogRecord() *Record
  • func (f *File) ShiftLogRecordByIndex(value string) *Record
• type IndexedRecord
• type ListenRequest
• type ListenResponse
• type ProcessingState
  • func (s *ProcessingState) Reset()
  • func (s *ProcessingState) Update(position, lineNumber int) (string, int)
• type Record
  • func (r *Record) AsMap() (map[string]interface{}, error)
• type RecordAssert
• type ResetRequest
• type ResetResponse
• type Type
  • func (t *Type) GetIndexExpr() (*regexp.Regexp, error)
  • func (t *Type) UseIndex() bool
• type TypeMeta
  • func NewTypeMeta(source *url.Resource, logType *Type) *TypeMeta
  • func (m *TypeMeta) Iterator() toolbox.Iterator
• type TypesMeta

Constants

const (
	//ServiceID represents log validator service id.
	ServiceID = "validator/log"
)

Functions

func New

func New() endly.Service

New creates a new log validator service.

Types

type AssertRequest

type AssertRequest struct {
	LogWaitTimeMs      int
	LogWaitRetryCount  int
	Description        string
	ExpectedLogRecords []*ExpectedRecord `required:"true" description:"expected log data"`
}

AssertRequest represents a log assert request

type AssertResponse

type AssertResponse struct {
	Validations []*assertly.Validation
}

AssertResponse represents a log assert response

func (*AssertResponse) Assertion

func (r *AssertResponse) Assertion() []*assertly.Validation

Assertion returns description with validation slice

type ExpectedRecord

type ExpectedRecord struct {
	TagID   string `description:"neatly tag id for matching validation summary"`
	Type    string `required:"true" description:"log type register with listener"`
	Records []interface{}
}

ExpectedRecord represents an expected log record.

type File

type File struct {
	URL     string
	Content string
	Name    string
	*Type
	ProcessingState *ProcessingState
	LastModified    time.Time
	Size            int
	Records         []*Record
	IndexedRecords  map[string]*Record
	Mutex           *sync.RWMutex
}

File represents a log file

func (*File) HasPendingLogs

func (f *File) HasPendingLogs() bool

HasPendingLogs returns true if file has pending validation records

func (*File) PushLogRecord

func (f *File) PushLogRecord(record *Record)

PushLogRecord appends provided log record to the records.

func (*File) Reset

func (f *File) Reset(object storage.Object)

Reset resets processing state

func (*File) ShiftLogRecord

func (f *File) ShiftLogRecord() *Record

ShiftLogRecord returns and remove the first log record if present

func (*File) ShiftLogRecordByIndex

func (f *File) ShiftLogRecordByIndex(value string) *Record

ShiftLogRecordByIndex returns and remove the first log record if present

type IndexedRecord

type IndexedRecord struct {
	*Record
	IndexValue string
}

IndexedRecord represents indexed log record

type ListenRequest

type ListenRequest struct {
	FrequencyMs int
	Source      *url.Resource `required:"true" description:"log location"`
	Types       []*Type       `required:"true" description:"log types"`
}

ListenRequest represents listen for a logs request.

type ListenResponse

type ListenResponse struct {
	Meta TypesMeta
}

ListenResponse represents a log validation listen response.

type ProcessingState

type ProcessingState struct {
	Line     int
	Position int
}

ProcessingState represents log processing state

func (*ProcessingState) Reset

func (s *ProcessingState) Reset()

Reset resets processing state

func (*ProcessingState) Update

func (s *ProcessingState) Update(position, lineNumber int) (string, int)

Update updates processed position and line number

type Record

type Record struct {
	URL    string
	Number int
	Line   string
}

Record represents a log record

func (*Record) AsMap

func (r *Record) AsMap() (map[string]interface{}, error)

AsMap returns log records as map

type RecordAssert

type RecordAssert struct {
	TagID    string
	Expected interface{}
	Actual   interface{}
}

RecordAssert represents log record assert

type ResetRequest

type ResetRequest struct {
	LogTypes []string `required:"true" description:"log types to reset"`
}

ResetRequest represents a log reset request

type ResetResponse

type ResetResponse struct {
	LogFiles []string
}

ResetResponse represents a log reset response

type Type

type Type struct {
	Name         string `required:"true" description:"log type name"`
	Format       string `description:"log format"`
	Mask         string `description:"expected log file mast"`
	Exclusion    string `description:"if specified, exclusion fragment can not match log record"`
	Inclusion    string `description:"if specified, inclusion fragment must match log record"`
	IndexRegExpr string `description:"provide expression for indexing log messages, in this case position based logging will not apply"` //provide expression for indexing log message, in this case position based logging will not apply

	UDF string `description:"registered user defined function to transform content file before applying validation i,e decompress"`
	// contains filtered or unexported fields
}

Type represents a log type

func (*Type) GetIndexExpr

func (t *Type) GetIndexExpr() (*regexp.Regexp, error)

GetIndexExpr returns index expression.

func (*Type) UseIndex

func (t *Type) UseIndex() bool

UseIndex returns true if index can be used.

type TypeMeta

type TypeMeta struct {
	Source   *url.Resource
	LogType  *Type
	LogFiles map[string]*File
}

TypeMeta represents a log type meta

func NewTypeMeta

func NewTypeMeta(source *url.Resource, logType *Type) *TypeMeta

NewTypeMeta creates a nre log type meta.

func (*TypeMeta) Iterator

func (m *TypeMeta) Iterator() toolbox.Iterator

Iterator returns log record iterator

type TypesMeta

type TypesMeta map[string]*TypeMeta

TypesMeta represents log type meta details