common

Documentation

Index

• Constants
• Variables
• type DataStream
• type DataStreamHints
• type S3DataStreamHints

Constants

const (
	MaxRetries     = 20 // setting Max Retries to a higher number - we'd like to retry VERY hard before failing.
	EventDelimiter = '\n'
)

const (
	OpLogNamespace  = "log_analysis"
	OpLogComponent  = "log_processor"
	OpLogServiceDim = "service"
)

labels for oplog

Variables

var (
	OpLogManager = oplog.NewManager(OpLogNamespace, OpLogComponent)

	OpLogLambdaServiceDim    = zap.String(OpLogServiceDim, "lambda")
	OpLogS3ServiceDim        = zap.String(OpLogServiceDim, "s3")
	OpLogSNSServiceDim       = zap.String(OpLogServiceDim, "sns")
	OpLogProcessorServiceDim = zap.String(OpLogServiceDim, "processor")
	OpLogGlueServiceDim      = zap.String(OpLogServiceDim, "glue")
)

var Session = session.Must(session.NewSession(aws.NewConfig().WithMaxRetries(MaxRetries)))

Session AWS Session that can be used by components of the system

Types

type DataStream

type DataStream struct {
	Reader io.Reader
	Hints  DataStreamHints
	// The log type if known
	// If it is nil, it means the log type hasn't been identified yet
	LogType *string
}

DataStream represents a data stream that read by the processor

type DataStreamHints

type DataStreamHints struct {
	S3 *S3DataStreamHints // if nil, no hint
}

Used in a DataStream as meta data to describe the data

type S3DataStreamHints

type S3DataStreamHints struct {
	Bucket      string
	Key         string
	ContentType string
}

Used in a DataStreamHints as meta data to describe the S3 object backing the stream