common

Documentation

Index

• Constants
• Variables
• func BuildJSON() jsoniter.API
• func Setup()
• type DataStream
• type EnvConfig

Constants

const (
	MaxRetries     = 13 // ~7'
	EventDelimiter = '\n'
)

const (
	OpLogNamespace  = "log_analysis"
	OpLogComponent  = "log_processor"
	OpLogServiceDim = "service"
)

labels for oplog

Variables

var (
	// Session and clients that can be used by components of the log processor
	Session      *session.Session
	LambdaClient lambdaiface.LambdaAPI
	S3Uploader   s3manageriface.UploaderAPI
	SqsClient    sqsiface.SQSAPI
	SnsClient    snsiface.SNSAPI

	Config EnvConfig

	SQSWaitTime int64 // set by env
)

var (
	OpLogManager = oplog.NewManager(OpLogNamespace, OpLogComponent)

	OpLogLambdaServiceDim    = zap.String(OpLogServiceDim, "lambda")
	OpLogS3ServiceDim        = zap.String(OpLogServiceDim, "s3")
	OpLogSNSServiceDim       = zap.String(OpLogServiceDim, "sns")
	OpLogProcessorServiceDim = zap.String(OpLogServiceDim, "processor")
	OpLogGlueServiceDim      = zap.String(OpLogServiceDim, "glue")
)

var (
	BytesProcessedLogger = metrics.MustStaticLogger([]metrics.DimensionSet{
		{
			"LogType",
		},
	}, []metrics.Metric{
		{
			Name: "BytesProcessed",
			Unit: metrics.UnitBytes,
		},
		{
			Name: "EventsProcessed",
			Unit: metrics.UnitCount,
		},
		{
			Name: "CombinedLatency",
			Unit: metrics.UnitMilliseconds,
		},
	})
)

Functions

func BuildJSON

func BuildJSON() jsoniter.API

BuildJSON returns a jsoniter.API instance that is configured to be used for decoding/encoding JSON log events.

func Setup

func Setup()

Types

type DataStream

type DataStream struct {
	Reader      io.Reader
	Source      *models.SourceIntegration
	S3ObjectKey string
	S3Bucket    string
	ContentType string
}

DataStream represents a data stream that read by the processor

type EnvConfig

type EnvConfig struct {
	AwsLambdaFunctionMemorySize int    `required:"true" split_words:"true"`
	ProcessedDataBucket         string `required:"true" split_words:"true"`
	SqsQueueURL                 string `required:"true" split_words:"true"`
	SqsDelaySec                 int64  `required:"true" split_words:"true"`
	SnsTopicARN                 string `required:"true" split_words:"true"`
}