common

Documentation

Index

• Constants
• Variables
• func ConfigForDataLakeWriters() jsoniter.API
• func Setup()
• type DataStream
• type EnvConfig

Constants

const (
	MaxRetries     = 13 // ~7'
	EventDelimiter = '\n'
)

const (
	OpLogNamespace  = "log_analysis"
	OpLogComponent  = "log_processor"
	OpLogServiceDim = "service"
)

labels for oplog

Variables

var (
	// Session and clients that can be used by components of the log processor
	// FIXME: these should be removed as globals
	Session      *session.Session
	LambdaClient lambdaiface.LambdaAPI
	S3Client     s3iface.S3API
	SqsClient    sqsiface.SQSAPI
	SnsClient    snsiface.SNSAPI

	Config EnvConfig
)

var (
	OpLogManager = oplog.NewManager(OpLogNamespace, OpLogComponent)

	OpLogLambdaServiceDim    = zap.String(OpLogServiceDim, "lambda")
	OpLogS3ServiceDim        = zap.String(OpLogServiceDim, "s3")
	OpLogSNSServiceDim       = zap.String(OpLogServiceDim, "sns")
	OpLogProcessorServiceDim = zap.String(OpLogServiceDim, "processor")
	OpLogGlueServiceDim      = zap.String(OpLogServiceDim, "glue")
)

var (
	BytesProcessedLogger = metrics.MustStaticLogger([]metrics.DimensionSet{
		{
			"LogType",
		},
	}, []metrics.Metric{
		{
			Name: "BytesProcessed",
			Unit: metrics.UnitBytes,
		},
		{
			Name: "EventsProcessed",
			Unit: metrics.UnitCount,
		},
		{
			Name: "CombinedLatency",
			Unit: metrics.UnitMilliseconds,
		},
	})
)

Functions

func ConfigForDataLakeWriters

func ConfigForDataLakeWriters() jsoniter.API

ConfigForDataLakeWriters returns a jsoniter.API configured to be used for JSON log events written to the data-lake.

WARNING: This is meant to be used for encoding ONLY FOR WRITING TO S3 Processed logs.

For all other uses please use pantherlog.ConfigJSON()
You should probably not use it.

func Setup

func Setup()

Types

type DataStream

type DataStream struct {
	Closer       io.Closer // cleans up resources
	Reader       io.Reader
	Source       *models.SourceIntegration
	S3ObjectKey  string
	S3Bucket     string
	S3ObjectSize int64
}

DataStream represents a data stream that read by the processor

type EnvConfig

type EnvConfig struct {
	AwsLambdaFunctionMemorySize int    `required:"true" split_words:"true"`
	ProcessedDataBucket         string `required:"true" split_words:"true"`
	SqsQueueURL                 string `required:"true" split_words:"true"`
	SqsBatchSize                int64  `required:"true" split_words:"true"`
	SnsTopicARN                 string `required:"true" split_words:"true"`
}