process

Documentation

Index

• Constants
• Variables
• func Handle(ctx context.Context, event *DataCatalogEvent) (err error)
• func HandleCreateTablesMessage(ctx context.Context, msg *CreateTablesMessage) error
• func HandleS3Notification(notification S3Notification) error
• func HandleSyncEvent(ctx context.Context, event *SyncEvent) error
• func HandleSyncTableEvent(ctx context.Context, event *SyncTableEvent) error
• func InvokeBackgroundSync(ctx context.Context, lambdaAPI lambdaiface.LambdaAPI, event *SyncEvent) (err error)
• func Setup()
• type CreateTablesMessage
  • func (m CreateTablesMessage) Send(sqsClient sqsiface.SQSAPI, queueURL string) error
• type DataCatalogEvent
• type S3Notification
  • func NewS3ObjectPutNotification(bucket, key string, nbytes int) *S3Notification
• type SyncEvent
• type SyncTableEvent

Constants

const (
	// The message type key to be used when sending SQS messages. Use it to distinguish
	// the type of the message payload.
	PantherMessageType = "PantherMessageType"
)

Variables

var CreateTableMessageAttribute = sqs.MessageAttributeValue{
	DataType:    aws.String("String"),
	StringValue: aws.String("CreateTables"),
}

CreateTableMessageAttribute is the SQS message attribute for the CreateTablesMessage.

Functions

func Handle

func Handle(ctx context.Context, event *DataCatalogEvent) (err error)

func HandleCreateTablesMessage

func HandleCreateTablesMessage(ctx context.Context, msg *CreateTablesMessage) error

func HandleS3Notification

func HandleS3Notification(notification S3Notification) error

func HandleSyncEvent

func HandleSyncEvent(ctx context.Context, event *SyncEvent) error

HandleSyncEvent handles a full database sync by invoking all required table sync events in the background

func HandleSyncTableEvent

func HandleSyncTableEvent(ctx context.Context, event *SyncTableEvent) error

HandleSyncTableEvent starts or continues a gluetasks.SyncTablePartitions task. nolint: nakedret

func InvokeBackgroundSync

func InvokeBackgroundSync(ctx context.Context, lambdaAPI lambdaiface.LambdaAPI, event *SyncEvent) (err error)

InvokeBackgroundSync triggers a database sync in the background. The sync is kicked-off using a Lambda call and will be a long running task. This function does not wait for it to finish. If no TraceID is provided this function will try to use the AWS request id.

func Setup

func Setup()

Types

type CreateTablesMessage

type CreateTablesMessage struct {
	LogTypes []string
	Sync     bool // if true issue a non-blocking sync of all partitions
}

CreateTablesMessage is the event that triggers the creation of Glue tables/views for logtypes.

func (CreateTablesMessage) Send

func (m CreateTablesMessage) Send(sqsClient sqsiface.SQSAPI, queueURL string) error

type DataCatalogEvent

type DataCatalogEvent struct {
	events.SQSEvent
	SyncDatabaseEvent   *SyncEvent
	SyncTablePartitions *SyncTableEvent
}

DataCatalogEvent is the event that this lambda accepts.

type S3Notification

type S3Notification struct {
	// https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
	Records []events.S3EventRecord
}

S3Notification is sent when new data is available in S3

func NewS3ObjectPutNotification

func NewS3ObjectPutNotification(bucket, key string, nbytes int) *S3Notification

type SyncEvent

type SyncEvent struct {
	// An identifier to use in order to keep track of all 'child' Lambda invocations for this sync.
	TraceID string
	// Which databases to sync
	DatabaseNames []string
	// Which log types to sync
	LogTypes []string
	// If set to true the sync will only scan for updates and will not modify Glue partitions
	DryRun bool
}

SyncEvent is a request to start a full database sync in the background.

type SyncTableEvent

type SyncTableEvent struct {
	// Use a common trace id (the SyncDatabase request id) for all events triggered by a sync event.
	// This is used in all logging done by the SyncTablePartitions task to be able to trace all lambda invocations
	// back to their original SyncDatabase request.
	TraceID string
	// NumCalls keeps track of the number of recursive calls for the specific sync table event.
	// It acts as a guard against infinite recursion.
	NumCalls int
	// NumTimeouts keeps track of how many times the last token was retried because of timeout.
	NumTimeouts int
	// Embed the full sync table partitions task state
	// This allows us to continue the task by recursively calling the lambda.
	// The task carries all status information over
	gluetasks.SyncTablePartitions
}

SyncTableEvent initializes or continues a gluetasks.SyncTablePartitions task