datacatalog

Documentation

Index

• type Client
  • func (c *Client) SendCreateTablesForLogTypes(ctx context.Context, logTypes ...string) error
  • func (c *Client) SendSyncDatabase(ctx context.Context, traceID string, requiredLogTypes []string) error
  • func (c *Client) SendSyncDatabasePartitions(ctx context.Context, event *SyncDatabasePartitionsEvent) error
• type CreateTablesEvent
• type LambdaHandler
  • func (h *LambdaHandler) HandleCreateTablesEvent(ctx context.Context, event *CreateTablesEvent) error
  • func (h *LambdaHandler) HandleS3Event(ctx context.Context, event *events.S3Event) (err error)
  • func (h *LambdaHandler) HandleS3EventRecord(ctx context.Context, event *events.S3EventRecord) error
  • func (h *LambdaHandler) HandleSQSEvent(ctx context.Context, event *events.SQSEvent) error
  • func (h *LambdaHandler) HandleSyncDatabaseEvent(ctx context.Context, event *SyncDatabaseEvent) error
  • func (h *LambdaHandler) HandleSyncDatabasePartitionsEvent(ctx context.Context, event *SyncDatabasePartitionsEvent) error
  • func (h *LambdaHandler) HandleSyncTableEvent(ctx context.Context, event *SyncTableEvent) error
  • func (h *LambdaHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error)
• type SyncDatabaseEvent
• type SyncDatabasePartitionsEvent
• type SyncTableEvent

Types

type Client

type Client struct {
	QueueURL string
	SQSAPI   sqsiface.SQSAPI
}

func (*Client) SendCreateTablesForLogTypes

func (c *Client) SendCreateTablesForLogTypes(ctx context.Context, logTypes ...string) error

func (*Client) SendSyncDatabase

func (c *Client) SendSyncDatabase(ctx context.Context, traceID string, requiredLogTypes []string) error

func (*Client) SendSyncDatabasePartitions

func (c *Client) SendSyncDatabasePartitions(ctx context.Context, event *SyncDatabasePartitionsEvent) error

type CreateTablesEvent

type CreateTablesEvent struct {
	LogTypes []string
}

type LambdaHandler

type LambdaHandler struct {
	ProcessedDataBucket   string
	AthenaWorkgroup       string
	QueueURL              string
	ListAvailableLogTypes func(ctx context.Context) ([]string, error)
	GlueClient            glueiface.GlueAPI
	Resolver              logtypes.Resolver
	AthenaClient          athenaiface.AthenaAPI
	SQSClient             sqsiface.SQSAPI
	Logger                *zap.Logger
	// contains filtered or unexported fields
}

func (*LambdaHandler) HandleCreateTablesEvent

func (h *LambdaHandler) HandleCreateTablesEvent(ctx context.Context, event *CreateTablesEvent) error

func (*LambdaHandler) HandleS3Event

func (h *LambdaHandler) HandleS3Event(ctx context.Context, event *events.S3Event) (err error)

func (*LambdaHandler) HandleS3EventRecord

func (h *LambdaHandler) HandleS3EventRecord(ctx context.Context, event *events.S3EventRecord) error

func (*LambdaHandler) HandleSQSEvent

func (h *LambdaHandler) HandleSQSEvent(ctx context.Context, event *events.SQSEvent) error

HandleSQSEvent handles messages in an SQS event.

func (*LambdaHandler) HandleSyncDatabaseEvent

func (h *LambdaHandler) HandleSyncDatabaseEvent(ctx context.Context, event *SyncDatabaseEvent) error

func (*LambdaHandler) HandleSyncDatabasePartitionsEvent

func (h *LambdaHandler) HandleSyncDatabasePartitionsEvent(ctx context.Context, event *SyncDatabasePartitionsEvent) error

HandleSyncDatabasePartitionsEvent handles a full database sync by invoking all required table sync events in the background

func (*LambdaHandler) HandleSyncTableEvent

func (h *LambdaHandler) HandleSyncTableEvent(ctx context.Context, event *SyncTableEvent) error

HandleSyncTableEvent starts or continues a gluetasks.SyncTablePartitions task. nolint: nakedret

func (*LambdaHandler) Invoke

func (h *LambdaHandler) Invoke(ctx context.Context, payload []byte) ([]byte, error)

Invoke implements lambda.Handler interface.

This is the main entry point for Lambda code.

type SyncDatabaseEvent

type SyncDatabaseEvent struct {
	TraceID          string
	RequiredLogTypes []string
}

type SyncDatabasePartitionsEvent

type SyncDatabasePartitionsEvent struct {
	// An identifier to use in order to keep track of all 'child' Lambda invocations for this sync.
	TraceID string
	// Which databases to sync
	DatabaseNames []string
	// Which log types to sync
	LogTypes []string
	// If set to true the sync will only scan for updates and will not modify Glue partitions
	DryRun bool
}

SyncDatabasePartitionsEvent is a request to sync all database partitions

type SyncTableEvent

type SyncTableEvent struct {
	// Use a common trace id (the SyncDatabase request id) for all events triggered by a sync event.
	// This is used in all logging done by the SyncTablePartitions task to be able to trace all lambda invocations
	// back to their original SyncDatabase request.
	TraceID string
	// NumCalls keeps track of the number of recursive calls for the specific sync table event.
	// It acts as a guard against infinite recursion.
	NumCalls int
	// NumTimeouts keeps track of how many times the last token was retried because of timeout.
	NumTimeouts int
	// Embed the full sync table partitions task state
	// This allows us to continue the task by recursively calling the lambda.
	// The task carries all status information over
	gluetasks.SyncTablePartitions
}

SyncTableEvent initializes or continues a gluetasks.SyncTablePartitions task