gluetasks

Documentation

Index

• type RecoverDatabaseTables
  • func (r *RecoverDatabaseTables) Run(ctx context.Context, glueAPI glueiface.GlueAPI, s3API s3iface.S3API, ...) error
• type RecoverStats
• type RecoverTablePartitions
  • func (r *RecoverTablePartitions) Run(ctx context.Context, apiGlue glueiface.GlueAPI, apiS3 s3iface.S3API, ...) error
• type SyncDatabaseTables
  • func (s *SyncDatabaseTables) Run(ctx context.Context, api glueiface.GlueAPI, log *zap.Logger) error
• type SyncStats
• type SyncTablePartitions
  • func (s *SyncTablePartitions) Run(ctx context.Context, api glueiface.GlueAPI, log *zap.Logger) error

Types

type RecoverDatabaseTables

type RecoverDatabaseTables struct {
	// DatabaseName scans this Glue database for missing partitions
	DatabaseName string
	// MatchPrefix will match tables whose name begins with this prefix
	MatchPrefix string
	// Start sets the start of the scan range
	Start time.Time
	// End sets the end of the scan range
	End time.Time
	// NumWorkers sets the number of parallel scans to run on each table
	NumWorkers int
	// DryRun is a flag to not modify any partitions
	DryRun bool
	// Stats holds the stats for all tables recovered
	Stats RecoverStats
}

func (*RecoverDatabaseTables) Run

func (r *RecoverDatabaseTables) Run(ctx context.Context, glueAPI glueiface.GlueAPI, s3API s3iface.S3API, log *zap.Logger) error

Run executes the recovery

type RecoverStats

type RecoverStats struct {
	NumRecovered int
	NumS3Hit     int
	NumFailed    int
	NumS3Miss    int
	NumProcessed int
}

type RecoverTablePartitions

type RecoverTablePartitions struct {
	DatabaseName string
	TableName    string
	NumWorkers   int
	DryRun       bool
	Start        time.Time
	End          time.Time
	LastDate     time.Time
	Stats        RecoverStats
}

RecoverTablePartitions scans a date range to recover missing partitions

func (*RecoverTablePartitions) Run

func (r *RecoverTablePartitions) Run(ctx context.Context, apiGlue glueiface.GlueAPI, apiS3 s3iface.S3API, log *zap.Logger) error

type SyncDatabaseTables

type SyncDatabaseTables struct {
	Start                time.Time
	MatchPrefix          string
	DatabaseName         string
	NumWorkers           int
	Stats                SyncStats
	DryRun               bool
	AfterTableCreateTime bool
}

func (*SyncDatabaseTables) Run

func (s *SyncDatabaseTables) Run(ctx context.Context, api glueiface.GlueAPI, log *zap.Logger) error

type SyncStats

type SyncStats struct {
	NumPages         int
	NumPartitions    int
	NumDiff          int
	NumSynced        int
	MinTime, MaxTime time.Time
}

type SyncTablePartitions

type SyncTablePartitions struct {
	DatabaseName         string
	TableName            string
	NumWorkers           int
	NextToken            string
	Stats                SyncStats
	AfterTableCreateTime bool
	DryRun               bool
}

func (*SyncTablePartitions) Run

func (s *SyncTablePartitions) Run(ctx context.Context, api glueiface.GlueAPI, log *zap.Logger) error