awsglue

Documentation

Index

• Constants
• type GlueMetadata
  • func NewGlueMetadata(databaseName, tableName, description string, timebin GlueTableTimebin, ...) (gm *GlueMetadata, err error)
  • func (gm *GlueMetadata) CreateJSONPartition(client glueiface.GlueAPI, t time.Time) error
  • func (gm *GlueMetadata) DatabaseName() string
  • func (gm *GlueMetadata) DeletePartition(client glueiface.GlueAPI, t time.Time) (output *glue.DeletePartitionOutput, err error)
  • func (gm *GlueMetadata) Description() string
  • func (gm *GlueMetadata) EventStruct() interface{}
  • func (gm *GlueMetadata) GetPartition(client glueiface.GlueAPI, t time.Time) (output *glue.GetPartitionOutput, err error)
  • func (gm *GlueMetadata) PartitionKeys() (partitions []Partition)
  • func (gm *GlueMetadata) PartitionPrefix(t time.Time) (prefix string)
  • func (gm *GlueMetadata) PartitionValues(t time.Time) (values []*string)
  • func (gm *GlueMetadata) S3Prefix() string
  • func (gm *GlueMetadata) SyncPartition(client glueiface.GlueAPI, t time.Time) error
  • func (gm *GlueMetadata) TableName() string
  • func (gm *GlueMetadata) Timebin() GlueTableTimebin
• type GlueTableTimebin
  • func (tb GlueTableTimebin) Next(t time.Time) (next time.Time)
  • func (tb GlueTableTimebin) Validate() (err error)
• type Partition

Constants

const (
	LogS3Prefix = "logs"

	TablesDatabaseName        = "panther_tables"
	TablesDatabaseDescription = "Holds tables generated by Panther data processing"

	ViewsDatabaseName        = "panther_views"
	ViewsDatabaseDescription = "Holds views useful for querying Panther tables"

	GlueTimestampType = "timestamp" // type in Glue tables for timestamps that we will re-map Go times
)

Types

type GlueMetadata

type GlueMetadata struct {
	// contains filtered or unexported fields
}

Meta data about Glue table over parser data written to S3 NOTE: this struct has all accessor behind functions to allow a lazy evaluation

so the cost of creating the schema is only when actually needing this information.

func NewGlueMetadata

func NewGlueMetadata(databaseName, tableName, description string, timebin GlueTableTimebin,
	timeUnpadded bool, eventStruct interface{}) (gm *GlueMetadata, err error)

func (*GlueMetadata) CreateJSONPartition

func (gm *GlueMetadata) CreateJSONPartition(client glueiface.GlueAPI, t time.Time) error

func (*GlueMetadata) DatabaseName

func (gm *GlueMetadata) DatabaseName() string

func (*GlueMetadata) DeletePartition

func (gm *GlueMetadata) DeletePartition(client glueiface.GlueAPI, t time.Time) (output *glue.DeletePartitionOutput, err error)

func (*GlueMetadata) Description

func (gm *GlueMetadata) Description() string

func (*GlueMetadata) EventStruct

func (gm *GlueMetadata) EventStruct() interface{}

func (*GlueMetadata) GetPartition

func (gm *GlueMetadata) GetPartition(client glueiface.GlueAPI, t time.Time) (output *glue.GetPartitionOutput, err error)

func (*GlueMetadata) PartitionKeys

func (gm *GlueMetadata) PartitionKeys() (partitions []Partition)

func (*GlueMetadata) PartitionPrefix

func (gm *GlueMetadata) PartitionPrefix(t time.Time) (prefix string)

Based on Timebin(), return an S3 prefix for objects

func (*GlueMetadata) PartitionValues

func (gm *GlueMetadata) PartitionValues(t time.Time) (values []*string)

Based on Timebin(), return an []*string values (used for Glue APIs)

func (*GlueMetadata) S3Prefix

func (gm *GlueMetadata) S3Prefix() string

func (*GlueMetadata) SyncPartition

func (gm *GlueMetadata) SyncPartition(client glueiface.GlueAPI, t time.Time) error

SyncPartition deletes and re-creates a partition using the latest table schema. Used when schemas change.

func (*GlueMetadata) TableName

func (gm *GlueMetadata) TableName() string

func (*GlueMetadata) Timebin

func (gm *GlueMetadata) Timebin() GlueTableTimebin

type GlueTableTimebin

type GlueTableTimebin int

Use this to tag the time partitioning used in a Glue table

const (
	GlueTableMonthly GlueTableTimebin = iota + 1
	GlueTableDaily
	GlueTableHourly
)

func (GlueTableTimebin) Next

func (tb GlueTableTimebin) Next(t time.Time) (next time.Time)

return the next time interval

func (GlueTableTimebin) Validate

func (tb GlueTableTimebin) Validate() (err error)

type Partition

type Partition struct {
	Name string
	Type string
}