awsglue

Documentation

Index

• Constants
• Variables
• func GetDataPrefix(databaseName string) string
• func GetPartitionPrefix(datatype models.DataType, logType string, timebin GlueTableTimebin, ...) string
• func GetTableName(logType string) string
• type GluePartition
  • func GetPartitionFromS3(s3Bucket, s3ObjectKey string) (*GluePartition, error)
  • func (gp *GluePartition) GetCompression() string
  • func (gp *GluePartition) GetDataFormat() string
  • func (gp *GluePartition) GetDatabase() string
  • func (gp *GluePartition) GetGlueTableMetadata() *GlueTableMetadata
  • func (gp *GluePartition) GetPartitionColumnsInfo() []PartitionColumnInfo
  • func (gp *GluePartition) GetPartitionLocation() string
  • func (gp *GluePartition) GetS3Bucket() string
  • func (gp *GluePartition) GetTable() string
  • func (gp *GluePartition) GetTime() time.Time
• type GlueTableMetadata
  • func NewGlueTableMetadata(datatype models.DataType, logType, logDescription string, ...) *GlueTableMetadata
  • func (gm *GlueTableMetadata) CreateJSONPartition(client glueiface.GlueAPI, t time.Time) (created bool, err error)
  • func (gm *GlueTableMetadata) DatabaseName() string
  • func (gm *GlueTableMetadata) Description() string
  • func (gm *GlueTableMetadata) EventStruct() interface{}
  • func (gm *GlueTableMetadata) GetPartition(client glueiface.GlueAPI, t time.Time) (output *glue.GetPartitionOutput, err error)
  • func (gm *GlueTableMetadata) GetPartitionPrefix(t time.Time) string
  • func (gm *GlueTableMetadata) LogType() string
  • func (gm *GlueTableMetadata) PartitionKeys() (partitions []PartitionKey)
  • func (gm *GlueTableMetadata) Prefix() string
  • func (gm *GlueTableMetadata) SyncPartitions(glueClient glueiface.GlueAPI, s3Client s3iface.S3API, startDate time.Time) error
  • func (gm *GlueTableMetadata) TableName() string
  • func (gm *GlueTableMetadata) Timebin() GlueTableTimebin
• type GlueTableTimebin
  • func (tb GlueTableTimebin) Next(t time.Time) (next time.Time)
  • func (tb GlueTableTimebin) PartitionValuesFromTime(t time.Time) (values []*string)
  • func (tb GlueTableTimebin) Validate() (err error)
• type PartitionColumnInfo
• type PartitionKey

Constants

const (
	LogProcessingDatabaseName        = "panther_logs"
	LogProcessingDatabaseDescription = "Holds tables with data from Panther log processing"

	RuleMatchDatabaseName        = "panther_rule_matches"
	RuleMatchDatabaseDescription = "Holds tables with data from Panther rule matching (same table structure as panther_logs)"

	ViewsDatabaseName        = "panther_views"
	ViewsDatabaseDescription = "Holds views useful for querying Panther data"

	TempDatabaseName        = "panther_temp"
	TempDatabaseDescription = "Holds temporary tables used for processing tasks"
)

Variables

var (
	// PantherDatabases is exposed as public var to allow code to get/lookup the Panther databases
	PantherDatabases = map[string]string{
		LogProcessingDatabaseName: LogProcessingDatabaseDescription,
		RuleMatchDatabaseName:     RuleMatchDatabaseDescription,
		ViewsDatabaseName:         ViewsDatabaseDescription,
	}
)

Functions

func GetDataPrefix

func GetDataPrefix(databaseName string) string

func GetPartitionPrefix

func GetPartitionPrefix(datatype models.DataType, logType string, timebin GlueTableTimebin, time time.Time) string

func GetTableName

func GetTableName(logType string) string

Types

type GluePartition

type GluePartition struct {
	// contains filtered or unexported fields
}

A partition in Glue containing Panther data

func GetPartitionFromS3

func GetPartitionFromS3(s3Bucket, s3ObjectKey string) (*GluePartition, error)

Gets the partition from S3bucket and S3 object key info. The s3Object key is expected to be in the the format `{logs,rules}/{table_name}/year=d{4}/month=d{2}/[day=d{2}/][hour=d{2}/]/{S+}.json.gz` otherwise an error is returned.

func (*GluePartition) GetCompression

func (gp *GluePartition) GetCompression() string

func (*GluePartition) GetDataFormat

func (gp *GluePartition) GetDataFormat() string

func (*GluePartition) GetDatabase

func (gp *GluePartition) GetDatabase() string

func (*GluePartition) GetGlueTableMetadata

func (gp *GluePartition) GetGlueTableMetadata() *GlueTableMetadata

func (*GluePartition) GetPartitionColumnsInfo

func (gp *GluePartition) GetPartitionColumnsInfo() []PartitionColumnInfo

func (*GluePartition) GetPartitionLocation

func (gp *GluePartition) GetPartitionLocation() string

func (*GluePartition) GetS3Bucket

func (gp *GluePartition) GetS3Bucket() string

func (*GluePartition) GetTable

func (gp *GluePartition) GetTable() string

func (*GluePartition) GetTime

func (gp *GluePartition) GetTime() time.Time

type GlueTableMetadata

type GlueTableMetadata struct {
	// contains filtered or unexported fields
}

Metadata about Glue table

func NewGlueTableMetadata

func NewGlueTableMetadata(
	datatype models.DataType, logType, logDescription string, timebin GlueTableTimebin, eventStruct interface{}) *GlueTableMetadata

Creates a new GlueTableMetadata object

func (*GlueTableMetadata) CreateJSONPartition

func (gm *GlueTableMetadata) CreateJSONPartition(client glueiface.GlueAPI, t time.Time) (created bool, err error)

func (*GlueTableMetadata) DatabaseName

func (gm *GlueTableMetadata) DatabaseName() string

func (*GlueTableMetadata) Description

func (gm *GlueTableMetadata) Description() string

func (*GlueTableMetadata) EventStruct

func (gm *GlueTableMetadata) EventStruct() interface{}

func (*GlueTableMetadata) GetPartition

func (gm *GlueTableMetadata) GetPartition(client glueiface.GlueAPI, t time.Time) (output *glue.GetPartitionOutput, err error)

get partition, return nil if it does not exist

func (*GlueTableMetadata) GetPartitionPrefix

func (gm *GlueTableMetadata) GetPartitionPrefix(t time.Time) string

Based on Timebin(), return an S3 prefix for objects of this table

func (*GlueTableMetadata) LogType

func (gm *GlueTableMetadata) LogType() string

func (*GlueTableMetadata) PartitionKeys

func (gm *GlueTableMetadata) PartitionKeys() (partitions []PartitionKey)

The partition keys for this table

func (*GlueTableMetadata) Prefix

func (gm *GlueTableMetadata) Prefix() string

All data for this table are stored in this S3 prefix

func (*GlueTableMetadata) SyncPartitions

func (gm *GlueTableMetadata) SyncPartitions(glueClient glueiface.GlueAPI, s3Client s3iface.S3API, startDate time.Time) error

SyncPartitions updates a table's partitions using the latest table schema. Used when schemas change.

func (*GlueTableMetadata) TableName

func (gm *GlueTableMetadata) TableName() string

func (*GlueTableMetadata) Timebin

func (gm *GlueTableMetadata) Timebin() GlueTableTimebin

type GlueTableTimebin

type GlueTableTimebin int

Use this to tag the time partitioning used in a GlueTableMetadata table

const (
	GlueTableMonthly GlueTableTimebin = iota + 1
	GlueTableDaily
	GlueTableHourly
)

func (GlueTableTimebin) Next

func (tb GlueTableTimebin) Next(t time.Time) (next time.Time)

Next returns the next time interval

func (GlueTableTimebin) PartitionValuesFromTime

func (tb GlueTableTimebin) PartitionValuesFromTime(t time.Time) (values []*string)

PartitionValuesFromTime returns an []*string values (used for Glue APIs)

func (GlueTableTimebin) Validate

func (tb GlueTableTimebin) Validate() (err error)

type PartitionColumnInfo

type PartitionColumnInfo struct {
	Key   string
	Value string
}

Contains information about partition columns

type PartitionKey

type PartitionKey struct {
	Name string
	Type string
}