awsglue

package
v1.1.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 22, 2020 License: AGPL-3.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source
const (
	LogProcessingDatabaseName        = "panther_logs"
	LogProcessingDatabaseDescription = "Holds tables with data from Panther log processing"

	RuleMatchDatabaseName        = "panther_rule_matches"
	RuleMatchDatabaseDescription = "Holds tables with data from Panther rule matching (same table structure as panther_logs)"

	ViewsDatabaseName        = "panther_views"
	ViewsDatabaseDescription = "Holds views useful for querying Panther data"
)

Variables

View Source
var (
	// PantherDatabases is exposed as public var to allow code to get/lookup the Panther databases
	PantherDatabases = map[string]string{
		LogProcessingDatabaseName: LogProcessingDatabaseDescription,
		RuleMatchDatabaseName:     RuleMatchDatabaseDescription,
		ViewsDatabaseName:         ViewsDatabaseDescription,
	}
)

Functions

func GetPartitionPrefix added in v1.0.0

func GetPartitionPrefix(datatype models.DataType, logType string, timebin GlueTableTimebin, time time.Time) string

func GetTableName added in v1.0.0

func GetTableName(logType string) string

Types

type GluePartition added in v1.0.0

type GluePartition struct {
	// contains filtered or unexported fields
}

A partition in Glue containing Panther data

func GetPartitionFromS3 added in v1.0.0

func GetPartitionFromS3(s3Bucket, s3ObjectKey string) (*GluePartition, error)

Gets the partition from S3bucket and S3 object key info. The s3Object key is expected to be in the the format `{logs,rules}/{table_name}/year=d{4}/month=d{2}/[day=d{2}/][hour=d{2}/]/{S+}.json.gz` otherwise an error is returned.

func (*GluePartition) CreatePartition added in v1.0.0

func (gp *GluePartition) CreatePartition(client glueiface.GlueAPI) error

Creates a new partition in Glue using the client provided.

func (*GluePartition) GetCompression added in v1.0.0

func (gp *GluePartition) GetCompression() string

func (*GluePartition) GetDataFormat added in v1.0.0

func (gp *GluePartition) GetDataFormat() string

func (*GluePartition) GetDatabase added in v1.0.0

func (gp *GluePartition) GetDatabase() string

func (*GluePartition) GetPartitionColumnsInfo added in v1.0.0

func (gp *GluePartition) GetPartitionColumnsInfo() []PartitionColumnInfo

func (*GluePartition) GetPartitionLocation added in v1.0.0

func (gp *GluePartition) GetPartitionLocation() string

func (*GluePartition) GetS3Bucket added in v1.0.0

func (gp *GluePartition) GetS3Bucket() string

func (*GluePartition) GetTable added in v1.0.0

func (gp *GluePartition) GetTable() string

type GlueTableMetadata added in v1.0.0

type GlueTableMetadata struct {
	// contains filtered or unexported fields
}

Metadata about Glue table

func NewGlueTableMetadata added in v1.0.0

func NewGlueTableMetadata(
	datatype models.DataType, logType, logDescription string, timebin GlueTableTimebin, eventStruct interface{}) *GlueTableMetadata

Creates a new GlueTableMetadata object

func (*GlueTableMetadata) CreateJSONPartition added in v1.0.0

func (gm *GlueTableMetadata) CreateJSONPartition(client glueiface.GlueAPI, t time.Time) error

func (*GlueTableMetadata) DatabaseName added in v1.0.0

func (gm *GlueTableMetadata) DatabaseName() string

func (*GlueTableMetadata) Description added in v1.0.0

func (gm *GlueTableMetadata) Description() string

func (*GlueTableMetadata) EventStruct added in v1.0.0

func (gm *GlueTableMetadata) EventStruct() interface{}

func (*GlueTableMetadata) GetPartitionPrefix added in v1.0.0

func (gm *GlueTableMetadata) GetPartitionPrefix(t time.Time) string

Based on Timebin(), return an S3 prefix for objects of this table

func (*GlueTableMetadata) LogType added in v1.0.0

func (gm *GlueTableMetadata) LogType() string

func (*GlueTableMetadata) PartitionKeys added in v1.0.0

func (gm *GlueTableMetadata) PartitionKeys() (partitions []PartitionKey)

The partition keys for this table

func (*GlueTableMetadata) Prefix added in v1.0.0

func (gm *GlueTableMetadata) Prefix() string

All data for this table are stored in this S3 prefix

func (*GlueTableMetadata) SyncPartitions added in v1.0.0

func (gm *GlueTableMetadata) SyncPartitions(glueClient glueiface.GlueAPI, s3Client s3iface.S3API, startDate time.Time) error

SyncPartitions updates a table's partitions using the latest table schema. Used when schemas change.

func (*GlueTableMetadata) TableName added in v1.0.0

func (gm *GlueTableMetadata) TableName() string

func (*GlueTableMetadata) Timebin added in v1.0.0

func (gm *GlueTableMetadata) Timebin() GlueTableTimebin

type GlueTableTimebin

type GlueTableTimebin int

Use this to tag the time partitioning used in a GlueTableMetadata table

const (
	GlueTableMonthly GlueTableTimebin = iota + 1
	GlueTableDaily
	GlueTableHourly
)

func (GlueTableTimebin) Next added in v0.3.0

func (tb GlueTableTimebin) Next(t time.Time) (next time.Time)

return the next time interval

func (GlueTableTimebin) Validate

func (tb GlueTableTimebin) Validate() (err error)

type PartitionColumnInfo added in v1.0.0

type PartitionColumnInfo struct {
	Key   string
	Value string
}

Contains information about partition columns

type PartitionKey added in v1.0.0

type PartitionKey struct {
	Name string
	Type string
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL