gluecf

Documentation

Overview

Package gluecf generates CloudFormation from Go objects to create AWS GlueTableMetadata objects.

Index

• Constants
• Variables
• func GenerateTables(tables []*awsglue.GlueTableMetadata) (cf []byte, err error)
• type Column
  • func InferJSONColumns(obj interface{}, customMappings ...CustomMapping) (cols []Column)
• type CustomMapping
• type Database
  • func NewDatabase(catalogID interface{}, name, description string) (db *Database)
• type DatabaseInput
• type DatabaseProperties
• type NewTableInput
• type SerdeInfo
• type StorageDescriptor
• type Table
  • func NewJSONLTable(input *NewTableInput) (db *Table)
  • func NewParquetTable(input *NewTableInput) (db *Table)
• type TableInput
• type TableProperties

Constants

const GlueTimestampType = "timestamp"

Glue tables typ for timestamps that we will re-map Go times

Variables

var (
	CatalogIDRef = cfngen.Ref{Ref: "AWS::AccountId"} // macro expand to accountId for CF

	// GlueMappings for custom Panther types.
	GlueMappings = []CustomMapping{
		{
			From: reflect.TypeOf(timestamp.RFC3339{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(timestamp.ANSICwithTZ{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(timestamp.UnixMillisecond{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(timestamp.FluentdTimestamp{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(timestamp.UnixFloat{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(timestamp.SuricataTimestamp{}),
			To:   GlueTimestampType,
		},
		{
			From: reflect.TypeOf(parsers.PantherAnyString{}),
			To:   "array<string>",
		},
		{
			From: reflect.TypeOf(jsoniter.RawMessage{}),
			To:   "string",
		},
		{
			From: reflect.TypeOf(*new(numerics.Integer)),
			To:   "bigint",
		},
		{
			From: reflect.TypeOf(*new(numerics.Int64)),
			To:   "bigint",
		},
	}

	// RuleMatchColumns are columns added by the rules engine
	RuleMatchColumns = []Column{
		{
			Name:    "p_rule_id",
			Type:    "string",
			Comment: "Rule id",
		},
		{
			Name:    "p_alert_id",
			Type:    "string",
			Comment: "Alert id",
		},
		{
			Name:    "p_alert_creation_time",
			Type:    "timestamp",
			Comment: "The time the alert was initially created (first match)",
		},
		{
			Name:    "p_alert_update_time",
			Type:    "timestamp",
			Comment: "The time the alert last updated (last match)",
		},
	}
)

Functions

func GenerateTables

func GenerateTables(tables []*awsglue.GlueTableMetadata) (cf []byte, err error)

Output CloudFormation for all 'tables'

Types

type Column

type Column struct {
	Name     string
	Type     string              // this is the Glue type
	Comment  string              `json:",omitempty"`
	Required bool                `json:"-"` // do NOT serialize! Not used for Glue CF (used for doc).
	Field    reflect.StructField `json:"-"` // do NOT serialize! Not used for Glue CF (used for doc).
}

func InferJSONColumns

func InferJSONColumns(obj interface{}, customMappings ...CustomMapping) (cols []Column)

Walk object, create columns using JSON Serde expected types, allow optional custom mappings

type CustomMapping

type CustomMapping struct {
	From reflect.Type // type to map (result of reflect.TypeOf() )
	To   string       // glue type to emit
}

type Database

type Database struct {
	Type       string
	DependsOn  []string `json:",omitempty"`
	Properties DatabaseProperties
}

func NewDatabase

func NewDatabase(catalogID interface{}, name, description string) (db *Database)

type DatabaseInput

type DatabaseInput struct {
	Name        interface{}
	Description string `json:",omitempty"`
}

Matches CF structure

type DatabaseProperties

type DatabaseProperties struct {
	CatalogID     interface{} `json:"CatalogId"` // required, string or Ref{}, need json tag to keep linter happy
	DatabaseInput DatabaseInput
}

type NewTableInput

type NewTableInput struct {
	CatalogID     interface{}
	DatabaseName  interface{}
	Name          interface{}
	Description   interface{}
	Location      interface{}
	Columns       []Column
	PartitionKeys []Column
}

inputs to table generation functions

type SerdeInfo

type SerdeInfo struct {
	SerializationLibrary string                 `json:",omitempty"`
	Parameters           map[string]interface{} `json:",omitempty"`
}

type StorageDescriptor

type StorageDescriptor struct {
	InputFormat            string
	OutputFormat           string
	Compressed             bool        `json:",omitempty"`
	Location               interface{} // required
	BucketColumns          []Column    `json:",omitempty"`
	SortColumns            []Column    `json:",omitempty"`
	StoredAsSubDirectories bool        `json:",omitempty"`
	SerdeInfo              SerdeInfo
	Columns                []Column
}

type Table

type Table struct {
	Type       string
	DependsOn  []string `json:",omitempty"`
	Properties TableProperties
}

func NewJSONLTable

func NewJSONLTable(input *NewTableInput) (db *Table)

func NewParquetTable

func NewParquetTable(input *NewTableInput) (db *Table)

type TableInput

type TableInput struct {
	TableType         string
	Name              interface{}
	Description       interface{} `json:",omitempty"`
	StorageDescriptor StorageDescriptor
	PartitionKeys     []Column `json:",omitempty"`
}

type TableProperties

type TableProperties struct {
	CatalogID    interface{} `json:"CatalogId"` // required,  string or Ref{}, need json tag to keep linter happy
	DatabaseName interface{}
	TableInput   TableInput
}