gluecf

Documentation

Overview

Package gluecf generates CloudFormation from Go objects to create AWS Glue objects.

Index

• Variables
• func GenerateCloudFormation(tables []*awsglue.GlueMetadata) (cf []byte, err error)
• type Column
  • func InferJSONColumns(obj interface{}, customMappings ...CustomMapping) (cols []Column)
• type CustomMapping
• type Database
  • func NewDatabase(catalogID interface{}, name, description string) (db *Database)
• type DatabaseInput
• type DatabaseProperties
• type NewTableInput
• type SerdeInfo
• type StorageDescriptor
• type Table
  • func NewJSONLTable(input *NewTableInput) (db *Table)
  • func NewParquetTable(input *NewTableInput) (db *Table)
• type TableInput
• type TableProperties

Variables

var (
	CatalogIDRef = cfngen.Ref{Ref: "AWS::AccountId"} // macro expand to accountId for CF

)

Functions

func GenerateCloudFormation

func GenerateCloudFormation(tables []*awsglue.GlueMetadata) (cf []byte, err error)

Output CloudFormation for all 'tables'

Types

type Column

type Column struct {
	Name    string
	Type    string
	Comment string `json:",omitempty"`
}

func InferJSONColumns

func InferJSONColumns(obj interface{}, customMappings ...CustomMapping) (cols []Column)

Walk object, create columns using JSON Serde expected types, allow optional custom mappings

type CustomMapping

type CustomMapping struct {
	From reflect.Type // type to map (result of reflect.TypeOf() )
	To   string       // glue type to emit
}

type Database

type Database struct {
	Type       string
	DependsOn  []string `json:",omitempty"`
	Properties DatabaseProperties
}

func NewDatabase

func NewDatabase(catalogID interface{}, name, description string) (db *Database)

type DatabaseInput

type DatabaseInput struct {
	Name        interface{}
	Description string `json:",omitempty"`
}

Matches CF structure

type DatabaseProperties

type DatabaseProperties struct {
	CatalogID     interface{} `json:"CatalogId"` // required, string or Ref{}, need json tag to keep linter happy
	DatabaseInput DatabaseInput
}

type NewTableInput

type NewTableInput struct {
	CatalogID     interface{}
	DatabaseName  interface{}
	Name          interface{}
	Description   interface{}
	Location      interface{}
	Columns       []Column
	PartitionKeys []Column
}

inputs to table generation functions

type SerdeInfo

type SerdeInfo struct {
	SerializationLibrary string                 `json:",omitempty"`
	Parameters           map[string]interface{} `json:",omitempty"`
}

type StorageDescriptor

type StorageDescriptor struct {
	InputFormat            string
	OutputFormat           string
	Compressed             bool        `json:",omitempty"`
	Location               interface{} // required
	BucketColumns          []Column    `json:",omitempty"`
	SortColumns            []Column    `json:",omitempty"`
	StoredAsSubDirectories bool        `json:",omitempty"`
	SerdeInfo              SerdeInfo
	Columns                []Column
}

type Table

type Table struct {
	Type       string
	DependsOn  []string `json:",omitempty"`
	Properties TableProperties
}

func NewJSONLTable

func NewJSONLTable(input *NewTableInput) (db *Table)

func NewParquetTable

func NewParquetTable(input *NewTableInput) (db *Table)

type TableInput

type TableInput struct {
	TableType         string
	Name              interface{}
	Description       interface{} `json:",omitempty"`
	StorageDescriptor StorageDescriptor
	PartitionKeys     []Column `json:",omitempty"`
}

type TableProperties

type TableProperties struct {
	CatalogID    interface{} `json:"CatalogId"` // required,  string or Ref{}, need json tag to keep linter happy
	DatabaseName interface{}
	TableInput   TableInput
}