schemarepository

Documentation

Index

• Constants
• Variables
• func LoadFileBatching(files []warehouseutils.LoadFileT, batchSize int) [][]warehouseutils.LoadFileT
• func UseGlue(w *warehouseutils.Warehouse) bool
• type GlueSchemaRepository
  • func NewGlueSchemaRepository(wh warehouseutils.Warehouse) (*GlueSchemaRepository, error)
  • func (gl *GlueSchemaRepository) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)
  • func (gl *GlueSchemaRepository) AlterColumn(tableName, columnName, columnType string) (model.AlterTableResponse, error)
  • func (gl *GlueSchemaRepository) CreateSchema() (err error)
  • func (gl *GlueSchemaRepository) CreateTable(tableName string, columnMap map[string]string) (err error)
  • func (gl *GlueSchemaRepository) FetchSchema(warehouse warehouseutils.Warehouse) (warehouseutils.SchemaT, warehouseutils.SchemaT, error)
  • func (gl *GlueSchemaRepository) RefreshPartitions(tableName string, loadFiles []warehouseutils.LoadFileT) error
• type LocalSchemaRepository
  • func NewLocalSchemaRepository(wh warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (*LocalSchemaRepository, error)
  • func (ls *LocalSchemaRepository) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)
  • func (ls *LocalSchemaRepository) AlterColumn(tableName, columnName, columnType string) (model.AlterTableResponse, error)
  • func (*LocalSchemaRepository) CreateSchema() (err error)
  • func (ls *LocalSchemaRepository) CreateTable(tableName string, columnMap map[string]string) (err error)
  • func (ls *LocalSchemaRepository) FetchSchema(_ warehouseutils.Warehouse) (warehouseutils.SchemaT, warehouseutils.SchemaT, error)
  • func (*LocalSchemaRepository) RefreshPartitions(_ string, _ []warehouseutils.LoadFileT) error
• type SchemaRepository
  • func NewSchemaRepository(wh warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (SchemaRepository, error)

Constants

const MAX_CHARACTER_LIMIT = 65535

Variables

var (
	PartitionFolderRegex = regexp.MustCompile(`.*/(?P<name>.*)=(?P<value>.*)$`)
	PartitionWindowRegex = regexp.MustCompile(`^(?P<name>.*)=(?P<value>.*)$`)
)

var UseGlueConfig = "useGlue"

var (
	VARCHAR_TYPE = fmt.Sprintf("varchar(%d)", MAX_CHARACTER_LIMIT)
)

Functions

func LoadFileBatching

func LoadFileBatching(files []warehouseutils.LoadFileT, batchSize int) [][]warehouseutils.LoadFileT

LoadFileBatching batches load files for refresh partitions

func UseGlue

func UseGlue(w *warehouseutils.Warehouse) bool

Types

type GlueSchemaRepository

type GlueSchemaRepository struct {
	GlueClient *glue.Glue

	Warehouse warehouseutils.Warehouse
	Namespace string
	Logger    logger.Logger
	// contains filtered or unexported fields
}

func NewGlueSchemaRepository

func NewGlueSchemaRepository(wh warehouseutils.Warehouse) (*GlueSchemaRepository, error)

func (*GlueSchemaRepository) AddColumns

func (gl *GlueSchemaRepository) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)

func (*GlueSchemaRepository) AlterColumn

func (gl *GlueSchemaRepository) AlterColumn(tableName, columnName, columnType string) (model.AlterTableResponse, error)

func (*GlueSchemaRepository) CreateSchema

func (gl *GlueSchemaRepository) CreateSchema() (err error)

func (*GlueSchemaRepository) CreateTable

func (gl *GlueSchemaRepository) CreateTable(tableName string, columnMap map[string]string) (err error)

func (*GlueSchemaRepository) FetchSchema

func (gl *GlueSchemaRepository) FetchSchema(warehouse warehouseutils.Warehouse) (warehouseutils.SchemaT, warehouseutils.SchemaT, error)

func (*GlueSchemaRepository) RefreshPartitions

func (gl *GlueSchemaRepository) RefreshPartitions(tableName string, loadFiles []warehouseutils.LoadFileT) error

RefreshPartitions takes a tableName and a list of loadFiles and refreshes all the partitions that are modified by the path in those loadFiles. It returns any error reported by Glue

type LocalSchemaRepository

type LocalSchemaRepository struct {
	// contains filtered or unexported fields
}

func NewLocalSchemaRepository

func NewLocalSchemaRepository(wh warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (*LocalSchemaRepository, error)

func (*LocalSchemaRepository) AddColumns

func (ls *LocalSchemaRepository) AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)

func (*LocalSchemaRepository) AlterColumn

func (ls *LocalSchemaRepository) AlterColumn(tableName, columnName, columnType string) (model.AlterTableResponse, error)

func (*LocalSchemaRepository) CreateSchema

func (*LocalSchemaRepository) CreateSchema() (err error)

func (*LocalSchemaRepository) CreateTable

func (ls *LocalSchemaRepository) CreateTable(tableName string, columnMap map[string]string) (err error)

func (*LocalSchemaRepository) FetchSchema

func (ls *LocalSchemaRepository) FetchSchema(_ warehouseutils.Warehouse) (warehouseutils.SchemaT, warehouseutils.SchemaT, error)

func (*LocalSchemaRepository) RefreshPartitions

func (*LocalSchemaRepository) RefreshPartitions(_ string, _ []warehouseutils.LoadFileT) error

type SchemaRepository

type SchemaRepository interface {
	FetchSchema(warehouse warehouseutils.Warehouse) (warehouseutils.SchemaT, warehouseutils.SchemaT, error)
	CreateSchema() (err error)
	CreateTable(tableName string, columnMap map[string]string) (err error)
	AddColumns(tableName string, columnsInfo []warehouseutils.ColumnInfo) (err error)
	AlterColumn(tableName, columnName, columnType string) (model.AlterTableResponse, error)
	RefreshPartitions(tableName string, loadFiles []warehouseutils.LoadFileT) error
}

func NewSchemaRepository

func NewSchemaRepository(wh warehouseutils.Warehouse, uploader warehouseutils.UploaderI) (SchemaRepository, error)