deltalake

Documentation

Index

• func WithConfig(h *Deltalake, config *config.Config)
• type Deltalake
  • func New() *Deltalake
  • func (d *Deltalake) AddColumns(ctx context.Context, tableName string, columnsInfo []warehouseutils.ColumnInfo) error
  • func (*Deltalake) AlterColumn(context.Context, string, string, string) (model.AlterTableResponse, error)
  • func (d *Deltalake) Cleanup(ctx context.Context)
  • func (d *Deltalake) Connect(_ context.Context, warehouse model.Warehouse) (warehouseclient.Client, error)
  • func (d *Deltalake) CrashRecover(ctx context.Context)
  • func (d *Deltalake) CreateSchema(ctx context.Context) error
  • func (d *Deltalake) CreateTable(ctx context.Context, tableName string, columns model.TableSchema) error
  • func (*Deltalake) DeleteBy(context.Context, []string, warehouseutils.DeleteByParams) error
  • func (*Deltalake) DownloadIdentityRules(context.Context, *misc.GZipWriter) error
  • func (d *Deltalake) DropTable(ctx context.Context, tableName string) error
  • func (*Deltalake) ErrorMappings() []model.JobError
  • func (d *Deltalake) FetchSchema(ctx context.Context) (model.Schema, model.Schema, error)
  • func (d *Deltalake) GetTotalCountInTable(ctx context.Context, tableName string) (int64, error)
  • func (*Deltalake) IsEmpty(context.Context, model.Warehouse) (bool, error)
  • func (*Deltalake) LoadIdentityMappingsTable(context.Context) error
  • func (*Deltalake) LoadIdentityMergeRulesTable(context.Context) error
  • func (d *Deltalake) LoadTable(ctx context.Context, tableName string) error
  • func (d *Deltalake) LoadTestTable(ctx context.Context, location, tableName string, _ map[string]interface{}, ...) error
  • func (d *Deltalake) LoadUserTables(ctx context.Context) map[string]error
  • func (d *Deltalake) SetConnectionTimeout(timeout time.Duration)
  • func (d *Deltalake) Setup(_ context.Context, warehouse model.Warehouse, uploader warehouseutils.Uploader) error
  • func (d *Deltalake) TestConnection(ctx context.Context, _ model.Warehouse) error

Functions

func WithConfig

func WithConfig(h *Deltalake, config *config.Config)

Types

type Deltalake

type Deltalake struct {
	DB                     *sqlmiddleware.DB
	Namespace              string
	ObjectStorage          string
	Warehouse              model.Warehouse
	Uploader               warehouseutils.Uploader
	ConnectTimeout         time.Duration
	Logger                 logger.Logger
	Stats                  stats.Stats
	LoadTableStrategy      string
	EnablePartitionPruning bool
	SlowQueryThreshold     time.Duration
	MaxRetries             int
	RetryMinWait           time.Duration
	RetryMaxWait           time.Duration
}

func New

func New() *Deltalake

func (*Deltalake) AddColumns

func (d *Deltalake) AddColumns(ctx context.Context, tableName string, columnsInfo []warehouseutils.ColumnInfo) error

AddColumns adds columns to the table.

func (*Deltalake) AlterColumn

func (*Deltalake) AlterColumn(context.Context, string, string, string) (model.AlterTableResponse, error)

AlterColumn alters a column in the warehouse

func (*Deltalake) Cleanup

func (d *Deltalake) Cleanup(ctx context.Context)

Cleanup cleans up the warehouse

func (*Deltalake) Connect

func (d *Deltalake) Connect(_ context.Context, warehouse model.Warehouse) (warehouseclient.Client, error)

Connect returns Client

func (*Deltalake) CrashRecover

func (d *Deltalake) CrashRecover(ctx context.Context)

CrashRecover crash recover scenarios

func (*Deltalake) CreateSchema

func (d *Deltalake) CreateSchema(ctx context.Context) error

CreateSchema creates a schema in the warehouse if it does not exist.

func (*Deltalake) CreateTable

func (d *Deltalake) CreateTable(ctx context.Context, tableName string, columns model.TableSchema) error

CreateTable creates a table in the warehouse.

func (*Deltalake) DeleteBy

func (*Deltalake) DeleteBy(context.Context, []string, warehouseutils.DeleteByParams) error

func (*Deltalake) DownloadIdentityRules

func (*Deltalake) DownloadIdentityRules(context.Context, *misc.GZipWriter) error

DownloadIdentityRules downloadchecking if schema exists identity rules

func (*Deltalake) DropTable

func (d *Deltalake) DropTable(ctx context.Context, tableName string) error

DropTable drops a table in the warehouse

func (*Deltalake) ErrorMappings

func (*Deltalake) ErrorMappings() []model.JobError

ErrorMappings returns the error mappings

func (*Deltalake) FetchSchema

func (d *Deltalake) FetchSchema(ctx context.Context) (model.Schema, model.Schema, error)

FetchSchema fetches the schema from the warehouse

func (*Deltalake) GetTotalCountInTable

func (d *Deltalake) GetTotalCountInTable(ctx context.Context, tableName string) (int64, error)

GetTotalCountInTable returns the total count in the table

func (*Deltalake) IsEmpty

func (*Deltalake) IsEmpty(context.Context, model.Warehouse) (bool, error)

IsEmpty checks if the warehouse is empty or not

func (*Deltalake) LoadIdentityMappingsTable

func (*Deltalake) LoadIdentityMappingsTable(context.Context) error

LoadIdentityMappingsTable loads identifies mappings table

func (*Deltalake) LoadIdentityMergeRulesTable

func (*Deltalake) LoadIdentityMergeRulesTable(context.Context) error

LoadIdentityMergeRulesTable loads identifies merge rules tables

func (*Deltalake) LoadTable

func (d *Deltalake) LoadTable(ctx context.Context, tableName string) error

LoadTable loads table for table name

func (*Deltalake) LoadTestTable

func (d *Deltalake) LoadTestTable(ctx context.Context, location, tableName string, _ map[string]interface{}, format string) error

LoadTestTable loads the test table

func (*Deltalake) LoadUserTables

func (d *Deltalake) LoadUserTables(ctx context.Context) map[string]error

LoadUserTables loads user tables

func (*Deltalake) SetConnectionTimeout

func (d *Deltalake) SetConnectionTimeout(timeout time.Duration)

SetConnectionTimeout sets the connection timeout

func (*Deltalake) Setup

func (d *Deltalake) Setup(_ context.Context, warehouse model.Warehouse, uploader warehouseutils.Uploader) error

Setup sets up the warehouse

func (*Deltalake) TestConnection

func (d *Deltalake) TestConnection(ctx context.Context, _ model.Warehouse) error

TestConnection tests the connection to the warehouse