format

Documentation

Index

• Constants
• Variables
• func MakeCSVBytes(ctx context.Context, records []opencdc.Record, csvColumnOrder []string, ...) (err error)
• type AvroRecordSchema
• type ConnectorColumns
  • func GetDataSchema(ctx context.Context, records []opencdc.Record, schema map[string]string, ...) ([]string, *ConnectorColumns, error)
• type Type
  • func (i Type) String() string

Constants

const (
	// numeric types.
	SnowflakeInteger = "INTEGER"
	SnowflakeFloat   = "FLOAT"
	SnowflakeFixed   = "FIXED"
	SnowflakeReal    = "REAL"

	// string & binary types.
	SnowflakeText    = "TEXT"
	SnowflakeBinary  = "BINARY"
	SnowflakeVarchar = "VARCHAR"

	// logical data types.
	SnowflakeBoolean = "BOOLEAN"

	// date & time types.
	SnowflakeTimestampLTZ = "TIMESTAMP_LTZ"
	SnowflakeTimestampNTZ = "TIMESTAMP_NTZ"
	SnowflakeTimestampTZ  = "TIMESTAMP_TZ"
	SnowflakeTime         = "TIME"
	SnowflakeDate         = "DATE"

	// semi-structured data types.
	SnowflakeVariant = "VARIANT"
	SnowflakeObject  = "OBJECT"
	SnowflakeArray   = "ARRAY"

	// geospatial data types.
	SnowflakeGeography = "GEOGRAPHY"
	SnowflakeGeometry  = "GEOMETRY"

	// vector data types.
	SnowflakeVector = "VECTOR"
)

const (
	AvroBoolean         = "boolean"
	AvroInt             = "int"
	AvroLong            = "long"
	AvroFloat           = "float"
	AvroDouble          = "double"
	AvroBytes           = "bytes"
	AvroString          = "string"
	AvroDecimal         = "decimal"
	AvroUUID            = "uuid"
	AvroDate            = "date"
	AvroTimeMillis      = "time-millis"
	AvroTimeMicros      = "time-micros"
	AvroTimestampMillis = "timestamp-millis"
	AvroTimestampMicros = "timestamp-micros"
	AvroRecord          = "record"
	AvroArray           = "array"
	AvroMap             = "map"
)

Variables

var All = []Type{
	TypeCSV,
}

var AvroToSnowflakeType = map[avro.Type]string{
	avro.Boolean: SnowflakeBoolean,
	avro.Int:     SnowflakeInteger,
	avro.Long:    SnowflakeInteger,
	avro.Float:   SnowflakeFloat,
	avro.Double:  SnowflakeFloat,
	avro.Bytes:   SnowflakeVarchar,
	avro.String:  SnowflakeVarchar,
	avro.Record:  SnowflakeObject,
	avro.Array:   SnowflakeArray,
	avro.Map:     SnowflakeObject,
}

AvroToSnowflakeType Map from Avro Types to Snowflake Types.

var SnowflakeTypeMapping = map[string]string{
	SnowflakeFixed:        SnowflakeInteger,
	SnowflakeReal:         SnowflakeFloat,
	SnowflakeText:         SnowflakeVarchar,
	SnowflakeBinary:       SnowflakeBinary,
	SnowflakeBoolean:      SnowflakeBoolean,
	SnowflakeDate:         SnowflakeDate,
	SnowflakeTimestampNTZ: SnowflakeTimestampNTZ,
	SnowflakeTimestampLTZ: SnowflakeTimestampLTZ,
	SnowflakeTimestampTZ:  SnowflakeTimestampTZ,
	SnowflakeVariant:      SnowflakeVariant,
	SnowflakeObject:       SnowflakeObject,
	SnowflakeArray:        SnowflakeArray,
	SnowflakeVector:       SnowflakeVector,
}

SnowflakeTypeMapping Map between snowflake retrieved types and connector defined types.

Functions

func MakeCSVBytes

func MakeCSVBytes(
	ctx context.Context,
	records []opencdc.Record,
	csvColumnOrder []string,
	meroxaColumns ConnectorColumns,
	schema map[string]string,
	primaryKey string,
	insertsBuf *bytes.Buffer,
	updatesBuf *bytes.Buffer,
	_ int,
) (err error)

TODO: refactor this function, make it more modular and readable.

Types

type AvroRecordSchema

type AvroRecordSchema struct {
	Name   string `json:"name"`
	Type   string `json:"type"`
	Fields []struct {
		Name string `json:"name"`
		Type string `json:"type"`
	} `json:"fields"`
}

type ConnectorColumns

type ConnectorColumns struct {
	OperationColumn string
	CreatedAtColumn string
	UpdatedAtColumn string
	DeletedAtColumn string
}

func GetDataSchema

func GetDataSchema(
	ctx context.Context,
	records []opencdc.Record,
	schema map[string]string,
	prefix string,
) ([]string, *ConnectorColumns, error)

type Type

type Type int

const (
	TypeCSV Type = iota
)

func (Type) String

func (i Type) String() string