encoding

Documentation

Index

• Constants
• func GetParquetValue(val interface{}, colType string) (retVal interface{}, err error)
• func Init()
• type CsvLoader
  • func NewCSVLoader(destType string, writer LoadFileWriter) *CsvLoader
  • func (loader *CsvLoader) AddColumn(_, _ string, val interface{})
  • func (loader *CsvLoader) AddEmptyColumn(columnName string)
  • func (loader *CsvLoader) AddRow(_, row []string)
  • func (*CsvLoader) GetLoadTimeFormat(string) string
  • func (loader *CsvLoader) IsLoadTimeColumn(columnName string) bool
  • func (loader *CsvLoader) Write() error
  • func (loader *CsvLoader) WriteToString() (string, error)
• type CsvReader
  • func NewCsvReader(r io.Reader) *CsvReader
  • func (csv *CsvReader) Read([]string) (record []string, err error)
• type EventLoader
  • func GetNewEventLoader(destinationType, loadFileType string, w LoadFileWriter) EventLoader
• type EventReader
  • func NewEventReader(r io.Reader, provider string) EventReader
• type JsonLoader
  • func NewJSONLoader(destType string, writer LoadFileWriter) *JsonLoader
  • func (loader *JsonLoader) AddColumn(columnName, _ string, val interface{})
  • func (loader *JsonLoader) AddEmptyColumn(columnName string)
  • func (loader *JsonLoader) AddRow(columnNames, row []string)
  • func (loader *JsonLoader) GetLoadTimeFormat(columnName string) string
  • func (loader *JsonLoader) IsLoadTimeColumn(columnName string) bool
  • func (loader *JsonLoader) Write() error
  • func (loader *JsonLoader) WriteToString() (string, error)
• type JsonReader
  • func NewJSONReader(r io.Reader) *JsonReader
  • func (js *JsonReader) Read(columnNames []string) ([]string, error)
• type LoadFileWriter
• type ParquetLoader
  • func NewParquetLoader(destType string, w LoadFileWriter) *ParquetLoader
  • func (loader *ParquetLoader) AddColumn(columnName, colType string, val interface{})
  • func (loader *ParquetLoader) AddEmptyColumn(columnName string)
  • func (*ParquetLoader) AddRow(_, _ []string)
  • func (*ParquetLoader) GetLoadTimeFormat(_ string) string
  • func (loader *ParquetLoader) IsLoadTimeColumn(columnName string) bool
  • func (loader *ParquetLoader) Write() error
  • func (*ParquetLoader) WriteToString() (string, error)
• type ParquetWriter
  • func CreateParquetWriter(schema model.TableSchema, outputFilePath, destType string) (*ParquetWriter, error)
  • func (p *ParquetWriter) Close() error
  • func (p *ParquetWriter) GetLoadFile() *os.File
  • func (*ParquetWriter) Write(_ []byte) (int, error)
  • func (*ParquetWriter) WriteGZ(_ string) error
  • func (p *ParquetWriter) WriteRow(row []interface{}) error

Constants

const (
	UUIDTsColumn     = "uuid_ts"
	LoadedAtColumn   = "loaded_at"
	BQLoadedAtFormat = "2006-01-02 15:04:05.999999 Z"
	BQUuidTSFormat   = "2006-01-02 15:04:05 Z"
)

const (
	ParquetInt64           = "type=INT64, repetitiontype=OPTIONAL"
	ParquetBoolean         = "type=BOOLEAN, repetitiontype=OPTIONAL"
	ParquetDouble          = "type=DOUBLE, repetitiontype=OPTIONAL"
	ParquetString          = "type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL"
	ParquetTimestampMicros = "type=INT64, convertedtype=TIMESTAMP_MICROS, repetitiontype=OPTIONAL"
)

Functions

func GetParquetValue

func GetParquetValue(val interface{}, colType string) (retVal interface{}, err error)

func Init

func Init()

Types

type CsvLoader

type CsvLoader struct {
	// contains filtered or unexported fields
}

CsvLoader is common for non-BQ warehouses. If you need any custom logic, either extend this or use destType and if/else/switch.

func NewCSVLoader

func NewCSVLoader(destType string, writer LoadFileWriter) *CsvLoader

func (*CsvLoader) AddColumn

func (loader *CsvLoader) AddColumn(_, _ string, val interface{})

func (*CsvLoader) AddEmptyColumn

func (loader *CsvLoader) AddEmptyColumn(columnName string)

func (*CsvLoader) AddRow

func (loader *CsvLoader) AddRow(_, row []string)

func (*CsvLoader) GetLoadTimeFormat

func (*CsvLoader) GetLoadTimeFormat(string) string

func (*CsvLoader) IsLoadTimeColumn

func (loader *CsvLoader) IsLoadTimeColumn(columnName string) bool

func (*CsvLoader) Write

func (loader *CsvLoader) Write() error

func (*CsvLoader) WriteToString

func (loader *CsvLoader) WriteToString() (string, error)

type CsvReader

type CsvReader struct {
	// contains filtered or unexported fields
}

func NewCsvReader

func NewCsvReader(r io.Reader) *CsvReader

func (*CsvReader) Read

func (csv *CsvReader) Read([]string) (record []string, err error)

type EventLoader

type EventLoader interface {
	IsLoadTimeColumn(columnName string) bool
	GetLoadTimeFormat(columnName string) string
	AddColumn(columnName, columnType string, val interface{})
	AddRow(columnNames, values []string)
	AddEmptyColumn(columnName string)
	WriteToString() (string, error)
	Write() error
}

func GetNewEventLoader

func GetNewEventLoader(destinationType, loadFileType string, w LoadFileWriter) EventLoader

type EventReader

type EventReader interface {
	Read(columnNames []string) (record []string, err error)
}

func NewEventReader

func NewEventReader(r io.Reader, provider string) EventReader

type JsonLoader

type JsonLoader struct {
	// contains filtered or unexported fields
}

func NewJSONLoader

func NewJSONLoader(destType string, writer LoadFileWriter) *JsonLoader

NewJSONLoader returns a new JsonLoader JsonLoader is only for BQ now. Treat this is as custom BQ loader. If more warehouses are added in the future, change this accordingly.

func (*JsonLoader) AddColumn

func (loader *JsonLoader) AddColumn(columnName, _ string, val interface{})

func (*JsonLoader) AddEmptyColumn

func (loader *JsonLoader) AddEmptyColumn(columnName string)

func (*JsonLoader) AddRow

func (loader *JsonLoader) AddRow(columnNames, row []string)

func (*JsonLoader) GetLoadTimeFormat

func (loader *JsonLoader) GetLoadTimeFormat(columnName string) string

func (*JsonLoader) IsLoadTimeColumn

func (loader *JsonLoader) IsLoadTimeColumn(columnName string) bool

func (*JsonLoader) Write

func (loader *JsonLoader) Write() error

func (*JsonLoader) WriteToString

func (loader *JsonLoader) WriteToString() (string, error)

type JsonReader

type JsonReader struct {
	// contains filtered or unexported fields
}

func NewJSONReader

func NewJSONReader(r io.Reader) *JsonReader

NewJSONReader returns a new JSON reader default scanner buffer maxCapacity is 64K set it to higher value to avoid read stop on read size error

func (*JsonReader) Read

func (js *JsonReader) Read(columnNames []string) ([]string, error)

type LoadFileWriter

type LoadFileWriter interface {
	WriteGZ(s string) error
	Write(p []byte) (int, error)
	WriteRow(r []interface{}) error
	Close() error
	GetLoadFile() *os.File
}

type ParquetLoader

type ParquetLoader struct {
	Schema     []string
	Values     []interface{}
	FileWriter LoadFileWriter
	// contains filtered or unexported fields
}

ParquetLoader is used for generating parquet load files.

func NewParquetLoader

func NewParquetLoader(destType string, w LoadFileWriter) *ParquetLoader

func (*ParquetLoader) AddColumn

func (loader *ParquetLoader) AddColumn(columnName, colType string, val interface{})

func (*ParquetLoader) AddEmptyColumn

func (loader *ParquetLoader) AddEmptyColumn(columnName string)

func (*ParquetLoader) AddRow

func (*ParquetLoader) AddRow(_, _ []string)

func (*ParquetLoader) GetLoadTimeFormat

func (*ParquetLoader) GetLoadTimeFormat(_ string) string

func (*ParquetLoader) IsLoadTimeColumn

func (loader *ParquetLoader) IsLoadTimeColumn(columnName string) bool

func (*ParquetLoader) Write

func (loader *ParquetLoader) Write() error

func (*ParquetLoader) WriteToString

func (*ParquetLoader) WriteToString() (string, error)

type ParquetWriter

type ParquetWriter struct {
	// contains filtered or unexported fields
}

func CreateParquetWriter

func CreateParquetWriter(schema model.TableSchema, outputFilePath, destType string) (*ParquetWriter, error)

func (*ParquetWriter) Close

func (p *ParquetWriter) Close() error

func (*ParquetWriter) GetLoadFile

func (p *ParquetWriter) GetLoadFile() *os.File

func (*ParquetWriter) Write

func (*ParquetWriter) Write(_ []byte) (int, error)

func (*ParquetWriter) WriteGZ

func (*ParquetWriter) WriteGZ(_ string) error

func (*ParquetWriter) WriteRow

func (p *ParquetWriter) WriteRow(row []interface{}) error