serializer

README

Serializer

This package defines serializers interfaces, that serializes ChangeItems to []bytes

Supported Formats:

• json
• tsv
• raw

#TODO

• interface for byte stream serialization
• add new serialization types

Documentation

Index

• Constants
• func BuildParquetSchema(tableSchema abstract.FastTableSchema) (*parquet.Schema, error)
• func NewCsvBatchSerializer(config *CsvBatchSerializerConfig) *csvBatchSerializer
• func NewCsvSerializer() *csvSerializer
• func NewCsvStreamSerializer(ostream io.Writer) *csvStreamSerializer
• func NewJSONBatchSerializer(config *JSONBatchSerializerConfig) *jsonBatchSerializer
• func NewJSONSerializer(conf *JSONSerializerConfig) *jsonSerializer
• func NewJSONStreamSerializer(ostream io.Writer, conf *JSONSerializerConfig) *jsonStreamSerializer
• func NewParquetBatchSerializer() *parquetBatchSerializer
• func NewParquetStreamSerializer(ostream io.Writer, schema *parquet.Schema, ...) (*parquetStreamSerializer, error)
• func NewRawBatchSerializer(config *RawBatchSerializerConfig) *rawBatchSerializer
• func NewRawSerializer(conf *RawSerializerConfig) *rawSerializer
• func NewRawStreamSerializer(ostream io.Writer, conf *RawSerializerConfig) *rawStreamSerializer
• type BatchSerializer
• type BatchSerializerConfig
• type CsvBatchSerializerConfig
• type JSONBatchSerializerConfig
• type JSONSerializerConfig
• type RawBatchSerializerConfig
• type RawSerializerConfig
• type Serializer
• type StreamSerializer

Constants

const (
	DefaultBatchSerializerThreshold = 25000
)

Functions

func BuildParquetSchema

func BuildParquetSchema(tableSchema abstract.FastTableSchema) (*parquet.Schema, error)

func NewCsvBatchSerializer

func NewCsvBatchSerializer(config *CsvBatchSerializerConfig) *csvBatchSerializer

func NewCsvSerializer

func NewCsvSerializer() *csvSerializer

func NewCsvStreamSerializer

func NewCsvStreamSerializer(ostream io.Writer) *csvStreamSerializer

func NewJSONBatchSerializer

func NewJSONBatchSerializer(config *JSONBatchSerializerConfig) *jsonBatchSerializer

func NewJSONSerializer

func NewJSONSerializer(conf *JSONSerializerConfig) *jsonSerializer

func NewJSONStreamSerializer

func NewJSONStreamSerializer(ostream io.Writer, conf *JSONSerializerConfig) *jsonStreamSerializer

func NewParquetBatchSerializer

func NewParquetBatchSerializer() *parquetBatchSerializer

func NewParquetStreamSerializer

func NewParquetStreamSerializer(ostream io.Writer, schema *parquet.Schema, tableSchema abstract.FastTableSchema) (*parquetStreamSerializer, error)

func NewRawBatchSerializer

func NewRawBatchSerializer(config *RawBatchSerializerConfig) *rawBatchSerializer

func NewRawSerializer

func NewRawSerializer(conf *RawSerializerConfig) *rawSerializer

func NewRawStreamSerializer

func NewRawStreamSerializer(ostream io.Writer, conf *RawSerializerConfig) *rawStreamSerializer

Types

type BatchSerializer

type BatchSerializer interface {
	Serialize(items []*abstract.ChangeItem) ([]byte, error)
}

type BatchSerializerConfig

type BatchSerializerConfig struct {
	Concurrency int
	Threshold   int

	DisableConcurrency bool
}

type CsvBatchSerializerConfig

type CsvBatchSerializerConfig struct {
	BatchConfig *BatchSerializerConfig
}

type JSONBatchSerializerConfig

type JSONBatchSerializerConfig struct {
	SerializerConfig *JSONSerializerConfig
	BatchConfig      *BatchSerializerConfig
}

type JSONSerializerConfig

type JSONSerializerConfig struct {
	UnsupportedItemKinds map[abstract.Kind]bool
	AddClosingNewLine    bool
	AnyAsString          bool
}

type RawBatchSerializerConfig

type RawBatchSerializerConfig struct {
	SerializerConfig *RawSerializerConfig
	BatchConfig      *BatchSerializerConfig
}

type RawSerializerConfig

type RawSerializerConfig struct {
	AddClosingNewLine bool
}

type Serializer

type Serializer interface {
	Serialize(item *abstract.ChangeItem) ([]byte, error)
}

type StreamSerializer

type StreamSerializer interface {
	Serialize(items []*abstract.ChangeItem) error
	Close() error
}