encoder

package
v0.2.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 30, 2024 License: Apache-2.0 Imports: 15 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CSVConcatenatingWriter added in v0.2.4 

type CSVConcatenatingWriter struct {
	HasHeader bool
	// contains filtered or unexported fields
}

CSV Concatenator

func NewCSVConcatenatingWriter added in v0.2.4 

func NewCSVConcatenatingWriter(output io.WriteCloser, hasHeader bool) *CSVConcatenatingWriter

NewCSVConcatenatingWriter creates a new CSVConcatenatingWriter.

func (*CSVConcatenatingWriter) Close added in v0.2.4 

func (m *CSVConcatenatingWriter) Close() error

Close completes the CSV writing process.

func (*CSVConcatenatingWriter) Write added in v0.2.4 

func (m *CSVConcatenatingWriter) Write(reader io.ReadCloser) (err error)

Write writes a part of a CSV file to the target output.

type CSVEncoderConfig added in v0.2.2 

type CSVEncoderConfig struct {
	Columns        []string `json:"columns"`
	HasHeader      bool     `json:"has_header"`
	Separator      string   `json:"separator"`
	FileEncoding   string   `json:"file_encoding"`
	ValidateFields bool     `json:"validate_fields"`
	IgnoreColumns  []string `json:"ignore_columns"`
}

func NewCSVEncoderConfig added in v0.2.2 

func NewCSVEncoderConfig(sourceConfig map[string]any) (*CSVEncoderConfig, error)

type CSVItem 

type CSVItem struct {
	// contains filtered or unexported fields
}

func (*CSVItem) GetPropertyNames 

func (item *CSVItem) GetPropertyNames() []string

func (*CSVItem) GetValue 

func (item *CSVItem) GetValue(key string) any

func (*CSVItem) NativeItem 

func (item *CSVItem) NativeItem() any

func (*CSVItem) SetValue 

func (item *CSVItem) SetValue(key string, value any)

type CSVItemFactory 

type CSVItemFactory struct{}

func (*CSVItemFactory) NewItem 

func (c *CSVItemFactory) NewItem() cdl.Item

type CSVItemIterator 

type CSVItemIterator struct {
	// contains filtered or unexported fields
}

func NewCSVItemIterator 

func NewCSVItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (*CSVItemIterator, error)

func (*CSVItemIterator) Close 

func (c *CSVItemIterator) Close() error

func (*CSVItemIterator) Read 

func (c *CSVItemIterator) Read() (cdl.Item, error)

type CSVItemWriter 

type CSVItemWriter struct {
	// contains filtered or unexported fields
}

func NewCSVItemWriter 

func NewCSVItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*CSVItemWriter, error)

func (*CSVItemWriter) Close 

func (c *CSVItemWriter) Close() error

func (*CSVItemWriter) Write 

func (c *CSVItemWriter) Write(item cdl.Item) error

type ConcatenatingWriter added in v0.2.4 

type ConcatenatingWriter interface {
	Write(reader io.ReadCloser) error
	Close() error
}

ConcatenatingWriter is an interface that defines the functions for concatenating data from many files into one file It ensures that the header / metadata is only written once and then appends all content from the files in accordance with the specific format, e.g. CSV, JSON, Parquet etc. Users of this interface should call Write for each file to be concatenated and Close when all files have been submitted for concatenated.

func NewConcatenatingWriter added in v0.2.5 

func NewConcatenatingWriter(sourceConfig map[string]any, output io.WriteCloser) (ConcatenatingWriter, error)

type FlatFileConfig added in v0.2.3 

type FlatFileConfig struct {
	Fields []FlatFileField `json:"fields"`
}

func NewFlatFileReadConfig added in v0.2.3 

func NewFlatFileReadConfig(sourceConfig map[string]any) (*FlatFileConfig, error)

func NewFlatFileWriteConfig added in v0.2.3 

func NewFlatFileWriteConfig(sourceConfig map[string]any) (*FlatFileConfig, error)

type FlatFileField added in v0.2.3 

type FlatFileField struct {
	Name      string `json:"name"`
	Length    int    `json:"length"`
	Ignore    bool   `json:"ignore"`
	NumberPad bool   `json:"number_pad"`
}

use position of field in the list to order fieldOrder

type FlatFileItem added in v0.2.3 

type FlatFileItem struct {
	// contains filtered or unexported fields
}

func (*FlatFileItem) GetPropertyNames added in v0.2.3 

func (item *FlatFileItem) GetPropertyNames() []string

func (*FlatFileItem) GetValue added in v0.2.3 

func (item *FlatFileItem) GetValue(key string) any

func (*FlatFileItem) NativeItem added in v0.2.3 

func (item *FlatFileItem) NativeItem() any

func (*FlatFileItem) SetValue added in v0.2.3 

func (item *FlatFileItem) SetValue(key string, value any)

type FlatFileItemFactory added in v0.2.3 

type FlatFileItemFactory struct{}

func (*FlatFileItemFactory) NewItem added in v0.2.3 

func (c *FlatFileItemFactory) NewItem() common_datalayer.Item

type FlatFileItemIterator added in v0.2.3 

type FlatFileItemIterator struct {
	// contains filtered or unexported fields
}

func NewFlatFileItemIterator added in v0.2.3 

func NewFlatFileItemIterator(sourceConfig map[string]any, data io.ReadCloser) (*FlatFileItemIterator, error)

func (*FlatFileItemIterator) Close added in v0.2.3 

func (c *FlatFileItemIterator) Close() error

func (*FlatFileItemIterator) Read added in v0.2.3 

func (c *FlatFileItemIterator) Read() (common_datalayer.Item, error)

type FlatFileItemWriter added in v0.2.3 

type FlatFileItemWriter struct {
	// contains filtered or unexported fields
}

func NewFlatFileItemWriter added in v0.2.3 

func NewFlatFileItemWriter(sourceConfig map[string]any, data io.WriteCloser, batchInfo *common_datalayer.BatchInfo) (*FlatFileItemWriter, error)

func (*FlatFileItemWriter) Close added in v0.2.3 

func (c *FlatFileItemWriter) Close() error

func (*FlatFileItemWriter) Write added in v0.2.3 

func (c *FlatFileItemWriter) Write(item common_datalayer.Item) error

type GenericConcatenatingWriter added in v0.2.4 

type GenericConcatenatingWriter struct {
	// contains filtered or unexported fields
}

GenericConcatenatingWriter implements the ConcatenatingWriter interface for simple concatenation. This can be used when all parts are headerless and can be concatenated without any special handling.

func NewGenericConcatenatingWriter added in v0.2.4 

func NewGenericConcatenatingWriter(output io.WriteCloser) *GenericConcatenatingWriter

NewGenericConcatenatingWriter creates a new GenericConcatenatingWriter.

func (*GenericConcatenatingWriter) Close added in v0.2.4 

func (m *GenericConcatenatingWriter) Close() error

Close finalizes the writing process by flushing the buffer and closing the output.

func (*GenericConcatenatingWriter) Write added in v0.2.4 

func (m *GenericConcatenatingWriter) Write(reader io.ReadCloser) (err error)

Write writes a part of a file to the target output by simple concatenation.

type ItemFactory 

type ItemFactory interface {
	NewItem() cdl.Item
}

func NewCSVItemFactory 

func NewCSVItemFactory() ItemFactory

func NewFlatFileItemFactory added in v0.2.3 

func NewFlatFileItemFactory() ItemFactory

func NewItemFactory 

func NewItemFactory(sourceConfig map[string]any) (ItemFactory, error)

func NewJsonItemFactory 

func NewJsonItemFactory() ItemFactory

func NewParquetItemFactory added in v0.2.3 

func NewParquetItemFactory() ItemFactory

type ItemIterator 

type ItemIterator interface {
	Read() (cdl.Item, error)
	Close() error
}

func NewItemIterator 

func NewItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (ItemIterator, error)

type ItemWriter 

type ItemWriter interface {
	Write(item cdl.Item) error
	Close() error
}

func NewItemWriter 

func NewItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (ItemWriter, error)

type JSONConcatenatingWriter added in v0.2.4 

type JSONConcatenatingWriter struct {
	// contains filtered or unexported fields
}

JSONConcatenatingWriter implements the ConcatenatingWriter interface for JSON arrays.

func NewJSONConcatenatingWriter added in v0.2.4 

func NewJSONConcatenatingWriter(output io.WriteCloser) *JSONConcatenatingWriter

func (*JSONConcatenatingWriter) Close added in v0.2.4 

func (m *JSONConcatenatingWriter) Close() error

Close finalizes the JSON writing process.

func (*JSONConcatenatingWriter) Write added in v0.2.4 

func (m *JSONConcatenatingWriter) Write(reader io.ReadCloser) (err error)

Write writes a part of a JSON array to the target output.

type JsonItem 

type JsonItem struct {
	// contains filtered or unexported fields
}

func (*JsonItem) GetPropertyNames 

func (item *JsonItem) GetPropertyNames() []string

func (*JsonItem) GetValue 

func (item *JsonItem) GetValue(key string) any

func (*JsonItem) NativeItem 

func (item *JsonItem) NativeItem() any

func (*JsonItem) SetValue 

func (item *JsonItem) SetValue(key string, value any)

type JsonItemFactory 

type JsonItemFactory struct{}

func (*JsonItemFactory) NewItem 

func (j *JsonItemFactory) NewItem() cdl.Item

type JsonItemIterator 

type JsonItemIterator struct {
	// contains filtered or unexported fields
}

func NewJsonItemIterator 

func NewJsonItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (*JsonItemIterator, error)

func (*JsonItemIterator) Close 

func (j *JsonItemIterator) Close() error

func (*JsonItemIterator) Read 

func (j *JsonItemIterator) Read() (cdl.Item, error)

type JsonItemWriter 

type JsonItemWriter struct {
	// contains filtered or unexported fields
}

func NewJsonItemWriter 

func NewJsonItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*JsonItemWriter, error)

func (*JsonItemWriter) Close 

func (j *JsonItemWriter) Close() error

func (*JsonItemWriter) Write 

func (j *JsonItemWriter) Write(item cdl.Item) error

type ParquetConcatenatingWriter added in v0.2.4 

type ParquetConcatenatingWriter struct {
	// contains filtered or unexported fields
}

ParquetConcatenatingWriter implements the ConcatenatingWriter interface for Parquet files.

func NewParquetConcatenatingWriter added in v0.2.4 

func NewParquetConcatenatingWriter(output io.WriteCloser) *ParquetConcatenatingWriter

NewParquetConcatenatingWriter creates a new ParquetConcatenatingWriter.

func (*ParquetConcatenatingWriter) Close added in v0.2.4 

func (m *ParquetConcatenatingWriter) Close() error

Close finalizes the Parquet writing process.

func (*ParquetConcatenatingWriter) Write added in v0.2.4 

func (m *ParquetConcatenatingWriter) Write(reader io.ReadCloser) (err error)

Write writes a part of a Parquet file to the target output.

type ParquetEncoderConfig added in v0.2.3 

type ParquetEncoderConfig struct {
	SchemaDef      *parquetschema.SchemaDefinition `json:"schema"`
	IgnoreColumns  []string                        `json:"ignore_columns"`
	FlushThreshold int64                           `json:"flush_threshold"`
}

func NewParquetDecoderConfig added in v0.2.3 

func NewParquetDecoderConfig(sourceConfig map[string]any) (*ParquetEncoderConfig, error)

func NewParquetEncoderConfig added in v0.2.3 

func NewParquetEncoderConfig(sourceConfig map[string]any) (*ParquetEncoderConfig, error)

type ParquetItem added in v0.2.3 

type ParquetItem struct {
	// contains filtered or unexported fields
}

func (*ParquetItem) GetPropertyNames added in v0.2.3 

func (item *ParquetItem) GetPropertyNames() []string

func (*ParquetItem) GetValue added in v0.2.3 

func (item *ParquetItem) GetValue(key string) any

func (*ParquetItem) NativeItem added in v0.2.3 

func (item *ParquetItem) NativeItem() any

func (*ParquetItem) SetValue added in v0.2.3 

func (item *ParquetItem) SetValue(key string, value any)

type ParquetItemFactory added in v0.2.3 

type ParquetItemFactory struct{}

func (*ParquetItemFactory) NewItem added in v0.2.3 

func (c *ParquetItemFactory) NewItem() cdl.Item

type ParquetItemIterator added in v0.2.3 

type ParquetItemIterator struct {
	// contains filtered or unexported fields
}

func NewParquetItemIterator added in v0.2.3 

func NewParquetItemIterator(sourceConfig map[string]any, data io.ReadCloser) (*ParquetItemIterator, error)

func (*ParquetItemIterator) Close added in v0.2.3 

func (c *ParquetItemIterator) Close() error

func (*ParquetItemIterator) Read added in v0.2.3 

func (c *ParquetItemIterator) Read() (cdl.Item, error)

type ParquetItemWriter added in v0.2.3 

type ParquetItemWriter struct {
	// contains filtered or unexported fields
}

func NewParquetItemWriter added in v0.2.3 

func NewParquetItemWriter(sourceConfig map[string]any, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*ParquetItemWriter, error)

func (*ParquetItemWriter) Close added in v0.2.3 

func (c *ParquetItemWriter) Close() error

func (*ParquetItemWriter) Write added in v0.2.3 

func (c *ParquetItemWriter) Write(item cdl.Item) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.