encoder

package
v0.2.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 15, 2024 License: Apache-2.0 Imports: 15 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type CSVEncoderConfig added in v0.2.2 

type CSVEncoderConfig struct {
	Columns        []string `json:"columns"`
	HasHeader      bool     `json:"has_header"`
	Separator      string   `json:"separator"`
	FileEncoding   string   `json:"file_encoding"`
	ValidateFields bool     `json:"validate_fields"`
	IgnoreColumns  []string `json:"ignore_columns"`
}

func NewCSVEncoderConfig added in v0.2.2 

func NewCSVEncoderConfig(sourceConfig map[string]any) (*CSVEncoderConfig, error)

type CSVItem 

type CSVItem struct {
	// contains filtered or unexported fields
}

func (*CSVItem) GetPropertyNames 

func (item *CSVItem) GetPropertyNames() []string

func (*CSVItem) GetValue 

func (item *CSVItem) GetValue(key string) any

func (*CSVItem) NativeItem 

func (item *CSVItem) NativeItem() any

func (*CSVItem) SetValue 

func (item *CSVItem) SetValue(key string, value any)

type CSVItemFactory 

type CSVItemFactory struct{}

func (*CSVItemFactory) NewItem 

func (c *CSVItemFactory) NewItem() cdl.Item

type CSVItemIterator 

type CSVItemIterator struct {
	// contains filtered or unexported fields
}

func NewCSVItemIterator 

func NewCSVItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (*CSVItemIterator, error)

func (*CSVItemIterator) Close 

func (c *CSVItemIterator) Close() error

func (*CSVItemIterator) Read 

func (c *CSVItemIterator) Read() (cdl.Item, error)

type CSVItemWriter 

type CSVItemWriter struct {
	// contains filtered or unexported fields
}

func NewCSVItemWriter 

func NewCSVItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*CSVItemWriter, error)

func (*CSVItemWriter) Close 

func (c *CSVItemWriter) Close() error

func (*CSVItemWriter) Write 

func (c *CSVItemWriter) Write(item cdl.Item) error

type FlatFileConfig added in v0.2.3 

type FlatFileConfig struct {
	Fields []FlatFileField `json:"fields"`
}

func NewFlatFileReadConfig added in v0.2.3 

func NewFlatFileReadConfig(sourceConfig map[string]any) (*FlatFileConfig, error)

func NewFlatFileWriteConfig added in v0.2.3 

func NewFlatFileWriteConfig(sourceConfig map[string]any) (*FlatFileConfig, error)

type FlatFileField added in v0.2.3 

type FlatFileField struct {
	Name      string `json:"name"`
	Length    int    `json:"length"`
	Ignore    bool   `json:"ignore"`
	NumberPad bool   `json:"number_pad"`
}

use position of field in the list to order fieldOrder

type FlatFileItem added in v0.2.3 

type FlatFileItem struct {
	// contains filtered or unexported fields
}

func (*FlatFileItem) GetPropertyNames added in v0.2.3 

func (item *FlatFileItem) GetPropertyNames() []string

func (*FlatFileItem) GetValue added in v0.2.3 

func (item *FlatFileItem) GetValue(key string) any

func (*FlatFileItem) NativeItem added in v0.2.3 

func (item *FlatFileItem) NativeItem() any

func (*FlatFileItem) SetValue added in v0.2.3 

func (item *FlatFileItem) SetValue(key string, value any)

type FlatFileItemFactory added in v0.2.3 

type FlatFileItemFactory struct{}

func (*FlatFileItemFactory) NewItem added in v0.2.3 

func (c *FlatFileItemFactory) NewItem() common_datalayer.Item

type FlatFileItemIterator added in v0.2.3 

type FlatFileItemIterator struct {
	// contains filtered or unexported fields
}

func NewFlatFileItemIterator added in v0.2.3 

func NewFlatFileItemIterator(sourceConfig map[string]any, data io.ReadCloser) (*FlatFileItemIterator, error)

func (*FlatFileItemIterator) Close added in v0.2.3 

func (c *FlatFileItemIterator) Close() error

func (*FlatFileItemIterator) Read added in v0.2.3 

func (c *FlatFileItemIterator) Read() (common_datalayer.Item, error)

type FlatFileItemWriter added in v0.2.3 

type FlatFileItemWriter struct {
	// contains filtered or unexported fields
}

func NewFlatFileItemWriter added in v0.2.3 

func NewFlatFileItemWriter(sourceConfig map[string]any, data io.WriteCloser, batchInfo *common_datalayer.BatchInfo) (*FlatFileItemWriter, error)

func (*FlatFileItemWriter) Close added in v0.2.3 

func (c *FlatFileItemWriter) Close() error

func (*FlatFileItemWriter) Write added in v0.2.3 

func (c *FlatFileItemWriter) Write(item common_datalayer.Item) error

type ItemFactory 

type ItemFactory interface {
	NewItem() cdl.Item
}

func NewCSVItemFactory 

func NewCSVItemFactory() ItemFactory

func NewFlatFileItemFactory added in v0.2.3 

func NewFlatFileItemFactory() ItemFactory

func NewItemFactory 

func NewItemFactory(sourceConfig map[string]any) (ItemFactory, error)

func NewJsonItemFactory 

func NewJsonItemFactory() ItemFactory

func NewParquetItemFactory added in v0.2.3 

func NewParquetItemFactory() ItemFactory

type ItemIterator 

type ItemIterator interface {
	Read() (cdl.Item, error)
	Close() error
}

func NewItemIterator 

func NewItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (ItemIterator, error)

type ItemWriter 

type ItemWriter interface {
	Write(item cdl.Item) error
	Close() error
}

func NewItemWriter 

func NewItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (ItemWriter, error)

type JsonItem 

type JsonItem struct {
	// contains filtered or unexported fields
}

func (*JsonItem) GetPropertyNames 

func (item *JsonItem) GetPropertyNames() []string

func (*JsonItem) GetValue 

func (item *JsonItem) GetValue(key string) any

func (*JsonItem) NativeItem 

func (item *JsonItem) NativeItem() any

func (*JsonItem) SetValue 

func (item *JsonItem) SetValue(key string, value any)

type JsonItemFactory 

type JsonItemFactory struct{}

func (*JsonItemFactory) NewItem 

func (j *JsonItemFactory) NewItem() cdl.Item

type JsonItemIterator 

type JsonItemIterator struct {
	// contains filtered or unexported fields
}

func NewJsonItemIterator 

func NewJsonItemIterator(sourceConfig map[string]any, logger cdl.Logger, data io.ReadCloser) (*JsonItemIterator, error)

func (*JsonItemIterator) Close 

func (j *JsonItemIterator) Close() error

func (*JsonItemIterator) Read 

func (j *JsonItemIterator) Read() (cdl.Item, error)

type JsonItemWriter 

type JsonItemWriter struct {
	// contains filtered or unexported fields
}

func NewJsonItemWriter 

func NewJsonItemWriter(sourceConfig map[string]any, logger cdl.Logger, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*JsonItemWriter, error)

func (*JsonItemWriter) Close 

func (j *JsonItemWriter) Close() error

func (*JsonItemWriter) Write 

func (j *JsonItemWriter) Write(item cdl.Item) error

type ParquetEncoderConfig added in v0.2.3 

type ParquetEncoderConfig struct {
	SchemaDef      *parquetschema.SchemaDefinition `json:"schema"`
	IgnoreColumns  []string                        `json:"ignore_columns"`
	FlushThreshold int64                           `json:"flush_threshold"`
}

func NewParquetDecoderConfig added in v0.2.3 

func NewParquetDecoderConfig(sourceConfig map[string]any) (*ParquetEncoderConfig, error)

func NewParquetEncoderConfig added in v0.2.3 

func NewParquetEncoderConfig(sourceConfig map[string]any) (*ParquetEncoderConfig, error)

type ParquetItem added in v0.2.3 

type ParquetItem struct {
	// contains filtered or unexported fields
}

func (*ParquetItem) GetPropertyNames added in v0.2.3 

func (item *ParquetItem) GetPropertyNames() []string

func (*ParquetItem) GetValue added in v0.2.3 

func (item *ParquetItem) GetValue(key string) any

func (*ParquetItem) NativeItem added in v0.2.3 

func (item *ParquetItem) NativeItem() any

func (*ParquetItem) SetValue added in v0.2.3 

func (item *ParquetItem) SetValue(key string, value any)

type ParquetItemFactory added in v0.2.3 

type ParquetItemFactory struct{}

func (*ParquetItemFactory) NewItem added in v0.2.3 

func (c *ParquetItemFactory) NewItem() cdl.Item

type ParquetItemIterator added in v0.2.3 

type ParquetItemIterator struct {
	// contains filtered or unexported fields
}

func NewParquetItemIterator added in v0.2.3 

func NewParquetItemIterator(sourceConfig map[string]any, data io.ReadCloser) (*ParquetItemIterator, error)

func (*ParquetItemIterator) Close added in v0.2.3 

func (c *ParquetItemIterator) Close() error

func (*ParquetItemIterator) Read added in v0.2.3 

func (c *ParquetItemIterator) Read() (cdl.Item, error)

type ParquetItemWriter added in v0.2.3 

type ParquetItemWriter struct {
	// contains filtered or unexported fields
}

func NewParquetItemWriter added in v0.2.3 

func NewParquetItemWriter(sourceConfig map[string]any, data io.WriteCloser, batchInfo *cdl.BatchInfo) (*ParquetItemWriter, error)

func (*ParquetItemWriter) Close added in v0.2.3 

func (c *ParquetItemWriter) Close() error

func (*ParquetItemWriter) Write added in v0.2.3 

func (c *ParquetItemWriter) Write(item cdl.Item) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.