writer

Documentation

Index

• type CSVWriter
  • func NewCSVWriter(md []string, pfile source.ParquetFile, np int64) (*CSVWriter, error)
  • func (self *CSVWriter) WriteString(recsi interface{}) error
• type JSONWriter
  • func NewJSONWriter(jsonSchema string, pfile source.ParquetFile, np int64) (*JSONWriter, error)
• type ParquetWriter
  • func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error)
  • func (self *ParquetWriter) Flush(flag bool) error
  • func (self *ParquetWriter) RenameSchema()
  • func (self *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error
  • func (self *ParquetWriter) Write(src interface{}) error
  • func (self *ParquetWriter) WriteStop() error

Types

type CSVWriter

type CSVWriter struct {
	ParquetWriter
}

func NewCSVWriter

func NewCSVWriter(md []string, pfile source.ParquetFile, np int64) (*CSVWriter, error)

Create CSV writer

func (*CSVWriter) WriteString

func (self *CSVWriter) WriteString(recsi interface{}) error

Write string values to parquet file

type JSONWriter

type JSONWriter struct {
	ParquetWriter
}

func NewJSONWriter

func NewJSONWriter(jsonSchema string, pfile source.ParquetFile, np int64) (*JSONWriter, error)

Create JSON writer

type ParquetWriter

type ParquetWriter struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	Offset          int64

	Objs              []interface{}
	ObjsSize          int64
	ObjSize           int64
	CheckSizeCritical int64

	PagesMapBuf map[string][]*layout.Page
	Size        int64
	NumRows     int64

	DictRecs map[string]*layout.DictRecType

	MarshalFunc func(src []interface{}, bgn int, end int, sh *schema.SchemaHandler) (*map[string]*layout.Table, error)
}

ParquetWriter is a writer parquet file

func NewParquetWriter

func NewParquetWriter(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetWriter, error)

Create a parquet handler. Obj is a object with tags or JSON schema string.

func (*ParquetWriter) Flush

func (self *ParquetWriter) Flush(flag bool) error

Flush the write buffer to parquet file

func (*ParquetWriter) RenameSchema

func (self *ParquetWriter) RenameSchema()

Rename schema name to exname in tags

func (*ParquetWriter) SetSchemaHandlerFromJSON

func (self *ParquetWriter) SetSchemaHandlerFromJSON(jsonSchema string) error

func (*ParquetWriter) Write

func (self *ParquetWriter) Write(src interface{}) error

Write one object to parquet file

func (*ParquetWriter) WriteStop

func (self *ParquetWriter) WriteStop() error

Write the footer and stop writing