internal

Documentation

Index

• func DecimalToFloat(fieldAttr ReinterpretField, iface interface{}) (*float64, error)
• func NewCSVWriter(uri string, option WriteOption, schema []string) (*writer.CSVWriter, error)
• func NewGenericWriter(uri string, option WriteOption, schema string) (*writer.ParquetWriter, error)
• func NewJSONWriter(uri string, option WriteOption, schema string) (*writer.JSONWriter, error)
• func NewParquetFileReader(URI string, option ReadOption) (*reader.ParquetReader, error)
• func NewParquetFileWriter(uri string, option WriteOption) (source.ParquetFile, error)
• func StringToBytes(fieldAttr ReinterpretField, value string) []byte
• func TimeUnitToTag(timeUnit *parquet.TimeUnit) string
• type JSONSchema
• type ReadOption
• type ReinterpretField
• type SchemaNode
  • func NewSchemaTree(reader *reader.ParquetReader, option SchemaOption) (*SchemaNode, error)
  • func (s SchemaNode) CSVSchema() (string, error)
  • func (s *SchemaNode) GetReinterpretFields(rootPath string, noInterimLayer bool) map[string]ReinterpretField
  • func (s SchemaNode) GoStruct() (string, error)
  • func (s SchemaNode) JSONSchema() string
• type SchemaOption
• type WriteOption

Functions

func DecimalToFloat

func DecimalToFloat(fieldAttr ReinterpretField, iface interface{}) (*float64, error)

func NewCSVWriter

func NewCSVWriter(uri string, option WriteOption, schema []string) (*writer.CSVWriter, error)

func NewGenericWriter

func NewGenericWriter(uri string, option WriteOption, schema string) (*writer.ParquetWriter, error)

func NewJSONWriter

func NewJSONWriter(uri string, option WriteOption, schema string) (*writer.JSONWriter, error)

func NewParquetFileReader

func NewParquetFileReader(URI string, option ReadOption) (*reader.ParquetReader, error)

func NewParquetFileWriter

func NewParquetFileWriter(uri string, option WriteOption) (source.ParquetFile, error)

func StringToBytes

func StringToBytes(fieldAttr ReinterpretField, value string) []byte

func TimeUnitToTag

func TimeUnitToTag(timeUnit *parquet.TimeUnit) string

Types

type JSONSchema

type JSONSchema struct {
	Tag    string
	Fields []JSONSchema `json:",omitempty"`
}

type ReadOption

type ReadOption struct {
	HTTPMultipleConnection bool              `help:"(HTTP URI only) use multiple HTTP connection." default:"false"`
	HTTPIgnoreTLSError     bool              `help:"(HTTP URI only) ignore TLS error." default:"false"`
	HTTPExtraHeaders       map[string]string `mapsep:"," help:"(HTTP URI only) extra HTTP headers." default:""`
	ObjectVersion          string            `help:"(S3 URI only) object version." default:""`
	Anonymous              bool              `help:"(S3 and Azure only) object is publicly accessible." default:"false"`
}

ReadOption includes options for read operation

type ReinterpretField

type ReinterpretField struct {
	ParquetType   parquet.Type
	ConvertedType parquet.ConvertedType
	Precision     int
	Scale         int
}

type SchemaNode

type SchemaNode struct {
	parquet.SchemaElement
	Parent   []string      `json:"-"`
	Children []*SchemaNode `json:"children,omitempty"`
}

func NewSchemaTree

func NewSchemaTree(reader *reader.ParquetReader, option SchemaOption) (*SchemaNode, error)

func (SchemaNode) CSVSchema

func (s SchemaNode) CSVSchema() (string, error)

func (*SchemaNode) GetReinterpretFields

func (s *SchemaNode) GetReinterpretFields(rootPath string, noInterimLayer bool) map[string]ReinterpretField

func (SchemaNode) GoStruct

func (s SchemaNode) GoStruct() (string, error)

func (SchemaNode) JSONSchema

func (s SchemaNode) JSONSchema() string

type SchemaOption

type SchemaOption struct {
	FailOnInt96 bool
}

type WriteOption

type WriteOption struct {
	Compression string `` /* 144-byte string literal not displayed */
}

WriteOption includes options for write operation