parquet

README

parquet-go

Modified version of https://github.com/xitongsys/parquet-go

Documentation

Index

• type GetReaderFunc
• type Reader
  • func NewReader(getReaderFunc GetReaderFunc, columnNames set.StringSet) (*Reader, error)
  • func (reader *Reader) Close() (err error)
  • func (reader *Reader) Read() (record *Record, err error)
• type Record
  • func (r *Record) Get(name string) (Value, bool)
  • func (r *Record) Range(f func(name string, value Value) bool)
  • func (r *Record) String() string
• type Value
  • func (value Value) MarshalJSON() (data []byte, err error)
• type Writer
  • func NewWriter(writeCloser io.WriteCloser, schemaTree *schema.Tree, rowGroupCount int) (*Writer, error)
  • func (writer *Writer) Close() (err error)
  • func (writer *Writer) Write(record map[string]*data.Column) (err error)
  • func (writer *Writer) WriteJSON(recordData []byte) (err error)

Types

type GetReaderFunc

type GetReaderFunc func(offset, length int64) (io.ReadCloser, error)

GetReaderFunc - function type returning io.ReadCloser for requested offset/length.

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader - denotes parquet file.

func NewReader

func NewReader(getReaderFunc GetReaderFunc, columnNames set.StringSet) (*Reader, error)

NewReader - creates new parquet reader. Reader calls getReaderFunc to get required data range for given columnNames. If columnNames is empty, all columns are used.

func (*Reader) Close

func (reader *Reader) Close() (err error)

Close - closes underneath readers.

func (*Reader) Read

func (reader *Reader) Read() (record *Record, err error)

Read - reads single record.

type Record

type Record struct {
	// contains filtered or unexported fields
}

Record - ordered parquet record.

func (*Record) Get

func (r *Record) Get(name string) (Value, bool)

Get - returns Value of name.

func (*Record) Range

func (r *Record) Range(f func(name string, value Value) bool)

Range - calls f sequentially for each name and value present in the record. If f returns false, range stops the iteration.

func (*Record) String

func (r *Record) String() string

String - returns string representation of this record.

type Value

type Value struct {
	Value interface{}
	Type  parquet.Type
}

Value - denotes column value

func (Value) MarshalJSON

func (value Value) MarshalJSON() (data []byte, err error)

MarshalJSON - encodes to JSON data

type Writer

type Writer struct {
	PageSize        int64
	RowGroupSize    int64
	CompressionType parquet.CompressionCodec
	// contains filtered or unexported fields
}

Writer - represents parquet writer.

func NewWriter

func NewWriter(writeCloser io.WriteCloser, schemaTree *schema.Tree, rowGroupCount int) (*Writer, error)

NewWriter - creates new parquet writer. Binary data of rowGroupCount records are written to writeCloser.

func (*Writer) Close

func (writer *Writer) Close() (err error)

Close - finalizes and closes writer. If any pending records are available, they are written here.

func (*Writer) Write

func (writer *Writer) Write(record map[string]*data.Column) (err error)

Write - writes a record represented in map.

func (*Writer) WriteJSON

func (writer *Writer) WriteJSON(recordData []byte) (err error)

WriteJSON - writes a record represented in JSON.