parquet

Documentation

Index

• Constants
• func JSONToArray(buffer []byte) ([]model.Dictionary, error)
• func JSONToPackedDictionary(jsonline []byte) (model.Dictionary, error)
• type Sink
  • func NewSink(file io.Writer, schema *arrow.Schema) (Sink, error)
  • func NewSinkWithContext(file io.Writer, schema *arrow.Schema, counter string) (Sink, error)
  • func (s Sink) Close() error
  • func (s Sink) Open() error
  • func (s Sink) ProcessDictionary(dictionary model.Entry) error
• type Source
  • func NewPackedSource(path string) (*Source, error)
  • func NewSource(path string) (*Source, error)
  • func (s *Source) Err() error
  • func (s *Source) Next() bool
  • func (s *Source) NextBatch() bool
  • func (s *Source) Open() error
  • func (s *Source) Schema() (*arrow.Schema, error)
  • func (s *Source) Value() model.Entry

Constants

const BufferSize = 4

Functions

func JSONToArray

func JSONToArray(buffer []byte) ([]model.Dictionary, error)

JSONToArray return a array of model.Dictionary from a jsonline

func JSONToPackedDictionary

func JSONToPackedDictionary(jsonline []byte) (model.Dictionary, error)

JSONToPackedDictionary return a packed model.Dictionary from a jsonline

Types

type Sink

type Sink struct {
	// contains filtered or unexported fields
}

func NewSink

func NewSink(file io.Writer, schema *arrow.Schema) (Sink, error)

NewSink creates a new Sink.

func NewSinkWithContext

func NewSinkWithContext(file io.Writer, schema *arrow.Schema, counter string) (Sink, error)

NewSinkWithContext creates a new Sink.

func (Sink) Close

func (s Sink) Close() error

func (Sink) Open

func (s Sink) Open() error

func (Sink) ProcessDictionary

func (s Sink) ProcessDictionary(dictionary model.Entry) error

type Source

type Source struct {
	// contains filtered or unexported fields
}

Source export line to JSON format.

func NewPackedSource

func NewPackedSource(path string) (*Source, error)

NewPackedSource creates a new packed Source.

func NewSource

func NewSource(path string) (*Source, error)

NewSource creates a new Source.

func (*Source) Err

func (s *Source) Err() error

func (*Source) Next

func (s *Source) Next() bool

func (*Source) NextBatch

func (s *Source) NextBatch() bool

NextBatch read next batch in parquet file

func (*Source) Open

func (s *Source) Open() error

func (*Source) Schema

func (s *Source) Schema() (*arrow.Schema, error)

func (*Source) Value

func (s *Source) Value() model.Entry