arrow3

README

arrow3

Build apache arrow records from protocol buffers

Documentation

Index

• Variables
• type Schema
  • func New[T proto.Message](mem memory.Allocator) (schema *Schema[T], err error)
  • func (s *Schema[T]) Append(value T)
  • func (s *Schema[T]) NewRecord() arrow.Record
  • func (s *Schema[T]) Parquet() *schema.Schema
  • func (s *Schema[T]) Proto(r arrow.Record, rows []int) []T
  • func (s *Schema[T]) Read(ctx context.Context, r parquet.ReaderAtSeeker, columns []int) (arrow.Record, error)
  • func (s *Schema[T]) Release()
  • func (s *Schema[T]) Schema() *arrow.Schema
  • func (s *Schema[T]) WriteParquet(w io.Writer) error
  • func (s *Schema[T]) WriteParquetRecords(w io.Writer, records ...arrow.Record) error

Variables

var ErrMxDepth = errors.New("max depth reached, either the message is deeply nested or a circular dependency was introduced")

Types

type Schema

type Schema[T proto.Message] struct {
	// contains filtered or unexported fields
}

func New

func New[T proto.Message](mem memory.Allocator) (schema *Schema[T], err error)

func (*Schema[T]) Append

func (s *Schema[T]) Append(value T)

Append appends protobuf value to the schema builder.This method is not safe for concurrent use.

func (*Schema[T]) NewRecord

func (s *Schema[T]) NewRecord() arrow.Record

NewRecord returns buffered builder value as an arrow.Record. The builder is reset and can be reused to build new records.

func (*Schema[T]) Parquet

func (s *Schema[T]) Parquet() *schema.Schema

Parquet returns schema as parquet schema

func (*Schema[T]) Proto

func (s *Schema[T]) Proto(r arrow.Record, rows []int) []T

Proto decodes rows and returns them as proto messages.

func (*Schema[T]) Read

func (s *Schema[T]) Read(ctx context.Context, r parquet.ReaderAtSeeker, columns []int) (arrow.Record, error)

func (*Schema[T]) Release

func (s *Schema[T]) Release()

func (*Schema[T]) Schema

func (s *Schema[T]) Schema() *arrow.Schema

Parquet returns schema as arrow schema

func (*Schema[T]) WriteParquet

func (s *Schema[T]) WriteParquet(w io.Writer) error

func (*Schema[T]) WriteParquetRecords

func (s *Schema[T]) WriteParquetRecords(w io.Writer, records ...arrow.Record) error