arrjson

Documentation

Overview

Package arrjson provides types and functions to encode and decode ARROW types and data to and from JSON files.

Index

• type Array
  • func (a *Array) MarshalJSON() ([]byte, error)
  • func (a *Array) UnmarshalJSON(b []byte) (err error)
• type Dictionary
• type Field
• type FieldDict
• type FieldWrapper
  • func (f FieldWrapper) MarshalJSON() ([]byte, error)
  • func (f *FieldWrapper) UnmarshalJSON(data []byte) error
• type Option
  • func WithAllocator(mem memory.Allocator) Option
  • func WithSchema(schema *arrow.Schema) Option
• type Reader
  • func NewReader(r io.Reader, opts ...Option) (*Reader, error)
  • func (r *Reader) NumRecords() int
  • func (r *Reader) Read() (arrow.Record, error)
  • func (r *Reader) ReadAt(index int) (arrow.Record, error)
  • func (r *Reader) Release()
  • func (r *Reader) Retain()
  • func (r *Reader) Schema() *arrow.Schema
• type Record
• type Schema
  • func (s Schema) MarshalJSON() ([]byte, error)
  • func (s *Schema) UnmarshalJSON(data []byte) error
• type Writer
  • func NewWriter(w io.Writer, schema *arrow.Schema) (*Writer, error)
  • func (w *Writer) Close() error
  • func (w *Writer) Write(rec arrow.Record) error

Types

type Array

type Array struct {
	Name     string                `json:"name"`
	Count    int                   `json:"count"`
	Valids   []int                 `json:"VALIDITY,omitempty"`
	Data     []interface{}         `json:"DATA,omitempty"`
	TypeID   []arrow.UnionTypeCode `json:"TYPE_ID,omitempty"`
	Offset   interface{}           `json:"OFFSET,omitempty"`
	Size     interface{}           `json:"SIZE,omitempty"`
	Children []Array               `json:"children,omitempty"`
	Variadic []string              `json:"VARIADIC_DATA_BUFFERS,omitempty"`
	Views    []interface{}         `json:"VIEWS,omitempty"`
}

func (*Array) MarshalJSON

func (a *Array) MarshalJSON() ([]byte, error)

func (*Array) UnmarshalJSON

func (a *Array) UnmarshalJSON(b []byte) (err error)

type Dictionary

type Dictionary struct {
	ID   int64  `json:"id"`
	Data Record `json:"data"`
}

type Field

type Field struct {
	Name string `json:"name"`

	// leave this as a json RawMessage in order to partially unmarshal as needed
	// during marshal/unmarshal time so we can determine what the structure is
	// actually expected to be.
	Type     json.RawMessage `json:"type"`
	Nullable bool            `json:"nullable"`
	Children []FieldWrapper  `json:"children"`

	Dictionary *FieldDict `json:"dictionary,omitempty"`
	Metadata   []metaKV   `json:"metadata,omitempty"`
	// contains filtered or unexported fields
}

type FieldDict

type FieldDict struct {
	ID   int             `json:"id"`
	Type json.RawMessage `json:"indexType"`

	Ordered bool `json:"isOrdered"`
	// contains filtered or unexported fields
}

type FieldWrapper

type FieldWrapper struct {
	Field
}

FieldWrapper gets used in order to hook into the JSON marshalling and unmarshalling without creating an infinite loop when dealing with the children fields.

func (FieldWrapper) MarshalJSON

func (f FieldWrapper) MarshalJSON() ([]byte, error)

func (*FieldWrapper) UnmarshalJSON

func (f *FieldWrapper) UnmarshalJSON(data []byte) error

type Option

type Option func(*config)

Option is a functional option to configure opening or creating Arrow files and streams.

func WithAllocator

func WithAllocator(mem memory.Allocator) Option

WithAllocator specifies the Arrow memory allocator used while building records.

func WithSchema

func WithSchema(schema *arrow.Schema) Option

WithSchema specifies the Arrow schema to be used for reading or writing.

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader

func NewReader(r io.Reader, opts ...Option) (*Reader, error)

func (*Reader) NumRecords

func (r *Reader) NumRecords() int

func (*Reader) Read

func (r *Reader) Read() (arrow.Record, error)

func (*Reader) ReadAt

func (r *Reader) ReadAt(index int) (arrow.Record, error)

func (*Reader) Release

func (r *Reader) Release()

Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.

func (*Reader) Retain

func (r *Reader) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

func (*Reader) Schema

func (r *Reader) Schema() *arrow.Schema

type Record

type Record struct {
	Count   int64   `json:"count"`
	Columns []Array `json:"columns"`
}

type Schema

type Schema struct {
	Fields []FieldWrapper `json:"fields"`

	Metadata []metaKV `json:"metadata,omitempty"`
	// contains filtered or unexported fields
}

func (Schema) MarshalJSON

func (s Schema) MarshalJSON() ([]byte, error)

func (*Schema) UnmarshalJSON

func (s *Schema) UnmarshalJSON(data []byte) error

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter(w io.Writer, schema *arrow.Schema) (*Writer, error)

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) Write

func (w *Writer) Write(rec arrow.Record) error