arrow

Documentation

Index

• Variables
• func CreateCsvReader(r io.Reader, schema *arrow.Schema, opts CsvOptions) *csv.Reader
• type ArraySlice
  • func (a ArraySlice) Release()
• type CastMeta
• type Castable
• type Caster
  • func NewCaster(allowedTypes []DataTyper) Caster
  • func (c Caster) CastArray(iface array.Interface, arr any) error
  • func (c Caster) IsCastable(iface array.Interface, arr any) error
• type CsvOptions
  • func (c CsvOptions) ToOptions() []csv.Option
• type DataTyper
• type ErrDataTypeMismatch
  • func (err *ErrDataTypeMismatch) Error() string
• type ErrNotCastable
  • func (err *ErrNotCastable) Error() string
• type Formatter
• type ID
• type Main
  • func NewMain(alloc memory.Allocator, meth *Methods, opts CsvOptions) *Main
  • func (m Main) Drop() error
  • func (m *Main) SetArrayPolicy(pol mapping.PolicyKind) error
  • func (m *Main) SetFormatterPolicy(pol mapping.PolicyKind) error
  • func (m *Main) SetRecordPolicy(pol mapping.PolicyKind) error
• type Methods
  • func (m Methods) DoCast(meta CastMeta, arrays mapping.Of[string, array.Interface]) (array.Record, error)
• type RecordStorage
• type Storage
  • func EmptyStorage() Storage
• type TypeSet
  • func (t TypeSet) Format(w io.Writer) error

Variables

var BuiltInArrayTypes = []DataTyper{
	&array.String{},
	&array.Boolean{},
	&array.Boolean{},
	&array.Float16{},
	&array.Float32{},
	&array.Float64{},
	&array.Int16{},
	&array.Int32{},
	&array.Int64{},
	&array.Uint16{},
	&array.Uint32{},
	&array.Uint64{},
	&array.Date32{},
	&array.Date64{},
	&array.Duration{},
}

var DefaultCaster = NewCaster(BuiltInArrayTypes)

var DefaultMethods = Methods{CreateCsvReader: CreateCsvReader, Cast: DefaultCaster.CastArray}

Functions

func CreateCsvReader

func CreateCsvReader(r io.Reader, schema *arrow.Schema, opts CsvOptions) *csv.Reader

Types

type ArraySlice

type ArraySlice struct {
	// contains filtered or unexported fields
}

func (ArraySlice) Release

func (a ArraySlice) Release()

type CastMeta

type CastMeta struct {
	Schema *arrow.Schema
	Arrays []any
}

type Castable

type Castable interface {
	CastMeta() CastMeta
}

type Caster

type Caster struct {
	// contains filtered or unexported fields
}

func NewCaster

func NewCaster(allowedTypes []DataTyper) Caster

func (Caster) CastArray

func (c Caster) CastArray(iface array.Interface, arr any) error

func (Caster) IsCastable

func (c Caster) IsCastable(iface array.Interface, arr any) error

type CsvOptions

type CsvOptions struct {
	// Allocator specifies the Arrow memory allocator used while building records.
	Allocator memory.Allocator
	// Comment specifies the comment character used while parsing CSV files.
	Comment rune
	// Comma specifies the fields separation character used while parsing CSV files.
	Comma rune
	// CRLF specifies the line terminator used while writing CSV files.
	// If useCRLF is true, \r\n is used as the line terminator, otherwise \n is used.
	CRLF bool
	// Chunk specifies the chunk size used while parsing CSV files.
	// If n is zero or 1, no chunking will take place and the reader will create
	// one record per row.
	// If n is greater than 1, chunks of n rows will be read.
	// If n is negative, the reader will load the whole CSV file into memory and
	// create one big record with all the rows.
	Chunk int
	// Header enables or disables CSV-header handling.
	Header bool
	// WithNullReader sets options for a CSV Reader pertaining to NULL value
	// handling. If stringsCanBeNull is true, then a string that matches one of the
	// nullValues set will be interpreted as NULL. Numeric columns will be checked
	// for nulls in all cases. If no nullValues arguments are passed in, the
	// defaults set in NewReader() will be kept.
	//
	// When no NULL values is given, the default set is taken from DefaultNullValues.
	StringsCanBeNull bool
	NullValues       []string
	// WithNullWriter sets the null string written for NULL values. The default is
	// set in NewWriter().
	WriterNullString string
}

func (CsvOptions) ToOptions

func (c CsvOptions) ToOptions() []csv.Option

type DataTyper

type DataTyper interface {
	DataType() arrow.DataType
}

type ErrDataTypeMismatch

type ErrDataTypeMismatch struct {
	Expected, Got arrow.DataType
}

func (*ErrDataTypeMismatch) Error

func (err *ErrDataTypeMismatch) Error() string

type ErrNotCastable

type ErrNotCastable struct {
	Type    reflect.Type
	TypeSet TypeSet
}

func (*ErrNotCastable) Error

func (err *ErrNotCastable) Error() string

type Formatter

type Formatter interface {
	FormatArray(w io.Writer, data *array.Data) (int, error)
	FormatRecord(w io.Writer, data *array.Record) (int, error)
}

type ID

type ID uint64

type Main

type Main struct {
	Storage    Storage
	Allocator  memory.Allocator
	Methods    *Methods
	CsvOptions CsvOptions
	// contains filtered or unexported fields
}

func NewMain

func NewMain(alloc memory.Allocator, meth *Methods, opts CsvOptions) *Main

func (Main) Drop

func (m Main) Drop() error

func (*Main) SetArrayPolicy

func (m *Main) SetArrayPolicy(pol mapping.PolicyKind) error

func (*Main) SetFormatterPolicy

func (m *Main) SetFormatterPolicy(pol mapping.PolicyKind) error

func (*Main) SetRecordPolicy

func (m *Main) SetRecordPolicy(pol mapping.PolicyKind) error

type Methods

type Methods struct {
	CreateCsvReader func(r io.Reader, schema *arrow.Schema, opts CsvOptions) *csv.Reader
	Cast            func(data array.Interface, arr any) error
	Mapping         *mapping.Methods
	Log             zerolog.Logger
}

func (Methods) DoCast

func (m Methods) DoCast(meta CastMeta, arrays mapping.Of[string, array.Interface]) (array.Record, error)

type RecordStorage

type RecordStorage struct {
	// contains filtered or unexported fields
}

type Storage

type Storage struct {
	Arrays     mapping.Mut[ID, array.Interface]
	Records    mapping.Mut[ID, array.Record]
	Formatters mapping.Mut[ID, Formatter]
}

func EmptyStorage

func EmptyStorage() Storage

type TypeSet

type TypeSet map[reflect.Type]struct{}

func (TypeSet) Format

func (t TypeSet) Format(w io.Writer) error