ipc

Documentation

Index

• Constants
• Variables
• type FileReader
  • func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error)
  • func (f *FileReader) Close() error
  • func (f *FileReader) NumDictionaries() int
  • func (f *FileReader) NumRecords() int
  • func (f *FileReader) Read() (rec array.Record, err error)
  • func (f *FileReader) ReadAt(i int64) (array.Record, error)
  • func (f *FileReader) Record(i int) (array.Record, error)
  • func (f *FileReader) Schema() *arrow.Schema
  • func (f *FileReader) Version() MetadataVersion
• type FileWriter
  • func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error)
  • func (f *FileWriter) Close() error
  • func (f *FileWriter) Write(rec array.Record) error
• type Message
  • func NewMessage(meta, body *memory.Buffer) *Message
  • func (msg *Message) BodyLen() int64
  • func (msg *Message) Release()
  • func (msg *Message) Retain()
  • func (msg *Message) Type() MessageType
  • func (msg *Message) Version() MetadataVersion
• type MessageReader
  • func NewMessageReader(r io.Reader) MessageReader
• type MessageType
  • func (m MessageType) String() string
• type MetadataVersion
  • func (m MetadataVersion) String() string
• type Option
  • func WithAllocator(mem memory.Allocator) Option
  • func WithFooterOffset(offset int64) Option
  • func WithSchema(schema *arrow.Schema) Option
• type Payload
  • func (p *Payload) Meta() *memory.Buffer
  • func (p *Payload) Release()
  • func (p *Payload) SerializeBody(w io.Writer) error
• type PayloadWriter
• type ReadAtSeeker
• type Reader
  • func NewReader(r io.Reader, opts ...Option) (*Reader, error)
  • func NewReaderFromMessageReader(r MessageReader, opts ...Option) (*Reader, error)
  • func (r *Reader) Err() error
  • func (r *Reader) Next() bool
  • func (r *Reader) Read() (array.Record, error)
  • func (r *Reader) Record() array.Record
  • func (r *Reader) Release()
  • func (r *Reader) Retain()
  • func (r *Reader) Schema() *arrow.Schema
• type Writer
  • func NewWriter(w io.Writer, opts ...Option) *Writer
  • func NewWriterWithPayloadWriter(pw PayloadWriter, opts ...Option) *Writer
  • func (w *Writer) Close() error
  • func (w *Writer) Write(rec array.Record) error

Constants

const (
	MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0
	MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0
	MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1
	MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0
)

const (
	MessageNone            = MessageType(flatbuf.MessageHeaderNONE)
	MessageSchema          = MessageType(flatbuf.MessageHeaderSchema)
	MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch)
	MessageRecordBatch     = MessageType(flatbuf.MessageHeaderRecordBatch)
	MessageTensor          = MessageType(flatbuf.MessageHeaderTensor)
	MessageSparseTensor    = MessageType(flatbuf.MessageHeaderSparseTensor)
)

Variables

var Magic = []byte("ARROW1")

Magic string identifying an Apache Arrow file.

Types

type FileReader

type FileReader struct {
	// contains filtered or unexported fields
}

FileReader is an Arrow file reader.

func NewFileReader

func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error)

NewFileReader opens an Arrow file using the provided reader r.

func (*FileReader) Close

func (f *FileReader) Close() error

Close cleans up resources used by the File. Close does not close the underlying reader.

func (*FileReader) NumDictionaries

func (f *FileReader) NumDictionaries() int

func (*FileReader) NumRecords

func (f *FileReader) NumRecords() int

func (*FileReader) Read

func (f *FileReader) Read() (rec array.Record, err error)

Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).

The returned record value is valid until the next call to Read. Users need to call Retain on that Record to keep it valid for longer.

func (*FileReader) ReadAt

func (f *FileReader) ReadAt(i int64) (array.Record, error)

ReadAt reads the i-th record from the underlying stream and an error, if any.

func (*FileReader) Record

func (f *FileReader) Record(i int) (array.Record, error)

Record returns the i-th record from the file. The returned value is valid until the next call to Record. Users need to call Retain on that Record to keep it valid for longer.

func (*FileReader) Schema

func (f *FileReader) Schema() *arrow.Schema

func (*FileReader) Version

func (f *FileReader) Version() MetadataVersion

type FileWriter

type FileWriter struct {
	// contains filtered or unexported fields
}

FileWriter is an Arrow file writer.

func NewFileWriter

func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error)

NewFileWriter opens an Arrow file using the provided writer w.

func (*FileWriter) Close

func (f *FileWriter) Close() error

func (*FileWriter) Write

func (f *FileWriter) Write(rec array.Record) error

type Message

type Message struct {
	// contains filtered or unexported fields
}

Message is an IPC message, including metadata and body.

func NewMessage

func NewMessage(meta, body *memory.Buffer) *Message

NewMessage creates a new message from the metadata and body buffers. NewMessage panics if any of these buffers is nil.

func (*Message) BodyLen

func (msg *Message) BodyLen() int64

func (*Message) Release

func (msg *Message) Release()

Release decreases the reference count by 1. Release may be called simultaneously from multiple goroutines. When the reference count goes to zero, the memory is freed.

func (*Message) Retain

func (msg *Message) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

func (*Message) Type

func (msg *Message) Type() MessageType

func (*Message) Version

func (msg *Message) Version() MetadataVersion

type MessageReader

type MessageReader interface {
	Message() (*Message, error)
	Release()
	Retain()
}

func NewMessageReader

func NewMessageReader(r io.Reader) MessageReader

NewMessageReader returns a reader that reads messages from an input stream.

type MessageType

type MessageType flatbuf.MessageHeader

MessageType represents the type of Message in an Arrow format.

func (MessageType) String

func (m MessageType) String() string

type MetadataVersion

type MetadataVersion flatbuf.MetadataVersion

MetadataVersion represents the Arrow metadata version.

func (MetadataVersion) String

func (m MetadataVersion) String() string

type Option

type Option func(*config)

Option is a functional option to configure opening or creating Arrow files and streams.

func WithAllocator

func WithAllocator(mem memory.Allocator) Option

WithAllocator specifies the Arrow memory allocator used while building records.

func WithFooterOffset

func WithFooterOffset(offset int64) Option

WithFooterOffset specifies the Arrow footer position in bytes.

func WithSchema

func WithSchema(schema *arrow.Schema) Option

WithSchema specifies the Arrow schema to be used for reading or writing.

type Payload

type Payload struct {
	// contains filtered or unexported fields
}

Payload is the underlying message object which is passed to the payload writer for actually writing out ipc messages

func (*Payload) Meta

func (p *Payload) Meta() *memory.Buffer

Meta returns the buffer containing the metadata for this payload, callers must call Release on the buffer

func (*Payload) Release

func (p *Payload) Release()

func (*Payload) SerializeBody

func (p *Payload) SerializeBody(w io.Writer) error

SerializeBody serializes the body buffers and writes them to the provided writer.

type PayloadWriter

type PayloadWriter interface {
	Start() error
	WritePayload(Payload) error
	Close() error
}

PayloadWriter is an interface for injecting a different payloadwriter allowing more reusability with the Writer object with other scenarios, such as with Flight data

type ReadAtSeeker

type ReadAtSeeker interface {
	io.Reader
	io.Seeker
	io.ReaderAt
}

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader reads records from an io.Reader. Reader expects a schema (plus any dictionaries) as the first messages in the stream, followed by records.

func NewReader

func NewReader(r io.Reader, opts ...Option) (*Reader, error)

NewReader returns a reader that reads records from an input stream.

func NewReaderFromMessageReader

func NewReaderFromMessageReader(r MessageReader, opts ...Option) (*Reader, error)

NewReaderFromMessageReader allows constructing a new reader object with the provided MessageReader allowing injection of reading messages other than by simple streaming bytes such as Arrow Flight which receives a protobuf message

func (*Reader) Err

func (r *Reader) Err() error

Err returns the last error encountered during the iteration over the underlying stream.

func (*Reader) Next

func (r *Reader) Next() bool

Next returns whether a Record could be extracted from the underlying stream.

func (*Reader) Read

func (r *Reader) Read() (array.Record, error)

Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).

func (*Reader) Record

func (r *Reader) Record() array.Record

Record returns the current record that has been extracted from the underlying stream. It is valid until the next call to Next.

func (*Reader) Release

func (r *Reader) Release()

Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.

func (*Reader) Retain

func (r *Reader) Retain()

Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.

func (*Reader) Schema

func (r *Reader) Schema() *arrow.Schema

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer is an Arrow stream writer.

func NewWriter

func NewWriter(w io.Writer, opts ...Option) *Writer

NewWriter returns a writer that writes records to the provided output stream.

func NewWriterWithPayloadWriter

func NewWriterWithPayloadWriter(pw PayloadWriter, opts ...Option) *Writer

NewWriterWithPayloadWriter constructs a writer with the provided payload writer instead of the default stream payload writer. This makes the writer more reusable such as by the Arrow Flight writer.

func (*Writer) Close

func (w *Writer) Close() error

func (*Writer) Write

func (w *Writer) Write(rec array.Record) error