Documentation
¶
Index ¶
- Constants
- Variables
- type FileReader
- func (f *FileReader) Close() error
- func (f *FileReader) NumDictionaries() int
- func (f *FileReader) NumRecords() int
- func (f *FileReader) Read() (rec array.Record, err error)
- func (f *FileReader) ReadAt(i int64) (array.Record, error)
- func (f *FileReader) Record(i int) (array.Record, error)
- func (f *FileReader) RecordAt(i int) (array.Record, error)
- func (f *FileReader) Schema() *arrow.Schema
- func (f *FileReader) Version() MetadataVersion
- type FileWriter
- type Message
- type MessageReader
- type MessageType
- type MetadataVersion
- type Option
- type Payload
- type PayloadWriter
- type ReadAtSeeker
- type Reader
- type RecordBatchReader
- type RecordBatchWriter
- type Writer
Constants ¶
const ( MetadataV1 = MetadataVersion(flatbuf.MetadataVersionV1) // version for Arrow-0.1.0 MetadataV2 = MetadataVersion(flatbuf.MetadataVersionV2) // version for Arrow-0.2.0 MetadataV3 = MetadataVersion(flatbuf.MetadataVersionV3) // version for Arrow-0.3.0 to 0.7.1 MetadataV4 = MetadataVersion(flatbuf.MetadataVersionV4) // version for >= Arrow-0.8.0 MetadataV5 = MetadataVersion(flatbuf.MetadataVersionV5) // version for >= Arrow-1.0.0, backward compatible with v4 )
const ( MessageNone = MessageType(flatbuf.MessageHeaderNONE) MessageSchema = MessageType(flatbuf.MessageHeaderSchema) MessageDictionaryBatch = MessageType(flatbuf.MessageHeaderDictionaryBatch) MessageRecordBatch = MessageType(flatbuf.MessageHeaderRecordBatch) MessageTensor = MessageType(flatbuf.MessageHeaderTensor) MessageSparseTensor = MessageType(flatbuf.MessageHeaderSparseTensor) )
const ( // constants for the extension type metadata keys for the type name and // any extension metadata to be passed to deserialize. ExtensionTypeKeyName = "ARROW:extension:name" ExtensionMetadataKeyName = "ARROW:extension:metadata" )
Variables ¶
var Magic = []byte("ARROW1")
Magic string identifying an Apache Arrow file.
Functions ¶
This section is empty.
Types ¶
type FileReader ¶
type FileReader struct {
// contains filtered or unexported fields
}
FileReader is an Arrow file reader.
func NewFileReader ¶
func NewFileReader(r ReadAtSeeker, opts ...Option) (*FileReader, error)
NewFileReader opens an Arrow file using the provided reader r.
func (*FileReader) Close ¶
func (f *FileReader) Close() error
Close cleans up resources used by the File. Close does not close the underlying reader.
func (*FileReader) NumDictionaries ¶
func (f *FileReader) NumDictionaries() int
func (*FileReader) NumRecords ¶
func (f *FileReader) NumRecords() int
func (*FileReader) Read ¶
func (f *FileReader) Read() (rec array.Record, err error)
Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
The returned record value is valid until the next call to Read. Users need to call Retain on that Record to keep it valid for longer.
func (*FileReader) ReadAt ¶
func (f *FileReader) ReadAt(i int64) (array.Record, error)
ReadAt reads the i-th record from the underlying stream and an error, if any.
func (*FileReader) Record ¶
func (f *FileReader) Record(i int) (array.Record, error)
Record returns the i-th record from the file. The returned value is valid until the next call to Record. Users need to call Retain on that Record to keep it valid for longer.
func (*FileReader) RecordAt ¶
func (f *FileReader) RecordAt(i int) (array.Record, error)
Record returns the i-th record from the file. Ownership is transferred to the caller and must call Release() to free the memory. This method is safe to call concurrently.
func (*FileReader) Schema ¶
func (f *FileReader) Schema() *arrow.Schema
func (*FileReader) Version ¶
func (f *FileReader) Version() MetadataVersion
type FileWriter ¶
type FileWriter struct {
// contains filtered or unexported fields
}
FileWriter is an Arrow file writer.
func NewFileWriter ¶
func NewFileWriter(w io.WriteSeeker, opts ...Option) (*FileWriter, error)
NewFileWriter opens an Arrow file using the provided writer w.
func (*FileWriter) Close ¶
func (f *FileWriter) Close() error
type Message ¶
type Message struct {
// contains filtered or unexported fields
}
Message is an IPC message, including metadata and body.
func NewMessage ¶
NewMessage creates a new message from the metadata and body buffers. NewMessage panics if any of these buffers is nil.
func (*Message) Release ¶
func (msg *Message) Release()
Release decreases the reference count by 1. Release may be called simultaneously from multiple goroutines. When the reference count goes to zero, the memory is freed.
func (*Message) Retain ¶
func (msg *Message) Retain()
Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.
func (*Message) Type ¶
func (msg *Message) Type() MessageType
func (*Message) Version ¶
func (msg *Message) Version() MetadataVersion
type MessageReader ¶
func NewMessageReader ¶
func NewMessageReader(r io.Reader, opts ...Option) MessageReader
NewMessageReader returns a reader that reads messages from an input stream.
type MessageType ¶
type MessageType flatbuf.MessageHeader
MessageType represents the type of Message in an Arrow format.
func (MessageType) String ¶
func (m MessageType) String() string
type MetadataVersion ¶
type MetadataVersion flatbuf.MetadataVersion
MetadataVersion represents the Arrow metadata version.
func (MetadataVersion) String ¶
func (m MetadataVersion) String() string
type Option ¶
type Option func(*config)
Option is a functional option to configure opening or creating Arrow files and streams.
func WithAllocator ¶
WithAllocator specifies the Arrow memory allocator used while building records.
func WithCompressConcurrency ¶
WithCompressConcurrency specifies a number of goroutines to spin up for concurrent compression of the body buffers when writing compress IPC records. If n <= 1 then compression will be done serially without goroutine parallelization. Default is 0.
func WithFooterOffset ¶
WithFooterOffset specifies the Arrow footer position in bytes.
func WithLZ4 ¶
func WithLZ4() Option
WithLZ4 tells the writer to use LZ4 Frame compression on the data buffers before writing. Requires >= Arrow 1.0.0 to read/decompress
func WithSchema ¶
WithSchema specifies the Arrow schema to be used for reading or writing.
type Payload ¶
type Payload struct {
// contains filtered or unexported fields
}
Payload is the underlying message object which is passed to the payload writer for actually writing out ipc messages
type PayloadWriter ¶
PayloadWriter is an interface for injecting a different payloadwriter allowing more reusability with the Writer object with other scenarios, such as with Flight data
type Reader ¶
type Reader struct {
// contains filtered or unexported fields
}
Reader reads records from an io.Reader. Reader expects a schema (plus any dictionaries) as the first messages in the stream, followed by records.
func NewReaderFromMessageReader ¶
func NewReaderFromMessageReader(r MessageReader, opts ...Option) (*Reader, error)
NewReaderFromMessageReader allows constructing a new reader object with the provided MessageReader allowing injection of reading messages other than by simple streaming bytes such as Arrow Flight which receives a protobuf message
func (*Reader) Err ¶
Err returns the last error encountered during the iteration over the underlying stream.
func (*Reader) Read ¶
Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
func (*Reader) Record ¶
Record returns the current record that has been extracted from the underlying stream. It is valid until the next call to Next.
func (*Reader) Release ¶
func (r *Reader) Release()
Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.
type RecordBatchReader ¶
type RecordBatchReader struct {
// contains filtered or unexported fields
}
func NewRecordBatchReader ¶
NewRecordBatchReader returns a reader that reads records from an input stream.
func (*RecordBatchReader) Next ¶
func (r *RecordBatchReader) Next() bool
Next returns whether a Record could be extracted from the underlying stream.
func (*RecordBatchReader) Read ¶
func (r *RecordBatchReader) Read() (array.Record, error)
Read reads the current record from the underlying stream and an error, if any. When the Reader reaches the end of the underlying stream, it returns (nil, io.EOF).
func (*RecordBatchReader) Record ¶
func (r *RecordBatchReader) Record() array.Record
Record returns the current record that has been extracted from the underlying stream. It is valid until the next call to Next.
func (*RecordBatchReader) Release ¶
func (r *RecordBatchReader) Release()
Release decreases the reference count by 1. When the reference count goes to zero, the memory is freed. Release may be called simultaneously from multiple goroutines.
func (*RecordBatchReader) Retain ¶
func (r *RecordBatchReader) Retain()
Retain increases the reference count by 1. Retain may be called simultaneously from multiple goroutines.
type RecordBatchWriter ¶
type RecordBatchWriter struct {
// contains filtered or unexported fields
}
func NewRecordBatchWriter ¶
func NewRecordBatchWriter(w io.Writer, opts ...Option) *RecordBatchWriter
func (*RecordBatchWriter) Close ¶
func (w *RecordBatchWriter) Close() error
type Writer ¶
type Writer struct {
// contains filtered or unexported fields
}
Writer is an Arrow stream writer.
func NewWriterWithPayloadWriter ¶
func NewWriterWithPayloadWriter(pw PayloadWriter, opts ...Option) *Writer
NewWriterWithPayloadWriter constructs a writer with the provided payload writer instead of the default stream payload writer. This makes the writer more reusable such as by the Arrow Flight writer.
Source Files
Directories