Documentation ¶
Index ¶
- func BoolType(se *sch.SchemaElement)
- func Float32Type(se *sch.SchemaElement)
- func Float64Type(se *sch.SchemaElement)
- func GetBools(r io.Reader, n int, pageSizes []int) ([]bool, error)
- func Int32Type(se *sch.SchemaElement)
- func Int64Type(se *sch.SchemaElement)
- func OptionalFieldSnappy(r *OptionalField)
- func OptionalFieldUncompressed(o *OptionalField)
- func ReadMetaData(r io.ReadSeeker) (*sch.FileMetaData, error)
- func RepetitionOptional(se *sch.SchemaElement)
- func RepetitionRequired(se *sch.SchemaElement)
- func RequiredFieldSnappy(r *RequiredField)
- func RequiredFieldUncompressed(r *RequiredField)
- func StringType(se *sch.SchemaElement)
- func Uint32Type(se *sch.SchemaElement)
- func Uint64Type(se *sch.SchemaElement)
- type Field
- type FieldFunc
- type Metadata
- func (m *Metadata) Footer(w io.Writer) error
- func (m *Metadata) PageHeader(r io.ReadSeeker) (*sch.PageHeader, error)
- func (m *Metadata) Pages() (map[string][]Page, error)
- func (m *Metadata) ReadFooter(r io.ReadSeeker) error
- func (m *Metadata) RowGroups() []RowGroup
- func (m *Metadata) Rows() int64
- func (m *Metadata) StartRowGroup(fields ...Field)
- func (m *Metadata) WritePageHeader(w io.Writer, col string, dataLen, compressedLen, count int, ...) error
- type OptionalField
- type Page
- type RequiredField
- type RowGroup
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func BoolType ¶
func BoolType(se *sch.SchemaElement)
func Float32Type ¶
func Float32Type(se *sch.SchemaElement)
func Float64Type ¶
func Float64Type(se *sch.SchemaElement)
func Int32Type ¶
func Int32Type(se *sch.SchemaElement)
func Int64Type ¶
func Int64Type(se *sch.SchemaElement)
func OptionalFieldSnappy ¶ added in v0.1.0
func OptionalFieldSnappy(r *OptionalField)
OptionalFieldSnappy sets the compression for a column to snappy It is an optional arg to NewOptionalField
func OptionalFieldUncompressed ¶ added in v0.1.0
func OptionalFieldUncompressed(o *OptionalField)
OptionalFieldUncompressed sets the compression to none It is an optional arg to NewOptionalField
func ReadMetaData ¶ added in v0.1.0
func ReadMetaData(r io.ReadSeeker) (*sch.FileMetaData, error)
ReadMetaData reads the FileMetaData from the end of a parquet file
func RepetitionOptional ¶
func RepetitionOptional(se *sch.SchemaElement)
func RepetitionRequired ¶
func RepetitionRequired(se *sch.SchemaElement)
func RequiredFieldSnappy ¶ added in v0.1.0
func RequiredFieldSnappy(r *RequiredField)
RequiredFieldSnappy sets the compression for a column to snappy It is an optional arg to NewRequiredField
func RequiredFieldUncompressed ¶ added in v0.1.0
func RequiredFieldUncompressed(r *RequiredField)
RequiredFieldUncompressed sets the compression to none It is an optional arg to NewRequiredField
func StringType ¶
func StringType(se *sch.SchemaElement)
func Uint32Type ¶
func Uint32Type(se *sch.SchemaElement)
func Uint64Type ¶
func Uint64Type(se *sch.SchemaElement)
Types ¶
type FieldFunc ¶
type FieldFunc func(*sch.SchemaElement)
FieldFunc is used to set some of the metadata for each column
type Metadata ¶
type Metadata struct {
// contains filtered or unexported fields
}
Metadata keeps track of the things that need to be kept track of in order to write the FileMetaData at the end of the parquet file.
func (*Metadata) PageHeader ¶
func (m *Metadata) PageHeader(r io.ReadSeeker) (*sch.PageHeader, error)
PageHeader reads the page header from a column page
func (*Metadata) ReadFooter ¶
func (m *Metadata) ReadFooter(r io.ReadSeeker) error
ReadFooter reads the parquet metadata
func (*Metadata) StartRowGroup ¶
StartRowGroup is called when starting a new row group
func (*Metadata) WritePageHeader ¶
func (m *Metadata) WritePageHeader(w io.Writer, col string, dataLen, compressedLen, count int, comp sch.CompressionCodec) error
WritePageHeader is called when no more data is written to a column chunk
type OptionalField ¶ added in v0.0.8
type OptionalField struct { Defs []int64 // contains filtered or unexported fields }
func NewOptionalField ¶ added in v0.0.8
func NewOptionalField(col string, opts ...func(*OptionalField)) OptionalField
func (*OptionalField) DoRead ¶ added in v0.0.8
func (f *OptionalField) DoRead(r io.ReadSeeker, meta *Metadata, pg Page) (io.Reader, []int, error)
DoRead is called by all optional fields. It reads the definition levels and uses them to interpret the raw data.
func (*OptionalField) DoWrite ¶ added in v0.0.8
DoWrite is called by all optional field types to write the definition levels and raw data to the io.Writer
func (*OptionalField) Name ¶ added in v0.0.8
func (f *OptionalField) Name() string
Name returns the column name of this field
func (*OptionalField) Values ¶ added in v0.0.8
func (f *OptionalField) Values() int
Values reads the definition levels and uses them to return the values from the page data.
type Page ¶ added in v0.1.0
type Page struct { // N is the number of values in the ColumnChunk N int Size int Offset int64 Codec sch.CompressionCodec }
Page keeps track of metadata for each ColumnChunk
type RequiredField ¶ added in v0.0.8
type RequiredField struct {
// contains filtered or unexported fields
}
RequiredField writes the raw data for required columns
func NewRequiredField ¶ added in v0.0.8
func NewRequiredField(col string, opts ...func(*RequiredField)) RequiredField
NewRequiredField creates a new required field.
func (*RequiredField) DoRead ¶ added in v0.0.8
func (f *RequiredField) DoRead(r io.ReadSeeker, meta *Metadata, pg Page) (io.Reader, []int, error)
func (*RequiredField) Name ¶ added in v0.0.8
func (f *RequiredField) Name() string
type RowGroup ¶ added in v0.0.8
type RowGroup struct { Rows int64 // contains filtered or unexported fields }
RowGroup wraps schema.RowGroup and adds accounting functions that are used to keep track of number of rows written, byte size, etc.
func (*RowGroup) Columns ¶ added in v0.0.8
func (r *RowGroup) Columns() []*sch.ColumnChunk
Columns returns the Columns of the row group.