parquet

README

Parquet

Parquet generates a parquet reader and writer for your custom type. See this for an example of how to use it.

Documentation

Index

• func BoolType(se *sch.SchemaElement)
• func Float32Type(se *sch.SchemaElement)
• func Float64Type(se *sch.SchemaElement)
• func GetBools(r io.Reader, n int) ([]bool, error)
• func Int32Type(se *sch.SchemaElement)
• func Int64Type(se *sch.SchemaElement)
• func ReadLevels(r io.Reader) ([]int64, int, error)
• func RepetitionOptional(se *sch.SchemaElement)
• func RepetitionRequired(se *sch.SchemaElement)
• func StringType(se *sch.SchemaElement)
• func Uint32Type(se *sch.SchemaElement)
• func Uint64Type(se *sch.SchemaElement)
• func WriteLevels(w io.Writer, vals []int64) error
• type Field
• type FieldFunc
• type Metadata
  • func New(fields ...Field) *Metadata
  • func (m *Metadata) Footer(w io.Writer) error
  • func (m *Metadata) Offsets() (map[string][]Position, error)
  • func (m *Metadata) PageHeader(r io.ReadSeeker) (*sch.PageHeader, error)
  • func (m *Metadata) ReadFooter(r io.ReadSeeker) error
  • func (m *Metadata) RowGroups() []*sch.RowGroup
  • func (m *Metadata) Rows() int64
  • func (m *Metadata) StartRowGroup(fields ...Field)
  • func (m *Metadata) WritePageHeader(w io.Writer, col string, dataLen, compressedLen, count int) error
• type Position

Functions

func BoolType

func BoolType(se *sch.SchemaElement)

func Float32Type

func Float32Type(se *sch.SchemaElement)

func Float64Type

func Float64Type(se *sch.SchemaElement)

func GetBools

func GetBools(r io.Reader, n int) ([]bool, error)

func Int32Type

func Int32Type(se *sch.SchemaElement)

func Int64Type

func Int64Type(se *sch.SchemaElement)

func ReadLevels

func ReadLevels(r io.Reader) ([]int64, int, error)

ReadLevels reads the RLE encoded definition levels

func RepetitionOptional

func RepetitionOptional(se *sch.SchemaElement)

func RepetitionRequired

func RepetitionRequired(se *sch.SchemaElement)

func StringType

func StringType(se *sch.SchemaElement)

func Uint32Type

func Uint32Type(se *sch.SchemaElement)

func Uint64Type

func Uint64Type(se *sch.SchemaElement)

func WriteLevels

func WriteLevels(w io.Writer, vals []int64) error

WriteLevels writes vals to w as RLE encoded data

Types

type Field

type Field struct {
	Name           string
	Type           FieldFunc
	RepetitionType FieldFunc
}

type FieldFunc

type FieldFunc func(*sch.SchemaElement)

type Metadata

type Metadata struct {
	// contains filtered or unexported fields
}

func New

func New(fields ...Field) *Metadata

func (*Metadata) Footer

func (m *Metadata) Footer(w io.Writer) error

func (*Metadata) Offsets

func (m *Metadata) Offsets() (map[string][]Position, error)

func (*Metadata) PageHeader

func (m *Metadata) PageHeader(r io.ReadSeeker) (*sch.PageHeader, error)

func (*Metadata) ReadFooter

func (m *Metadata) ReadFooter(r io.ReadSeeker) error

func (*Metadata) RowGroups

func (m *Metadata) RowGroups() []*sch.RowGroup

func (*Metadata) Rows

func (m *Metadata) Rows() int64

func (*Metadata) StartRowGroup

func (m *Metadata) StartRowGroup(fields ...Field)

func (*Metadata) WritePageHeader

func (m *Metadata) WritePageHeader(w io.Writer, col string, dataLen, compressedLen, count int) error

WritePageHeader indicates you are done writing this columns's chunk

type Position

type Position struct {
	N      int
	Size   int
	Offset int64
}