performance

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 17, 2021 License: MIT Imports: 9 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func BoolType 

func BoolType(se *sch.SchemaElement)

func Float32Type 

func Float32Type(se *sch.SchemaElement)

func Float64Type 

func Float64Type(se *sch.SchemaElement)

func Gzip 

func Gzip(p *ParquetWriter) error

func Int32Type 

func Int32Type(se *sch.SchemaElement)

func Int64Type 

func Int64Type(se *sch.SchemaElement)

func MaxPageSize 

func MaxPageSize(m int) func(*ParquetWriter) error

MaxPageSize is the maximum number of rows in each row groups' page.

func Snappy 

func Snappy(p *ParquetWriter) error

func StringType 

func StringType(se *sch.SchemaElement)

func Uint32Type 

func Uint32Type(se *sch.SchemaElement)

func Uint64Type 

func Uint64Type(se *sch.SchemaElement)

func Uncompressed 

func Uncompressed(p *ParquetWriter) error

Types

type BoolField 

type BoolField struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewBoolField 

func NewBoolField(read func(r Message) bool, write func(r *Message, vals []bool), path []string, opts ...func(*parquet.RequiredField)) *BoolField

func (*BoolField) Add 

func (f *BoolField) Add(r Message)

func (*BoolField) Levels 

func (f *BoolField) Levels() ([]uint8, []uint8)

func (*BoolField) Read 

func (f *BoolField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*BoolField) Scan 

func (f *BoolField) Scan(r *Message)

func (*BoolField) Schema 

func (f *BoolField) Schema() parquet.Field

func (*BoolField) Write 

func (f *BoolField) Write(w io.Writer, meta *parquet.Metadata) error

type BoolOptionalField 

type BoolOptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewBoolOptionalField 

func NewBoolOptionalField(read func(r Message) ([]bool, []uint8, []uint8), write func(r *Message, vals []bool, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *BoolOptionalField

func (*BoolOptionalField) Add 

func (f *BoolOptionalField) Add(r Message)

func (*BoolOptionalField) Levels 

func (f *BoolOptionalField) Levels() ([]uint8, []uint8)

func (*BoolOptionalField) Read 

func (f *BoolOptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*BoolOptionalField) Scan 

func (f *BoolOptionalField) Scan(r *Message)

func (*BoolOptionalField) Schema 

func (f *BoolOptionalField) Schema() parquet.Field

func (*BoolOptionalField) Write 

func (f *BoolOptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Field 

type Field interface {
	Add(r Message)
	Write(w io.Writer, meta *parquet.Metadata) error
	Schema() parquet.Field
	Scan(r *Message)
	Read(r io.ReadSeeker, pg parquet.Page) error
	Name() string
	Levels() ([]uint8, []uint8)
}

func Fields 

func Fields(compression compression) []Field

type Float32Field 

type Float32Field struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewFloat32Field 

func NewFloat32Field(read func(r Message) float32, write func(r *Message, vals []float32), path []string, opts ...func(*parquet.RequiredField)) *Float32Field

func (*Float32Field) Add 

func (f *Float32Field) Add(r Message)

func (*Float32Field) Levels 

func (f *Float32Field) Levels() ([]uint8, []uint8)

func (*Float32Field) Read 

func (f *Float32Field) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Float32Field) Scan 

func (f *Float32Field) Scan(r *Message)

func (*Float32Field) Schema 

func (f *Float32Field) Schema() parquet.Field

func (*Float32Field) Write 

func (f *Float32Field) Write(w io.Writer, meta *parquet.Metadata) error

type Float32OptionalField 

type Float32OptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewFloat32OptionalField 

func NewFloat32OptionalField(read func(r Message) ([]float32, []uint8, []uint8), write func(r *Message, vals []float32, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *Float32OptionalField

func (*Float32OptionalField) Add 

func (f *Float32OptionalField) Add(r Message)

func (*Float32OptionalField) Levels 

func (f *Float32OptionalField) Levels() ([]uint8, []uint8)

func (*Float32OptionalField) Read 

func (f *Float32OptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Float32OptionalField) Scan 

func (f *Float32OptionalField) Scan(r *Message)

func (*Float32OptionalField) Schema 

func (f *Float32OptionalField) Schema() parquet.Field

func (*Float32OptionalField) Write 

func (f *Float32OptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Float64Field 

type Float64Field struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewFloat64Field 

func NewFloat64Field(read func(r Message) float64, write func(r *Message, vals []float64), path []string, opts ...func(*parquet.RequiredField)) *Float64Field

func (*Float64Field) Add 

func (f *Float64Field) Add(r Message)

func (*Float64Field) Levels 

func (f *Float64Field) Levels() ([]uint8, []uint8)

func (*Float64Field) Read 

func (f *Float64Field) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Float64Field) Scan 

func (f *Float64Field) Scan(r *Message)

func (*Float64Field) Schema 

func (f *Float64Field) Schema() parquet.Field

func (*Float64Field) Write 

func (f *Float64Field) Write(w io.Writer, meta *parquet.Metadata) error

type Float64OptionalField 

type Float64OptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewFloat64OptionalField 

func NewFloat64OptionalField(read func(r Message) ([]float64, []uint8, []uint8), write func(r *Message, vals []float64, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *Float64OptionalField

func (*Float64OptionalField) Add 

func (f *Float64OptionalField) Add(r Message)

func (*Float64OptionalField) Levels 

func (f *Float64OptionalField) Levels() ([]uint8, []uint8)

func (*Float64OptionalField) Read 

func (f *Float64OptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Float64OptionalField) Scan 

func (f *Float64OptionalField) Scan(r *Message)

func (*Float64OptionalField) Schema 

func (f *Float64OptionalField) Schema() parquet.Field

func (*Float64OptionalField) Write 

func (f *Float64OptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Int32Field 

type Int32Field struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewInt32Field 

func NewInt32Field(read func(r Message) int32, write func(r *Message, vals []int32), path []string, opts ...func(*parquet.RequiredField)) *Int32Field

func (*Int32Field) Add 

func (f *Int32Field) Add(r Message)

func (*Int32Field) Levels 

func (f *Int32Field) Levels() ([]uint8, []uint8)

func (*Int32Field) Read 

func (f *Int32Field) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Int32Field) Scan 

func (f *Int32Field) Scan(r *Message)

func (*Int32Field) Schema 

func (f *Int32Field) Schema() parquet.Field

func (*Int32Field) Write 

func (f *Int32Field) Write(w io.Writer, meta *parquet.Metadata) error

type Int32OptionalField 

type Int32OptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewInt32OptionalField 

func NewInt32OptionalField(read func(r Message) ([]int32, []uint8, []uint8), write func(r *Message, vals []int32, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *Int32OptionalField

func (*Int32OptionalField) Add 

func (f *Int32OptionalField) Add(r Message)

func (*Int32OptionalField) Levels 

func (f *Int32OptionalField) Levels() ([]uint8, []uint8)

func (*Int32OptionalField) Read 

func (f *Int32OptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Int32OptionalField) Scan 

func (f *Int32OptionalField) Scan(r *Message)

func (*Int32OptionalField) Schema 

func (f *Int32OptionalField) Schema() parquet.Field

func (*Int32OptionalField) Write 

func (f *Int32OptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Int64Field 

type Int64Field struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewInt64Field 

func NewInt64Field(read func(r Message) int64, write func(r *Message, vals []int64), path []string, opts ...func(*parquet.RequiredField)) *Int64Field

func (*Int64Field) Add 

func (f *Int64Field) Add(r Message)

func (*Int64Field) Levels 

func (f *Int64Field) Levels() ([]uint8, []uint8)

func (*Int64Field) Read 

func (f *Int64Field) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Int64Field) Scan 

func (f *Int64Field) Scan(r *Message)

func (*Int64Field) Schema 

func (f *Int64Field) Schema() parquet.Field

func (*Int64Field) Write 

func (f *Int64Field) Write(w io.Writer, meta *parquet.Metadata) error

type Int64OptionalField 

type Int64OptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewInt64OptionalField 

func NewInt64OptionalField(read func(r Message) ([]int64, []uint8, []uint8), write func(r *Message, vals []int64, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *Int64OptionalField

func (*Int64OptionalField) Add 

func (f *Int64OptionalField) Add(r Message)

func (*Int64OptionalField) Levels 

func (f *Int64OptionalField) Levels() ([]uint8, []uint8)

func (*Int64OptionalField) Read 

func (f *Int64OptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*Int64OptionalField) Scan 

func (f *Int64OptionalField) Scan(r *Message)

func (*Int64OptionalField) Schema 

func (f *Int64OptionalField) Schema() parquet.Field

func (*Int64OptionalField) Write 

func (f *Int64OptionalField) Write(w io.Writer, meta *parquet.Metadata) error

type Levels 

type Levels struct {
	Name string
	Defs []uint8
	Reps []uint8
}

type ParquetReader 

type ParquetReader struct {
	// contains filtered or unexported fields
}

ParquetReader reads one page from a row group.

func NewParquetReader 

func NewParquetReader(r io.ReadSeeker, opts ...func(*ParquetReader)) (*ParquetReader, error)

func (*ParquetReader) Error 

func (p *ParquetReader) Error() error

func (*ParquetReader) Levels 

func (p *ParquetReader) Levels() []Levels

func (*ParquetReader) Next 

func (p *ParquetReader) Next() bool

func (*ParquetReader) Rows 

func (p *ParquetReader) Rows() int64

func (*ParquetReader) Scan 

func (p *ParquetReader) Scan(x *Message)

type ParquetWriter 

type ParquetWriter struct {
	// contains filtered or unexported fields
}

ParquetWriter reprents a row group

func NewParquetWriter 

func NewParquetWriter(w io.Writer, opts ...func(*ParquetWriter) error) (*ParquetWriter, error)

func (*ParquetWriter) Add 

func (p *ParquetWriter) Add(rec Message)

func (*ParquetWriter) Close 

func (p *ParquetWriter) Close() error

func (*ParquetWriter) Write 

func (p *ParquetWriter) Write() error

type StringField 

type StringField struct {
	parquet.RequiredField
	// contains filtered or unexported fields
}

func NewStringField 

func NewStringField(read func(r Message) string, write func(r *Message, vals []string), path []string, opts ...func(*parquet.RequiredField)) *StringField

func (*StringField) Add 

func (f *StringField) Add(r Message)

func (*StringField) Levels 

func (f *StringField) Levels() ([]uint8, []uint8)

func (*StringField) Read 

func (f *StringField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*StringField) Scan 

func (f *StringField) Scan(r *Message)

func (*StringField) Schema 

func (f *StringField) Schema() parquet.Field

func (*StringField) Write 

func (f *StringField) Write(w io.Writer, meta *parquet.Metadata) error

type StringOptionalField 

type StringOptionalField struct {
	parquet.OptionalField
	// contains filtered or unexported fields
}

func NewStringOptionalField 

func NewStringOptionalField(read func(r Message) ([]string, []uint8, []uint8), write func(r *Message, vals []string, defs, reps []uint8) (int, int), path []string, types []int, opts ...func(*parquet.OptionalField)) *StringOptionalField

func (*StringOptionalField) Add 

func (f *StringOptionalField) Add(r Message)

func (*StringOptionalField) Levels 

func (f *StringOptionalField) Levels() ([]uint8, []uint8)

func (*StringOptionalField) Read 

func (f *StringOptionalField) Read(r io.ReadSeeker, pg parquet.Page) error

func (*StringOptionalField) Scan 

func (f *StringOptionalField) Scan(r *Message)

func (*StringOptionalField) Schema 

func (f *StringOptionalField) Schema() parquet.Field

func (*StringOptionalField) Write 

func (f *StringOptionalField) Write(w io.Writer, meta *parquet.Metadata) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.