pqutil

Documentation

Index

• Variables
• func GetCompression(codec string) (compress.Compression, error)
• func LogicalOrConvertedAnnotation(node pqschema.Node) string
• func LookupGroupNode(schema *pqschema.Schema, name string) (*pqschema.GroupNode, bool)
• func LookupListElementNode(sc *pqschema.Schema, name string) (*pqschema.PrimitiveNode, bool)
• func LookupNode(schema *pqschema.Schema, name string) (pqschema.Node, bool)
• func LookupPrimitiveNode(schema *pqschema.Schema, name string) (*pqschema.PrimitiveNode, bool)
• func ParquetSchemaString(schema *pqschema.Schema) string
• func TransformByColumn(config *TransformConfig) error
• type ArrowSchemaBuilder
  • func NewArrowSchemaBuilder() *ArrowSchemaBuilder
  • func (b *ArrowSchemaBuilder) Add(record map[string]any) error
  • func (b *ArrowSchemaBuilder) AddGeometry(name string, encoding string) error
  • func (b *ArrowSchemaBuilder) Has(name string) bool
  • func (b *ArrowSchemaBuilder) Ready() bool
  • func (b *ArrowSchemaBuilder) Schema() (*arrow.Schema, error)
• type ColumnTransformer
• type SchemaTransformer
• type TransformConfig

Variables

var ParquetStringType = pqschema.StringLogicalType{}

Functions

func GetCompression

func GetCompression(codec string) (compress.Compression, error)

func LogicalOrConvertedAnnotation

func LogicalOrConvertedAnnotation(node pqschema.Node) string

func LookupGroupNode

func LookupGroupNode(schema *pqschema.Schema, name string) (*pqschema.GroupNode, bool)

func LookupListElementNode

func LookupListElementNode(sc *pqschema.Schema, name string) (*pqschema.PrimitiveNode, bool)

func LookupNode

func LookupNode(schema *pqschema.Schema, name string) (pqschema.Node, bool)

func LookupPrimitiveNode

func LookupPrimitiveNode(schema *pqschema.Schema, name string) (*pqschema.PrimitiveNode, bool)

func ParquetSchemaString

func ParquetSchemaString(schema *pqschema.Schema) string

ParquetSchemaString generates a string representation of the schema as documented in https://pkg.go.dev/github.com/fraugster/parquet-go/parquetschema

func TransformByColumn

func TransformByColumn(config *TransformConfig) error

Types

type ArrowSchemaBuilder

type ArrowSchemaBuilder struct {
	// contains filtered or unexported fields
}

func NewArrowSchemaBuilder

func NewArrowSchemaBuilder() *ArrowSchemaBuilder

func (*ArrowSchemaBuilder) Add

func (b *ArrowSchemaBuilder) Add(record map[string]any) error

func (*ArrowSchemaBuilder) AddGeometry

func (b *ArrowSchemaBuilder) AddGeometry(name string, encoding string) error

func (*ArrowSchemaBuilder) Has

func (b *ArrowSchemaBuilder) Has(name string) bool

func (*ArrowSchemaBuilder) Ready

func (b *ArrowSchemaBuilder) Ready() bool

func (*ArrowSchemaBuilder) Schema

func (b *ArrowSchemaBuilder) Schema() (*arrow.Schema, error)

type ColumnTransformer

type ColumnTransformer func(*arrow.Field, *arrow.Field, *arrow.Chunked) (*arrow.Chunked, error)

type SchemaTransformer

type SchemaTransformer func(*file.Reader) (*schema.Schema, error)

type TransformConfig

type TransformConfig struct {
	Reader          parquet.ReaderAtSeeker
	Writer          io.Writer
	Compression     *compress.Compression
	RowGroupLength  int
	TransformSchema SchemaTransformer
	TransformColumn ColumnTransformer
	BeforeClose     func(*file.Reader, *pqarrow.FileWriter) error
}