pqarrow

Documentation

Index

• func ArrowScalarToParquetValue(sc scalar.Scalar) (parquet.Value, error)
• func ColToWriter(col int, writers []MultiColumnWriter) writer.ValueWriter
• func ParquetRowGroupToArrowSchema(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema, ...) (*arrow.Schema, error)
• func ParquetSchemaToArrowSchema(ctx context.Context, schema *parquet.Schema, s *dynparquet.Schema, ...) (*arrow.Schema, error)
• func Project(r arrow.Record, projections []logicalplan.Expr) arrow.Record
• func RecordDynamicCols(record arrow.Record) (columns map[string][]string)
• func RecordToDynamicRow(pqSchema *parquet.Schema, record arrow.Record, dyncols map[string][]string, ...) (*dynparquet.DynamicRow, error)
• func RecordToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, r arrow.Record) error
• func RecordToRow(final *parquet.Schema, record arrow.Record, index int) (parquet.Row, error)
• func RecordsToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, recs []arrow.Record) error
• func SerializeRecord(r arrow.Record, schema *dynparquet.Schema) (*dynparquet.SerializedBuffer, error)
• func SingleMatchingColumn(distinctColumns []logicalplan.Expr, fields []parquet.Field) bool
• type MultiColumnWriter
• type ParquetConverter
  • func NewParquetConverter(pool memory.Allocator, iterOpts logicalplan.IterOptions) *ParquetConverter
  • func (c *ParquetConverter) Close()
  • func (c *ParquetConverter) Convert(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema) error
  • func (c *ParquetConverter) Fields() []builder.ColumnBuilder
  • func (c *ParquetConverter) NewRecord() arrow.Record
  • func (c *ParquetConverter) NumRows() int
  • func (c *ParquetConverter) Reset()
• type PreExprVisitorFunc
  • func (f PreExprVisitorFunc) PostVisit(_ logicalplan.Expr) bool
  • func (f PreExprVisitorFunc) PreVisit(expr logicalplan.Expr) bool
  • func (f PreExprVisitorFunc) Visit(_ logicalplan.Expr) bool

Functions

func ArrowScalarToParquetValue

func ArrowScalarToParquetValue(sc scalar.Scalar) (parquet.Value, error)

func ColToWriter

func ColToWriter(col int, writers []MultiColumnWriter) writer.ValueWriter

func ParquetRowGroupToArrowSchema

func ParquetRowGroupToArrowSchema(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema, options logicalplan.IterOptions) (*arrow.Schema, error)

ParquetRowGroupToArrowSchema converts a parquet row group to an arrow schema.

func ParquetSchemaToArrowSchema

func ParquetSchemaToArrowSchema(ctx context.Context, schema *parquet.Schema, s *dynparquet.Schema, options logicalplan.IterOptions) (*arrow.Schema, error)

func Project

func Project(r arrow.Record, projections []logicalplan.Expr) arrow.Record

Project will project the record according to the given projections.

func RecordDynamicCols

func RecordDynamicCols(record arrow.Record) (columns map[string][]string)

func RecordToDynamicRow

func RecordToDynamicRow(pqSchema *parquet.Schema, record arrow.Record, dyncols map[string][]string, index int) (*dynparquet.DynamicRow, error)

func RecordToFile

func RecordToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, r arrow.Record) error

func RecordToRow

func RecordToRow(final *parquet.Schema, record arrow.Record, index int) (parquet.Row, error)

RecordToRow converts an arrow record with dynamic columns into a row using a dynamic parquet schema.

func RecordsToFile

func RecordsToFile(schema *dynparquet.Schema, w dynparquet.ParquetWriter, recs []arrow.Record) error

func SerializeRecord

func SerializeRecord(r arrow.Record, schema *dynparquet.Schema) (*dynparquet.SerializedBuffer, error)

func SingleMatchingColumn

func SingleMatchingColumn(distinctColumns []logicalplan.Expr, fields []parquet.Field) bool

SingleMatchingColumn returns true if there is only a single matching column for the given column matchers.

Types

type MultiColumnWriter

type MultiColumnWriter struct {
	// contains filtered or unexported fields
}

type ParquetConverter

type ParquetConverter struct {
	// contains filtered or unexported fields
}

ParquetConverter converts parquet.RowGroups into arrow.Records. The converted results are accumulated in the converter and can be retrieved by calling NewRecord, at which point the converter is reset.

func NewParquetConverter

func NewParquetConverter(
	pool memory.Allocator,
	iterOpts logicalplan.IterOptions,
) *ParquetConverter

func (*ParquetConverter) Close

func (c *ParquetConverter) Close()

func (*ParquetConverter) Convert

func (c *ParquetConverter) Convert(ctx context.Context, rg parquet.RowGroup, s *dynparquet.Schema) error

func (*ParquetConverter) Fields

func (c *ParquetConverter) Fields() []builder.ColumnBuilder

func (*ParquetConverter) NewRecord

func (c *ParquetConverter) NewRecord() arrow.Record

func (*ParquetConverter) NumRows

func (c *ParquetConverter) NumRows() int

func (*ParquetConverter) Reset

func (c *ParquetConverter) Reset()

type PreExprVisitorFunc

type PreExprVisitorFunc func(expr logicalplan.Expr) bool

func (PreExprVisitorFunc) PostVisit

func (f PreExprVisitorFunc) PostVisit(_ logicalplan.Expr) bool

func (PreExprVisitorFunc) PreVisit

func (f PreExprVisitorFunc) PreVisit(expr logicalplan.Expr) bool

func (PreExprVisitorFunc) Visit

func (f PreExprVisitorFunc) Visit(_ logicalplan.Expr) bool