parts

Documentation

Index

• func FindMaximumNonOverlappingSet(schema *dynparquet.Schema, parts []Part) ([]Part, []Part, error)
• type Option
  • func WithCompactionLevel(level int) Option
  • func WithRelease(release func()) Option
• type Part
  • func NewArrowPart(tx uint64, record arrow.Record, size uint64, schema *dynparquet.Schema, ...) Part
  • func NewParquetPart(tx uint64, buf *dynparquet.SerializedBuffer, options ...Option) Part
• type PartSorter
  • func NewPartSorter(schema *dynparquet.Schema, parts []Part) *PartSorter
  • func (p *PartSorter) Err() error
  • func (p *PartSorter) Len() int
  • func (p *PartSorter) Less(i, j int) bool
  • func (p *PartSorter) Swap(i, j int)

Functions

func FindMaximumNonOverlappingSet

func FindMaximumNonOverlappingSet(schema *dynparquet.Schema, parts []Part) ([]Part, []Part, error)

FindMaximumNonOverlappingSet removes the minimum number of parts from the given slice in order to return the maximum non-overlapping set of parts. The function returns the non-overlapping parts first and any overlapping parts second. The parts returned are in sorted order according to their Least row.

Types

type Option

type Option func(*basePart)

func WithCompactionLevel

func WithCompactionLevel(level int) Option

func WithRelease

func WithRelease(release func()) Option

type Part

type Part interface {
	// Record returns the Arrow record for the part. If the part is not an Arrow
	// record part, nil is returned.
	Record() arrow.Record
	Release()
	Retain()
	SerializeBuffer(schema *dynparquet.Schema, w dynparquet.ParquetWriter) error
	AsSerializedBuffer(schema *dynparquet.Schema) (*dynparquet.SerializedBuffer, error)
	NumRows() int64
	Size() int64
	CompactionLevel() int
	TX() uint64
	Least() (*dynparquet.DynamicRow, error)
	Most() (*dynparquet.DynamicRow, error)
	OverlapsWith(schema *dynparquet.Schema, otherPart Part) (bool, error)
	Write(io.Writer) error
}

func NewArrowPart

func NewArrowPart(tx uint64, record arrow.Record, size uint64, schema *dynparquet.Schema, options ...Option) Part

NewArrowPart returns a new Arrow part.

func NewParquetPart

func NewParquetPart(tx uint64, buf *dynparquet.SerializedBuffer, options ...Option) Part

type PartSorter

type PartSorter struct {
	// contains filtered or unexported fields
}

func NewPartSorter

func NewPartSorter(schema *dynparquet.Schema, parts []Part) *PartSorter

func (*PartSorter) Err

func (p *PartSorter) Err() error

func (*PartSorter) Len

func (p *PartSorter) Len() int

func (*PartSorter) Less

func (p *PartSorter) Less(i, j int) bool

func (*PartSorter) Swap

func (p *PartSorter) Swap(i, j int)