goparquet

Aug 18, 2022

Changes in this version

type FileReaderOption

+ func WithMaximumMemorySize(maxSizeBytes uint64) FileReaderOption

Apr 21, 2022

Changes in this version

type Column

+ func (c *Column) Path() ColumnPath

+ type ColumnPath []string

+ func (c ColumnPath) Equal(d ColumnPath) bool

+ func (c ColumnPath) HasPrefix(d ColumnPath) bool

type FileReader

+ func (f *FileReader) ColumnMetaDataByPath(path ColumnPath) (map[string]string, error)

+ func (f *FileReader) GetColumnByName(name string) *Column

+ func (f *FileReader) GetColumnByPath(path ColumnPath) *Column

+ func (f *FileReader) SetSelectedColumns(cols ...string)

+ func (f *FileReader) SetSelectedColumnsByPath(cols ...ColumnPath)

type FileReaderOption

+ func WithColumnPaths(columns ...ColumnPath) FileReaderOption

type FileWriter

+ func (fw *FileWriter) AddColumnByPath(path ColumnPath, col *Column) error

+ func (fw *FileWriter) AddGroupByPath(path ColumnPath, rep parquet.FieldRepetitionType) error

+ func (fw *FileWriter) Columns() []*Column

+ func (fw *FileWriter) GetColumnByName(name string) *Column

+ func (fw *FileWriter) GetColumnByPath(path ColumnPath) *Column

+ func (fw *FileWriter) SetSchemaDefinition(schemaDef *parquetschema.SchemaDefinition) error

type FlushRowGroupOption

+ func WithRowGroupMetaDataForColumnPath(path ColumnPath, kv map[string]string) FlushRowGroupOption

Feb 18, 2022

Changes in this version

type FileReader

+ func NewFileReaderWithOptions(r io.ReadSeeker, readerOptions ...FileReaderOption) (*FileReader, error)

+ func (f *FileReader) Columns() []*Column

+ func (f *FileReader) GetSchemaDefinition() *parquetschema.SchemaDefinition

+ type FileReaderOption func(*fileReaderOptions) error

+ func WithCRC32Validation(enable bool) FileReaderOption

+ func WithColumns(columns ...string) FileReaderOption

+ func WithFileMetaData(metaData *parquet.FileMetaData) FileReaderOption

+ func WithReaderContext(ctx context.Context) FileReaderOption

type FileWriterOption

+ func WithCRC(enableCRC bool) FileWriterOption

Feb 10, 2022

Changes in this version

type FileWriter

+ func (fw *FileWriter) AddColumn(path string, col *Column) error

+ func (fw *FileWriter) AddGroup(path string, rep parquet.FieldRepetitionType) error

+ func (fw *FileWriter) GetSchemaDefinition() *parquetschema.SchemaDefinition

type FileWriterOption

+ func WithMaxPageSize(size int64) FileWriterOption

Oct 29, 2021

Changes in this version

+ func ReadFileMetaDataWithContext(ctx context.Context, r io.ReadSeeker, extraValidation bool) (*parquet.FileMetaData, error)

type FileReader

+ func NewFileReaderWithContext(ctx context.Context, r io.ReadSeeker, columns ...string) (*FileReader, error)

+ func (f *FileReader) NextRowWithContext(ctx context.Context) (map[string]interface{}, error)

+ func (f *FileReader) PreLoadWithContext(ctx context.Context) error

+ func (f *FileReader) RowGroupNumRowsWithContext(ctx context.Context) (int64, error)

+ func (f *FileReader) SeekToRowGroupWithContext(ctx context.Context, rowGroupPosition int) error

type FileWriter

+ func (fw *FileWriter) CloseWithContext(ctx context.Context, opts ...FlushRowGroupOption) error

+ func (fw *FileWriter) FlushRowGroupWithContext(ctx context.Context, opts ...FlushRowGroupOption) error

type FileWriterOption

+ func WithWriterContext(ctx context.Context) FileWriterOption

Oct 6, 2021

Changes in this version

+ func IsAfterUnixEpoch(t time.Time) bool

+ func ReadFileMetaData(r io.ReadSeeker, extraValidation bool) (*parquet.FileMetaData, error)

type FileReader

+ func NewFileReaderWithMetaData(r io.ReadSeeker, meta *parquet.FileMetaData, columns ...string) (*FileReader, error)

+ func (f *FileReader) SeekToRowGroup(rowGroupPosition int) error

type SchemaReader

+ SetSelectedColumns func(selected ...string)

Nov 4, 2020

Changes in this version

+ func GetRegisteredBlockCompressors() map[parquet.CompressionCodec]BlockCompressor

May 26, 2020

Changes in this version

type FileReader

+ func (f *FileReader) ColumnMetaData(colName string) (map[string]string, error)

+ func (f *FileReader) MetaData() map[string]string

Apr 23, 2020

Changes in this version

+ var DefaultHashFunc func([]byte) interface

+ func Int96ToTime(parquetDate [12]byte) time.Time

+ func RegisterBlockCompressor(method parquet.CompressionCodec, compressor BlockCompressor)

+ func TimeToInt96(t time.Time) [12]byte

+ type BlockCompressor interface

+ CompressBlock func([]byte) ([]byte, error)

+ DecompressBlock func([]byte) ([]byte, error)

+ type Column struct

+ func NewDataColumn(store *ColumnStore, rep parquet.FieldRepetitionType) *Column

+ func NewListColumn(element *Column, rep parquet.FieldRepetitionType) (*Column, error)

+ func NewMapColumn(key, value *Column, rep parquet.FieldRepetitionType) (*Column, error)

+ func (c *Column) Children() []*Column

+ func (c *Column) ChildrenCount() int

+ func (c *Column) DataColumn() bool

+ func (c *Column) Element() *parquet.SchemaElement

+ func (c *Column) FlatName() string

+ func (c *Column) Index() int

+ func (c *Column) MaxDefinitionLevel() uint16

+ func (c *Column) MaxRepetitionLevel() uint16

+ func (c *Column) Name() string

+ func (c *Column) RepetitionType() *parquet.FieldRepetitionType

+ func (c *Column) Type() *parquet.Type

+ type ColumnParameters struct

+ ConvertedType *parquet.ConvertedType

+ FieldID *int32

+ LogicalType *parquet.LogicalType

+ Precision *int32

+ Scale *int32

+ TypeLength *int32

+ type ColumnStore struct

+ func NewBooleanStore(enc parquet.Encoding, params *ColumnParameters) (*ColumnStore, error)

+ func NewByteArrayStore(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewDoubleStore(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewFixedByteArrayStore(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewFloatStore(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewInt32Store(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewInt64Store(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ func NewInt96Store(enc parquet.Encoding, allowDict bool, params *ColumnParameters) (*ColumnStore, error)

+ type FileReader struct

+ func NewFileReader(r io.ReadSeeker, columns ...string) (*FileReader, error)

+ func (f *FileReader) CurrentRowGroup() *parquet.RowGroup

+ func (f *FileReader) NextRow() (map[string]interface{}, error)

+ func (f *FileReader) NumRows() int64

+ func (f *FileReader) PreLoad() error

+ func (f *FileReader) RowGroupCount() int

+ func (f *FileReader) RowGroupNumRows() (int64, error)

+ func (f *FileReader) SkipRowGroup()

+ type FileWriter struct

+ func NewFileWriter(w io.Writer, options ...FileWriterOption) *FileWriter

+ func (fw *FileWriter) AddData(m map[string]interface{}) error

+ func (fw *FileWriter) Close(opts ...FlushRowGroupOption) error

+ func (fw *FileWriter) CurrentFileSize() int64

+ func (fw *FileWriter) CurrentRowGroupSize() int64

+ func (fw *FileWriter) FlushRowGroup(opts ...FlushRowGroupOption) error

+ type FileWriterOption func(fw *FileWriter)

+ func FileVersion(version int32) FileWriterOption

+ func WithCompressionCodec(codec parquet.CompressionCodec) FileWriterOption

+ func WithCreator(createdBy string) FileWriterOption

+ func WithDataPageV2() FileWriterOption

+ func WithMaxRowGroupSize(size int64) FileWriterOption

+ func WithMetaData(data map[string]string) FileWriterOption

+ func WithSchemaDefinition(sd *parquetschema.SchemaDefinition) FileWriterOption

+ type FlushRowGroupOption func(h *flushRowGroupOptionHandle)

+ func WithRowGroupMetaData(kv map[string]string) FlushRowGroupOption

+ func WithRowGroupMetaDataForColumn(col string, kv map[string]string) FlushRowGroupOption

+ type SchemaCommon interface

+ Columns func() []*Column

+ GetColumnByName func(path string) *Column

+ GetSchemaDefinition func() *parquetschema.SchemaDefinition

+ SetSchemaDefinition func(*parquetschema.SchemaDefinition) error

+ type SchemaReader interface

+ type SchemaWriter interface

+ AddColumn func(path string, col *Column) error

+ AddData func(m map[string]interface{}) error

+ AddGroup func(path string, rep parquet.FieldRepetitionType) error

+ DataSize func() int64