storage

Documentation

Overview

Package storage contains functions that work with external data

Index

• func ParquetGuessCapiType(se *pgparquet.SchemaElement) (sc.TableFieldType, error)
• func ParquetReadBool(val any, se *pgparquet.SchemaElement) (bool, error)
• func ParquetReadDateTime(val any, se *pgparquet.SchemaElement) (time.Time, error)
• func ParquetReadDecimal2(val any, se *pgparquet.SchemaElement) (decimal.Decimal, error)
• func ParquetReadFloat(val any, se *pgparquet.SchemaElement) (float64, error)
• func ParquetReadInt(val any, se *pgparquet.SchemaElement) (int64, error)
• func ParquetReadString(val any, se *pgparquet.SchemaElement) (string, error)
• func ParquetWriterDecimal2(dec decimal.Decimal) any
• func ParquetWriterMilliTs(t time.Time) any
• type GuessedField
  • func CsvGuessFields(filePath string, csvHeaderLineIdx int, csvFirstDataLineIdx int, ...) ([]*GuessedField, error)
  • func ParquetGuessFields(filePath string) ([]*GuessedField, error)
• type ParquetWriter
  • func NewParquetWriter(ioWriter io.Writer, codec sc.ParquetCodecType) (*ParquetWriter, error)
  • func (w *ParquetWriter) AddColumn(name string, fieldType sc.TableFieldType) error
  • func (w *ParquetWriter) Close() error

Functions

func ParquetGuessCapiType

func ParquetGuessCapiType(se *pgparquet.SchemaElement) (sc.TableFieldType, error)

func ParquetReadBool

func ParquetReadBool(val any, se *pgparquet.SchemaElement) (bool, error)

func ParquetReadDateTime

func ParquetReadDateTime(val any, se *pgparquet.SchemaElement) (time.Time, error)

func ParquetReadDecimal2

func ParquetReadDecimal2(val any, se *pgparquet.SchemaElement) (decimal.Decimal, error)

func ParquetReadFloat

func ParquetReadFloat(val any, se *pgparquet.SchemaElement) (float64, error)

func ParquetReadInt

func ParquetReadInt(val any, se *pgparquet.SchemaElement) (int64, error)

func ParquetReadString

func ParquetReadString(val any, se *pgparquet.SchemaElement) (string, error)

func ParquetWriterDecimal2

func ParquetWriterDecimal2(dec decimal.Decimal) any

func ParquetWriterMilliTs

func ParquetWriterMilliTs(t time.Time) any

Types

type GuessedField

type GuessedField struct {
	OriginalHeader string
	CapiName       string
	Type           sc.TableFieldType
	Format         string
}

func CsvGuessFields

func CsvGuessFields(filePath string, csvHeaderLineIdx int, csvFirstDataLineIdx int, csvSeparator string) ([]*GuessedField, error)

func ParquetGuessFields

func ParquetGuessFields(filePath string) ([]*GuessedField, error)

type ParquetWriter

type ParquetWriter struct {
	FileWriter *gp.FileWriter
	StoreMap   map[string]*gp.ColumnStore // TODO: consider using w.FileWriter.GetColumnByName() instead and abandon ParquetWriter
}

func NewParquetWriter

func NewParquetWriter(ioWriter io.Writer, codec sc.ParquetCodecType) (*ParquetWriter, error)

func (*ParquetWriter) AddColumn

func (w *ParquetWriter) AddColumn(name string, fieldType sc.TableFieldType) error

func (*ParquetWriter) Close

func (w *ParquetWriter) Close() error