common

Documentation

Index

• Constants
• func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int, el *parquet.SchemaElement) ([]interface{}, error)
• func CmpIntBinary(as string, bs string, order string, signed bool) bool
• func DeepCopy(src, dst interface{})
• func HeadToUpper(str string) string
• func Max(table FuncTable, a interface{}, b interface{}) interface{}
• func Min(table FuncTable, a interface{}, b interface{}) interface{}
• func NewLogicalTypeFromConvertedType(schemaElement *parquet.SchemaElement, info *Tag) *parquet.LogicalType
• func NewLogicalTypeFromFieldsMap(mp map[string]string) (*parquet.LogicalType, error)
• func NewSchemaElementFromTagMap(info *Tag) (*parquet.SchemaElement, error)
• func NewTable(rowLen, colLen int) [][]interface{}
• func PathStrIndex(str string) int
• func PathToStr(path []string) string
• func ReformPathStr(pathStr string) string
• func SizeOf(val reflect.Value) int64
• func Str2Bool(val string) (bool, error)
• func Str2Int32(val string) (int32, error)
• func StrToPath(str string) []string
• func StringToVariableName(str string) string
• func TransposeTable(table [][]interface{}) [][]interface{}
• type FuncTable
  • func FindFuncTable(pT *parquet.Type, cT *parquet.ConvertedType, logT *parquet.LogicalType) FuncTable
• type Tag
  • func GetKeyTagMap(src *Tag) *Tag
  • func GetValueTagMap(src *Tag) *Tag
  • func NewTag() *Tag
  • func StringToTag(tag string) (*Tag, error)

Constants

const PAR_GO_PATH_DELIMITER = "\x01"

Functions

func ArrowColToParquetCol

func ArrowColToParquetCol(field arrow.Field, col array.Interface, len int,
	el *parquet.SchemaElement) ([]interface{}, error)

ArrowColToParquetCol creates column with native parquet values from column with arrow values.

If a single record is not valid by the arrow definitions we assign it default value which we chose.

func CmpIntBinary

func CmpIntBinary(as string, bs string, order string, signed bool) bool

func DeepCopy

func DeepCopy(src, dst interface{})

func HeadToUpper

func HeadToUpper(str string) string

Convert the first letter of a string to uppercase

func Max

func Max(table FuncTable, a interface{}, b interface{}) interface{}

func Min

func Min(table FuncTable, a interface{}, b interface{}) interface{}

func NewLogicalTypeFromConvertedType

func NewLogicalTypeFromConvertedType(schemaElement *parquet.SchemaElement, info *Tag) *parquet.LogicalType

func NewLogicalTypeFromFieldsMap

func NewLogicalTypeFromFieldsMap(mp map[string]string) (*parquet.LogicalType, error)

func NewSchemaElementFromTagMap

func NewSchemaElementFromTagMap(info *Tag) (*parquet.SchemaElement, error)

func NewTable

func NewTable(rowLen, colLen int) [][]interface{}

NewTable creates empty table with transposed columns and records

func PathStrIndex

func PathStrIndex(str string) int

Get the pathStr index in a path

func PathToStr

func PathToStr(path []string) string

Convert path slice to string

func ReformPathStr

func ReformPathStr(pathStr string) string

. -> \x01

func SizeOf

func SizeOf(val reflect.Value) int64

Get the size of a parquet value

func Str2Bool

func Str2Bool(val string) (bool, error)

func Str2Int32

func Str2Int32(val string) (int32, error)

func StrToPath

func StrToPath(str string) []string

Convert string to path slice

func StringToVariableName

func StringToVariableName(str string) string

Convert string to a golang variable name

func TransposeTable

func TransposeTable(table [][]interface{}) [][]interface{}

TransposeTable transposes a table's rows and columns once per arrow record. We need to transpose the rows and columns because parquet-go library writes data row by row while the arrow library provides the data column by column.

Types

type FuncTable

type FuncTable interface {
	LessThan(a interface{}, b interface{}) bool
	MinMaxSize(minVal interface{}, maxVal interface{}, val interface{}) (interface{}, interface{}, int32)
}

func FindFuncTable

func FindFuncTable(pT *parquet.Type, cT *parquet.ConvertedType, logT *parquet.LogicalType) FuncTable

type Tag

type Tag struct {
	InName string
	ExName string

	Type      string
	KeyType   string
	ValueType string

	ConvertedType      string
	KeyConvertedType   string
	ValueConvertedType string

	Length      int32
	KeyLength   int32
	ValueLength int32

	Scale      int32
	KeyScale   int32
	ValueScale int32

	Precision      int32
	KeyPrecision   int32
	ValuePrecision int32

	IsAdjustedToUTC      bool
	KeyIsAdjustedToUTC   bool
	ValueIsAdjustedToUTC bool

	FieldID      int32
	KeyFieldID   int32
	ValueFieldID int32

	Encoding      parquet.Encoding
	KeyEncoding   parquet.Encoding
	ValueEncoding parquet.Encoding

	OmitStats      bool
	KeyOmitStats   bool
	ValueOmitStats bool

	RepetitionType      parquet.FieldRepetitionType
	KeyRepetitionType   parquet.FieldRepetitionType
	ValueRepetitionType parquet.FieldRepetitionType

	LogicalTypeFields      map[string]string
	KeyLogicalTypeFields   map[string]string
	ValueLogicalTypeFields map[string]string
}

`parquet:"name=Name, type=FIXED_LEN_BYTE_ARRAY, length=12"`

func GetKeyTagMap

func GetKeyTagMap(src *Tag) *Tag

Get key tag map for map

func GetValueTagMap

func GetValueTagMap(src *Tag) *Tag

Get value tag map for map

func NewTag

func NewTag() *Tag

func StringToTag

func StringToTag(tag string) (*Tag, error)