Documentation ¶
Index ¶
- Constants
- Variables
- func BuildPositionAccessors(schema *Schema) (map[int]*Accessor, error)
- func IndexByID(schema *Schema) (map[int]NestedField, error)
- func IndexByName(schema *Schema) (map[string]int, error)
- func IndexNameByID(schema *Schema) (map[int]string, error)
- func Visit[T any](sc *Schema, visitor SchemaVisitor[T]) (res T, err error)
- type Accessor
- type AfterFieldVisitor
- type AfterListElementVisitor
- type AfterMapKeyVisitor
- type AfterMapValueVisitor
- type BeforeFieldVisitor
- type BeforeListElementVisitor
- type BeforeMapKeyVisitor
- type BeforeMapValueVisitor
- type BinaryType
- type BooleanType
- type BucketTransform
- type DataFile
- type Date
- type DateType
- type DayTransform
- type DecimalType
- type FileFormat
- type FixedType
- type Float32Type
- type Float64Type
- type HourTransform
- type IdentityTransform
- type Int32Type
- type Int64Type
- type ListType
- type ManifestContent
- type ManifestEntry
- type ManifestEntryContent
- type ManifestEntryStatus
- type ManifestFile
- type MapType
- type MonthTransform
- type NestedField
- type NestedType
- type PartitionField
- type PartitionSpec
- func (ps *PartitionSpec) CompatibleWith(other *PartitionSpec) bool
- func (ps *PartitionSpec) Equals(other PartitionSpec) bool
- func (ps *PartitionSpec) Field(i int) PartitionField
- func (ps *PartitionSpec) FieldsBySourceID(fieldID int) []PartitionField
- func (ps *PartitionSpec) ID() int
- func (ps *PartitionSpec) IsUnpartitioned() bool
- func (ps *PartitionSpec) LastAssignedFieldID() int
- func (ps PartitionSpec) MarshalJSON() ([]byte, error)
- func (ps *PartitionSpec) NumFields() int
- func (ps *PartitionSpec) PartitionType(schema *Schema) StructType
- func (ps PartitionSpec) String() string
- func (ps *PartitionSpec) UnmarshalJSON(b []byte) error
- type PrimitiveType
- type Properties
- type Row
- type Schema
- func (s *Schema) AccessorForField(id int) (*Accessor, error)
- func (s *Schema) AsStruct() StructType
- func (s *Schema) Children() []NestedField
- func (s *Schema) Equals(other *Schema) bool
- func (s *Schema) Field(i int) NestedField
- func (s *Schema) FindColumnName(fieldID int) (string, bool)
- func (s *Schema) FindFieldByID(id int) (NestedField, bool)
- func (s *Schema) FindFieldByName(name string) (NestedField, bool)
- func (s *Schema) FindFieldByNameCaseInsensitive(name string) (NestedField, bool)
- func (s *Schema) FindTypeByID(id int) (Type, bool)
- func (s *Schema) FindTypeByName(name string) (Type, bool)
- func (s *Schema) FindTypeByNameCaseInsensitive(name string) (Type, bool)
- func (s *Schema) HighestFieldID() int
- func (s *Schema) MarshalJSON() ([]byte, error)
- func (s *Schema) NumFields() int
- func (s *Schema) Select(caseSensitive bool, names ...string) (*Schema, error)
- func (s *Schema) Type() string
- func (s *Schema) UnmarshalJSON(b []byte) error
- type SchemaVisitor
- type StringType
- type StructType
- type Time
- type TimeType
- type Timestamp
- type TimestampType
- type TimestampTzType
- type Transform
- type TruncateTransform
- type Type
- type UUIDType
- type VoidTransform
- type YearTransform
Constants ¶
View Source
const (
InitialPartitionSpecID = 0
)
View Source
const Version = "0.1.0"
Variables ¶
View Source
var ( ErrInvalidTypeString = errors.New("invalid type") ErrNotImplemented = errors.New("not implemented") ErrInvalidArgument = errors.New("invalid argument") )
View Source
var (
ErrInvalidSchema = errors.New("invalid schema")
)
View Source
var (
ErrInvalidTransform = errors.New("invalid transform syntax")
)
View Source
var PositionalDeleteSchema = NewSchema(0, NestedField{ID: 2147483546, Type: PrimitiveTypes.String, Name: "file_path", Required: true}, NestedField{ID: 2147483545, Type: PrimitiveTypes.Int32, Name: "pos", Required: true}, )
View Source
var PrimitiveTypes = struct { Bool Type Int32 Type Int64 Type Float32 Type Float64 Type Date Type Time Type Timestamp Type TimestampTz Type String Type Binary Type UUID Type }{ Bool: BooleanType{}, Int32: Int32Type{}, Int64: Int64Type{}, Float32: Float32Type{}, Float64: Float64Type{}, Date: DateType{}, Time: TimeType{}, Timestamp: TimestampType{}, TimestampTz: TimestampTzType{}, String: StringType{}, Binary: BinaryType{}, UUID: UUIDType{}, }
View Source
var (
UnpartitionedPartitionSpec = &PartitionSpec{id: 0}
)
Functions ¶
func BuildPositionAccessors ¶
Types ¶
type AfterFieldVisitor ¶
type AfterFieldVisitor interface {
AfterField(field NestedField)
}
type AfterListElementVisitor ¶
type AfterListElementVisitor interface {
AfterListElement(elem NestedField)
}
type AfterMapKeyVisitor ¶
type AfterMapKeyVisitor interface {
AfterMapKey(key NestedField)
}
type AfterMapValueVisitor ¶
type AfterMapValueVisitor interface {
AfterMapValue(value NestedField)
}
type BeforeFieldVisitor ¶
type BeforeFieldVisitor interface {
BeforeField(field NestedField)
}
type BeforeListElementVisitor ¶
type BeforeListElementVisitor interface {
BeforeListElement(elem NestedField)
}
type BeforeMapKeyVisitor ¶
type BeforeMapKeyVisitor interface {
BeforeMapKey(key NestedField)
}
type BeforeMapValueVisitor ¶
type BeforeMapValueVisitor interface {
BeforeMapValue(value NestedField)
}
type BinaryType ¶
type BinaryType struct{}
func (BinaryType) Equals ¶
func (BinaryType) Equals(other Type) bool
func (BinaryType) String ¶
func (BinaryType) String() string
func (BinaryType) Type ¶
func (BinaryType) Type() string
type BooleanType ¶
type BooleanType struct{}
func (BooleanType) Equals ¶
func (BooleanType) Equals(other Type) bool
func (BooleanType) String ¶
func (BooleanType) String() string
func (BooleanType) Type ¶
func (BooleanType) Type() string
type BucketTransform ¶
type BucketTransform struct {
N int
}
func (BucketTransform) MarshalText ¶
func (t BucketTransform) MarshalText() ([]byte, error)
func (BucketTransform) String ¶
func (t BucketTransform) String() string
type DataFile ¶
type DataFile interface { ContentType() ManifestEntryContent FilePath() string FileFormat() FileFormat Partition() map[string]any Count() int64 FileSizeBytes() int64 ColumnSizes() map[int]int64 ValueCounts() map[int]int64 NullValueCounts() map[int]int64 NaNValueCounts() map[int]int64 DistinctValueCounts() map[int]int64 LowerBoundValues() map[int][]byte UpperBoundValues() map[int][]byte MetadataKey() []byte Splits() []int64 EqualityFieldIDs() []int SortOrderID() *int }
type DayTransform ¶
type DayTransform struct{}
func (DayTransform) MarshalText ¶
func (t DayTransform) MarshalText() ([]byte, error)
func (DayTransform) String ¶
func (DayTransform) String() string
type DecimalType ¶
type DecimalType struct {
// contains filtered or unexported fields
}
func DecimalTypeOf ¶
func DecimalTypeOf(prec, scale int) DecimalType
func (DecimalType) Equals ¶
func (d DecimalType) Equals(other Type) bool
func (DecimalType) Precision ¶
func (d DecimalType) Precision() int
func (DecimalType) Scale ¶
func (d DecimalType) Scale() int
func (DecimalType) String ¶
func (d DecimalType) String() string
func (DecimalType) Type ¶
func (d DecimalType) Type() string
type FileFormat ¶
type FileFormat string
const ( AvroFile FileFormat = "AVRO" OrcFile FileFormat = "ORC" ParquetFile FileFormat = "PARQUET" )
type FixedType ¶
type FixedType struct {
// contains filtered or unexported fields
}
func FixedTypeOf ¶
type Float32Type ¶
type Float32Type struct{}
func (Float32Type) Equals ¶
func (Float32Type) Equals(other Type) bool
func (Float32Type) String ¶
func (Float32Type) String() string
func (Float32Type) Type ¶
func (Float32Type) Type() string
type Float64Type ¶
type Float64Type struct{}
func (Float64Type) Equals ¶
func (Float64Type) Equals(other Type) bool
func (Float64Type) String ¶
func (Float64Type) String() string
func (Float64Type) Type ¶
func (Float64Type) Type() string
type HourTransform ¶
type HourTransform struct{}
func (HourTransform) MarshalText ¶
func (t HourTransform) MarshalText() ([]byte, error)
func (HourTransform) String ¶
func (HourTransform) String() string
type IdentityTransform ¶
type IdentityTransform struct{}
func (IdentityTransform) MarshalText ¶
func (t IdentityTransform) MarshalText() ([]byte, error)
func (IdentityTransform) String ¶
func (IdentityTransform) String() string
type ListType ¶
type ListType struct { ElementID int `json:"element-id"` Element Type `json:"element"` ElementRequired bool `json:"element-required"` }
func (*ListType) Children ¶
func (l *ListType) Children() []NestedField
func (*ListType) ElementField ¶
func (l *ListType) ElementField() NestedField
func (*ListType) UnmarshalJSON ¶
type ManifestContent ¶
type ManifestContent int32
const ( ManifestContentData ManifestContent = 0 ManifestContentDeletes ManifestContent = 1 )
type ManifestEntry ¶
type ManifestEntry interface { Status() ManifestEntryStatus SnapshotID() int64 SequenceNum() int64 FileSequenceNum() *int64 DataFile() DataFile // contains filtered or unexported methods }
type ManifestEntryContent ¶
type ManifestEntryContent int8
const ( EntryContentData ManifestEntryContent = 0 EntryContentPosDeletes ManifestEntryContent = 1 EntryContentEqDeletes ManifestEntryContent = 2 )
type ManifestEntryStatus ¶
type ManifestEntryStatus int8
const ( EntryStatusEXISTING ManifestEntryStatus = 0 EntryStatusADDED ManifestEntryStatus = 1 EntryStatusDELETED ManifestEntryStatus = 2 )
type ManifestFile ¶
type ManifestFile interface { Version() int FilePath() string Length() int64 PartitionID() int32 ManifestContent() ManifestContent SnapshotID() int64 AddedDataFiles() int32 ExistingDataFiles() int32 DeletedDataFiles() int32 AddedRows() int64 ExistingRows() int64 DeletedRows() int64 SequenceNum() int64 MinSequenceNum() int64 Metadata() []byte PartitionList() []fieldSummary HasAddedFiles() bool HasExistingFiles() bool FetchEntries(fs iceio.IO, discardDeleted bool) ([]ManifestEntry, error) }
func ReadManifestList ¶
func ReadManifestList(in io.Reader) ([]ManifestFile, error)
type MapType ¶
type MapType struct { KeyID int `json:"key-id"` KeyType Type `json:"key"` ValueID int `json:"value-id"` ValueType Type `json:"value"` ValueRequired bool `json:"value-required"` }
func (*MapType) Children ¶
func (m *MapType) Children() []NestedField
func (*MapType) KeyField ¶
func (m *MapType) KeyField() NestedField
func (*MapType) UnmarshalJSON ¶
func (*MapType) ValueField ¶
func (m *MapType) ValueField() NestedField
type MonthTransform ¶
type MonthTransform struct{}
func (MonthTransform) MarshalText ¶
func (t MonthTransform) MarshalText() ([]byte, error)
func (MonthTransform) String ¶
func (MonthTransform) String() string
type NestedField ¶
type NestedField struct { Type `json:"-"` ID int `json:"id"` Name string `json:"name"` Required bool `json:"required"` Doc string `json:"doc,omitempty"` InitialDefault any `json:"initial-default,omitempty"` WriteDefault any `json:"write-default,omitempty"` }
func (*NestedField) Equals ¶
func (n *NestedField) Equals(other NestedField) bool
func (*NestedField) MarshalJSON ¶
func (n *NestedField) MarshalJSON() ([]byte, error)
func (NestedField) String ¶
func (n NestedField) String() string
func (*NestedField) UnmarshalJSON ¶
func (n *NestedField) UnmarshalJSON(b []byte) error
type NestedType ¶
type NestedType interface { Type Children() []NestedField }
type PartitionField ¶
type PartitionField struct { SourceID int `json:"source-id"` FieldID int `json:"field-id"` Name string `json:"name"` Transform Transform `json:"transform"` }
func (*PartitionField) String ¶
func (p *PartitionField) String() string
func (*PartitionField) UnmarshalJSON ¶
func (p *PartitionField) UnmarshalJSON(b []byte) error
type PartitionSpec ¶
type PartitionSpec struct {
// contains filtered or unexported fields
}
func NewPartitionSpec ¶
func NewPartitionSpec(fields ...PartitionField) PartitionSpec
func NewPartitionSpecID ¶
func NewPartitionSpecID(id int, fields ...PartitionField) PartitionSpec
func (*PartitionSpec) CompatibleWith ¶
func (ps *PartitionSpec) CompatibleWith(other *PartitionSpec) bool
func (*PartitionSpec) Equals ¶
func (ps *PartitionSpec) Equals(other PartitionSpec) bool
func (*PartitionSpec) Field ¶
func (ps *PartitionSpec) Field(i int) PartitionField
func (*PartitionSpec) FieldsBySourceID ¶
func (ps *PartitionSpec) FieldsBySourceID(fieldID int) []PartitionField
func (*PartitionSpec) ID ¶
func (ps *PartitionSpec) ID() int
func (*PartitionSpec) IsUnpartitioned ¶
func (ps *PartitionSpec) IsUnpartitioned() bool
func (*PartitionSpec) LastAssignedFieldID ¶
func (ps *PartitionSpec) LastAssignedFieldID() int
func (PartitionSpec) MarshalJSON ¶
func (ps PartitionSpec) MarshalJSON() ([]byte, error)
func (*PartitionSpec) NumFields ¶
func (ps *PartitionSpec) NumFields() int
func (*PartitionSpec) PartitionType ¶
func (ps *PartitionSpec) PartitionType(schema *Schema) StructType
func (PartitionSpec) String ¶
func (ps PartitionSpec) String() string
func (*PartitionSpec) UnmarshalJSON ¶
func (ps *PartitionSpec) UnmarshalJSON(b []byte) error
type PrimitiveType ¶
type PrimitiveType interface { Type // contains filtered or unexported methods }
type Properties ¶
type Schema ¶
type Schema struct { ID int `json:"schema-id"` IdentifierFieldIDs []int `json:"identifier-field-ids"` // contains filtered or unexported fields }
func NewSchema ¶
func NewSchema(id int, fields ...NestedField) *Schema
func NewSchemaWithIdentifiers ¶
func NewSchemaWithIdentifiers(id int, identifierIDs []int, fields ...NestedField) *Schema
func PruneColumns ¶
func (*Schema) AsStruct ¶
func (s *Schema) AsStruct() StructType
func (*Schema) Children ¶
func (s *Schema) Children() []NestedField
func (*Schema) Field ¶
func (s *Schema) Field(i int) NestedField
func (*Schema) FindFieldByID ¶
func (s *Schema) FindFieldByID(id int) (NestedField, bool)
func (*Schema) FindFieldByName ¶
func (s *Schema) FindFieldByName(name string) (NestedField, bool)
func (*Schema) FindFieldByNameCaseInsensitive ¶
func (s *Schema) FindFieldByNameCaseInsensitive(name string) (NestedField, bool)
func (*Schema) FindTypeByNameCaseInsensitive ¶
func (*Schema) HighestFieldID ¶
func (*Schema) MarshalJSON ¶
func (*Schema) UnmarshalJSON ¶
type SchemaVisitor ¶
type SchemaVisitor[T any] interface { Schema(schema *Schema, structResult T) T Struct(st StructType, fieldResults []T) T Field(field NestedField, fieldResult T) T List(list ListType, elemResult T) T Map(mapType MapType, keyResult, valueResult T) T Primitive(p PrimitiveType) T }
type StringType ¶
type StringType struct{}
func (StringType) Equals ¶
func (StringType) Equals(other Type) bool
func (StringType) String ¶
func (StringType) String() string
func (StringType) Type ¶
func (StringType) Type() string
type StructType ¶
type StructType struct {
Fields []NestedField `json:"fields"`
}
func (*StructType) Children ¶
func (s *StructType) Children() []NestedField
func (*StructType) Equals ¶
func (s *StructType) Equals(other Type) bool
func (*StructType) MarshalJSON ¶
func (s *StructType) MarshalJSON() ([]byte, error)
func (*StructType) String ¶
func (s *StructType) String() string
func (*StructType) Type ¶
func (*StructType) Type() string
type TimestampType ¶
type TimestampType struct{}
func (TimestampType) Equals ¶
func (TimestampType) Equals(other Type) bool
func (TimestampType) String ¶
func (TimestampType) String() string
func (TimestampType) Type ¶
func (TimestampType) Type() string
type TimestampTzType ¶
type TimestampTzType struct{}
func (TimestampTzType) Equals ¶
func (TimestampTzType) Equals(other Type) bool
func (TimestampTzType) String ¶
func (TimestampTzType) String() string
func (TimestampTzType) Type ¶
func (TimestampTzType) Type() string
type Transform ¶
type Transform interface { fmt.Stringer encoding.TextMarshaler }
func ParseTransform ¶
type TruncateTransform ¶
type TruncateTransform struct {
W int
}
func (TruncateTransform) MarshalText ¶
func (t TruncateTransform) MarshalText() ([]byte, error)
func (TruncateTransform) String ¶
func (t TruncateTransform) String() string
type VoidTransform ¶
type VoidTransform struct{}
func (VoidTransform) MarshalText ¶
func (t VoidTransform) MarshalText() ([]byte, error)
func (VoidTransform) String ¶
func (VoidTransform) String() string
type YearTransform ¶
type YearTransform struct{}
func (YearTransform) MarshalText ¶
func (t YearTransform) MarshalText() ([]byte, error)
func (YearTransform) String ¶
func (YearTransform) String() string
Click to show internal directories.
Click to hide internal directories.