schema

package
v0.0.24 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 12, 2024 License: Apache-2.0 Imports: 9 Imported by: 65

Documentation

Overview

Package schema defines schema of YT tables.

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func CompareRows 

func CompareRows(a, b []any, s *Schema) int

CompareRows orders two set of row keys according to schema.

Keys may contain:

  • nil
  • bool
  • string
  • []byte
  • float32, float64
  • int, int32, int64, int16, int8
  • uint, uint32, uint64, uint16, uint8

Passing any other value to this function results in panic.

Currently, this function does not handle NaN-s correctly.

The result will be 0 if a==b, -1 if a < b, and +1 if a > b.

Types

type AggregateFunction 

type AggregateFunction string
const (
	AggregateSum AggregateFunction = "sum"
	AggregateMin AggregateFunction = "min"
	AggregateMax AggregateFunction = "max"
)

func (AggregateFunction) MarshalText 

func (a AggregateFunction) MarshalText() (text []byte, err error)

func (*AggregateFunction) UnmarshalText 

func (a *AggregateFunction) UnmarshalText(text []byte) error

type Any 

type Any any

type Column 

type Column struct {
	Name string `yson:"name"`

	// Type and Required represent "legacy" row schema.
	//
	// When creating Column fill either Type and Required, or ComplexType but not both.
	//
	// When reading schema from YT, both "legacy" and "new" fields are available.
	Type     Type `yson:"type,omitempty"`
	Required bool `yson:"required,omitempty"`

	// ComplexType is "new" row schema.
	//
	//   Type == TypeInt64 && Required == true is equivalent to ComplexType == TypeInt64
	//   Type == TypeInt64 && Required == false is equivalent to ComplexType == Optional{Item: TypeInt64}
	ComplexType ComplexType `yson:"type_v3,omitempty"`

	SortOrder SortOrder `yson:"sort_order,omitempty"`

	// Lock specifies lock group.
	//
	// Used by the MVCC mechanism of dynamic tables. Columns from different lock groups might be modified concurrently.
	Lock       string            `yson:"lock,omitempty"`
	Expression string            `yson:"expression,omitempty"`
	Aggregate  AggregateFunction `yson:"aggregate,omitempty"`

	// Group specifies column group. When using columnar format (optimize_for=scan), columns groups are stored together.
	Group string `yson:"group,omitempty"`
}

Column specifies schema of a single column.

See https://yt.yandex-team.ru/docs/description/storage/static_schema

func (Column) MarshalYSON 

func (c Column) MarshalYSON(w *yson.Writer) error

func (Column) NormalizeType 

func (c Column) NormalizeType() Column

type ComplexType 

type ComplexType interface {
	// contains filtered or unexported methods
}

type Date 

type Date uint64

Date is YT type representing number of days since beginning of the unix epoch.

func NewDate 

func NewDate(t time.Time) (Date, error)

func (Date) Time 

func (t Date) Time() time.Time

type Datetime 

type Datetime uint64

Datetime is YT type representing number of seconds since beginning of the unix epoch.

func NewDatetime 

func NewDatetime(t time.Time) (Datetime, error)

func (Datetime) Time 

func (t Datetime) Time() time.Time

type Decimal 

type Decimal struct {
	Precision int `yson:"precision"`
	Scale     int `yson:"scale"`
}

func (Decimal) MarshalYSON 

func (d Decimal) MarshalYSON(w *yson.Writer) error

type Dict 

type Dict struct {
	Key   ComplexType `yson:"key"`
	Value ComplexType `yson:"value"`
}

func (Dict) MarshalYSON 

func (d Dict) MarshalYSON(w *yson.Writer) error

type Interval 

type Interval int64

Interval is YT type representing distance between two Timestamps-s in microseconds.

func NewInterval 

func NewInterval(d time.Duration) (Interval, error)

func (Interval) Duration 

func (i Interval) Duration() time.Duration

type List 

type List struct {
	Item ComplexType `yson:"item"`
}

func (List) MarshalYSON 

func (l List) MarshalYSON(w *yson.Writer) error

type Optional 

type Optional struct {
	Item ComplexType `yson:"item"`
}

func (Optional) MarshalYSON 

func (o Optional) MarshalYSON(w *yson.Writer) error

type RangeError 

type RangeError struct {
	MinValue any
	MaxValue any
}

func (RangeError) Error 

func (r RangeError) Error() string

type Schema 

type Schema struct {
	// Schema is strict by default. Change this option only if you know that you are doing.
	Strict     *bool `yson:"strict,attr,omitempty"`
	UniqueKeys bool  `yson:"unique_keys,attr"`

	Columns []Column `yson:",value"`
}

Schema describe schema of YT table.

Schema is strict by default.

See https://yt.yandex-team.ru/docs/description/storage/static_schema

func Infer 

func Infer(value any) (s Schema, err error)

Infer infers Schema from go struct.

By default Infer creates column for each struct field.

Column name is inferred from the value of yson tag, by the usual rules of the yson.Marshal.

`yson:",key"` tag corresponds to sort_order=ascending scheme attribute.

`ytgroup:""` tag is used to fill "group" scheme attribute.

Type of the column corresponds to go type of the field.

Mapping between go and YT types is defined as follows:

Go types int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64 and bool are mapped to equivalent YT types. NOTE: int8 and uint8 types are not supported.

Go type []byte is mapped to YT type string.

Go types implementing encoding.BinaryMarshaler interface are mapper to YT type string.

Go type string is mapped to YT type utf8.

Go types implementing encoding.TextMarshaler interface are mapped to YT type utf8.

Example 
package main

import (
	"fmt"

	"go.ytsaurus.tech/yt/go/schema"
)

type MyStruct struct {
	IntColumn    int
	AnyValue     any
	Optional     *int
	StringColumn string `yson:"custom_column_name"`
	SkipMe       int    `yson:"-"`
}

var MySchema = schema.MustInfer(&MyStruct{})

func main() {
	for _, column := range MySchema.Columns {
		var kind string
		if column.Required {
			kind = "required"
		} else {
			kind = "optional"
		}

		fmt.Printf("%q is %s column of type %s\n", column.Name, kind, column.Type)
	}

}

Output:

"IntColumn" is required column of type int64
"AnyValue" is optional column of type any
"Optional" is optional column of type int64
"custom_column_name" is required column of type utf8

func InferMap 

func InferMap(value any) (s Schema, err error)

InferMap infers Schema from go map[string]any.

InferMap creates column for each key value pair. Column name inferred from key itself, and column type inferred from the type of value.

To avoid ambiguity key type should always be string, while value type doesn't matter.

func MergeSchemas 

func MergeSchemas(lhs, rhs Schema) Schema

MergeSchemas merges two schemas inferred from rows

If column have different types, then result column will be of type Any. Result schema does not have key columns, and columns have following order: columns from `lhs` schema, then columns from `rhs` schema that does not exist in `lhs` schema.

func MustInfer 

func MustInfer(value any) Schema

MustInfer infers Schema from go struct.

MustInfer panics on errors.

func MustInferMap 

func MustInferMap(value any) (s Schema)

MustInferMap infers Schema from go map[string]any.

MustInferMap panics on errors.

func (Schema) Append 

func (s Schema) Append(column ...Column) Schema

func (Schema) Copy 

func (s Schema) Copy() Schema

func (Schema) Equal 

func (s Schema) Equal(other Schema) bool

func (*Schema) IsStrict 

func (s *Schema) IsStrict() bool

func (Schema) KeyColumns 

func (s Schema) KeyColumns() []string

KeyColumns returns list of columns names marked as sorted.

func (Schema) Normalize 

func (s Schema) Normalize() Schema

Normalize converts in-memory Schema to canonical representation.

  • Strict flag is explicitly set to %true.
  • Old types are converted to new types.

func (Schema) Prepend 

func (s Schema) Prepend(column ...Column) Schema

func (Schema) SortedBy 

func (s Schema) SortedBy(keyColumns ...string) Schema

SortedBy returns copy of the schema, with keyColumns (and only them) marked as sorted.

func (Schema) WithUniqueKeys 

func (s Schema) WithUniqueKeys() Schema

WithUniqueKeys returns copy of schema with UniqueKeys attribute set.

type SortOrder 

type SortOrder string
const (
	SortNone       SortOrder = ""
	SortAscending  SortOrder = "ascending"
	SortDescending SortOrder = "descending"
)

func (SortOrder) MarshalText 

func (o SortOrder) MarshalText() (text []byte, err error)

func (*SortOrder) UnmarshalText 

func (o *SortOrder) UnmarshalText(text []byte) error

type Struct 

type Struct struct {
	Members []StructMember `yson:"members"`
}

func (Struct) MarshalYSON 

func (s Struct) MarshalYSON(w *yson.Writer) error

type StructMember 

type StructMember struct {
	Name string      `yson:"name"`
	Type ComplexType `yson:"type"`
}

func (StructMember) MarshalYSON 

func (m StructMember) MarshalYSON(w *yson.Writer) error

type Tagged 

type Tagged struct {
	Tag  string      `yson:"tag"`
	Item ComplexType `yson:"item"`
}

func (Tagged) MarshalYSON 

func (t Tagged) MarshalYSON(w *yson.Writer) error

type Timestamp 

type Timestamp uint64

Timestamp is YT type representing number of microseconds since beginning of the unix epoch.

func NewTimestamp 

func NewTimestamp(t time.Time) (Timestamp, error)

func (Timestamp) Time 

func (t Timestamp) Time() time.Time

type Tuple 

type Tuple struct {
	Elements []TupleElement `yson:"elements"`
}

func (Tuple) MarshalYSON 

func (t Tuple) MarshalYSON(w *yson.Writer) error

type TupleElement 

type TupleElement struct {
	Type ComplexType `yson:"type"`
}

func (TupleElement) MarshalYSON 

func (e TupleElement) MarshalYSON(w *yson.Writer) error

type Type 

type Type string
const (
	TypeInt64   Type = "int64"
	TypeInt32   Type = "int32"
	TypeInt16   Type = "int16"
	TypeInt8    Type = "int8"
	TypeUint64  Type = "uint64"
	TypeUint32  Type = "uint32"
	TypeUint16  Type = "uint16"
	TypeUint8   Type = "uint8"
	TypeFloat32 Type = "float"
	TypeFloat64 Type = "double"

	TypeBytes  Type = "string"
	TypeString Type = "utf8"

	// NOTE: "boolean" was renamed into "bool" in type_v3, and "any" was renamed into "yson"
	//
	// We keep old in-memory representation for this types for compatibility reasons.
	// Code that only operates with constants provided by this package should not care.
	TypeBoolean Type = "boolean"

	TypeAny Type = "any"

	TypeDate      Type = "date"
	TypeDatetime  Type = "datetime"
	TypeTimestamp Type = "timestamp"
	TypeInterval  Type = "interval"
)

func (Type) MarshalText 

func (t Type) MarshalText() (text []byte, err error)

func (Type) String 

func (t Type) String() string

func (*Type) UnmarshalText 

func (t *Type) UnmarshalText(text []byte) error

type TypeName 

type TypeName string
const (
	TypeNameDecimal  TypeName = "decimal"
	TypeNameOptional TypeName = "optional"
	TypeNameList     TypeName = "list"
	TypeNameStruct   TypeName = "struct"
	TypeNameTuple    TypeName = "tuple"
	TypeNameVariant  TypeName = "variant"
	TypeNameDict     TypeName = "dict"
	TypeNameTagged   TypeName = "tagged"
)

type Variant 

type Variant struct {
	Members  []StructMember `yson:"members"`
	Elements []TupleElement `yson:"elements"`
}

func (Variant) MarshalYSON 

func (v Variant) MarshalYSON(w *yson.Writer) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.