skiff

Documentation

Overview

Package skiff implements YT skiff format.

Skiff provides very efficient encoding and decoding, but requires schema negotiation and does not support schema evolution.

Package skiff implements efficient serialization format, optimized for YT.

Index

• type Decoder
  • func NewDecoder(r io.Reader, format Format) (*Decoder, error)
  • func (d *Decoder) Err() error
  • func (d *Decoder) KeySwitch() bool
  • func (d *Decoder) Next() bool
  • func (d *Decoder) RangeIndex() int
  • func (d *Decoder) RowIndex() int64
  • func (d *Decoder) Scan(value any) (err error)
  • func (d *Decoder) TableIndex() int
• type Encoder
  • func NewEncoder(w io.Writer, schema Schema) (*Encoder, error)
  • func (e *Encoder) Flush() error
  • func (e *Encoder) Write(value any) error
  • func (e *Encoder) WriteRow(cols []any) error
• type Format
• type Schema
  • func FromTableSchema(schema schema.Schema) Schema
  • func OptionalColumn(name string, typ WireType) Schema
  • func (c Schema) IsSystem() bool
• type WireType
  • func FromYTType(typ schema.Type) WireType
  • func (t WireType) IsSimple() bool
  • func (t WireType) MarshalYSON(w *yson.Writer) error
  • func (t WireType) String() string
  • func (t *WireType) UnmarshalYSON(data []byte) error

Types

type Decoder

type Decoder struct {
	// contains filtered or unexported fields
}

Decoder reads stream of skiff tuples.

Streaming format is specific for YT job input.

func NewDecoder

func NewDecoder(r io.Reader, format Format) (*Decoder, error)

NewDecoder creates decoder for reading rows from input stream formatted by format.

Each table schema in format must start with three system columns.

func (*Decoder) Err

func (d *Decoder) Err() error

func (*Decoder) KeySwitch

func (d *Decoder) KeySwitch() bool

func (*Decoder) Next

func (d *Decoder) Next() bool

func (*Decoder) RangeIndex

func (d *Decoder) RangeIndex() int

func (*Decoder) RowIndex

func (d *Decoder) RowIndex() int64

func (*Decoder) Scan

func (d *Decoder) Scan(value any) (err error)

Scan unmarshals current record into the value.

func (*Decoder) TableIndex

func (d *Decoder) TableIndex() int

type Encoder

type Encoder struct {
	// contains filtered or unexported fields
}

func NewEncoder

func NewEncoder(w io.Writer, schema Schema) (*Encoder, error)

NewEncoder creates encoder for writing rows into w.

func (*Encoder) Flush

func (e *Encoder) Flush() error

func (*Encoder) Write

func (e *Encoder) Write(value any) error

func (*Encoder) WriteRow

func (e *Encoder) WriteRow(cols []any) error

type Format

type Format struct {
	// name is always equal to string "skiff"
	Name string `yson:",value"`

	// either skiff.Schema of reference into registry
	TableSchemas []any `yson:"table_skiff_schemas,attr"`

	// schemas shared between multiple tables
	SchemaRegistry map[string]*Schema `yson:"skiff_schema_registry,attr"`
}

Format describes skiff schemas for the stream.

type Schema

type Schema struct {
	Type     WireType `yson:"wire_type"`
	Name     string   `yson:"name,omitempty"`
	Children []Schema `yson:"children,omitempty"`
}

Schema describes wire format for the single value.

func FromTableSchema

func FromTableSchema(schema schema.Schema) Schema

FromTableSchema creates skiff schema from table schema.

func OptionalColumn

func OptionalColumn(name string, typ WireType) Schema

func (Schema) IsSystem

func (c Schema) IsSystem() bool

type WireType

type WireType int

const (
	TypeNothing WireType = iota
	TypeBoolean
	TypeInt64
	TypeUint64
	TypeDouble
	TypeString32
	TypeYSON32
	TypeVariant8
	TypeVariant16
	TypeRepeatedVariant16
	TypeTuple
)

func FromYTType

func FromYTType(typ schema.Type) WireType

FromYTType returns skiff wire type used for transferring YT type.

func (WireType) IsSimple

func (t WireType) IsSimple() bool

func (WireType) MarshalYSON

func (t WireType) MarshalYSON(w *yson.Writer) error

func (WireType) String

func (t WireType) String() string

func (*WireType) UnmarshalYSON

func (t *WireType) UnmarshalYSON(data []byte) error