block

Documentation

Index

• func Read(buffer []byte, desiredSchema typeof.Schema) (column.Columns, error)
• func Transform(filter *typeof.Schema, computed ...computed.Computed) applyFunc
• type Block
  • func FromBatchBy(batch *talaria.Batch, partitionBy string, filter *typeof.Schema, ...) ([]Block, error)
  • func FromBuffer(b []byte) (block Block, err error)
  • func FromCSVBy(input []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)
  • func FromColumns(key string, columns column.Columns) (blk Block, err error)
  • func FromOrcBy(payload []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)
  • func FromParquetBy(payload []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)
  • func FromRequestBy(request *talaria.IngestRequest, partitionBy string, filter *typeof.Schema, ...) ([]Block, error)
  • func FromRequestWithTable(request *talaria.IngestWithTableRequest, partitionBy string, ...) ([]Block, error)
  • func FromURLBy(uri string, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)
  • func (b *Block) Encode() ([]byte, error)
  • func (b *Block) LastRow() (map[string]interface{}, error)
  • func (b *Block) Min(column string) (int64, bool)
  • func (b *Block) Schema() typeof.Schema
  • func (b *Block) Select(columns typeof.Schema) (column.Columns, error)
• type Row
  • func FromBlockBy(blk Block, schema typeof.Schema) ([]Row, error)
  • func NewRow(schema typeof.Schema, capacity int) Row
  • func (r Row) AppendTo(cols column.Columns) (size int)
  • func (r Row) Set(k string, v interface{})

Functions

func Read

func Read(buffer []byte, desiredSchema typeof.Schema) (column.Columns, error)

Read decodes the block and selects the columns

func Transform

func Transform(filter *typeof.Schema, computed ...computed.Computed) applyFunc

Transform runs the computed Values and overwrites/appends them to the set.

Types

type Block

type Block struct {
	Size    int64          // The unencoded size of the block
	Key     nocopy.String  // The key of the block
	Columns nocopy.ByteMap // The encoded column metadata
	Data    nocopy.Bytes   // The set of columnar data
	Expires int64          // The expiration time for the block, in unix seconds
	// contains filtered or unexported fields
}

Block represents a serialized block

func FromBatchBy

func FromBatchBy(batch *talaria.Batch, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)

FromBatchBy creates a block from a talaria protobuf-encoded batch. It repartitions the batch by a given partition key at the same time.

func FromBuffer

func FromBuffer(b []byte) (block Block, err error)

FromBuffer unmarshals a block from a in-memory buffer.

func FromCSVBy

func FromCSVBy(input []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)

FromCSVBy creates a block from a comma-separated file. It repartitions the batch by a given partition key at the same time.

func FromColumns

func FromColumns(key string, columns column.Columns) (blk Block, err error)

FromColumns creates a block from a set of presto named columns

func FromOrcBy

func FromOrcBy(payload []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)

FromOrcBy decodes a set of blocks from an orc file and repartitions it by the specified partition key.

func FromParquetBy

func FromParquetBy(payload []byte, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)

FromParquetBy decodes a set of blocks from a Parquet file and repartitions it by the specified partition key.

func FromRequestBy

func FromRequestBy(request *talaria.IngestRequest, partitionBy string, filter *typeof.Schema, funcs ...applyFunc) ([]Block, error)

FromRequestBy creates a block from a talaria protobuf-encoded request. It repartitions the batch by a given partition key at the same time.

func FromRequestWithTable

func FromRequestWithTable(request *talaria.IngestWithTableRequest, partitionBy string, filter *typeof.Schema, funcs ...applyFunc) ([]Block, error)

FromRequestByTable creates a block from a talaria protobuf-encoded request. It repartitions the batch by a given partition key at the same time.

func FromURLBy

func FromURLBy(uri string, partitionBy string, filter *typeof.Schema, apply applyFunc) ([]Block, error)

FromURLBy creates a block from a remote url which should be loaded. It repartitions the batch by a given partition key at the same time.

func (*Block) Encode

func (b *Block) Encode() ([]byte, error)

Encode encodes the block as bytes

func (*Block) LastRow

func (b *Block) LastRow() (map[string]interface{}, error)

LastRow returns the last row of the block

func (*Block) Min

func (b *Block) Min(column string) (int64, bool)

Min selects the smallest value for a column (must be an integer or a bigint)

func (*Block) Schema

func (b *Block) Schema() typeof.Schema

Schema returns a schema of the block.

func (*Block) Select

func (b *Block) Select(columns typeof.Schema) (column.Columns, error)

Select selects a set of thrift columns

type Row

type Row struct {
	Values map[string]interface{}
	Schema typeof.Schema
}

Row represents a single row on which we can perform transformations

func FromBlockBy

func FromBlockBy(blk Block, schema typeof.Schema) ([]Row, error)

FromBlockBy creates and returns a list of new block.Row for a block.

func NewRow

func NewRow(schema typeof.Schema, capacity int) Row

NewRow creates a new row with a schema and a capacity

func (Row) AppendTo

func (r Row) AppendTo(cols column.Columns) (size int)

AppendTo appends the entire row to the column set

func (Row) Set

func (r Row) Set(k string, v interface{})

Set sets the key/value pair