gbigtable

Documentation

Index

• Constants
• Variables
• func NewLoaderFactory(ctx context.Context, config Config) (entity.LoaderFactory, error)
• type BigTableAdminClient
• type BigTableClient
• type BigTableTable
• type ColumnFamily
• type ColumnQualifier
• type Config
• type GarbageCollectionPolicy
• type NameFromId
• type Query
• type QueryType
• type RowKey
• type RowKeyValue
• type SinkConfig
  • func NewSinkConfig(spec *entity.Spec) (sc SinkConfig, err error)
• type Table
• type Whitelist

Constants

const (
	PreDefinedRowKeyTimestampIso      = "timestampIso"
	PreDefinedRowKeyUuid              = "uuid"
	PreDefinedRowKeyInvertedTimestamp = "invertedTimestamp"
	PreDefinedRowKeyKeysInMap         = "keysInMap"

	GarbageCollectionPolicyMaxVersions = "maxVersions"
	GarbageCollectionPolicyMaxAge      = "maxAge"
)

Variables

var ErrNotApplicable = errors.New("not applicable")

Functions

func NewLoaderFactory

func NewLoaderFactory(ctx context.Context, config Config) (entity.LoaderFactory, error)

Types

type BigTableAdminClient

type BigTableAdminClient interface {
	Tables(ctx context.Context) ([]string, error)
	CreateTable(ctx context.Context, table string) error
	TableInfo(ctx context.Context, table string) (*bigtable.TableInfo, error)
	CreateColumnFamily(ctx context.Context, table, family string) error
	SetGCPolicy(ctx context.Context, table, family string, policy bigtable.GCPolicy) error
	Close() error
}

type BigTableClient

type BigTableClient interface {
	Open(table string) *bigtable.Table
	Close() error
}

type BigTableTable

type BigTableTable interface {
	Apply(ctx context.Context, row string, m *bigtable.Mutation, opts ...bigtable.ApplyOption) (err error)

	// A missing row will return a zero-length map and a nil error.
	ReadRow(ctx context.Context, row string, opts ...bigtable.ReadOption) (bigtable.Row, error)
}

type ColumnFamily

type ColumnFamily struct {
	Name                    string                   `json:"name"`
	GarbageCollectionPolicy *GarbageCollectionPolicy `json:"garbageCollectionPolicy"`
	ColumnQualifiers        []ColumnQualifier        `json:"columnQualifiers"`
}

type ColumnQualifier

type ColumnQualifier struct {
	Id         string      `json:"id"`
	Name       string      `json:"name,omitempty"`
	NameFromId *NameFromId `json:"nameFromId,omitempty"`
}

The Id field can be used directly in the Transformed map to fetch the value to be inserted The Name field is the actual CQ name to be used in the table. Either Name or NameFromId must be present, not both.

type Config

type Config struct {

	// ProjectId (required) specifies GCP project ID for this deployment.
	ProjectId string

	// InstanceId (required) specifies BigTable instance ID
	InstanceId string
}

type GarbageCollectionPolicy

type GarbageCollectionPolicy struct {
	Type  string `json:"type"`
	Value int    `json:"value"`
}

TODO: Add support for Intersection and Union policies The following types are supported: - MaxVersions: where Value takes an integer of number of old versions to keep (-1) - MaxAge: where Value takes an integer of number of hours before deleting the data.

type NameFromId

type NameFromId struct {
	Prefix       string `json:"prefix"`
	SuffixFromId string `json:"suffixFromId"`
}

Creates a Column/CQ name from id outputs in transloaded event map

type Query

type Query struct {
	Type    QueryType
	Table   string
	RowKey  string
	LatestN int
}

type QueryType

type QueryType int

const (
	InvalidQueryType QueryType = iota
	KeyValue
	LatestN
	All
)

type RowKey

type RowKey struct {
	Predefined string   `json:"predefined,omitempty"`
	Keys       []string `json:"keys,omitempty"`
	Delimiter  string   `json:"delimiter,omitempty"`

	// Only required when using the Predefined rowkey option "keysInMap". This id should map to the transformed
	// output map item specified in ExtractItemsFromArray.Id
	MapId string `json:"mapId,omitempty"`
}

RowKey specifies how the row-key should be generated for BigTable sinks. If one of the Predefined options are set, that will be used. Currently available Predefined options are:

"timestampIso"
"invertedTimestamp"
"uuid"
"keysInMap"

If Predefined is not set, the Keys array should be used to specify which extracted fields from the event should be used. TODO: Add padding config

type RowKeyValue

type RowKeyValue struct {
	RowKey string
	Value  []byte
}

type SinkConfig

type SinkConfig struct {
	Tables []Table `json:"tables,omitempty"`
}

SinkConfig specifies the schema for the "customConfig" field in the "sink" section of the stream spec. It enables arbitrary connector specific fields to be present in the stream spec.

func NewSinkConfig

func NewSinkConfig(spec *entity.Spec) (sc SinkConfig, err error)

type Table

type Table struct {
	Name string `json:"name"`

	// Table spec for BigTable are built up by RowKey and ColumnFamilies
	RowKey         RowKey         `json:"rowKey"`
	ColumnFamilies []ColumnFamily `json:"columnFamilies"`

	// Only input transformations satisfying the whitelist key/value filter will be
	// processed by the sink (mostly needed in multi-table Sink specs)
	Whitelist *Whitelist `json:"whitelist,omitempty"`
}

type Whitelist

type Whitelist struct {
	Id     string   `json:"id"`
	Type   string   `json:"type"`
	Values []string `json:"values"`
}