mapping

Documentation

Overview

Package mapping provides the API to convert data coming from Big Table into a data.Set.

The mapping system is based on a set of rules described inside a JSON mapping file, here's an example:

{
  "raws": {
	"ui": "user_id"
  },
  "mapped": {
	"oi": {
	  "name": "is_opted_in",
	  "values": {
		"0": "false",
		"1": "true"
	  }
	}
  },
  "reversed": [
	{
	  "name": "order_status",
	  "values": {
		"1": "pending_payment",
		"2": "failed",
		"3": "processing",
		"4": "completed",
		"5": "on_hold",
		"6": "canceled",
		"7": "refunded"
	  }
	}
  ]
}

The mapping has 3 sections:

• raws: only the column qualifier is translated from its short version to a meaningful name.
• mapped: the column qualifier is translated from its short version to a meaningful name, and the value is translated from its short-value to the full value.
• reversed: the column qualifier contains the real data which is mapped to a value as described in the "values" property and the column name is taken from the "name" attribute.

Considering the mapping above, let's say we have a row with the following data coming from Big Table:

{
  "ui": "12345",
  "oi": "1",
  "3": "1"
}

The mapping system will map it to a new data.Event containing the following data:

{
  "user_id": "12345",
  "is_opted_in": "true",
  "order_status": "processing"
}

Index

• func NewGCSBucketGetter(gcreds *GcloudCreds, bucketName string) (*gcsBucketGetter, *storage.Client, error)
• func NewGCSBucketGetterFromEnvironment(bucketName string) (*gcsBucketGetter, *storage.Client, error)
• func NewGCSBucketGetterWithClient(client *storage.Client, bucketName string) (*gcsBucketGetter, error)
• func Unwrap(err error) error
• func UnwrapAll(err error) error
• type GcloudCreds
• type Map
• type Mapper
  • func NewMapper(mapping *Mapping) *Mapper
  • func (m *Mapper) GetMappedEvents(items []bigtable.ReadItem) ([]string, []*data.Event)
  • func (m *Mapper) GetMutations(eventSet *data.Set) map[string]*bigtable.Mutation
• type Mapping
  • func LoadMapping(c []byte) (*Mapping, error)
  • func LoadMappingFromFile(path string) (*Mapping, error)
  • func LoadMappingIO(reader io.ReadCloser) (*Mapping, error)
  • func LoadMappingVersion(c []byte, version string) (*Mapping, error)
• type Reader
  • func NewReader(gcreds *GcloudCreds, bucketName string) (*Reader, *storage.Client, error)
  • func (r *Reader) Load(ctx context.Context, eventFamily string, version string, environment string) (*Mapping, error)
• type Writer
  • func NewWriter(gcreds *GcloudCreds, bucketName string) (*Writer, *storage.Client, error)
  • func NewWriterFromGCSClient(gbSW func(ctx context.Context, fileName string) io.WriteCloser, ...) (*Writer, error)
  • func (w *Writer) Upload(ctx context.Context, eventFamily, version string, environment string, ...) error

Functions

func NewGCSBucketGetter

func NewGCSBucketGetter(gcreds *GcloudCreds, bucketName string) (*gcsBucketGetter, *storage.Client, error)

func NewGCSBucketGetterFromEnvironment

func NewGCSBucketGetterFromEnvironment(bucketName string) (*gcsBucketGetter, *storage.Client, error)

func NewGCSBucketGetterWithClient

func NewGCSBucketGetterWithClient(client *storage.Client, bucketName string) (*gcsBucketGetter, error)

func Unwrap

func Unwrap(err error) error

Unwrap calls std_errors.Unwrap.

func UnwrapAll

func UnwrapAll(err error) error

UnwrapAll unwraps all nested errors, and returns the last one.

Types

type GcloudCreds

type GcloudCreds struct {
	Type         string `json:"type"`
	ProjectID    string `json:"project_id"`
	PrivateKeyID string `json:"private_key_id"`
	PrivateKey   string `json:"private_key"`
	ClientEmail  string `json:"client_email"`
	ClientID     string `json:"client_id"`
	AuthURI      string `json:"auth_uri"`
	TokenURI     string `json:"token_uri"`
	AuthProvider string `json:"auth_provider_x509_cert_url"`
	CertURL      string `json:"client_x509_cert_url"`
}

type Map

type Map struct {
	Name   string            `json:"name"`
	Values map[string]string `json:"values"`
}

Map is used to map a column to a set of string values.

type Mapper

type Mapper struct {
	// mapping coming from the JSON file
	*Mapping
	// contains filtered or unexported fields
}

Mapper is in charge of translating data from Big Table into a human-readable format.

func NewMapper

func NewMapper(mapping *Mapping) *Mapper

func (*Mapper) GetMappedEvents

func (m *Mapper) GetMappedEvents(items []bigtable.ReadItem) ([]string, []*data.Event)

GetMappedEvents translates a slice of bigtable.ReadItem into a slice of data.Event. It uses the Mapping to know which columns to seek and each event is identified by the timestamp of the bigtable.ReadItem. So assuming there's a slice of 20 bigtable.ReadItem with the same timestamp, then the returned slice will have 1 data.Event containing a slice of 20 Cells.

func (*Mapper) GetMutations

func (m *Mapper) GetMutations(eventSet *data.Set) map[string]*bigtable.Mutation

type Mapping

type Mapping struct {
	// columns that are taken as there are in Big Table
	Raws map[string]string `json:"raws"`
	// columns that needs to be mapped to a string value
	Mapped map[string]Map `json:"mapped"`
	// columns featuring a reversed mapping, meaning that the column qualifier is the data
	Reversed []Map `json:"reversed"`
}

Mapping describes the mapping between data stored in Big Table and its human-readable representation.

func LoadMapping

func LoadMapping(c []byte) (*Mapping, error)

LoadMapping loads a mapping from a slice of bytes. You can use this function if you prefer to open the mapping file yourself.

func LoadMappingFromFile

func LoadMappingFromFile(path string) (*Mapping, error)

LoadMappingFromFile loads a mapping from a file.

func LoadMappingIO

func LoadMappingIO(reader io.ReadCloser) (*Mapping, error)

LoadMappingIO loads a mapping from a IO reader.

func LoadMappingVersion

func LoadMappingVersion(c []byte, version string) (*Mapping, error)

LoadMappingVersion loads a mapping from a slice of bytes and its version. You can use this function if you prefer to open the mapping file yourself.

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func NewReader

func NewReader(gcreds *GcloudCreds, bucketName string) (*Reader, *storage.Client, error)

func (*Reader) Load

func (r *Reader) Load(ctx context.Context, eventFamily string, version string, environment string) (*Mapping, error)

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter(gcreds *GcloudCreds, bucketName string) (*Writer, *storage.Client, error)

func NewWriterFromGCSClient

func NewWriterFromGCSClient(gbSW func(ctx context.Context, fileName string) io.WriteCloser, gbSL func(ctx context.Context, fileName string) (io.ReadCloser, error)) (*Writer, error)

func (*Writer) Upload

func (w *Writer) Upload(ctx context.Context, eventFamily, version string, environment string, writeMapping *Mapping, forceUpload bool) error