transformer

Documentation

Index

• Constants
• Variables
• type ColumnTransformer
  • func GetMappingTransform(tType string, config MappingTransformerConfig) ColumnTransformer
  • func GetSingleValueTransform(tType string, geoip geoip.GeoLookup) ColumnTransformer
• type ErrNotTracked
  • func (t ErrNotTracked) Error() string
• type ErrSkippedColumn
  • func (t ErrSkippedColumn) Error() string
• type EventMetadataConfigLoader
• type MappingTransformerConfig
• type RedshiftTransformer
  • func (t *RedshiftTransformer) Consume(event *parser.MixpanelEvent) *writer.WriteRequest
• type RedshiftType
  • func (r *RedshiftType) Format(eventProperties map[string]interface{}) (string, string, error)
• type SchemaConfigLoader
• type Transformer
  • func NewRedshiftTransformer(configs SchemaConfigLoader, eventMetadataConfigs EventMetadataConfigLoader, ...) Transformer

Constants

const (
	// RedshiftDatetimeIngestString is the format of timestamps that Redshift understands.
	RedshiftDatetimeIngestString = "2006-01-02 15:04:05.999"

	// FloatLowerBound is the minimum float value to allow.
	// Redshift and Go appear to differ on floating point representation
	// we use 10^-300 here as a stop gap estimation.
	FloatLowerBound = 10e-300
)

Variables

var (
	// ErrUnknownTransform is when the transform from blueprint is unknown.
	ErrUnknownTransform = errors.New("Unrecognized transform")
	// ErrColumnNotFound is when a property from blueprint is not on an event.
	ErrColumnNotFound = errors.New("Property Not Found")
)

var (
	// ErrIDSet means we didn't have to do a lookup.
	ErrIDSet = errors.New("id was set")

	// ErrBadLookupValue means the lookup value is not usable.
	ErrBadLookupValue = errors.New("bad lookup value")

	// ErrEmptyLookupValue means the lookup value is not usable.
	ErrEmptyLookupValue = errors.New("empty lookup value")

	// ErrLocalCacheHit means the value was in the local cache.
	ErrLocalCacheHit = errors.New("local cache hit")

	// ErrRemoteCacheHit means the value was in the remote cache.
	ErrRemoteCacheHit = errors.New("remote cache hit")

	// ErrFetchSuccess means we were able to fetch the correct value.
	ErrFetchSuccess = errors.New("fetch success")

	// ErrFetchFailure means we were unable to fetch the correct value.
	ErrFetchFailure = errors.New("fetch failure")

	// ErrCacheSetFailure means we were unable to store the lookup value in the cache.
	ErrCacheSetFailure = errors.New("cache set failure")
)

var (
	// ErrEmptyRequest indicates the Event field is not set.
	ErrEmptyRequest = errors.New("Event field is not set")
)

var PST = getPST()

PST is the timezone used for everything.

Types

type ColumnTransformer

type ColumnTransformer func([]interface{}) (string, error)

ColumnTransformer takes an event property and transforms it to a string.

func GetMappingTransform

func GetMappingTransform(tType string, config MappingTransformerConfig) ColumnTransformer

GetMappingTransform returns a mapping transformer for a given identifier string.

func GetSingleValueTransform

func GetSingleValueTransform(tType string, geoip geoip.GeoLookup) ColumnTransformer

GetSingleValueTransform returns us a single value Transformer for a given identifier string.

type ErrNotTracked

type ErrNotTracked struct {
	What string
}

ErrNotTracked indicates an event type is not tracked.

func (ErrNotTracked) Error

func (t ErrNotTracked) Error() string

Error returns information on which event type is not being tracked.

type ErrSkippedColumn

type ErrSkippedColumn struct {
	What string
}

ErrSkippedColumn indicates an event is missing one or more columns.

func (ErrSkippedColumn) Error

func (t ErrSkippedColumn) Error() string

Error returns information on one column which is missing.

type EventMetadataConfigLoader

type EventMetadataConfigLoader interface {
	GetMetadataValueByType(string, string) string
}

EventMetadataConfigLoader returns event metadata for a given event and metadata type

type MappingTransformerConfig

type MappingTransformerConfig struct {
	Fetcher     lookup.ValueFetcher // used to fetch a value with with support of extra columns
	LocalCache  cache.StringCache   // an in-memory cache to avoid fetching.
	RemoteCache cache.StringCache   // an external cache to avoid fetching.
	Stats       reporter.StatsLogger
}

MappingTransformerConfig contains the required configuration for a mapping transformer to work.

type RedshiftTransformer

type RedshiftTransformer struct {
	Configs              SchemaConfigLoader
	EventMetadataConfigs EventMetadataConfigLoader
	// contains filtered or unexported fields
}

RedshiftTransformer turns MixpanelEvents into WriteRequests using the given SchemaConfigLoader.

func (*RedshiftTransformer) Consume

func (t *RedshiftTransformer) Consume(event *parser.MixpanelEvent) *writer.WriteRequest

Consume transforms a MixpanelEvent into a WriteRequest.

type RedshiftType

type RedshiftType struct {
	Transformer       ColumnTransformer
	InboundName       string
	OutboundName      string
	SupportingColumns []string
}

RedshiftType combines a way to get the input to the ColumnTransformer. Basically it performs Transformer(Event[EventProperty]) -> Column with the help of the values of the SupportingColumns provided.

func (*RedshiftType) Format

func (r *RedshiftType) Format(eventProperties map[string]interface{}) (string, string, error)

Format finds the column to transform and returns the outbound column name and transformed value.

type SchemaConfigLoader

type SchemaConfigLoader interface {
	GetColumnsForEvent(string) ([]RedshiftType, error)
	GetVersionForEvent(string) int
}

SchemaConfigLoader returns columns (transformers) or versions for given event types.

type Transformer

type Transformer interface {
	Consume(*parser.MixpanelEvent) *writer.WriteRequest
}

Transformer converts a MixpanelEvent into a WriteRequest.

func NewRedshiftTransformer

func NewRedshiftTransformer(configs SchemaConfigLoader, eventMetadataConfigs EventMetadataConfigLoader, stats reporter.StatsLogger) Transformer

NewRedshiftTransformer creates a new RedshiftTransformer using the given SchemaConfigLoader and EventMetadataConfigLoader