rules

Documentation

Index

• Variables
• func AddLocalJobConfig(serviceConfig config.ServiceConfig, jobConfigs JobConfigs) error
• func NoOp(from interface{}, ctx map[string]string) (interface{}, error)
• type Assignment
  • func NewAssignmentFromController(from *models.IngestionAssignment) (*Assignment, error)
• type DestinationConfig
• type JobAresConfig
• type JobConfig
  • func CloneJobConfig(src *JobConfig, serviceConfig config.ServiceConfig, aresCluster string) (*JobConfig, error)
  • func (j *JobConfig) GetColumnDict() map[string]int
  • func (j *JobConfig) GetDestinations() map[string]*DestinationConfig
  • func (j *JobConfig) GetPrimaryKeyBytes() int
  • func (j *JobConfig) GetPrimaryKeys() map[string]int
  • func (j *JobConfig) GetTranformations() map[string]*TransformationConfig
  • func (j *JobConfig) PopulateAresTableConfig() error
  • func (j *JobConfig) SetPrimaryKeyBytes(primaryKeyBytes int)
• type JobConfigs
• type Params
• type Result
  • func NewJobConfigs(params Params) (Result, error)
• type TransformationConfig
  • func (t TransformationConfig) Transform(from interface{}) (to interface{}, err error)

Variables

var Module = fx.Options(
	fx.Provide(
		NewJobConfigs,
	),
)

Module configures JobConfigs.

Functions

func AddLocalJobConfig

func AddLocalJobConfig(serviceConfig config.ServiceConfig, jobConfigs JobConfigs) error

AddLocalJobConfig creates a list of jobConfigs from local configuration file

func NoOp

func NoOp(from interface{}, ctx map[string]string) (interface{}, error)

NoOp is a transformation preserve value but it can rename the source value into different alias

Types

type Assignment

type Assignment struct {
	// Subscriber is ares-subscriber instance name
	Subscriber string `json:"subscriber"`
	// Jobs is a list of jobConfigs for the ares-subscriber instance
	Jobs []*JobConfig `json:"jobs"`
	// AresClusters is a table of aresClusters for the ares-subscriber instance
	AresClusters map[string]config.SinkConfig `json:"instances"`
}

Assignment defines the job assignment of the ares-subscriber

func NewAssignmentFromController

func NewAssignmentFromController(from *models.IngestionAssignment) (*Assignment, error)

NewAssignmentFromController parse controller assignment and create Assignment rule

type DestinationConfig

type DestinationConfig struct {
	// Table is ares table
	Table string `json:"table" yaml:"table"`
	// Column is ares table column name
	Column string `json:"column" yaml:"column"`
	// UpdateMode is column's upsert mode
	UpdateMode memCom.ColumnUpdateMode `json:"update_mode,omitempty" yaml:"update_mode,omitempty"`
}

DestinationConfig defines the configuration needed to save data in ares

type JobAresConfig

type JobAresConfig map[string]*JobConfig

JobAresConfig contains configuration and information for each Ares cluster and job configuration.

type JobConfig

type JobConfig struct {
	models.JobConfig
	// contains filtered or unexported fields
}

JobConfig wraps job config controller

func CloneJobConfig

func CloneJobConfig(src *JobConfig, serviceConfig config.ServiceConfig, aresCluster string) (*JobConfig, error)

CloneJobConfig deep copy jobConfig

func (*JobConfig) GetColumnDict

func (j *JobConfig) GetColumnDict() map[string]int

GetColumnDict returns a job's columnDict definition

func (*JobConfig) GetDestinations

func (j *JobConfig) GetDestinations() map[string]*DestinationConfig

GetDestinations returns a job's destination definition

func (*JobConfig) GetPrimaryKeyBytes

func (j *JobConfig) GetPrimaryKeyBytes() int

GetPrimaryKeyBytes returns the number of bytes needed by primaryKey

func (*JobConfig) GetPrimaryKeys

func (j *JobConfig) GetPrimaryKeys() map[string]int

GetPrimaryKeys returns a job's primaryKeys definition

func (*JobConfig) GetTranformations

func (j *JobConfig) GetTranformations() map[string]*TransformationConfig

GetTranformations returns a job's tranformation definition

func (*JobConfig) PopulateAresTableConfig

func (j *JobConfig) PopulateAresTableConfig() error

PopulateAresTableConfig populates information into jobConfig fields

func (*JobConfig) SetPrimaryKeyBytes

func (j *JobConfig) SetPrimaryKeyBytes(primaryKeyBytes int)

SetPrimaryKeyBytes sets the number of bytes needed by primaryKey

type JobConfigs

type JobConfigs map[string]JobAresConfig

JobConfigs contains configuration and information for all jobs and therir destination ares clusters.

type Params

type Params struct {
	fx.In

	ServiceConfig config.ServiceConfig
}

Params defines the base objects for jobConfigs.

type Result

type Result struct {
	fx.Out

	JobConfigs JobConfigs
}

Result defines the objects that the rules module provides.

func NewJobConfigs

func NewJobConfigs(params Params) (Result, error)

NewJobConfigs creates JobConfigs object

type TransformationConfig

type TransformationConfig struct {
	// Type of transformationConfig to apply for the column,
	// like timestamp, uuid, uuid_hll etc
	Type string `json:"type" yaml:"type"`
	// Source is the field name to read the value from
	Source string `json:"source" yaml:"source"`
	// Default value to use if value is empty
	Default string `json:"default" yaml:"default"`
	// Context help complex transformations to define information
	// needed to parse the values
	Context map[string]string
}

TransformationConfig defiines the configuration needed to generate a specific transformation

func (TransformationConfig) Transform

func (t TransformationConfig) Transform(from interface{}) (to interface{}, err error)

Transform converts source to destination data