utils

Documentation

Index

• Variables
• func DefaultSchemaValidator(ctx context.Context, table *toolkit.Table, properties *TransformerProperties, ...) (toolkit.ValidationWarnings, error)
• func NoiseDateV2(r *rand.Rand, ratio time.Duration, t *time.Time) *time.Time
• func NoiseFloat(r *rand.Rand, ratio float64, value float64, precision int) float64
• func NoiseInt(r *rand.Rand, ratio float64, value int64) int64
• func RandomBool(r *rand.Rand) bool
• func RandomDate(r *rand.Rand, min, max *time.Time) *time.Time
• func RandomFloat(r *rand.Rand, min, max float64, precision int) float64
• func RandomInt(r *rand.Rand, min, max int64) int64
• func RandomString(randGen *rand.Rand, minLength, maxLength int64, symbols []rune, buf []rune) string
• func Round(precision int, num float64) float64
• func TruncateDate(part *string, t *time.Time) (*time.Time, error)
• func ValidateSchema(table *toolkit.Table, column *toolkit.Column, ...) toolkit.ValidationWarnings
• type CancelFunction
• type CmdTransformerBase
  • func NewCmdTransformerBase(name string, expectedExitCode int, rowTransformationTimeout time.Duration, ...) *CmdTransformerBase
  • func (ctb *CmdTransformerBase) BaseDone() error
  • func (ctb *CmdTransformerBase) BaseInit(executable string, args []string) error
  • func (ctb *CmdTransformerBase) BaseInitWithContext(ctx context.Context, executable string, args []string) error
  • func (ctb *CmdTransformerBase) ReceiveStderrLine(ctx context.Context) (line []byte, err error)
  • func (ctb *CmdTransformerBase) ReceiveStdoutLine(ctx context.Context) (line []byte, err error)
  • func (ctb *CmdTransformerBase) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)
• type Definition
  • func NewDefinition(properties *TransformerProperties, newTransformerFunc NewTransformerFunc, ...) *Definition
  • func (d *Definition) Instance(ctx context.Context, driver *toolkit.Driver, ...) (Transformer, toolkit.ValidationWarnings, error)
  • func (d *Definition) SetSchemaValidator(v SchemaValidationFunc) *Definition
• type NewTransformerFunc
• type SchemaValidationFunc
• type Transformer
• type TransformerProperties
  • func NewTransformerProperties(name, description string) *TransformerProperties
• type TransformerRegistry
  • func NewTransformerRegistry() *TransformerRegistry
  • func (tm *TransformerRegistry) Get(name string) (*Definition, bool)
  • func (tm *TransformerRegistry) MustRegister(definition *Definition)
  • func (tm *TransformerRegistry) Register(definition *Definition) error

Variables

var DefaultTransformerRegistry = NewTransformerRegistry()

var ErrRowTransformationTimeout = errors.New("row transformation timeout")

Functions

func DefaultSchemaValidator

func DefaultSchemaValidator(
	ctx context.Context, table *toolkit.Table, properties *TransformerProperties, parameters map[string]*toolkit.Parameter, types []*toolkit.Type,
) (toolkit.ValidationWarnings, error)

func NoiseDateV2

func NoiseDateV2(r *rand.Rand, ratio time.Duration, t *time.Time) *time.Time

func NoiseFloat

func NoiseFloat(r *rand.Rand, ratio float64, value float64, precision int) float64

func NoiseInt

func NoiseInt(r *rand.Rand, ratio float64, value int64) int64

func RandomBool

func RandomBool(r *rand.Rand) bool

func RandomDate

func RandomDate(r *rand.Rand, min, max *time.Time) *time.Time

func RandomFloat

func RandomFloat(r *rand.Rand, min, max float64, precision int) float64

func RandomInt

func RandomInt(r *rand.Rand, min, max int64) int64

func RandomString

func RandomString(randGen *rand.Rand, minLength, maxLength int64, symbols []rune, buf []rune) string

func Round

func Round(precision int, num float64) float64

func TruncateDate

func TruncateDate(part *string, t *time.Time) (*time.Time, error)

TruncateDate - truncate date till the provided part of date

func ValidateSchema

func ValidateSchema(
	table *toolkit.Table, column *toolkit.Column, columnProperties *toolkit.ColumnProperties,
) toolkit.ValidationWarnings

Types

type CancelFunction

type CancelFunction func() error

type CmdTransformerBase

type CmdTransformerBase struct {
	Name                     string
	Cmd                      *exec.Cmd
	ExpectedExitCode         int
	Driver                   *toolkit.Driver
	Api                      toolkit.InteractionApi
	ProcessedLines           int
	RowTransformationTimeout time.Duration

	StdoutReader *bufio.Reader
	StderrReader *bufio.Reader
	StdinWriter  io.Writer
	Cancel       CancelFunction
	// contains filtered or unexported fields
}

func NewCmdTransformerBase

func NewCmdTransformerBase(
	name string,
	expectedExitCode int,
	rowTransformationTimeout time.Duration,
	driver *toolkit.Driver,
	api toolkit.InteractionApi,
) *CmdTransformerBase

func (*CmdTransformerBase) BaseDone

func (ctb *CmdTransformerBase) BaseDone() error

func (*CmdTransformerBase) BaseInit

func (ctb *CmdTransformerBase) BaseInit(executable string, args []string) error

func (*CmdTransformerBase) BaseInitWithContext

func (ctb *CmdTransformerBase) BaseInitWithContext(ctx context.Context, executable string, args []string) error

func (*CmdTransformerBase) ReceiveStderrLine

func (ctb *CmdTransformerBase) ReceiveStderrLine(ctx context.Context) (line []byte, err error)

func (*CmdTransformerBase) ReceiveStdoutLine

func (ctb *CmdTransformerBase) ReceiveStdoutLine(ctx context.Context) (line []byte, err error)

func (*CmdTransformerBase) Transform

func (ctb *CmdTransformerBase) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type Definition

type Definition struct {
	Properties      *TransformerProperties `json:"properties"`
	New             NewTransformerFunc     `json:"-"`
	Parameters      []*toolkit.Parameter   `json:"parameters"`
	SchemaValidator SchemaValidationFunc   `json:"-"`
}

func NewDefinition

func NewDefinition(
	properties *TransformerProperties, newTransformerFunc NewTransformerFunc,
	parameters ...*toolkit.Parameter,
) *Definition

func (*Definition) Instance

func (d *Definition) Instance(
	ctx context.Context, driver *toolkit.Driver, rawParams map[string]toolkit.ParamsValue, types []*toolkit.Type,
) (Transformer, toolkit.ValidationWarnings, error)

func (*Definition) SetSchemaValidator

func (d *Definition) SetSchemaValidator(v SchemaValidationFunc) *Definition

type NewTransformerFunc

type NewTransformerFunc func(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (
	Transformer, toolkit.ValidationWarnings, error,
)

NewTransformerFunc - make new transformer. This function receives Driver for making some steps for validation or anything else. parameters - the map of the parsed parameters, for get an appropriate parameter find it in the map by the Name. All those parameters has been defined in the Definition object of the transformer

type SchemaValidationFunc

type SchemaValidationFunc func(ctx context.Context, table *toolkit.Table, properties *TransformerProperties, parameters map[string]*toolkit.Parameter, types []*toolkit.Type) (toolkit.ValidationWarnings, error)

type Transformer

type Transformer interface {
	Init(ctx context.Context) error
	Done(ctx context.Context) error
	Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)
	GetAffectedColumns() map[int]string
}

type TransformerProperties

type TransformerProperties struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	IsCustom    bool   `json:"is_custom"`
}

func NewTransformerProperties

func NewTransformerProperties(
	name, description string,
) *TransformerProperties

type TransformerRegistry

type TransformerRegistry struct {
	M map[string]*Definition
}

func NewTransformerRegistry

func NewTransformerRegistry() *TransformerRegistry

func (*TransformerRegistry) Get

func (tm *TransformerRegistry) Get(name string) (*Definition, bool)

func (*TransformerRegistry) MustRegister

func (tm *TransformerRegistry) MustRegister(definition *Definition)

func (*TransformerRegistry) Register

func (tm *TransformerRegistry) Register(definition *Definition) error