transformers

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 24, 2024 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	JsonDeleteOpName = "delete"
	JsonSetOpName    = "set"
)
View Source 
const (
	MPassword   string = "password"
	MName       string = "name"
	MAddress    string = "addr"
	MEmail      string = "email"
	MMobile     string = "mobile"
	MTelephone  string = "tel"
	MID         string = "id"
	MCreditCard string = "credit_cart"
	MURL        string = "url"
	MDefault    string = "default"
)

Variables

View Source 
var CmdTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		cmdTransformerName,
		"Transform data via external program using stdin and stdout interaction",
	),

	NewCmd,

	toolkit.MustNewParameter(
		"columns",
		"affected column names. If empty use the whole tuple."+
			"The structure:"+
			`{`+
			`"name": "type:string, required:true, description: column Name",`+
			`"not_affected": "type:bool, required:false, description: is column affected in transformation", `+
			`"skip_original_data":  "type:bool, required:false, description: is original data required for transformer",`+
			`"skip_on_null_input":  "type:bool, required:false, description: skip transformation on null input"`+
			`}`,
	).SetDefaultValue([]byte("[]")),

	toolkit.MustNewParameter(
		"executable",
		"path to executable file",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"args",
		"list of parameters for executable file",
	).SetDefaultValue([]byte("[]")),

	toolkit.MustNewParameter(
		"driver",
		"row driver with parameters that is used for interacting with cmd. The default is csv. "+
			`The structure is:`+
			`{"name": "text|csv|json", "params": { "format": "[text|bytes]"} }`,
	).SetDefaultValue([]byte(`{"name": "csv"}`)),

	toolkit.MustNewParameter(
		"validate",
		"try to encode-decode data received from cmd",
	).SetDefaultValue([]byte("false")),

	toolkit.MustNewParameter(
		"timeout",
		"timeout for sending and receiving data from cmd",
	).SetDefaultValue([]byte("2s")),

	toolkit.MustNewParameter(
		"expected_exit_code",
		"expected exit code",
	).SetDefaultValue([]byte("0")),

	toolkit.MustNewParameter(
		"skip_on_behaviour",
		"skip transformation call if one of the provided columns has null value (any) or each of the provided"+
			" column has null values (all). This option works together with skip_on_null_input parameter on columns."+
			`Possible values: "all", "any"`,
	).SetDefaultValue([]byte("all")).
		SetRawValueValidator(cmdValidateSkipBehaviour),
)
View Source 
var DictTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Dict",
		"Replace values matched by dictionary keys",
	),

	NewDictTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"values",
		`map of value to replace in format: {"string": "string"}". The string with value "\N" supposed to be NULL value`,
	).SetRequired(true),

	toolkit.MustNewParameter(
		"default",
		`default value if not any value has been matched with dict. The string with value "\N" supposed to be NULL value. Default is empty`,
	).SetRequired(false),

	toolkit.MustNewParameter(
		"fail_not_matched",
		`fail if value is not matched with dict otherwise keep value`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),
	toolkit.MustNewParameter(
		"validate",
		`perform encode-decode procedure using column type, ensuring that value has correct type`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var FakerTransformersDes = map[string]*FakerTransformerDef{

	"RandomLatitude": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%f", faker.Latitude())
		},
		SupportedTypes: []string{"float4", "float8", "numeric"},
		Description:    "Generates a random latitude value.",
	},
	"RandomLongitude": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%f", faker.Longitude())
		},
		SupportedTypes: []string{"float4", "float8", "numeric"},
		Description:    "Generates a random longitude value.",
	},

	"RandomUnixTime": {
		Generator: func(opts ...options.OptionFunc) string {
			return fmt.Sprintf("%d", faker.UnixTime())
		},
		SupportedTypes: []string{"int4", "int8", "numeric"},
		Description:    "Generates a random Unix timestamp.",
	},
	"RandomMonthName": {
		Generator:      faker.MonthName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates the name of a random month.",
	},
	"RandomYearString": {
		Generator:      faker.YearString,
		SupportedTypes: []string{"text", "varchar", "int2", "int4", "int8", "numeric"},
		Description:    "Generates a random year as a string.",
	},
	"RandomDayOfWeek": {
		Generator:      faker.DayOfWeek,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random day of the week.",
	},
	"RandomDayOfMonth": {
		Generator:      faker.DayOfMonth,
		SupportedTypes: []string{"text", "varchar", "int2", "int4", "int8", "numeric"},
		Description:    "Generates a random day of the month.",
	},
	"RandomCentury": {
		Generator:      faker.Century,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random century.",
	},
	"RandomTimezone": {
		Generator:      faker.Timezone,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random timezone.",
	},

	"RandomEmail": {
		Generator:      faker.Email,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random email address.",
	},
	"RandomMacAddress": {
		Generator:      faker.MacAddress,
		SupportedTypes: []string{"text", "varchar", "macaddr", "macaddr8"},
		Description:    "Generates a random MAC address.",
	},
	"RandomDomainName": {
		Generator:      faker.DomainName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random domain name.",
	},
	"RandomURL": {
		Generator:      faker.URL,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random URL.",
	},
	"RandomUsername": {
		Generator:      faker.Username,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random username.",
	},
	"RandomIPv4": {
		Generator:      faker.IPv4,
		SupportedTypes: []string{"text", "varchar", "inet"},
		Description:    "Generates a random IPv4 address.",
	},
	"RandomIPv6": {
		Generator:      faker.IPv6,
		SupportedTypes: []string{"text", "varchar", "inet"},
		Description:    "Generates a random IPv6 address.",
	},
	"RandomPassword": {
		Generator:      faker.Password,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random password.",
	},

	"RandomWord": {
		Generator:      faker.Word,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random word.",
	},
	"RandomSentence": {
		Generator:      faker.Sentence,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random sentence.",
	},
	"RandomParagraph": {
		Generator:      faker.Paragraph,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random paragraph.",
	},

	"RandomCCType": {
		Generator:      faker.CCType,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random credit card type.",
	},
	"RandomCCNumber": {
		Generator:      faker.CCNumber,
		SupportedTypes: []string{"text", "varchar", "int4", "int8", "numeric"},
		Description:    "Generates a random credit card number.",
	},
	"RandomCurrency": {
		Generator:      faker.Currency,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random currency code.",
	},
	"RandomAmountWithCurrency": {
		Generator:      faker.AmountWithCurrency,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random monetary amount with currency.",
	},

	"RandomTitleMale": {
		Generator:      faker.TitleMale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random title for males.",
	},
	"RandomTitleFemale": {
		Generator:      faker.TitleFemale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random title for females.",
	},
	"RandomFirstName": {
		Generator:      faker.FirstName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random first name.",
	},
	"RandomFirstNameMale": {
		Generator:      faker.FirstNameMale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random male first name.",
	},
	"RandomFirstNameFemale": {
		Generator:      faker.FirstNameFemale,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random female first name.",
	},
	"RandomLastName": {
		Generator:      faker.LastName,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random last name.",
	},
	"RandomName": {
		Generator:      faker.Name,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a full random name.",
	},

	"RandomPhoneNumber": {
		Generator:      faker.Phonenumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random phone number.",
	},
	"RandomTollFreePhoneNumber": {
		Generator:      faker.TollFreePhoneNumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random toll-free phone number.",
	},
	"RandomE164PhoneNumber": {
		Generator:      faker.E164PhoneNumber,
		SupportedTypes: []string{"text", "varchar"},
		Description:    "Generates a random phone number in E.164 format.",
	},
}
View Source 
var HashTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Hash",
		"Generate hash of the text value using Scrypt hash function under the hood",
	),

	NewHashTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"salt",
		"salt for hash function base64 encoded",
	),
)
View Source 
var JsonTransformerDefinition = utils.NewDefinition(

	utils.NewTransformerProperties(
		"Json",
		"Update json document",
	),

	NewJsonTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("json", "jsonb"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"operations",
		`list of operations that contains editing operation [{"operation": "set|delete", "path": "path to the part of the document", "value": "value in any type - string, int, float, list, object, null", "value_template": "go template", "error_not_exist", "raise error if not exists - boolean"}]`,
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"apply changes in value is null",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var MaskingTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Masking",
		"Mask a value using one of masking type",
	),

	NewMaskingTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"type",
		"logical type of attribute (default, password, name, addr, email, mobile, tel, id, credit, url)",
	).SetRawValueValidator(maskerTypeValidator).
		SetDefaultValue(toolkit.ParamsValue(MDefault)),
)
View Source 
var NoiseDateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseDate",
		"Add some random value (shift value) in the provided interval",
	),

	NewNoiseDateTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("date", "timestamp", "timestamptz").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random duration for noise",
	).SetRequired(true).
		SetCastDbType("interval"),

	toolkit.MustNewParameter(
		"truncate",
		fmt.Sprintf("truncate date till the part (%s)", strings.Join(truncateParts, ", ")),
	).SetRawValueValidator(ValidateDateTruncationParameterValue),
)
View Source 
var NoiseFloatTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseFloat",
		"Make noise float for int",
	),
	NewNoiseFloatTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("float4", "float8", "numeric").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random percentage for noise",
	).SetDefaultValue(toolkit.ParamsValue("0.1")),

	toolkit.MustNewParameter(
		"precision",
		"precision of noised float value (number of digits after coma)",
	).SetDefaultValue(toolkit.ParamsValue("4")),
)
View Source 
var NoiseIntTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"NoiseInt",
		"Make noise value for int",
	),

	NewNoiseIntTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("int2", "int4", "int8").
		SetSkipOnNull(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"ratio",
		"max random percentage for noise",
	).SetRequired(true).
		SetDefaultValue(toolkit.ParamsValue("0.1")),
)
View Source 
var RandomBoolTransformerDefinition = utils.NewDefinition(

	utils.NewTransformerProperties(
		"RandomBool",
		"Generate random bool",
	),

	NewRandomBoolTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("bool"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomChoiceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomChoice",
		"Replace values chosen randomly from list",
	),

	NewRandomChoiceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"values",
		`list of values in any format. The string with value "\N" supposed to be NULL value`,
	).SetRequired(true).
		SetUnmarshaller(randomChoiceValuesUnmarshaller),

	toolkit.MustNewParameter(
		"validate",
		`perform decode procedure via PostgreSQL driver using column type, ensuring that value has correct type`,
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("true")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomDateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomDate",
		"Generate random date in the provided interval",
	),

	NewRandomDateTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("date", "timestamp", "timestamptz"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min threshold date (and/or time) of random value",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"max",
		"max threshold date  (and/or time) of random value",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"truncate",
		fmt.Sprintf("truncate date till the part (%s)", strings.Join(truncateParts, ", ")),
	).SetRawValueValidator(ValidateDateTruncationParameterValue),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomFloatTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomFloat",
		"Generate random float",
	),

	NewRandomFloatTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("float4", "float8", "numeric"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min float threshold of random value. The value range depends on column type",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max",
		"min float threshold of random value. The value range depends on column type",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"precision",
		"precision of random float value (number of digits after coma)",
	).SetDefaultValue(toolkit.ParamsValue("4")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomIntTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomInt",
		"Generate random int value from min to max",
	),

	NewRandomIntTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("int2", "int4", "int8", "numeric"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min",
		"min int value threshold",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max",
		"max int value threshold",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomStringTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomString",
		"Generate random string",
	),

	NewRandomStringTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"min_length",
		"min length of string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"max_length",
		"max length of string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"symbols",
		"the characters range for random string",
	).SetDefaultValue(toolkit.ParamsValue("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RandomUuidTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RandomUuid",
		"Generate random uuid",
	),

	NewRandomUuidTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("text", "varchar", "uuid"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var RealAddressTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RealAddress",
		"Generate a real address",
	),

	NewRealAddressTransformer,

	toolkit.MustNewParameter(
		"columns",
		"affected column names."+
			"The structure:"+
			`{`+
			`"name": "type:string, required:true, description: column Name",`+
			`"template": "type:string, required:true, description: gotemplate with real address attributes injections",`+
			`"keep_null": "type:bool, required:false, description: keep null values",`+
			`}`,
	).SetRequired(true),
)
View Source 
var RegexpReplaceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"RegexpReplace",
		"Replace string using regular expression",
	),

	NewRegexpReplaceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetAllowedColumnTypes("varchar", "text", "bpchar"),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"regexp",
		"regular expression",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"replace",
		"replacement value",
	).SetRequired(true),
)
View Source 
var ReplaceTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Replace",
		"Replace column value to the provided",
	),

	NewReplaceTransformer,

	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"value",
		"value to replace",
	).SetRequired(true).
		SetLinkParameter("column"),

	toolkit.MustNewParameter(
		"keep_null",
		"indicates that NULL values must not be replaced with transformed values",
	).SetDefaultValue(toolkit.ParamsValue("true")),

	toolkit.MustNewParameter(
		"validate",
		"validate the value via driver decoding procedure",
	).SetDefaultValue(toolkit.ParamsValue("true")),
)
View Source 
var SetNullTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"SetNull",
		"Set NULL value",
	),
	NewSetNullTransformer,
	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true).
		SetNullable(true),
	).SetRequired(true),
)
View Source 
var TemplateRecordTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"TemplateRecord",
		"Modify the record using gotemplate",
	),
	NewTemplateRecordTransformer,

	toolkit.MustNewParameter(
		"template",
		"gotemplate string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"validate",
		"validate template result via PostgreSQL driver decoding",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("false")),

	toolkit.MustNewParameter(
		"columns",
		"columns that supposed to be affected by the template. The list of columns will be checked for constraint violation",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("[]")),
)
View Source 
var TemplateTransformerDefinition = utils.NewDefinition(
	utils.NewTransformerProperties(
		"Template",
		"Modify the value using gotemplate",
	),
	NewTemplateTransformer,
	toolkit.MustNewParameter(
		"column",
		"column name",
	).SetIsColumn(toolkit.NewColumnProperties().
		SetAffected(true),
	).SetRequired(true),

	toolkit.MustNewParameter(
		"template",
		"gotemplate string",
	).SetRequired(true),

	toolkit.MustNewParameter(
		"validate",
		"validate template result via PostgreSQL driver decoding",
	).SetRequired(false).
		SetDefaultValue(toolkit.ParamsValue("false")),
)

Functions

func MakeNewFakeTransformerFunction added in v0.1.1 

func MakeNewFakeTransformerFunction(generator FakerFunc) utils.NewTransformerFunc

func NewCmd 

func NewCmd(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewDictTransformer 

func NewDictTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewFakeTransformer added in v0.1.1 

func NewFakeTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter, generator FakerFunc,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewHashTransformer 

func NewHashTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewJsonTransformer 

func NewJsonTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewMaskingTransformer 

func NewMaskingTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseDateTransformer 

func NewNoiseDateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseFloatTransformer 

func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewNoiseIntTransformer 

func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomBoolTransformer 

func NewRandomBoolTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomChoiceTransformer 

func NewRandomChoiceTransformer(
	ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter,
) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomDateTransformer 

func NewRandomDateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomFloatTransformer 

func NewRandomFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomIntTransformer 

func NewRandomIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomStringTransformer 

func NewRandomStringTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRandomUuidTransformer 

func NewRandomUuidTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRealAddressTransformer added in v0.1.1 

func NewRealAddressTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewRegexpReplaceTransformer 

func NewRegexpReplaceTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewReplaceTransformer 

func NewReplaceTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewSetNullTransformer 

func NewSetNullTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewTemplateRecordTransformer 

func NewTemplateRecordTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func NewTemplateTransformer 

func NewTemplateTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]*toolkit.Parameter) (utils.Transformer, toolkit.ValidationWarnings, error)

func ValidateDateTruncationParameterValue 

func ValidateDateTruncationParameterValue(p *toolkit.Parameter, v toolkit.ParamsValue) (toolkit.ValidationWarnings, error)

Types

type Cmd 

type Cmd struct {
	*utils.CmdTransformerBase

	Columns []*Column
	// contains filtered or unexported fields
}

func (*Cmd) Done 

func (c *Cmd) Done(ctx context.Context) error

func (*Cmd) GetAffectedColumns 

func (c *Cmd) GetAffectedColumns() map[int]string

func (*Cmd) Init 

func (c *Cmd) Init(ctx context.Context) error

func (*Cmd) Transform 

func (c *Cmd) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type Column 

type Column struct {
	Name             string `json:"name"`
	NotAffected      bool   `json:"not_affected"`
	SkipOnNullInput  bool   `json:"skip_on_null_input"`
	SkipOriginalData bool   `json:"skip_original_data"`
}

type ColumnContext 

type ColumnContext struct {
	// contains filtered or unexported fields
}

func NewColumnContext 

func NewColumnContext(columnType string, columnName string) *ColumnContext

func (*ColumnContext) DecodeValue 

func (cc *ColumnContext) DecodeValue(v any) (any, error)

func (*ColumnContext) DecodeValueByColumn 

func (cc *ColumnContext) DecodeValueByColumn(name string, v any) (any, error)

func (*ColumnContext) DecodeValueByType 

func (cc *ColumnContext) DecodeValueByType(name string, v any) (any, error)

func (*ColumnContext) EncodeValue 

func (cc *ColumnContext) EncodeValue(v any) (any, error)

func (*ColumnContext) EncodeValueByColumn 

func (cc *ColumnContext) EncodeValueByColumn(name string, v any) (any, error)

func (*ColumnContext) EncodeValueByType 

func (cc *ColumnContext) EncodeValueByType(name string, v any) (any, error)

func (*ColumnContext) GetColumnRawValue 

func (cc *ColumnContext) GetColumnRawValue(name string) (any, error)

func (*ColumnContext) GetColumnType 

func (cc *ColumnContext) GetColumnType() string

func (*ColumnContext) GetColumnValue 

func (cc *ColumnContext) GetColumnValue(name string) (any, error)

func (*ColumnContext) GetRawValue 

func (cc *ColumnContext) GetRawValue() (any, error)

func (*ColumnContext) GetValue 

func (cc *ColumnContext) GetValue() (any, error)

type DictTransformer 

type DictTransformer struct {
	// contains filtered or unexported fields
}

func (*DictTransformer) Done 

func (ht *DictTransformer) Done(ctx context.Context) error

func (*DictTransformer) GetAffectedColumns 

func (ht *DictTransformer) GetAffectedColumns() map[int]string

func (*DictTransformer) Init 

func (ht *DictTransformer) Init(ctx context.Context) error

func (*DictTransformer) Transform 

func (ht *DictTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type FakeTransformer added in v0.1.1 

type FakeTransformer struct {
	// contains filtered or unexported fields
}

func (*FakeTransformer) Done added in v0.1.1 

func (fts *FakeTransformer) Done(ctx context.Context) error

func (*FakeTransformer) GetAffectedColumns added in v0.1.1 

func (fts *FakeTransformer) GetAffectedColumns() map[int]string

func (*FakeTransformer) Init added in v0.1.1 

func (fts *FakeTransformer) Init(ctx context.Context) error

func (*FakeTransformer) Transform added in v0.1.1 

func (fts *FakeTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type FakerFunc added in v0.1.1 

type FakerFunc func(opts ...options.OptionFunc) string

type FakerTransformerDef added in v0.1.1 

type FakerTransformerDef struct {
	SupportedTypes []string
	Description    string
	Generator      FakerFunc
}

type HashTransformer 

type HashTransformer struct {
	// contains filtered or unexported fields
}

func (*HashTransformer) Done 

func (ht *HashTransformer) Done(ctx context.Context) error

func (*HashTransformer) GetAffectedColumns 

func (ht *HashTransformer) GetAffectedColumns() map[int]string

func (*HashTransformer) Init 

func (ht *HashTransformer) Init(ctx context.Context) error

func (*HashTransformer) Transform 

func (ht *HashTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type JsonContext 

type JsonContext struct {
	// contains filtered or unexported fields
}

func NewJsonContext 

func NewJsonContext() *JsonContext

func (*JsonContext) DecodeValueByColumn 

func (jc *JsonContext) DecodeValueByColumn(name string, v any) (any, error)

func (*JsonContext) DecodeValueByType 

func (jc *JsonContext) DecodeValueByType(name string, v any) (any, error)

func (*JsonContext) EncodeValueByColumn 

func (jc *JsonContext) EncodeValueByColumn(name string, v any) (any, error)

func (*JsonContext) EncodeValueByType 

func (jc *JsonContext) EncodeValueByType(name string, v any) (any, error)

func (*JsonContext) GetColumnRawValue 

func (jc *JsonContext) GetColumnRawValue(name string) (any, error)

func (*JsonContext) GetColumnValue 

func (jc *JsonContext) GetColumnValue(name string) (any, error)

func (*JsonContext) GetOriginalValue 

func (jc *JsonContext) GetOriginalValue() any

func (*JsonContext) GetPath 

func (jc *JsonContext) GetPath() string

func (*JsonContext) OriginalValueExists 

func (jc *JsonContext) OriginalValueExists() bool

type JsonTransformer 

type JsonTransformer struct {
	// contains filtered or unexported fields
}

func (*JsonTransformer) Done 

func (jt *JsonTransformer) Done(ctx context.Context) error

func (*JsonTransformer) GetAffectedColumns 

func (jt *JsonTransformer) GetAffectedColumns() map[int]string

func (*JsonTransformer) Init 

func (jt *JsonTransformer) Init(ctx context.Context) error

func (*JsonTransformer) Transform 

func (jt *JsonTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type MaskingTransformer 

type MaskingTransformer struct {
	// contains filtered or unexported fields
}

func (*MaskingTransformer) Done 

func (mt *MaskingTransformer) Done(ctx context.Context) error

func (*MaskingTransformer) GetAffectedColumns 

func (mt *MaskingTransformer) GetAffectedColumns() map[int]string

func (*MaskingTransformer) Init 

func (mt *MaskingTransformer) Init(ctx context.Context) error

func (*MaskingTransformer) Transform 

func (mt *MaskingTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseDateTransformer 

type NoiseDateTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseDateTransformer) Done 

func (ndt *NoiseDateTransformer) Done(ctx context.Context) error

func (*NoiseDateTransformer) GetAffectedColumns 

func (ndt *NoiseDateTransformer) GetAffectedColumns() map[int]string

func (*NoiseDateTransformer) Init 

func (ndt *NoiseDateTransformer) Init(ctx context.Context) error

func (*NoiseDateTransformer) Transform 

func (ndt *NoiseDateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseFloatTransformer 

type NoiseFloatTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseFloatTransformer) Done 

func (nft *NoiseFloatTransformer) Done(ctx context.Context) error

func (*NoiseFloatTransformer) GetAffectedColumns 

func (nft *NoiseFloatTransformer) GetAffectedColumns() map[int]string

func (*NoiseFloatTransformer) Init 

func (nft *NoiseFloatTransformer) Init(ctx context.Context) error

func (*NoiseFloatTransformer) Transform 

func (nft *NoiseFloatTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type NoiseFloatTransformerParams 

type NoiseFloatTransformerParams struct {
	Ratio     float64 `mapstructure:"ratio" validate:"required,min=0,max=1"`
	Precision int16   `mapstructure:"precision"`
	Nullable  bool    `mapstructure:"nullable"`
	Fraction  float32 `mapstructure:"fraction"`
}

type NoiseIntTransformer 

type NoiseIntTransformer struct {
	// contains filtered or unexported fields
}

func (*NoiseIntTransformer) Done 

func (nit *NoiseIntTransformer) Done(ctx context.Context) error

func (*NoiseIntTransformer) GetAffectedColumns 

func (nit *NoiseIntTransformer) GetAffectedColumns() map[int]string

func (*NoiseIntTransformer) Init 

func (nit *NoiseIntTransformer) Init(ctx context.Context) error

func (*NoiseIntTransformer) Transform 

func (nit *NoiseIntTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type Operation 

type Operation struct {
	Operation     string      `mapstructure:"operation"`
	Value         interface{} `mapstructure:"value,omitempty"`
	ValueTemplate string      `mapstructure:"value_template,omitempty"`
	Path          string      `mapstructure:"path"`
	ErrorNotExist bool        `mapstructure:"error_not_exist"`
	// contains filtered or unexported fields
}

func (*Operation) Apply 

func (o *Operation) Apply(inp []byte, tctx *JsonContext, buf *bytes.Buffer) ([]byte, error)

type RandomBoolTransformer 

type RandomBoolTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomBoolTransformer) Done 

func (rbt *RandomBoolTransformer) Done(ctx context.Context) error

func (*RandomBoolTransformer) GetAffectedColumns 

func (rbt *RandomBoolTransformer) GetAffectedColumns() map[int]string

func (*RandomBoolTransformer) Init 

func (rbt *RandomBoolTransformer) Init(ctx context.Context) error

func (*RandomBoolTransformer) Transform 

func (rbt *RandomBoolTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomChoiceTransformer 

type RandomChoiceTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomChoiceTransformer) Done 

func (rct *RandomChoiceTransformer) Done(ctx context.Context) error

func (*RandomChoiceTransformer) GetAffectedColumns 

func (rct *RandomChoiceTransformer) GetAffectedColumns() map[int]string

func (*RandomChoiceTransformer) Init 

func (rct *RandomChoiceTransformer) Init(ctx context.Context) error

func (*RandomChoiceTransformer) Transform 

func (rct *RandomChoiceTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomDateTransformer 

type RandomDateTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomDateTransformer) Done 

func (rdt *RandomDateTransformer) Done(ctx context.Context) error

func (*RandomDateTransformer) GetAffectedColumns 

func (rdt *RandomDateTransformer) GetAffectedColumns() map[int]string

func (*RandomDateTransformer) Init 

func (rdt *RandomDateTransformer) Init(ctx context.Context) error

func (*RandomDateTransformer) Transform 

func (rdt *RandomDateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomDateTransformerParams 

type RandomDateTransformerParams struct {
	Min      string  `mapstructure:"min" validate:"required"`
	Max      string  `mapstructure:"max" validate:"required"`
	Truncate string  `mapstructure:"truncate" validate:"omitempty,oneof=year month day hour second nano"`
	Nullable bool    `mapstructure:"nullable"`
	Fraction float32 `mapstructure:"fraction"`
}

type RandomFloatTransformer 

type RandomFloatTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomFloatTransformer) Done 

func (rft *RandomFloatTransformer) Done(ctx context.Context) error

func (*RandomFloatTransformer) GetAffectedColumns 

func (rft *RandomFloatTransformer) GetAffectedColumns() map[int]string

func (*RandomFloatTransformer) Init 

func (rft *RandomFloatTransformer) Init(ctx context.Context) error

func (*RandomFloatTransformer) Transform 

func (rft *RandomFloatTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomIntTransformer 

type RandomIntTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomIntTransformer) Done 

func (rit *RandomIntTransformer) Done(ctx context.Context) error

func (*RandomIntTransformer) GetAffectedColumns 

func (rit *RandomIntTransformer) GetAffectedColumns() map[int]string

func (*RandomIntTransformer) Init 

func (rit *RandomIntTransformer) Init(ctx context.Context) error

func (*RandomIntTransformer) Transform 

func (rit *RandomIntTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomStringTransformer 

type RandomStringTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomStringTransformer) Done 

func (rst *RandomStringTransformer) Done(ctx context.Context) error

func (*RandomStringTransformer) GetAffectedColumns 

func (rst *RandomStringTransformer) GetAffectedColumns() map[int]string

func (*RandomStringTransformer) Init 

func (rst *RandomStringTransformer) Init(ctx context.Context) error

func (*RandomStringTransformer) Transform 

func (rst *RandomStringTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RandomUuidTransformer 

type RandomUuidTransformer struct {
	// contains filtered or unexported fields
}

func (*RandomUuidTransformer) Done 

func (rut *RandomUuidTransformer) Done(ctx context.Context) error

func (*RandomUuidTransformer) GetAffectedColumns 

func (rut *RandomUuidTransformer) GetAffectedColumns() map[int]string

func (*RandomUuidTransformer) Init 

func (rut *RandomUuidTransformer) Init(ctx context.Context) error

func (*RandomUuidTransformer) Transform 

func (rut *RandomUuidTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RealAddressColumn added in v0.1.1 

type RealAddressColumn struct {
	Name     string `json:"name"`
	KeepNull bool   `json:"keep_null"`
	Template string `json:"template"`
	// contains filtered or unexported fields
}

type RealAddressTransformer added in v0.1.1 

type RealAddressTransformer struct {
	// contains filtered or unexported fields
}

func (*RealAddressTransformer) Done added in v0.1.1 

func (rat *RealAddressTransformer) Done(ctx context.Context) error

func (*RealAddressTransformer) GetAffectedColumns added in v0.1.1 

func (rat *RealAddressTransformer) GetAffectedColumns() map[int]string

func (*RealAddressTransformer) Init added in v0.1.1 

func (rat *RealAddressTransformer) Init(ctx context.Context) error

func (*RealAddressTransformer) Transform added in v0.1.1 

func (rat *RealAddressTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type RealAddressValue added in v0.1.1 

type RealAddressValue struct {
	Address    string  `json:"address1"`
	City       string  `json:"city"`
	State      string  `json:"state"`
	PostalCode string  `json:"postalCode"`
	Latitude   float64 `json:"lat"`
	Longitude  float64 `json:"lng"`
}

type RegexpReplaceTransformer 

type RegexpReplaceTransformer struct {
	// contains filtered or unexported fields
}

func (*RegexpReplaceTransformer) Done 

func (rrt *RegexpReplaceTransformer) Done(ctx context.Context) error

func (*RegexpReplaceTransformer) GetAffectedColumns 

func (rrt *RegexpReplaceTransformer) GetAffectedColumns() map[int]string

func (*RegexpReplaceTransformer) Init 

func (rrt *RegexpReplaceTransformer) Init(ctx context.Context) error

func (*RegexpReplaceTransformer) Transform 

func (rrt *RegexpReplaceTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type ReplaceTransformer 

type ReplaceTransformer struct {
	// contains filtered or unexported fields
}

func (*ReplaceTransformer) Done 

func (rt *ReplaceTransformer) Done(ctx context.Context) error

func (*ReplaceTransformer) GetAffectedColumns 

func (rt *ReplaceTransformer) GetAffectedColumns() map[int]string

func (*ReplaceTransformer) Init 

func (rt *ReplaceTransformer) Init(ctx context.Context) error

func (*ReplaceTransformer) Transform 

func (rt *ReplaceTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type SetNullTransformer 

type SetNullTransformer struct {
	// contains filtered or unexported fields
}

func (*SetNullTransformer) Done 

func (sut *SetNullTransformer) Done(ctx context.Context) error

func (*SetNullTransformer) GetAffectedColumns 

func (sut *SetNullTransformer) GetAffectedColumns() map[int]string

func (*SetNullTransformer) Init 

func (sut *SetNullTransformer) Init(ctx context.Context) error

func (*SetNullTransformer) Transform 

func (sut *SetNullTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type TemplateRecordTransformer 

type TemplateRecordTransformer struct {
	// contains filtered or unexported fields
}

func (*TemplateRecordTransformer) Done 

func (sut *TemplateRecordTransformer) Done(ctx context.Context) error

func (*TemplateRecordTransformer) GetAffectedColumns 

func (sut *TemplateRecordTransformer) GetAffectedColumns() map[int]string

func (*TemplateRecordTransformer) Init 

func (sut *TemplateRecordTransformer) Init(ctx context.Context) error

func (*TemplateRecordTransformer) Transform 

func (sut *TemplateRecordTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

type TemplateTransformer 

type TemplateTransformer struct {
	// contains filtered or unexported fields
}

func (*TemplateTransformer) Done 

func (sut *TemplateTransformer) Done(ctx context.Context) error

func (*TemplateTransformer) GetAffectedColumns 

func (sut *TemplateTransformer) GetAffectedColumns() map[int]string

func (*TemplateTransformer) Init 

func (sut *TemplateTransformer) Init(ctx context.Context) error

func (*TemplateTransformer) Transform 

func (sut *TemplateTransformer) Transform(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error)

Source Files

View all Source files

Directories

Path Synopsis
template

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.