transforms

Documentation

Index

• Constants
• Variables
• func Add(name string, creator Creator)
• func GetTransformerOptions() map[string][]Option
• func GetTransformerUsages() KeyValueSlice
• func SetError(errNum int, currentErr error, errType int, key string) (int, error)
• func SetStatsInfo(err error, stats StatsInfo, errNum, dataLen int64, transformType string) (StatsInfo, error)
• type Creator
• type Initializer
• type RawTransformInfo
• type RawTransformResult
• type RawTransformResultSlice
  • func (slice RawTransformResultSlice) Len() int
  • func (slice RawTransformResultSlice) Less(i, j int) bool
  • func (slice RawTransformResultSlice) Swap(i, j int)
• type ServerTansformer
• type StatsTransformer
• type TransformInfo
• type TransformResult
• type TransformResultSlice
  • func (slice TransformResultSlice) Len() int
  • func (slice TransformResultSlice) Less(i, j int) bool
  • func (slice TransformResultSlice) Swap(i, j int)
• type Transformer

Constants

const (
	TransformAt = "transform_at"

	TransformTypeString  = "string"
	TransformTypeLong    = "long"
	TransformTypeFloat   = "float"
	TransformTypeBoolean = "bool"
	TransformTypeByte    = "[]byte"
)

const (
	StageBeforeParser = "before_parser"
	StageAfterParser  = "after_parser"
)

const (
	General = iota
	GetErr
	SetErr
)

Variables

var (
	KeyStage = Option{
		KeyName:       "stage",
		Element:       Radio,
		ChooseOnly:    true,
		ChooseOptions: []interface{}{StageAfterParser, StageBeforeParser},
		Default:       StageAfterParser,
		DefaultNoUse:  false,
		Description:   "transform运行的阶段(parser前还是parser后)(stage)",
		ToolTip:       "transform在parser前或在parser后进行运行",
		Type:          TransformTypeString,
		Advance:       true,
	}
	KeyFieldName = Option{
		KeyName:      "key",
		ChooseOnly:   false,
		Default:      "",
		Required:     true,
		Placeholder:  "my_field_keyname",
		DefaultNoUse: true,
		Description:  "要进行Transform变化的键(key)",
		ToolTip:      "对该字段的值进行transform变换",
		Type:         TransformTypeString,
	}
	KeyFieldNew = Option{
		KeyName:      "new",
		ChooseOnly:   false,
		Default:      "",
		Placeholder:  "new_field_keyname",
		DefaultNoUse: false,
		Description:  "新的字段名(new)",
		CheckRegex:   CheckPatternKey,
		ToolTip:      "生成的字段名称，不改变原有的字段",
		Type:         TransformTypeString,
	}
	KeyFieldNewRequired = Option{
		KeyName:      "new",
		ChooseOnly:   false,
		Default:      "",
		Required:     true,
		Placeholder:  "new_field_keyname",
		DefaultNoUse: false,
		Description:  "新的字段名(new)",
		CheckRegex:   CheckPatternKey,
		ToolTip:      "生成的字段名称，不改变原有的字段",
		Type:         TransformTypeString,
	}
	KeyTimezoneoffset = Option{
		KeyName:    "offset",
		ChooseOnly: true,
		ChooseOptions: []interface{}{0, -1, -2, -3, -4,
			-5, -6, -7, -8, -9, -10, -11, -12,
			1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12},
		Default:      0,
		DefaultNoUse: false,
		Description:  "时区偏移量(offset)",
		Advance:      true,
		CheckRegex:   "*",
		Type:         TransformTypeLong,
		ToolTip:      "如果key中带有时区信息，则以该时区作为offset的基础时区，否则以UTC时区为基础时区",
	}
	KeyKeepString = Option{
		KeyName:       "keep",
		Element:       Radio,
		ChooseOnly:    true,
		Default:       false,
		ChooseOptions: []interface{}{false, true},
		DefaultNoUse:  false,
		Description:   "值解析为字符串，不进行转换",
		Advance:       true,
		Type:          TransformTypeBoolean,
	}
	KeyOverride = Option{
		KeyName:       "override",
		Element:       Radio,
		ChooseOnly:    true,
		ChooseOptions: []interface{}{false, true},
		Default:       false,
		DefaultNoUse:  false,
		Description:   "要进行Transform变化的键已存在时，是否覆盖原有的值(override)",
		Type:          TransformTypeBoolean,
		Advance:       true,
	}
	KeyDiscardkey = Option{
		KeyName:       "discard_key",
		Element:       Radio,
		ChooseOnly:    true,
		ChooseOptions: []interface{}{false, true},
		Default:       false,
		DefaultNoUse:  false,
		Description:   "删除原始key指定的字段名和字段值",
		Type:          TransformTypeBoolean,
		Advance:       true,
	}
)

var Transformers = map[string]Creator{}

Functions

func Add

func Add(name string, creator Creator)

func GetTransformerOptions

func GetTransformerOptions() map[string][]Option

func GetTransformerUsages

func GetTransformerUsages() KeyValueSlice

func SetError

func SetError(errNum int, currentErr error, errType int, key string) (int, error)

func SetStatsInfo

func SetStatsInfo(err error, stats StatsInfo, errNum, dataLen int64, transformType string) (StatsInfo, error)

Types

type Creator

type Creator func() Transformer

type Initializer

type Initializer interface {
	Init() error
}

transformer 初始化方法接口,err 不为空表示初始化失败

type RawTransformInfo

type RawTransformInfo struct {
	CurData string
	Index   int
}

type RawTransformResult

type RawTransformResult struct {
	Index   int
	CurData string
	Err     error
	ErrNum  int
}

type RawTransformResultSlice

type RawTransformResultSlice []RawTransformResult

func (RawTransformResultSlice) Len

func (slice RawTransformResultSlice) Len() int

func (RawTransformResultSlice) Less

func (slice RawTransformResultSlice) Less(i, j int) bool

func (RawTransformResultSlice) Swap

func (slice RawTransformResultSlice) Swap(i, j int)

type ServerTansformer

type ServerTansformer interface {
	ServerConfig() map[string]interface{}
}

type StatsTransformer

type StatsTransformer interface {
	SetStats(string) StatsInfo
}

StatsTransformer 代表了一个带有统计功能的转换器

type TransformInfo

type TransformInfo struct {
	CurData Data
	Index   int
}

type TransformResult

type TransformResult struct {
	Index    int
	CurData  Data
	CurDatas []Data
	Err      error
	ErrNum   int
}

type TransformResultSlice

type TransformResultSlice []TransformResult

func (TransformResultSlice) Len

func (slice TransformResultSlice) Len() int

func (TransformResultSlice) Less

func (slice TransformResultSlice) Less(i, j int) bool

func (TransformResultSlice) Swap

func (slice TransformResultSlice) Swap(i, j int)

type Transformer

type Transformer interface {
	Description() string
	SampleConfig() string
	ConfigOptions() []Option
	Type() string
	Transform([]Data) ([]Data, error)
	RawTransform([]string) ([]string, error)
	Stage() string
	Stats() StatsInfo
}

Transformer plugin做数据变换的接口 注意： transform的规则是，出错要把数据原样返回