config

Documentation

Index

• type JobCfg
  • func FromTaskCfgs(taskCfgs []*TaskCfg) *JobCfg
  • func (c *JobCfg) Clone() (*JobCfg, error)
  • func (c *JobCfg) Decode(content []byte) error
  • func (c *JobCfg) DecodeFile(fpath string) error
  • func (c *JobCfg) ToTaskCfg() *TaskCfg
  • func (c *JobCfg) ToTaskCfgs() map[string]*TaskCfg
  • func (c *JobCfg) Yaml() ([]byte, error)
• type TaskCfg
  • func (c *TaskCfg) ToDMSubTaskCfg(jobID string) *dmconfig.SubTaskConfig
  • func (c *TaskCfg) ToJobCfg() *JobCfg
• type UpstreamCfg

Types

type JobCfg

type JobCfg struct {
	TaskMode                  string                                `yaml:"task-mode" toml:"task-mode" json:"task-mode"`
	ShardMode                 string                                `yaml:"shard-mode" toml:"shard-mode" json:"shard-mode"` // when `shard-mode` set, we always enable sharding support.
	StrictOptimisticShardMode bool                                  `yaml:"strict-optimistic-shard-mode" toml:"strict-optimistic-shard-mode" json:"strict-optimistic-shard-mode"`
	IgnoreCheckingItems       []string                              `yaml:"ignore-checking-items" toml:"ignore-checking-items" json:"ignore-checking-items"`
	Timezone                  string                                `yaml:"timezone" toml:"timezone" json:"timezone"`
	CollationCompatible       string                                `yaml:"collation_compatible" toml:"collation_compatible" json:"collation_compatible"`
	TargetDB                  *dbconfig.DBConfig                    `yaml:"target-database" toml:"target-database" json:"target-database"`
	ShadowTableRules          []string                              `yaml:"shadow-table-rules" toml:"shadow-table-rules" json:"shadow-table-rules"`
	TrashTableRules           []string                              `yaml:"trash-table-rules" toml:"trash-table-rules" json:"trash-table-rules"`
	Filters                   map[string]*bf.BinlogEventRule        `yaml:"filters" toml:"filters" json:"filters"`
	ExprFilter                map[string]*dmconfig.ExpressionFilter `yaml:"expression-filter" toml:"expression-filter" json:"expression-filter"`
	BAList                    map[string]*filter.Rules              `yaml:"block-allow-list" toml:"block-allow-list" json:"block-allow-list"`
	Mydumpers                 map[string]*dmconfig.MydumperConfig   `yaml:"mydumpers" toml:"mydumpers" json:"mydumpers"`
	Loaders                   map[string]*dmconfig.LoaderConfig     `yaml:"loaders" toml:"loaders" json:"loaders"`
	Syncers                   map[string]*dmconfig.SyncerConfig     `yaml:"syncers" toml:"syncers" json:"syncers"`
	Routes                    map[string]*router.TableRule          `yaml:"routes" toml:"routes" json:"routes"`
	Validators                map[string]*dmconfig.ValidatorConfig  `yaml:"validators" toml:"validators" json:"validators"`
	// remove source config, use db config instead.
	Upstreams []*UpstreamCfg `yaml:"upstreams" toml:"upstreams" json:"upstreams"`

	// no need experimental features?
	Experimental struct {
		AsyncCheckpointFlush bool `yaml:"async-checkpoint-flush" toml:"async-checkpoint-flush" json:"async-checkpoint-flush"`
	} `yaml:"experimental" toml:"experimental" json:"experimental"`

	// remove them later
	MetaSchema     string                  `yaml:"meta-schema" toml:"meta-schema" json:"meta-schema"`
	OnlineDDL      bool                    `yaml:"online-ddl" toml:"online-ddl" json:"online-ddl"`
	ColumnMappings map[string]*column.Rule `yaml:"column-mappings" toml:"column-mappings" json:"column-mappings"`

	ModRevision uint64 `yaml:"mod-revision" toml:"mod-revision" json:"mod-revision"`
}

JobCfg copies from SubTaskConfig and removes some deprecated fields. It represents a DM subtask with multiple source configs embedded as Upstreams. DISCUSS: support command line args. e.g. --start-time.

func FromTaskCfgs

func FromTaskCfgs(taskCfgs []*TaskCfg) *JobCfg

FromTaskCfgs converts task configs to a jobCfg.

func (*JobCfg) Clone

func (c *JobCfg) Clone() (*JobCfg, error)

Clone returns a deep copy of JobCfg

func (*JobCfg) Decode

func (c *JobCfg) Decode(content []byte) error

Decode unmarshals the content into JobCfg and calls adjust() on it. TODO: unify config type Now, dmJobmaster use yaml, dmWorker use toml, and lib use json...

func (*JobCfg) DecodeFile

func (c *JobCfg) DecodeFile(fpath string) error

DecodeFile reads file content from a given path and decodes it.

func (*JobCfg) ToTaskCfg

func (c *JobCfg) ToTaskCfg() *TaskCfg

ToTaskCfg converts JobCfg to TaskCfg.

func (*JobCfg) ToTaskCfgs

func (c *JobCfg) ToTaskCfgs() map[string]*TaskCfg

ToTaskCfgs converts job config to a map, mapping from upstream source id to task config.

func (*JobCfg) Yaml

func (c *JobCfg) Yaml() ([]byte, error)

Yaml serializes the JobCfg into a YAML document.

type TaskCfg

type TaskCfg struct {
	JobCfg

	// FIXME: remove this item after fix https://github.com/pingcap/tiflow/issues/7304
	NeedExtStorage bool
}

TaskCfg shares same struct as JobCfg, but it only serves one upstream. TaskCfg can be converted to an equivalent DM subtask by ToDMSubTaskCfg. TaskCfg add some internal config for jobmaster/worker.

func (*TaskCfg) ToDMSubTaskCfg

func (c *TaskCfg) ToDMSubTaskCfg(jobID string) *dmconfig.SubTaskConfig

ToDMSubTaskCfg adapts a TaskCfg to a SubTaskCfg for worker now. TODO: fully support all fields

func (*TaskCfg) ToJobCfg

func (c *TaskCfg) ToJobCfg() *JobCfg

ToJobCfg converts TaskCfg to JobCfg.

type UpstreamCfg

type UpstreamCfg struct {
	dmconfig.MySQLInstance `yaml:",inline" toml:",inline" json:",inline"`
	DBCfg                  *dbconfig.DBConfig `yaml:"db-config" toml:"db-config" json:"db-config"`
	ServerID               uint32             `yaml:"server-id" toml:"server-id" json:"server-id"`
	Flavor                 string             `yaml:"flavor" toml:"flavor" json:"flavor"`
	EnableGTID             bool               `yaml:"enable-gtid" toml:"enable-gtid" json:"enable-gtid"`
	CaseSensitive          bool               `yaml:"case-sensitive" toml:"case-sensitive" json:"case-sensitive"`
}

UpstreamCfg copies the needed fields from DM SourceCfg and MySQLInstance part of DM task config.