config

package
v0.0.0-...-4b22ae8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 13, 2024 License: Apache-2.0 Imports: 48 Imported by: 2

Documentation

Index

Constants

View Source 
const (
	AllChecking                  = "all"
	DumpPrivilegeChecking        = "dump_privilege"
	ReplicationPrivilegeChecking = "replication_privilege"
	VersionChecking              = "version"
	ServerIDChecking             = "server_id"
	BinlogEnableChecking         = "binlog_enable"
	BinlogFormatChecking         = "binlog_format"
	BinlogRowImageChecking       = "binlog_row_image"
	TableSchemaChecking          = "table_schema"
	ShardTableSchemaChecking     = "schema_of_shard_tables"
	ShardAutoIncrementIDChecking = "auto_increment_ID"
	OnlineDDLChecking            = "online_ddl"
	BinlogDBChecking             = "binlog_db"
	MetaPositionChecking         = "meta_position"
	ConnNumberChecking           = "conn_number"
	TargetDBPrivilegeChecking    = "target_privilege"
	// lighting prechecks.
	LightningEmptyRegionChecking        = "empty_region"
	LightningRegionDistributionChecking = "region_distribution"
	LightningDownstreamVersionChecking  = "downstream_version"
	LightningFreeSpaceChecking          = "free_space"
	LightningMutexFeatureChecking       = "downstream_mutex_features"
	LightningTableEmptyChecking         = "downstream_table_empty"
)

DM definition checking items. Don't forget to update AllCheckingItems and LightningPrechecks.

View Source 
const (
	ModeAll       = "all"
	ModeFull      = "full"
	ModeIncrement = "incremental"
	ModeDump      = "dump"
	ModeLoad      = "load"
	ModeLoadSync  = "load&sync"

	DefaultShadowTableRules = "^_(.+)_(?:new|gho)$"
	DefaultTrashTableRules  = "^_(.+)_(?:ghc|del|old)$"

	ShadowTableRules              = "shadow-table-rules"
	TrashTableRules               = "trash-table-rules"
	TiDBLightningCheckpointPrefix = "tidb_lightning_checkpoint_"
)

task modes.

View Source 
const (
	GHOST = "gh-ost"
	PT    = "pt"
)

Online DDL Scheme.

View Source 
const (
	ShardPessimistic = "pessimistic"
	ShardOptimistic  = "optimistic"
)

shard DDL mode.

View Source 
const (
	LooseCollationCompatible  = "loose"
	StrictCollationCompatible = "strict"
)

collation_compatible.

View Source 
const (
	ValidationNone = "none"
	ValidationFast = "fast"
	ValidationFull = "full"

	DefaultValidatorWorkerCount       = 4
	DefaultValidatorValidateInterval  = 10 * time.Second
	DefaultValidatorCheckInterval     = 5 * time.Second
	DefaultValidatorRowErrorDelay     = 30 * time.Minute
	DefaultValidatorMetaFlushInterval = 5 * time.Minute
	DefaultValidatorBatchQuerySize    = 100
	DefaultValidatorMaxPendingRowSize = "500m"

	ValidatorMaxAccumulatedRow = 100000
	// PendingRow is substantial in this version (in sysbench test)
	// set to MaxInt temporaly and reset in the future.
	DefaultValidatorMaxPendingRow = math.MaxInt32
)
View Source 
const (
	OpLevelRequired = "required"
	OpLevelOptional = "optional"
	OpLevelOff      = "off"
)
View Source 
const MaxSourceIDLength = 32

MaxSourceIDLength is the max length for dm-worker source id.

Variables

View Source 
var (
	DefaultCheckInterval           = 5 * time.Second
	DefaultBackoffRollback         = 5 * time.Minute
	DefaultBackoffMin              = 1 * time.Second
	DefaultBackoffMax              = 5 * time.Minute
	DefaultBackoffJitter           = true
	DefaultBackoffFactor   float64 = 2
)

Backoff related constants.

View Source 
var AllCheckingItems = map[string]string{
	AllChecking:                  "all checking items",
	DumpPrivilegeChecking:        "dump privileges of source DB checking item",
	ReplicationPrivilegeChecking: "replication privileges of source DB checking item",
	VersionChecking:              "MySQL/MariaDB version checking item",
	ServerIDChecking:             "server_id checking item",
	BinlogEnableChecking:         "binlog enable checking item",
	BinlogFormatChecking:         "binlog format checking item",
	BinlogRowImageChecking:       "binlog row image checking item",
	TableSchemaChecking:          "table schema compatibility checking item",
	ShardTableSchemaChecking:     "consistent schema of shard tables checking item",
	ShardAutoIncrementIDChecking: "conflict auto increment ID of shard tables checking item",
	OnlineDDLChecking:            "online ddl checking item",
	BinlogDBChecking:             "binlog db checking item",
	MetaPositionChecking:         "meta position valid checking item",
	ConnNumberChecking:           "connection number checking item",
	TargetDBPrivilegeChecking:    "privileges of target DB checking item",

	LightningEmptyRegionChecking:        "physical import mode empty region checking item",
	LightningRegionDistributionChecking: "physical import mode region distribution checking item",
	LightningDownstreamVersionChecking:  "physical import mode downstream TiDB/PD/TiKV version checking item",
	LightningFreeSpaceChecking:          "downstream free space checking item",
	LightningMutexFeatureChecking:       "physical import mode downstream incompatible feature checking item",
	LightningTableEmptyChecking:         "physical import mode downstream table empty checking item",
}

AllCheckingItems contains all checking items.

View Source 
var LightningPrechecks = []string{
	LightningEmptyRegionChecking,
	LightningRegionDistributionChecking,
	LightningDownstreamVersionChecking,
	LightningFreeSpaceChecking,
	LightningMutexFeatureChecking,
	LightningTableEmptyChecking,
}

LightningPrechecks returns all checking items for lightning.

View Source 
var ObfuscatedPasswordForFeedback string = "******"

ObfuscatedPasswordForFeedback is the source encryption password that returns to the foreground. PM's requirement, we always return obfuscated password to users.

View Source 
var SampleSourceConfig string

SampleSourceConfig is sample config file of source. The embed source.yaml is a copy of dm/master/source.yaml, because embed can only match regular files in the current directory and subdirectories.

View Source 
var SampleSubtaskConfig string

SampleSubtaskConfig is the content of subtask.toml in current folder.

Functions

func AdjustTargetDBSessionCfg 

func AdjustTargetDBSessionCfg(dbConfig *dbconfig.DBConfig, version *semver.Version)

AdjustTargetDBSessionCfg adjust session cfg of TiDB.

func FetchTimeZoneSetting 

func FetchTimeZoneSetting(ctx context.Context, db *sql.DB) (string, error)

FetchTimeZoneSetting fetch target db global time_zone setting. TODO: move GetTimeZoneOffset and FormatTimeZoneOffset from TiDB to tiflow.

func FilterCheckingItems 

func FilterCheckingItems(ignoredItems []string) map[string]string

FilterCheckingItems filters ignored items from all checking items.

func GetDBConfigForTest 

func GetDBConfigForTest() dbconfig.DBConfig

GetDBConfigForTest is a helper function to get db config for unit test .

func GetTargetDBCfgFromOpenAPITask 

func GetTargetDBCfgFromOpenAPITask(task *openapi.Task) *dbconfig.DBConfig

GetTargetDBCfgFromOpenAPITask gets target db config.

func HasDump 

func HasDump(taskMode string) bool

HasDump returns true if taskMode contains dump unit.

func HasLoad 

func HasLoad(taskMode string) bool

HasLoad returns true if taskMode contains load unit.

func HasSync 

func HasSync(taskMode string) bool

HasSync returns true if taskMode contains sync unit.

func NewLoaderConfigForDowngrade 

func NewLoaderConfigForDowngrade(loaderConfigs map[string]*LoaderConfig) map[string]*LoaderConfigForDowngrade

func NewSyncerConfigsForDowngrade 

func NewSyncerConfigsForDowngrade(syncerConfigs map[string]*SyncerConfig) map[string]*SyncerConfigForDowngrade

NewSyncerConfigsForDowngrade converts SyncerConfig to SyncerConfigForDowngrade.

func SourceCfgToOpenAPISource 

func SourceCfgToOpenAPISource(cfg *SourceConfig) openapi.Source

SourceCfgToOpenAPISource converter SourceConfig to openapi.Source.

func SubTaskConfigsToOpenAPITask 

func SubTaskConfigsToOpenAPITask(subTaskConfigList []*SubTaskConfig) *openapi.Task

SubTaskConfigsToOpenAPITask gets openapi task from sub task configs.

func SubTaskConfigsToOpenAPITaskList 

func SubTaskConfigsToOpenAPITaskList(subTaskConfigMap map[string]map[string]*SubTaskConfig) []*openapi.Task

SubTaskConfigsToOpenAPITaskList gets openapi task from sub task configs. subTaskConfigMap: taskName -> sourceName -> SubTaskConfig.

func SupportCheckingItems 

func SupportCheckingItems() string

SupportCheckingItems returns all supporting checking item.

func TaskConfigToOpenAPITask 

func TaskConfigToOpenAPITask(c *TaskConfig, sourceCfgMap map[string]*SourceConfig) (*openapi.Task, error)

TaskConfigToOpenAPITask converts TaskConfig to an openapi task.

func ValidateCheckingItem 

func ValidateCheckingItem(item string) error

ValidateCheckingItem validates checking item.

Types

type CheckerConfig 

type CheckerConfig struct {
	CheckEnable     bool     `yaml:"check-enable" toml:"check-enable" json:"check-enable"`
	BackoffRollback Duration `yaml:"backoff-rollback" toml:"backoff-rollback" json:"backoff-rollback"`
	BackoffMax      Duration `yaml:"backoff-max" toml:"backoff-max" json:"backoff-max"`
	// unexpose config
	CheckInterval Duration `yaml:"check-interval" toml:"check-interval" json:"-"`
	BackoffMin    Duration `yaml:"backoff-min" toml:"backoff-min" json:"-"`
	BackoffJitter bool     `yaml:"backoff-jitter" toml:"backoff-jitter" json:"-"`
	BackoffFactor float64  `yaml:"backoff-factor" toml:"backoff-factor" json:"-"`
}

CheckerConfig is configuration used for TaskStatusChecker.

func (*CheckerConfig) Adjust 

func (cc *CheckerConfig) Adjust()

Adjust sets default value for field: CheckInterval/BackoffMin/BackoffJitter/BackoffFactor.

type Duration 

type Duration struct {
	time.Duration
}

Duration is used to hold a time.Duration field.

func (*Duration) MarshalJSON 

func (d *Duration) MarshalJSON() ([]byte, error)

MarshalJSON hacks to satisfy the json.Marshaler interface.

func (Duration) MarshalText 

func (d Duration) MarshalText() ([]byte, error)

MarshalText hacks to satisfy the encoding.TextMarshaler interface For MarshalText, we should use (d Duration) which can be used by both pointer and instance.

func (*Duration) UnmarshalText 

func (d *Duration) UnmarshalText(text []byte) error

UnmarshalText hacks to satisfy the encoding.TextUnmarshaler interface For UnmarshalText, we should use (d *Duration) to change the value of this instance instead of the copy.

type ExpressionFilter 

type ExpressionFilter struct {
	Schema             string `yaml:"schema" toml:"schema" json:"schema"`
	Table              string `yaml:"table" toml:"table" json:"table"`
	InsertValueExpr    string `yaml:"insert-value-expr" toml:"insert-value-expr" json:"insert-value-expr"`
	UpdateOldValueExpr string `yaml:"update-old-value-expr" toml:"update-old-value-expr" json:"update-old-value-expr"`
	UpdateNewValueExpr string `yaml:"update-new-value-expr" toml:"update-new-value-expr" json:"update-new-value-expr"`
	DeleteValueExpr    string `yaml:"delete-value-expr" toml:"delete-value-expr" json:"delete-value-expr"`
}

ExpressionFilter represents a filter that will be applied on row changes. one ExpressionFilter can only have one of (insert, update, delete) expressions. there are two update expressions, which form an AND logic. If user omits one expression, DM will use "TRUE" for it.

type LoadMode 

type LoadMode string

LoadMode defines different mode used in load phase. 

const (
	// LoadModeSQL means write data by sql statements, uses tidb-lightning tidb backend to load data.
	// deprecated, use LoadModeLogical instead.
	LoadModeSQL LoadMode = "sql"
	// LoadModeLoader is the legacy sql mode, use loader to load data. this should be replaced by LoadModeLogical mode.
	// deprecated, use LoadModeLogical instead.
	LoadModeLoader LoadMode = "loader"
	// LoadModeLogical means use tidb backend of lightning to load data, which uses SQL to load data.
	LoadModeLogical LoadMode = "logical"
	// LoadModePhysical means use local backend of lightning to load data, which ingest SST files to load data.
	LoadModePhysical LoadMode = "physical"
)

type LoaderConfig 

type LoaderConfig struct {
	PoolSize           int      `yaml:"pool-size" toml:"pool-size" json:"pool-size"`
	Dir                string   `yaml:"dir" toml:"dir" json:"dir"`
	SortingDirPhysical string   `yaml:"sorting-dir-physical" toml:"sorting-dir-physical" json:"sorting-dir-physical"`
	SQLMode            string   `yaml:"-" toml:"-" json:"-"` // wrote by dump unit (DM op) or jobmaster (DM in engine)
	ImportMode         LoadMode `yaml:"import-mode" toml:"import-mode" json:"import-mode"`
	// deprecated, use OnDuplicateLogical instead.
	OnDuplicate         LogicalDuplicateResolveType  `yaml:"on-duplicate" toml:"on-duplicate" json:"on-duplicate"`
	OnDuplicateLogical  LogicalDuplicateResolveType  `yaml:"on-duplicate-logical" toml:"on-duplicate-logical" json:"on-duplicate-logical"`
	OnDuplicatePhysical PhysicalDuplicateResolveType `yaml:"on-duplicate-physical" toml:"on-duplicate-physical" json:"on-duplicate-physical"`
	DiskQuotaPhysical   config.ByteSize              `yaml:"disk-quota-physical" toml:"disk-quota-physical" json:"disk-quota-physical"`
	ChecksumPhysical    PhysicalPostOpLevel          `yaml:"checksum-physical" toml:"checksum-physical" json:"checksum-physical"`
	Analyze             PhysicalPostOpLevel          `yaml:"analyze" toml:"analyze" json:"analyze"`
	RangeConcurrency    int                          `yaml:"range-concurrency" toml:"range-concurrency" json:"range-concurrency"`
	CompressKVPairs     string                       `yaml:"compress-kv-pairs" toml:"compress-kv-pairs" json:"compress-kv-pairs"`
	PDAddr              string                       `yaml:"pd-addr" toml:"pd-addr" json:"pd-addr"`
}

LoaderConfig represents loader process unit's specific config.

func DefaultLoaderConfig 

func DefaultLoaderConfig() LoaderConfig

DefaultLoaderConfig return default loader config for task.

func (*LoaderConfig) UnmarshalYAML 

func (m *LoaderConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML.

type LoaderConfigForDowngrade 

type LoaderConfigForDowngrade struct {
	PoolSize int    `yaml:"pool-size" toml:"pool-size" json:"pool-size"`
	Dir      string `yaml:"dir" toml:"dir" json:"dir"`
}

LoaderConfigForDowngrade is the base configuration for loader in v2.0. This config is used for downgrade(config export) from a higher dmctl version. When we add any new config item into LoaderConfig, we should update it also.

type LogicalDuplicateResolveType 

type LogicalDuplicateResolveType string

LogicalDuplicateResolveType defines the duplication resolution when meet duplicate rows for logical import. 

const (
	// OnDuplicateReplace represents replace the old row with new data.
	OnDuplicateReplace LogicalDuplicateResolveType = "replace"
	// OnDuplicateError represents return an error when meet duplicate row.
	OnDuplicateError LogicalDuplicateResolveType = "error"
	// OnDuplicateIgnore represents ignore the new data when meet duplicate row.
	OnDuplicateIgnore LogicalDuplicateResolveType = "ignore"
)

type Meta 

type Meta struct {
	BinLogName string `toml:"binlog-name" yaml:"binlog-name"`
	BinLogPos  uint32 `toml:"binlog-pos" yaml:"binlog-pos"`
	BinLogGTID string `toml:"binlog-gtid" yaml:"binlog-gtid"`
}

Meta represents binlog's meta pos NOTE: refine to put these config structs into pkgs NOTE: now, syncer does not support GTID mode and which is supported by relay.

func (*Meta) Verify 

func (m *Meta) Verify() error

Verify does verification on configs NOTE: we can't decide to verify `binlog-name` or `binlog-gtid` until being bound to a source (with `enable-gtid` set).

type MySQLInstance 

type MySQLInstance struct {
	// it represents a MySQL/MariaDB instance or a replica group
	SourceID    string   `yaml:"source-id"`
	Meta        *Meta    `yaml:"meta"`
	FilterRules []string `yaml:"filter-rules"`
	// deprecated
	ColumnMappingRules []string `yaml:"column-mapping-rules"`
	RouteRules         []string `yaml:"route-rules"`
	ExpressionFilters  []string `yaml:"expression-filters"`

	// black-white-list is deprecated, use block-allow-list instead
	BWListName string `yaml:"black-white-list"`
	BAListName string `yaml:"block-allow-list"`

	MydumperConfigName string          `yaml:"mydumper-config-name"`
	Mydumper           *MydumperConfig `yaml:"mydumper"`
	// MydumperThread is alias for Threads in MydumperConfig, and its priority is higher than Threads
	MydumperThread int `yaml:"mydumper-thread"`

	LoaderConfigName string        `yaml:"loader-config-name"`
	Loader           *LoaderConfig `yaml:"loader"`
	// LoaderThread is alias for PoolSize in LoaderConfig, and its priority is higher than PoolSize
	LoaderThread int `yaml:"loader-thread"`

	SyncerConfigName string        `yaml:"syncer-config-name"`
	Syncer           *SyncerConfig `yaml:"syncer"`
	// SyncerThread is alias for WorkerCount in SyncerConfig, and its priority is higher than WorkerCount
	SyncerThread int `yaml:"syncer-thread"`

	ContinuousValidatorConfigName string          `yaml:"validator-config-name"`
	ContinuousValidator           ValidatorConfig `yaml:"-"`
}

MySQLInstance represents a sync config of a MySQL instance.

func (*MySQLInstance) VerifyAndAdjust 

func (m *MySQLInstance) VerifyAndAdjust() error

VerifyAndAdjust does verification on configs, and adjust some configs.

type MySQLInstanceForDowngrade 

type MySQLInstanceForDowngrade struct {
	SourceID           string          `yaml:"source-id"`
	Meta               *Meta           `yaml:"meta"`
	FilterRules        []string        `yaml:"filter-rules"`
	ColumnMappingRules []string        `yaml:"column-mapping-rules"`
	RouteRules         []string        `yaml:"route-rules"`
	BWListName         string          `yaml:"black-white-list"`
	BAListName         string          `yaml:"block-allow-list"`
	MydumperConfigName string          `yaml:"mydumper-config-name"`
	Mydumper           *MydumperConfig `yaml:"mydumper"`
	MydumperThread     int             `yaml:"mydumper-thread"`
	LoaderConfigName   string          `yaml:"loader-config-name"`
	Loader             *LoaderConfig   `yaml:"loader"`
	LoaderThread       int             `yaml:"loader-thread"`
	SyncerConfigName   string          `yaml:"syncer-config-name"`
	Syncer             *SyncerConfig   `yaml:"syncer"`
	SyncerThread       int             `yaml:"syncer-thread"`
	// new config item
	ExpressionFilters []string `yaml:"expression-filters,omitempty"`
}

MySQLInstanceForDowngrade represents a sync config of a MySQL instance for downgrade.

func NewMySQLInstancesForDowngrade 

func NewMySQLInstancesForDowngrade(mysqlInstances []*MySQLInstance) []*MySQLInstanceForDowngrade

NewMySQLInstancesForDowngrade creates []* MySQLInstanceForDowngrade.

type MydumperConfig 

type MydumperConfig struct {
	MydumperPath  string `yaml:"mydumper-path" toml:"mydumper-path" json:"mydumper-path"`    // mydumper binary path
	Threads       int    `yaml:"threads" toml:"threads" json:"threads"`                      // -t, --threads
	ChunkFilesize string `yaml:"chunk-filesize" toml:"chunk-filesize" json:"chunk-filesize"` // -F, --chunk-filesize
	StatementSize uint64 `yaml:"statement-size" toml:"statement-size" json:"statement-size"` // -S, --statement-size
	Rows          uint64 `yaml:"rows" toml:"rows" json:"rows"`                               // -r, --rows
	Where         string `yaml:"where" toml:"where" json:"where"`                            // --where

	SkipTzUTC bool   `yaml:"skip-tz-utc" toml:"skip-tz-utc" json:"skip-tz-utc"` // --skip-tz-utc
	ExtraArgs string `yaml:"extra-args" toml:"extra-args" json:"extra-args"`    // other extra args

}

MydumperConfig represents mydumper process unit's specific config.

func DefaultMydumperConfig 

func DefaultMydumperConfig() MydumperConfig

DefaultMydumperConfig return default mydumper config for task.

func (*MydumperConfig) UnmarshalYAML 

func (m *MydumperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML.

type PhysicalDuplicateResolveType 

type PhysicalDuplicateResolveType string

PhysicalDuplicateResolveType defines the duplication resolution when meet duplicate rows for physical import. 

const (
	// OnDuplicateNone represents do nothing when meet duplicate row and the task will continue.
	OnDuplicateNone PhysicalDuplicateResolveType = "none"
	// OnDuplicateManual represents that task should be paused when meet duplicate row to let user handle it manually.
	OnDuplicateManual PhysicalDuplicateResolveType = "manual"
)

type PhysicalPostOpLevel 

type PhysicalPostOpLevel string

PhysicalPostOpLevel defines the configuration of checksum/analyze of physical import.

type PurgeConfig 

type PurgeConfig struct {
	Interval    int64 `yaml:"interval" toml:"interval" json:"interval"`             // check whether need to purge at this @Interval (seconds)
	Expires     int64 `yaml:"expires" toml:"expires" json:"expires"`                // if file's modified time is older than @Expires (hours), then it can be purged
	RemainSpace int64 `yaml:"remain-space" toml:"remain-space" json:"remain-space"` // if remain space in @RelayBaseDir less than @RemainSpace (GB), then it can be purged
}

PurgeConfig is the configuration for Purger.

type SourceConfig 

type SourceConfig struct {
	Enable     bool `yaml:"enable" toml:"enable" json:"enable"`
	EnableGTID bool `yaml:"enable-gtid" toml:"enable-gtid" json:"enable-gtid"`
	// deprecated
	AutoFixGTID bool   `yaml:"auto-fix-gtid" toml:"auto-fix-gtid" json:"auto-fix-gtid"`
	RelayDir    string `yaml:"relay-dir" toml:"relay-dir" json:"relay-dir"`
	// deprecated
	MetaDir string `yaml:"meta-dir" toml:"meta-dir" json:"meta-dir"`
	Flavor  string `yaml:"flavor" toml:"flavor" json:"flavor"`
	// deprecated
	Charset string `yaml:"charset" toml:"charset" json:"charset"`

	EnableRelay bool `yaml:"enable-relay" toml:"enable-relay" json:"enable-relay"`
	// relay synchronous starting point (if specified)
	RelayBinLogName string `yaml:"relay-binlog-name" toml:"relay-binlog-name" json:"relay-binlog-name"`
	RelayBinlogGTID string `yaml:"relay-binlog-gtid" toml:"relay-binlog-gtid" json:"relay-binlog-gtid"`
	// only use when the source is bound to a worker, do not marsh it
	UUIDSuffix int `yaml:"-" toml:"-" json:"-"`

	SourceID string            `yaml:"source-id" toml:"source-id" json:"source-id"`
	From     dbconfig.DBConfig `yaml:"from" toml:"from" json:"from"`

	// config items for purger
	Purge PurgeConfig `yaml:"purge" toml:"purge" json:"purge"`

	// config items for task status checker
	Checker CheckerConfig `yaml:"checker" toml:"checker" json:"checker"`

	// id of the worker on which this task run
	ServerID uint32 `yaml:"server-id" toml:"server-id" json:"server-id"`

	// deprecated tracer, to keep compatibility with older version
	Tracer map[string]interface{} `yaml:"tracer" toml:"tracer" json:"-"`

	CaseSensitive bool                  `yaml:"case-sensitive" toml:"case-sensitive" json:"case-sensitive"`
	Filters       []*bf.BinlogEventRule `yaml:"filters" toml:"filters" json:"filters"`
}

SourceConfig is the configuration for source.

func LoadFromFile 

func LoadFromFile(path string) (*SourceConfig, error)

LoadFromFile loads config from file.

func NewSourceConfig 

func NewSourceConfig() *SourceConfig

NewSourceConfig creates a new base config for upstream MySQL/MariaDB source.

func OpenAPISourceToSourceCfg 

func OpenAPISourceToSourceCfg(source openapi.Source) *SourceConfig

OpenAPISourceToSourceCfg converter openapi.Source to SourceConfig.

func SourceCfgFromYaml 

func SourceCfgFromYaml(content string) (*SourceConfig, error)

SourceCfgFromYaml parses flag definitions from the argument list, content should be yaml format.

func SourceCfgFromYamlAndVerify 

func SourceCfgFromYamlAndVerify(content string) (*SourceConfig, error)

SourceCfgFromYamlAndVerify does SourceCfgFromYaml and Verify.

func (*SourceConfig) Adjust 

func (c *SourceConfig) Adjust(ctx context.Context, db *conn.BaseDB) (err error)

Adjust flavor and server-id of SourceConfig.

func (*SourceConfig) AdjustCaseSensitive 

func (c *SourceConfig) AdjustCaseSensitive(ctx context.Context, db *conn.BaseDB) (err error)

AdjustCaseSensitive adjust CaseSensitive from DB.

func (*SourceConfig) AdjustFlavor 

func (c *SourceConfig) AdjustFlavor(ctx context.Context, db *conn.BaseDB) (err error)

AdjustFlavor adjust Flavor from DB.

func (*SourceConfig) AdjustServerID 

func (c *SourceConfig) AdjustServerID(ctx context.Context, db *conn.BaseDB) error

AdjustServerID adjust server id from DB.

func (*SourceConfig) Clone 

func (c *SourceConfig) Clone() *SourceConfig

Clone clones a config.

func (*SourceConfig) EncodeToml 

func (c *SourceConfig) EncodeToml() (string, error)

EncodeToml encodes config.

func (*SourceConfig) FromToml 

func (c *SourceConfig) FromToml(content string) error

FromToml parses flag definitions from the argument list. accept toml content for legacy use (mainly used by etcd).

func (*SourceConfig) GenerateDBConfig 

func (c *SourceConfig) GenerateDBConfig() *dbconfig.DBConfig

GenerateDBConfig creates DBConfig for DB.

func (*SourceConfig) GetDecryptedClone 

func (c *SourceConfig) GetDecryptedClone() *SourceConfig

GetDecryptedClone returns a decrypted config replica in config.

func (*SourceConfig) String 

func (c *SourceConfig) String() string

func (*SourceConfig) Toml 

func (c *SourceConfig) Toml() (string, error)

Toml returns TOML format representation of config.

func (*SourceConfig) Verify 

func (c *SourceConfig) Verify() error

Verify verifies the config.

func (*SourceConfig) Yaml 

func (c *SourceConfig) Yaml() (string, error)

Yaml returns YAML format representation of config.

func (*SourceConfig) YamlForDowngrade 

func (c *SourceConfig) YamlForDowngrade() (string, error)

YamlForDowngrade returns YAML format represents of config for downgrade.

type SourceConfigForDowngrade 

type SourceConfigForDowngrade struct {
	Enable          bool                   `yaml:"enable,omitempty"`
	EnableGTID      bool                   `yaml:"enable-gtid"`
	RelayDir        string                 `yaml:"relay-dir"`
	Flavor          string                 `yaml:"flavor"`
	Charset         string                 `yaml:"charset"`
	EnableRelay     bool                   `yaml:"enable-relay"`
	RelayBinLogName string                 `yaml:"relay-binlog-name"`
	RelayBinlogGTID string                 `yaml:"relay-binlog-gtid"`
	UUIDSuffix      int                    `yaml:"-"`
	SourceID        string                 `yaml:"source-id"`
	From            dbconfig.DBConfig      `yaml:"from"`
	Purge           PurgeConfig            `yaml:"purge"`
	Checker         CheckerConfig          `yaml:"checker"`
	ServerID        uint32                 `yaml:"server-id"`
	Tracer          map[string]interface{} `yaml:"tracer"`
	// any new config item, we mark it omitempty
	CaseSensitive bool                  `yaml:"case-sensitive,omitempty"`
	Filters       []*bf.BinlogEventRule `yaml:"filters,omitempty"`
}

SourceConfigForDowngrade is the base configuration for source in v2.0. This config is used for downgrade(config export) from a higher dmctl version. When we add any new config item into SourceConfig, we should update it also.

func NewSourceConfigForDowngrade 

func NewSourceConfigForDowngrade(sourceCfg *SourceConfig) *SourceConfigForDowngrade

NewSourceConfigForDowngrade creates a new base config for downgrade.

func (*SourceConfigForDowngrade) Yaml 

func (c *SourceConfigForDowngrade) Yaml() (string, error)

Yaml returns YAML format representation of the config.

type SubTaskConfig 

type SubTaskConfig struct {

	// when in sharding, multi dm-workers do one task
	IsSharding                bool   `toml:"is-sharding" json:"is-sharding"`
	ShardMode                 string `toml:"shard-mode" json:"shard-mode"`
	StrictOptimisticShardMode bool   `toml:"strict-optimistic-shard-mode" json:"strict-optimistic-shard-mode"`
	OnlineDDL                 bool   `toml:"online-ddl" json:"online-ddl"`

	// pt/gh-ost name rule, support regex
	ShadowTableRules []string `yaml:"shadow-table-rules" toml:"shadow-table-rules" json:"shadow-table-rules"`
	TrashTableRules  []string `yaml:"trash-table-rules" toml:"trash-table-rules" json:"trash-table-rules"`

	// deprecated
	OnlineDDLScheme string `toml:"online-ddl-scheme" json:"online-ddl-scheme"`

	// handle schema/table name mode, and only for schema/table name/pattern
	// if case insensitive, we would convert schema/table name/pattern to lower case
	CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"`

	// default "loose" handle create sql by original sql, will not add default collation as upstream
	// "strict" will add default collation as upstream, and downstream will occur error when downstream don't support
	CollationCompatible string `yaml:"collation_compatible" toml:"collation_compatible" json:"collation_compatible"`

	Name string `toml:"name" json:"name"`
	Mode string `toml:"mode" json:"mode"`
	//  treat it as hidden configuration
	IgnoreCheckingItems []string `toml:"ignore-checking-items" json:"ignore-checking-items"`
	// it represents a MySQL/MariaDB instance or a replica group
	SourceID   string `toml:"source-id" json:"source-id"`
	ServerID   uint32 `toml:"server-id" json:"server-id"`
	Flavor     string `toml:"flavor" json:"flavor"`
	MetaSchema string `toml:"meta-schema" json:"meta-schema"`
	// deprecated
	HeartbeatUpdateInterval int `toml:"heartbeat-update-interval" json:"heartbeat-update-interval"`
	// deprecated
	HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"`
	// deprecated
	EnableHeartbeat bool   `toml:"enable-heartbeat" json:"enable-heartbeat"`
	Timezone        string `toml:"timezone" json:"timezone"`

	Meta *Meta `toml:"meta" json:"meta"`

	// RelayDir get value from dm-worker config
	RelayDir string `toml:"relay-dir" json:"relay-dir"`

	// UseRelay get value from dm-worker's relayEnabled
	UseRelay bool              `toml:"use-relay" json:"use-relay"`
	From     dbconfig.DBConfig `toml:"from" json:"from"`
	To       dbconfig.DBConfig `toml:"to" json:"to"`

	RouteRules  []*router.TableRule   `toml:"route-rules" json:"route-rules"`
	FilterRules []*bf.BinlogEventRule `toml:"filter-rules" json:"filter-rules"`
	// deprecated
	ColumnMappingRules []*column.Rule      `toml:"mapping-rule" json:"mapping-rule"`
	ExprFilter         []*ExpressionFilter `yaml:"expression-filter" toml:"expression-filter" json:"expression-filter"`

	// black-white-list is deprecated, use block-allow-list instead
	BWList *filter.Rules `toml:"black-white-list" json:"black-white-list"`
	BAList *filter.Rules `toml:"block-allow-list" json:"block-allow-list"`

	MydumperConfig // Mydumper configuration
	LoaderConfig   // Loader configuration
	SyncerConfig   // Syncer configuration
	ValidatorCfg   ValidatorConfig

	// compatible with standalone dm unit
	LogLevel  string `toml:"log-level" json:"log-level"`
	LogFile   string `toml:"log-file" json:"log-file"`
	LogFormat string `toml:"log-format" json:"log-format"`
	LogRotate string `toml:"log-rotate" json:"log-rotate"`

	PprofAddr  string `toml:"pprof-addr" json:"pprof-addr"`
	StatusAddr string `toml:"status-addr" json:"status-addr"`

	ConfigFile string `toml:"-" json:"config-file"`

	CleanDumpFile bool `toml:"clean-dump-file" json:"clean-dump-file"`

	// deprecated, will auto discover SQL mode
	EnableANSIQuotes bool `toml:"ansi-quotes" json:"ansi-quotes"`

	// which DM worker is running the subtask, this will be injected when the real worker starts running the subtask(StartSubTask).
	WorkerName string `toml:"-" json:"-"`
	// task experimental configs
	Experimental struct {
		AsyncCheckpointFlush bool `yaml:"async-checkpoint-flush" toml:"async-checkpoint-flush" json:"async-checkpoint-flush"`
	} `yaml:"experimental" toml:"experimental" json:"experimental"`

	// members below are injected by dataflow engine
	ExtStorage      extstorage.ExternalStorage `toml:"-" json:"-"`
	MetricsFactory  promutil.Factory           `toml:"-" json:"-"`
	FrameworkLogger *zap.Logger                `toml:"-" json:"-"`
	// members below are injected by dataflow engine
	// UUID should be unique in one go runtime.
	// IOTotalBytes is used build TCPConnWithIOCounter and UUID is used to as a
	// key to let MySQL driver to find the right TCPConnWithIOCounter.
	// It will meter TCP io usage to downstream of the subtask
	UUID         string         `toml:"uuid" json:"-"`
	IOTotalBytes *atomic.Uint64 `toml:"io-total-bytes" json:"io-total-bytes"`

	// DumpUUID as same as UUID
	// DumpIOTotalBytes meter TCP io usage from upstream of the subtask, other same as IOTotalBytes
	// e.g., pulling binlog
	DumpUUID         string         `toml:"dump-uuid" json:"-"`
	DumpIOTotalBytes *atomic.Uint64 `toml:"dump-io-total-bytes" json:"dump-io-total-bytes"`
	// contains filtered or unexported fields
}

SubTaskConfig is the configuration for SubTask.

func NewSubTaskConfig 

func NewSubTaskConfig() *SubTaskConfig

NewSubTaskConfig creates a new SubTaskConfig.

func OpenAPITaskToSubTaskConfigs 

func OpenAPITaskToSubTaskConfigs(task *openapi.Task, toDBCfg *dbconfig.DBConfig, sourceCfgMap map[string]*SourceConfig) (
	[]*SubTaskConfig, error,
)

OpenAPITaskToSubTaskConfigs generates sub task configs by openapi.Task.

func TaskConfigToSubTaskConfigs 

func TaskConfigToSubTaskConfigs(c *TaskConfig, sources map[string]dbconfig.DBConfig) ([]*SubTaskConfig, error)

TaskConfigToSubTaskConfigs generates sub task configs by TaskConfig.

func (*SubTaskConfig) Adjust 

func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error

Adjust adjusts and verifies configs.

func (*SubTaskConfig) Clone 

func (c *SubTaskConfig) Clone() (*SubTaskConfig, error)

Clone returns a replica of SubTaskConfig.

func (*SubTaskConfig) Decode 

func (c *SubTaskConfig) Decode(data string, verifyDecryptPassword bool) error

Decode loads config from file data. Note: The atomic.Uint64 fields (IOTotalBytes and DumpIOTotalBytes) will not be populated from the TOML data since they cannot be decoded by toml.Decode(). As a result, these fields will remain uninitialized (zero value) after decoding.

func (*SubTaskConfig) DecodeFile 

func (c *SubTaskConfig) DecodeFile(fpath string, verifyDecryptPassword bool) error

DecodeFile loads and decodes config from file.

func (*SubTaskConfig) DecryptedClone 

func (c *SubTaskConfig) DecryptedClone() (*SubTaskConfig, error)

DecryptedClone tries to decrypt db password in config.

func (*SubTaskConfig) GetFlagSet 

func (c *SubTaskConfig) GetFlagSet() *flag.FlagSet

GetFlagSet provides the pointer of subtask's flag set.

func (*SubTaskConfig) InitIOCounters 

func (c *SubTaskConfig) InitIOCounters()

InitIOCounters init io counter and uuid for syncer.

func (*SubTaskConfig) Parse 

func (c *SubTaskConfig) Parse(arguments []string, verifyDecryptPassword bool) error

Parse parses flag definitions from the argument list.

func (*SubTaskConfig) SetFlagSet 

func (c *SubTaskConfig) SetFlagSet(flagSet *flag.FlagSet)

SetFlagSet writes back the flag set.

func (*SubTaskConfig) String 

func (c *SubTaskConfig) String() string

String returns the config's json string.

func (*SubTaskConfig) Toml 

func (c *SubTaskConfig) Toml() (string, error)

Toml returns TOML format representation of config. Note: The atomic.Uint64 fields (IOTotalBytes and DumpIOTotalBytes) are not encoded in the TOML output because they do not implement the necessary marshaling interfaces. As a result, these fields will not be included in the TOML representation.

type SyncerConfig 

type SyncerConfig struct {
	MetaFile    string `yaml:"meta-file" toml:"meta-file" json:"meta-file"` // meta filename, used only when load SubConfig directly
	WorkerCount int    `yaml:"worker-count" toml:"worker-count" json:"worker-count"`
	Batch       int    `yaml:"batch" toml:"batch" json:"batch"`
	QueueSize   int    `yaml:"queue-size" toml:"queue-size" json:"queue-size"`
	// checkpoint flush interval in seconds.
	CheckpointFlushInterval int `yaml:"checkpoint-flush-interval" toml:"checkpoint-flush-interval" json:"checkpoint-flush-interval"`
	// TODO: add this two new config items for openapi.
	Compact      bool `yaml:"compact" toml:"compact" json:"compact"`
	MultipleRows bool `yaml:"multiple-rows" toml:"multiple-rows" json:"multiple-rows"`

	// deprecated
	MaxRetry int `yaml:"max-retry" toml:"max-retry" json:"max-retry"`

	// deprecated
	AutoFixGTID bool `yaml:"auto-fix-gtid" toml:"auto-fix-gtid" json:"auto-fix-gtid"`
	EnableGTID  bool `yaml:"enable-gtid" toml:"enable-gtid" json:"enable-gtid"`
	// deprecated
	DisableCausality bool   `yaml:"disable-detect" toml:"disable-detect" json:"disable-detect"`
	SafeMode         bool   `yaml:"safe-mode" toml:"safe-mode" json:"safe-mode"`
	SafeModeDuration string `yaml:"safe-mode-duration" toml:"safe-mode-duration" json:"safe-mode-duration"`
	// deprecated, use `ansi-quotes` in top level config instead
	EnableANSIQuotes bool `yaml:"enable-ansi-quotes" toml:"enable-ansi-quotes" json:"enable-ansi-quotes"`
}

SyncerConfig represents syncer process unit's specific config.

func DefaultSyncerConfig 

func DefaultSyncerConfig() SyncerConfig

DefaultSyncerConfig return default syncer config for task.

func (*SyncerConfig) UnmarshalYAML 

func (m *SyncerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML.

type SyncerConfigForDowngrade 

type SyncerConfigForDowngrade struct {
	MetaFile                string `yaml:"meta-file"`
	WorkerCount             int    `yaml:"worker-count"`
	Batch                   int    `yaml:"batch"`
	QueueSize               int    `yaml:"queue-size"`
	CheckpointFlushInterval int    `yaml:"checkpoint-flush-interval"`
	MaxRetry                int    `yaml:"max-retry"`
	EnableGTID              bool   `yaml:"enable-gtid"`
	DisableCausality        bool   `yaml:"disable-detect"`
	SafeMode                bool   `yaml:"safe-mode"`
	EnableANSIQuotes        bool   `yaml:"enable-ansi-quotes"`

	SafeModeDuration string `yaml:"safe-mode-duration,omitempty"`
	Compact          bool   `yaml:"compact,omitempty"`
	MultipleRows     bool   `yaml:"multipleRows,omitempty"`
}

SyncerConfigForDowngrade is the base configuration for syncer in v2.0. This config is used for downgrade(config export) from a higher dmctl version. When we add any new config item into SyncerConfig, we should update it also.

type TaskCliArgs 

type TaskCliArgs struct {
	StartTime        string `toml:"start-time" json:"start_time"`
	SafeModeDuration string `toml:"safe-mode-duration" json:"safe_mode_duration"`
	WaitTimeOnStop   string `toml:"wait-time-on-stop" json:"wait_time_on_stop"`
}

TaskCliArgs is the task command line arguments, these arguments have higher priority than the config file and downstream checkpoint, but may need to be removed after the first time they take effect.

func OpenAPIStartTaskReqToTaskCliArgs 

func OpenAPIStartTaskReqToTaskCliArgs(req openapi.StartTaskRequest) (*TaskCliArgs, error)

func OpenAPIStopTaskReqToTaskCliArgs 

func OpenAPIStopTaskReqToTaskCliArgs(req openapi.StopTaskRequest) (*TaskCliArgs, error)

func (*TaskCliArgs) Decode 

func (t *TaskCliArgs) Decode(data []byte) error

Decode load a json representation of TaskCliArgs.

func (*TaskCliArgs) ToJSON 

func (t *TaskCliArgs) ToJSON() (string, error)

ToJSON returns json marshal result.

func (*TaskCliArgs) Verify 

func (t *TaskCliArgs) Verify() error

Verify checks if all fields are legal.

type TaskConfig 

type TaskConfig struct {
	*flag.FlagSet `yaml:"-" toml:"-" json:"-"`

	Name                      string `yaml:"name" toml:"name" json:"name"`
	TaskMode                  string `yaml:"task-mode" toml:"task-mode" json:"task-mode"`
	IsSharding                bool   `yaml:"is-sharding" toml:"is-sharding" json:"is-sharding"`
	ShardMode                 string `yaml:"shard-mode" toml:"shard-mode" json:"shard-mode"` // when `shard-mode` set, we always enable sharding support.
	StrictOptimisticShardMode bool   `yaml:"strict-optimistic-shard-mode" toml:"strict-optimistic-shard-mode" json:"strict-optimistic-shard-mode"`
	// treat it as hidden configuration
	IgnoreCheckingItems []string `yaml:"ignore-checking-items" toml:"ignore-checking-items" json:"ignore-checking-items"`
	// we store detail status in meta
	// don't save configuration into it
	MetaSchema string `yaml:"meta-schema" toml:"meta-schema" json:"meta-schema"`
	// deprecated
	EnableHeartbeat bool `yaml:"enable-heartbeat" toml:"enable-heartbeat" json:"enable-heartbeat"`
	// deprecated
	HeartbeatUpdateInterval int `yaml:"heartbeat-update-interval" toml:"heartbeat-update-interval" json:"heartbeat-update-interval"`
	// deprecated
	HeartbeatReportInterval int    `yaml:"heartbeat-report-interval" toml:"heartbeat-report-interval" json:"heartbeat-report-interval"`
	Timezone                string `yaml:"timezone" toml:"timezone" json:"timezone"`

	// handle schema/table name mode, and only for schema/table name
	// if case insensitive, we would convert schema/table name to lower case
	CaseSensitive bool `yaml:"case-sensitive" toml:"case-sensitive" json:"case-sensitive"`

	// default "loose" handle create sql by original sql, will not add default collation as upstream
	// "strict" will add default collation as upstream, and downstream will occur error when downstream don't support
	CollationCompatible string `yaml:"collation_compatible" toml:"collation_compatible" json:"collation_compatible"`

	TargetDB *dbconfig.DBConfig `yaml:"target-database" toml:"target-database" json:"target-database"`

	MySQLInstances []*MySQLInstance `yaml:"mysql-instances" toml:"mysql-instances" json:"mysql-instances"`

	OnlineDDL bool `yaml:"online-ddl" toml:"online-ddl" json:"online-ddl"`
	// pt/gh-ost name rule,support regex
	ShadowTableRules []string `yaml:"shadow-table-rules" toml:"shadow-table-rules" json:"shadow-table-rules"`
	TrashTableRules  []string `yaml:"trash-table-rules" toml:"trash-table-rules" json:"trash-table-rules"`

	// deprecated
	OnlineDDLScheme string `yaml:"online-ddl-scheme" toml:"online-ddl-scheme" json:"online-ddl-scheme"`

	Routes  map[string]*router.TableRule   `yaml:"routes" toml:"routes" json:"routes"`
	Filters map[string]*bf.BinlogEventRule `yaml:"filters" toml:"filters" json:"filters"`
	// deprecated
	ColumnMappings map[string]*column.Rule      `yaml:"column-mappings" toml:"column-mappings" json:"column-mappings"`
	ExprFilter     map[string]*ExpressionFilter `yaml:"expression-filter" toml:"expression-filter" json:"expression-filter"`

	// black-white-list is deprecated, use block-allow-list instead
	BWList map[string]*filter.Rules `yaml:"black-white-list" toml:"black-white-list" json:"black-white-list"`
	BAList map[string]*filter.Rules `yaml:"block-allow-list" toml:"block-allow-list" json:"block-allow-list"`

	Mydumpers  map[string]*MydumperConfig  `yaml:"mydumpers" toml:"mydumpers" json:"mydumpers"`
	Loaders    map[string]*LoaderConfig    `yaml:"loaders" toml:"loaders" json:"loaders"`
	Syncers    map[string]*SyncerConfig    `yaml:"syncers" toml:"syncers" json:"syncers"`
	Validators map[string]*ValidatorConfig `yaml:"validators" toml:"validators" json:"validators"`

	CleanDumpFile bool `yaml:"clean-dump-file" toml:"clean-dump-file" json:"clean-dump-file"`
	// deprecated
	EnableANSIQuotes bool `yaml:"ansi-quotes" toml:"ansi-quotes" json:"ansi-quotes"`

	// deprecated, replaced by `start-task --remove-meta`
	RemoveMeta bool `yaml:"remove-meta"`

	// task experimental configs
	Experimental struct {
		AsyncCheckpointFlush bool `yaml:"async-checkpoint-flush" toml:"async-checkpoint-flush" json:"async-checkpoint-flush"`
	} `yaml:"experimental" toml:"experimental" json:"experimental"`
}

TaskConfig is the configuration for Task.

func NewTaskConfig 

func NewTaskConfig() *TaskConfig

NewTaskConfig creates a TaskConfig.

func OpenAPITaskToTaskConfig 

func OpenAPITaskToTaskConfig(task *openapi.Task, sourceCfgMap map[string]*SourceConfig) (*TaskConfig, error)

OpenAPITaskToTaskConfig converts an openapi task to TaskConfig.

func SubTaskConfigsToTaskConfig 

func SubTaskConfigsToTaskConfig(stCfgs ...*SubTaskConfig) *TaskConfig

SubTaskConfigsToTaskConfig constructs task configs from a list of valid subtask configs.

func (*TaskConfig) Adjust 

func (c *TaskConfig) Adjust() error

Adjust adjusts and verifies config.

func (*TaskConfig) DecodeFile 

func (c *TaskConfig) DecodeFile(fpath string) error

DecodeFile loads and decodes config from file.

func (*TaskConfig) FromYaml 

func (c *TaskConfig) FromYaml(data string) error

FromYaml loads config from file data.

func (*TaskConfig) JSON 

func (c *TaskConfig) JSON() string

JSON returns the config's json string.

func (*TaskConfig) RawDecode 

func (c *TaskConfig) RawDecode(data string) error

RawDecode loads config from file data.

func (*TaskConfig) String 

func (c *TaskConfig) String() string

String returns the config's yaml string.

func (*TaskConfig) YamlForDowngrade 

func (c *TaskConfig) YamlForDowngrade() (string, error)

YamlForDowngrade returns YAML format represents of config for downgrade.

type TaskConfigForDowngrade 

type TaskConfigForDowngrade struct {
	Name                    string                               `yaml:"name"`
	TaskMode                string                               `yaml:"task-mode"`
	IsSharding              bool                                 `yaml:"is-sharding"`
	ShardMode               string                               `yaml:"shard-mode"`
	IgnoreCheckingItems     []string                             `yaml:"ignore-checking-items"`
	MetaSchema              string                               `yaml:"meta-schema"`
	EnableHeartbeat         bool                                 `yaml:"enable-heartbeat"`
	HeartbeatUpdateInterval int                                  `yaml:"heartbeat-update-interval"`
	HeartbeatReportInterval int                                  `yaml:"heartbeat-report-interval"`
	Timezone                string                               `yaml:"timezone"`
	CaseSensitive           bool                                 `yaml:"case-sensitive"`
	TargetDB                *dbconfig.DBConfig                   `yaml:"target-database"`
	OnlineDDLScheme         string                               `yaml:"online-ddl-scheme"`
	Routes                  map[string]*router.TableRule         `yaml:"routes"`
	Filters                 map[string]*bf.BinlogEventRule       `yaml:"filters"`
	ColumnMappings          map[string]*column.Rule              `yaml:"column-mappings"`
	BWList                  map[string]*filter.Rules             `yaml:"black-white-list"`
	BAList                  map[string]*filter.Rules             `yaml:"block-allow-list"`
	Mydumpers               map[string]*MydumperConfig           `yaml:"mydumpers"`
	Loaders                 map[string]*LoaderConfigForDowngrade `yaml:"loaders"`
	Syncers                 map[string]*SyncerConfigForDowngrade `yaml:"syncers"`
	CleanDumpFile           bool                                 `yaml:"clean-dump-file"`
	EnableANSIQuotes        bool                                 `yaml:"ansi-quotes"`
	RemoveMeta              bool                                 `yaml:"remove-meta"`
	// new config item
	MySQLInstances            []*MySQLInstanceForDowngrade `yaml:"mysql-instances"`
	ExprFilter                map[string]*ExpressionFilter `yaml:"expression-filter,omitempty"`
	OnlineDDL                 bool                         `yaml:"online-ddl,omitempty"`
	ShadowTableRules          []string                     `yaml:"shadow-table-rules,omitempty"`
	TrashTableRules           []string                     `yaml:"trash-table-rules,omitempty"`
	StrictOptimisticShardMode bool                         `yaml:"strict-optimistic-shard-mode,omitempty"`
}

TaskConfigForDowngrade is the base configuration for task in v2.0. This config is used for downgrade(config export) from a higher dmctl version. When we add any new config item into SourceConfig, we should update it also.

func NewTaskConfigForDowngrade 

func NewTaskConfigForDowngrade(taskConfig *TaskConfig) *TaskConfigForDowngrade

NewTaskConfigForDowngrade create new TaskConfigForDowngrade.

func (*TaskConfigForDowngrade) Yaml 

func (c *TaskConfigForDowngrade) Yaml() (string, error)

Yaml returns YAML format representation of config.

type ValidatorConfig 

type ValidatorConfig struct {
	Mode               string   `yaml:"mode" toml:"mode" json:"mode"`
	WorkerCount        int      `yaml:"worker-count" toml:"worker-count" json:"worker-count"`
	ValidateInterval   Duration `yaml:"validate-interval" toml:"validate-interval" json:"validate-interval"`
	CheckInterval      Duration `yaml:"check-interval" toml:"check-interval" json:"check-interval"`
	RowErrorDelay      Duration `yaml:"row-error-delay" toml:"row-error-delay" json:"row-error-delay"`
	MetaFlushInterval  Duration `yaml:"meta-flush-interval" toml:"meta-flush-interval" json:"meta-flush-interval"`
	BatchQuerySize     int      `yaml:"batch-query-size" toml:"batch-query-size" json:"batch-query-size"`
	MaxPendingRowSize  string   `yaml:"max-pending-row-size" toml:"max-pending-row-size" json:"max-pending-row-size"`
	MaxPendingRowCount int      `yaml:"max-pending-row-count" toml:"max-pending-row-count" json:"max-pending-row-count"`
	StartTime          string   `yaml:"-" toml:"start-time" json:"-"`
}

func (*ValidatorConfig) Adjust 

func (v *ValidatorConfig) Adjust() error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.