Documentation
¶
Index ¶
Constants ¶
View Source
const ( // LocalFilePerm is the permission for local files LocalFilePerm os.FileMode = 0o644 // LogFileName is the filename of the log LogFileName = "sync_diff.log" // UnifiedTimeZone is the time zone UnifiedTimeZone string = "+0:00" )
Variables ¶
This section is empty.
Functions ¶
func DecryptOrPlaintext ¶
DecryptOrPlaintext tries to decrypt base64 encoded ciphertext to plaintext or return plaintext.
func ExportTemplateConfig ¶
ExportTemplateConfig print the configType
Types ¶
type Config ¶
type Config struct {
*flag.FlagSet `json:"-"`
// log level
LogLevel string `toml:"-" json:"-"`
// how many goroutines are created to check data
CheckThreadCount int `toml:"check-thread-count" json:"check-thread-count"`
// how many goroutines are created to split chunk. A goroutine splits one table at a time.
SplitThreadCount int `toml:"-" json:"split-thread-count"`
// set true if want to compare rows
// set false won't compare rows.
ExportFixSQL bool `toml:"export-fix-sql" json:"export-fix-sql"`
// only check table struct without table data.
CheckStructOnly bool `toml:"check-struct-only" json:"check-struct-only"`
// experimental feature: only check table data without table struct
CheckDataOnly bool `toml:"check-data-only" json:"-"`
// skip validation for tables that don't exist upstream or downstream
SkipNonExistingTable bool `toml:"skip-non-existing-table" json:"-"`
// DMAddr is dm-master's address, the format should like "http://127.0.0.1:8261"
DMAddr string `toml:"dm-addr" json:"dm-addr"`
// DMTask string `toml:"dm-task" json:"dm-task"`
DMTask string `toml:"dm-task" json:"dm-task"`
DataSources map[string]*DataSource `toml:"data-sources" json:"data-sources"`
Routes map[string]*router.TableRule `toml:"routes" json:"routes"`
TableConfigs map[string]*TableConfig `toml:"table-configs" json:"table-configs"`
Task TaskConfig `toml:"task" json:"task"`
// config file
ConfigFile string
// export a template config file
Template string `toml:"-" json:"-"`
// print version if set true
PrintVersion bool
}
Config is the configuration.
func (*Config) CheckConfig ¶
CheckConfig check whether the config is vaild
type DBConfig ¶
type DBConfig struct {
Host string `toml:"host" json:"host" yaml:"host"`
Port int `toml:"port" json:"port" yaml:"port"`
User string `toml:"user" json:"user" yaml:"user"`
Password string `toml:"password" json:"-" yaml:"password"` // omit it for privacy
// deprecated, mysql driver could automatically fetch this value
MaxAllowedPacket *int `toml:"max-allowed-packet" json:"max-allowed-packet" yaml:"max-allowed-packet"`
Session map[string]string `toml:"session" json:"session" yaml:"session"`
// security config
Security *security.Security `toml:"security" json:"security" yaml:"security"`
}
DBConfig is the DB configuration.
type DataSource ¶
type DataSource struct {
Host string `toml:"host" json:"host"`
Port int `toml:"port" json:"port"`
User string `toml:"user" json:"user"`
Password utils.SecretString `toml:"password" json:"password"`
SQLMode string `toml:"sql-mode" json:"sql-mode"`
Snapshot string `toml:"snapshot" json:"snapshot"`
Security *Security `toml:"security" json:"security"`
RouteRules []string `toml:"route-rules" json:"route-rules"`
Router *router.Table
RouteTargetSet map[string]struct{} `json:"-"`
Conn *sql.DB
}
DataSource represents the Source Config.
func (*DataSource) IsAutoSnapshot ¶
func (d *DataSource) IsAutoSnapshot() bool
IsAutoSnapshot returns true if the tidb_snapshot is expected to automatically be set from the syncpoint from the target TiDB instance.
func (*DataSource) RegisterTLS ¶
func (d *DataSource) RegisterTLS() error
RegisterTLS register TLS config for driver
func (*DataSource) SetSnapshot ¶
func (d *DataSource) SetSnapshot(newSnapshot string)
SetSnapshot changes the snapshot in configuration. This is typically used with the auto-snapshot feature.
func (*DataSource) ToDBConfig ¶
func (d *DataSource) ToDBConfig() *dbutil.DBConfig
ToDBConfig get the current config from data source
func (*DataSource) ToDriverConfig ¶
func (d *DataSource) ToDriverConfig() *mysql.Config
ToDriverConfig get the driver config
type Security ¶
type Security struct {
TLSName string `json:"tls-name"`
CAPath string `toml:"ca-path" json:"ca-path"`
CertPath string `toml:"cert-path" json:"cert-path"`
KeyPath string `toml:"key-path" json:"key-path"`
// raw content
CABytes string `toml:"ca-bytes" json:"ca-bytes"`
CertBytes string `toml:"cert-bytes" json:"cert-bytes"`
KeyBytes string `toml:"key-bytes" json:"key-bytes"`
}
Security is the wrapper for TLS Security
type SubTaskConfig ¶
type SubTaskConfig struct {
// when in sharding, multi dm-workers do one task
IsSharding bool `toml:"is-sharding" json:"is-sharding"`
ShardMode string `toml:"shard-mode" json:"shard-mode"`
StrictOptimisticShardMode bool `toml:"strict-optimistic-shard-mode" json:"strict-optimistic-shard-mode"`
OnlineDDL bool `toml:"online-ddl" json:"online-ddl"`
// pt/gh-ost name rule, support regex
ShadowTableRules []string `yaml:"shadow-table-rules" toml:"shadow-table-rules" json:"shadow-table-rules"`
TrashTableRules []string `yaml:"trash-table-rules" toml:"trash-table-rules" json:"trash-table-rules"`
// deprecated
OnlineDDLScheme string `toml:"online-ddl-scheme" json:"online-ddl-scheme"`
// handle schema/table name mode, and only for schema/table name/pattern
// if case insensitive, we would convert schema/table name/pattern to lower case
CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"`
// default "loose" handle create sql by original sql, will not add default collation as upstream
// "strict" will add default collation as upstream, and downstream will occur error when downstream don't support
CollationCompatible string `yaml:"collation_compatible" toml:"collation_compatible" json:"collation_compatible"`
Name string `toml:"name" json:"name"`
Mode string `toml:"mode" json:"mode"`
// treat it as hidden configuration
IgnoreCheckingItems []string `toml:"ignore-checking-items" json:"ignore-checking-items"`
// it represents a MySQL/MariaDB instance or a replica group
SourceID string `toml:"source-id" json:"source-id"`
ServerID uint32 `toml:"server-id" json:"server-id"`
Flavor string `toml:"flavor" json:"flavor"`
MetaSchema string `toml:"meta-schema" json:"meta-schema"`
// deprecated
HeartbeatUpdateInterval int `toml:"heartbeat-update-interval" json:"heartbeat-update-interval"`
// deprecated
HeartbeatReportInterval int `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"`
// deprecated
EnableHeartbeat bool `toml:"enable-heartbeat" json:"enable-heartbeat"`
Timezone string `toml:"timezone" json:"timezone"`
// RelayDir get value from dm-worker config
RelayDir string `toml:"relay-dir" json:"relay-dir"`
// UseRelay get value from dm-worker's relayEnabled
UseRelay bool `toml:"use-relay" json:"use-relay"`
From DBConfig `toml:"from" json:"from"`
To DBConfig `toml:"to" json:"to"`
RouteRules []*router.TableRule `toml:"route-rules" json:"route-rules"`
// FilterRules []*bf.BinlogEventRule `toml:"filter-rules" json:"filter-rules"`
// deprecated
ColumnMappingRules []*column.Rule `toml:"mapping-rule" json:"mapping-rule"`
// black-white-list is deprecated, use block-allow-list instead
BWList *filter.Rules `toml:"black-white-list" json:"black-white-list"`
BAList *filter.Rules `toml:"block-allow-list" json:"block-allow-list"`
// compatible with standalone dm unit
LogLevel string `toml:"log-level" json:"log-level"`
LogFile string `toml:"log-file" json:"log-file"`
LogFormat string `toml:"log-format" json:"log-format"`
LogRotate string `toml:"log-rotate" json:"log-rotate"`
PprofAddr string `toml:"pprof-addr" json:"pprof-addr"`
StatusAddr string `toml:"status-addr" json:"status-addr"`
ConfigFile string `toml:"-" json:"config-file"`
CleanDumpFile bool `toml:"clean-dump-file" json:"clean-dump-file"`
// deprecated, will auto discover SQL mode
EnableANSIQuotes bool `toml:"ansi-quotes" json:"ansi-quotes"`
// which DM worker is running the subtask, this will be injected when the real worker starts running the subtask(StartSubTask).
WorkerName string `toml:"-" json:"-"`
// task experimental configs
Experimental struct {
AsyncCheckpointFlush bool `yaml:"async-checkpoint-flush" toml:"async-checkpoint-flush" json:"async-checkpoint-flush"`
} `yaml:"experimental" toml:"experimental" json:"experimental"`
}
SubTaskConfig is the configuration for SubTask.
type TableConfig ¶
type TableConfig struct {
// table's filter to tell us which table should adapt to this config.
TargetTables []string `toml:"target-tables" json:"target-tables"`
// Internally used to indicate which specified table in target is using this config.
Schema string
Table string
// Internally used to distinguish different config.
ConfigIndex int
// Internally used to valid config.
HasMatched bool
// columns be ignored, will not check this column's data
IgnoreColumns []string `toml:"ignore-columns"`
// field should be the primary key, unique key or field with index
Fields []string `toml:"index-fields"`
// select range, for example: "age > 10 AND age < 20"
Range string `toml:"range"`
TargetTableInfo *model.TableInfo
// collation config in mysql/tidb
Collation string `toml:"collation"`
// specify the chunksize for the table
ChunkSize int64 `toml:"chunk-size" json:"chunk-size"`
}
TableConfig is the config of table.
func (*TableConfig) Valid ¶
func (t *TableConfig) Valid() bool
Valid returns true if table's config is valide.
type TaskConfig ¶
type TaskConfig struct {
Source []string `toml:"source-instances" json:"source-instances"`
Routes []string `toml:"source-routes" json:"source-routes"`
Target string `toml:"target-instance" json:"target-instance"`
CheckTables []string `toml:"target-check-tables" json:"target-check-tables"`
TableConfigs []string `toml:"target-configs" json:"target-configs"`
// OutputDir include these
// 1. checkpoint Dir
// 2. fix-target-sql Dir
// 3. summary file
// 4. sync diff log file
// 5. fix
OutputDir string `toml:"output-dir" json:"output-dir"`
SourceInstances []*DataSource
TargetInstance *DataSource
TargetTableConfigs []*TableConfig
TargetCheckTables filter.Filter
FixDir string
CheckpointDir string
HashFile string
}
TaskConfig is the config for sync diff
func (*TaskConfig) ComputeConfigHash ¶
func (t *TaskConfig) ComputeConfigHash() (string, error)
ComputeConfigHash compute the hash according to the task if ConfigHash is as same as checkpoint.hash we think the second sync diff can use the checkpoint.
func (*TaskConfig) Init ¶
func (t *TaskConfig) Init( dataSources map[string]*DataSource, tableConfigs map[string]*TableConfig, ) (err error)
Init return a new config
Source Files
Click to show internal directories.
Click to hide internal directories.