config

package
v1.0.7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 21, 2021 License: Apache-2.0 Imports: 18 Imported by: 14

Documentation

Index

Constants

View Source 
const (
	AllChecking                  = "all"
	DumpPrivilegeChecking        = "dump_privilege"
	ReplicationPrivilegeChecking = "replication_privilege"
	VersionChecking              = "version"
	BinlogEnableChecking         = "binlog_enable"
	BinlogFormatChecking         = "binlog_format"
	BinlogRowImageChecking       = "binlog_row_image"
	TableSchemaChecking          = "table_schema"
	ShardTableSchemaChecking     = "schema_of_shard_tables"
	ShardAutoIncrementIDChecking = "auto_increment_ID"
)

DM definition checking items refer github.com/pingcap/tidb-tools/pkg/check

View Source 
const (
	ModeAll       = "all"
	ModeFull      = "full"
	ModeIncrement = "incremental"
)

task modes

View Source 
const (
	GHOST = "gh-ost"
	PT    = "pt"
)

Online DDL Scheme

View Source 
const MaxSourceIDLength = 32

MaxSourceIDLength is the max length for dm-worker source id

Variables

View Source 
var AllCheckingItems = map[string]string{
	AllChecking:                  "all checking items",
	DumpPrivilegeChecking:        "dump privileges of source DB checking item",
	ReplicationPrivilegeChecking: "replication privileges of source DB checking item",
	VersionChecking:              "MySQL/MariaDB version checking item",
	BinlogEnableChecking:         "binlog enable checking item",
	BinlogFormatChecking:         "binlog format checking item",
	BinlogRowImageChecking:       "binlog row image checking item",
	TableSchemaChecking:          "table schema compatibility checking item",
	ShardTableSchemaChecking:     "consistent schema of shard tables checking item",
	ShardAutoIncrementIDChecking: "conflict auto increment ID of shard tables checking item",
}

AllCheckingItems contains all checking items

Functions

func FilterCheckingItems 

func FilterCheckingItems(ignoredItems []string) map[string]string

FilterCheckingItems filters ignored items from all checking items

func SupportCheckingItems 

func SupportCheckingItems() string

SupportCheckingItems returns all supporting checking item

func ValidateCheckingItem 

func ValidateCheckingItem(item string) error

ValidateCheckingItem validates checking item

Types

type DBConfig 

type DBConfig struct {
	Host             string            `toml:"host" json:"host" yaml:"host"`
	Port             int               `toml:"port" json:"port" yaml:"port"`
	User             string            `toml:"user" json:"user" yaml:"user"`
	Password         string            `toml:"password" json:"-" yaml:"password"` // omit it for privacy
	MaxAllowedPacket *int              `toml:"max-allowed-packet" json:"max-allowed-packet" yaml:"max-allowed-packet"`
	Session          map[string]string `toml:"session" json:"session" yaml:"session"`

	RawDBCfg *RawDBConfig `toml:"-" json:"-" yaml:"-"`
}

DBConfig is the DB configuration.

func (*DBConfig) Adjust 

func (db *DBConfig) Adjust()

Adjust adjusts the config.

func (*DBConfig) Decode 

func (db *DBConfig) Decode(data string) error

Decode loads config from file data

func (*DBConfig) String 

func (db *DBConfig) String() string

func (*DBConfig) Toml 

func (db *DBConfig) Toml() (string, error)

Toml returns TOML format representation of config

type LoaderConfig 

type LoaderConfig struct {
	PoolSize int    `yaml:"pool-size" toml:"pool-size" json:"pool-size"`
	Dir      string `yaml:"dir" toml:"dir" json:"dir"`
}

LoaderConfig represents loader process unit's specific config

func (*LoaderConfig) UnmarshalYAML 

func (m *LoaderConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML

type Meta 

type Meta struct {
	BinLogName string `yaml:"binlog-name"`
	BinLogPos  uint32 `yaml:"binlog-pos"`
}

Meta represents binlog's meta pos NOTE: refine to put these config structs into pkgs NOTE: now, syncer does not support GTID mode and which is supported by relay

func (*Meta) Verify 

func (m *Meta) Verify() error

Verify does verification on configs

type MySQLInstance 

type MySQLInstance struct {
	// it represents a MySQL/MariaDB instance or a replica group
	SourceID           string   `yaml:"source-id"`
	Meta               *Meta    `yaml:"meta"`
	FilterRules        []string `yaml:"filter-rules"`
	ColumnMappingRules []string `yaml:"column-mapping-rules"`
	RouteRules         []string `yaml:"route-rules"`
	BWListName         string   `yaml:"black-white-list"`

	MydumperConfigName string          `yaml:"mydumper-config-name"`
	Mydumper           *MydumperConfig `yaml:"mydumper"`
	// MydumperThread is alias for Threads in MydumperConfig, and its priority is higher than Threads
	MydumperThread int `yaml:"mydumper-thread"`

	LoaderConfigName string        `yaml:"loader-config-name"`
	Loader           *LoaderConfig `yaml:"loader"`
	// LoaderThread is alias for PoolSize in LoaderConfig, and its priority is higher than PoolSize
	LoaderThread int `yaml:"loader-thread"`

	SyncerConfigName string        `yaml:"syncer-config-name"`
	Syncer           *SyncerConfig `yaml:"syncer"`
	// SyncerThread is alias for WorkerCount in SyncerConfig, and its priority is higher than WorkerCount
	SyncerThread int `yaml:"syncer-thread"`
}

MySQLInstance represents a sync config of a MySQL instance

func (*MySQLInstance) Verify 

func (m *MySQLInstance) Verify() error

Verify does verification on configs

type MydumperConfig 

type MydumperConfig struct {
	MydumperPath  string `yaml:"mydumper-path" toml:"mydumper-path" json:"mydumper-path"`    // mydumper binary path
	Threads       int    `yaml:"threads" toml:"threads" json:"threads"`                      // -t, --threads
	ChunkFilesize int64  `yaml:"chunk-filesize" toml:"chunk-filesize" json:"chunk-filesize"` // -F, --chunk-filesize
	SkipTzUTC     bool   `yaml:"skip-tz-utc" toml:"skip-tz-utc" json:"skip-tz-utc"`          // --skip-tz-utc
	ExtraArgs     string `yaml:"extra-args" toml:"extra-args" json:"extra-args"`             // other extra args

}

MydumperConfig represents mydumper process unit's specific config

func (*MydumperConfig) UnmarshalYAML 

func (m *MydumperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML

type RawDBConfig added in v1.0.2 

type RawDBConfig struct {
	MaxIdleConns int
	ReadTimeout  string
	WriteTimeout string
}

RawDBConfig contains some low level database config

func DefaultRawDBConfig added in v1.0.2 

func DefaultRawDBConfig() *RawDBConfig

DefaultRawDBConfig returns a default raw database config

func (*RawDBConfig) SetMaxIdleConns added in v1.0.2 

func (c *RawDBConfig) SetMaxIdleConns(value int) *RawDBConfig

SetMaxIdleConns set maxIdleConns for raw database config set value <= 0 then no idle connections are retained. set value > 0 then `value` idle connections are retained.

func (*RawDBConfig) SetReadTimeout added in v1.0.2 

func (c *RawDBConfig) SetReadTimeout(readTimeout string) *RawDBConfig

SetReadTimeout set readTimeout for raw database config

func (*RawDBConfig) SetWriteTimeout added in v1.0.2 

func (c *RawDBConfig) SetWriteTimeout(writeTimeout string) *RawDBConfig

SetWriteTimeout set writeTimeout for raw database config

type SubTaskConfig 

type SubTaskConfig struct {

	// when in sharding, multi dm-workers do one task
	IsSharding      bool   `toml:"is-sharding" json:"is-sharding"`
	OnlineDDLScheme string `toml:"online-ddl-scheme" json:"online-ddl-scheme"`

	// handle schema/table name mode, and only for schema/table name/pattern
	// if case insensitive, we would convert schema/table name/pattern to lower case
	CaseSensitive bool `toml:"case-sensitive" json:"case-sensitive"`

	Name string `toml:"name" json:"name"`
	Mode string `toml:"mode" json:"mode"`
	//  treat it as hidden configuration
	IgnoreCheckingItems []string `toml:"ignore-checking-items" json:"ignore-checking-items"`
	// it represents a MySQL/MariaDB instance or a replica group
	SourceID                string `toml:"source-id" json:"source-id"`
	ServerID                uint32 `toml:"server-id" json:"server-id"`
	Flavor                  string `toml:"flavor" json:"flavor"`
	MetaSchema              string `toml:"meta-schema" json:"meta-schema"`
	RemoveMeta              bool   `toml:"remove-meta" json:"remove-meta"`
	DisableHeartbeat        bool   `toml:"disable-heartbeat" json:"disable-heartbeat"` //  deprecated, use !enable-heartbeat instead
	HeartbeatUpdateInterval int    `toml:"heartbeat-update-interval" json:"heartbeat-update-interval"`
	HeartbeatReportInterval int    `toml:"heartbeat-report-interval" json:"heartbeat-report-interval"`
	EnableHeartbeat         bool   `toml:"enable-heartbeat" json:"enable-heartbeat"`
	Meta                    *Meta  `toml:"meta" json:"meta"`
	Timezone                string `toml:"timezone" josn:"timezone"`

	BinlogType string `toml:"binlog-type" json:"binlog-type"`
	// RelayDir get value from dm-worker config
	RelayDir string   `toml:"relay-dir" json:"relay-dir"`
	From     DBConfig `toml:"from" json:"from"`
	To       DBConfig `toml:"to" json:"to"`

	RouteRules         []*router.TableRule   `toml:"route-rules" json:"route-rules"`
	FilterRules        []*bf.BinlogEventRule `toml:"filter-rules" json:"filter-rules"`
	ColumnMappingRules []*column.Rule        `toml:"mapping-rule" json:"mapping-rule"`
	BWList             *filter.Rules         `toml:"black-white-list" json:"black-white-list"`

	MydumperConfig // Mydumper configuration
	LoaderConfig   // Loader configuration
	SyncerConfig   // Syncer configuration

	// compatible with standalone dm unit
	LogLevel  string `toml:"log-level" json:"log-level"`
	LogFile   string `toml:"log-file" json:"log-file"`
	LogRotate string `toml:"log-rotate" json:"log-rotate"`

	PprofAddr  string `toml:"pprof-addr" json:"pprof-addr"`
	StatusAddr string `toml:"status-addr" json:"status-addr"`

	ConfigFile string `toml:"-" json:"config-file"`
	// contains filtered or unexported fields
}

SubTaskConfig is the configuration for SubTask

func NewSubTaskConfig 

func NewSubTaskConfig() *SubTaskConfig

NewSubTaskConfig creates a new SubTaskConfig

func (*SubTaskConfig) Adjust 

func (c *SubTaskConfig) Adjust(verifyDecryptPassword bool) error

Adjust adjusts configs

func (*SubTaskConfig) Clone 

func (c *SubTaskConfig) Clone() (*SubTaskConfig, error)

Clone returns a replica of SubTaskConfig

func (*SubTaskConfig) Decode 

func (c *SubTaskConfig) Decode(data string, verifyDecryptPassword bool) error

Decode loads config from file data

func (*SubTaskConfig) DecodeFile 

func (c *SubTaskConfig) DecodeFile(fpath string, verifyDecryptPassword bool) error

DecodeFile loads and decodes config from file

func (*SubTaskConfig) DecryptPassword 

func (c *SubTaskConfig) DecryptPassword() (*SubTaskConfig, error)

DecryptPassword tries to decrypt db password in config

func (*SubTaskConfig) Parse 

func (c *SubTaskConfig) Parse(arguments []string, verifyDecryptPassword bool) error

Parse parses flag definitions from the argument list.

func (*SubTaskConfig) String 

func (c *SubTaskConfig) String() string

String returns the config's json string

func (*SubTaskConfig) Toml 

func (c *SubTaskConfig) Toml() (string, error)

Toml returns TOML format representation of config

type SyncerConfig 

type SyncerConfig struct {
	MetaFile    string `yaml:"meta-file" toml:"meta-file" json:"meta-file"` // meta filename, used only when load SubConfig directly
	WorkerCount int    `yaml:"worker-count" toml:"worker-count" json:"worker-count"`
	Batch       int    `yaml:"batch" toml:"batch" json:"batch"`
	QueueSize   int    `yaml:"queue-size" toml:"queue-size" json:"queue-size"`
	// checkpoint flush interval in seconds.
	CheckpointFlushInterval int `yaml:"checkpoint-flush-interval" toml:"checkpoint-flush-interval" json:"checkpoint-flush-interval"`

	// deprecated
	MaxRetry int `yaml:"max-retry" toml:"max-retry" json:"max-retry"`

	// refine following configs to top level configs?
	AutoFixGTID      bool `yaml:"auto-fix-gtid" toml:"auto-fix-gtid" json:"auto-fix-gtid"`
	EnableGTID       bool `yaml:"enable-gtid" toml:"enable-gtid" json:"enable-gtid"`
	DisableCausality bool `yaml:"disable-detect" toml:"disable-detect" json:"disable-detect"`
	SafeMode         bool `yaml:"safe-mode" toml:"safe-mode" json:"safe-mode"`
	EnableANSIQuotes bool `yaml:"enable-ansi-quotes" toml:"enable-ansi-quotes" json:"enable-ansi-quotes"`
}

SyncerConfig represents syncer process unit's specific config

func (*SyncerConfig) UnmarshalYAML 

func (m *SyncerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements Unmarshaler.UnmarshalYAML

type TaskConfig 

type TaskConfig struct {
	*flag.FlagSet `yaml:"-"`

	Name       string `yaml:"name"`
	TaskMode   string `yaml:"task-mode"`
	IsSharding bool   `yaml:"is-sharding"`
	//  treat it as hidden configuration
	IgnoreCheckingItems []string `yaml:"ignore-checking-items"`
	// we store detail status in meta
	// don't save configuration into it
	MetaSchema string `yaml:"meta-schema"`
	// remove meta from downstreaming database
	// now we delete checkpoint and online ddl information
	RemoveMeta              bool   `yaml:"remove-meta"`
	DisableHeartbeat        bool   `yaml:"disable-heartbeat"` //  deprecated, use !enable-heartbeat instead
	EnableHeartbeat         bool   `yaml:"enable-heartbeat"`
	HeartbeatUpdateInterval int    `yaml:"heartbeat-update-interval"`
	HeartbeatReportInterval int    `yaml:"heartbeat-report-interval"`
	Timezone                string `yaml:"timezone"`

	// handle schema/table name mode, and only for schema/table name
	// if case insensitive, we would convert schema/table name to lower case
	CaseSensitive bool `yaml:"case-sensitive"`

	TargetDB *DBConfig `yaml:"target-database"`

	MySQLInstances []*MySQLInstance `yaml:"mysql-instances"`

	OnlineDDLScheme string `yaml:"online-ddl-scheme"`

	Routes         map[string]*router.TableRule   `yaml:"routes"`
	Filters        map[string]*bf.BinlogEventRule `yaml:"filters"`
	ColumnMappings map[string]*column.Rule        `yaml:"column-mappings"`
	BWList         map[string]*filter.Rules       `yaml:"black-white-list"`

	Mydumpers map[string]*MydumperConfig `yaml:"mydumpers"`
	Loaders   map[string]*LoaderConfig   `yaml:"loaders"`
	Syncers   map[string]*SyncerConfig   `yaml:"syncers"`
}

TaskConfig is the configuration for Task

func NewTaskConfig 

func NewTaskConfig() *TaskConfig

NewTaskConfig creates a TaskConfig

func (*TaskConfig) Decode 

func (c *TaskConfig) Decode(data string) error

Decode loads config from file data

func (*TaskConfig) DecodeFile 

func (c *TaskConfig) DecodeFile(fpath string) error

DecodeFile loads and decodes config from file

func (*TaskConfig) String 

func (c *TaskConfig) String() string

String returns the config's yaml string

func (*TaskConfig) SubTaskConfigs 

func (c *TaskConfig) SubTaskConfigs(sources map[string]DBConfig) ([]*SubTaskConfig, error)

SubTaskConfigs generates sub task configs

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.