config

package
v5.3.0-alpha+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 2, 2021 License: Apache-2.0 Imports: 29 Imported by: 90

Documentation

Index

Constants

View Source 
const (
	// ImportMode defines mode of import for tikv.
	ImportMode = "import"
	// NormalMode defines mode of normal for tikv.
	NormalMode = "normal"

	// BackendTiDB is a constant for choosing the "TiDB" backend in the configuration.
	BackendTiDB = "tidb"
	// BackendImporter is a constant for choosing the "Importer" backend in the configuration.
	BackendImporter = "importer"
	// BackendLocal is a constant for choosing the "Local" backup in the configuration.
	// In this mode, we write & sort kv pairs with local storage and directly write them to tikv.
	BackendLocal = "local"

	// CheckpointDriverMySQL is a constant for choosing the "MySQL" checkpoint driver in the configuration.
	CheckpointDriverMySQL = "mysql"
	// CheckpointDriverFile is a constant for choosing the "File" checkpoint driver in the configuration.
	CheckpointDriverFile = "file"

	// ReplaceOnDup indicates using REPLACE INTO to insert data
	ReplaceOnDup = "replace"
	// IgnoreOnDup indicates using INSERT IGNORE INTO to insert data
	IgnoreOnDup = "ignore"
	// ErrorOnDup indicates using INSERT INTO to insert data, which would violate PK or UNIQUE constraint
	ErrorOnDup = "error"
)

Variables

View Source 
var (
	DefaultFilter = []string{
		"*.*",
		"!mysql.*",
		"!sys.*",
		"!INFORMATION_SCHEMA.*",
		"!PERFORMANCE_SCHEMA.*",
		"!METRICS_SCHEMA.*",
		"!INSPECTION_SCHEMA.*",
	}
)

Functions

This section is empty.

Types

type AllIgnoreColumns 

type AllIgnoreColumns []*IgnoreColumns

func (AllIgnoreColumns) GetIgnoreColumns 

func (igCols AllIgnoreColumns) GetIgnoreColumns(db string, table string, caseSensitive bool) (*IgnoreColumns, error)

GetIgnoreColumns gets Ignore config by schema name/regex and table name/regex.

type ByteSize 

type ByteSize int64

ByteSize is an alias of int64 which accepts human-friendly strings like '10G' when read from TOML. 

const (
	// mydumper
	ReadBlockSize   ByteSize = 64 * units.KiB
	MinRegionSize   ByteSize = 256 * units.MiB
	MaxRegionSize   ByteSize = 256 * units.MiB
	SplitRegionSize ByteSize = 96 * units.MiB

	BufferSizeScale = 5

	DefaultBatchSize ByteSize = 100 * units.GiB
)

func (*ByteSize) UnmarshalJSON 

func (size *ByteSize) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler (for testing)

func (*ByteSize) UnmarshalText 

func (size *ByteSize) UnmarshalText(b []byte) error

UnmarshalText implements encoding.TextUnmarshaler

type CSVConfig 

type CSVConfig struct {
	Separator       string `toml:"separator" json:"separator"`
	Delimiter       string `toml:"delimiter" json:"delimiter"`
	Terminator      string `toml:"terminator" json:"terminator"`
	Null            string `toml:"null" json:"null"`
	Header          bool   `toml:"header" json:"header"`
	TrimLastSep     bool   `toml:"trim-last-separator" json:"trim-last-separator"`
	NotNull         bool   `toml:"not-null" json:"not-null"`
	BackslashEscape bool   `toml:"backslash-escape" json:"backslash-escape"`
}

type Checkpoint 

type Checkpoint struct {
	Schema           string `toml:"schema" json:"schema"`
	DSN              string `toml:"dsn" json:"-"` // DSN may contain password, don't expose this to JSON.
	Driver           string `toml:"driver" json:"driver"`
	Enable           bool   `toml:"enable" json:"enable"`
	KeepAfterSuccess bool   `toml:"keep-after-success" json:"keep-after-success"`
}

type Config 

type Config struct {
	TaskID int64 `toml:"-" json:"id"`

	App  Lightning `toml:"lightning" json:"lightning"`
	TiDB DBStore   `toml:"tidb" json:"tidb"`

	Checkpoint   Checkpoint          `toml:"checkpoint" json:"checkpoint"`
	Mydumper     MydumperRuntime     `toml:"mydumper" json:"mydumper"`
	TikvImporter TikvImporter        `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  PostRestore         `toml:"post-restore" json:"post-restore"`
	Cron         Cron                `toml:"cron" json:"cron"`
	Routes       []*router.TableRule `toml:"routes" json:"routes"`
	Security     Security            `toml:"security" json:"security"`

	BWList filter.MySQLReplicationRules `toml:"black-white-list" json:"black-white-list"`
}

func NewConfig 

func NewConfig() *Config

func (*Config) Adjust 

func (cfg *Config) Adjust(ctx context.Context) error

Adjust fixes the invalid or unspecified settings to reasonable valid values.

func (*Config) AdjustCheckPoint 

func (cfg *Config) AdjustCheckPoint()

func (*Config) AdjustMydumper 

func (cfg *Config) AdjustMydumper()

func (*Config) CheckAndAdjustFilePath 

func (cfg *Config) CheckAndAdjustFilePath() error

func (*Config) CheckAndAdjustForLocalBackend 

func (cfg *Config) CheckAndAdjustForLocalBackend() error

func (*Config) CheckAndAdjustSecurity 

func (cfg *Config) CheckAndAdjustSecurity() error

func (*Config) CheckAndAdjustTiDBPort 

func (cfg *Config) CheckAndAdjustTiDBPort(ctx context.Context, mustHaveInternalConnections bool) error

func (*Config) DefaultVarsForImporterAndLocalBackend 

func (cfg *Config) DefaultVarsForImporterAndLocalBackend(ctx context.Context)

func (*Config) DefaultVarsForTiDBBackend 

func (cfg *Config) DefaultVarsForTiDBBackend()

func (*Config) HasLegacyBlackWhiteList 

func (cfg *Config) HasLegacyBlackWhiteList() bool

HasLegacyBlackWhiteList checks whether the deprecated [black-white-list] section was defined.

func (*Config) LoadFromGlobal 

func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error

LoadFromGlobal resets the current configuration to the global settings.

func (*Config) LoadFromTOML 

func (cfg *Config) LoadFromTOML(data []byte) error

LoadFromTOML overwrites the current configuration by the TOML data If data contains toml items not in Config and GlobalConfig, return an error If data contains toml items not in Config, thus won't take effect, warn user

func (*Config) String 

func (cfg *Config) String() string

func (*Config) ToTLS 

func (cfg *Config) ToTLS() (*common.TLS, error)

type Cron 

type Cron struct {
	SwitchMode     Duration `toml:"switch-mode" json:"switch-mode"`
	LogProgress    Duration `toml:"log-progress" json:"log-progress"`
	CheckDiskQuota Duration `toml:"check-disk-quota" json:"check-disk-quota"`
}

type DBStore 

type DBStore struct {
	Host       string    `toml:"host" json:"host"`
	Port       int       `toml:"port" json:"port"`
	User       string    `toml:"user" json:"user"`
	Psw        string    `toml:"password" json:"-"`
	StatusPort int       `toml:"status-port" json:"status-port"`
	PdAddr     string    `toml:"pd-addr" json:"pd-addr"`
	StrSQLMode string    `toml:"sql-mode" json:"sql-mode"`
	TLS        string    `toml:"tls" json:"tls"`
	Security   *Security `toml:"security" json:"security"`

	SQLMode          mysql.SQLMode `toml:"-" json:"-"`
	MaxAllowedPacket uint64        `toml:"max-allowed-packet" json:"max-allowed-packet"`

	DistSQLScanConcurrency     int `toml:"distsql-scan-concurrency" json:"distsql-scan-concurrency"`
	BuildStatsConcurrency      int `toml:"build-stats-concurrency" json:"build-stats-concurrency"`
	IndexSerialScanConcurrency int `toml:"index-serial-scan-concurrency" json:"index-serial-scan-concurrency"`
	ChecksumTableConcurrency   int `toml:"checksum-table-concurrency" json:"checksum-table-concurrency"`
}

type Duration 

type Duration struct {
	time.Duration
}

A duration which can be deserialized from a TOML string. Implemented as https://github.com/BurntSushi/toml#using-the-encodingtextunmarshaler-interface

func (*Duration) MarshalJSON 

func (d *Duration) MarshalJSON() ([]byte, error)

func (Duration) MarshalText 

func (d Duration) MarshalText() ([]byte, error)

func (*Duration) UnmarshalText 

func (d *Duration) UnmarshalText(text []byte) error

type FileRouteRule 

type FileRouteRule struct {
	Pattern     string `json:"pattern" toml:"pattern" yaml:"pattern"`
	Path        string `json:"path" toml:"path" yaml:"path"`
	Schema      string `json:"schema" toml:"schema" yaml:"schema"`
	Table       string `json:"table" toml:"table" yaml:"table"`
	Type        string `json:"type" toml:"type" yaml:"type"`
	Key         string `json:"key" toml:"key" yaml:"key"`
	Compression string `json:"compression" toml:"compression" yaml:"compression"`
}

type GlobalCheckpoint 

type GlobalCheckpoint struct {
	Enable bool `toml:"enable" json:"enable"`
}

type GlobalConfig 

type GlobalConfig struct {
	App          GlobalLightning   `toml:"lightning" json:"lightning"`
	Checkpoint   GlobalCheckpoint  `toml:"checkpoint" json:"checkpoint"`
	TiDB         GlobalTiDB        `toml:"tidb" json:"tidb"`
	Mydumper     GlobalMydumper    `toml:"mydumper" json:"mydumper"`
	TikvImporter GlobalImporter    `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  GlobalPostRestore `toml:"post-restore" json:"post-restore"`
	Security     Security          `toml:"security" json:"security"`

	ConfigFileContent []byte
}

func LoadGlobalConfig 

func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalConfig, error)

LoadGlobalConfig reads the arguments and fills in the GlobalConfig.

func Must 

func Must(cfg *GlobalConfig, err error) *GlobalConfig

Must should be called after LoadGlobalConfig(). If LoadGlobalConfig() returns any error, this function will exit the program with an appropriate exit code.

func NewGlobalConfig 

func NewGlobalConfig() *GlobalConfig

type GlobalImporter 

type GlobalImporter struct {
	Addr        string `toml:"addr" json:"addr"`
	Backend     string `toml:"backend" json:"backend"`
	SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
}

type GlobalLightning 

type GlobalLightning struct {
	log.Config
	StatusAddr        string `toml:"status-addr" json:"status-addr"`
	ServerMode        bool   `toml:"server-mode" json:"server-mode"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`

	// The legacy alias for setting "status-addr". The value should always the
	// same as StatusAddr, and will not be published in the JSON encoding.
	PProfPort int `toml:"pprof-port" json:"-"`
}

type GlobalMydumper 

type GlobalMydumper struct {
	SourceDir string `toml:"data-source-dir" json:"data-source-dir"`
	// Deprecated
	NoSchema      bool             `toml:"no-schema" json:"no-schema"`
	Filter        []string         `toml:"filter" json:"filter"`
	IgnoreColumns []*IgnoreColumns `toml:"ignore-columns" json:"ignore-columns"`
}

type GlobalPostRestore 

type GlobalPostRestore struct {
	Checksum PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze  PostOpLevel `toml:"analyze" json:"analyze"`
}

type GlobalTiDB 

type GlobalTiDB struct {
	Host       string `toml:"host" json:"host"`
	Port       int    `toml:"port" json:"port"`
	User       string `toml:"user" json:"user"`
	Psw        string `toml:"password" json:"-"`
	StatusPort int    `toml:"status-port" json:"status-port"`
	PdAddr     string `toml:"pd-addr" json:"pd-addr"`
	LogLevel   string `toml:"log-level" json:"log-level"`
}

type IgnoreColumns 

type IgnoreColumns struct {
	DB          string   `toml:"db" json:"db"`
	Table       string   `toml:"table" json:"table"`
	TableFilter []string `toml:"table-filter" json:"table-filter"`
	Columns     []string `toml:"columns" json:"columns"`
}

type Lightning 

type Lightning struct {
	TableConcurrency  int    `toml:"table-concurrency" json:"table-concurrency"`
	IndexConcurrency  int    `toml:"index-concurrency" json:"index-concurrency"`
	RegionConcurrency int    `toml:"region-concurrency" json:"region-concurrency"`
	IOConcurrency     int    `toml:"io-concurrency" json:"io-concurrency"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`
	MetaSchemaName    string `toml:"meta-schema-name" json:"meta-schema-name"`
}

type List 

type List struct {
	// contains filtered or unexported fields
}

List is a goroutine-safe FIFO list of *Config, which supports removal from the middle. The list is not expected to be very long.

func NewConfigList 

func NewConfigList() *List

NewConfigList creates a new ConfigList instance.

func (*List) AllIDs 

func (cl *List) AllIDs() []int64

AllIDs returns a list of all task IDs in the list.

func (*List) Get 

func (cl *List) Get(taskID int64) (*Config, bool)

Get obtains a task from the list given its task ID. If the task ID did not exist, the returned bool field will be false.

func (*List) MoveToBack 

func (cl *List) MoveToBack(taskID int64) bool

MoveToBack moves a task to the back of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) MoveToFront 

func (cl *List) MoveToFront(taskID int64) bool

MoveToFront moves a task to the front of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) Pop 

func (cl *List) Pop(ctx context.Context) (*Config, error)

Pop removes a configuration from the front of the list. If the list is empty, this method will block until either another goroutines calls Push() or the input context expired.

If the context expired, the error field will contain the error from context.

func (*List) Push 

func (cl *List) Push(cfg *Config)

Push adds a configuration to the end of the list. The field `cfg.TaskID` will be modified to include a unique ID to identify this task.

func (*List) Remove 

func (cl *List) Remove(taskID int64) bool

Remove removes a task from the list given its task ID. Returns true if a task is successfully removed, false if the task ID did not exist.

type MydumperRuntime 

type MydumperRuntime struct {
	ReadBlockSize    ByteSize         `toml:"read-block-size" json:"read-block-size"`
	BatchSize        ByteSize         `toml:"batch-size" json:"batch-size"`
	BatchImportRatio float64          `toml:"batch-import-ratio" json:"batch-import-ratio"`
	SourceDir        string           `toml:"data-source-dir" json:"data-source-dir"`
	CharacterSet     string           `toml:"character-set" json:"character-set"`
	CSV              CSVConfig        `toml:"csv" json:"csv"`
	MaxRegionSize    ByteSize         `toml:"max-region-size" json:"max-region-size"`
	Filter           []string         `toml:"filter" json:"filter"`
	FileRouters      []*FileRouteRule `toml:"files" json:"files"`
	// Deprecated: only used to keep the compatibility.
	NoSchema         bool             `toml:"no-schema" json:"no-schema"`
	CaseSensitive    bool             `toml:"case-sensitive" json:"case-sensitive"`
	StrictFormat     bool             `toml:"strict-format" json:"strict-format"`
	DefaultFileRules bool             `toml:"default-file-rules" json:"default-file-rules"`
	IgnoreColumns    AllIgnoreColumns `toml:"ignore-data-columns" json:"ignore-data-columns"`
}

type PostOpLevel 

type PostOpLevel int
const (
	OpLevelOff PostOpLevel = iota
	OpLevelOptional
	OpLevelRequired
)

func (*PostOpLevel) FromStringValue 

func (t *PostOpLevel) FromStringValue(s string) error

parser command line parameter

func (*PostOpLevel) MarshalJSON 

func (t *PostOpLevel) MarshalJSON() ([]byte, error)

func (PostOpLevel) MarshalText 

func (t PostOpLevel) MarshalText() ([]byte, error)

func (PostOpLevel) String 

func (t PostOpLevel) String() string

func (*PostOpLevel) UnmarshalJSON 

func (t *PostOpLevel) UnmarshalJSON(data []byte) error

func (*PostOpLevel) UnmarshalTOML 

func (t *PostOpLevel) UnmarshalTOML(v interface{}) error

type PostRestore 

type PostRestore struct {
	Checksum          PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze           PostOpLevel `toml:"analyze" json:"analyze"`
	Level1Compact     bool        `toml:"level-1-compact" json:"level-1-compact"`
	PostProcessAtLast bool        `toml:"post-process-at-last" json:"post-process-at-last"`
	Compact           bool        `toml:"compact" json:"compact"`
}

PostRestore has some options which will be executed after kv restored.

type Security 

type Security struct {
	CAPath   string `toml:"ca-path" json:"ca-path"`
	CertPath string `toml:"cert-path" json:"cert-path"`
	KeyPath  string `toml:"key-path" json:"key-path"`
	// RedactInfoLog indicates that whether enabling redact log
	RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"`
}

func (*Security) RegisterMySQL 

func (sec *Security) RegisterMySQL() error

RegistersMySQL registers (or deregisters) the TLS config with name "cluster" for use in `sql.Open()`. This method is goroutine-safe.

type TikvImporter 

type TikvImporter struct {
	Addr               string   `toml:"addr" json:"addr"`
	Backend            string   `toml:"backend" json:"backend"`
	OnDuplicate        string   `toml:"on-duplicate" json:"on-duplicate"`
	MaxKVPairs         int      `toml:"max-kv-pairs" json:"max-kv-pairs"`
	SendKVPairs        int      `toml:"send-kv-pairs" json:"send-kv-pairs"`
	RegionSplitSize    ByteSize `toml:"region-split-size" json:"region-split-size"`
	SortedKVDir        string   `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
	DiskQuota          ByteSize `toml:"disk-quota" json:"disk-quota"`
	RangeConcurrency   int      `toml:"range-concurrency" json:"range-concurrency"`
	DuplicateDetection bool     `toml:"duplicate-detection" json:"duplicate-detection"`

	EngineMemCacheSize      ByteSize `toml:"engine-mem-cache-size" json:"engine-mem-cache-size"`
	LocalWriterMemCacheSize ByteSize `toml:"local-writer-mem-cache-size" json:"local-writer-mem-cache-size"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.