config

package
v1.1.0-beta.0...-a878e1f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 19, 2024 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ImportMode defines mode of import for tikv.
	ImportMode = "import"
	// NormalMode defines mode of normal for tikv.
	NormalMode = "normal"

	// BackendTiDB is a constant for choosing the "TiDB" backend in the configuration.
	BackendTiDB = "tidb"
	// BackendLocal is a constant for choosing the "Local" backup in the configuration.
	// In this mode, we write & sort kv pairs with local storage and directly write them to tikv.
	BackendLocal = "local"

	// CheckpointDriverMySQL is a constant for choosing the "MySQL" checkpoint driver in the configuration.
	CheckpointDriverMySQL = "mysql"
	// CheckpointDriverFile is a constant for choosing the "File" checkpoint driver in the configuration.
	CheckpointDriverFile = "file"

	// KVWriteBatchSize batch size when write to TiKV.
	// this is the default value of linux send buffer size(net.ipv4.tcp_wmem) too.
	KVWriteBatchSize        = 16 * units.KiB
	DefaultRangeConcurrency = 16

	DefaultTableConcurrency = 6

	DefaultRegionCheckBackoffLimit = 1800
	DefaultRegionSplitBatchSize    = 4096

	DefaultRecordDuplicateThreshold = 10000

	DefaultEngineMemCacheSize      = 512 * units.MiB
	DefaultLocalWriterMemCacheSize = 128 * units.MiB
	DefaultBlockSize               = 16 * units.KiB

	DefaultSwitchTiKVModeInterval = 5 * time.Minute
)

constants for config items

View Source 
const (
	DefaultBatchImportRatio = 0.75

	ReadBlockSize ByteSize = 64 * units.KiB
	// SplitRegionSize is the default coprocessor.region-split-keys value before TiKV v8.4.0 See:
	// 	https://github.com/tikv/tikv/blob/e030a0aae9622f3774df89c62f21b2171a72a69e/etc/config-template.toml#L360
	// lower the max-key-count to avoid tikv trigger region auto split
	SplitRegionSize         ByteSize = 96 * units.MiB
	SplitRegionKeys         int      = 1_280_000
	MaxSplitRegionSizeRatio int      = 10
)

some constants

Variables

View Source 
var (
	DefaultGrpcKeepaliveParams = grpc.WithKeepaliveParams(keepalive.ClientParameters{
		Time:                1 * time.Minute,
		Timeout:             2 * time.Minute,
		PermitWithoutStream: false,
	})
	// BufferSizeScale is the factor of block buffer size
	BufferSizeScale           = int64(5)
	DefaultBatchSize ByteSize = 100 * units.GiB
	MaxRegionSize    ByteSize = 256 * units.MiB
)

static vars

Functions

func GetDefaultFilter 

func GetDefaultFilter() []string

GetDefaultFilter gets the default table filter used in Lightning. It clones the original default filter, so that the original value won't be changed when the returned slice's element is changed.

Types

type AllIgnoreColumns 

type AllIgnoreColumns []*IgnoreColumns

AllIgnoreColumns is a slice of IgnoreColumns.

func (AllIgnoreColumns) GetIgnoreColumns 

func (igCols AllIgnoreColumns) GetIgnoreColumns(db string, table string, caseSensitive bool) (*IgnoreColumns, error)

GetIgnoreColumns gets Ignore config by schema name/regex and table name/regex.

type ByteSize 

type ByteSize int64

ByteSize is an alias of int64 which accepts human-friendly strings like '10G' when read from TOML.

func (*ByteSize) UnmarshalJSON 

func (size *ByteSize) UnmarshalJSON(b []byte) error

UnmarshalJSON implements json.Unmarshaler (for testing)

func (*ByteSize) UnmarshalText 

func (size *ByteSize) UnmarshalText(b []byte) error

UnmarshalText implements encoding.TextUnmarshaler

type CSVConfig 

type CSVConfig struct {
	// Separator, Delimiter and Terminator should all be in utf8mb4 encoding.
	Separator         string              `toml:"separator" json:"separator"`
	Delimiter         string              `toml:"delimiter" json:"delimiter"`
	Terminator        string              `toml:"terminator" json:"terminator"`
	Null              StringOrStringSlice `toml:"null" json:"null"`
	Header            bool                `toml:"header" json:"header"`
	HeaderSchemaMatch bool                `toml:"header-schema-match" json:"header-schema-match"`
	TrimLastSep       bool                `toml:"trim-last-separator" json:"trim-last-separator"`
	NotNull           bool                `toml:"not-null" json:"not-null"`
	// deprecated, use `escaped-by` instead.
	BackslashEscape bool `toml:"backslash-escape" json:"backslash-escape"`
	// EscapedBy has higher priority than BackslashEscape, currently it must be a single character if set.
	EscapedBy string `toml:"escaped-by" json:"escaped-by"`

	// hide these options for lightning configuration file, they can only be used by LOAD DATA
	// https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-field-line-handling
	StartingBy     string `toml:"-" json:"-"`
	AllowEmptyLine bool   `toml:"-" json:"-"`
	// For non-empty Delimiter (for example quotes), null elements inside quotes are not considered as null except for
	// `\N` (when escape-by is `\`). That is to say, `\N` is special for null because it always means null.
	QuotedNullIsText bool `toml:"-" json:"-"`
	// ref https://dev.mysql.com/doc/refman/8.0/en/load-data.html
	// > If the field begins with the ENCLOSED BY character, instances of that character are recognized as terminating a
	// > field value only if followed by the field or line TERMINATED BY sequence.
	// This means we will meet unescaped quote in a quoted field
	// > The "BIG" boss      -> The "BIG" boss
	// This means we will meet unescaped quote in a unquoted field
	UnescapedQuote bool `toml:"-" json:"-"`
}

CSVConfig is the config for CSV files.

type Charset 

type Charset int

Charset defines character set 

const (
	Binary Charset = iota
	UTF8MB4
	GB18030
	GBK
	Latin1
	ASCII
)

Charset constants

func ParseCharset 

func ParseCharset(dataCharacterSet string) (Charset, error)

ParseCharset parser character set for string

func (Charset) String 

func (c Charset) String() string

String return the string value of charset

type Checkpoint 

type Checkpoint struct {
	Schema           string                    `toml:"schema" json:"schema"`
	DSN              string                    `toml:"dsn" json:"-"` // DSN may contain password, don't expose this to JSON.
	MySQLParam       *common.MySQLConnectParam `toml:"-" json:"-"`   // For some security reason, we use MySQLParam instead of DSN.
	Driver           string                    `toml:"driver" json:"driver"`
	Enable           bool                      `toml:"enable" json:"enable"`
	KeepAfterSuccess CheckpointKeepStrategy    `toml:"keep-after-success" json:"keep-after-success"`
}

Checkpoint is the config for checkpoint.

type CheckpointKeepStrategy 

type CheckpointKeepStrategy int

CheckpointKeepStrategy represents the strategy to keep checkpoint data. 

const (
	// CheckpointRemove remove checkpoint data
	CheckpointRemove CheckpointKeepStrategy = iota
	// CheckpointRename keep by rename checkpoint file/db according to task id
	CheckpointRename
	// CheckpointOrigin keep checkpoint data unchanged
	CheckpointOrigin
)

func (*CheckpointKeepStrategy) FromStringValue 

func (t *CheckpointKeepStrategy) FromStringValue(s string) error

FromStringValue parser command line parameter.

func (*CheckpointKeepStrategy) MarshalJSON 

func (t *CheckpointKeepStrategy) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler interface.

func (CheckpointKeepStrategy) MarshalText 

func (t CheckpointKeepStrategy) MarshalText() ([]byte, error)

MarshalText implements encoding.TextMarshaler interface.

func (CheckpointKeepStrategy) String 

func (t CheckpointKeepStrategy) String() string

String implements fmt.Stringer interface.

func (*CheckpointKeepStrategy) UnmarshalJSON 

func (t *CheckpointKeepStrategy) UnmarshalJSON(data []byte) error

UnmarshalJSON implements json.Unmarshaler interface.

func (*CheckpointKeepStrategy) UnmarshalTOML 

func (t *CheckpointKeepStrategy) UnmarshalTOML(v any) error

UnmarshalTOML implements toml.Unmarshaler interface.

type CompressionType 

type CompressionType int

CompressionType is the config type of compression algorithm. 

const (
	// CompressionNone means no compression.
	CompressionNone CompressionType = iota
	// CompressionGzip means gzip compression.
	CompressionGzip
)

func (*CompressionType) FromStringValue 

func (t *CompressionType) FromStringValue(s string) error

FromStringValue parses a string to CompressionType.

func (*CompressionType) MarshalJSON 

func (t *CompressionType) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler.

func (CompressionType) MarshalText 

func (t CompressionType) MarshalText() ([]byte, error)

MarshalText implements encoding.TextMarshaler.

func (CompressionType) String 

func (t CompressionType) String() string

String implements fmt.Stringer.

func (*CompressionType) UnmarshalJSON 

func (t *CompressionType) UnmarshalJSON(data []byte) error

UnmarshalJSON implements json.Unmarshaler.

func (*CompressionType) UnmarshalTOML 

func (t *CompressionType) UnmarshalTOML(v any) error

UnmarshalTOML implements toml.Unmarshaler.

type Config 

type Config struct {
	TaskID int64 `toml:"-" json:"id"`

	App  Lightning `toml:"lightning" json:"lightning"`
	TiDB DBStore   `toml:"tidb" json:"tidb"`

	Checkpoint   Checkpoint      `toml:"checkpoint" json:"checkpoint"`
	Mydumper     MydumperRuntime `toml:"mydumper" json:"mydumper"`
	TikvImporter TikvImporter    `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  PostRestore     `toml:"post-restore" json:"post-restore"`
	Cron         Cron            `toml:"cron" json:"cron"`
	Routes       Routes          `toml:"routes" json:"routes"`
	Security     Security        `toml:"security" json:"security"`
	Conflict     Conflict        `toml:"conflict" json:"conflict"`
}

Config is the configuration.

func NewConfig 

func NewConfig() *Config

NewConfig creates a new Config.

func (*Config) Adjust 

func (cfg *Config) Adjust(ctx context.Context) error

Adjust fixes the invalid or unspecified settings to reasonable valid values, and checks for illegal configuration.

func (*Config) AdjustForDDL 

func (cfg *Config) AdjustForDDL() error

AdjustForDDL acts like Adjust, but DDL will not use some functionalities so those members are skipped in adjusting.

func (*Config) LoadFromGlobal 

func (cfg *Config) LoadFromGlobal(global *GlobalConfig) error

LoadFromGlobal resets the current configuration to the global settings.

func (*Config) LoadFromTOML 

func (cfg *Config) LoadFromTOML(data []byte) error

LoadFromTOML overwrites the current configuration by the TOML data If data contains toml items not in Config and GlobalConfig, return an error If data contains toml items not in Config, thus won't take effect, warn user

func (*Config) String 

func (cfg *Config) String() string

String implements fmt.Stringer interface.

func (*Config) ToTLS 

func (cfg *Config) ToTLS() (*common.TLS, error)

ToTLS creates a common.TLS from the config.

type Conflict 

type Conflict struct {
	Strategy                     DuplicateResolutionAlgorithm `toml:"strategy" json:"strategy"`
	PrecheckConflictBeforeImport bool                         `toml:"precheck-conflict-before-import" json:"precheck-conflict-before-import"`
	Threshold                    int64                        `toml:"threshold" json:"threshold"`
	MaxRecordRows                int64                        `toml:"max-record-rows" json:"max-record-rows"`
}

Conflict is the config section for PK/UK conflict related configurations.

type Cron 

type Cron struct {
	SwitchMode     Duration `toml:"switch-mode" json:"switch-mode"`
	LogProgress    Duration `toml:"log-progress" json:"log-progress"`
	CheckDiskQuota Duration `toml:"check-disk-quota" json:"check-disk-quota"`
}

Cron is the config for cron.

type DBStore 

type DBStore struct {
	Host       string    `toml:"host" json:"host"`
	Port       int       `toml:"port" json:"port"`
	User       string    `toml:"user" json:"user"`
	Psw        string    `toml:"password" json:"-"`
	StatusPort int       `toml:"status-port" json:"status-port"`
	PdAddr     string    `toml:"pd-addr" json:"pd-addr"`
	StrSQLMode string    `toml:"sql-mode" json:"sql-mode"`
	TLS        string    `toml:"tls" json:"tls"`
	Security   *Security `toml:"security" json:"security"`

	SQLMode          mysql.SQLMode `toml:"-" json:"-"`
	MaxAllowedPacket uint64        `toml:"max-allowed-packet" json:"max-allowed-packet"`

	DistSQLScanConcurrency     int               `toml:"distsql-scan-concurrency" json:"distsql-scan-concurrency"`
	BuildStatsConcurrency      int               `toml:"build-stats-concurrency" json:"build-stats-concurrency"`
	IndexSerialScanConcurrency int               `toml:"index-serial-scan-concurrency" json:"index-serial-scan-concurrency"`
	ChecksumTableConcurrency   int               `toml:"checksum-table-concurrency" json:"checksum-table-concurrency"`
	Vars                       map[string]string `toml:"session-vars" json:"vars"`

	IOTotalBytes *atomic.Uint64 `toml:"-" json:"-"`
	UUID         string         `toml:"-" json:"-"`
}

DBStore is the database connection information.

type DuplicateResolutionAlgorithm 

type DuplicateResolutionAlgorithm int

DuplicateResolutionAlgorithm is the config type of how to resolve duplicates. 

const (
	// NoneOnDup does nothing when detecting duplicate.
	NoneOnDup DuplicateResolutionAlgorithm = iota
	// ReplaceOnDup indicates using REPLACE INTO to insert data for TiDB backend.
	// ReplaceOnDup records all duplicate records, remove some rows with conflict
	// and reserve other rows that can be kept and not cause conflict anymore for local backend.
	// Users need to analyze the lightning_task_info.conflict_error_v3 table to check whether the reserved data
	// cater to their need and check whether they need to add back the correct rows.
	ReplaceOnDup
	// IgnoreOnDup indicates using INSERT IGNORE INTO to insert data for TiDB backend.
	// Local backend does not support IgnoreOnDup.
	IgnoreOnDup
	// ErrorOnDup indicates using INSERT INTO to insert data for TiDB backend, which would violate PK or UNIQUE constraint when detecting duplicate.
	// ErrorOnDup reports an error after detecting the first conflict and stops the import process for local backend.
	ErrorOnDup
)

func (*DuplicateResolutionAlgorithm) FromStringValue 

func (dra *DuplicateResolutionAlgorithm) FromStringValue(s string) error

FromStringValue parses the string value to the DuplicateResolutionAlgorithm.

func (*DuplicateResolutionAlgorithm) MarshalJSON 

func (dra *DuplicateResolutionAlgorithm) MarshalJSON() ([]byte, error)

MarshalJSON implements the json.Marshaler interface.

func (DuplicateResolutionAlgorithm) MarshalText 

func (dra DuplicateResolutionAlgorithm) MarshalText() ([]byte, error)

MarshalText implements the encoding.TextMarshaler interface.

func (DuplicateResolutionAlgorithm) String 

func (dra DuplicateResolutionAlgorithm) String() string

String implements the fmt.Stringer interface.

func (*DuplicateResolutionAlgorithm) UnmarshalJSON 

func (dra *DuplicateResolutionAlgorithm) UnmarshalJSON(data []byte) error

UnmarshalJSON implements the json.Unmarshaler interface.

func (*DuplicateResolutionAlgorithm) UnmarshalTOML 

func (dra *DuplicateResolutionAlgorithm) UnmarshalTOML(v any) error

UnmarshalTOML implements the toml.Unmarshaler interface.

type Duration 

type Duration struct {
	time.Duration
}

Duration which can be deserialized from a TOML string. Implemented as https://github.com/BurntSushi/toml#using-the-encodingtextunmarshaler-interface

func (*Duration) MarshalJSON 

func (d *Duration) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler.

func (Duration) MarshalText 

func (d Duration) MarshalText() ([]byte, error)

MarshalText implements encoding.TextMarshaler.

func (*Duration) UnmarshalText 

func (d *Duration) UnmarshalText(text []byte) error

UnmarshalText implements encoding.TextUnmarshaler.

type FileRouteRule 

type FileRouteRule struct {
	Pattern     string `json:"pattern" toml:"pattern" yaml:"pattern"`
	Path        string `json:"path" toml:"path" yaml:"path"`
	Schema      string `json:"schema" toml:"schema" yaml:"schema"`
	Table       string `json:"table" toml:"table" yaml:"table"`
	Type        string `json:"type" toml:"type" yaml:"type"`
	Key         string `json:"key" toml:"key" yaml:"key"`
	Compression string `json:"compression" toml:"compression" yaml:"compression"`
	// unescape the schema/table name only used in lightning's internal logic now.
	Unescape bool `json:"-" toml:"-" yaml:"-"`
}

FileRouteRule is the rule for routing files.

type GlobalCheckpoint 

type GlobalCheckpoint struct {
	Enable bool `toml:"enable" json:"enable"`
}

GlobalCheckpoint is the global configuration of checkpoint.

type GlobalConfig 

type GlobalConfig struct {
	App          GlobalLightning   `toml:"lightning" json:"lightning"`
	Checkpoint   GlobalCheckpoint  `toml:"checkpoint" json:"checkpoint"`
	TiDB         GlobalTiDB        `toml:"tidb" json:"tidb"`
	Mydumper     GlobalMydumper    `toml:"mydumper" json:"mydumper"`
	TikvImporter GlobalImporter    `toml:"tikv-importer" json:"tikv-importer"`
	PostRestore  GlobalPostRestore `toml:"post-restore" json:"post-restore"`
	Security     Security          `toml:"security" json:"security"`

	ConfigFileContent []byte
}

GlobalConfig is the global configuration of lightning.

func LoadGlobalConfig 

func LoadGlobalConfig(args []string, extraFlags func(*flag.FlagSet)) (*GlobalConfig, error)

LoadGlobalConfig reads the arguments and fills in the GlobalConfig.

func Must 

func Must(cfg *GlobalConfig, err error) *GlobalConfig

Must should be called after LoadGlobalConfig(). If LoadGlobalConfig() returns any error, this function will exit the program with an appropriate exit code.

func NewGlobalConfig 

func NewGlobalConfig() *GlobalConfig

NewGlobalConfig creates a new GlobalConfig with default values.

type GlobalImporter 

type GlobalImporter struct {
	Backend     string `toml:"backend" json:"backend"`
	SortedKVDir string `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
}

GlobalImporter is the global configuration of tikv-importer.

type GlobalLightning 

type GlobalLightning struct {
	log.Config
	StatusAddr        string `toml:"status-addr" json:"status-addr"`
	ServerMode        bool   `toml:"server-mode" json:"server-mode"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`

	// The legacy alias for setting "status-addr". The value should always the
	// same as StatusAddr, and will not be published in the JSON encoding.
	PProfPort int `toml:"pprof-port" json:"-"`
}

GlobalLightning is the global configuration of lightning.

type GlobalMydumper 

type GlobalMydumper struct {
	SourceDir string `toml:"data-source-dir" json:"data-source-dir"`
	// Deprecated
	NoSchema      bool             `toml:"no-schema" json:"no-schema"`
	Filter        []string         `toml:"filter" json:"filter"`
	IgnoreColumns []*IgnoreColumns `toml:"ignore-columns" json:"ignore-columns"`
}

GlobalMydumper is the global configuration of mydumper.

type GlobalPostRestore 

type GlobalPostRestore struct {
	Checksum PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze  PostOpLevel `toml:"analyze" json:"analyze"`
}

GlobalPostRestore is the global configuration of post-restore.

type GlobalTiDB 

type GlobalTiDB struct {
	Host       string `toml:"host" json:"host"`
	Port       int    `toml:"port" json:"port"`
	User       string `toml:"user" json:"user"`
	Psw        string `toml:"password" json:"-"`
	StatusPort int    `toml:"status-port" json:"status-port"`
	PdAddr     string `toml:"pd-addr" json:"pd-addr"`
	LogLevel   string `toml:"log-level" json:"log-level"`
}

GlobalTiDB is the global configuration of TiDB.

type IgnoreColumns 

type IgnoreColumns struct {
	DB          string   `toml:"db" json:"db"`
	Table       string   `toml:"table" json:"table"`
	TableFilter []string `toml:"table-filter" json:"table-filter"`
	Columns     []string `toml:"columns" json:"columns"`
}

IgnoreColumns is the config for ignoring columns.

func (*IgnoreColumns) ColumnsMap 

func (ic *IgnoreColumns) ColumnsMap() map[string]struct{}

ColumnsMap returns a map of columns.

type Lightning 

type Lightning struct {
	TableConcurrency  int    `toml:"table-concurrency" json:"table-concurrency"`
	IndexConcurrency  int    `toml:"index-concurrency" json:"index-concurrency"`
	RegionConcurrency int    `toml:"region-concurrency" json:"region-concurrency"`
	IOConcurrency     int    `toml:"io-concurrency" json:"io-concurrency"`
	CheckRequirements bool   `toml:"check-requirements" json:"check-requirements"`
	MetaSchemaName    string `toml:"meta-schema-name" json:"meta-schema-name"`

	MaxError MaxError `toml:"max-error" json:"max-error"`
	// deprecated, use Conflict.MaxRecordRows instead
	MaxErrorRecords    int64  `toml:"max-error-records" json:"max-error-records"`
	TaskInfoSchemaName string `toml:"task-info-schema-name" json:"task-info-schema-name"`
}

Lightning is the root configuration of lightning.

type List 

type List struct {
	// contains filtered or unexported fields
}

List is a goroutine-safe FIFO list of *Config, which supports removal from the middle. The list is not expected to be very long.

func NewConfigList 

func NewConfigList() *List

NewConfigList creates a new ConfigList instance.

func (*List) AllIDs 

func (cl *List) AllIDs() []int64

AllIDs returns a list of all task IDs in the list.

func (*List) Get 

func (cl *List) Get(taskID int64) (*Config, bool)

Get obtains a task from the list given its task ID. If the task ID did not exist, the returned bool field will be false.

func (*List) MoveToBack 

func (cl *List) MoveToBack(taskID int64) bool

MoveToBack moves a task to the back of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) MoveToFront 

func (cl *List) MoveToFront(taskID int64) bool

MoveToFront moves a task to the front of the list. Returns true if the task is successfully moved (including no-op), false if the task ID did not exist.

func (*List) Pop 

func (cl *List) Pop(ctx context.Context) (*Config, error)

Pop removes a configuration from the front of the list. If the list is empty, this method will block until either another goroutines calls Push() or the input context expired.

If the context expired, the error field will contain the error from context.

func (*List) Push 

func (cl *List) Push(cfg *Config)

Push adds a configuration to the end of the list. The field `cfg.TaskID` will be modified to include a unique ID to identify this task.

func (*List) Remove 

func (cl *List) Remove(taskID int64) bool

Remove removes a task from the list given its task ID. Returns true if a task is successfully removed, false if the task ID did not exist.

type MaxError 

type MaxError struct {
	// Syntax is the maximum number of syntax errors accepted.
	// When tolerated, the file chunk causing syntax error will be skipped, and adds 1 to the counter.
	// TODO Currently this is hard-coded to zero.
	Syntax atomic.Int64 `toml:"syntax" json:"-"`

	// Charset is the maximum number of character-set conversion errors accepted.
	// When tolerated, and `data-invalid-char-replace` is not changed from "\ufffd",
	// every invalid byte in the source file will be converted to U+FFFD and adds 1 to the counter.
	// Note that a failed conversion a column's character set (e.g. UTF8-to-GBK conversion)
	// is counted as a type error, not a charset error.
	// TODO character-set conversion is not yet implemented.
	Charset atomic.Int64 `toml:"charset" json:"-"`

	// Type is the maximum number of type errors accepted.
	// This includes strict-mode errors such as zero in dates, integer overflow, character string too long, etc.
	// In TiDB backend, this also includes all possible SQL errors raised from INSERT,
	// such as unique key conflict when `on-duplicate` is set to `error`.
	// When tolerated, the row causing the error will be skipped, and adds 1 to the counter.
	// The default value is zero, which means that such errors are not tolerated.
	Type atomic.Int64 `toml:"type" json:"type"`

	// deprecated, use `conflict.threshold` instead.
	// Conflict is the maximum number of unique key conflicts in local backend accepted.
	// When tolerated, every pair of conflict adds 1 to the counter.
	// Those pairs will NOT be deleted from the target. Conflict resolution is performed separately.
	// The default value is max int64, which means conflict errors will be recorded as much as possible.
	// Sometime the actual number of conflict record logged will be greater than the value configured here,
	// because conflict error data are recorded batch by batch.
	// If the limit is reached in a single batch, the entire batch of records will be persisted before an error is reported.
	Conflict atomic.Int64 `toml:"conflict" json:"conflict"`
}

MaxError configures the maximum number of acceptable errors per kind.

func (*MaxError) UnmarshalTOML 

func (cfg *MaxError) UnmarshalTOML(v any) error

UnmarshalTOML implements toml.Unmarshaler interface.

type MydumperRuntime 

type MydumperRuntime struct {
	ReadBlockSize    ByteSize         `toml:"read-block-size" json:"read-block-size"`
	BatchSize        ByteSize         `toml:"batch-size" json:"batch-size"`
	BatchImportRatio float64          `toml:"batch-import-ratio" json:"batch-import-ratio"`
	SourceID         string           `toml:"source-id" json:"source-id"`
	SourceDir        string           `toml:"data-source-dir" json:"data-source-dir"`
	CharacterSet     string           `toml:"character-set" json:"character-set"`
	CSV              CSVConfig        `toml:"csv" json:"csv"`
	MaxRegionSize    ByteSize         `toml:"max-region-size" json:"max-region-size"`
	Filter           []string         `toml:"filter" json:"filter"`
	FileRouters      []*FileRouteRule `toml:"files" json:"files"`
	// Deprecated: only used to keep the compatibility.
	NoSchema         bool             `toml:"no-schema" json:"no-schema"`
	CaseSensitive    bool             `toml:"case-sensitive" json:"case-sensitive"`
	StrictFormat     bool             `toml:"strict-format" json:"strict-format"`
	DefaultFileRules bool             `toml:"default-file-rules" json:"default-file-rules"`
	IgnoreColumns    AllIgnoreColumns `toml:"ignore-data-columns" json:"ignore-data-columns"`
	// DataCharacterSet is the character set of the source file. Only CSV files are supported now. The following options are supported.
	//   - utf8mb4
	//   - GB18030
	//   - GBK: an extension of the GB2312 character set and is also known as Code Page 936.
	//   - latin1: IANA Windows1252
	//   - binary: no attempt to convert the encoding.
	// Leave DataCharacterSet empty will make it use `binary` by default.
	DataCharacterSet string `toml:"data-character-set" json:"data-character-set"`
	// DataInvalidCharReplace is the replacement characters for non-compatible characters, which shouldn't duplicate with the separators or line breaks.
	// Changing the default value will result in increased parsing time. Non-compatible characters do not cause an increase in error.
	DataInvalidCharReplace string `toml:"data-invalid-char-replace" json:"data-invalid-char-replace"`
}

MydumperRuntime is the runtime config for mydumper.

type PausePDSchedulerScope 

type PausePDSchedulerScope string

PausePDSchedulerScope the scope when pausing pd schedulers. 

const (
	// PausePDSchedulerScopeTable pause scheduler by adding schedule=deny label to target key range of the table.
	PausePDSchedulerScopeTable PausePDSchedulerScope = "table"
	// PausePDSchedulerScopeGlobal pause scheduler by remove global schedulers.
	// schedulers removed includes:
	// 	- balance-leader-scheduler
	// 	- balance-hot-region-scheduler
	// 	- balance-region-scheduler
	// 	- shuffle-leader-scheduler
	// 	- shuffle-region-scheduler
	// 	- shuffle-hot-region-scheduler
	// and we also set configs below:
	// 	- max-merge-region-keys = 0
	// 	- max-merge-region-size = 0
	// 	- leader-schedule-limit = min(40, <store-count> * <current value of leader-schedule-limit>)
	// 	- region-schedule-limit = min(40, <store-count> * <current value of region-schedule-limit>)
	// 	- max-snapshot-count = min(40, <store-count> * <current value of max-snapshot-count>)
	// 	- enable-location-replacement = false
	// 	- max-pending-peer-count = math.MaxInt32
	// see br/pkg/pdutil/pd.go for more detail.
	PausePDSchedulerScopeGlobal PausePDSchedulerScope = "global"
)

constants for PausePDSchedulerScope.

type PostOpLevel 

type PostOpLevel int

PostOpLevel represents the level of post-operation. 

const (
	OpLevelOff PostOpLevel = iota
	OpLevelOptional
	OpLevelRequired
)

PostOpLevel constants.

func (*PostOpLevel) FromStringValue 

func (t *PostOpLevel) FromStringValue(s string) error

FromStringValue parse command line parameter.

func (*PostOpLevel) MarshalJSON 

func (t *PostOpLevel) MarshalJSON() ([]byte, error)

MarshalJSON implements json.Marshaler interface.

func (PostOpLevel) MarshalText 

func (t PostOpLevel) MarshalText() ([]byte, error)

MarshalText implements encoding.TextMarshaler interface.

func (PostOpLevel) String 

func (t PostOpLevel) String() string

String returns the string representation of the level.

func (*PostOpLevel) UnmarshalJSON 

func (t *PostOpLevel) UnmarshalJSON(data []byte) error

UnmarshalJSON implements json.Unmarshaler interface.

func (*PostOpLevel) UnmarshalTOML 

func (t *PostOpLevel) UnmarshalTOML(v any) error

UnmarshalTOML implements toml.Unmarshaler interface.

type PostRestore 

type PostRestore struct {
	Checksum          PostOpLevel `toml:"checksum" json:"checksum"`
	Analyze           PostOpLevel `toml:"analyze" json:"analyze"`
	Level1Compact     bool        `toml:"level-1-compact" json:"level-1-compact"`
	PostProcessAtLast bool        `toml:"post-process-at-last" json:"post-process-at-last"`
	Compact           bool        `toml:"compact" json:"compact"`
	ChecksumViaSQL    bool        `toml:"checksum-via-sql" json:"checksum-via-sql"`
}

PostRestore has some options which will be executed after kv restored.

type Routes 

type Routes []*router.TableRule

Routes is a alias of []*router.TableRule. It's used to attach method to []*router.TableRule.

type Security 

type Security struct {
	CAPath   string `toml:"ca-path" json:"ca-path"`
	CertPath string `toml:"cert-path" json:"cert-path"`
	KeyPath  string `toml:"key-path" json:"key-path"`
	// RedactInfoLog indicates that whether enabling redact log
	RedactInfoLog bool `toml:"redact-info-log" json:"redact-info-log"`

	TLSConfig                *tls.Config `toml:"-" json:"-"`
	AllowFallbackToPlaintext bool        `toml:"-" json:"-"`

	// When DM/engine uses lightning as a library, it can directly pass in the content
	CABytes   []byte `toml:"-" json:"-"`
	CertBytes []byte `toml:"-" json:"-"`
	KeyBytes  []byte `toml:"-" json:"-"`
}

Security is the config for security.

func (*Security) BuildTLSConfig 

func (sec *Security) BuildTLSConfig() error

BuildTLSConfig builds the tls config which is used by SQL drier later.

type StringOrStringSlice 

type StringOrStringSlice []string

StringOrStringSlice can unmarshal a TOML string as string slice with one element.

func (*StringOrStringSlice) UnmarshalTOML 

func (s *StringOrStringSlice) UnmarshalTOML(in any) error

UnmarshalTOML implements the toml.Unmarshaler interface.

type TikvImporter 

type TikvImporter struct {
	// Deprecated: only used to keep the compatibility.
	Addr    string `toml:"addr" json:"addr"`
	Backend string `toml:"backend" json:"backend"`
	// deprecated, use Conflict.Strategy instead.
	OnDuplicate DuplicateResolutionAlgorithm `toml:"on-duplicate" json:"on-duplicate"`
	MaxKVPairs  int                          `toml:"max-kv-pairs" json:"max-kv-pairs"`
	// deprecated
	SendKVPairs             int             `toml:"send-kv-pairs" json:"send-kv-pairs"`
	SendKVSize              ByteSize        `toml:"send-kv-size" json:"send-kv-size"`
	CompressKVPairs         CompressionType `toml:"compress-kv-pairs" json:"compress-kv-pairs"`
	RegionSplitSize         ByteSize        `toml:"region-split-size" json:"region-split-size"`
	RegionSplitKeys         int             `toml:"region-split-keys" json:"region-split-keys"`
	RegionSplitBatchSize    int             `toml:"region-split-batch-size" json:"region-split-batch-size"`
	RegionSplitConcurrency  int             `toml:"region-split-concurrency" json:"region-split-concurrency"`
	RegionCheckBackoffLimit int             `toml:"region-check-backoff-limit" json:"region-check-backoff-limit"`
	SortedKVDir             string          `toml:"sorted-kv-dir" json:"sorted-kv-dir"`
	DiskQuota               ByteSize        `toml:"disk-quota" json:"disk-quota"`
	RangeConcurrency        int             `toml:"range-concurrency" json:"range-concurrency"`
	// deprecated, use Conflict.Strategy instead.
	DuplicateResolution DuplicateResolutionAlgorithm `toml:"duplicate-resolution" json:"duplicate-resolution"`
	// deprecated, use ParallelImport instead.
	IncrementalImport bool   `toml:"incremental-import" json:"incremental-import"`
	ParallelImport    bool   `toml:"parallel-import" json:"parallel-import"`
	KeyspaceName      string `toml:"keyspace-name" json:"keyspace-name"`
	AddIndexBySQL     bool   `toml:"add-index-by-sql" json:"add-index-by-sql"`

	EngineMemCacheSize      ByteSize `toml:"engine-mem-cache-size" json:"engine-mem-cache-size"`
	LocalWriterMemCacheSize ByteSize `toml:"local-writer-mem-cache-size" json:"local-writer-mem-cache-size"`
	StoreWriteBWLimit       ByteSize `toml:"store-write-bwlimit" json:"store-write-bwlimit"`
	LogicalImportBatchSize  ByteSize `toml:"logical-import-batch-size" json:"logical-import-batch-size"`
	LogicalImportBatchRows  int      `toml:"logical-import-batch-rows" json:"logical-import-batch-rows"`

	// default is PausePDSchedulerScopeTable to compatible with previous version(>= 6.1)
	PausePDSchedulerScope PausePDSchedulerScope `toml:"pause-pd-scheduler-scope" json:"pause-pd-scheduler-scope"`
	BlockSize             ByteSize              `toml:"block-size" json:"block-size"`
}

TikvImporter is the config for tikv-importer.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.