task

package
v1.1.0-beta.0...-652cee2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 27, 2024 License: Apache-2.0, Apache-2.0 Imports: 102 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	FullBackupCmd  = "Full Backup"
	DBBackupCmd    = "Database Backup"
	TableBackupCmd = "Table Backup"
	RawBackupCmd   = "Raw Backup"
	TxnBackupCmd   = "Txn Backup"
)
View Source 
const (
	SchemeLocal = "local"
	SchemeAWS   = "aws-kms"
	SchemeAzure = "azure-kms"
	SchemeGCP   = "gcp-kms"

	AWSVendor      = "aws"
	AWSRegion      = "REGION"
	AWSEndpoint    = "ENDPOINT"
	AWSAccessKeyId = "AWS_ACCESS_KEY_ID"
	AWSSecretKey   = "AWS_SECRET_ACCESS_KEY"

	AzureVendor       = "azure"
	AzureTenantID     = "AZURE_TENANT_ID"
	AzureClientID     = "AZURE_CLIENT_ID"
	AzureClientSecret = "AZURE_CLIENT_SECRET"
	AzureVaultName    = "AZURE_VAULT_NAME"

	GCPVendor      = "gcp"
	GCPCredentials = "CREDENTIALS"
)
View Source 
const (

	// FlagMergeRegionSizeBytes is the flag name of merge small regions by size
	FlagMergeRegionSizeBytes = "merge-region-size-bytes"
	// FlagMergeRegionKeyCount is the flag name of merge small regions by key count
	FlagMergeRegionKeyCount = "merge-region-key-count"
	// FlagPDConcurrency controls concurrency pd-relative operations like split & scatter.
	FlagPDConcurrency = "pd-concurrency"
	// FlagStatsConcurrency controls concurrency to restore statistic.
	FlagStatsConcurrency = "stats-concurrency"
	// FlagBatchFlushInterval controls after how long the restore batch would be auto sended.
	FlagBatchFlushInterval = "batch-flush-interval"
	// FlagDdlBatchSize controls batch ddl size to create a batch of tables
	FlagDdlBatchSize = "ddl-batch-size"
	// FlagWithPlacementPolicy corresponds to tidb config with-tidb-placement-mode
	// current only support STRICT or IGNORE, the default is STRICT according to tidb.
	FlagWithPlacementPolicy = "with-tidb-placement-mode"
	// FlagKeyspaceName corresponds to tidb config keyspace-name
	FlagKeyspaceName = "keyspace-name"

	// FlagWaitTiFlashReady represents whether wait tiflash replica ready after table restored and checksumed.
	FlagWaitTiFlashReady = "wait-tiflash-ready"

	// FlagStreamStartTS and FlagStreamRestoreTS is used for log restore timestamp range.
	FlagStreamStartTS   = "start-ts"
	FlagStreamRestoreTS = "restored-ts"
	// FlagStreamFullBackupStorage is used for log restore, represents the full backup storage.
	FlagStreamFullBackupStorage = "full-backup-storage"
	// FlagPiTRBatchCount and FlagPiTRBatchSize are used for restore log with batch method.
	FlagPiTRBatchCount  = "pitr-batch-count"
	FlagPiTRBatchSize   = "pitr-batch-size"
	FlagPiTRConcurrency = "pitr-concurrency"

	FlagResetSysUsers = "reset-sys-users"
)
View Source 
const (
	FullRestoreCmd  = "Full Restore"
	DBRestoreCmd    = "DataBase Restore"
	TableRestoreCmd = "Table Restore"
	PointRestoreCmd = "Point Restore"
	RawRestoreCmd   = "Raw Restore"
	TxnRestoreCmd   = "Txn Restore"
)
View Source 
const (
	FlagChecksum = "checksum"
)

Variables

View Source 
var (
	StreamStart    = "log start"
	StreamStop     = "log stop"
	StreamPause    = "log pause"
	StreamResume   = "log resume"
	StreamStatus   = "log status"
	StreamTruncate = "log truncate"
	StreamMetadata = "log metadata"
	StreamCtl      = "log advancer"
)
View Source 
var StreamCommandMap = map[string]func(c context.Context, g glue.Glue, cmdName string, cfg *StreamConfig) error{
	StreamStart:    RunStreamStart,
	StreamStop:     RunStreamStop,
	StreamPause:    RunStreamPause,
	StreamResume:   RunStreamResume,
	StreamStatus:   RunStreamStatus,
	StreamTruncate: RunStreamTruncate,
	StreamMetadata: RunStreamMetadata,
	StreamCtl:      RunStreamAdvancer,
}

Functions

func CheckDDLJobByRules 

func CheckDDLJobByRules(srcDDLJobs []*model.Job, rules ...DDLJobFilterRule) error

CheckDDLJobByRules if one of rules returns true, the job in srcDDLJobs will be filtered.

func CheckNewCollationEnable 

func CheckNewCollationEnable(
	backupNewCollationEnable string,
	g glue.Glue,
	storage kv.Storage,
	CheckRequirements bool,
) (bool, error)

func CheckRestoreDBAndTable 

func CheckRestoreDBAndTable(schemas []*metautil.Database, cfg *RestoreConfig) error

CheckRestoreDBAndTable is used to check whether the restore dbs or tables have been backup

func CheckStoreSpace 

func CheckStoreSpace(necessary uint64, store *http.StoreInfo) error

func DDLJobBlockListRule 

func DDLJobBlockListRule(ddlJob *model.Job) bool

DDLJobBlockListRule rule for filter ddl job with type in block list.

func DDLJobLogIncrementalCompactBlockListRule 

func DDLJobLogIncrementalCompactBlockListRule(ddlJob *model.Job) bool

func DefineBackupEBSFlags 

func DefineBackupEBSFlags(flags *pflag.FlagSet)

DefineBackupEBSFlags defines common flags for the backup command.

func DefineBackupFlags 

func DefineBackupFlags(flags *pflag.FlagSet)

DefineBackupFlags defines common flags for the backup command.

func DefineCommonFlags 

func DefineCommonFlags(flags *pflag.FlagSet)

DefineCommonFlags defines the flags common to all BRIE commands.

func DefineDatabaseFlags 

func DefineDatabaseFlags(command *cobra.Command)

DefineDatabaseFlags defines the required --db flag for `db` subcommand.

func DefineFilterFlags 

func DefineFilterFlags(command *cobra.Command, defaultFilter []string, setHidden bool)

DefineFilterFlags defines the --filter and --case-sensitive flags for `full` subcommand.

func DefineRawBackupFlags 

func DefineRawBackupFlags(command *cobra.Command)

DefineRawBackupFlags defines common flags for the backup command.

func DefineRawRestoreFlags 

func DefineRawRestoreFlags(command *cobra.Command)

DefineRawRestoreFlags defines common flags for the backup command.

func DefineRestoreCommonFlags 

func DefineRestoreCommonFlags(flags *pflag.FlagSet)

DefineRestoreCommonFlags defines common flags for the restore command.

func DefineRestoreFlags 

func DefineRestoreFlags(flags *pflag.FlagSet)

DefineRestoreFlags defines common flags for the restore tidb command.

func DefineRestoreSnapshotFlags 

func DefineRestoreSnapshotFlags(command *cobra.Command)

DefineRestoreSnapshotFlags defines common flags for the backup command.

func DefineStreamCommonFlags 

func DefineStreamCommonFlags(flags *pflag.FlagSet)

DefineStreamCommonFlags define common flags for `stream task`

func DefineStreamPauseFlags 

func DefineStreamPauseFlags(flags *pflag.FlagSet)

func DefineStreamRestoreFlags 

func DefineStreamRestoreFlags(command *cobra.Command)

DefineStreamRestoreFlags defines for the restore log command.

func DefineStreamStartFlags 

func DefineStreamStartFlags(flags *pflag.FlagSet)

DefineStreamStartFlags defines flags used for `stream start`

func DefineStreamStatusCommonFlags 

func DefineStreamStatusCommonFlags(flags *pflag.FlagSet)

func DefineStreamTruncateLogFlags 

func DefineStreamTruncateLogFlags(flags *pflag.FlagSet)

func DefineTableFlags 

func DefineTableFlags(command *cobra.Command)

DefineTableFlags defines the required --db and --table flags for `table` subcommand.

func DefineTxnBackupFlags 

func DefineTxnBackupFlags(command *cobra.Command)

DefineTxnBackupFlags defines common flags for the backup command.

func EstimateRangeSize 

func EstimateRangeSize(files []*backuppb.File) int

EstimateRangeSize estimates the total range count by file.

func EstimateTiflashUsage 

func EstimateTiflashUsage(tables []*metautil.Table, storeCnt uint64) uint64

func EstimateTikvUsage 

func EstimateTikvUsage(files []*backuppb.File, replicaCnt uint64, storeCnt uint64) uint64

func Exhaust 

func Exhaust(ec <-chan error) []error

Exhaust drains all remaining errors in the channel, into a slice of errors.

func FilterDDLJobByRules 

func FilterDDLJobByRules(srcDDLJobs []*model.Job, rules ...DDLJobFilterRule) (dstDDLJobs []*model.Job)

FilterDDLJobByRules if one of rules returns true, the job in srcDDLJobs will be filtered.

func FilterDDLJobs 

func FilterDDLJobs(allDDLJobs []*model.Job, tables []*metautil.Table) (ddlJobs []*model.Job)

FilterDDLJobs filters ddl jobs.

func GetCipherKeyContent 

func GetCipherKeyContent(cipherKey, cipherKeyFile string) ([]byte, error)

func GetKeepalive 

func GetKeepalive(cfg *Config) keepalive.ClientParameters

GetKeepalive get the keepalive info from the config.

func GetStorage 

func GetStorage(
	ctx context.Context,
	storageName string,
	cfg *Config,
) (*backuppb.StorageBackend, storage.ExternalStorage, error)

GetStorage gets the storage backend from the config.

func HiddenFlagsForStream 

func HiddenFlagsForStream(flags *pflag.FlagSet)

HiddenFlagsForStream temporary hidden flags that stream cmd not support.

func IsStreamRestore 

func IsStreamRestore(cmdName string) bool

IsStreamRestore checks the command is `restore point`

func LogArguments 

func LogArguments(cmd *cobra.Command)

LogArguments prints origin command arguments.

func NewMgr 

func NewMgr(ctx context.Context,
	g glue.Glue, pds []string,
	tlsConfig TLSConfig,
	keepalive keepalive.ClientParameters,
	checkRequirements bool,
	needDomain bool,
	versionCheckerType conn.VersionCheckerType,
) (*conn.Mgr, error)

NewMgr creates a new mgr at the given PD address.

func NewStreamMgr 

func NewStreamMgr(ctx context.Context, cfg *StreamConfig, g glue.Glue, isStreamStart bool) (*streamMgr, error)

func ParseTLSTripleFromFlags 

func ParseTLSTripleFromFlags(flags *pflag.FlagSet) (ca, cert, key string, err error)

ParseTLSTripleFromFlags parses the (ca, cert, key) triple from flags.

func ParseTSString 

func ParseTSString(ts string, tzCheck bool) (uint64, error)

ParseTSString port from tidb setSnapshotTS.

func PreCheckTableClusterIndex 

func PreCheckTableClusterIndex(
	tables []*metautil.Table,
	ddlJobs []*model.Job,
	dom *domain.Domain,
) error

PreCheckTableClusterIndex checks whether backup tables and existed tables have different cluster index options。

func PreCheckTableTiFlashReplica 

func PreCheckTableTiFlashReplica(
	ctx context.Context,
	pdClient pd.Client,
	tables []*metautil.Table,
	recorder *tiflashrec.TiFlashRecorder,
) error

PreCheckTableTiFlashReplica checks whether TiFlash replica is less than TiFlash node.

func ReadBackupMeta 

func ReadBackupMeta(
	ctx context.Context,
	fileName string,
	cfg *Config,
) (*backuppb.StorageBackend, storage.ExternalStorage, *backuppb.BackupMeta, error)

ReadBackupMeta reads the backupmeta file from the storage.

func ReadBackupMetaData 

func ReadBackupMetaData(ctx context.Context, s storage.ExternalStorage) (uint64, int, error)

func RunBackup 

func RunBackup(c context.Context, g glue.Glue, cmdName string, cfg *BackupConfig) error

RunBackup starts a backup task inside the current goroutine.

func RunBackupEBS 

func RunBackupEBS(c context.Context, g glue.Glue, cfg *BackupConfig) error

RunBackupEBS starts a backup task to backup volume vai EBS snapshot.

func RunBackupRaw 

func RunBackupRaw(c context.Context, g glue.Glue, cmdName string, cfg *RawKvConfig) error

RunBackupRaw starts a backup task inside the current goroutine.

func RunBackupTxn 

func RunBackupTxn(c context.Context, g glue.Glue, cmdName string, cfg *TxnKvConfig) error

RunBackupTxn starts a backup task inside the current goroutine.

func RunResolveKvData 

func RunResolveKvData(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error

RunResolveKvData starts a restore task inside the current goroutine.

func RunRestore 

func RunRestore(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error

RunRestore starts a restore task inside the current goroutine.

func RunRestoreEBSMeta 

func RunRestoreEBSMeta(c context.Context, g glue.Glue, cmdName string, cfg *RestoreConfig) error

RunRestoreEBSMeta phase 1 of EBS based restore

func RunRestoreRaw 

func RunRestoreRaw(c context.Context, g glue.Glue, cmdName string, cfg *RestoreRawConfig) (err error)

RunRestoreRaw starts a raw kv restore task inside the current goroutine.

func RunRestoreTxn 

func RunRestoreTxn(c context.Context, g glue.Glue, cmdName string, cfg *Config) (err error)

RunRestoreTxn starts a txn kv restore task inside the current goroutine.

func RunStreamAdvancer 

func RunStreamAdvancer(c context.Context, g glue.Glue, cmdName string, cfg *StreamConfig) error

func RunStreamCommand 

func RunStreamCommand(
	ctx context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamCommand run all kinds of `stream task`

func RunStreamMetadata 

func RunStreamMetadata(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

func RunStreamPause 

func RunStreamPause(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamPause specifies pausing a stream task.

func RunStreamRestore 

func RunStreamRestore(
	c context.Context,
	mgr *conn.Mgr,
	g glue.Glue,
	cfg *RestoreConfig,
) (err error)

RunStreamRestore restores stream log.

func RunStreamResume 

func RunStreamResume(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamResume specifies resuming a stream task.

func RunStreamStart 

func RunStreamStart(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamStart specifies starting a stream task

func RunStreamStatus 

func RunStreamStatus(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamStatus get status for a specific stream task

func RunStreamStop 

func RunStreamStop(
	c context.Context,
	g glue.Glue,
	cmdName string,
	cfg *StreamConfig,
) error

RunStreamStop specifies stoping a stream task

func RunStreamTruncate 

func RunStreamTruncate(c context.Context, g glue.Glue, cmdName string, cfg *StreamConfig) (err error)

RunStreamTruncate truncates the log that belong to (0, until-ts)

func ShiftTS 

func ShiftTS(startTS uint64) uint64

ShiftTS gets a smaller shiftTS than startTS. It has a safe duration between shiftTS and startTS for trasaction.

Types

type BackupConfig 

type BackupConfig struct {
	Config

	TimeAgo          time.Duration     `json:"time-ago" toml:"time-ago"`
	BackupTS         uint64            `json:"backup-ts" toml:"backup-ts"`
	LastBackupTS     uint64            `json:"last-backup-ts" toml:"last-backup-ts"`
	GCTTL            int64             `json:"gc-ttl" toml:"gc-ttl"`
	RemoveSchedulers bool              `json:"remove-schedulers" toml:"remove-schedulers"`
	IgnoreStats      bool              `json:"ignore-stats" toml:"ignore-stats"`
	UseBackupMetaV2  bool              `json:"use-backupmeta-v2"`
	UseCheckpoint    bool              `json:"use-checkpoint" toml:"use-checkpoint"`
	ReplicaReadLabel map[string]string `json:"replica-read-label" toml:"replica-read-label"`
	TableConcurrency uint              `json:"table-concurrency" toml:"table-concurrency"`
	CompressionConfig

	// for ebs-based backup
	FullBackupType          FullBackupType `json:"full-backup-type" toml:"full-backup-type"`
	VolumeFile              string         `json:"volume-file" toml:"volume-file"`
	SkipAWS                 bool           `json:"skip-aws" toml:"skip-aws"`
	CloudAPIConcurrency     uint           `json:"cloud-api-concurrency" toml:"cloud-api-concurrency"`
	ProgressFile            string         `json:"progress-file" toml:"progress-file"`
	SkipPauseGCAndScheduler bool           `json:"skip-pause-gc-and-scheduler" toml:"skip-pause-gc-and-scheduler"`
}

BackupConfig is the configuration specific for backup tasks.

func DefaultBackupConfig 

func DefaultBackupConfig(commonConfig Config) BackupConfig

func (*BackupConfig) Adjust 

func (cfg *BackupConfig) Adjust()

Adjust is use for BR(binary) and BR in TiDB. When new config was add and not included in parser. we should set proper value in this function. so that both binary and TiDB will use same default value.

func (*BackupConfig) Hash 

func (cfg *BackupConfig) Hash() ([]byte, error)

a rough hash for checkpoint checker

func (*BackupConfig) ParseFromFlags 

func (cfg *BackupConfig) ParseFromFlags(flags *pflag.FlagSet, skipCommonConfig bool) error

ParseFromFlags parses the backup-related flags from the flag set.

type CompressionConfig 

type CompressionConfig struct {
	CompressionType  backuppb.CompressionType `json:"compression-type" toml:"compression-type"`
	CompressionLevel int32                    `json:"compression-level" toml:"compression-level"`
}

CompressionConfig is the configuration for sst file compression.

type Config 

type Config struct {
	storage.BackendOptions

	Storage             string    `json:"storage" toml:"storage"`
	PD                  []string  `json:"pd" toml:"pd"`
	TLS                 TLSConfig `json:"tls" toml:"tls"`
	RateLimit           uint64    `json:"rate-limit" toml:"rate-limit"`
	ChecksumConcurrency uint      `json:"checksum-concurrency" toml:"checksum-concurrency"`
	TableConcurrency    uint      `json:"table-concurrency" toml:"table-concurrency"`
	Concurrency         uint32    `json:"concurrency" toml:"concurrency"`
	Checksum            bool      `json:"checksum" toml:"checksum"`
	SendCreds           bool      `json:"send-credentials-to-tikv" toml:"send-credentials-to-tikv"`
	// LogProgress is true means the progress bar is printed to the log instead of stdout.
	LogProgress bool `json:"log-progress" toml:"log-progress"`

	// CaseSensitive should not be used.
	//
	// Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field
	// should be removed after TiDB upgrades the BR dependency.
	CaseSensitive bool

	// NoCreds means don't try to load cloud credentials
	NoCreds bool `json:"no-credentials" toml:"no-credentials"`

	CheckRequirements bool `json:"check-requirements" toml:"check-requirements"`
	// EnableOpenTracing is whether to enable opentracing
	EnableOpenTracing bool `json:"enable-opentracing" toml:"enable-opentracing"`
	// SkipCheckPath skips verifying the path
	// deprecated
	SkipCheckPath bool `json:"skip-check-path" toml:"skip-check-path"`
	// Filter should not be used, use TableFilter instead.
	//
	// Deprecated: This field is kept only to satisfy the cyclic dependency with TiDB. This field
	// should be removed after TiDB upgrades the BR dependency.
	Filter filter.MySQLReplicationRules

	FilterStr          []string      `json:"filter-strings" toml:"filter-strings"`
	TableFilter        filter.Filter `json:"-" toml:"-"`
	SwitchModeInterval time.Duration `json:"switch-mode-interval" toml:"switch-mode-interval"`
	// Schemas is a database name set, to check whether the restore database has been backup
	Schemas map[string]struct{}
	// Tables is a table name set, to check whether the restore table has been backup
	Tables map[string]struct{}

	// GrpcKeepaliveTime is the interval of pinging the server.
	GRPCKeepaliveTime time.Duration `json:"grpc-keepalive-time" toml:"grpc-keepalive-time"`
	// GrpcKeepaliveTimeout is the max time a grpc conn can keep idel before killed.
	GRPCKeepaliveTimeout time.Duration `json:"grpc-keepalive-timeout" toml:"grpc-keepalive-timeout"`

	// Plaintext data key mainly used for full/snapshot backup and restore.
	CipherInfo backuppb.CipherInfo `json:"-" toml:"-"`

	// Could be used in log backup and restore but not recommended in a serious environment since data key is stored
	// in PD in plaintext.
	LogBackupCipherInfo backuppb.CipherInfo `json:"-" toml:"-"`

	// Master key based encryption for log restore.
	// More than one can be specified for log restore if master key rotated during log backup.
	MasterKeyConfig backuppb.MasterKeyConfig `json:"master-key-config" toml:"master-key-config"`

	// whether there's explicit filter
	ExplicitFilter bool `json:"-" toml:"-"`

	// KeyspaceName is the name of the keyspace of the task
	KeyspaceName string `json:"keyspace-name" toml:"keyspace-name"`

	// Metadata download batch size, such as metadata for log restore
	MetadataDownloadBatchSize uint `json:"metadata-download-batch-size" toml:"metadata-download-batch-size"`
}

Config is the common configuration for all BRIE tasks.

func DefaultConfig 

func DefaultConfig() Config

func (*Config) OverrideDefaultForBackup 

func (cfg *Config) OverrideDefaultForBackup()

OverrideDefaultForBackup override common config for backup tasks

func (*Config) ParseFromFlags 

func (cfg *Config) ParseFromFlags(flags *pflag.FlagSet) error

ParseFromFlags parses the config from the flag set.

type DDLJobFilterRule 

type DDLJobFilterRule func(ddlJob *model.Job) bool

type FullBackupType 

type FullBackupType string

FullBackupType type when doing full backup or restore 

const (
	FullBackupTypeKV  FullBackupType = "kv" // default type
	FullBackupTypeEBS FullBackupType = "aws-ebs"
)

func (FullBackupType) Valid 

func (t FullBackupType) Valid() bool

Valid whether the type is valid

type PiTRTaskInfo 

type PiTRTaskInfo struct {
	CheckpointInfo      *checkpoint.CheckpointTaskInfoForLogRestore
	NeedFullRestore     bool
	FullRestoreCheckErr error
}

type RawKvConfig 

type RawKvConfig struct {
	Config

	StartKey []byte `json:"start-key" toml:"start-key"`
	EndKey   []byte `json:"end-key" toml:"end-key"`
	CF       string `json:"cf" toml:"cf"`
	CompressionConfig
	RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"`
}

RawKvConfig is the common config for rawkv backup and restore.

func (*RawKvConfig) ParseBackupConfigFromFlags 

func (cfg *RawKvConfig) ParseBackupConfigFromFlags(flags *pflag.FlagSet) error

ParseBackupConfigFromFlags parses the backup-related flags from the flag set.

func (*RawKvConfig) ParseFromFlags 

func (cfg *RawKvConfig) ParseFromFlags(flags *pflag.FlagSet) error

ParseFromFlags parses the raw kv backup&restore common flags from the flag set.

type RestoreCommonConfig 

type RestoreCommonConfig struct {
	Online              bool                     `json:"online" toml:"online"`
	Granularity         string                   `json:"granularity" toml:"granularity"`
	ConcurrencyPerStore pconfig.ConfigTerm[uint] `json:"tikv-max-restore-concurrency" toml:"tikv-max-restore-concurrency"`

	// MergeSmallRegionSizeBytes is the threshold of merging small regions (Default 96MB, region split size).
	// MergeSmallRegionKeyCount is the threshold of merging smalle regions (Default 960_000, region split key count).
	// See https://github.com/tikv/tikv/blob/v4.0.8/components/raftstore/src/coprocessor/config.rs#L35-L38
	MergeSmallRegionSizeBytes pconfig.ConfigTerm[uint64] `json:"merge-region-size-bytes" toml:"merge-region-size-bytes"`
	MergeSmallRegionKeyCount  pconfig.ConfigTerm[uint64] `json:"merge-region-key-count" toml:"merge-region-key-count"`

	// determines whether enable restore sys table on default, see fullClusterRestore in restore/client.go
	WithSysTable bool `json:"with-sys-table" toml:"with-sys-table"`

	ResetSysUsers []string `json:"reset-sys-users" toml:"reset-sys-users"`
}

RestoreCommonConfig is the common configuration for all BR restore tasks.

func (*RestoreCommonConfig) ParseFromFlags 

func (cfg *RestoreCommonConfig) ParseFromFlags(flags *pflag.FlagSet) error

ParseFromFlags parses the config from the flag set.

type RestoreConfig 

type RestoreConfig struct {
	Config
	RestoreCommonConfig

	NoSchema           bool          `json:"no-schema" toml:"no-schema"`
	LoadStats          bool          `json:"load-stats" toml:"load-stats"`
	PDConcurrency      uint          `json:"pd-concurrency" toml:"pd-concurrency"`
	StatsConcurrency   uint          `json:"stats-concurrency" toml:"stats-concurrency"`
	BatchFlushInterval time.Duration `json:"batch-flush-interval" toml:"batch-flush-interval"`
	// DdlBatchSize use to define the size of batch ddl to create tables
	DdlBatchSize uint `json:"ddl-batch-size" toml:"ddl-batch-size"`

	WithPlacementPolicy string `json:"with-tidb-placement-mode" toml:"with-tidb-placement-mode"`

	// FullBackupStorage is used to  run `restore full` before `restore log`.
	// if it is empty, directly take restoring log justly.
	FullBackupStorage string `json:"full-backup-storage" toml:"full-backup-storage"`

	// AllowPITRFromIncremental indicates whether this restore should enter a compatibility mode for incremental restore.
	// In this restore mode, the restore will not perform timestamp rewrite on the incremental data.
	AllowPITRFromIncremental bool `json:"allow-pitr-from-incremental" toml:"allow-pitr-from-incremental"`

	// [startTs, RestoreTS] is used to `restore log` from StartTS to RestoreTS.
	StartTS uint64 `json:"start-ts" toml:"start-ts"`
	// if not specified system will restore to the max TS available
	RestoreTS uint64 `json:"restore-ts" toml:"restore-ts"`

	PitrBatchCount  uint32 `json:"pitr-batch-count" toml:"pitr-batch-count"`
	PitrBatchSize   uint32 `json:"pitr-batch-size" toml:"pitr-batch-size"`
	PitrConcurrency uint32 `json:"-" toml:"-"`

	UseCheckpoint bool `json:"use-checkpoint" toml:"use-checkpoint"`

	WaitTiflashReady bool `json:"wait-tiflash-ready" toml:"wait-tiflash-ready"`

	// for ebs-based restore
	FullBackupType      FullBackupType        `json:"full-backup-type" toml:"full-backup-type"`
	Prepare             bool                  `json:"prepare" toml:"prepare"`
	OutputFile          string                `json:"output-file" toml:"output-file"`
	SkipAWS             bool                  `json:"skip-aws" toml:"skip-aws"`
	CloudAPIConcurrency uint                  `json:"cloud-api-concurrency" toml:"cloud-api-concurrency"`
	VolumeType          pconfig.EBSVolumeType `json:"volume-type" toml:"volume-type"`
	VolumeIOPS          int64                 `json:"volume-iops" toml:"volume-iops"`
	VolumeThroughput    int64                 `json:"volume-throughput" toml:"volume-throughput"`
	VolumeEncrypted     bool                  `json:"volume-encrypted" toml:"volume-encrypted"`
	ProgressFile        string                `json:"progress-file" toml:"progress-file"`
	TargetAZ            string                `json:"target-az" toml:"target-az"`
	UseFSR              bool                  `json:"use-fsr" toml:"use-fsr"`
	// contains filtered or unexported fields
}

RestoreConfig is the configuration specific for restore tasks.

func DefaultRestoreConfig 

func DefaultRestoreConfig(commonConfig Config) RestoreConfig

func (*RestoreConfig) Adjust 

func (cfg *RestoreConfig) Adjust()

Adjust is use for BR(binary) and BR in TiDB. When new config was added and not included in parser. we should set proper value in this function. so that both binary and TiDB will use same default value.

func (*RestoreConfig) ParseFromFlags 

func (cfg *RestoreConfig) ParseFromFlags(flags *pflag.FlagSet, skipCommonConfig bool) error

ParseFromFlags parses the restore-related flags from the flag set.

func (*RestoreConfig) ParseStreamRestoreFlags 

func (cfg *RestoreConfig) ParseStreamRestoreFlags(flags *pflag.FlagSet) error

ParseStreamRestoreFlags parses the `restore stream` flags from the flag set.

type RestoreFunc 

type RestoreFunc func(string) error

func KeepGcDisabled 

func KeepGcDisabled(g glue.Glue, store kv.Storage) (RestoreFunc, string, error)

KeepGcDisabled keeps GC disabled and return a function that used to gc enabled. gc.ratio-threshold = "-1.0", which represents disable gc in TiKV.

type RestoreRawConfig 

type RestoreRawConfig struct {
	RawKvConfig
	RestoreCommonConfig
}

RestoreRawConfig is the configuration specific for raw kv restore tasks.

func (*RestoreRawConfig) ParseFromFlags 

func (cfg *RestoreRawConfig) ParseFromFlags(flags *pflag.FlagSet) error

ParseFromFlags parses the backup-related flags from the flag set.

type StreamConfig 

type StreamConfig struct {
	Config

	TaskName string `json:"task-name" toml:"task-name"`

	// StartTS usually equals the tso of full-backup, but user can reset it
	StartTS uint64 `json:"start-ts" toml:"start-ts"`
	EndTS   uint64 `json:"end-ts" toml:"end-ts"`
	// SafePointTTL ensures TiKV can scan entries not being GC at [startTS, currentTS]
	SafePointTTL int64 `json:"safe-point-ttl" toml:"safe-point-ttl"`

	// Spec for the command `truncate`, we should truncate the until when?
	Until              uint64 `json:"until" toml:"until"`
	DryRun             bool   `json:"dry-run" toml:"dry-run"`
	SkipPrompt         bool   `json:"skip-prompt" toml:"skip-prompt"`
	CleanUpCompactions bool   `json:"clean-up-compactions" toml:"clean-up-compactions"`

	// Spec for the command `status`.
	JSONOutput bool `json:"json-output" toml:"json-output"`

	// Spec for the command `advancer`.
	AdvancerCfg advancercfg.Config `json:"advancer-config" toml:"advancer-config"`
}

StreamConfig specifies the configure about backup stream

func (*StreamConfig) ParseStreamCommonFromFlags 

func (cfg *StreamConfig) ParseStreamCommonFromFlags(flags *pflag.FlagSet) error

ParseStreamCommonFromFlags parse parameters for `stream task`

func (*StreamConfig) ParseStreamPauseFromFlags 

func (cfg *StreamConfig) ParseStreamPauseFromFlags(flags *pflag.FlagSet) error

ParseStreamPauseFromFlags parse parameters for `stream pause`

func (*StreamConfig) ParseStreamStartFromFlags 

func (cfg *StreamConfig) ParseStreamStartFromFlags(flags *pflag.FlagSet) error

ParseStreamStartFromFlags parse parameters for `stream start`

func (*StreamConfig) ParseStreamStatusFromFlags 

func (cfg *StreamConfig) ParseStreamStatusFromFlags(flags *pflag.FlagSet) error

func (*StreamConfig) ParseStreamTruncateFromFlags 

func (cfg *StreamConfig) ParseStreamTruncateFromFlags(flags *pflag.FlagSet) error

type TLSConfig 

type TLSConfig struct {
	CA   string `json:"ca" toml:"ca"`
	Cert string `json:"cert" toml:"cert"`
	Key  string `json:"key" toml:"key"`
}

TLSConfig is the common configuration for TLS connection.

func (*TLSConfig) IsEnabled 

func (tls *TLSConfig) IsEnabled() bool

IsEnabled checks if TLS open or not.

func (*TLSConfig) ParseFromFlags 

func (tls *TLSConfig) ParseFromFlags(flags *pflag.FlagSet) (err error)

ParseFromFlags parses the TLS config from the flag set.

func (*TLSConfig) ToKVSecurity 

func (tls *TLSConfig) ToKVSecurity() config.Security

Convert the TLS config to the PD security option.

func (*TLSConfig) ToPDSecurityOption 

func (tls *TLSConfig) ToPDSecurityOption() pd.SecurityOption

Convert the TLS config to the PD security option.

func (*TLSConfig) ToTLSConfig 

func (tls *TLSConfig) ToTLSConfig() (*tls.Config, error)

ToTLSConfig generate tls.Config.

type TxnKvConfig 

type TxnKvConfig struct {
	Config

	StartKey     []byte `json:"start-key" toml:"start-key"`
	EndKey       []byte `json:"end-key" toml:"end-key"`
	StartVersion int64  `json:"start-version" toml:"start-version"`

	CompressionConfig
	RemoveSchedulers bool `json:"remove-schedulers" toml:"remove-schedulers"`
}

TxnKvConfig is the common config for txn kv backup and restore.

func (*TxnKvConfig) Adjust 

func (cfg *TxnKvConfig) Adjust()

Adjust Txn kv config for backup

func (*TxnKvConfig) ParseBackupConfigFromFlags 

func (cfg *TxnKvConfig) ParseBackupConfigFromFlags(flags *pflag.FlagSet) error

ParseBackupConfigFromFlags parses the backup-related flags from the flag set.

func (*TxnKvConfig) ParseFromFlags 

func (cfg *TxnKvConfig) ParseFromFlags(flags *pflag.FlagSet) error

ParseFromFlags parses the txn kv backup&restore common flags from the flag set.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.