restore

Documentation

Index

• func AssertUserDBsEmpty(dom *domain.Domain) error
• func GetFileRangeKey(f string) string
• func GetTS(ctx context.Context, pdClient pd.Client) (uint64, error)
• func GetTSWithRetry(ctx context.Context, pdClient pd.Client) (uint64, error)
• func GetTableSchema(dom *domain.Domain, dbName pmodel.CIStr, tableName pmodel.CIStr) (*model.TableInfo, error)
• func MarshalLogObjectForFiles(batchFileSet BatchBackupFileSet, encoder zapcore.ObjectEncoder) error
• func RestorePostWork(ctx context.Context, switcher *ImportModeSwitcher, ...)
• func RestorePreWork(ctx context.Context, mgr *conn.Mgr, switcher *ImportModeSwitcher, ...) (pdutil.UndoFunc, *pdutil.ClusterConfig, error)
• func TransferBoolToValue(enable bool) string
• func ZapBatchBackupFileSet(batchFileSet BatchBackupFileSet) zap.Field
• type BackupFileSet
  • func CreateUniqueFileSets(files []*backuppb.File) []BackupFileSet
  • func NewFileSet(files []*backuppb.File, rules *utils.RewriteRules) BackupFileSet
• type BalancedFileImporter
• type BatchBackupFileSet
• type FileImporter
• type Granularity
• type ImportModeSwitcher
  • func NewImportModeSwitcher(pdClient pd.Client, switchModeInterval time.Duration, tlsConf *tls.Config) *ImportModeSwitcher
  • func (switcher *ImportModeSwitcher) GoSwitchToImportMode(ctx context.Context) error
  • func (switcher *ImportModeSwitcher) SwitchToNormalMode(ctx context.Context) error
• type MultiTablesRestorer
  • func (m *MultiTablesRestorer) Close() error
  • func (m *MultiTablesRestorer) GoRestore(onProgress func(int64), batchFileSets ...BatchBackupFileSet) (err error)
  • func (m *MultiTablesRestorer) WaitUntilFinish() error
• type PipelineRestorerWrapper
  • func (p *PipelineRestorerWrapper[T]) WithSplit(ctx context.Context, i iter.TryNextor[T], strategy split.SplitStrategy[T]) iter.TryNextor[T]
• type SimpleRestorer
  • func (s *SimpleRestorer) Close() error
  • func (s *SimpleRestorer) GoRestore(onProgress func(int64), batchFileSets ...BatchBackupFileSet) error
  • func (s *SimpleRestorer) WaitUntilFinish() error
• type SstRestorer
  • func NewMultiTablesRestorer(ctx context.Context, fileImporter BalancedFileImporter, ...) SstRestorer
  • func NewSimpleSstRestorer(ctx context.Context, fileImporter FileImporter, workerPool *util.WorkerPool, ...) SstRestorer
• type UniqueTableName

Functions

func AssertUserDBsEmpty

func AssertUserDBsEmpty(dom *domain.Domain) error

AssertUserDBsEmpty check whether user dbs exist in the cluster

func GetFileRangeKey

func GetFileRangeKey(f string) string

GetFileRangeKey is used to reduce the checkpoint number, because we combine the write cf/default cf into one restore file group. during full restore, so we can reduce the checkpoint number with the common prefix of the file.

func GetTS

func GetTS(ctx context.Context, pdClient pd.Client) (uint64, error)

GetTS gets a new timestamp from PD.

func GetTSWithRetry

func GetTSWithRetry(ctx context.Context, pdClient pd.Client) (uint64, error)

GetTSWithRetry gets a new timestamp with retry from PD.

func GetTableSchema

func GetTableSchema(
	dom *domain.Domain,
	dbName pmodel.CIStr,
	tableName pmodel.CIStr,
) (*model.TableInfo, error)

GetTableSchema returns the schema of a table from TiDB.

func MarshalLogObjectForFiles

func MarshalLogObjectForFiles(batchFileSet BatchBackupFileSet, encoder zapcore.ObjectEncoder) error

MarshalLogObjectForFiles is an internal util function to zap something having `Files` field.

func RestorePostWork

func RestorePostWork(
	ctx context.Context,
	switcher *ImportModeSwitcher,
	restoreSchedulers pdutil.UndoFunc,
	isOnline bool,
)

RestorePostWork executes some post work after restore. TODO: aggregate all lifetime manage methods into batcher's context manager field.

func RestorePreWork

func RestorePreWork(
	ctx context.Context,
	mgr *conn.Mgr,
	switcher *ImportModeSwitcher,
	isOnline bool,
	switchToImport bool,
) (pdutil.UndoFunc, *pdutil.ClusterConfig, error)

RestorePreWork executes some prepare work before restore. TODO make this function returns a restore post work.

func TransferBoolToValue

func TransferBoolToValue(enable bool) string

func ZapBatchBackupFileSet

func ZapBatchBackupFileSet(batchFileSet BatchBackupFileSet) zap.Field

Types

type BackupFileSet

type BackupFileSet struct {
	// TableID only valid in 3.4.5.
	// For Raw/Txn KV, table id is always 0
	TableID int64

	// For log Backup Changes, this field is null.
	SSTFiles []*backuppb.File

	// RewriteRules is the rewrite rules for the specify table.
	// because these rules belongs to the *one table*.
	// we can hold them here.
	RewriteRules *utils.RewriteRules
}

BackupFileSet represents the batch files to be restored for a table. Current, we have 5 type files 1. Raw KV(sst files) 2. Txn KV(sst files) 3. Database KV backup(sst files) 4. Compacted Log backups(sst files)

func CreateUniqueFileSets

func CreateUniqueFileSets(files []*backuppb.File) []BackupFileSet

CreateUniqueFileSets used for Raw/Txn non-tableID files converts a slice of files into a slice of unique BackupFileSets, where each BackupFileSet contains a single file.

func NewFileSet

func NewFileSet(files []*backuppb.File, rules *utils.RewriteRules) BackupFileSet

type BalancedFileImporter

type BalancedFileImporter interface {
	FileImporter

	// PauseForBackpressure manages concurrency by controlling when imports can proceed,
	// ensuring load is distributed evenly across storage nodes.
	PauseForBackpressure()
}

BalancedFileImporter is a wrapper around FileImporter that adds concurrency controls. It ensures that file imports are balanced across storage nodes, which is particularly useful in MultiTablesRestorer scenarios where concurrency management is critical for efficiency.

type BatchBackupFileSet

type BatchBackupFileSet []BackupFileSet

type FileImporter

type FileImporter interface {
	// Import uploads and imports the provided backup file sets into storage.
	// The ctx parameter provides context for managing request scope.
	Import(ctx context.Context, fileSets ...BackupFileSet) error

	// Close releases any resources used by the importer client.
	Close() error
}

FileImporter is a low-level interface for handling the import of backup files into storage (e.g., TiKV). It is primarily used by the importer client to manage raw and transactional SST file imports.

type Granularity

type Granularity string

deprecated parameter

const (
	FineGrained   Granularity = "fine-grained"
	CoarseGrained Granularity = "coarse-grained"
)

type ImportModeSwitcher

type ImportModeSwitcher struct {
	// contains filtered or unexported fields
}

func NewImportModeSwitcher

func NewImportModeSwitcher(
	pdClient pd.Client,
	switchModeInterval time.Duration,
	tlsConf *tls.Config,
) *ImportModeSwitcher

func (*ImportModeSwitcher) GoSwitchToImportMode

func (switcher *ImportModeSwitcher) GoSwitchToImportMode(
	ctx context.Context,
) error

GoSwitchToImportMode switch tikv cluster to import mode.

func (*ImportModeSwitcher) SwitchToNormalMode

func (switcher *ImportModeSwitcher) SwitchToNormalMode(ctx context.Context) error

switchToNormalMode switch tikv cluster to normal mode.

type MultiTablesRestorer

type MultiTablesRestorer struct {
	// contains filtered or unexported fields
}

func (*MultiTablesRestorer) Close

func (m *MultiTablesRestorer) Close() error

func (*MultiTablesRestorer) GoRestore

func (m *MultiTablesRestorer) GoRestore(onProgress func(int64), batchFileSets ...BatchBackupFileSet) (err error)

func (*MultiTablesRestorer) WaitUntilFinish

func (m *MultiTablesRestorer) WaitUntilFinish() error

type PipelineRestorerWrapper

type PipelineRestorerWrapper[T any] struct {
	split.PipelineRegionsSplitter
}

func (*PipelineRestorerWrapper[T]) WithSplit

func (p *PipelineRestorerWrapper[T]) WithSplit(ctx context.Context, i iter.TryNextor[T], strategy split.SplitStrategy[T]) iter.TryNextor[T]

WithSplit processes items using a split strategy within a pipeline. It iterates over items, accumulating them until a split condition is met. When a split is required, it executes the split operation on the accumulated items.

type SimpleRestorer

type SimpleRestorer struct {
	// contains filtered or unexported fields
}

func (*SimpleRestorer) Close

func (s *SimpleRestorer) Close() error

func (*SimpleRestorer) GoRestore

func (s *SimpleRestorer) GoRestore(onProgress func(int64), batchFileSets ...BatchBackupFileSet) error

func (*SimpleRestorer) WaitUntilFinish

func (s *SimpleRestorer) WaitUntilFinish() error

type SstRestorer

type SstRestorer interface {
	// GoRestore imports the specified backup file sets into TiKV asynchronously.
	// The onProgress function is called with progress updates as files are processed.
	GoRestore(onProgress func(int64), batchFileSets ...BatchBackupFileSet) error

	// WaitUntilFinish blocks until all pending restore files have completed processing.
	WaitUntilFinish() error

	// Close releases any resources associated with the restoration process.
	Close() error
}

SstRestorer defines the essential methods required for restoring SST files in various backup formats: 1. Raw backup SST files 2. Transactional (Txn) backup SST files 3. TiDB backup SST files 4. Log-compacted SST files

It serves as a high-level interface for restoration, supporting implementations such as simpleRestorer and MultiTablesRestorer. SstRestorer includes FileImporter for handling raw, transactional, and compacted SSTs, and MultiTablesRestorer for TiDB-specific backups.

func NewMultiTablesRestorer

func NewMultiTablesRestorer(
	ctx context.Context,
	fileImporter BalancedFileImporter,
	workerPool *util.WorkerPool,
	checkpointRunner *checkpoint.CheckpointRunner[checkpoint.RestoreKeyType, checkpoint.RestoreValueType],
) SstRestorer

func NewSimpleSstRestorer

func NewSimpleSstRestorer(
	ctx context.Context,
	fileImporter FileImporter,
	workerPool *util.WorkerPool,
	checkpointRunner *checkpoint.CheckpointRunner[checkpoint.RestoreKeyType, checkpoint.RestoreValueType],
) SstRestorer

type UniqueTableName

type UniqueTableName struct {
	DB    string
	Table string
}