Documentation ¶
Index ¶
- Constants
- func CheckSysTableCompatibility(dom *domain.Domain, tables []*metautil.Table) error
- func MarshalLogObjectForFiles(files []TableIDWithFiles, encoder zapcore.ObjectEncoder) error
- func SortAndValidateFileRanges(createdTables []*CreatedTable, allFiles []*backuppb.File, ...) ([][]byte, [][]TableIDWithFiles, error)
- type CreatedTable
- type KvMode
- type PhysicalTable
- type PlacementRuleManager
- type RewriteMode
- type SnapClient
- func (rc *SnapClient) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error
- func (rc *SnapClient) CheckTargetClusterFresh(ctx context.Context) error
- func (rc *SnapClient) Close()
- func (rc *SnapClient) CreateDatabases(ctx context.Context, dbs []*metautil.Database) error
- func (rc *SnapClient) CreatePolicies(ctx context.Context, policyMap *sync.Map) error
- func (rc *SnapClient) CreateTables(ctx context.Context, tables []*metautil.Table, newTS uint64) ([]*CreatedTable, error)
- func (rc *SnapClient) EnableSkipCreateSQL()
- func (rc *SnapClient) ExecDDLs(ctx context.Context, ddlJobs []*model.Job) error
- func (rc *SnapClient) GetBatchDdlSize() uint
- func (rc *SnapClient) GetClusterID(ctx context.Context) uint64
- func (rc *SnapClient) GetDDLJobs() []*model.Job
- func (rc *SnapClient) GetDatabases() []*metautil.Database
- func (rc *SnapClient) GetDomain() *domain.Domain
- func (rc *SnapClient) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error)
- func (rc *SnapClient) GetPlacementPolicies() (*sync.Map, error)
- func (rc *SnapClient) GetRewriteMode() RewriteMode
- func (rc *SnapClient) GetSupportPolicy() bool
- func (rc *SnapClient) GetTLSConfig() *tls.Config
- func (rc *SnapClient) GoUpdateMetaAndLoadStats(ctx context.Context, s storage.ExternalStorage, inCh <-chan *CreatedTable, ...) chan *CreatedTable
- func (rc *SnapClient) GoValidateChecksum(ctx context.Context, inCh <-chan *CreatedTable, kvClient kv.Client, ...) chan *CreatedTable
- func (rc *SnapClient) GoWaitTiFlashReady(ctx context.Context, inCh <-chan *CreatedTable, updateCh glue.Progress, ...) chan *CreatedTable
- func (rc *SnapClient) HasBackedUpSysDB() bool
- func (rc *SnapClient) Init(g glue.Glue, store kv.Storage) error
- func (rc *SnapClient) InitCheckpoint(ctx context.Context, g glue.Glue, store kv.Storage, ...) (checkpointSetWithTableID map[int64]map[string]struct{}, ...)
- func (rc *SnapClient) InitFullClusterRestore(explicitFilter bool)
- func (rc *SnapClient) IsFull() bool
- func (rc *SnapClient) IsFullClusterRestore() bool
- func (rc *SnapClient) IsIncremental() bool
- func (rc *SnapClient) IsRawKvMode() bool
- func (rc *SnapClient) IsSkipCreateSQL() bool
- func (rc *SnapClient) LoadSchemaIfNeededAndInitClient(c context.Context, backupMeta *backuppb.BackupMeta, ...) error
- func (rc *SnapClient) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool
- func (rc *SnapClient) ResetSpeedLimit(ctx context.Context) error
- func (rc *SnapClient) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error
- func (rc *SnapClient) RestoreRaw(ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, ...) error
- func (rc *SnapClient) RestoreSSTFiles(ctx context.Context, tableIDWithFilesGroup [][]TableIDWithFiles, ...) (retErr error)
- func (rc *SnapClient) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (rerr error)
- func (rc *SnapClient) RestoreTables(ctx context.Context, placementRuleManager PlacementRuleManager, ...) error
- func (rc *SnapClient) SetBatchDdlSize(batchDdlsize uint)
- func (rc *SnapClient) SetConcurrencyPerStore(c uint)
- func (rc *SnapClient) SetCrypter(crypter *backuppb.CipherInfo)
- func (rc *SnapClient) SetPlacementPolicyMode(withPlacementPolicy string)
- func (rc *SnapClient) SetPolicyMap(p *sync.Map)
- func (rc *SnapClient) SetRateLimit(rateLimit uint64)
- func (rc *SnapClient) SetRewriteMode(ctx context.Context)
- func (rc *SnapClient) SetWithSysTable(withSysTable bool)
- func (rc *SnapClient) SplitPoints(ctx context.Context, sortedSplitKeys [][]byte, updateCh glue.Progress, ...) error
- func (rc *SnapClient) WaitForFilesRestored(ctx context.Context, files []*backuppb.File, updateCh glue.Progress) error
- func (rc *SnapClient) WaitForFinishCheckpoint(ctx context.Context, flush bool)
- type SnapFileImporter
- func (importer *SnapFileImporter) Close() error
- func (importer *SnapFileImporter) ImportSSTFiles(ctx context.Context, filesGroup []TableIDWithFiles, ...) error
- func (importer *SnapFileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error
- func (importer *SnapFileImporter) SetRawRange(startKey, endKey []byte) error
- func (importer *SnapFileImporter) ShouldBlock() bool
- func (importer *SnapFileImporter) WaitUntilUnblock()
- type TableIDWithFiles
Constants ¶
const MergedRangeCountThreshold = 1536
If there are many tables with only a few rows, the number of merged SSTs will be too large. So set a threshold to avoid it.
Variables ¶
This section is empty.
Functions ¶
func MarshalLogObjectForFiles ¶
func MarshalLogObjectForFiles(files []TableIDWithFiles, encoder zapcore.ObjectEncoder) error
MarshalLogObjectForFiles is an internal util function to zap something having `Files` field.
func SortAndValidateFileRanges ¶
func SortAndValidateFileRanges( createdTables []*CreatedTable, allFiles []*backuppb.File, checkpointSetWithTableID map[int64]map[string]struct{}, splitSizeBytes, splitKeyCount uint64, splitOnTable bool, updateCh glue.Progress, ) ([][]byte, [][]TableIDWithFiles, error)
SortAndValidateFileRanges sort, merge and validate files by tables and yields tables with range.
Types ¶
type CreatedTable ¶
type CreatedTable struct { RewriteRule *restoreutils.RewriteRules Table *model.TableInfo OldTable *metautil.Table }
CreatedTable is a table created on restore process, but not yet filled with data.
type PhysicalTable ¶
type PhysicalTable struct { NewPhysicalID int64 OldPhysicalID int64 RewriteRules *restoreutils.RewriteRules }
type PlacementRuleManager ¶
type PlacementRuleManager interface { SetPlacementRule(ctx context.Context, tables []*CreatedTable) error ResetPlacementRules(ctx context.Context) error }
PlacementRuleManager manages to set the placement rule of tables to label constraint key `exclusive`, and unset the rule.
type RewriteMode ¶
type RewriteMode int
RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules.
const ( // RewriteModeLegacy means no rewrite rule is applied. RewriteModeLegacy RewriteMode = iota // RewriteModeKeyspace means the rewrite rule could be applied to keyspace. RewriteModeKeyspace )
type SnapClient ¶
type SnapClient struct {
// contains filtered or unexported fields
}
func NewRestoreClient ¶
func NewRestoreClient( pdClient pd.Client, pdHTTPCli pdhttp.Client, tlsConf *tls.Config, keepaliveConf keepalive.ClientParameters, ) *SnapClient
NewRestoreClient returns a new RestoreClient.
func (*SnapClient) AllocTableIDs ¶
AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in the download stage.
func (*SnapClient) CheckTargetClusterFresh ¶
func (rc *SnapClient) CheckTargetClusterFresh(ctx context.Context) error
CheckTargetClusterFresh check whether the target cluster is fresh or not if there's no user dbs or tables, we take it as a fresh cluster, although user may have created some users or made other changes.
func (*SnapClient) CreateDatabases ¶
CreateDatabases creates databases. If the client has the db pool, it would create it.
func (*SnapClient) CreatePolicies ¶
CreatePolicies creates all policies in full restore.
func (*SnapClient) CreateTables ¶
func (rc *SnapClient) CreateTables( ctx context.Context, tables []*metautil.Table, newTS uint64, ) ([]*CreatedTable, error)
CreateTables create tables, and generate their information. this function will use workers as the same number of sessionPool, leave sessionPool nil to send DDLs sequential.
func (*SnapClient) EnableSkipCreateSQL ¶
func (rc *SnapClient) EnableSkipCreateSQL()
EnableSkipCreateSQL sets switch of skip create schema and tables.
func (*SnapClient) GetBatchDdlSize ¶
func (rc *SnapClient) GetBatchDdlSize() uint
func (*SnapClient) GetClusterID ¶
func (rc *SnapClient) GetClusterID(ctx context.Context) uint64
GetClusterID gets the cluster id from down-stream cluster.
func (*SnapClient) GetDDLJobs ¶
func (rc *SnapClient) GetDDLJobs() []*model.Job
GetDDLJobs returns ddl jobs.
func (*SnapClient) GetDatabases ¶
func (rc *SnapClient) GetDatabases() []*metautil.Database
GetDatabases returns all databases.
func (*SnapClient) GetDomain ¶
func (rc *SnapClient) GetDomain() *domain.Domain
func (*SnapClient) GetFilesInRawRange ¶
func (rc *SnapClient) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error)
GetFilesInRawRange gets all files that are in the given range or intersects with the given range.
func (*SnapClient) GetPlacementPolicies ¶
func (rc *SnapClient) GetPlacementPolicies() (*sync.Map, error)
GetPlacementPolicies returns policies.
func (*SnapClient) GetRewriteMode ¶
func (rc *SnapClient) GetRewriteMode() RewriteMode
func (*SnapClient) GetSupportPolicy ¶
func (rc *SnapClient) GetSupportPolicy() bool
GetSupportPolicy tells whether target tidb support placement policy.
func (*SnapClient) GetTLSConfig ¶
func (rc *SnapClient) GetTLSConfig() *tls.Config
GetTLSConfig returns the tls config.
func (*SnapClient) GoUpdateMetaAndLoadStats ¶
func (rc *SnapClient) GoUpdateMetaAndLoadStats( ctx context.Context, s storage.ExternalStorage, inCh <-chan *CreatedTable, errCh chan<- error, statsConcurrency uint, loadStats bool, ) chan *CreatedTable
func (*SnapClient) GoValidateChecksum ¶
func (rc *SnapClient) GoValidateChecksum( ctx context.Context, inCh <-chan *CreatedTable, kvClient kv.Client, errCh chan<- error, updateCh glue.Progress, concurrency uint, ) chan *CreatedTable
GoValidateChecksum forks a goroutine to validate checksum after restore. it returns a channel fires a struct{} when all things get done.
func (*SnapClient) GoWaitTiFlashReady ¶
func (rc *SnapClient) GoWaitTiFlashReady( ctx context.Context, inCh <-chan *CreatedTable, updateCh glue.Progress, errCh chan<- error, ) chan *CreatedTable
func (*SnapClient) HasBackedUpSysDB ¶
func (rc *SnapClient) HasBackedUpSysDB() bool
HasBackedUpSysDB whether we have backed up system tables br backs system tables up since 5.1.0
func (*SnapClient) InitCheckpoint ¶
func (rc *SnapClient) InitCheckpoint( ctx context.Context, g glue.Glue, store kv.Storage, config *pdutil.ClusterConfig, checkpointFirstRun bool, ) (checkpointSetWithTableID map[int64]map[string]struct{}, checkpointClusterConfig *pdutil.ClusterConfig, err error)
InitCheckpoint initialize the checkpoint status for the cluster. If the cluster is restored for the first time, it will initialize the checkpoint metadata. Otherwrise, it will load checkpoint metadata and checkpoint ranges/checksum from the external storage.
func (*SnapClient) InitFullClusterRestore ¶
func (rc *SnapClient) InitFullClusterRestore(explicitFilter bool)
InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed
func (*SnapClient) IsFull ¶
func (rc *SnapClient) IsFull() bool
IsFull returns whether this backup is full.
func (*SnapClient) IsFullClusterRestore ¶
func (rc *SnapClient) IsFullClusterRestore() bool
func (*SnapClient) IsIncremental ¶
func (rc *SnapClient) IsIncremental() bool
IsIncremental returns whether this backup is incremental.
func (*SnapClient) IsRawKvMode ¶
func (rc *SnapClient) IsRawKvMode() bool
IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.
func (*SnapClient) IsSkipCreateSQL ¶
func (rc *SnapClient) IsSkipCreateSQL() bool
IsSkipCreateSQL returns whether we need skip create schema and tables in restore.
func (*SnapClient) LoadSchemaIfNeededAndInitClient ¶
func (rc *SnapClient) LoadSchemaIfNeededAndInitClient( c context.Context, backupMeta *backuppb.BackupMeta, backend *backuppb.StorageBackend, reader *metautil.MetaReader, loadStats bool) error
LoadSchemaIfNeededAndInitClient loads schemas from BackupMeta to initialize RestoreClient.
func (*SnapClient) NeedCheckFreshCluster ¶
func (rc *SnapClient) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool
NeedCheckFreshCluster is every time. except restore from a checkpoint or user has not set filter argument.
func (*SnapClient) ResetSpeedLimit ¶
func (rc *SnapClient) ResetSpeedLimit(ctx context.Context) error
func (*SnapClient) ResetTS ¶
func (rc *SnapClient) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error
ResetTS resets the timestamp of PD to a bigger value.
func (*SnapClient) RestoreRaw ¶
func (rc *SnapClient) RestoreRaw( ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, updateCh glue.Progress, ) error
RestoreRaw tries to restore raw keys in the specified range.
func (*SnapClient) RestoreSSTFiles ¶
func (rc *SnapClient) RestoreSSTFiles( ctx context.Context, tableIDWithFilesGroup [][]TableIDWithFiles, updateCh glue.Progress, ) (retErr error)
RestoreSSTFiles tries to do something prepare work, such as set speed limit, and restore the files.
func (*SnapClient) RestoreSystemSchemas ¶
RestoreSystemSchemas restores the system schema(i.e. the `mysql` schema). Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.
func (*SnapClient) RestoreTables ¶
func (rc *SnapClient) RestoreTables( ctx context.Context, placementRuleManager PlacementRuleManager, createdTables []*CreatedTable, allFiles []*backuppb.File, checkpointSetWithTableID map[int64]map[string]struct{}, splitSizeBytes, splitKeyCount uint64, splitOnTable bool, updateCh glue.Progress, ) error
func (*SnapClient) SetBatchDdlSize ¶
func (rc *SnapClient) SetBatchDdlSize(batchDdlsize uint)
func (*SnapClient) SetConcurrencyPerStore ¶
func (rc *SnapClient) SetConcurrencyPerStore(c uint)
SetConcurrencyPerStore sets the concurrency of download files for each store.
func (*SnapClient) SetCrypter ¶
func (rc *SnapClient) SetCrypter(crypter *backuppb.CipherInfo)
func (*SnapClient) SetPlacementPolicyMode ¶
func (rc *SnapClient) SetPlacementPolicyMode(withPlacementPolicy string)
SetPlacementPolicyMode to policy mode.
func (*SnapClient) SetPolicyMap ¶
func (rc *SnapClient) SetPolicyMap(p *sync.Map)
SetPolicyMap set policyMap.
func (*SnapClient) SetRateLimit ¶
func (rc *SnapClient) SetRateLimit(rateLimit uint64)
func (*SnapClient) SetRewriteMode ¶
func (rc *SnapClient) SetRewriteMode(ctx context.Context)
TODO: remove this check and return RewriteModeKeyspace
func (*SnapClient) SetWithSysTable ¶
func (rc *SnapClient) SetWithSysTable(withSysTable bool)
func (*SnapClient) SplitPoints ¶
func (rc *SnapClient) SplitPoints( ctx context.Context, sortedSplitKeys [][]byte, updateCh glue.Progress, isRawKv bool, ) error
SplitRanges implements TiKVRestorer. It splits region by data range after rewrite.
func (*SnapClient) WaitForFilesRestored ¶
func (*SnapClient) WaitForFinishCheckpoint ¶
func (rc *SnapClient) WaitForFinishCheckpoint(ctx context.Context, flush bool)
type SnapFileImporter ¶
type SnapFileImporter struct {
// contains filtered or unexported fields
}
func NewSnapFileImporter ¶
func NewSnapFileImporter( ctx context.Context, metaClient split.SplitClient, importClient importclient.ImporterClient, backend *backuppb.StorageBackend, isRawKvMode bool, isTxnKvMode bool, tikvStores []*metapb.Store, rewriteMode RewriteMode, concurrencyPerStore uint, ) (*SnapFileImporter, error)
func (*SnapFileImporter) Close ¶
func (importer *SnapFileImporter) Close() error
func (*SnapFileImporter) ImportSSTFiles ¶
func (importer *SnapFileImporter) ImportSSTFiles( ctx context.Context, filesGroup []TableIDWithFiles, cipher *backuppb.CipherInfo, apiVersion kvrpcpb.APIVersion, ) error
ImportSSTFiles tries to import a file. Assert 1: All rewrite rules must contain raw key prefix. Assert 2: len(filesGroup[any].Files) > 0.
func (*SnapFileImporter) SetDownloadSpeedLimit ¶
func (importer *SnapFileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error
func (*SnapFileImporter) SetRawRange ¶
func (importer *SnapFileImporter) SetRawRange(startKey, endKey []byte) error
SetRawRange sets the range to be restored in raw kv mode.
func (*SnapFileImporter) ShouldBlock ¶
func (importer *SnapFileImporter) ShouldBlock() bool
func (*SnapFileImporter) WaitUntilUnblock ¶
func (importer *SnapFileImporter) WaitUntilUnblock()
type TableIDWithFiles ¶
type TableIDWithFiles struct { TableID int64 Files []*backuppb.File // RewriteRules is the rewrite rules for the specify table. // because these rules belongs to the *one table*. // we can hold them here. RewriteRules *restoreutils.RewriteRules }