snapclient

package
v1.1.0-beta.0...-3047328 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 11, 2024 License: Apache-2.0, Apache-2.0 Imports: 68 Imported by: 0

Documentation

Index

Constants

View Source 
const MergedRangeCountThreshold = 1536

If there are many tables with only a few rows, the number of merged SSTs will be too large. So set a threshold to avoid it.

Variables

This section is empty.

Functions

func CheckSysTableCompatibility 

func CheckSysTableCompatibility(dom *domain.Domain, tables []*metautil.Table) error

func MarshalLogObjectForFiles 

func MarshalLogObjectForFiles(files []TableIDWithFiles, encoder zapcore.ObjectEncoder) error

MarshalLogObjectForFiles is an internal util function to zap something having `Files` field.

func SortAndValidateFileRanges 

func SortAndValidateFileRanges(
	createdTables []*CreatedTable,
	allFiles []*backuppb.File,
	checkpointSetWithTableID map[int64]map[string]struct{},
	splitSizeBytes, splitKeyCount uint64,
	splitOnTable bool,
	updateCh glue.Progress,
) ([][]byte, [][]TableIDWithFiles, error)

SortAndValidateFileRanges sort, merge and validate files by tables and yields tables with range.

Types

type CreatedTable 

type CreatedTable struct {
	RewriteRule *restoreutils.RewriteRules
	Table       *model.TableInfo
	OldTable    *metautil.Table
}

CreatedTable is a table created on restore process, but not yet filled with data.

type KvMode 

type KvMode int
const (
	TiDB KvMode = iota
	Raw
	Txn
)

type PhysicalTable 

type PhysicalTable struct {
	NewPhysicalID int64
	OldPhysicalID int64
	RewriteRules  *restoreutils.RewriteRules
}

type PlacementRuleManager 

type PlacementRuleManager interface {
	SetPlacementRule(ctx context.Context, tables []*CreatedTable) error
	ResetPlacementRules(ctx context.Context) error
}

PlacementRuleManager manages to set the placement rule of tables to label constraint key `exclusive`, and unset the rule.

func NewPlacementRuleManager 

func NewPlacementRuleManager(ctx context.Context, pdClient pd.Client, pdHTTPCli pdhttp.Client, tlsConf *tls.Config, isOnline bool) (PlacementRuleManager, error)

NewPlacementRuleManager sets and unset placement rules for online restore.

type RewriteMode 

type RewriteMode int

RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules. 

const (
	// RewriteModeLegacy means no rewrite rule is applied.
	RewriteModeLegacy RewriteMode = iota

	// RewriteModeKeyspace means the rewrite rule could be applied to keyspace.
	RewriteModeKeyspace
)

type SnapClient 

type SnapClient struct {
	// contains filtered or unexported fields
}

func NewRestoreClient 

func NewRestoreClient(
	pdClient pd.Client,
	pdHTTPCli pdhttp.Client,
	tlsConf *tls.Config,
	keepaliveConf keepalive.ClientParameters,
) *SnapClient

NewRestoreClient returns a new RestoreClient.

func (*SnapClient) AllocTableIDs 

func (rc *SnapClient) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error

AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in the download stage.

func (*SnapClient) CheckTargetClusterFresh 

func (rc *SnapClient) CheckTargetClusterFresh(ctx context.Context) error

CheckTargetClusterFresh check whether the target cluster is fresh or not if there's no user dbs or tables, we take it as a fresh cluster, although user may have created some users or made other changes.

func (*SnapClient) Close 

func (rc *SnapClient) Close()

Close a client.

func (*SnapClient) CreateDatabases 

func (rc *SnapClient) CreateDatabases(ctx context.Context, dbs []*metautil.Database) error

CreateDatabases creates databases. If the client has the db pool, it would create it.

func (*SnapClient) CreatePolicies 

func (rc *SnapClient) CreatePolicies(ctx context.Context, policyMap *sync.Map) error

CreatePolicies creates all policies in full restore.

func (*SnapClient) CreateTables 

func (rc *SnapClient) CreateTables(
	ctx context.Context,
	tables []*metautil.Table,
	newTS uint64,
) ([]*CreatedTable, error)

CreateTables create tables, and generate their information. this function will use workers as the same number of sessionPool, leave sessionPool nil to send DDLs sequential.

func (*SnapClient) EnableSkipCreateSQL 

func (rc *SnapClient) EnableSkipCreateSQL()

EnableSkipCreateSQL sets switch of skip create schema and tables.

func (*SnapClient) ExecDDLs 

func (rc *SnapClient) ExecDDLs(ctx context.Context, ddlJobs []*model.Job) error

ExecDDLs executes the queries of the ddl jobs.

func (*SnapClient) GetBatchDdlSize 

func (rc *SnapClient) GetBatchDdlSize() uint

func (*SnapClient) GetClusterID 

func (rc *SnapClient) GetClusterID(ctx context.Context) uint64

GetClusterID gets the cluster id from down-stream cluster.

func (*SnapClient) GetDDLJobs 

func (rc *SnapClient) GetDDLJobs() []*model.Job

GetDDLJobs returns ddl jobs.

func (*SnapClient) GetDatabases 

func (rc *SnapClient) GetDatabases() []*metautil.Database

GetDatabases returns all databases.

func (*SnapClient) GetDomain 

func (rc *SnapClient) GetDomain() *domain.Domain

func (*SnapClient) GetFilesInRawRange 

func (rc *SnapClient) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error)

GetFilesInRawRange gets all files that are in the given range or intersects with the given range.

func (*SnapClient) GetPlacementPolicies 

func (rc *SnapClient) GetPlacementPolicies() (*sync.Map, error)

GetPlacementPolicies returns policies.

func (*SnapClient) GetRewriteMode 

func (rc *SnapClient) GetRewriteMode() RewriteMode

func (*SnapClient) GetSupportPolicy 

func (rc *SnapClient) GetSupportPolicy() bool

GetSupportPolicy tells whether target tidb support placement policy.

func (*SnapClient) GetTLSConfig 

func (rc *SnapClient) GetTLSConfig() *tls.Config

GetTLSConfig returns the tls config.

func (*SnapClient) GoUpdateMetaAndLoadStats 

func (rc *SnapClient) GoUpdateMetaAndLoadStats(
	ctx context.Context,
	s storage.ExternalStorage,
	inCh <-chan *CreatedTable,
	errCh chan<- error,
	statsConcurrency uint,
	loadStats bool,
) chan *CreatedTable

func (*SnapClient) GoValidateChecksum 

func (rc *SnapClient) GoValidateChecksum(
	ctx context.Context,
	inCh <-chan *CreatedTable,
	kvClient kv.Client,
	errCh chan<- error,
	updateCh glue.Progress,
	concurrency uint,
) chan *CreatedTable

GoValidateChecksum forks a goroutine to validate checksum after restore. it returns a channel fires a struct{} when all things get done.

func (*SnapClient) GoWaitTiFlashReady 

func (rc *SnapClient) GoWaitTiFlashReady(
	ctx context.Context,
	inCh <-chan *CreatedTable,
	updateCh glue.Progress,
	errCh chan<- error,
) chan *CreatedTable

func (*SnapClient) HasBackedUpSysDB 

func (rc *SnapClient) HasBackedUpSysDB() bool

HasBackedUpSysDB whether we have backed up system tables br backs system tables up since 5.1.0

func (*SnapClient) Init 

func (rc *SnapClient) Init(g glue.Glue, store kv.Storage) error

Init create db connection and domain for storage.

func (*SnapClient) InitBackupMeta 

func (rc *SnapClient) InitBackupMeta(
	c context.Context,
	backupMeta *backuppb.BackupMeta,
	backend *backuppb.StorageBackend,
	reader *metautil.MetaReader,
	loadStats bool) error

InitBackupMeta loads schemas from BackupMeta to initialize RestoreClient.

func (*SnapClient) InitCheckpoint 

func (rc *SnapClient) InitCheckpoint(
	ctx context.Context,
	g glue.Glue, store kv.Storage,
	config *pdutil.ClusterConfig,
	checkpointFirstRun bool,
) (checkpointSetWithTableID map[int64]map[string]struct{}, checkpointClusterConfig *pdutil.ClusterConfig, err error)

InitCheckpoint initialize the checkpoint status for the cluster. If the cluster is restored for the first time, it will initialize the checkpoint metadata. Otherwrise, it will load checkpoint metadata and checkpoint ranges/checksum from the external storage.

func (*SnapClient) InitFullClusterRestore 

func (rc *SnapClient) InitFullClusterRestore(explicitFilter bool)

InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed

func (*SnapClient) IsFull 

func (rc *SnapClient) IsFull() bool

IsFull returns whether this backup is full.

func (*SnapClient) IsFullClusterRestore 

func (rc *SnapClient) IsFullClusterRestore() bool

func (*SnapClient) IsIncremental 

func (rc *SnapClient) IsIncremental() bool

IsIncremental returns whether this backup is incremental.

func (*SnapClient) IsRawKvMode 

func (rc *SnapClient) IsRawKvMode() bool

IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.

func (*SnapClient) IsSkipCreateSQL 

func (rc *SnapClient) IsSkipCreateSQL() bool

IsSkipCreateSQL returns whether we need skip create schema and tables in restore.

func (*SnapClient) NeedCheckFreshCluster 

func (rc *SnapClient) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool

NeedCheckFreshCluster is every time. except restore from a checkpoint or user has not set filter argument.

func (*SnapClient) ResetSpeedLimit 

func (rc *SnapClient) ResetSpeedLimit(ctx context.Context) error

func (*SnapClient) ResetTS 

func (rc *SnapClient) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error

ResetTS resets the timestamp of PD to a bigger value.

func (*SnapClient) RestoreRaw 

func (rc *SnapClient) RestoreRaw(
	ctx context.Context, startKey []byte, endKey []byte, files []*backuppb.File, updateCh glue.Progress,
) error

RestoreRaw tries to restore raw keys in the specified range.

func (*SnapClient) RestoreSSTFiles 

func (rc *SnapClient) RestoreSSTFiles(
	ctx context.Context,
	tableIDWithFilesGroup [][]TableIDWithFiles,
	updateCh glue.Progress,
) (retErr error)

RestoreSSTFiles tries to do something prepare work, such as set speed limit, and restore the files.

func (*SnapClient) RestoreSystemSchemas 

func (rc *SnapClient) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (rerr error)

RestoreSystemSchemas restores the system schema(i.e. the `mysql` schema). Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.

func (*SnapClient) RestoreTables 

func (rc *SnapClient) RestoreTables(
	ctx context.Context,
	placementRuleManager PlacementRuleManager,
	createdTables []*CreatedTable,
	allFiles []*backuppb.File,
	checkpointSetWithTableID map[int64]map[string]struct{},
	splitSizeBytes, splitKeyCount uint64,
	splitOnTable bool,
	updateCh glue.Progress,
) error

func (*SnapClient) SetBatchDdlSize 

func (rc *SnapClient) SetBatchDdlSize(batchDdlsize uint)

func (*SnapClient) SetConcurrencyPerStore 

func (rc *SnapClient) SetConcurrencyPerStore(c uint)

SetConcurrencyPerStore sets the concurrency of download files for each store.

func (*SnapClient) SetCrypter 

func (rc *SnapClient) SetCrypter(crypter *backuppb.CipherInfo)

func (*SnapClient) SetPlacementPolicyMode 

func (rc *SnapClient) SetPlacementPolicyMode(withPlacementPolicy string)

SetPlacementPolicyMode to policy mode.

func (*SnapClient) SetPolicyMap 

func (rc *SnapClient) SetPolicyMap(p *sync.Map)

SetPolicyMap set policyMap.

func (*SnapClient) SetRateLimit 

func (rc *SnapClient) SetRateLimit(rateLimit uint64)

func (*SnapClient) SetRewriteMode 

func (rc *SnapClient) SetRewriteMode(ctx context.Context)

TODO: remove this check and return RewriteModeKeyspace

func (*SnapClient) SetWithSysTable 

func (rc *SnapClient) SetWithSysTable(withSysTable bool)

func (*SnapClient) SplitPoints 

func (rc *SnapClient) SplitPoints(
	ctx context.Context,
	sortedSplitKeys [][]byte,
	updateCh glue.Progress,
	isRawKv bool,
) error

SplitRanges implements TiKVRestorer. It splits region by data range after rewrite.

func (*SnapClient) WaitForFilesRestored 

func (rc *SnapClient) WaitForFilesRestored(ctx context.Context, files []*backuppb.File, updateCh glue.Progress) error

func (*SnapClient) WaitForFinishCheckpoint 

func (rc *SnapClient) WaitForFinishCheckpoint(ctx context.Context, flush bool)

type SnapFileImporter 

type SnapFileImporter struct {
	// contains filtered or unexported fields
}

func NewSnapFileImporter 

func NewSnapFileImporter(
	ctx context.Context,
	metaClient split.SplitClient,
	importClient importclient.ImporterClient,
	backend *backuppb.StorageBackend,
	isRawKvMode bool,
	isTxnKvMode bool,
	tikvStores []*metapb.Store,
	rewriteMode RewriteMode,
	concurrencyPerStore uint,
) (*SnapFileImporter, error)

func (*SnapFileImporter) Close 

func (importer *SnapFileImporter) Close() error

func (*SnapFileImporter) ImportSSTFiles 

func (importer *SnapFileImporter) ImportSSTFiles(
	ctx context.Context,
	filesGroup []TableIDWithFiles,
	cipher *backuppb.CipherInfo,
	apiVersion kvrpcpb.APIVersion,
) error

ImportSSTFiles tries to import a file. Assert 1: All rewrite rules must contain raw key prefix. Assert 2: len(filesGroup[any].Files) > 0.

func (*SnapFileImporter) SetDownloadSpeedLimit 

func (importer *SnapFileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error

func (*SnapFileImporter) SetRawRange 

func (importer *SnapFileImporter) SetRawRange(startKey, endKey []byte) error

SetRawRange sets the range to be restored in raw kv mode.

func (*SnapFileImporter) ShouldBlock 

func (importer *SnapFileImporter) ShouldBlock() bool

func (*SnapFileImporter) WaitUntilUnblock 

func (importer *SnapFileImporter) WaitUntilUnblock()

type TableIDWithFiles 

type TableIDWithFiles struct {
	TableID int64

	Files []*backuppb.File
	// RewriteRules is the rewrite rules for the specify table.
	// because these rules belongs to the *one table*.
	// we can hold them here.
	RewriteRules *restoreutils.RewriteRules
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.