snapclient

package
v1.1.0-beta.0...-68ac9ec Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 10, 2024 License: Apache-2.0, Apache-2.0 Imports: 67 Imported by: 0

Documentation

Index

Constants

View Source
const MergedRangeCountThreshold = 1536

If there are many tables with only a few rows, the number of merged SSTs will be too large. So set a threshold to avoid it.

Variables

This section is empty.

Functions

func CheckSysTableCompatibility

func CheckSysTableCompatibility(dom *domain.Domain, tables []*metautil.Table) error

func SetSpeedLimitFn

func SetSpeedLimitFn(ctx context.Context, stores []*metapb.Store, pool *tidbutil.WorkerPool) func(*SnapFileImporter, uint64) error

func SortAndValidateFileRanges

func SortAndValidateFileRanges(
	createdTables []*CreatedTable,
	allFiles []*backuppb.File,
	checkpointSetWithTableID map[int64]map[string]struct{},
	splitSizeBytes, splitKeyCount uint64,
	splitOnTable bool,
	onProgress func(int64),
) ([][]byte, []restore.BatchBackupFileSet, error)

SortAndValidateFileRanges sort, merge and validate files by tables and yields tables with range.

Types

type CreatedTable

type CreatedTable struct {
	RewriteRule *restoreutils.RewriteRules
	Table       *model.TableInfo
	OldTable    *metautil.Table
}

CreatedTable is a table created on restore process, but not yet filled with data.

type KvMode

type KvMode int
const (
	TiDBFull KvMode = iota
	Raw
	Txn
	TiDBCompcated
)

type PhysicalTable

type PhysicalTable struct {
	NewPhysicalID int64
	OldPhysicalID int64
	RewriteRules  *restoreutils.RewriteRules
}

type PlacementRuleManager

type PlacementRuleManager interface {
	SetPlacementRule(ctx context.Context, tables []*CreatedTable) error
	ResetPlacementRules(ctx context.Context) error
}

PlacementRuleManager manages to set the placement rule of tables to label constraint key `exclusive`, and unset the rule.

func NewPlacementRuleManager

func NewPlacementRuleManager(ctx context.Context, pdClient pd.Client, pdHTTPCli pdhttp.Client, tlsConf *tls.Config, isOnline bool) (PlacementRuleManager, error)

NewPlacementRuleManager sets and unset placement rules for online restore.

type RewriteMode

type RewriteMode int

RewriteMode is a mode flag that tells the TiKV how to handle the rewrite rules.

const (
	// RewriteModeLegacy means no rewrite rule is applied.
	RewriteModeLegacy RewriteMode = iota

	// RewriteModeKeyspace means the rewrite rule could be applied to keyspace.
	RewriteModeKeyspace
)

type SnapClient

type SnapClient struct {
	// contains filtered or unexported fields
}

func NewRestoreClient

func NewRestoreClient(
	pdClient pd.Client,
	pdHTTPCli pdhttp.Client,
	tlsConf *tls.Config,
	keepaliveConf keepalive.ClientParameters,
) *SnapClient

NewRestoreClient returns a new RestoreClient.

func (*SnapClient) AllocTableIDs

func (rc *SnapClient) AllocTableIDs(ctx context.Context, tables []*metautil.Table) error

AllocTableIDs would pre-allocate the table's origin ID if exists, so that the TiKV doesn't need to rewrite the key in the download stage.

func (*SnapClient) CheckTargetClusterFresh

func (rc *SnapClient) CheckTargetClusterFresh(ctx context.Context) error

CheckTargetClusterFresh check whether the target cluster is fresh or not if there's no user dbs or tables, we take it as a fresh cluster, although user may have created some users or made other changes.

func (*SnapClient) Close

func (rc *SnapClient) Close()

Close a client.

func (*SnapClient) CreateDatabases

func (rc *SnapClient) CreateDatabases(ctx context.Context, dbs []*metautil.Database) error

CreateDatabases creates databases. If the client has the db pool, it would create it.

func (*SnapClient) CreatePolicies

func (rc *SnapClient) CreatePolicies(ctx context.Context, policyMap *sync.Map) error

CreatePolicies creates all policies in full restore.

func (*SnapClient) CreateTables

func (rc *SnapClient) CreateTables(
	ctx context.Context,
	tables []*metautil.Table,
	newTS uint64,
) ([]*CreatedTable, error)

CreateTables create tables, and generate their information. this function will use workers as the same number of sessionPool, leave sessionPool nil to send DDLs sequential.

func (*SnapClient) EnableSkipCreateSQL

func (rc *SnapClient) EnableSkipCreateSQL()

EnableSkipCreateSQL sets switch of skip create schema and tables.

func (*SnapClient) ExecDDLs

func (rc *SnapClient) ExecDDLs(ctx context.Context, ddlJobs []*model.Job) error

ExecDDLs executes the queries of the ddl jobs.

func (*SnapClient) GetBatchDdlSize

func (rc *SnapClient) GetBatchDdlSize() uint

func (*SnapClient) GetClusterID

func (rc *SnapClient) GetClusterID(ctx context.Context) uint64

GetClusterID gets the cluster id from down-stream cluster.

func (*SnapClient) GetDDLJobs

func (rc *SnapClient) GetDDLJobs() []*model.Job

GetDDLJobs returns ddl jobs.

func (*SnapClient) GetDatabases

func (rc *SnapClient) GetDatabases() []*metautil.Database

GetDatabases returns all databases.

func (*SnapClient) GetDomain

func (rc *SnapClient) GetDomain() *domain.Domain

func (*SnapClient) GetFilesInRawRange

func (rc *SnapClient) GetFilesInRawRange(startKey []byte, endKey []byte, cf string) ([]*backuppb.File, error)

GetFilesInRawRange gets all files that are in the given range or intersects with the given range.

func (*SnapClient) GetPlacementPolicies

func (rc *SnapClient) GetPlacementPolicies() (*sync.Map, error)

GetPlacementPolicies returns policies.

func (*SnapClient) GetRestorer

func (rc *SnapClient) GetRestorer() restore.SstRestorer

func (*SnapClient) GetRewriteMode

func (rc *SnapClient) GetRewriteMode() RewriteMode

func (*SnapClient) GetSupportPolicy

func (rc *SnapClient) GetSupportPolicy() bool

GetSupportPolicy tells whether target tidb support placement policy.

func (*SnapClient) GetTLSConfig

func (rc *SnapClient) GetTLSConfig() *tls.Config

GetTLSConfig returns the tls config.

func (*SnapClient) GoUpdateMetaAndLoadStats

func (rc *SnapClient) GoUpdateMetaAndLoadStats(
	ctx context.Context,
	s storage.ExternalStorage,
	inCh <-chan *CreatedTable,
	errCh chan<- error,
	statsConcurrency uint,
	loadStats bool,
) chan *CreatedTable

func (*SnapClient) GoValidateChecksum

func (rc *SnapClient) GoValidateChecksum(
	ctx context.Context,
	inCh <-chan *CreatedTable,
	kvClient kv.Client,
	errCh chan<- error,
	updateCh glue.Progress,
	concurrency uint,
) chan *CreatedTable

GoValidateChecksum forks a goroutine to validate checksum after restore. it returns a channel fires a struct{} when all things get done.

func (*SnapClient) GoWaitTiFlashReady

func (rc *SnapClient) GoWaitTiFlashReady(
	ctx context.Context,
	inCh <-chan *CreatedTable,
	updateCh glue.Progress,
	errCh chan<- error,
) chan *CreatedTable

func (*SnapClient) HasBackedUpSysDB

func (rc *SnapClient) HasBackedUpSysDB() bool

HasBackedUpSysDB whether we have backed up system tables br backs system tables up since 5.1.0

func (*SnapClient) Init

func (rc *SnapClient) Init(g glue.Glue, store kv.Storage) error

Init create db connection and domain for storage.

func (*SnapClient) InitCheckpoint

func (rc *SnapClient) InitCheckpoint(
	ctx context.Context,
	g glue.Glue, store kv.Storage,
	config *pdutil.ClusterConfig,
	checkpointFirstRun bool,
) (checkpointSetWithTableID map[int64]map[string]struct{}, checkpointClusterConfig *pdutil.ClusterConfig, err error)

InitCheckpoint initialize the checkpoint status for the cluster. If the cluster is restored for the first time, it will initialize the checkpoint metadata. Otherwrise, it will load checkpoint metadata and checkpoint ranges/checksum from the external storage.

func (*SnapClient) InitFullClusterRestore

func (rc *SnapClient) InitFullClusterRestore(explicitFilter bool)

InitFullClusterRestore init fullClusterRestore and set SkipGrantTable as needed

func (*SnapClient) IsFull

func (rc *SnapClient) IsFull() bool

IsFull returns whether this backup is full.

func (*SnapClient) IsFullClusterRestore

func (rc *SnapClient) IsFullClusterRestore() bool

func (*SnapClient) IsIncremental

func (rc *SnapClient) IsIncremental() bool

IsIncremental returns whether this backup is incremental.

func (*SnapClient) IsRawKvMode

func (rc *SnapClient) IsRawKvMode() bool

IsRawKvMode checks whether the backup data is in raw kv format, in which case transactional recover is forbidden.

func (*SnapClient) IsSkipCreateSQL

func (rc *SnapClient) IsSkipCreateSQL() bool

IsSkipCreateSQL returns whether we need skip create schema and tables in restore.

func (*SnapClient) LoadSchemaIfNeededAndInitClient

func (rc *SnapClient) LoadSchemaIfNeededAndInitClient(
	c context.Context,
	backupMeta *backuppb.BackupMeta,
	backend *backuppb.StorageBackend,
	reader *metautil.MetaReader,
	loadStats bool,
	RawStartKey []byte,
	RawEndKey []byte,
) error

LoadSchemaIfNeededAndInitClient loads schemas from BackupMeta to initialize RestoreClient.

func (*SnapClient) NeedCheckFreshCluster

func (rc *SnapClient) NeedCheckFreshCluster(ExplicitFilter bool, firstRun bool) bool

NeedCheckFreshCluster is every time. except restore from a checkpoint or user has not set filter argument.

func (*SnapClient) ResetTS

func (rc *SnapClient) ResetTS(ctx context.Context, pdCtrl *pdutil.PdController) error

ResetTS resets the timestamp of PD to a bigger value.

func (*SnapClient) RestoreSSTFiles

func (rc *SnapClient) RestoreSSTFiles(
	ctx context.Context,
	tableIDWithFilesGroup []restore.BatchBackupFileSet,
	onProgress func(int64),
) (retErr error)

RestoreSSTFiles tries to do something prepare work, such as set speed limit, and restore the files.

func (*SnapClient) RestoreSystemSchemas

func (rc *SnapClient) RestoreSystemSchemas(ctx context.Context, f filter.Filter) (rerr error)

RestoreSystemSchemas restores the system schema(i.e. the `mysql` schema). Detail see https://github.com/pingcap/br/issues/679#issuecomment-762592254.

func (*SnapClient) RestoreTables

func (rc *SnapClient) RestoreTables(
	ctx context.Context,
	placementRuleManager PlacementRuleManager,
	createdTables []*CreatedTable,
	allFiles []*backuppb.File,
	checkpointSetWithTableID map[int64]map[string]struct{},
	splitSizeBytes, splitKeyCount uint64,
	splitOnTable bool,
	onProgress func(int64),
) error

func (*SnapClient) SetBatchDdlSize

func (rc *SnapClient) SetBatchDdlSize(batchDdlsize uint)

func (*SnapClient) SetConcurrencyPerStore

func (rc *SnapClient) SetConcurrencyPerStore(c uint)

SetConcurrencyPerStore sets the concurrency of download files for each store.

func (*SnapClient) SetCrypter

func (rc *SnapClient) SetCrypter(crypter *backuppb.CipherInfo)

func (*SnapClient) SetPlacementPolicyMode

func (rc *SnapClient) SetPlacementPolicyMode(withPlacementPolicy string)

SetPlacementPolicyMode to policy mode.

func (*SnapClient) SetPolicyMap

func (rc *SnapClient) SetPolicyMap(p *sync.Map)

SetPolicyMap set policyMap.

func (*SnapClient) SetRateLimit

func (rc *SnapClient) SetRateLimit(rateLimit uint64)

func (*SnapClient) SetRewriteMode

func (rc *SnapClient) SetRewriteMode(ctx context.Context)

TODO: remove this check and return RewriteModeKeyspace

func (*SnapClient) SetWithSysTable

func (rc *SnapClient) SetWithSysTable(withSysTable bool)

func (*SnapClient) SplitPoints

func (rc *SnapClient) SplitPoints(
	ctx context.Context,
	sortedSplitKeys [][]byte,
	onProgress func(int64),
	isRawKv bool,
) error

SplitRanges implements TiKVRestorer. It splits region by data range after rewrite.

func (*SnapClient) WaitForFinishCheckpoint

func (rc *SnapClient) WaitForFinishCheckpoint(ctx context.Context, flush bool)

type SnapFileImporter

type SnapFileImporter struct {
	// contains filtered or unexported fields
}

func NewSnapFileImporter

func NewSnapFileImporter(
	ctx context.Context,
	apiVersion kvrpcpb.APIVersion,
	kvMode KvMode,
	options *SnapFileImporterOptions,
) (*SnapFileImporter, error)

func (*SnapFileImporter) CheckMultiIngestSupport

func (importer *SnapFileImporter) CheckMultiIngestSupport(ctx context.Context, tikvStores []*metapb.Store) error

CheckMultiIngestSupport checks whether all stores support multi-ingest

func (*SnapFileImporter) Close

func (importer *SnapFileImporter) Close() error

func (*SnapFileImporter) Import

func (importer *SnapFileImporter) Import(
	ctx context.Context,
	backupFileSets ...restore.BackupFileSet,
) error

Import tries to import a file. Assert 1: All rewrite rules must contain raw key prefix. Assert 2: len(filesGroup[any].Files) > 0.

func (*SnapFileImporter) PauseForBackpressure

func (importer *SnapFileImporter) PauseForBackpressure()

func (*SnapFileImporter) SetDownloadSpeedLimit

func (importer *SnapFileImporter) SetDownloadSpeedLimit(ctx context.Context, storeID, rateLimit uint64) error

func (*SnapFileImporter) SetRawRange

func (importer *SnapFileImporter) SetRawRange(startKey, endKey []byte) error

SetRawRange sets the range to be restored in raw kv mode.

func (*SnapFileImporter) ShouldBlock

func (importer *SnapFileImporter) ShouldBlock() bool

type SnapFileImporterOptions

type SnapFileImporterOptions struct {
	// contains filtered or unexported fields
}

func NewSnapFileImporterOptions

func NewSnapFileImporterOptions(
	cipher *backuppb.CipherInfo,
	metaClient split.SplitClient,
	importClient importclient.ImporterClient,
	backend *backuppb.StorageBackend,
	rewriteMode RewriteMode,
	tikvStores []*metapb.Store,
	concurrencyPerStore uint,
	createCallbacks []func(*SnapFileImporter) error,
	closeCallbacks []func(*SnapFileImporter) error,
) *SnapFileImporterOptions

func NewSnapFileImporterOptionsForTest

func NewSnapFileImporterOptionsForTest(
	splitClient split.SplitClient,
	importClient importclient.ImporterClient,
	tikvStores []*metapb.Store,
	rewriteMode RewriteMode,
	concurrencyPerStore uint,
) *SnapFileImporterOptions

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL