restore

package
v4.0.0-beta.2.0...-c7f9382 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 13, 2021 License: Apache-2.0 Imports: 71 Imported by: 0

Documentation

Index

Constants

View Source 
const (

	// OnlineBytesLimitation/OnlineKeysLimitation is the statistics of
	// Bytes/Keys used per region from pdWriteFlow/pdReadFlow
	// this determines whether the cluster has some region that have other loads
	// and might influence the import task in the future.
	OnlineBytesLimitation = 10 * units.MiB
	OnlineKeysLimitation  = 5000
)
View Source 
const (
	FullLevelCompact = -1
	Level1Compact    = 1
)
View Source 
const (

	// CreateTableMetadataTable stores the per-table sub jobs information used by TiDB Lightning
	CreateTableMetadataTable = `` /* 647-byte string literal not displayed */

	// CreateTaskMetaTable stores the pre-lightning metadata used by TiDB Lightning
	CreateTaskMetaTable = `` /* 317-byte string literal not displayed */

)

Variables

View Source 
var DeliverPauser = common.NewPauser()

DeliverPauser is a shared pauser to pause progress to (*chunkRestore).encodeLoop

Functions

func AlterAutoIncrement 

func AlterAutoIncrement(ctx context.Context, g glue.SQLExecutor, tableName string, incr int64) error

AlterAutoIncrement rebase the table auto increment id

NOTE: since tidb can make sure the auto id is always be rebase even if the `incr` value is smaller the the auto incremanet base in tidb side, we needn't fetch currently auto increment value here. See: https://github.com/pingcap/tidb/blob/64698ef9a3358bfd0fdc323996bb7928a56cadca/ddl/ddl_api.go#L2528-L2533

func AlterAutoRandom 

func AlterAutoRandom(ctx context.Context, g glue.SQLExecutor, tableName string, randomBase int64) error

func DBFromConfig 

func DBFromConfig(dsn config.DBStore) (*sql.DB, error)

func InitSchema 

func InitSchema(ctx context.Context, g glue.Glue, database string, tablesSchema map[string]string) error

func LoadSchemaInfo 

func LoadSchemaInfo(
	ctx context.Context,
	schemas []*mydump.MDDatabaseMeta,
	getTables func(context.Context, string) ([]*model.TableInfo, error),
) (map[string]*checkpoints.TidbDBInfo, error)

func ObtainGCLifeTime 

func ObtainGCLifeTime(ctx context.Context, db *sql.DB) (string, error)

func ObtainImportantVariables 

func ObtainImportantVariables(ctx context.Context, g glue.SQLExecutor) map[string]string

func ObtainNewCollationEnabled 

func ObtainNewCollationEnabled(ctx context.Context, g glue.SQLExecutor) bool

func UpdateGCLifeTime 

func UpdateGCLifeTime(ctx context.Context, db *sql.DB, gcLifeTime string) error

Types

type CheckType 

type CheckType string
const (
	Critical CheckType = "critical"
	Warn     CheckType = "performance"
)

type ChecksumManager 

type ChecksumManager interface {
	Checksum(ctx context.Context, tableInfo *checkpoints.TidbTableInfo) (*RemoteChecksum, error)
}

type Controller 

type Controller struct {
	// contains filtered or unexported fields
}

func NewRestoreController 

func NewRestoreController(
	ctx context.Context,
	dbMetas []*mydump.MDDatabaseMeta,
	cfg *config.Config,
	s storage.ExternalStorage,
	g glue.Glue,
) (*Controller, error)

func NewRestoreControllerWithPauser 

func NewRestoreControllerWithPauser(
	ctx context.Context,
	dbMetas []*mydump.MDDatabaseMeta,
	cfg *config.Config,
	s storage.ExternalStorage,
	pauser *common.Pauser,
	g glue.Glue,
) (*Controller, error)

func (*Controller) CheckpointIsValid 

func (rc *Controller) CheckpointIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta) ([]string, bool, error)

CheckpointIsValid checks whether we can start this import with this checkpoint.

func (*Controller) Close 

func (rc *Controller) Close()

func (*Controller) ClusterIsAvailable 

func (rc *Controller) ClusterIsAvailable(ctx context.Context) error

ClusterIsAvailable check cluster is available to import data. this test can be skipped.

func (*Controller) ClusterResource 

func (rc *Controller) ClusterResource(ctx context.Context, localSource int64) error

ClusterResource check cluster has enough resource to import data. this test can by skipped.

func (*Controller) DataCheck 

func (rc *Controller) DataCheck(ctx context.Context) error

DataCheck checks the data schema which needs #rc.restoreSchema finished.

func (*Controller) EstimateSourceData 

func (rc *Controller) EstimateSourceData(ctx context.Context) (int64, error)

func (*Controller) HasLargeCSV 

func (rc *Controller) HasLargeCSV(dbMetas []*mydump.MDDatabaseMeta) error

HasLargeCSV checks whether input csvs is fit for Lightning import. If strictFormat is false, and csv file is large. Lightning will have performance issue. this test cannot be skipped.

func (*Controller) LocalResource 

func (rc *Controller) LocalResource(ctx context.Context, sourceSize int64) error

LocalResource checks the local node has enough resources for this import when local backend enabled;

func (*Controller) Run 

func (rc *Controller) Run(ctx context.Context) error

func (*Controller) SampleDataFromTable 

func (rc *Controller) SampleDataFromTable(ctx context.Context, dbName string, tableMeta *mydump.MDTableMeta, tableInfo *model.TableInfo) error

func (*Controller) SchemaIsValid 

func (rc *Controller) SchemaIsValid(ctx context.Context, tableInfo *mydump.MDTableMeta) ([]string, error)

SchemaIsValid checks the import file and cluster schema is match.

func (*Controller) StoragePermission 

func (rc *Controller) StoragePermission(ctx context.Context) error

StoragePermission checks whether Lightning has enough permission to storage. this test cannot be skipped.

type RemoteChecksum 

type RemoteChecksum struct {
	Schema     string
	Table      string
	Checksum   uint64
	TotalKVs   uint64
	TotalBytes uint64
}

RemoteChecksum represents a checksum result got from tidb.

func DoChecksum 

func DoChecksum(ctx context.Context, table *checkpoints.TidbTableInfo) (*RemoteChecksum, error)

DoChecksum do checksum for tables. table should be in <db>.<table>, format. e.g. foo.bar

type SimpleTemplate 

type SimpleTemplate struct {
	// contains filtered or unexported fields
}

func (*SimpleTemplate) Collect 

func (c *SimpleTemplate) Collect(t CheckType, passed bool, msg string)

func (*SimpleTemplate) FailedCount 

func (c *SimpleTemplate) FailedCount(t CheckType) int

func (*SimpleTemplate) Output 

func (c *SimpleTemplate) Output() string

func (*SimpleTemplate) Success 

func (c *SimpleTemplate) Success() bool

type TableRestore 

type TableRestore struct {
	// contains filtered or unexported fields
}

func NewTableRestore 

func NewTableRestore(
	tableName string,
	tableMeta *mydump.MDTableMeta,
	dbInfo *checkpoints.TidbDBInfo,
	tableInfo *checkpoints.TidbTableInfo,
	cp *checkpoints.TableCheckpoint,
	ignoreColumns []string,
) (*TableRestore, error)

func (*TableRestore) Close 

func (tr *TableRestore) Close()

func (*TableRestore) RebaseChunkRowIDs 

func (t *TableRestore) RebaseChunkRowIDs(cp *checkpoints.TableCheckpoint, rowIDBase int64)

type Template 

type Template interface {
	// Collect mainly collect performance related checks' results and critical level checks' results.
	// If the performance is not as expect. It will output a warn to user and it won't break the whole import task.
	// if one of critical check not passed. it will stop import task.
	Collect(t CheckType, passed bool, msg string)

	// Success represents the whole check has passed or not.
	Success() bool

	// FailedCount represents (the warn check failed count, the critical check failed count)
	FailedCount(t CheckType) int

	// Output print all checks results.
	Output() string
}

func NewSimpleTemplate 

func NewSimpleTemplate() Template

type TiDBManager 

type TiDBManager struct {
	// contains filtered or unexported fields
}

func NewTiDBManager 

func NewTiDBManager(dsn config.DBStore, tls *common.TLS) (*TiDBManager, error)

func NewTiDBManagerWithDB 

func NewTiDBManagerWithDB(db *sql.DB, sqlMode mysql.SQLMode) *TiDBManager

NewTiDBManagerWithDB creates a new TiDB manager with an existing database connection.

func (*TiDBManager) Close 

func (timgr *TiDBManager) Close()

func (*TiDBManager) DropTable 

func (timgr *TiDBManager) DropTable(ctx context.Context, tableName string) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.