base

package
v0.0.0-...-474ea0c Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 28, 2023 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	HTTPStatusOK       = 200
	MaxEventsBatchSize = 1000
	ETAUnknown         = math.MinInt64
)

Variables

This section is empty.

Functions

func FileExists 

func FileExists(fileName string) bool

func NewDefaultLogger 

func NewDefaultLogger() *simpleLogger

func PrettifyDurationOutput 

func PrettifyDurationOutput(d time.Duration) string

func StringContainsAll 

func StringContainsAll(s string, substrings ...string) bool

StringContainsAll returns true if `s` contains all non empty given `substrings` The function returns `false` if no non-empty arguments are given.

func TouchFile 

func TouchFile(fileName string) error

func ValidateConnection 

func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error)

Types

type ContextConfig 

type ContextConfig struct {
	Client struct {
		User     string
		Password string
	}
	Osc struct {
		Chunk_Size            int64
		Max_Lag_Millis        int64
		Replication_Lag_Query string
		Max_Load              string
	}
}

type CutOver 

type CutOver int
const (
	CutOverAtomic CutOver = iota
	CutOverTwoStep
)

type LoadMap 

type LoadMap map[string]int64

LoadMap is a mapping of status variable & threshold e.g. [Threads_connected: 100, Threads_running: 50]

func NewLoadMap 

func NewLoadMap() LoadMap

func ParseLoadMap 

func ParseLoadMap(loadList string) (LoadMap, error)

NewLoadMap parses a `--*-load` flag (e.g. `--max-load`), which is in multiple key-value format, such as: 

'Threads_running=100,Threads_connected=500'

func (*LoadMap) Duplicate 

func (this *LoadMap) Duplicate() LoadMap

Duplicate creates a clone of this map

func (*LoadMap) String 

func (this *LoadMap) String() string

String() returns a string representation of this map

type Logger 

type Logger interface {
	Debug(args ...interface{})
	Debugf(format string, args ...interface{})
	Info(args ...interface{})
	Infof(format string, args ...interface{})
	Warning(args ...interface{}) error
	Warningf(format string, args ...interface{}) error
	Error(args ...interface{}) error
	Errorf(format string, args ...interface{}) error
	Errore(err error) error
	Fatal(args ...interface{}) error
	Fatalf(format string, args ...interface{}) error
	Fatale(err error) error
	SetLevel(level log.LogLevel)
	SetPrintStackTrace(printStackTraceFlag bool)
}

type MigrationContext 

type MigrationContext struct {
	Uuid string

	DatabaseName          string
	OriginalTableName     string
	AlterStatement        string
	AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement

	CountTableRows           bool
	ConcurrentCountTableRows bool
	AllowedRunningOnMaster   bool
	AllowedMasterMaster      bool
	SwitchToRowBinlogFormat  bool
	AssumeRBR                bool
	SkipForeignKeyChecks     bool
	SkipStrictMode           bool
	AllowZeroInDate          bool
	NullableUniqueKeyAllowed bool
	ApproveRenamedColumns    bool
	SkipRenamedColumns       bool
	IsTungsten               bool
	DiscardForeignKeys       bool
	AliyunRDS                bool
	GoogleCloudPlatform      bool
	AzureMySQL               bool
	AttemptInstantDDL        bool

	ConfigFile        string
	CliUser           string
	CliPassword       string
	UseTLS            bool
	TLSAllowInsecure  bool
	TLSCACertificate  string
	TLSCertificate    string
	TLSKey            string
	CliMasterUser     string
	CliMasterPassword string

	HeartbeatIntervalMilliseconds int64

	ChunkSize int64

	MaxLagMillisecondsThrottleThreshold int64

	ThrottleFlagFile           string
	ThrottleAdditionalFlagFile string

	IgnoreHTTPErrors        bool
	ThrottleCommandedByUser int64
	HibernateUntil          int64

	CriticalLoadIntervalMilliseconds int64
	CriticalLoadHibernateSeconds     int64
	PostponeCutOverFlagFile          string
	CutOverLockTimeoutSeconds        int64
	CutOverExponentialBackoff        bool
	ExponentialBackoffMaxInterval    int64
	ForceNamedCutOverCommand         bool
	ForceNamedPanicCommand           bool
	PanicFlagFile                    string
	HooksPath                        string
	HooksHintMessage                 string
	HooksHintOwner                   string
	HooksHintToken                   string
	HooksStatusIntervalSec           int64

	DropServeSocket bool
	ServeSocketFile string
	ServeTCPPort    int64

	Noop                         bool
	TestOnReplica                bool
	MigrateOnReplica             bool
	TestOnReplicaSkipReplicaStop bool
	OkToDropTable                bool
	InitiallyDropOldTable        bool
	InitiallyDropGhostTable      bool
	TimestampOldTable            bool // Should old table name include a timestamp
	CutOverType                  CutOver
	ReplicaServerId              uint

	Hostname                  string
	AssumeMasterHostname      string
	ApplierTimeZone           string
	TableEngine               string
	RowsEstimate              int64
	RowsDeltaEstimate         int64
	UsedRowsEstimateMethod    RowsEstimateMethod
	HasSuperPrivilege         bool
	OriginalBinlogFormat      string
	OriginalBinlogRowImage    string
	InspectorConnectionConfig *mysql.ConnectionConfig
	InspectorMySQLVersion     string
	ApplierConnectionConfig   *mysql.ConnectionConfig
	ApplierMySQLVersion       string
	StartTime                 time.Time
	RowCopyStartTime          time.Time
	RowCopyEndTime            time.Time
	LockTablesStartTime       time.Time
	RenameTablesStartTime     time.Time
	RenameTablesEndTime       time.Time

	CurrentLag int64

	ThrottleHTTPIntervalMillis int64
	ThrottleHTTPStatusCode     int64
	ThrottleHTTPTimeoutMillis  int64

	TotalRowsCopied       int64
	TotalDMLEventsApplied int64
	DMLBatchSize          int64

	IsPostponingCutOver                    int64
	CountingRowsFlag                       int64
	AllEventsUpToLockProcessedInjectedFlag int64
	CleanupImminentFlag                    int64
	UserCommandedUnpostponeFlag            int64
	CutOverCompleteFlag                    int64
	InCutOverCriticalSectionFlag           int64
	PanicAbort                             chan error

	OriginalTableColumnsOnApplier          *sql.ColumnList
	OriginalTableColumns                   *sql.ColumnList
	OriginalTableVirtualColumns            *sql.ColumnList
	OriginalTableUniqueKeys                [](*sql.UniqueKey)
	OriginalTableAutoIncrement             uint64
	GhostTableColumns                      *sql.ColumnList
	GhostTableVirtualColumns               *sql.ColumnList
	GhostTableUniqueKeys                   [](*sql.UniqueKey)
	UniqueKey                              *sql.UniqueKey
	SharedColumns                          *sql.ColumnList
	ColumnRenameMap                        map[string]string
	DroppedColumnsMap                      map[string]bool
	MappedSharedColumns                    *sql.ColumnList
	MigrationRangeMinValues                *sql.ColumnValues
	MigrationRangeMaxValues                *sql.ColumnValues
	Iteration                              int64
	MigrationIterationRangeMinValues       *sql.ColumnValues
	MigrationIterationRangeMaxValues       *sql.ColumnValues
	ForceTmpTableName                      string
	Where                                  string
	ForceQueryMigrationRangeValuesOnMaster bool

	BinlogSyncerMaxReconnectAttempts int

	Log Logger
	// contains filtered or unexported fields
}

MigrationContext has the general, global state of migration. It is used by all components throughout the migration process.

func NewMigrationContext 

func NewMigrationContext() *MigrationContext

func (*MigrationContext) AddThrottleControlReplicaKey 

func (this *MigrationContext) AddThrottleControlReplicaKey(key mysql.InstanceKey) error

func (*MigrationContext) ApplyCredentials 

func (this *MigrationContext) ApplyCredentials()

ApplyCredentials sorts out the credentials between the config file and the CLI flags

func (*MigrationContext) CancelTableRowsCount 

func (this *MigrationContext) CancelTableRowsCount()

CancelTableRowsCount cancels the CountTableRows query context. It is safe to call function even when IsCountingTableRows is false.

func (*MigrationContext) ElapsedRowCopyTime 

func (this *MigrationContext) ElapsedRowCopyTime() time.Duration

ElapsedRowCopyTime returns time since starting to copy chunks of rows

func (*MigrationContext) ElapsedTime 

func (this *MigrationContext) ElapsedTime() time.Duration

ElapsedTime returns time since very beginning of the process

func (*MigrationContext) GetApplierHostname 

func (this *MigrationContext) GetApplierHostname() string

GetApplierHostname is a safe access method to the applier hostname

func (*MigrationContext) GetChangelogTableName 

func (this *MigrationContext) GetChangelogTableName() string

GetChangelogTableName generates the name of changelog table, based on original table name or a given table name.

func (*MigrationContext) GetControlReplicasLagResult 

func (this *MigrationContext) GetControlReplicasLagResult() mysql.ReplicationLagResult

func (*MigrationContext) GetCriticalLoad 

func (this *MigrationContext) GetCriticalLoad() LoadMap

func (*MigrationContext) GetCurrentLagDuration 

func (this *MigrationContext) GetCurrentLagDuration() time.Duration

func (*MigrationContext) GetETADuration 

func (this *MigrationContext) GetETADuration() time.Duration

func (*MigrationContext) GetETASeconds 

func (this *MigrationContext) GetETASeconds() int64

func (*MigrationContext) GetGhostTableName 

func (this *MigrationContext) GetGhostTableName() string

GetGhostTableName generates the name of ghost table, based on original table name or a given table name

func (*MigrationContext) GetInspectorHostname 

func (this *MigrationContext) GetInspectorHostname() string

GetInspectorHostname is a safe access method to the inspector hostname

func (*MigrationContext) GetIteration 

func (this *MigrationContext) GetIteration() int64

func (*MigrationContext) GetLastHeartbeatOnChangelogTime 

func (this *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time

func (*MigrationContext) GetMaxLoad 

func (this *MigrationContext) GetMaxLoad() LoadMap

func (*MigrationContext) GetNiceRatio 

func (this *MigrationContext) GetNiceRatio() float64

func (*MigrationContext) GetOldTableName 

func (this *MigrationContext) GetOldTableName() string

GetOldTableName generates the name of the "old" table, into which the original table is renamed.

func (*MigrationContext) GetProgressPct 

func (this *MigrationContext) GetProgressPct() float64

func (*MigrationContext) GetRecentBinlogCoordinates 

func (this *MigrationContext) GetRecentBinlogCoordinates() mysql.BinlogCoordinates

func (*MigrationContext) GetThrottleControlReplicaKeys 

func (this *MigrationContext) GetThrottleControlReplicaKeys() *mysql.InstanceKeyMap

func (*MigrationContext) GetThrottleGeneralCheckResult 

func (this *MigrationContext) GetThrottleGeneralCheckResult() *ThrottleCheckResult

func (*MigrationContext) GetThrottleHTTP 

func (this *MigrationContext) GetThrottleHTTP() string

func (*MigrationContext) GetThrottleQuery 

func (this *MigrationContext) GetThrottleQuery() string

func (*MigrationContext) GetTotalRowsCopied 

func (this *MigrationContext) GetTotalRowsCopied() int64

GetTotalRowsCopied returns the accurate number of rows being copied (affected) This is not exactly the same as the rows being iterated via chunks, but potentially close enough

func (*MigrationContext) GetVoluntaryLockName 

func (this *MigrationContext) GetVoluntaryLockName() string

GetVoluntaryLockName returns a name of a voluntary lock to be used throughout the swap-tables process.

func (*MigrationContext) HasMigrationRange 

func (this *MigrationContext) HasMigrationRange() bool

HasMigrationRange tells us whether there's a range to iterate for copying rows. It will be `false` if the table is initially empty

func (*MigrationContext) InspectorIsAlsoApplier 

func (this *MigrationContext) InspectorIsAlsoApplier() bool

InspectorIsAlsoApplier is `true` when the both inspector and applier are the same database instance. This would be true when running directly on master or when testing on replica.

func (*MigrationContext) IsCountingTableRows 

func (this *MigrationContext) IsCountingTableRows() bool

IsCountingTableRows returns true if the migration has a table count query running

func (*MigrationContext) IsThrottled 

func (this *MigrationContext) IsThrottled() (bool, string, ThrottleReasonHint)

func (*MigrationContext) IsTransactionalTable 

func (this *MigrationContext) IsTransactionalTable() bool

func (*MigrationContext) MarkPointOfInterest 

func (this *MigrationContext) MarkPointOfInterest() int64

func (*MigrationContext) MarkRowCopyEndTime 

func (this *MigrationContext) MarkRowCopyEndTime()

ElapsedRowCopyTime returns time since starting to copy chunks of rows

func (*MigrationContext) MarkRowCopyStartTime 

func (this *MigrationContext) MarkRowCopyStartTime()

MarkRowCopyStartTime

func (*MigrationContext) MaxRetries 

func (this *MigrationContext) MaxRetries() int64

func (*MigrationContext) ReadConfigFile 

func (this *MigrationContext) ReadConfigFile() error

ReadConfigFile attempts to read the config file, if it exists

func (*MigrationContext) ReadCriticalLoad 

func (this *MigrationContext) ReadCriticalLoad(criticalLoadList string) error

ReadMaxLoad parses the `--max-load` flag, which is in multiple key-value format, such as: 'Threads_running=100,Threads_connected=500' It only applies changes in case there's no parsing error.

func (*MigrationContext) ReadMaxLoad 

func (this *MigrationContext) ReadMaxLoad(maxLoadList string) error

ReadMaxLoad parses the `--max-load` flag, which is in multiple key-value format, such as: 'Threads_running=100,Threads_connected=500' It only applies changes in case there's no parsing error.

func (*MigrationContext) ReadThrottleControlReplicaKeys 

func (this *MigrationContext) ReadThrottleControlReplicaKeys(throttleControlReplicas string) error

func (*MigrationContext) RequiresBinlogFormatChange 

func (this *MigrationContext) RequiresBinlogFormatChange() bool

RequiresBinlogFormatChange is `true` when the original binlog format isn't `ROW`

func (*MigrationContext) SetChunkSize 

func (this *MigrationContext) SetChunkSize(chunkSize int64)

func (*MigrationContext) SetConnectionConfig 

func (this *MigrationContext) SetConnectionConfig(storageEngine string) error

func (*MigrationContext) SetControlReplicasLagResult 

func (this *MigrationContext) SetControlReplicasLagResult(lagResult *mysql.ReplicationLagResult)

func (*MigrationContext) SetCountTableRowsCancelFunc 

func (this *MigrationContext) SetCountTableRowsCancelFunc(f func())

SetCountTableRowsCancelFunc sets the cancel function for the CountTableRows query context

func (*MigrationContext) SetCutOverLockTimeoutSeconds 

func (this *MigrationContext) SetCutOverLockTimeoutSeconds(timeoutSeconds int64) error

func (*MigrationContext) SetDMLBatchSize 

func (this *MigrationContext) SetDMLBatchSize(batchSize int64)

func (*MigrationContext) SetDefaultNumRetries 

func (this *MigrationContext) SetDefaultNumRetries(retries int64)

func (*MigrationContext) SetETADuration 

func (this *MigrationContext) SetETADuration(etaDuration time.Duration)

func (*MigrationContext) SetExponentialBackoffMaxInterval 

func (this *MigrationContext) SetExponentialBackoffMaxInterval(intervalSeconds int64) error

func (*MigrationContext) SetHeartbeatIntervalMilliseconds 

func (this *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64)

func (*MigrationContext) SetIgnoreHTTPErrors 

func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool)

func (*MigrationContext) SetLastHeartbeatOnChangelogTime 

func (this *MigrationContext) SetLastHeartbeatOnChangelogTime(t time.Time)

func (*MigrationContext) SetMaxLagMillisecondsThrottleThreshold 

func (this *MigrationContext) SetMaxLagMillisecondsThrottleThreshold(maxLagMillisecondsThrottleThreshold int64)

func (*MigrationContext) SetNiceRatio 

func (this *MigrationContext) SetNiceRatio(newRatio float64)

func (*MigrationContext) SetProgressPct 

func (this *MigrationContext) SetProgressPct(progressPct float64)

func (*MigrationContext) SetRecentBinlogCoordinates 

func (this *MigrationContext) SetRecentBinlogCoordinates(coordinates mysql.BinlogCoordinates)

func (*MigrationContext) SetThrottleGeneralCheckResult 

func (this *MigrationContext) SetThrottleGeneralCheckResult(checkResult *ThrottleCheckResult) *ThrottleCheckResult

func (*MigrationContext) SetThrottleHTTP 

func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string)

func (*MigrationContext) SetThrottleQuery 

func (this *MigrationContext) SetThrottleQuery(newQuery string)

func (*MigrationContext) SetThrottled 

func (this *MigrationContext) SetThrottled(throttle bool, reason string, reasonHint ThrottleReasonHint)

func (*MigrationContext) SetupTLS 

func (this *MigrationContext) SetupTLS() error

func (*MigrationContext) TimeSinceLastHeartbeatOnChangelog 

func (this *MigrationContext) TimeSinceLastHeartbeatOnChangelog() time.Duration

func (*MigrationContext) TimeSincePointOfInterest 

func (this *MigrationContext) TimeSincePointOfInterest() time.Duration

type RowsEstimateMethod 

type RowsEstimateMethod string

RowsEstimateMethod is the type of row number estimation 

const (
	TableStatusRowsEstimate RowsEstimateMethod = "TableStatusRowsEstimate"
	ExplainRowsEstimate     RowsEstimateMethod = "ExplainRowsEstimate"
	CountRowsEstimate       RowsEstimateMethod = "CountRowsEstimate"
)

type ThrottleCheckResult 

type ThrottleCheckResult struct {
	ShouldThrottle bool
	Reason         string
	ReasonHint     ThrottleReasonHint
}

func NewThrottleCheckResult 

func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleReasonHint) *ThrottleCheckResult

type ThrottleReasonHint 

type ThrottleReasonHint string
const (
	NoThrottleReasonHint                 ThrottleReasonHint = "NoThrottleReasonHint"
	UserCommandThrottleReasonHint        ThrottleReasonHint = "UserCommandThrottleReasonHint"
	LeavingHibernationThrottleReasonHint ThrottleReasonHint = "LeavingHibernationThrottleReasonHint"
)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.