checkpoint

Documentation

Index

• Constants
• func SaveCheckpointMetadata(ctx context.Context, s storage.ExternalStorage, meta *CheckpointMetadata) error
• func WalkCheckpointFile(ctx context.Context, s storage.ExternalStorage, cipher *backuppb.CipherInfo, ...) (time.Duration, error)
• type CheckpointData
• type CheckpointLock
• type CheckpointMessage
• type CheckpointMetadata
  • func LoadCheckpointMetadata(ctx context.Context, s storage.ExternalStorage) (*CheckpointMetadata, error)
• type CheckpointRunner
  • func StartCheckpointRunner(ctx context.Context, storage storage.ExternalStorage, ...) (*CheckpointRunner, error)
  • func StartCheckpointRunnerForTest(ctx context.Context, storage storage.ExternalStorage, ...) (*CheckpointRunner, error)
  • func (r *CheckpointRunner) Append(ctx context.Context, groupKey string, startKey []byte, endKey []byte, ...) error
  • func (r *CheckpointRunner) FlushChecksum(ctx context.Context, tableID int64, crc64xor uint64, totalKvs uint64, ...) error
  • func (r *CheckpointRunner) WaitForFinish(ctx context.Context)
• type ChecksumInfo
• type ChecksumItem
• type ChecksumItems
• type ChecksumRunner
  • func NewChecksumRunner() *ChecksumRunner
  • func (cr *ChecksumRunner) FlushChecksum(ctx context.Context, s storage.ExternalStorage, tableID int64, crc64xor uint64, ...) error
  • func (cr *ChecksumRunner) RecordError(err error)
• type GlobalTimer
• type RangeGroupData
• type RangeGroups

Constants

const (
	CheckpointMetaPath = "checkpoint.meta"
	CheckpointDir      = "/checkpoints"

	CheckpointDataDir     = CheckpointDir + "/data"
	CheckpointChecksumDir = CheckpointDir + "/checksum"
	CheckpointLockPath    = CheckpointDir + "/checkpoint.lock"
)

const MaxChecksumTotalCost float64 = 60.0

Functions

func SaveCheckpointMetadata

func SaveCheckpointMetadata(ctx context.Context, s storage.ExternalStorage, meta *CheckpointMetadata) error

save the checkpoint metadata into the external storage

func WalkCheckpointFile

func WalkCheckpointFile(ctx context.Context, s storage.ExternalStorage, cipher *backuppb.CipherInfo, fn func(groupKey string, rg *rtree.Range)) (time.Duration, error)

walk the whole checkpoint range files and retrieve the metadatat of backed up ranges and return the total time cost in the past executions

Types

type CheckpointData

type CheckpointData struct {
	DureTime        time.Duration     `json:"dure-time"`
	RangeGroupMetas []*RangeGroupData `json:"range-group-metas"`
}

type CheckpointLock

type CheckpointLock struct {
	LockId   uint64 `json:"lock-id"`
	ExpireAt int64  `json:"expire-at"`
}

type CheckpointMessage

type CheckpointMessage struct {
	// start-key of the origin range
	GroupKey string

	Group *rtree.Range
}

type CheckpointMetadata

type CheckpointMetadata struct {
	GCServiceId string        `json:"gc-service-id"`
	ConfigHash  []byte        `json:"config-hash"`
	BackupTS    uint64        `json:"backup-ts"`
	Ranges      []rtree.Range `json:"ranges"`

	CheckpointChecksum map[int64]*ChecksumItem    `json:"-"`
	CheckpointDataMap  map[string]rtree.RangeTree `json:"-"`
}

func LoadCheckpointMetadata

func LoadCheckpointMetadata(ctx context.Context, s storage.ExternalStorage) (*CheckpointMetadata, error)

load checkpoint metadata from the external storage

type CheckpointRunner

type CheckpointRunner struct {
	// contains filtered or unexported fields
}

func StartCheckpointRunner

func StartCheckpointRunner(ctx context.Context, storage storage.ExternalStorage, cipher *backuppb.CipherInfo, timer GlobalTimer) (*CheckpointRunner, error)

func StartCheckpointRunnerForTest

func StartCheckpointRunnerForTest(ctx context.Context, storage storage.ExternalStorage, cipher *backuppb.CipherInfo, tick time.Duration, timer GlobalTimer) (*CheckpointRunner, error)

only for test

func (*CheckpointRunner) Append

func (r *CheckpointRunner) Append(
	ctx context.Context,
	groupKey string,
	startKey []byte,
	endKey []byte,
	files []*backuppb.File,
) error

func (*CheckpointRunner) FlushChecksum

func (r *CheckpointRunner) FlushChecksum(ctx context.Context, tableID int64, crc64xor uint64, totalKvs uint64, totalBytes uint64, timeCost float64) error

func (*CheckpointRunner) WaitForFinish

func (r *CheckpointRunner) WaitForFinish(ctx context.Context)

Note: Cannot be parallel with `Append` function

type ChecksumInfo

type ChecksumInfo struct {
	Content  []byte `json:"content"`
	Checksum []byte `json:"checksum"`
}

type ChecksumItem

type ChecksumItem struct {
	TableID    int64  `json:"table-id"`
	Crc64xor   uint64 `json:"crc64-xor"`
	TotalKvs   uint64 `json:"total-kvs"`
	TotalBytes uint64 `json:"total-bytes"`
}

type ChecksumItems

type ChecksumItems struct {
	Items []*ChecksumItem `json:"checksum-items"`
}

type ChecksumRunner

type ChecksumRunner struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewChecksumRunner

func NewChecksumRunner() *ChecksumRunner

func (*ChecksumRunner) FlushChecksum

func (cr *ChecksumRunner) FlushChecksum(
	ctx context.Context,
	s storage.ExternalStorage,
	tableID int64,
	crc64xor uint64,
	totalKvs uint64,
	totalBytes uint64,
	timeCost float64,
) error

FlushChecksum save the checksum in the memory temporarily and flush to the external storage if checksum take much time

func (*ChecksumRunner) RecordError

func (cr *ChecksumRunner) RecordError(err error)

type GlobalTimer

type GlobalTimer interface {
	GetTS(context.Context) (int64, int64, error)
}

type RangeGroupData

type RangeGroupData struct {
	RangeGroupsEncriptedData []byte
	Checksum                 []byte
	CipherIv                 []byte

	Size int
}

type RangeGroups

type RangeGroups struct {
	GroupKey string         `json:"group-key"`
	Groups   []*rtree.Range `json:"groups"`
}