metautil

package
v1.1.0-beta.0...-70f5218 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 28, 2024 License: Apache-2.0, Apache-2.0 Imports: 29 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// LockFile represents file name
	LockFile = "backup.lock"
	// MetaFile represents file name
	MetaFile = "backupmeta"
	// MetaJSONFile represents backup meta json file name
	MetaJSONFile = "jsons/backupmeta.json"
	// MaxBatchSize represents the internal channel buffer size of MetaWriter and MetaReader.
	MaxBatchSize = 1024

	// MetaFileSize represents the limit size of one MetaFile
	MetaFileSize = 128 * units.MiB

	// CrypterIvLen represents the length of iv of crypter method
	CrypterIvLen = 16
)
View Source 
const (
	// MetaV1 represents the old version of backupmeta.
	// because the old version doesn't have version field, so set it to 0 for compatibility.
	MetaV1 = iota
	// MetaV2 represents the new version of backupmeta.
	MetaV2
)
View Source 
const (
	// JSONFileFormat represents json file name format
	JSONFileFormat = "jsons/%s.json"
)

Variables

This section is empty.

Functions

func ArchiveSize 

func ArchiveSize(files []*backuppb.File) uint64

ArchiveSize return the size of Archive data

func DecodeMetaFile 

func DecodeMetaFile(
	ctx context.Context,
	s storage.ExternalStorage,
	cipher *backuppb.CipherInfo,
	metaIndex *backuppb.MetaFile,
) error

DecodeMetaFile decodes the meta file to json format, it is called by br debug

func DecodeStatsFile 

func DecodeStatsFile(
	ctx context.Context,
	s storage.ExternalStorage,
	cipher *backuppb.CipherInfo,
	schemas []*backuppb.Schema,
) error

DecodeStatsFile decodes the stats file to json format, it is called by br debug

func DecryptFullBackupMetaIfNeeded 

func DecryptFullBackupMetaIfNeeded(metaData []byte, cipherInfo *backuppb.CipherInfo) ([]byte, error)

func Encrypt 

func Encrypt(content []byte, cipher *backuppb.CipherInfo) (encryptedContent, iv []byte, err error)

Encrypt encrypts the content according to CipherInfo.

func LoadBackupTables 

func LoadBackupTables(ctx context.Context, reader *MetaReader, loadStats bool) (map[string]*Database, error)

LoadBackupTables loads schemas from BackupMeta.

func NewSizedMetaFile 

func NewSizedMetaFile(sizeLimit int) *sizedMetaFile

NewSizedMetaFile represents the sizedMetaFile.

func PitrIDMapsFilename 

func PitrIDMapsFilename(clusterID, restoreTS uint64) string

PitrIDMapsFilename is filename that used to save id maps in pitr.

func RestoreStats 

func RestoreStats(
	ctx context.Context,
	storage storage.ExternalStorage,
	cipher *backuppb.CipherInfo,
	statsHandler *handle.Handle,
	newTableInfo *model.TableInfo,
	statsFileIndexes []*backuppb.StatsFileIndex,
	rewriteIDMap map[int64]int64,
) error

func SkipFiles 

func SkipFiles(conf *readSchemaConfig)

SkipFiles is the configuration which will make the schema reader skip all files. This is useful when only schema information is needed.

func SkipStats 

func SkipStats(conf *readSchemaConfig)

Types

type AppendOp 

type AppendOp int

AppendOp represents the operation type of meta. 

const (
	// AppendMetaFile represents the MetaFile type.
	AppendMetaFile AppendOp = 0
	// AppendDataFile represents the DataFile type.
	// it records the file meta from tikv.
	AppendDataFile AppendOp = 1
	// AppendSchema represents the schema from tidb.
	AppendSchema AppendOp = 2
	// AppendDDL represents the ddls before last backup.
	AppendDDL AppendOp = 3
)

type ChecksumStats 

type ChecksumStats struct {
	Crc64Xor   uint64
	TotalKvs   uint64
	TotalBytes uint64
}

func CalculateChecksumStatsOnFiles 

func CalculateChecksumStatsOnFiles(files []*backuppb.File) ChecksumStats

CalculateChecksumStatsOnFiles returns the ChecksumStats for the given files

func (ChecksumStats) ChecksumExists 

func (stats ChecksumStats) ChecksumExists() bool

type Database 

type Database struct {
	Info   *model.DBInfo
	Tables []*Table
}

Database wraps the schema and tables of a database.

func (*Database) GetTable 

func (db *Database) GetTable(name string) *Table

GetTable returns a table of the database by name.

type MetaReader 

type MetaReader struct {
	// contains filtered or unexported fields
}

MetaReader wraps a reader to read both old and new version of backupmeta.

func NewMetaReader 

func NewMetaReader(
	backupMeta *backuppb.BackupMeta,
	storage storage.ExternalStorage,
	cipher *backuppb.CipherInfo) *MetaReader

NewMetaReader creates MetaReader.

func (*MetaReader) GetBasic 

func (reader *MetaReader) GetBasic() backuppb.BackupMeta

GetBasic returns a basic copy of the backup meta.

func (*MetaReader) ReadDDLs 

func (reader *MetaReader) ReadDDLs(ctx context.Context) ([]byte, error)

ReadDDLs reads the ddls from the backupmeta. This function is compatible with the old backupmeta.

func (*MetaReader) ReadSchemasFiles 

func (reader *MetaReader) ReadSchemasFiles(ctx context.Context, output chan<- *Table, opts ...ReadSchemaOption) error

ReadSchemasFiles reads the schema and datafiles from the backupmeta. This function is compatible with the old backupmeta.

type MetaWriter 

type MetaWriter struct {
	// contains filtered or unexported fields
}

MetaWriter represents wraps a writer, and the MetaWriter should be compatible with old version of backupmeta.

func NewMetaWriter 

func NewMetaWriter(
	storage storage.ExternalStorage,
	metafileSizeLimit int,
	useV2Meta bool,
	metaFileName string,
	cipher *backuppb.CipherInfo,
) *MetaWriter

NewMetaWriter creates MetaWriter.

func (*MetaWriter) ArchiveSize 

func (writer *MetaWriter) ArchiveSize() uint64

ArchiveSize represents the size of ArchiveSize.

func (*MetaWriter) Backupmeta 

func (writer *MetaWriter) Backupmeta() *backuppb.BackupMeta

Backupmeta clones a backupmeta.

func (*MetaWriter) FinishWriteMetas 

func (writer *MetaWriter) FinishWriteMetas(ctx context.Context, op AppendOp) error

FinishWriteMetas close the channel in StartWriteMetasAsync and flush the buffered data.

func (*MetaWriter) FlushBackupMeta 

func (writer *MetaWriter) FlushBackupMeta(ctx context.Context) error

FlushBackupMeta flush the `backupMeta` to `ExternalStorage`

func (*MetaWriter) MetaFilesSize 

func (writer *MetaWriter) MetaFilesSize() uint64

MetaFilesSize represents the size of meta files from backupmeta v2, must be called after everything finishes by `FinishWriteMetas`.

func (*MetaWriter) NewStatsWriter 

func (writer *MetaWriter) NewStatsWriter() *StatsWriter

NewStatsWriter wraps the new function of stats writer

func (*MetaWriter) Send 

func (writer *MetaWriter) Send(m any, _ AppendOp) error

Send sends the item to buffer.

func (*MetaWriter) StartWriteMetasAsync 

func (writer *MetaWriter) StartWriteMetasAsync(ctx context.Context, op AppendOp)

StartWriteMetasAsync writes four kind of meta into backupmeta. 1. file 2. schema 3. ddl 4. rawRange( raw kv ) when useBackupMetaV2 enabled, it will generate multi-level index backupmetav2. else it will generate backupmeta as before for compatibility. User should call FinishWriteMetas after StartWriterMetasAsync.

func (*MetaWriter) Update 

func (writer *MetaWriter) Update(f func(m *backuppb.BackupMeta))

Update updates some property of backupmeta.

type ReadSchemaOption 

type ReadSchemaOption func(*readSchemaConfig)

ReadSchemaOption describes some extra option of reading the config.

type StatsWriter 

type StatsWriter struct {
	// contains filtered or unexported fields
}

A lightweight function wrapper to dump the statistic

func (*StatsWriter) BackupStats 

func (s *StatsWriter) BackupStats(ctx context.Context, jsonTable *statsutil.JSONTable, physicalID int64) error

func (*StatsWriter) BackupStatsDone 

func (s *StatsWriter) BackupStatsDone(ctx context.Context) ([]*backuppb.StatsFileIndex, error)

type Table 

type Table struct {
	DB               *model.DBInfo
	Info             *model.TableInfo
	Crc64Xor         uint64
	TotalKvs         uint64
	TotalBytes       uint64
	Files            []*backuppb.File
	TiFlashReplicas  int
	Stats            *util.JSONTable
	StatsFileIndexes []*backuppb.StatsFileIndex
}

Table wraps the schema and files of a table.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.