redo

Documentation

Index

• Constants
• Variables
• func FixLocalScheme(uri *url.URL)
• func IsBlackholeStorage(scheme string) bool
• func IsConsistentEnabled(level string) bool
• func IsExternalStorage(scheme string) bool
• func IsLocalStorage(scheme string) bool
• func IsValidConsistentLevel(level string) bool
• func IsValidConsistentStorage(scheme string) bool
• func ParseLogFileName(name string) (uint64, string, error)
• func ValidateStorage(uri *url.URL) error
• type ConsistentLevelType
• type ConsistentStorage

Constants

const (
	// DefaultTimeout is the default timeout for writing external storage.
	DefaultTimeout = 5 * time.Minute
	// CloseTimeout is the default timeout for close redo writer.
	CloseTimeout = 15 * time.Second

	// FlushWarnDuration is the warning duration for flushing external storage.
	FlushWarnDuration = time.Second * 20
	// DefaultFlushIntervalInMs is the default flush interval for redo log.
	DefaultFlushIntervalInMs = 2000
	// DefaultMetaFlushIntervalInMs is the default flush interval for redo meta.
	DefaultMetaFlushIntervalInMs = 200
	// MinFlushIntervalInMs is the minimum flush interval for redo log.
	MinFlushIntervalInMs = 50

	// DefaultEncodingWorkerNum is the default number of encoding workers.
	DefaultEncodingWorkerNum = 16
	// DefaultEncodingInputChanSize is the default size of input channel for encoding worker.
	DefaultEncodingInputChanSize = 128
	// DefaultEncodingOutputChanSize is the default size of output channel for encoding worker.
	DefaultEncodingOutputChanSize = 2048
	// DefaultFlushWorkerNum is the default number of flush workers.
	// Maximum allocated memory is flushWorkerNum*maxLogSize, which is
	// `8*64MB = 512MB` by default.
	DefaultFlushWorkerNum = 8

	// DefaultFileMode is the default mode when operation files
	DefaultFileMode = 0o644
	// DefaultDirMode is the default mode when operation dir
	DefaultDirMode = 0o755

	// TmpEXT is the file ext of log file before safely wrote to disk
	TmpEXT = ".tmp"
	// LogEXT is the file ext of log file after safely wrote to disk
	LogEXT = ".log"
	// MetaEXT is the meta file ext of meta file after safely wrote to disk
	MetaEXT = ".meta"
	// SortLogEXT is the sorted log file ext of log file after safely wrote to disk
	SortLogEXT = ".sort"

	// MinSectorSize is minimum sector size used when flushing log so that log can safely
	// distinguish between torn writes and ordinary data corruption.
	MinSectorSize = 512
	// PageBytes is the alignment for flushing records to the backing Writer.
	// It should be a multiple of the minimum sector size so that log can safely
	// distinguish between torn writes and ordinary data corruption.
	PageBytes = 8 * MinSectorSize
	// Megabyte is the size of 1MB
	Megabyte int64 = 1024 * 1024
)

const (
	// RedoMetaFileType is the default file type of meta file
	RedoMetaFileType = "meta"
	// RedoRowLogFileType is the default file type of row log file
	RedoRowLogFileType = "row"
	// RedoDDLLogFileType is the default file type of ddl log file
	RedoDDLLogFileType = "ddl"
)

const (
	// RedoLogFileFormatV1 was used before v6.1.0, which doesn't contain namespace information
	// layout: captureID_changefeedID_fileType_maxEventCommitTs_uuid.fileExtName
	RedoLogFileFormatV1 = "%s_%s_%s_%d_%s%s"
	// RedoLogFileFormatV2 is available since v6.1.0, which contains namespace information
	// layout: captureID_namespace_changefeedID_fileType_maxEventCommitTs_uuid.fileExtName
	RedoLogFileFormatV2 = "%s_%s_%s_%s_%d_%s%s"
	// RedoMetaFileFormat is the format of redo meta file, which contains namespace information.
	// layout: captureID_namespace_changefeedID_fileType_uuid.fileExtName
	RedoMetaFileFormat = "%s_%s_%s_%s_%s%s"
)

Variables

var (
	// DefaultGCIntervalInMs defines GC interval in meta manager, which can be changed in tests.
	DefaultGCIntervalInMs = 5000 // 5 seconds
	// DefaultMaxLogSize is the default max size of log file
	DefaultMaxLogSize = int64(64)
)

var InitExternalStorage = func(ctx context.Context, uri url.URL) (storage.ExternalStorage, error) {
	s, err := util.GetExternalStorageWithDefaultTimeout(ctx, uri.String())
	if err != nil {
		return nil, errors.WrapError(errors.ErrStorageInitialize, err,
			fmt.Sprintf("can't init external storage for %s", uri.String()))
	}
	return s, nil
}

InitExternalStorage init an external storage.

Functions

func FixLocalScheme

func FixLocalScheme(uri *url.URL)

FixLocalScheme convert local scheme to externally compatible scheme.

func IsBlackholeStorage

func IsBlackholeStorage(scheme string) bool

IsBlackholeStorage returns whether a blackhole storage is used.

func IsConsistentEnabled

func IsConsistentEnabled(level string) bool

IsConsistentEnabled returns whether the consistent feature is enabled.

func IsExternalStorage

func IsExternalStorage(scheme string) bool

IsExternalStorage returns whether an external storage is used.

func IsLocalStorage

func IsLocalStorage(scheme string) bool

IsLocalStorage returns whether a local storage is used.

func IsValidConsistentLevel

func IsValidConsistentLevel(level string) bool

IsValidConsistentLevel checks whether a given consistent level is valid

func IsValidConsistentStorage

func IsValidConsistentStorage(scheme string) bool

IsValidConsistentStorage checks whether a give consistent storage is valid.

func ParseLogFileName

func ParseLogFileName(name string) (uint64, string, error)

ParseLogFileName extract the commitTs, fileType from log fileName

func ValidateStorage

func ValidateStorage(uri *url.URL) error

ValidateStorage validates the storage used by redo.

Types

type ConsistentLevelType

type ConsistentLevelType string

ConsistentLevelType is the level of redo log consistent level.

const (
	// ConsistentLevelNone no consistent guarantee.
	ConsistentLevelNone ConsistentLevelType = "none"
	// ConsistentLevelEventual eventual consistent.
	ConsistentLevelEventual ConsistentLevelType = "eventual"
)

type ConsistentStorage

type ConsistentStorage string

ConsistentStorage is the type of consistent storage.