split

package
v1.1.0-beta.0...-04608f4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 25, 2024 License: Apache-2.0, Apache-2.0 Imports: 42 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SplitRetryInterval    = 50 * time.Millisecond
	SplitMaxRetryInterval = 4 * time.Second

	// it takes 30 minutes to scatter regions when each TiKV has 400k regions
	ScatterWaitUpperInterval = 30 * time.Minute

	ScanRegionPaginationLimit = 128
)

Constants for split retry machinery.

Variables

View Source 
var (
	WaitRegionOnlineAttemptTimes = config.DefaultRegionCheckBackoffLimit
	SplitRetryTimes              = 150
)
View Source 
var (
	ErrBackoff             = errors.New("found backoff error")
	ErrBackoffAndDontCount = errors.New("found backoff error but don't count")
)

Functions

func CheckRegionEpoch 

func CheckRegionEpoch(_new, _old *RegionInfo) bool

CheckRegionEpoch check region epoch.

func PdErrorCanRetry 

func PdErrorCanRetry(err error) bool

PdErrorCanRetry when pd error retry.

Types

type BackoffMayNotCountBackoffer 

type BackoffMayNotCountBackoffer struct {
	// contains filtered or unexported fields
}

BackoffMayNotCountBackoffer is a backoffer but it may not increase the retry counter. It should be used with ErrBackoff or ErrBackoffAndDontCount.

func NewBackoffMayNotCountBackoffer 

func NewBackoffMayNotCountBackoffer() *BackoffMayNotCountBackoffer

NewBackoffMayNotCountBackoffer creates a new backoffer that may backoff or retry.

TODO: currently it has the same usage as NewWaitRegionOnlineBackoffer so we don't expose its inner settings.

func (*BackoffMayNotCountBackoffer) Attempt 

func (b *BackoffMayNotCountBackoffer) Attempt() int

Attempt implements utils.Backoffer.

func (*BackoffMayNotCountBackoffer) NextBackoff 

func (b *BackoffMayNotCountBackoffer) NextBackoff(err error) time.Duration

NextBackoff implements utils.Backoffer. For BackoffMayNotCountBackoffer, only ErrBackoff and ErrBackoffAndDontCount is meaningful.

type ClientOptionalParameter 

type ClientOptionalParameter func(*pdClient)

func WithOnSplit 

func WithOnSplit(onSplit func(key [][]byte)) ClientOptionalParameter

WithOnSplit sets a callback function to be called after each split.

func WithRawKV 

func WithRawKV() ClientOptionalParameter

WithRawKV sets the client to use raw kv mode.

type ExponentialBackoffer 

type ExponentialBackoffer struct {
	Attempts    int
	BaseBackoff time.Duration
}

ExponentialBackoffer trivially retry any errors it meets. It's useful when the caller has handled the errors but only want to a more semantic backoff implementation.

func (*ExponentialBackoffer) Attempt 

func (b *ExponentialBackoffer) Attempt() int

Attempt returns the remain attempt times

func (*ExponentialBackoffer) NextBackoff 

func (b *ExponentialBackoffer) NextBackoff(error) time.Duration

NextBackoff returns a duration to wait before retrying again.

type MockPDClientForSplit 

type MockPDClientForSplit struct {
	pd.Client

	Regions *pdtypes.RegionTree
	// contains filtered or unexported fields
}

MockPDClientForSplit is a mock PD client for testing split and scatter.

func NewMockPDClientForSplit 

func NewMockPDClientForSplit() *MockPDClientForSplit

NewMockPDClientForSplit creates a new MockPDClientForSplit.

func (*MockPDClientForSplit) BatchScanRegions 

func (c *MockPDClientForSplit) BatchScanRegions(
	_ context.Context,
	keyRanges []pd.KeyRange,
	limit int,
	_ ...pd.GetRegionOption,
) ([]*pd.Region, error)

func (*MockPDClientForSplit) GetOperator 

func (c *MockPDClientForSplit) GetOperator(_ context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error)

func (*MockPDClientForSplit) GetRegionByID 

func (c *MockPDClientForSplit) GetRegionByID(_ context.Context, regionID uint64, _ ...pd.GetRegionOption) (*pd.Region, error)

func (*MockPDClientForSplit) ScanRegions 

func (c *MockPDClientForSplit) ScanRegions(
	_ context.Context,
	key, endKey []byte,
	limit int,
	_ ...pd.GetRegionOption,
) ([]*pd.Region, error)

func (*MockPDClientForSplit) ScatterRegion 

func (c *MockPDClientForSplit) ScatterRegion(_ context.Context, regionID uint64) error

func (*MockPDClientForSplit) ScatterRegions 

func (c *MockPDClientForSplit) ScatterRegions(_ context.Context, regionIDs []uint64, _ ...pd.RegionsOption) (*pdpb.ScatterRegionResponse, error)

func (*MockPDClientForSplit) SetRegions 

func (c *MockPDClientForSplit) SetRegions(boundaries [][]byte) []*metapb.Region

func (*MockPDClientForSplit) SplitRegion 

func (c *MockPDClientForSplit) SplitRegion(
	region *RegionInfo,
	keys [][]byte,
	isRawKV bool,
) (bool, *kvrpcpb.SplitRegionResponse, error)

type RegionInfo 

type RegionInfo struct {
	Region       *metapb.Region
	Leader       *metapb.Peer
	PendingPeers []*metapb.Peer
	DownPeers    []*metapb.Peer
}

RegionInfo includes a region and the leader of the region.

func PaginateScanRegion 

func PaginateScanRegion(
	ctx context.Context, client SplitClient, startKey, endKey []byte, limit int,
) ([]*RegionInfo, error)

PaginateScanRegion scan regions with a limit pagination and return all regions at once. The returned regions are continuous and cover the key range. If not, or meet errors, it will retry internally.

func ScanRegionsWithRetry 

func ScanRegionsWithRetry(
	ctx context.Context, client SplitClient, startKey, endKey []byte, limit int,
) ([]*RegionInfo, error)

func (*RegionInfo) ContainsInterior 

func (region *RegionInfo) ContainsInterior(key []byte) bool

ContainsInterior returns whether the region contains the given key, and also that the key does not fall on the boundary (start key) of the region.

func (*RegionInfo) ToZapFields 

func (region *RegionInfo) ToZapFields() zap.Field

ToZapFields returns zap fields for the RegionInfo. It can handle nil RegionInfo.

type SplitClient 

type SplitClient interface {
	// GetStore gets a store by a store id.
	GetStore(ctx context.Context, storeID uint64) (*metapb.Store, error)
	// GetRegion gets a region which includes a specified key.
	GetRegion(ctx context.Context, key []byte) (*RegionInfo, error)
	// GetRegionByID gets a region by a region id.
	GetRegionByID(ctx context.Context, regionID uint64) (*RegionInfo, error)
	// SplitKeysAndScatter splits the related regions of the keys and scatters the
	// new regions. It returns the new regions that need to be called with
	// WaitRegionsScattered.
	SplitKeysAndScatter(ctx context.Context, sortedSplitKeys [][]byte) ([]*RegionInfo, error)

	// SplitWaitAndScatter splits a region from a batch of keys, waits for the split
	// is finished, and scatters the new regions. It will return the original region,
	// new regions and error. The input keys should not be encoded.
	//
	// The split step has a few retry times. If it meets error, the error is returned
	// directly.
	//
	// The split waiting step has a backoff retry logic, if split has made progress,
	// it will not increase the retry counter. Otherwise, it will retry for about 1h.
	// If the retry is timeout, it will log a warning and continue.
	//
	// The scatter step has a few retry times. If it meets error, it will log a
	// warning and continue.
	// TODO(lance6716): remove this function in interface after BR uses SplitKeysAndScatter.
	SplitWaitAndScatter(ctx context.Context, region *RegionInfo, keys [][]byte) ([]*RegionInfo, error)
	// GetOperator gets the status of operator of the specified region.
	GetOperator(ctx context.Context, regionID uint64) (*pdpb.GetOperatorResponse, error)
	// ScanRegions gets a list of regions, starts from the region that contains key.
	// Limit limits the maximum number of regions returned.
	ScanRegions(ctx context.Context, key, endKey []byte, limit int) ([]*RegionInfo, error)
	// GetPlacementRule loads a placement rule from PD.
	GetPlacementRule(ctx context.Context, groupID, ruleID string) (*pdhttp.Rule, error)
	// SetPlacementRule insert or update a placement rule to PD.
	SetPlacementRule(ctx context.Context, rule *pdhttp.Rule) error
	// DeletePlacementRule removes a placement rule from PD.
	DeletePlacementRule(ctx context.Context, groupID, ruleID string) error
	// SetStoresLabel add or update specified label of stores. If labelValue
	// is empty, it clears the label.
	SetStoresLabel(ctx context.Context, stores []uint64, labelKey, labelValue string) error
	// WaitRegionsScattered waits for an already started scatter region action to
	// finish. Internally it will backoff and retry at the maximum internal of 2
	// seconds. If the scatter makes progress during the retry, it will not decrease
	// the retry counter. If there's always no progress, it will retry for about 1h.
	// Caller can set the context timeout to control the max waiting time.
	//
	// The first return value is always the number of regions that are not finished
	// scattering no matter what the error is.
	WaitRegionsScattered(ctx context.Context, regionInfos []*RegionInfo) (notFinished int, err error)
}

SplitClient is an external client used by RegionSplitter.

func NewClient 

func NewClient(
	client pd.Client,
	httpCli pdhttp.Client,
	tlsConf *tls.Config,
	splitBatchKeyCnt int,
	splitConcurrency int,
	opts ...ClientOptionalParameter,
) SplitClient

NewClient creates a SplitClient.

splitBatchKeyCnt controls how many keys are sent to TiKV in a batch in split region API. splitConcurrency controls how many regions are split concurrently.

type WaitRegionOnlineBackoffer 

type WaitRegionOnlineBackoffer struct {
	Stat utils.RetryState
}

func NewWaitRegionOnlineBackoffer 

func NewWaitRegionOnlineBackoffer() *WaitRegionOnlineBackoffer

NewWaitRegionOnlineBackoffer create a backoff to wait region online.

func (*WaitRegionOnlineBackoffer) Attempt 

func (b *WaitRegionOnlineBackoffer) Attempt() int

Attempt returns the remain attempt times

func (*WaitRegionOnlineBackoffer) NextBackoff 

func (b *WaitRegionOnlineBackoffer) NextBackoff(err error) time.Duration

NextBackoff returns a duration to wait before retrying again

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.