local

Documentation

Index

• Constants
• func NewLocalBackend(ctx context.Context, tls *common.TLS, pdAddr string, cfg *config.TikvImporter, ...) (backend.Backend, error)
• func VerifyRLimit(estimateMaxFiles Rlim_t) error
• type DuplicateManager
  • func NewDuplicateManager(db *pebble.DB, splitCli restore.SplitClient, ts uint64, tls *common.TLS, ...) (*DuplicateManager, error)
  • func (manager *DuplicateManager) CollectDuplicateRowsFromLocalIndex(ctx context.Context, tbl table.Table, db *pebble.DB) error
  • func (manager *DuplicateManager) CollectDuplicateRowsFromTiKV(ctx context.Context, tbl table.Table) error
  • func (manager *DuplicateManager) RepairDuplicateData() error
  • func (manager *DuplicateManager) ReportDuplicateData() error
• type DuplicateRequest
• type File
  • func (e *File) Cleanup(dataDir string) error
  • func (e *File) Close() error
  • func (e *File) Exist(dataDir string) error
• type KeyAdapter
• type Range
• type RangePropertiesCollector
  • func (c *RangePropertiesCollector) Add(key pebble.InternalKey, value []byte) error
  • func (c *RangePropertiesCollector) Finish(userProps map[string]string) error
  • func (c *RangePropertiesCollector) Name() string
• type Rlim_t
  • func GetSystemRLimit() (Rlim_t, error)
• type Writer
  • func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, rows kv.Rows) error
  • func (w *Writer) Close(ctx context.Context) (backend.ChunkFlushStatus, error)
  • func (w *Writer) IsSynced() bool

Constants

const (
	SplitRetryTimes = 8
)

Functions

func NewLocalBackend

func NewLocalBackend(
	ctx context.Context,
	tls *common.TLS,
	pdAddr string,
	cfg *config.TikvImporter,
	enableCheckpoint bool,
	g glue.Glue,
	maxOpenFiles int,
) (backend.Backend, error)

NewLocalBackend creates new connections to tikv.

func VerifyRLimit

func VerifyRLimit(estimateMaxFiles Rlim_t) error

VerifyRLimit checks whether the open-file limit is large enough. In Local-backend, we need to read and write a lot of L0 SST files, so we need to check system max open files limit.

Types

type DuplicateManager

type DuplicateManager struct {
	// contains filtered or unexported fields
}

func NewDuplicateManager

func NewDuplicateManager(
	db *pebble.DB,
	splitCli restore.SplitClient,
	ts uint64,
	tls *common.TLS,
	regionConcurrency int) (*DuplicateManager, error)

func (*DuplicateManager) CollectDuplicateRowsFromLocalIndex

func (manager *DuplicateManager) CollectDuplicateRowsFromLocalIndex(
	ctx context.Context,
	tbl table.Table,
	db *pebble.DB,
) error

Collect rows by read the index in db.

func (*DuplicateManager) CollectDuplicateRowsFromTiKV

func (manager *DuplicateManager) CollectDuplicateRowsFromTiKV(ctx context.Context, tbl table.Table) error

func (*DuplicateManager) RepairDuplicateData

func (manager *DuplicateManager) RepairDuplicateData() error

func (*DuplicateManager) ReportDuplicateData

func (manager *DuplicateManager) ReportDuplicateData() error

type DuplicateRequest

type DuplicateRequest struct {
	// contains filtered or unexported fields
}

type File

type File struct {
	UUID uuid.UUID
	// contains filtered or unexported fields
}

func (*File) Cleanup

func (e *File) Cleanup(dataDir string) error

Cleanup remove meta and db files

func (*File) Close

func (e *File) Close() error

func (*File) Exist

func (e *File) Exist(dataDir string) error

Exist checks if db folder existing (meta sometimes won't flush before lightning exit)

type KeyAdapter

type KeyAdapter interface {
	// Encode encodes key with rowID and offset. It guarantees the encoded key is in ascending order for comparison.
	// `buf` is used to buffer data to avoid the cost of make slice.
	// Implementations of Encode must not reuse the key for encoding.
	Encode(buf []byte, key []byte, rowID int64, offset int64) []byte

	// Decode decodes the original key. `buf` is used to buffer data to avoid the cost of make slice.
	// Implementations of Decode must not reuse the data for decoding.
	Decode(buf []byte, data []byte) (key []byte, rowID int64, offset int64, err error)

	// EncodedLen returns the encoded key length.
	EncodedLen(key []byte) int
}

KeyAdapter is used to encode and decode keys.

type Range

type Range struct {
	// contains filtered or unexported fields
}

Range record start and end key for localStoreDir.DB so we can write it to tikv in streaming

type RangePropertiesCollector

type RangePropertiesCollector struct {
	// contains filtered or unexported fields
}

func (*RangePropertiesCollector) Add

func (c *RangePropertiesCollector) Add(key pebble.InternalKey, value []byte) error

implement `pebble.TablePropertyCollector` implement `TablePropertyCollector.Add`

func (*RangePropertiesCollector) Finish

func (c *RangePropertiesCollector) Finish(userProps map[string]string) error

func (*RangePropertiesCollector) Name

func (c *RangePropertiesCollector) Name() string

The name of the property collector.

type Rlim_t

type Rlim_t = uint64

func GetSystemRLimit

func GetSystemRLimit() (Rlim_t, error)

type Writer

type Writer struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*Writer) AppendRows

func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, rows kv.Rows) error

func (*Writer) Close

func (w *Writer) Close(ctx context.Context) (backend.ChunkFlushStatus, error)

func (*Writer) IsSynced

func (w *Writer) IsSynced() bool