local

Documentation

Index

• Constants
• func GetSystemRLimit() (uint64, error)
• func NewLocalBackend(ctx context.Context, tls *common.TLS, pdAddr string, cfg *config.TikvImporter, ...) (backend.Backend, error)
• func VerifyRLimit(estimateMaxFiles uint64) error
• type File
  • func (e *File) Cleanup(dataDir string) error
  • func (e *File) Close() error
  • func (e *File) Exist(dataDir string) error
• type Range
• type RangePropertiesCollector
  • func (c *RangePropertiesCollector) Add(key pebble.InternalKey, value []byte) error
  • func (c *RangePropertiesCollector) Finish(userProps map[string]string) error
  • func (c *RangePropertiesCollector) Name() string
• type Writer
  • func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, ...) error
  • func (w *Writer) AppendRowsAsync(ctx context.Context, tableName string, columnNames []string, ts uint64, ...) error
  • func (w *Writer) Close() error

Constants

const (
	SplitRetryTimes = 8
)

Functions

func GetSystemRLimit

func GetSystemRLimit() (uint64, error)

func NewLocalBackend

func NewLocalBackend(
	ctx context.Context,
	tls *common.TLS,
	pdAddr string,
	cfg *config.TikvImporter,
	enableCheckpoint bool,
	g glue.Glue,
	maxOpenFiles int,
) (backend.Backend, error)

NewLocalBackend creates new connections to tikv.

func VerifyRLimit

func VerifyRLimit(estimateMaxFiles uint64) error

VerifyRLimit checks whether the open-file limit is large enough. In Local-backend, we need to read and write a lot of L0 SST files, so we need to check system max open files limit.

Types

type File

type File struct {
	UUID uuid.UUID
	// contains filtered or unexported fields
}

func (*File) Cleanup

func (e *File) Cleanup(dataDir string) error

Cleanup remove meta and db files

func (*File) Close

func (e *File) Close() error

func (*File) Exist

func (e *File) Exist(dataDir string) error

Exist checks if db folder existing (meta sometimes won't flush before lightning exit)

type Range

type Range struct {
	// contains filtered or unexported fields
}

Range record start and end key for localStoreDir.DB so we can write it to tikv in streaming

type RangePropertiesCollector

type RangePropertiesCollector struct {
	// contains filtered or unexported fields
}

func (*RangePropertiesCollector) Add

func (c *RangePropertiesCollector) Add(key pebble.InternalKey, value []byte) error

implement `pebble.TablePropertyCollector` implement `TablePropertyCollector.Add`

func (*RangePropertiesCollector) Finish

func (c *RangePropertiesCollector) Finish(userProps map[string]string) error

func (*RangePropertiesCollector) Name

func (c *RangePropertiesCollector) Name() string

The name of the property collector.

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func (*Writer) AppendRows

func (w *Writer) AppendRows(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error

TODO: replace the implementation back with `AppendRowsAsync` after addressing the performance issue.

func (*Writer) AppendRowsAsync

func (w *Writer) AppendRowsAsync(ctx context.Context, tableName string, columnNames []string, ts uint64, rows kv.Rows) error

TODO: temporarily replace this async append rows with the former write-batch approach before addressing the performance issue.

func (*Writer) Close

func (w *Writer) Close() error