loader

Documentation

Index

• func GetLightningConfig(globalCfg *lcfg.GlobalConfig, subtaskCfg *config.SubTaskConfig) (*lcfg.Config, error)
• func GetTaskInfoSchemaName(dmMetaSchema, taskName string) string
• func MakeGlobalConfig(cfg *config.SubTaskConfig) *lcfg.GlobalConfig
• func RegisterMetrics(registry *prometheus.Registry)
• type LightningCheckpointList
  • func NewLightningCheckpointList(db *conn.BaseDB, taskName string, sourceName string, metaSchema string, ...) *LightningCheckpointList
  • func (cp *LightningCheckpointList) Close()
  • func (cp *LightningCheckpointList) Prepare(ctx context.Context) error
  • func (cp *LightningCheckpointList) RegisterCheckPoint(ctx context.Context) error
  • func (cp *LightningCheckpointList) UpdateStatus(ctx context.Context, status lightingLoadStatus) error
• type LightningLoader
  • func NewLightning(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName string) *LightningLoader
  • func (l *LightningLoader) Close()
  • func (l *LightningLoader) Init(ctx context.Context) (err error)
  • func (l *LightningLoader) IsFreshTask(ctx context.Context) (bool, error)
  • func (l *LightningLoader) Kill()
  • func (l *LightningLoader) Pause()
  • func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult)
  • func (l *LightningLoader) Resume(ctx context.Context, pr chan pb.ProcessResult)
  • func (l *LightningLoader) Status(_ *binlog.SourceStatus) interface{}
  • func (l *LightningLoader) Type() pb.UnitType
  • func (l *LightningLoader) Update(ctx context.Context, cfg *config.SubTaskConfig) error

Functions

func GetLightningConfig

func GetLightningConfig(globalCfg *lcfg.GlobalConfig, subtaskCfg *config.SubTaskConfig) (*lcfg.Config, error)

GetLightningConfig returns the lightning task config for the lightning global config and DM subtask config.

func GetTaskInfoSchemaName

func GetTaskInfoSchemaName(dmMetaSchema, taskName string) string

GetTaskInfoSchemaName is used to assign to TikvImporter.DuplicateResolution in lightning config.

func MakeGlobalConfig

func MakeGlobalConfig(cfg *config.SubTaskConfig) *lcfg.GlobalConfig

MakeGlobalConfig converts subtask config to lightning global config.

func RegisterMetrics

func RegisterMetrics(registry *prometheus.Registry)

RegisterMetrics registers metrics.

Types

type LightningCheckpointList

type LightningCheckpointList struct {
	// contains filtered or unexported fields
}

func NewLightningCheckpointList

func NewLightningCheckpointList(
	db *conn.BaseDB,
	taskName string,
	sourceName string,
	metaSchema string,
	logger log.Logger,
) *LightningCheckpointList

func (*LightningCheckpointList) Close

func (cp *LightningCheckpointList) Close()

Close implements CheckPoint.Close.

func (*LightningCheckpointList) Prepare

func (cp *LightningCheckpointList) Prepare(ctx context.Context) error

func (*LightningCheckpointList) RegisterCheckPoint

func (cp *LightningCheckpointList) RegisterCheckPoint(ctx context.Context) error

func (*LightningCheckpointList) UpdateStatus

func (cp *LightningCheckpointList) UpdateStatus(ctx context.Context, status lightingLoadStatus) error

type LightningLoader

type LightningLoader struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

LightningLoader can load your mydumper data into TiDB database.

func NewLightning

func NewLightning(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName string) *LightningLoader

NewLightning creates a new Loader importing data with lightning.

func (*LightningLoader) Close

func (l *LightningLoader) Close()

Close does graceful shutdown.

func (*LightningLoader) Init

func (l *LightningLoader) Init(ctx context.Context) (err error)

Init initializes loader for a load task, but not start Process. if fail, it should not call l.Close.

func (*LightningLoader) IsFreshTask

func (l *LightningLoader) IsFreshTask(ctx context.Context) (bool, error)

IsFreshTask implements Unit.IsFreshTask.

func (*LightningLoader) Kill

func (l *LightningLoader) Kill()

Kill does ungraceful shutdown.

func (*LightningLoader) Pause

func (l *LightningLoader) Pause()

Pause pauses the process, and it can be resumed later should cancel context from external.

func (*LightningLoader) Process

func (l *LightningLoader) Process(ctx context.Context, pr chan pb.ProcessResult)

Process implements Unit.Process.

func (*LightningLoader) Resume

func (l *LightningLoader) Resume(ctx context.Context, pr chan pb.ProcessResult)

Resume resumes the paused process.

func (*LightningLoader) Status

func (l *LightningLoader) Status(_ *binlog.SourceStatus) interface{}

Status returns the unit's current status.

func (*LightningLoader) Type

func (l *LightningLoader) Type() pb.UnitType

Type implements Unit.Type.

func (*LightningLoader) Update

func (l *LightningLoader) Update(ctx context.Context, cfg *config.SubTaskConfig) error

Update implements Unit.Update now, only support to update config for routes, filters, column-mappings, block-allow-list now no config diff implemented, so simply re-init use new config no binlog filter for loader need to update.