diff

README

diff

introduction

diff is a library to provide a function to compare tables:

• support comapre table's data and struct
• can generate sql for target table to fix data
• support compare tables with different name
• support compare one target table with multipe source tables

To comapre tables, you should construct a TableDiff struct:

// TableDiff saves config for diff table
type TableDiff struct {
	// source tables
	SourceTables []*TableInstance
	// target table
	TargetTable *TableInstance

	// columns be ignored, will not check this column's data, 
	// but may use these columns as split field or order by key.
	IgnoreColumns []string

	// columns be removed, will remove these columns from table info,
	// and will not check these columns' data, will not use these columns as split field or order by key too.
	RemoveColumns []string

	// field should be the primary key, unique key or field with index
	Field string

	// select range, for example: "age > 10 AND age < 20"
	Range string

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int

	// sampling check percent, for example 10 means only check 10% data
	Sample int

	// how many goroutines are created to check data
	CheckThreadCount int

	// set true if target-db and source-db all support tidb implicit column "_tidb_rowid"
	UseRowID bool

	// set false if want to comapre the data directly
    UseChecksum bool
    
    // collation config in mysql/tidb, should corresponding to charset.
	Collation string

	// ignore check table's struct
	IgnoreStructCheck bool

	// ignore check table's data
	IgnoreDataCheck bool
}

Then call TableDiff's function Equal to get the compare result. The Equal function define as:

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (structEqual bool, dataEqual bool, err error)

Documentation

Index

• type CheckJob
  • func GenerateCheckJob(table *TableInstance, splitFields, limits string, chunkSize int, ...) ([]*CheckJob, error)
• type RowData
• type RowDatas
  • func (r RowDatas) Len() int
  • func (r RowDatas) Less(i, j int) bool
  • func (r *RowDatas) Pop() interface{}
  • func (r *RowDatas) Push(x interface{})
  • func (r RowDatas) Swap(i, j int)
• type TableDiff
  • func (t *TableDiff) CheckTableData(ctx context.Context) (bool, error)
  • func (t *TableDiff) CheckTableStruct(ctx context.Context) (bool, error)
  • func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (bool, bool, error)
  • func (t *TableDiff) EqualTableData(ctx context.Context) (equal bool, err error)
  • func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error)
• type TableInstance

Types

type CheckJob

type CheckJob struct {
	Schema string
	Table  string
	Where  string
	Args   []string
}

CheckJob is the struct of job for check

func GenerateCheckJob

func GenerateCheckJob(table *TableInstance, splitFields, limits string, chunkSize int, collation string, useTiDBStatsInfo bool) ([]*CheckJob, error)

GenerateCheckJob generates some CheckJobs.

type RowData

type RowData struct {
	Data   map[string]*dbutil.ColumnData
	Source string
}

RowData is the struct of rows selected from mysql/tidb

type RowDatas

type RowDatas struct {
	Rows         []RowData
	OrderKeyCols []*model.ColumnInfo
}

RowDatas is a heap of MergeItems.

func (RowDatas) Len

func (r RowDatas) Len() int

func (RowDatas) Less

func (r RowDatas) Less(i, j int) bool

func (*RowDatas) Pop

func (r *RowDatas) Pop() interface{}

Pop implements heap.Interface's Pop function

func (*RowDatas) Push

func (r *RowDatas) Push(x interface{})

Push implements heap.Interface's Push function

func (RowDatas) Swap

func (r RowDatas) Swap(i, j int)

type TableDiff

type TableDiff struct {
	// source tables
	SourceTables []*TableInstance
	// target table
	TargetTable *TableInstance

	// columns be ignored
	IgnoreColumns []string

	// columns be removed
	RemoveColumns []string

	// field should be the primary key, unique key or field with index
	Fields string

	// select range, for example: "age > 10 AND age < 20"
	Range string

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int

	// sampling check percent, for example 10 means only check 10% data
	Sample int

	// how many goroutines are created to check data
	CheckThreadCount int

	// set true if target-db and source-db all support tidb implicit column "_tidb_rowid"
	UseRowID bool

	// set false if want to comapre the data directly
	UseChecksum bool

	// set true if just want compare data by checksum, will skip select data when checksum is not equal
	OnlyUseChecksum bool

	// collation config in mysql/tidb, should corresponding to charset.
	Collation string

	// ignore check table's struct
	IgnoreStructCheck bool

	// ignore check table's data
	IgnoreDataCheck bool

	// get tidb statistics information from which table instance. if is nil, will split chunk by random.
	TiDBStatsSource *TableInstance
	// contains filtered or unexported fields
}

TableDiff saves config for diff table

func (*TableDiff) CheckTableData

func (t *TableDiff) CheckTableData(ctx context.Context) (bool, error)

CheckTableData checks table's data

func (*TableDiff) CheckTableStruct

func (t *TableDiff) CheckTableStruct(ctx context.Context) (bool, error)

CheckTableStruct checks table's struct

func (*TableDiff) Equal

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (bool, bool, error)

Equal tests whether two database have same data and schema.

func (*TableDiff) EqualTableData

func (t *TableDiff) EqualTableData(ctx context.Context) (equal bool, err error)

EqualTableData checks data is equal or not.

func (*TableDiff) WriteSqls

func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error)

WriteSqls write sqls to file

type TableInstance

type TableInstance struct {
	Conn   *sql.DB
	Schema string
	Table  string
	// contains filtered or unexported fields
}

TableInstance record a table instance