diff

README

diff

introduction

diff is a library to provide a function to compare tables:

• support comapre table's data and struct
• can generate sql for target table to fix data
• support compare tables with different name
• support compare one target table with multipe source tables

To comapre tables, you should construct a TableDiff struct:

// TableDiff saves config for diff table
type TableDiff struct {
	// source tables
	SourceTables []*TableInstance
	// target table
	TargetTable *TableInstance

	// columns be ignored, will not check this column's data, 
	// but may use these columns as split field or order by key.
	IgnoreColumns []string

	// columns be removed, will remove these columns from table info,
	// and will not check these columns' data, will not use these columns as split field or order by key too.
	RemoveColumns []string

	// field should be the primary key, unique key or field with index
	Field string

	// select range, for example: "age > 10 AND age < 20"
	Range string

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int

	// sampling check percent, for example 10 means only check 10% data
	Sample int

	// how many goroutines are created to check data
	CheckThreadCount int

	// set true if target-db and source-db all support tidb implicit column "_tidb_rowid"
	UseRowID bool

	// set false if want to comapre the data directly
    UseChecksum bool
    
    // collation config in mysql/tidb, should corresponding to charset.
	Collation string

	// ignore check table's struct
	IgnoreStructCheck bool

	// ignore check table's data
	IgnoreDataCheck bool
}

Then call TableDiff's function Equal to get the compare result. The Equal function define as:

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (structEqual bool, dataEqual bool, err error)

Documentation

Index

• type Bound
• type ChunkRange
  • func NewChunkRange(mode string) *ChunkRange
  • func SplitChunks(ctx context.Context, table *TableInstance, splitFields, limits string, ...) (chunks []*ChunkRange, err error)
  • func (c *ChunkRange) String() string
• type RowData
• type RowDatas
  • func (r RowDatas) Len() int
  • func (r RowDatas) Less(i, j int) bool
  • func (r *RowDatas) Pop() interface{}
  • func (r *RowDatas) Push(x interface{})
  • func (r RowDatas) Swap(i, j int)
• type TableDiff
  • func (t *TableDiff) CheckTableData(ctx context.Context) (equal bool, err error)
  • func (t *TableDiff) CheckTableStruct(ctx context.Context) (bool, error)
  • func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (bool, bool, error)
  • func (t *TableDiff) LoadCheckpoint(ctx context.Context) ([]*ChunkRange, error)
  • func (t *TableDiff) UpdateSummaryInfo(ctx context.Context) chan bool
  • func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error) chan bool
• type TableInstance

Types

type Bound

type Bound struct {
	Column      string `json:"column"`
	Lower       string `json:"lower"`
	LowerSymbol string `json:"lower-symbol"`
	Upper       string `json:"upper"`
	UpperSymbol string `json:"upper-symbol"`
}

Bound represents a bound for a column

type ChunkRange

type ChunkRange struct {
	ID     int      `json:"id"`
	Bounds []*Bound `json:"bounds"`
	Mode   string   `json:"mode"`

	Where string   `json:"where"`
	Args  []string `json:"args"`

	State string `json:"state"`
}

ChunkRange represents chunk range

func NewChunkRange

func NewChunkRange(mode string) *ChunkRange

NewChunkRange return a ChunkRange.

func SplitChunks

func SplitChunks(ctx context.Context, table *TableInstance, splitFields, limits string, chunkSize int, collation string, useTiDBStatsInfo bool) (chunks []*ChunkRange, err error)

SplitChunks splits the table to some chunks.

func (*ChunkRange) String

func (c *ChunkRange) String() string

String returns the string of ChunkRange, used for log.

type RowData

type RowData struct {
	Data   map[string]*dbutil.ColumnData
	Source string
}

RowData is the struct of rows selected from mysql/tidb

type RowDatas

type RowDatas struct {
	Rows         []RowData
	OrderKeyCols []*model.ColumnInfo
}

RowDatas is a heap of MergeItems.

func (RowDatas) Len

func (r RowDatas) Len() int

func (RowDatas) Less

func (r RowDatas) Less(i, j int) bool

func (*RowDatas) Pop

func (r *RowDatas) Pop() interface{}

Pop implements heap.Interface's Pop function

func (*RowDatas) Push

func (r *RowDatas) Push(x interface{})

Push implements heap.Interface's Push function

func (RowDatas) Swap

func (r RowDatas) Swap(i, j int)

type TableDiff

type TableDiff struct {
	// source tables
	SourceTables []*TableInstance `json:"source-tables"`
	// target table
	TargetTable *TableInstance `json:"target-table"`

	// columns be ignored
	IgnoreColumns []string `json:"-"`

	// columns be removed
	RemoveColumns []string `json:"-"`

	// field should be the primary key, unique key or field with index
	Fields string `json:"fields"`

	// select range, for example: "age > 10 AND age < 20"
	Range string `json:"range"`

	// for example, the whole data is [1...100]
	// we can split these data to [1...10], [11...20], ..., [91...100]
	// the [1...10] is a chunk, and it's chunk size is 10
	// size of the split chunk
	ChunkSize int `json:"chunk-size"`

	// sampling check percent, for example 10 means only check 10% data
	Sample int `json:"sample"`

	// how many goroutines are created to check data
	CheckThreadCount int `json:"-"`

	// set true if target-db and source-db all support tidb implicit column "_tidb_rowid"
	UseRowID bool `json:"use-rowid"`

	// set false if want to comapre the data directly
	UseChecksum bool `json:"-"`

	// set true if just want compare data by checksum, will skip select data when checksum is not equal
	OnlyUseChecksum bool `json:"-"`

	// collation config in mysql/tidb, should corresponding to charset.
	Collation string `json:"collation"`

	// ignore check table's struct
	IgnoreStructCheck bool `json:"-"`

	// ignore check table's data
	IgnoreDataCheck bool `json:"-"`

	// set true will continue check from the latest checkpoint
	UseCheckpoint bool `json:"use-checkpoint"`

	// get tidb statistics information from which table instance. if is nil, will split chunk by random.
	TiDBStatsSource *TableInstance `json:"tidb-stats-source"`
	// contains filtered or unexported fields
}

TableDiff saves config for diff table

func (*TableDiff) CheckTableData

func (t *TableDiff) CheckTableData(ctx context.Context) (equal bool, err error)

CheckTableData checks table's data

func (*TableDiff) CheckTableStruct

func (t *TableDiff) CheckTableStruct(ctx context.Context) (bool, error)

CheckTableStruct checks table's struct

func (*TableDiff) Equal

func (t *TableDiff) Equal(ctx context.Context, writeFixSQL func(string) error) (bool, bool, error)

Equal tests whether two database have same data and schema.

func (*TableDiff) LoadCheckpoint

func (t *TableDiff) LoadCheckpoint(ctx context.Context) ([]*ChunkRange, error)

LoadCheckpoint do some prepare work before check data, like adjust config and create checkpoint table

func (*TableDiff) UpdateSummaryInfo

func (t *TableDiff) UpdateSummaryInfo(ctx context.Context) chan bool

func (*TableDiff) WriteSqls

func (t *TableDiff) WriteSqls(ctx context.Context, writeFixSQL func(string) error) chan bool

WriteSqls write sqls to file

type TableInstance

type TableInstance struct {
	Conn       *sql.DB `json:"-"`
	Schema     string  `json:"schema"`
	Table      string  `json:"table"`
	InstanceID string  `json:"instance-id"`
	// contains filtered or unexported fields
}

TableInstance record a table instance