binlog

package module
v0.0.0-...-b009a82 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 16, 2018 License: Apache-2.0 Imports: 12 Imported by: 0

README

go-mysql-binlog

License Go Report Card

MySQL binary log analyzer in Golang.

中文说明

Example

func main() {
	decoder, err := binlog.NewBinFileDecoder("./testdata/mysql-bin.000004")
	if err != nil {
		panic(err)
		return
	}
    
	num := 0
	maxEventCount := 0
	err = decoder.WalkEvent(func(event *binlog.BinEvent) (isContinue bool, err error) {
		fmt.Printf("Got %s: \n\t", binlog.EventType2Str[event.Header.EventType])
		fmt.Println(event.Header)
		
		// show details if you need
		// if event.Body != nil {
		// 	pretty.Println(event.Body)
		// }
		//
		
		fmt.Println(strings.Repeat("=", 100))
		count ++
		return maxEventCount > num || maxEventCount == 0, nil
	}, nil)
    
	if err != nil {
		panic(err)
	}

}
Output:
Got FORMAT_DESCRIPTION_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:119, LogPos:123, Flag:0x1
====================================================================================================
Got PREVIOUS_GTIDS_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:31, LogPos:154, Flag:0x80
====================================================================================================
Got ANONYMOUS_GTID_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:65, LogPos:219, Flag:0x0
====================================================================================================
Got QUERY_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:79, LogPos:298, Flag:0x8
====================================================================================================
Got TABLE_MAP_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:64, LogPos:362, Flag:0x0
====================================================================================================
Got WRITE_ROWS_EVENTv2: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:197, LogPos:559, Flag:0x0
====================================================================================================
Got XID_EVENT: 
	Time:2018-09-22 18:24:30 +0800 CST, ServerID:1537611870, EventSize:31, LogPos:590, Flag:0x0
====================================================================================================

Progress

EventType Supported
UNKNOWN_EVENT ✔
START_EVENT_V3
QUERY_EVENT ✔
STOP_EVENT
ROTATE_EVENT ✔
INTVAR_EVENT ✔
LOAD_EVENT
SLAVE_EVENT
CREATE_FILE_EVENT
APPEND_BLOCK_EVENT
EXEC_LOAD_EVENT
DELETE_FILE_EVENT
NEW_LOAD_EVENT
RAND_EVENT
USER_VAR_EVENT
FORMAT_DESCRIPTION_EVENT ✔
XID_EVENT ✔
BEGIN_LOAD_QUERY_EVENT
EXECUTE_LOAD_QUERY_EVENT
TABLE_MAP_EVENT ✔
WRITE_ROWS_EVENTv0
UPDATE_ROWS_EVENTv0
DELETE_ROWS_EVENTv0
WRITE_ROWS_EVENTv1
UPDATE_ROWS_EVENTv1
DELETE_ROWS_EVENTv1
INCIDENT_EVENT
HEARTBEAT_EVENT
IGNORABLE_EVENT
ROWS_QUERY_EVENT
WRITE_ROWS_EVENTv2
UPDATE_ROWS_EVENTv2
DELETE_ROWS_EVENTv2
GTID_EVENT ✔
ANONYMOUS_GTID_EVENT ✔
PREVIOUS_GTIDS_EVENT ✔

TODO

  1. Support all mysql binlog event.
  2. Get binlog event through network connections.
  3. Multi threads binlog dumper.
  4. Flash back base on row format binary log.
  5. more.

Documentation

Index

Constants

View Source 
const (
	MySQLTypeDecimal   = 0x00
	MySQLTypeTiny      = 0x01
	MySQLTypeShort     = 0x02
	MySQLTypeLong      = 0x03
	MySQLTypeFloat     = 0x04
	MySQLTypeDouble    = 0x05
	MySQLTypeNull      = 0x06
	MySQLTypeTimestamp = 0x07
	MySQLTypeLonglong  = 0x08
	MySQLTypeInt24     = 0x09
	MySQLTypeDate      = 0x0a
	MySQLTypeTime      = 0x0b
	MySQLTypeDatetime  = 0x0c
	MySQLTypeYear      = 0x0d
	MySQLTypeNewDate   = 0x0e
	MySQLTypeVarchar   = 0x0f
	MySQLTypeBit       = 0x10

	// mysql 5.6
	MySQLTypeTimestamp2 = 0x11
	MySQLTypeDatetime2  = 0x12
	MySQLTypeTime2      = 0x13

	// start with 0xf5
	MySQLTypeJSON       = 0xf5
	MySQLTypeNewDecimal = 0xf6
	MySQLTypeEnum       = 0xf7
	MySQLTypeSet        = 0xf8
	MySQLTypeTinyBlob   = 0xf9
	MySQLTypeMediumBlob = 0xfa
	MySQLTypeLongBlob   = 0xfb
	MySQLTypeBlob       = 0xfc
	MySQLTypeVarString  = 0xfd
	MySQLTypeString     = 0xfe
	MySQLTypeGeometry   = 0xff
)

https://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::MYSQL_TYPE_STRING

View Source 
const (
	UnknownEvent           = 0x00
	StartEventV3           = 0x01
	QueryEvent             = 0x02
	StopEvent              = 0x03
	RotateEvent            = 0x04
	IntvarEvent            = 0x05
	LoadEvent              = 0x06
	SlaveEvent             = 0x07
	CreateFileEvent        = 0x08
	AppendBlockEvent       = 0x09
	ExecLoadEvent          = 0x0a
	DeleteFileEvent        = 0x0b
	NewLoadEvent           = 0x0c
	RandEvent              = 0x0d
	UserVarEvent           = 0x0e
	FormatDescriptionEvent = 0x0f
	XIDEvent               = 0x10
	BeginLoadQueryEvent    = 0x11
	ExecuteLoadQueryEvent  = 0x12
	TableMapEvent          = 0x13
	WriteRowsEventV0       = 0x14
	UpdateRowsEventV0      = 0x15
	DeleteRowsEventV0      = 0x16
	WriteRowsEventV1       = 0x17
	UpdateRowsEventV1      = 0x18
	DeleteRowsEventV1      = 0x19
	IncidentEvent          = 0x1a
	HeartbeatEvent         = 0x1b
	IgnorableEvent         = 0x1c
	RowsQueryEvent         = 0x1d
	WriteRowsEventV2       = 0x1e
	UpdateRowsEventV2      = 0x1f
	DeleteRowsEventV2      = 0x20
	GTIDEvent              = 0x21
	AnonymousGTIDEvent     = 0x22
	PreviousGTIDEvent      = 0x23
)

https://dev.mysql.com/doc/internals/en/binlog-event-type.html

View Source 
const (
	BinlogChecksumAlgOff   byte = 0
	BinlogChecksumAlgCRC32 byte = 1
	BinlogChecksumAlgUndef byte = 255
)

BINGLOG_CHECKSUM_ALG

View Source 
const (
	QFlags2Code            = 0x00
	QSQLModeCode           = 0x01
	QCatalog               = 0x02
	QAutoIncrement         = 0x03
	QCharsetCode           = 0x04
	QTimeZoneCode          = 0x05
	QCatalogNZCode         = 0x06
	QLCTimeNamesCode       = 0x07
	QCharsetDatabaseCode   = 0x08
	QTableMapForUpdateCode = 0x09
	QMasterDataWrittenCode = 0x0a
	QInvokers              = 0x0b
	QUpdatedDBNames        = 0x0c
	QMicroseconds          = 0x0d
)

QUERY_EVENT status_vars

Variables

View Source 
var EventType2Str = map[uint8]string{
	UnknownEvent:           "UNKNOWN_EVENT",
	StartEventV3:           "START_EVENT_V3",
	QueryEvent:             "QUERY_EVENT",
	StopEvent:              "STOP_EVENT",
	RotateEvent:            "ROTATE_EVENT",
	IntvarEvent:            "INTVAR_EVENT",
	LoadEvent:              "LOAD_EVENT",
	SlaveEvent:             "SLAVE_EVENT",
	CreateFileEvent:        "CREATE_FILE_EVENT",
	AppendBlockEvent:       "APPEND_BLOCK_EVENT",
	ExecLoadEvent:          "EXEC_LOAD_EVENT",
	DeleteFileEvent:        "DELETE_FILE_EVENT",
	NewLoadEvent:           "NEW_LOAD_EVENT",
	RandEvent:              "RAND_EVENT",
	UserVarEvent:           "USER_VAR_EVENT",
	FormatDescriptionEvent: "FORMAT_DESCRIPTION_EVENT",
	XIDEvent:               "XID_EVENT",
	BeginLoadQueryEvent:    "BEGIN_LOAD_QUERY_EVENT",
	ExecuteLoadQueryEvent:  "EXECUTE_LOAD_QUERY_EVENT",
	TableMapEvent:          "TABLE_MAP_EVENT",
	WriteRowsEventV0:       "WRITE_ROWS_EVENTv0",
	UpdateRowsEventV0:      "UPDATE_ROWS_EVENTv0",
	DeleteRowsEventV0:      "DELETE_ROWS_EVENTv0",
	WriteRowsEventV1:       "WRITE_ROWS_EVENTv1",
	UpdateRowsEventV1:      "UPDATE_ROWS_EVENTv1",
	DeleteRowsEventV1:      "DELETE_ROWS_EVENTv1",
	IncidentEvent:          "INCIDENT_EVENT",
	HeartbeatEvent:         "HEARTBEAT_EVENT",
	IgnorableEvent:         "IGNORABLE_EVENT",
	RowsQueryEvent:         "ROWS_QUERY_EVENT",
	WriteRowsEventV2:       "WRITE_ROWS_EVENTv2",
	UpdateRowsEventV2:      "UPDATE_ROWS_EVENTv2",
	DeleteRowsEventV2:      "DELETE_ROWS_EVENTv2",
	GTIDEvent:              "GTID_EVENT",
	AnonymousGTIDEvent:     "ANONYMOUS_GTID_EVENT",
	PreviousGTIDEvent:      "PREVIOUS_GTIDS_EVENT",
}

EventType2Str mapping the name of binary log event type

View Source 
var QStatusKey2Str = map[uint8]string{
	QFlags2Code:            "Q_FLAGS2_CODE",
	QSQLModeCode:           "Q_SQL_MODE_CODE",
	QCatalog:               "Q_CATALOG",
	QAutoIncrement:         "Q_AUTO_INCREMENT",
	QCharsetCode:           "Q_CHARSET_CODE",
	QTimeZoneCode:          "Q_TIME_ZONE_CODE",
	QCatalogNZCode:         "Q_CATALOG_NZ_CODE",
	QLCTimeNamesCode:       "Q_LC_TIME_NAMES_CODE",
	QCharsetDatabaseCode:   "Q_CHARSET_DATABASE_CODE",
	QTableMapForUpdateCode: "Q_TABLE_MAP_FOR_UPDATE_CODE",
	QMasterDataWrittenCode: "Q_MASTER_DATA_WRITTEN_CODE",
	QInvokers:              "Q_INVOKERS",
	QUpdatedDBNames:        "Q_UPDATED_DB_NAMES",
	QMicroseconds:          "Q_MICROSECONDS",
}

QStatusKey2Str is the name of status_vars

Functions

func ChecksumValidate 

func ChecksumValidate(checksumType byte, expectedChecksum []byte, data []byte) bool

ChecksumValidate will validate binary log event checksum This information is from 'github.com/siddontang/go-mysql/replication/parser.go' mysql use zlib's CRC32 implementation, which uses polynomial 0xedb88320UL. reference: https://github.com/madler/zlib/blob/master/crc32.c https://github.com/madler/zlib/blob/master/doc/rfc1952.txt#L419

func FixedLengthInt 

func FixedLengthInt(buf []byte) uint64

FixedLengthInt will turn byte to uint64 this function is from 'github.com/siddontang/go-mysql/replication/util.go'

func LengthEncodedInt 

func LengthEncodedInt(b []byte) (num uint64, isNull bool, n int)

LengthEncodedInt will decode byte to uint64 this function is from 'github.com/siddontang/go-mysql/replication/util.go'

func LengthEnodedString 

func LengthEnodedString(b []byte) ([]byte, bool, int, error)

LengthEnodedString will decode bytes

func ReadNBytes 

func ReadNBytes(rd io.Reader, size int64) ([]byte, error)

ReadNBytes read n bytes from io.Reader

Types

type BaseEventBody 

type BaseEventBody struct{}

BaseEventBody is base off all events

type BinEvent 

type BinEvent struct {
	Header       *BinEventHeader
	Body         BinEventBody
	ChecksumType byte
	ChecksumVal  []byte
}

BinEvent binary log event definition

func (*BinEvent) Validation 

func (event *BinEvent) Validation(bin *BinaryLogInfo, header, body []byte) ([]byte, error)

Validation event validity check

type BinEventBody 

type BinEventBody interface {
	// contains filtered or unexported methods
}

BinEventBody describe event body

type BinEventHeader 

type BinEventHeader struct {
	Timestamp int64
	EventType uint8
	ServerID  int64
	EventSize int64
	LogPos    int64
	Flag      uint16
}

BinEventHeader binary log header definition https://dev.mysql.com/doc/internals/en/binlog-event-header.html

func (*BinEventHeader) String 

func (header *BinEventHeader) String() string

String interface implement

func (*BinEventHeader) Type 

func (header *BinEventHeader) Type() string

Type function will translate event type into string

type BinEventUnParsed 

type BinEventUnParsed struct {
	BaseEventBody
	Data []byte
}

BinEventUnParsed save the event data when the event type not supported yet.

type BinFileDecoder 

type BinFileDecoder struct {
	Path string // binary log path

	// binary log reading options
	Option *BinReaderOption

	// file object
	BinFile *os.File

	*BinaryLogInfo
	// contains filtered or unexported fields
}

BinFileDecoder will mapping a binary log file, decode binary log event

func NewBinFileDecoder 

func NewBinFileDecoder(path string, options ...*BinReaderOption) (*BinFileDecoder, error)

NewBinFileDecoder return a BinFileDecoder with binary log file path

func (*BinFileDecoder) DecodeEvent 

func (decoder *BinFileDecoder) DecodeEvent() (*BinEvent, error)

DecodeEvent will decode a single event from binary log

func (*BinFileDecoder) WalkEvent 

func (decoder *BinFileDecoder) WalkEvent(f func(event *BinEvent) (isContinue bool, err error)) error

WalkEvent will walk all events for binary log which in io.Reader This function will return isFinish bool and err error.

type BinFmtDescEvent 

type BinFmtDescEvent struct {
	BaseEventBody
	BinlogVersion     int
	MySQLVersion      string
	CreateTime        int64
	EventHeaderLength int64
	EventTypeHeader   []byte
	// contains filtered or unexported fields
}

BinFmtDescEvent is the definition of FORMAT_DESCRIPTION_EVENT https://dev.mysql.com/doc/internals/en/format-description-event.html

type BinIntvarEvent 

type BinIntvarEvent struct {
	BaseEventBody
	Type  uint8
	Value uint64
}

BinIntvarEvent is the definition of INTVAR_EVENT https://dev.mysql.com/doc/internals/en/xid-event.html Transaction ID for 2PC, written whenever a COMMIT is expected.

type BinPreGTIDsEvent 

type BinPreGTIDsEvent struct{ BaseEventBody }

BinPreGTIDsEvent is the definition of PREVIOUS_GTIDS_EVENT TODO: PREVIOUS_GTIDS_EVENT

type BinQueryEvent 

type BinQueryEvent struct {
	BaseEventBody
	SlaveProxyID  int64
	ExecutionTime int64
	ErrorCode     uint16

	StatusVars []byte
	Schema     string
	Query      string
	// contains filtered or unexported fields
}

BinQueryEvent is the definition of QUERY_EVENT https://dev.mysql.com/doc/internals/en/query-event.html

func (*BinQueryEvent) Statue 

func (event *BinQueryEvent) Statue() error

Statue will format status_vars of QUERY_EVENT TODO decode QUERY_EVENT status_var

type BinReaderOption 

type BinReaderOption struct {
	StartPos  int64
	EndPos    int64
	StartTime time.Time
	EndTime   time.Time
}

BinReaderOption will describe the details to tell decoders when it should start and when stop. with time [start, end)

func (*BinReaderOption) Start 

func (o *BinReaderOption) Start(header *BinEventHeader) bool

Start return bool of if start decoding

func (*BinReaderOption) Stop 

func (o *BinReaderOption) Stop(header *BinEventHeader) bool

Stop return bool of if stop decoding

type BinRotateEvent 

type BinRotateEvent struct {
	BaseEventBody
	Position uint64
	FileName string
}

BinRotateEvent is the definition of ROTATE_EVENT https://dev.mysql.com/doc/internals/en/rotate-event.html The rotate event is added to the binlog as last event to tell the reader what binlog to request next.

type BinRowsEvent 

type BinRowsEvent struct {
	BaseEventBody
	// header
	Version int
	TableID uint64

	Flags uint16
	// if version == 2
	ExtraData []byte

	// body
	ColumnCount    uint64
	ColumnsBitmap1 []byte
	// if UPDATE_ROWS_EVENTv1 or v2
	ColumnsBitmap2 []byte
	// contains filtered or unexported fields
}

BinRowsEvent describe MySQL ROWS_EVENT https://dev.mysql.com/doc/internals/en/rows-event.html

func (*BinRowsEvent) Init 

func (e *BinRowsEvent) Init(h *BinFmtDescEvent, eventType uint8) *BinRowsEvent

Init BinRowsEvent, adding version and table_id length

type BinTableMapEvent 

type BinTableMapEvent struct {
	BaseEventBody
	TableID uint64

	Flags         uint16
	Schema        string
	Table         string
	ColumnCount   uint64
	ColumnTypeDef []byte
	ColumnMetaDef []uint16
	NullBitmap    []byte
	// contains filtered or unexported fields
}

BinTableMapEvent is the definition of TABLE_MAP_EVENT https://dev.mysql.com/doc/internals/en/table-map-event.html

func (*BinTableMapEvent) Init 

func (e *BinTableMapEvent) Init(h *BinFmtDescEvent) *BinTableMapEvent

Init BinTableMapEvent tableIDLen

type BinXIDEvent 

type BinXIDEvent struct {
	BaseEventBody
	XID uint64
}

BinXIDEvent is the definition of XID_EVENT https://dev.mysql.com/doc/internals/en/xid-event.html Transaction ID for 2PC, written whenever a COMMIT is expected.

type BinaryLogInfo 

type BinaryLogInfo struct {
	// contains filtered or unexported fields
}

BinaryLogInfo is the base decoder of all types

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.