dbsyncer

Documentation

Overview

Package dbsyncer implements blockchain data synchronisation to relational database.

Source Files

• config.go : includes configurations, mostly related to the connected database
• dbsync.go : implements data synchronisation operations
• dbsync_context.go : provides context for chain event, block header, transactions and bulk inserts
• dbsync_multi.go : supports parallel synchronisation
• gen_config.go : is automatically generated from config.go
• query_engine.go : supports query level requests and results
• tx_record.go : manages transaction data handling
• utils.go : includes utility functions for dbsyncer package

Index

• Constants
• Variables
• func MakeSummaryDBRow(sa SummaryArguments) (cols string, vals []interface{}, count int, err error)
• func MakeTxDBRow(block *types.Block, txKey uint64, tx *types.Transaction, ...) (string, []interface{}, TxMapArguments, SummaryArguments, error)
• func MakeTxMappingRow(tm TxMapArguments) (cols string, vals []interface{}, count int, err error)
• type BulkInsertQuery
• type BulkInsertRequest
• type BulkInsertResult
• type DBConfig
  • func (d DBConfig) MarshalTOML() (interface{}, error)
  • func (d *DBConfig) UnmarshalTOML(unmarshal func(interface{}) error) error
• type DBSyncer
  • func NewDBSyncer(ctx *node.ServiceContext, cfg *DBConfig) (*DBSyncer, error)
  • func (ds *DBSyncer) APIs() []rpc.API
  • func (ds *DBSyncer) Components() []interface{}
  • func (ds *DBSyncer) HandleChainEvent(block *types.Block) error
  • func (ds *DBSyncer) HandleChainEventContext(block *types.Block) error
  • func (ds *DBSyncer) HandleChainEventParallel(block *types.Block) error
  • func (ds *DBSyncer) HandleDiffBlock(block *types.Block)
  • func (ds *DBSyncer) HandleLogsEvent(logs []*types.Log) error
  • func (ds *DBSyncer) Ping()
  • func (ds *DBSyncer) Protocols() []p2p.Protocol
  • func (ds *DBSyncer) SetComponents(components []interface{})
  • func (ds *DBSyncer) Start(server p2p.Server) error
  • func (ds *DBSyncer) Stop() error
  • func (ds *DBSyncer) SyncTransactions(block *types.Block) error
• type MakeQueryRequest
• type MakeQueryResult
• type QueryEngine
• type SummaryArguments
• type TxMapArguments
• type TxReceiptArgument

Constants

const (
	BLOCK_MODE = "block"
	HEAD_MODE  = "head"
)

const (
	// mysql driver (go) has query parameter max size (65535)
	// transaction record has 15 parameters
	BULK_INSERT_SIZE = 3000
	TX_KEY_FACTOR    = 100000
)

Variables

var DefaultDBConfig = &DBConfig{

	EnabledDBSyncer: false,
	EnabledLogMode:  false,

	DBPort: "3306",

	MaxIdleConns:     50,
	MaxOpenConns:     30,
	ConnMaxLifetime:  1 * time.Hour,
	BlockChannelSize: 5,

	GenQueryThread: 100,
	InsertThread:   30,

	BulkInsertSize: 200,

	Mode:      "multi",
	EventMode: HEAD_MODE,

	MaxBlockDiff: 0,
}

Functions

func MakeSummaryDBRow

func MakeSummaryDBRow(sa SummaryArguments) (cols string, vals []interface{}, count int, err error)

func MakeTxDBRow

func MakeTxDBRow(block *types.Block, txKey uint64, tx *types.Transaction, receipt *types.Receipt) (string, []interface{}, TxMapArguments, SummaryArguments, error)

func MakeTxMappingRow

func MakeTxMappingRow(tm TxMapArguments) (cols string, vals []interface{}, count int, err error)

Types

type BulkInsertQuery

type BulkInsertQuery struct {
	// contains filtered or unexported fields
}

type BulkInsertRequest

type BulkInsertRequest struct {
	// contains filtered or unexported fields
}

type BulkInsertResult

type BulkInsertResult struct {
	// contains filtered or unexported fields
}

type DBConfig

type DBConfig struct {
	EnabledDBSyncer bool
	EnabledLogMode  bool

	// DB Config
	DBHost     string `toml:",omitempty"`
	DBPort     string `toml:",omitempty"`
	DBUser     string `toml:",omitempty"`
	DBPassword string `toml:",omitempty"`
	DBName     string `toml:",omitempty"`

	MaxIdleConns     int           `toml:",omitempty"`
	MaxOpenConns     int           `toml:",omitempty"`
	ConnMaxLifetime  time.Duration `toml:",omitempty"`
	BlockChannelSize int           `toml:",omitempty"`

	GenQueryThread int `toml:",omitempty"`
	InsertThread   int `toml:",omitempty"`

	BulkInsertSize int `toml:",omitempty"`

	Mode      string `toml:",omitempty"`
	EventMode string `toml:",omitempty"`

	MaxBlockDiff uint64 `toml:",omitempty"`
}

func (DBConfig) MarshalTOML

func (d DBConfig) MarshalTOML() (interface{}, error)

MarshalTOML marshals as TOML.

func (*DBConfig) UnmarshalTOML

func (d *DBConfig) UnmarshalTOML(unmarshal func(interface{}) error) error

UnmarshalTOML unmarshals from TOML.

type DBSyncer

type DBSyncer struct {
	HandleBlock func(block *types.Block) error
	// contains filtered or unexported fields
}

func NewDBSyncer

func NewDBSyncer(ctx *node.ServiceContext, cfg *DBConfig) (*DBSyncer, error)

func (*DBSyncer) APIs

func (ds *DBSyncer) APIs() []rpc.API

func (*DBSyncer) Components

func (ds *DBSyncer) Components() []interface{}

func (*DBSyncer) HandleChainEvent

func (ds *DBSyncer) HandleChainEvent(block *types.Block) error

func (*DBSyncer) HandleChainEventContext

func (ds *DBSyncer) HandleChainEventContext(block *types.Block) error

HandleChainEventContext supports 2PC Commit (insert block + insert txs) for data consistency @TODO-Klaytn improve performance, too slower than HanleChainEvent()

func (*DBSyncer) HandleChainEventParallel

func (ds *DBSyncer) HandleChainEventParallel(block *types.Block) error

func (*DBSyncer) HandleDiffBlock

func (ds *DBSyncer) HandleDiffBlock(block *types.Block)

func (*DBSyncer) HandleLogsEvent

func (ds *DBSyncer) HandleLogsEvent(logs []*types.Log) error

func (*DBSyncer) Ping

func (ds *DBSyncer) Ping()

func (*DBSyncer) Protocols

func (ds *DBSyncer) Protocols() []p2p.Protocol

func (*DBSyncer) SetComponents

func (ds *DBSyncer) SetComponents(components []interface{})

func (*DBSyncer) Start

func (ds *DBSyncer) Start(server p2p.Server) error

func (*DBSyncer) Stop

func (ds *DBSyncer) Stop() error

func (*DBSyncer) SyncTransactions

func (ds *DBSyncer) SyncTransactions(block *types.Block) error

type MakeQueryRequest

type MakeQueryRequest struct {
	// contains filtered or unexported fields
}

type MakeQueryResult

type MakeQueryResult struct {
	// contains filtered or unexported fields
}

type QueryEngine

type QueryEngine struct {
	// contains filtered or unexported fields
}

QueryEngine is a helper structure to concurrently making insert query

type SummaryArguments

type SummaryArguments struct {
	// contains filtered or unexported fields
}

type TxMapArguments

type TxMapArguments struct {
	// contains filtered or unexported fields
}

type TxReceiptArgument

type TxReceiptArgument struct {
	// contains filtered or unexported fields
}