dbutils

package
v0.0.0-...-1f13f73 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 4, 2021 License: GPL-3.0 Imports: 8 Imported by: 0

Documentation

Index

Constants

View Source
const (

	//HashedAccountsBucket
	// key - address hash
	// value - account encoded for storage
	// Contains Storage:
	//key - address hash + incarnation + storage key hash
	//value - storage value(common.hash)
	HashedAccountsBucket   = "hashed_accounts"
	HashedStorageBucket    = "hashed_storage"
	CurrentStateBucketOld2 = "CST2"
)
View Source
const (
	// DatabaseInfoBucket is used to store information about data layout.
	DatabaseInfoBucket        = "DBINFO"
	SnapshotInfoBucket        = "SNINFO"
	BittorrentInfoBucket      = "BTINFO"
	HeadersSnapshotInfoBucket = "hSNINFO"
	BodiesSnapshotInfoBucket  = "bSNINFO"
	StateSnapshotInfoBucket   = "sSNINFO"

	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
	HeaderPrefixOld    = "h" // block_num_u64 + hash -> header
	HeaderNumberBucket = "H" // headerNumberPrefix + hash -> num (uint64 big endian)

	HeaderCanonicalBucket = "canonical_headers" // block_num_u64 -> header hash
	HeadersBucket         = "headers"           // block_num_u64 + hash -> header (RLP)
	HeaderTDBucket        = "header_to_td"      // block_num_u64 + hash -> td (RLP)

	BlockBodyPrefix     = "b"      // block_num_u64 + hash -> block body
	EthTx               = "eth_tx" // tbl_sequence_u64 -> rlp(tx)
	BlockReceiptsPrefix = "r"      // block_num_u64 -> canonical block receipts (non-canonical are not stored)
	Log                 = "log"    // block_num_u64 + txId -> logs of transaction

	// Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic'
	// [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN)
	// indices are sharded - because some bitmaps are >1Mb and when new incoming blocks process it
	//	 updates ~300 of bitmaps - by append small amount new values. It cause much big writes (LMDB does copy-on-write).
	//
	// if last existing shard size merge it with delta
	// if serialized size of delta > ShardLimit - break down to multiple shards
	// shard number - it's biggest value in bitmap
	LogTopicIndex   = "log_topic_index"
	LogAddressIndex = "log_address_index"

	// CallTraceSet is the name of the table that contain the mapping of block number to the set (sorted) of all accounts
	// touched by call traces. It is DupSort-ed table
	// 8-byte BE block nunber -> account address -> two bits (one for "from", another for "to")
	CallTraceSet = "call_trace_set"
	// Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex
	// Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses
	CallFromIndex = "call_from_index"
	CallToIndex   = "call_to_index"

	TxLookupPrefix  = "l" // hash -> transaction/receipt lookup metadata
	BloomBitsPrefix = "B" // bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits

	PreimagePrefix = "secure-key-"      // preimagePrefix + hash -> preimage
	ConfigPrefix   = "ethereum-config-" // config prefix for the db

	// Chain index prefixes (use `i` + single byte to avoid mixing data types).
	BloomBitsIndexPrefix = "iB" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress

	// Progress of sync stages: stageName -> stageData
	SyncStageProgress     = "SSP2"
	SyncStageProgressOld1 = "SSP"
	// Position to where to unwind sync stages: stageName -> stageData
	SyncStageUnwind     = "SSU2"
	SyncStageUnwindOld1 = "SSU"

	CliqueBucket             = "clique-"
	CliqueSeparateBucket     = "clique-snapshots-"
	CliqueSnapshotBucket     = "snap"
	CliqueLastSnapshotBucket = "lastSnap"

	// this bucket stored in separated database
	InodesBucket = "inodes"

	// Transaction senders - stored separately from the block bodies
	Senders = "txSenders" // block_num_u64 + blockHash -> sendersList (no serialization format, every 20 bytes is new sender)

	// headBlockKey tracks the latest know full block's hash.
	HeadBlockKey = "LastBlock"

	// migrationName -> serialized SyncStageProgress and SyncStageUnwind buckets
	// it stores stages progress to understand in which context was executed migration
	// in case of bug-report developer can ask content of this bucket
	Migrations = "migrations"

	Sequence      = "sequence" // tbl_name -> seq_u64
	HeadHeaderKey = "LastHeader"
)
View Source
const AccountChangeSetBucket = "PLAIN-ACS"

AccountChangeSetBucket and StorageChangeSetBucket - of block N store values of state before block N changed them. Because values "after" change stored in PlainState. Logical format:

key - blockNum_u64 + key_in_plain_state
value - value_in_plain_state_before_blockNum_changes

Example: If block N changed account A from value X to Y. Then:

AccountChangeSetBucket has record: bigEndian(N) + A -> X
PlainStateBucket has record: A -> Y

See also: docs/programmers_guide/db_walkthrough.MD#table-history-of-accounts

As you can see if block N changes much accounts - then all records have repetitive prefix `bigEndian(N)`. MDBX can store such prefixes only once - by DupSort feature (see `docs/programmers_guide/dupsort.md`). Both buckets are DupSort-ed and have physical format: AccountChangeSetBucket:

key - blockNum_u64
value - address + account(encoded)

StorageChangeSetBucket:

key - blockNum_u64 + address + incarnation_u64
value - plain_storage_key + value
View Source
const IntermediateTrieHashBucketOld2 = "iTh2"
View Source
const NumberLength = 8
View Source
const PlainContractCodeBucket = "PLAIN-contractCode"

PlainContractCodeBucket - key - address+incarnation value - code hash

View Source
const PlainStateBucket = "PLAIN-CST2"

PlainStateBucket logical layout:

Contains Accounts:
  key - address (unhashed)
  value - account encoded for storage
Contains Storage:
  key - address (unhashed) + incarnation + storage key (unhashed)
  value - storage value(common.hash)

Physical layout:

PlainStateBucket and HashedStorageBucket utilises DupSort feature of LMDB (store multiple values inside 1 key).

-------------------------------------------------------------

key              |            value

------------------------------------------------------------- [acc_hash] | [acc_value] [acc_hash]+[inc] | [storage1_hash]+[storage1_value]

| [storage2_hash]+[storage2_value] // this value has no own key. it's 2nd value of [acc_hash]+[inc] key.
| [storage3_hash]+[storage3_value]
| ...

[acc_hash]+[old_inc] | [storage1_hash]+[storage1_value]

| ...

[acc2_hash] | [acc2_value]

...
View Source
const PlainStateBucketOld1 = "PLAIN-CST"
View Source
const StorageChangeSetBucket = "PLAIN-SCS"
View Source
const TrieOfAccountsBucket = "trie_account"

TrieOfAccountsBucket and TrieOfStorageBucket hasState,groups - mark prefixes existing in hashed_account table hasTree - mark prefixes existing in trie_account table (not related with branchNodes) hasHash - mark prefixes which hashes are saved in current trie_account record (actually only hashes of branchNodes can be saved) @see UnmarshalTrieNode @see integrity.Trie

+-----------------------------------------------------------------------------------------------------+ | DB record: 0x0B, hasState: 0b1011, hasTree: 0b1001, hasHash: 0b1001, hashes: [x,x] | +-----------------------------------------------------------------------------------------------------+

|                                           |                               |
v                                           |                               v

+---------------------------------------------+ | +--------------------------------------+ | DB record: 0x0B00, hasState: 0b10001 | | | DB record: 0x0B03, hasState: 0b10010 | | hasTree: 0, hasHash: 0b10000, hashes: [x] | | | hasTree: 0, hasHash: 0, hashes: [] | +---------------------------------------------+ | +--------------------------------------+

|                    |                              |                         |                  |
v                    v                              v                         v                  v

+------------------+ +----------------------+ +---------------+ +---------------+ +---------------+ | Account: | | BranchNode: 0x0B0004 | | Account: | | Account: | | Account: | | 0x0B0000... | | has no record in | | 0x0B01... | | 0x0B0301... | | 0x0B0304... | | in HashedAccount | | TrieAccount | | | | | | | +------------------+ +----------------------+ +---------------+ +---------------+ +---------------+

                           |                |
                           v                v
		           +---------------+  +---------------+
		           | Account:      |  | Account:      |
		           | 0x0B000400... |  | 0x0B000401... |
		           +---------------+  +---------------+

Invariants: - hasTree is subset of hasState - hasHash is subset of hasState - first level in account_trie always exists if hasState>0 - TrieStorage record of account.root (length=40) must have +1 hash - it's account.root - each record in TrieAccount table must have parent (may be not direct) and this parent must have correct bit in hasTree bitmap - if hasState has bit - then HashedAccount table must have record according to this bit - each TrieAccount record must cover some state (means hasState is always > 0) - TrieAccount records with length=1 can satisfy (hasBranch==0&&hasHash==0) condition - Other records in TrieAccount and TrieStorage must (hasTree!=0 || hasHash!=0)

View Source
const TrieOfStorageBucket = "trie_storage"

Variables

View Source
var (

	//key - contract code hash
	//value - contract code
	CodeBucket = "CODE"

	//key - addressHash+incarnation
	//value - code hash
	ContractCodeBucket = "contractCode"

	// IncarnationMapBucket for deleted accounts
	//key - address
	//value - incarnation of account when it was last deleted
	IncarnationMapBucket = "incarnationMap"

	//TEVMCodeStatusBucket -
	//key - encoded timestamp(block number)
	//value - contract codes hashes: [code_hash1]+[code_hash2]
	ContractTEVMCodeStatusBucket = "TEVMCodeStatus"

	//TEVMCodeBucket -
	//key - contract code hash
	//value - contract EVTM code
	ContractTEVMCodeBucket = "TEVMCode"
)
View Source
var (
	//StorageModeHistory - does node save history.
	StorageModeHistory = []byte("smHistory")
	//StorageModeReceipts - does node save receipts.
	StorageModeReceipts = []byte("smReceipts")
	//StorageModeTxIndex - does node save transactions index.
	StorageModeTxIndex = []byte("smTxIndex")
	//StorageModeCallTraces - does not build index of call traces
	StorageModeCallTraces = []byte("smCallTraces")
	//StorageModeTEVM - does not translate EVM to TEVM
	StorageModeTEVM = []byte("smTEVM")

	DBSchemaVersionKey = []byte("dbVersion")

	SnapshotHeadersHeadNumber = "SnapshotLastHeaderNumber"
	SnapshotHeadersHeadHash   = "SnapshotLastHeaderHash"
	SnapshotBodyHeadNumber    = "SnapshotLastBodyNumber"
	SnapshotBodyHeadHash      = "SnapshotLastBodyHash"

	BittorrentPeerID            = "peerID"
	CurrentHeadersSnapshotHash  = []byte("CurrentHeadersSnapshotHash")
	CurrentHeadersSnapshotBlock = []byte("CurrentHeadersSnapshotBlock")
)

Keys

View Source
var AccountsHistoryBucket = "hAT"

AccountsHistoryBucket and StorageHistoryBucket - indices designed to serve next 2 type of requests: 1. what is smallest block number >= X where account A changed 2. get last shard of A - to append there new block numbers

Task 1. is part of "get historical state" operation (see `core/state:GetAsOf`): If `db.Seek(A+bigEndian(X))` returns non-last shard -

then get block number from shard value Y := RoaringBitmap(shard_value).GetGte(X)
and with Y go to ChangeSets: db.Get(ChangeSets, Y+A)

If `db.Seek(A+bigEndian(X))` returns last shard -

then we go to PlainState: db.Get(PlainState, A)

Format:

  • index split to shards by 2Kb - RoaringBitmap encoded sorted list of block numbers (to avoid performance degradation of popular accounts or look deep into history. Also 2Kb allows avoid Overflow pages inside DB.)
  • if shard is not last - then key has suffix 8 bytes = bigEndian(max_block_num_in_this_shard)
  • if shard is last - then key has suffix 8 bytes = 0xFF

It allows:

  • server task 1. by 1 db operation db.Seek(A+bigEndian(X))
  • server task 2. by 1 db operation db.Get(A+0xFF)

see also: docs/programmers_guide/db_walkthrough.MD#table-change-sets

AccountsHistoryBucket:

key - address + shard_id_u64
value - roaring bitmap  - list of block where it changed

StorageHistoryBucket

key - address + storage_key + shard_id_u64
value - roaring bitmap - list of block where it changed

Buckets - list of all buckets. App will panic if some bucket is not in this list. This list will be sorted in `init` method. BucketsConfigs - can be used to find index in sorted version of Buckets list by name

View Source
var BucketsConfigs = BucketsCfg{
	CurrentStateBucketOld2: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                72,
		DupToLen:                  40,
	},
	HashedStorageBucket: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                72,
		DupToLen:                  40,
	},
	AccountChangeSetBucket: {
		Flags: DupSort,
	},
	StorageChangeSetBucket: {
		Flags: DupSort,
	},
	PlainStateBucket: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                60,
		DupToLen:                  28,
	},
	IntermediateTrieHashBucketOld2: {
		Flags:               DupSort,
		CustomDupComparator: DupCmpSuffix32,
	},
	CallTraceSet: {
		Flags: DupSort,
	},
	ContractTEVMCodeStatusBucket: {
		Flags: DupSort,
	},
}
View Source
var DBSchemaVersionLMDB = types.VersionReply{Major: 1, Minor: 1, Patch: 0}

DBSchemaVersion

View Source
var DBSchemaVersionMDBX = types.VersionReply{Major: 2, Minor: 1, Patch: 0}

DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration

View Source
var ErrInvalidSize = errors.New("bit endian number has an invalid size")
View Source
var Rename = map[string]string{
	PlainStateBucket:          "PlainState",
	PlainContractCodeBucket:   "PlainCodeHash",
	AccountChangeSetBucket:    "AccountChangeSet",
	StorageChangeSetBucket:    "StorageChangeSet",
	HashedAccountsBucket:      "HashedAccount",
	HashedStorageBucket:       "HashedStorage",
	AccountsHistoryBucket:     "AccountHistory",
	StorageHistoryBucket:      "StorageHistory",
	CodeBucket:                "Code",
	ContractCodeBucket:        "HashedCodeHash",
	IncarnationMapBucket:      "IncarnationMap",
	TrieOfAccountsBucket:      "TrieAccount",
	TrieOfStorageBucket:       "TrieStorage",
	DatabaseInfoBucket:        "DbInfo",
	SnapshotInfoBucket:        "SnapshotInfo",
	BittorrentInfoBucket:      "BittorrentInfo",
	HeadersSnapshotInfoBucket: "HeadersSnapshotInfo",
	BodiesSnapshotInfoBucket:  "BodiesSnapshotInfo",
	StateSnapshotInfoBucket:   "StateSnapshotInfo",
	HeaderNumberBucket:        "HeaderNumber",
	HeaderCanonicalBucket:     "CanonicalHeader",
	HeadersBucket:             "Header",
	HeaderTDBucket:            "HeadersTotalDifficulty",
	BlockBodyPrefix:           "BlockBody",
	EthTx:                     "BlockTransaction",
	BlockReceiptsPrefix:       "Receipt",
	Log:                       "TransactionLog",
	LogTopicIndex:             "LogTopicIndex",
	LogAddressIndex:           "LogAddressIndex",
	CallTraceSet:              "CallTraceSet",
	CallFromIndex:             "CallFromIndex",
	CallToIndex:               "CallToIndex",
	TxLookupPrefix:            "BlockTransactionLookup",
	BloomBitsPrefix:           "BloomBits",
	PreimagePrefix:            "Preimage",
	ConfigPrefix:              "Config",
	BloomBitsIndexPrefix:      "BloomBitsIndex",
	SyncStageProgress:         "SyncStage",
	SyncStageUnwind:           "SyncStageUnwind",
	CliqueBucket:              "Clique",
	CliqueSeparateBucket:      "CliqueSeparate",
	CliqueSnapshotBucket:      "CliqueSnapshot",
	CliqueLastSnapshotBucket:  "CliqueLastSnapshot",
	InodesBucket:              "Inode",
	Senders:                   "TxSender",
	HeadBlockKey:              "LastBlock",
	Migrations:                "Migration",
	Sequence:                  "Sequence",
	HeadHeaderKey:             "LastHeader",
}
View Source
var StorageHistoryBucket = "hST"

Functions

func AccountIndexChunkKey

func AccountIndexChunkKey(key []byte, blockNumber uint64) []byte

func BlockBodyKey

func BlockBodyKey(number uint64, hash common.Hash) []byte

BlockBodyKey = num (uint64 big endian) + hash

func BloomBitsKey

func BloomBitsKey(bit uint, section uint64, hash common.Hash) []byte

bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash

func CompositeKeySuffix

func CompositeKeySuffix(key []byte, timestamp uint64) (composite, encodedTS []byte)

Key + blockNum

func CompositeKeyWithoutIncarnation

func CompositeKeyWithoutIncarnation(key []byte) []byte

func DecodeBlockNumber

func DecodeBlockNumber(number []byte) (uint64, error)

func DecodeTimestamp

func DecodeTimestamp(suffix []byte) (uint64, []byte)

func DefaultCmpFunc

func DefaultCmpFunc(k1, k2, v1, v2 []byte) int

func DefaultDupCmpFunc

func DefaultDupCmpFunc(k1, k2, v1, v2 []byte) int

func EncodeBlockNumber

func EncodeBlockNumber(number uint64) []byte

EncodeBlockNumber encodes a block number as big endian uint64

func EncodeTimestamp

func EncodeTimestamp(timestamp uint64) []byte

EncodeTimestamp has the property: if a < b, then Encoding(a) < Encoding(b) lexicographically

func GenerateCompositeStorageKey

func GenerateCompositeStorageKey(addressHash common.Hash, incarnation uint64, seckey common.Hash) []byte

AddrHash + incarnation + KeyHash For contract storage

func GenerateCompositeStoragePrefix

func GenerateCompositeStoragePrefix(addressHash []byte, incarnation uint64, storageHashPrefix []byte) []byte

AddrHash + incarnation + StorageHashPrefix

func GenerateCompositeTrieKey

func GenerateCompositeTrieKey(addressHash common.Hash, seckey common.Hash) []byte

AddrHash + KeyHash Only for trie

func GenerateStoragePrefix

func GenerateStoragePrefix(addressHash []byte, incarnation uint64) []byte

address hash + incarnation prefix

func HeaderKey

func HeaderKey(number uint64, hash common.Hash) []byte

HeaderKey = num (uint64 big endian) + hash

func LogKey

func LogKey(blockNumber uint64, txId uint32) []byte

LogKey = blockN (uint64 big endian) + txId (uint32 big endian)

func NextNibblesSubtree

func NextNibblesSubtree(in []byte, out *[]byte) bool

NextNibblesSubtree does []byte++. Returns false if overflow.

func NextSubtree

func NextSubtree(in []byte) ([]byte, bool)

NextSubtree does []byte++. Returns false if overflow.

func ParseCompositeStorageKey

func ParseCompositeStorageKey(compositeKey []byte) (common.Hash, uint64, common.Hash)

func ParseStoragePrefix

func ParseStoragePrefix(prefix []byte) (common.Hash, uint64)

func PlainGenerateCompositeStorageKey

func PlainGenerateCompositeStorageKey(address []byte, incarnation uint64, key []byte) []byte

AddrHash + incarnation + KeyHash For contract storage (for plain state)

func PlainGenerateStoragePrefix

func PlainGenerateStoragePrefix(address []byte, incarnation uint64) []byte

address hash + incarnation prefix (for plain state)

func PlainParseCompositeStorageKey

func PlainParseCompositeStorageKey(compositeKey []byte) (common.Address, uint64, common.Hash)

func PlainParseStoragePrefix

func PlainParseStoragePrefix(prefix []byte) (common.Address, uint64)

func ReceiptsKey

func ReceiptsKey(blockNumber uint64) []byte

ReceiptsKey = blockN (uint64 big endian)

func StorageIndexChunkKey

func StorageIndexChunkKey(key []byte, blockNumber uint64) []byte

func UpdateBucketsList

func UpdateBucketsList(newBucketCfg BucketsCfg)

Types

type Bucket

type Bucket string

type BucketConfigItem

type BucketConfigItem struct {
	Flags BucketFlags
	// AutoDupSortKeysConversion - enables some keys transformation - to change db layout without changing app code.
	// Use it wisely - it helps to do experiments with DB format faster, but better reduce amount of Magic in app.
	// If good DB format found, push app code to accept this format and then disable this property.
	AutoDupSortKeysConversion bool
	IsDeprecated              bool
	DBI                       DBI
	// DupFromLen - if user provide key of this length, then next transformation applied:
	// v = append(k[DupToLen:], v...)
	// k = k[:DupToLen]
	// And opposite at retrieval
	// Works only if AutoDupSortKeysConversion enabled
	DupFromLen          int
	DupToLen            int
	CustomComparator    CustomComparator
	CustomDupComparator CustomComparator
}

type BucketFlags

type BucketFlags uint
const (
	Default    BucketFlags = 0x00
	ReverseKey BucketFlags = 0x02
	DupSort    BucketFlags = 0x04
	IntegerKey BucketFlags = 0x08
	IntegerDup BucketFlags = 0x20
	ReverseDup BucketFlags = 0x40
)

type BucketsCfg

type BucketsCfg map[string]BucketConfigItem

func DefaultBuckets

func DefaultBuckets() BucketsCfg

type CmpFunc

type CmpFunc func(k1, k2, v1, v2 []byte) int

type CustomComparator

type CustomComparator string
const (
	DefaultCmp     CustomComparator = ""
	DupCmpSuffix32 CustomComparator = "dup_cmp_suffix32"
)

type DBI

type DBI uint

type Suffix

type Suffix []byte

func ToSuffix

func ToSuffix(b []byte) Suffix

func (Suffix) Add

func (s Suffix) Add(key []byte) Suffix

func (Suffix) KeyCount

func (s Suffix) KeyCount() uint32

func (Suffix) MultiAdd

func (s Suffix) MultiAdd(keys [][]byte) Suffix

func (Suffix) Walk

func (s Suffix) Walk(f func(k []byte) error) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL