dbutils

package
v0.0.0-...-140c642 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 24, 2021 License: GPL-3.0 Imports: 9 Imported by: 0

Documentation

Index

Constants

View Source
const (
	ItemLen      = 3
	MaxChunkSize = 1000
)

Variables

View Source
var (
	//PlainContractCodeBucket -
	//key - address+incarnation
	//value - code hash
	PlainContractCodeBucket = "PLAIN-contractCode"

	// PlainAccountChangeSetBucket keeps changesets of accounts ("plain state")
	// key - encoded timestamp(block number)
	// value - encoded ChangeSet{k - address v - account(encoded).
	PlainAccountChangeSetBucket = "PLAIN-ACS"

	// PlainStorageChangeSetBucket keeps changesets of storage ("plain state")
	// key - encoded timestamp(block number)
	// value - encoded ChangeSet{k - plainCompositeKey(for storage) v - originalValue(common.Hash)}.
	PlainStorageChangeSetBucket = "PLAIN-SCS"

	//HashedAccountsBucket
	// key - address hash
	// value - account encoded for storage
	// Contains Storage:
	//key - address hash + incarnation + storage key hash
	//value - storage value(common.hash)
	HashedAccountsBucket   = "hashed_accounts"
	HashedStorageBucket    = "hashed_storage"
	CurrentStateBucketOld2 = "CST2"
	CurrentStateBucketOld1 = "CST"

	//key - address + shard_id_u64
	//value - roaring bitmap  - list of block where it changed
	AccountsHistoryBucket = "hAT"

	//key - address + storage_key + shard_id_u64
	//value - roaring bitmap - list of block where it changed
	StorageHistoryBucket = "hST"

	//key - contract code hash
	//value - contract code
	CodeBucket = "CODE"

	//key - addressHash+incarnation
	//value - code hash
	ContractCodeBucket = "contractCode"

	// IncarnationMapBucket for deleted accounts
	//key - address
	//value - incarnation of account when it was last deleted
	IncarnationMapBucket = "incarnationMap"
)
View Source
var (
	// DatabaseInfoBucket is used to store information about data layout.
	DatabaseInfoBucket        = "DBINFO"
	SnapshotInfoBucket        = "SNINFO"
	HeadersSnapshotInfoBucket = "hSNINFO"
	BodiesSnapshotInfoBucket  = "bSNINFO"
	StateSnapshotInfoBucket   = "sSNINFO"

	// databaseVerisionKey tracks the current database version.
	DatabaseVerisionKey = "DatabaseVersion"

	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
	HeaderPrefix       = "h"         // block_num_u64 + hash -> header
	HeaderTDSuffix     = []byte("t") // block_num_u64 + hash + headerTDSuffix -> td
	HeaderHashSuffix   = []byte("n") // block_num_u64 + headerHashSuffix -> hash
	HeaderNumberPrefix = "H"         // headerNumberPrefix + hash -> num (uint64 big endian)

	BlockBodyPrefix     = "b"      // block_num_u64 + hash -> block body
	EthTx               = "eth_tx" // tbl_sequence_u64 -> rlp(tx)
	BlockReceiptsPrefix = "r"      // block_num_u64 + hash -> block receipts
	Log                 = "log"    // block_num_u64 + hash -> block receipts

	// Stores bitmap indices - in which block numbers saw logs of given 'address' or 'topic'
	// [addr or topic] + [2 bytes inverted shard number] -> bitmap(blockN)
	// indices are sharded - because some bitmaps are >1Mb and when new incoming blocks process it
	//	 updates ~300 of bitmaps - by append small amount new values. It cause much big writes (LMDB does copy-on-write).
	//
	// if last existing shard size merge it with delta
	// if serialized size of delta > ShardLimit - break down to multiple shards
	// shard number - it's biggest value in bitmap
	LogTopicIndex   = "log_topic_index"
	LogAddressIndex = "log_address_index"

	// Indices for call traces - have the same format as LogTopicIndex and LogAddressIndex
	// Store bitmap indices - in which block number we saw calls from (CallFromIndex) or to (CallToIndex) some addresses
	CallFromIndex = "call_from_index"
	CallToIndex   = "call_to_index"

	TxLookupPrefix  = "l" // txLookupPrefix + hash -> transaction/receipt lookup metadata
	BloomBitsPrefix = "B" // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits

	PreimagePrefix = "secure-key-"      // preimagePrefix + hash -> preimage
	ConfigPrefix   = "ethereum-config-" // config prefix for the db

	// Chain index prefixes (use `i` + single byte to avoid mixing data types).
	BloomBitsIndexPrefix = "iB" // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress

	// Progress of sync stages: stageName -> stageData
	SyncStageProgress     = "SSP2"
	SyncStageProgressOld1 = "SSP"
	// Position to where to unwind sync stages: stageName -> stageData
	SyncStageUnwind     = "SSU2"
	SyncStageUnwindOld1 = "SSU"

	CliqueBucket = "clique-"

	// this bucket stored in separated database
	InodesBucket = "inodes"

	// Transaction senders - stored separately from the block bodies
	Senders = "txSenders"

	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
	FastTrieProgressKey = "TrieSync"
	// headBlockKey tracks the latest know full block's hash.
	HeadBlockKey = "LastBlock"
	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
	HeadFastBlockKey = "LastFast"

	InvalidBlock    = "InvalidBlock"     // Inherited from go-ethereum, not used in turbo-geth yet
	UncleanShutdown = "unclean-shutdown" // Inherited from go-ethereum, not used in turbo-geth yet

	// migrationName -> serialized SyncStageProgress and SyncStageUnwind buckets
	// it stores stages progress to understand in which context was executed migration
	// in case of bug-report developer can ask content of this bucket
	Migrations = "migrations"

	Sequence = "sequence" // tbl_name -> seq_u64

)
View Source
var (
	// last  block that was pruned
	// it's saved one in 5 minutes
	LastPrunedBlockKey = []byte("LastPrunedBlock")
	//StorageModeHistory - does node save history.
	StorageModeHistory = []byte("smHistory")
	//StorageModeReceipts - does node save receipts.
	StorageModeReceipts = []byte("smReceipts")
	//StorageModeTxIndex - does node save transactions index.
	StorageModeTxIndex = []byte("smTxIndex")
	//StorageModeCallTraces - does not build index of call traces
	StorageModeCallTraces = []byte("smCallTraces")

	HeadHeaderKey = "LastHeader"

	SnapshotHeadersHeadNumber = "SnapshotLastHeaderNumber"
	SnapshotHeadersHeadHash   = "SnapshotLastHeaderHash"
	SnapshotBodyHeadNumber    = "SnapshotLastBodyNumber"
	SnapshotBodyHeadHash      = "SnapshotLastBodyHash"
)

Keys

View Source
var (
	PreimageCounter    = metrics.NewRegisteredCounter("db/preimage/total", nil)
	PreimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
)

Metrics

Buckets - list of all buckets. App will panic if some bucket is not in this list. This list will be sorted in `init` method. BucketsConfigs - can be used to find index in sorted version of Buckets list by name

View Source
var BucketsConfigs = BucketsCfg{
	CurrentStateBucketOld2: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                72,
		DupToLen:                  40,
	},
	HashedStorageBucket: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                72,
		DupToLen:                  40,
	},
	PlainAccountChangeSetBucket: {
		Flags: DupSort,
	},
	PlainStorageChangeSetBucket: {
		Flags: DupSort,
	},
	PlainStateBucket: {
		Flags:                     DupSort,
		AutoDupSortKeysConversion: true,
		DupFromLen:                60,
		DupToLen:                  28,
	},
	IntermediateTrieHashBucketOld2: {
		Flags:               DupSort,
		CustomDupComparator: DupCmpSuffix32,
	},
	InvalidBlock: {},
}

DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration

View Source
var IntermediateTrieHashBucketOld1 = "iTh"
View Source
var IntermediateTrieHashBucketOld2 = "iTh2"
View Source
var PlainStateBucket = "PLAIN-CST2"

PlainStateBucket Logical layout:

Contains Accounts:
  key - address (unhashed)
  value - account encoded for storage
Contains Storage:
  key - address (unhashed) + incarnation + storage key (unhashed)
  value - storage value(common.hash)

Physical layout:

PlainStateBucket and HashedStorageBucket utilises DupSort feature of LMDB (store multiple values inside 1 key).

-------------------------------------------------------------

key              |            value

------------------------------------------------------------- [acc_hash] | [acc_value] [acc_hash]+[inc] | [storage1_hash]+[storage1_value]

| [storage2_hash]+[storage2_value] // this value has no own key. it's 2nd value of [acc_hash]+[inc] key.
| [storage3_hash]+[storage3_value]
| ...

[acc_hash]+[old_inc] | [storage1_hash]+[storage1_value]

| ...

[acc2_hash] | [acc2_value]

...
View Source
var PlainStateBucketOld1 = "PLAIN-CST"
View Source
var TrieOfAccountsBucket = "trie_account"

TrieOfAccountsBucket and TrieOfStorageBucket hasState,groups - mark prefixes existing in hashed_account table hasTree - mark prefixes existing in trie_account table (not related with branchNodes) hasHash - mark prefixes which hashes are saved in current trie_account record (actually only hashes of branchNodes can be saved) @see UnmarshalTrieNode @see integrity.Trie

+-----------------------------------------------------------------------------------------------------+ | DB record: 0x0B, hasState: 0b1011, hasTree: 0b1001, hasHash: 0b1001, hashes: [x,x] | +-----------------------------------------------------------------------------------------------------+

|                                           |                               |
v                                           |                               v

+---------------------------------------------+ | +--------------------------------------+ | DB record: 0x0B00, hasState: 0b10001 | | | DB record: 0x0B03, hasState: 0b10010 | | hasTree: 0, hasHash: 0b10000, hashes: [x] | | | hasTree: 0, hasHash: 0, hashes: [] | +---------------------------------------------+ | +--------------------------------------+

|                    |                              |                         |                  |
v                    v                              v                         v                  v

+------------------+ +----------------------+ +---------------+ +---------------+ +---------------+ | Account: | | BranchNode: 0x0B0004 | | Account: | | Account: | | Account: | | 0x0B0000... | | has no record in | | 0x0B01... | | 0x0B0301... | | 0x050304... | | in HashedAccount | | TrieAccount | | | | | | | +------------------+ +----------------------+ +---------------+ +---------------+ +---------------+

                           |                |
                           v                v
		           +---------------+  +---------------+
		           | Account:      |  | Account:      |
		           | 0x0B000400... |  | 0x0B000401... |
		           +---------------+  +---------------+

Invariants: - hasTree is subset of hasState - hasHash is subset of hasState - first level in account_trie always exists if hasState>0 - TrieStorage record of account.root (length=40) must have +1 hash - it's account.root - each record in TrieAccount table must have parent (may be not direct) and this parent must have correct bit in hasTree bitmap - if hasState has bit - then HashedAccount table must have record according to this bit - each TrieAccount record must cover some state (means hasState is always > 0) - TrieAccount records with length=1 can satisfy (hasBranch==0&&hasHash==0) condition - Other records in TrieAccount and TrieStorage must (hasTree!=0 || hasHash!=0)

View Source
var TrieOfStorageBucket = "trie_storage"

Functions

func BlockBodyKey

func BlockBodyKey(number uint64, hash common.Hash) []byte

blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash

func BloomBitsKey

func BloomBitsKey(bit uint, section uint64, hash common.Hash) []byte

bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash

func ChangeSetByIndexBucket

func ChangeSetByIndexBucket(storage bool) string

func CheckCanonicalKey

func CheckCanonicalKey(k []byte) bool

func CheckNewIndexChunk

func CheckNewIndexChunk(b []byte, v uint64) bool

func CompositeKeySuffix

func CompositeKeySuffix(key []byte, timestamp uint64) (composite, encodedTS []byte)

Key + blockNum

func CompositeKeyWithoutIncarnation

func CompositeKeyWithoutIncarnation(key []byte) []byte

func CurrentChunkKey

func CurrentChunkKey(key []byte) []byte

func DecodeTimestamp

func DecodeTimestamp(suffix []byte) (uint64, []byte)

func DefaultCmpFunc

func DefaultCmpFunc(k1, k2, v1, v2 []byte) int

func DefaultDupCmpFunc

func DefaultDupCmpFunc(k1, k2, v1, v2 []byte) int

func EncodeBlockNumber

func EncodeBlockNumber(number uint64) []byte

EncodeBlockNumber encodes a block number as big endian uint64

func EncodeTimestamp

func EncodeTimestamp(timestamp uint64) []byte

EncodeTimestamp has the property: if a < b, then Encoding(a) < Encoding(b) lexicographically

func GenerateCompositeStorageKey

func GenerateCompositeStorageKey(addressHash common.Hash, incarnation uint64, seckey common.Hash) []byte

AddrHash + incarnation + KeyHash For contract storage

func GenerateCompositeStoragePrefix

func GenerateCompositeStoragePrefix(addressHash []byte, incarnation uint64, storageHashPrefix []byte) []byte

AddrHash + incarnation + StorageHashPrefix

func GenerateCompositeTrieKey

func GenerateCompositeTrieKey(addressHash common.Hash, seckey common.Hash) []byte

AddrHash + KeyHash Only for trie

func GenerateStoragePrefix

func GenerateStoragePrefix(addressHash []byte, incarnation uint64) []byte

address hash + incarnation prefix

func HeaderHashKey

func HeaderHashKey(number uint64) []byte

headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix

func HeaderKey

func HeaderKey(number uint64, hash common.Hash) []byte

headerKey = headerPrefix + num (uint64 big endian) + hash

func HeaderTDKey

func HeaderTDKey(number uint64, hash common.Hash) []byte

headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix

func IndexChunkKey

func IndexChunkKey(key []byte, blockNumber uint64) []byte

func IndexChunkKey32

func IndexChunkKey32(key []byte, blockNumber uint32) []byte

func IsHeaderHashKey

func IsHeaderHashKey(k []byte) bool

func IsHeaderKey

func IsHeaderKey(k []byte) bool

func IsHeaderTDKey

func IsHeaderTDKey(k []byte) bool

func LogKey

func LogKey(blockNumber uint64, txId uint32) []byte

LogKey = blockN (uint64 big endian) + txId (uint32 big endian)

func NextNibblesSubtree

func NextNibblesSubtree(in []byte, out *[]byte) bool

NextNibblesSubtree does []byte++. Returns false if overflow.

func NextSubtree

func NextSubtree(in []byte) ([]byte, bool)

NextSubtree does []byte++. Returns false if overflow.

func ParseCompositeStorageKey

func ParseCompositeStorageKey(compositeKey []byte) (common.Hash, uint64, common.Hash)

func ParseStoragePrefix

func ParseStoragePrefix(prefix []byte) (common.Hash, uint64)

func PlainGenerateCompositeStorageKey

func PlainGenerateCompositeStorageKey(address []byte, incarnation uint64, key []byte) []byte

AddrHash + incarnation + KeyHash For contract storage (for plain state)

func PlainGenerateStoragePrefix

func PlainGenerateStoragePrefix(address []byte, incarnation uint64) []byte

address hash + incarnation prefix (for plain state)

func PlainParseCompositeStorageKey

func PlainParseCompositeStorageKey(compositeKey []byte) (common.Address, uint64, common.Hash)

func PlainParseStoragePrefix

func PlainParseStoragePrefix(prefix []byte) (common.Address, uint64)

func ReceiptsKey

func ReceiptsKey(blockNumber uint64) []byte

ReceiptsKey = blockN (uint64 big endian)

func UpdateBucketsList

func UpdateBucketsList(newBucketCfg BucketsCfg)

Types

type Bucket

type Bucket string

type BucketConfigItem

type BucketConfigItem struct {
	Flags BucketFlags
	// AutoDupSortKeysConversion - enables some keys transformation - to change db layout without changing app code.
	// Use it wisely - it helps to do experiments with DB format faster, but better reduce amount of Magic in app.
	// If good DB format found, push app code to accept this format and then disable this property.
	AutoDupSortKeysConversion bool
	IsDeprecated              bool
	DBI                       DBI
	// DupFromLen - if user provide key of this length, then next transformation applied:
	// v = append(k[DupToLen:], v...)
	// k = k[:DupToLen]
	// And opposite at retrieval
	// Works only if AutoDupSortKeysConversion enabled
	DupFromLen          int
	DupToLen            int
	CustomComparator    CustomComparator
	CustomDupComparator CustomComparator
}

type BucketFlags

type BucketFlags uint
const (
	Default    BucketFlags = 0x00
	ReverseKey BucketFlags = 0x02
	DupSort    BucketFlags = 0x04
	IntegerKey BucketFlags = 0x08
	IntegerDup BucketFlags = 0x20
	ReverseDup BucketFlags = 0x40
)

type BucketsCfg

type BucketsCfg map[string]BucketConfigItem

func DefaultBuckets

func DefaultBuckets() BucketsCfg

type CmpFunc

type CmpFunc func(k1, k2, v1, v2 []byte) int

type CustomComparator

type CustomComparator string
const (
	DefaultCmp     CustomComparator = ""
	DupCmpSuffix32 CustomComparator = "dup_cmp_suffix32"
)

type DBI

type DBI uint

type HistoryIndexBytes

type HistoryIndexBytes []byte

func NewHistoryIndex

func NewHistoryIndex() HistoryIndexBytes

func WrapHistoryIndex

func WrapHistoryIndex(b []byte) HistoryIndexBytes

func (HistoryIndexBytes) Append

func (hi HistoryIndexBytes) Append(v uint64, emptyValue bool) HistoryIndexBytes

func (HistoryIndexBytes) Decode

func (hi HistoryIndexBytes) Decode() ([]uint64, []bool, error)

decode is used for debugging and in tests

func (HistoryIndexBytes) Key

func (hi HistoryIndexBytes) Key(key []byte) ([]byte, error)

func (HistoryIndexBytes) LastElement

func (hi HistoryIndexBytes) LastElement() (uint64, bool)

func (HistoryIndexBytes) Len

func (hi HistoryIndexBytes) Len() int

func (HistoryIndexBytes) Search

func (hi HistoryIndexBytes) Search(v uint64) (uint64, bool, bool)

Search looks for the element which is equal or greater of given timestamp

func (HistoryIndexBytes) String

func (hi HistoryIndexBytes) String() string

func (HistoryIndexBytes) TruncateGreater

func (hi HistoryIndexBytes) TruncateGreater(lower uint64) HistoryIndexBytes

Truncate all the timestamps that are strictly greater than the given bound

type Suffix

type Suffix []byte

func ToSuffix

func ToSuffix(b []byte) Suffix

func (Suffix) Add

func (s Suffix) Add(key []byte) Suffix

func (Suffix) KeyCount

func (s Suffix) KeyCount() uint32

func (Suffix) MultiAdd

func (s Suffix) MultiAdd(keys [][]byte) Suffix

func (Suffix) Walk

func (s Suffix) Walk(f func(k []byte) error) error

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL