dataRetriever

Documentation

Index

• Constants
• Variables
• func SetEpochHandlerToHdrResolver(resolversContainer ResolversContainer, epochHandler EpochHandler) error
• type ChainStorer
  • func NewChainStorer() *ChainStorer
  • func (bc *ChainStorer) AddStorer(key UnitType, s storage.Storer)
  • func (bc *ChainStorer) CloseAll() error
  • func (bc *ChainStorer) Destroy() error
  • func (bc *ChainStorer) Get(unitType UnitType, key []byte) ([]byte, error)
  • func (bc *ChainStorer) GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)
  • func (bc *ChainStorer) GetStorer(unitType UnitType) storage.Storer
  • func (bc *ChainStorer) Has(unitType UnitType, key []byte) error
  • func (bc *ChainStorer) IsInterfaceNil() bool
  • func (bc *ChainStorer) Put(unitType UnitType, key []byte, value []byte) error
• type DataPacker
• type DataRetriever
• type EpochHandler
• type EpochProviderByNonce
• type HeaderResolver
• type HeadersPool
• type IntRandomizer
• type MessageHandler
• type MiniBlocksResolver
• type Notifier
• type PeerListCreator
• type PoolsHolder
• type RequestData
  • func (rd *RequestData) Unmarshal(marshalizer marshal.Marshalizer, message p2p.MessageP2P) error
• type RequestDataType
  • func (rdt RequestDataType) String() string
• type RequestedItemsHandler
• type Resolver
• type ResolversContainer
• type ResolversContainerFactory
• type ResolversFinder
• type ShardIdHashMap
• type ShardedDataCacherNotifier
• type StorageService
• type StorageType
• type TopicHandler
• type TopicMessageHandler
• type TopicResolverSender
• type TransactionCacher
• type TrieDataGetter
• type UnitType

Constants

const TxPoolLargeNumOfTxsForASender = uint32(500)

TxPoolLargeNumOfTxsForASender instructs tx pool eviction algorithm to tag a sender with more transactions than this value as a "sender with a large number of transactions"

const TxPoolMinSizeInBytes = uint32(40960)

TxPoolMinSizeInBytes is the lower limit of the tx cache / eviction parameter "sizeInBytes"

const TxPoolNumSendersToEvictInOneStep = uint32(100)

TxPoolNumSendersToEvictInOneStep instructs tx pool eviction algorithm to remove this many senders when eviction takes place

const TxPoolNumTxsToEvictFromASender = uint32(100)

TxPoolNumTxsToEvictFromASender instructs tx pool eviction algorithm to remove this many transactions for "a sender with a large number of transactions" when eviction takes place

Variables

var ErrBadRequest = errors.New("request should not be done as it doesn't follow the protocol")

ErrBadRequest signals that the request should not have happened

var ErrCacheConfigInvalidEconomics = errors.New("cache-economics parameter is not valid")

ErrCacheConfigInvalidEconomics signals that an economics parameter required by the cache is invalid

var ErrCacheConfigInvalidSharding = errors.New("cache-sharding parameter is not valid")

ErrCacheConfigInvalidSharding signals that a sharding parameter required by the cache is invalid

var ErrCacheConfigInvalidShards = errors.New("cache parameter [shards] is not valid, it must be a positive number")

ErrCacheConfigInvalidShards signals that the cache parameter "shards" is invalid

var ErrCacheConfigInvalidSize = errors.New("cache parameter [size] is not valid, it must be a positive number")

ErrCacheConfigInvalidSize signals that the cache parameter "size" is invalid

var ErrCacheConfigInvalidSizeInBytes = errors.New("cache parameter [sizeInBytes] is not valid, it must be a positive, and large enough number")

ErrCacheConfigInvalidSizeInBytes signals that the cache parameter "sizeInBytes" is invalid

var ErrContainerKeyAlreadyExists = errors.New("provided key already exists in container")

ErrContainerKeyAlreadyExists signals that an element was already set in the container's map

var ErrEmptyMiniBlockSlice = errors.New("empty mini block slice")

ErrEmptyMiniBlockSlice signals that an operation has been attempted with an empty mini block slice

var ErrInvalidContainerKey = errors.New("element does not exist in container")

ErrInvalidContainerKey signals that an element does not exist in the container's map

var ErrInvalidMaxTxRequest = errors.New("max tx request number is invalid")

ErrInvalidMaxTxRequest signals that max tx request is too small

var ErrInvalidNonceByteSlice = errors.New("invalid nonce byte slice")

ErrInvalidNonceByteSlice signals that an invalid byte slice has been provided and an uint64 can not be decoded from that byte slice

var ErrInvalidRequestType = errors.New("invalid request type")

ErrInvalidRequestType signals that a request on a topic sends an invalid type

var ErrLenMismatch = errors.New("lengths mismatch")

ErrLenMismatch signals that 2 or more slices have different lengths

var ErrNilBlockBodyPool = errors.New("nil block body pool")

ErrNilBlockBodyPool signals that a nil block body pool has been provided

var ErrNilBlockBodyStorage = errors.New("nil block body storage")

ErrNilBlockBodyStorage signals that a nil block body storage has been provided

var ErrNilCacher = errors.New("nil cacher")

ErrNilCacher signals that a nil cache has been provided

var ErrNilContainerElement = errors.New("element cannot be nil")

ErrNilContainerElement signals when trying to add a nil element in the container

var ErrNilCurrBlockTxs = errors.New("nil current block txs holder")

ErrNilCurrBlockTxs signals that nil current blocks txs holder was provided

var ErrNilDataPacker = errors.New("nil data packer provided")

ErrNilDataPacker signals that a nil data packer has been provided

var ErrNilDataPoolHolder = errors.New("nil data pool holder")

ErrNilDataPoolHolder signals that the data pool holder is nil

var ErrNilDataToProcess = errors.New("nil data to process")

ErrNilDataToProcess signals that nil data was provided

var ErrNilEpochHandler = errors.New("nil epoch handler")

ErrNilEpochHandler signals that epoch handler is nil

var ErrNilHeadersDataPool = errors.New("nil headers data pool")

ErrNilHeadersDataPool signals that a nil header pool has been provided

var ErrNilHeadersNoncesStorage = errors.New("nil headers nonces storage")

ErrNilHeadersNoncesStorage signals that a nil header-nonce storage has been provided

var ErrNilHeadersStorage = errors.New("nil headers storage")

ErrNilHeadersStorage signals that a nil header storage has been provided

var ErrNilMarshalizer = errors.New("nil Marshalizer")

ErrNilMarshalizer signals that an operation has been attempted to or with a nil Marshalizer implementation

var ErrNilMessage = errors.New("nil message")

ErrNilMessage signals that a nil message has been received

var ErrNilMessenger = errors.New("nil Messenger")

ErrNilMessenger signals that a nil Messenger object was provided

var ErrNilMetaBlockPool = errors.New("nil meta block data pool")

ErrNilMetaBlockPool signals that a nil meta block data pool was provided

var ErrNilPeerChangeBlockDataPool = errors.New("nil peer change block data pool")

ErrNilPeerChangeBlockDataPool signals that a nil peer change pool has been provided

var ErrNilPeerListCreator = errors.New("nil peer list creator provided")

ErrNilPeerListCreator signals that a nil peer list creator implementation has been provided

var ErrNilRandomizer = errors.New("nil randomizer")

ErrNilRandomizer signals that a nil randomizer has been provided

var ErrNilRequestedItemsHandler = errors.New("nil requested items handler")

ErrNilRequestedItemsHandler signals that a nil requested items handler was provided

var ErrNilResolverContainer = errors.New("nil resolver container")

ErrNilResolverContainer signals that a nil resolver container was provided

var ErrNilResolverFinder = errors.New("nil resolvers finder")

ErrNilResolverFinder signals that a nil resolver finder has been provided

var ErrNilResolverSender = errors.New("nil resolver sender")

ErrNilResolverSender signals that a nil resolver sender object has been provided

var ErrNilRewardTransactionPool = errors.New("nil reward transaction data pool")

ErrNilRewardTransactionPool signals that a nil reward transactions pool has been provided

var ErrNilShardCoordinator = errors.New("nil shard coordinator")

ErrNilShardCoordinator signals that an operation has been attempted to or with a nil shard coordinator

var ErrNilStore = errors.New("nil data storage service")

ErrNilStore signals that the provided storage service is nil

var ErrNilTrieDataGetter = errors.New("nil trie data getter provided")

ErrNilTrieDataGetter signals that a nil trie data getter has been provided

var ErrNilTrieNodesPool = errors.New("nil trie nodes data pool")

ErrNilTrieNodesPool signals that a nil trie nodes data pool was provided

var ErrNilTxBlockDataPool = errors.New("nil tx block data pool")

ErrNilTxBlockDataPool signals that a nil tx block body pool has been provided

var ErrNilTxDataPool = errors.New("nil transaction data pool")

ErrNilTxDataPool signals that a nil transaction pool has been provided

var ErrNilTxStorage = errors.New("nil transaction storage")

ErrNilTxStorage signals that a nil transaction storage has been provided

var ErrNilUint64ByteSliceConverter = errors.New("nil byte slice converter")

ErrNilUint64ByteSliceConverter signals that a nil byte slice converter was provided

var ErrNilUnsignedTransactionPool = errors.New("nil unsigned transactions data pool")

ErrNilUnsignedTransactionPool signals that a nil unsigned transactions pool has been provided

var ErrNilValue = errors.New("nil value")

ErrNilValue signals the value is nil

var ErrNoConnectedPeerToSendRequest = errors.New("connected peers list is empty. Can not send request")

ErrNoConnectedPeerToSendRequest signals that the connected peers list is empty and can not send request

var ErrNoSuchStorageUnit = errors.New("no such unit type")

ErrNoSuchStorageUnit defines the error for using an invalid storage unit

var ErrRequestTypeNotImplemented = errors.New("request type is not implemented")

ErrRequestTypeNotImplemented signals that a not implemented type of request has been received

var ErrResolveTypeUnknown = errors.New("unknown resolve type")

ErrResolveTypeUnknown signals that an unknown resolve type was provided

var ErrTxNotFoundInBlockPool = errors.New("cannot find tx in current block pool")

ErrTxNotFoundInBlockPool signals the value is nil

var ErrUnmarshalMBHashes = errors.New("could not unmarshal miniblock hashes")

ErrUnmarshalMBHashes signals the value is nil

var ErrWrongTypeInContainer = errors.New("wrong type of object inside container")

ErrWrongTypeInContainer signals that a wrong type of object was found in container

Functions

func SetEpochHandlerToHdrResolver

func SetEpochHandlerToHdrResolver(
	resolversContainer ResolversContainer,
	epochHandler EpochHandler,
) error

SetEpochHandlerToHdrResolver sets the epoch handler to the metablock hdr resolver

Types

type ChainStorer

type ChainStorer struct {
	// contains filtered or unexported fields
}

ChainStorer is a StorageService implementation that can hold multiple storages

grouped by storage unit type

func NewChainStorer

func NewChainStorer() *ChainStorer

NewChainStorer returns a new initialised ChainStorer

func (*ChainStorer) AddStorer

func (bc *ChainStorer) AddStorer(key UnitType, s storage.Storer)

AddStorer will add a new storer to the chain map

func (*ChainStorer) CloseAll

func (bc *ChainStorer) CloseAll() error

CloseAll will close all the active units

func (*ChainStorer) Destroy

func (bc *ChainStorer) Destroy() error

Destroy removes the underlying files/resources used by the storage service

func (*ChainStorer) Get

func (bc *ChainStorer) Get(unitType UnitType, key []byte) ([]byte, error)

Get returns the value for the given key if found in the selected storage unit, nil otherwise. It can return an error if the provided unit type is not supported or if the storage unit underlying implementation reports an error

func (*ChainStorer) GetAll

func (bc *ChainStorer) GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)

GetAll gets all the elements with keys in the keys array, from the selected storage unit It can report an error if the provided unit type is not supported, if there is a missing key in the unit, or if the underlying implementation of the storage unit reports an error.

func (*ChainStorer) GetStorer

func (bc *ChainStorer) GetStorer(unitType UnitType) storage.Storer

GetStorer returns the storer from the chain map or nil if the storer was not found

func (*ChainStorer) Has

func (bc *ChainStorer) Has(unitType UnitType, key []byte) error

Has returns true if the key is found in the selected Unit or false otherwise It can return an error if the provided unit type is not supported or if the underlying implementation of the storage unit reports an error.

func (*ChainStorer) IsInterfaceNil

func (bc *ChainStorer) IsInterfaceNil() bool

IsInterfaceNil returns true if there is no value under the interface

func (*ChainStorer) Put

func (bc *ChainStorer) Put(unitType UnitType, key []byte, value []byte) error

Put stores the key, value pair in the selected storage unit It can return an error if the provided unit type is not supported or if the storage unit underlying implementation reports an error

type DataPacker

type DataPacker interface {
	PackDataInChunks(data [][]byte, limit int) ([][]byte, error)
	IsInterfaceNil() bool
}

DataPacker can split a large slice of byte slices in smaller packets

type DataRetriever

type DataRetriever interface {
	// Get methods searches for data in storage units and returns results, it is a blocking function
	Get(keys [][]byte, identifier string, lowestLevel StorageType, haveTime func() time.Duration) (map[string]interface{}, [][]byte, error)
	// Has searches for a value identifier by a key in storage
	Has(key []byte, identifier string, level StorageType) (StorageType, error)
	// HasOrAdd searches and adds a value if not exist in storage
	HasOrAdd(key []byte, value interface{}, identifier string, level StorageType)
	// Remove deletes an element from storage level
	Remove(key []byte, identifier string, lowestLevel StorageType) error
	// Put saves a key-value pair into storage
	Put(key []byte, value interface{}, identifier string, level StorageType) error
	// Keys returns all the keys from an identifier and storage type
	Keys(identifier string, level StorageType)
	// Request searches for data in specified storage level, if not present launches threads to search in network
	Request(keys [][]byte, identifier string, level StorageType, haveTime func() time.Duration, callbackHandler func(key []byte)) (map[string]interface{}, [][]byte, error)
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

DataRetriever interface provides functionality over high level data request component

type EpochHandler

type EpochHandler interface {
	Epoch() uint32
	IsInterfaceNil() bool
}

EpochHandler defines the functionality to get the current epoch

type EpochProviderByNonce

type EpochProviderByNonce interface {
	EpochForNonce(nonce uint64) (uint32, error)
	IsInterfaceNil() bool
}

EpochProviderByNonce defines the functionality needed for calculating an epoch based on nonce

type HeaderResolver

type HeaderResolver interface {
	Resolver
	RequestDataFromNonce(nonce uint64, epoch uint32) error
	RequestDataFromEpoch(identifier []byte) error
	SetEpochHandler(epochHandler EpochHandler) error
}

HeaderResolver defines what a block header resolver should do

type HeadersPool

type HeadersPool interface {
	Clear()
	AddHeader(headerHash []byte, header data.HeaderHandler)
	RemoveHeaderByHash(headerHash []byte)
	RemoveHeaderByNonceAndShardId(headerNonce uint64, shardId uint32)
	GetHeadersByNonceAndShardId(headerNonce uint64, shardId uint32) ([]data.HeaderHandler, [][]byte, error)
	GetHeaderByHash(hash []byte) (data.HeaderHandler, error)
	RegisterHandler(handler func(headerHandler data.HeaderHandler, headerHash []byte))
	Nonces(shardId uint32) []uint64
	Len() int
	MaxSize() int
	IsInterfaceNil() bool
}

HeadersPool defines what a headers pool structure can perform

type IntRandomizer

type IntRandomizer interface {
	Intn(n int) (int, error)
	IsInterfaceNil() bool
}

IntRandomizer interface provides functionality over generating integer numbers

type MessageHandler

type MessageHandler interface {
	ConnectedPeersOnTopic(topic string) []p2p.PeerID
	SendToConnectedPeer(topic string, buff []byte, peerID p2p.PeerID) error
	IsInterfaceNil() bool
}

MessageHandler defines the functionality needed by structs to send data to other peers

type MiniBlocksResolver

type MiniBlocksResolver interface {
	Resolver
	RequestDataFromHashArray(hashes [][]byte, epoch uint32) error
	GetMiniBlocks(hashes [][]byte) (block.MiniBlockSlice, [][]byte)
	GetMiniBlocksFromPool(hashes [][]byte) (block.MiniBlockSlice, [][]byte)
}

MiniBlocksResolver defines what a mini blocks resolver should do

type Notifier

type Notifier interface {
	RegisterHandler(func(key []byte))
	IsInterfaceNil() bool
}

Notifier defines a way to register funcs that get called when something useful happens

type PeerListCreator

type PeerListCreator interface {
	PeerList() []p2p.PeerID
	IsInterfaceNil() bool
}

PeerListCreator is used to create a peer list

type PoolsHolder

type PoolsHolder interface {
	Transactions() ShardedDataCacherNotifier
	UnsignedTransactions() ShardedDataCacherNotifier
	RewardTransactions() ShardedDataCacherNotifier
	Headers() HeadersPool
	MiniBlocks() storage.Cacher
	PeerChangesBlocks() storage.Cacher
	TrieNodes() storage.Cacher
	CurrentBlockTxs() TransactionCacher
	IsInterfaceNil() bool
}

PoolsHolder defines getters for data pools

type RequestData

type RequestData struct {
	Value []byte
	Type  RequestDataType
	Epoch uint32
}

RequestData holds the requested data This struct will be serialized and sent to the other peers

func (*RequestData) Unmarshal

func (rd *RequestData) Unmarshal(marshalizer marshal.Marshalizer, message p2p.MessageP2P) error

Unmarshal sets the fields according to p2p.MessageP2P.Data() contents Errors if something went wrong

type RequestDataType

type RequestDataType byte

RequestDataType represents the data type for the requested data

const (
	// HashType indicates that the request data object is of type hash
	HashType RequestDataType = iota + 1
	// HashArrayType that the request data object contains a serialised array of hashes
	HashArrayType
	// NonceType indicates that the request data object is of type nonce (uint64)
	NonceType
	// EpochType indicates that the request data object is of type epoch
	EpochType
)

func (RequestDataType) String

func (rdt RequestDataType) String() string

type RequestedItemsHandler

type RequestedItemsHandler interface {
	Add(key string) error
	Has(key string) bool
	Sweep()
	IsInterfaceNil() bool
}

RequestedItemsHandler can determine if a certain key has or not been requested

type Resolver

type Resolver interface {
	RequestDataFromHash(hash []byte, epoch uint32) error
	ProcessReceivedMessage(message p2p.MessageP2P, broadcastHandler func(buffToSend []byte)) error
	IsInterfaceNil() bool
}

Resolver defines what a data resolver should do

type ResolversContainer

type ResolversContainer interface {
	Get(key string) (Resolver, error)
	Add(key string, val Resolver) error
	AddMultiple(keys []string, resolvers []Resolver) error
	Replace(key string, val Resolver) error
	Remove(key string)
	Len() int
	IsInterfaceNil() bool
}

ResolversContainer defines a resolvers holder data type with basic functionality

type ResolversContainerFactory

type ResolversContainerFactory interface {
	Create() (ResolversContainer, error)
	IsInterfaceNil() bool
}

ResolversContainerFactory defines the functionality to create a resolvers container

type ResolversFinder

type ResolversFinder interface {
	ResolversContainer
	IntraShardResolver(baseTopic string) (Resolver, error)
	MetaChainResolver(baseTopic string) (Resolver, error)
	CrossShardResolver(baseTopic string, crossShard uint32) (Resolver, error)
}

ResolversFinder extends a container resolver and have 2 additional functionality

type ShardIdHashMap

type ShardIdHashMap interface {
	Load(shardId uint32) ([]byte, bool)
	Store(shardId uint32, hash []byte)
	Range(f func(shardId uint32, hash []byte) bool)
	Delete(shardId uint32)
	IsInterfaceNil() bool
}

ShardIdHashMap represents a map for shardId and hash

type ShardedDataCacherNotifier

type ShardedDataCacherNotifier interface {
	Notifier

	ShardDataStore(cacheId string) (c storage.Cacher)
	AddData(key []byte, data interface{}, cacheId string)
	SearchFirstData(key []byte) (value interface{}, ok bool)
	RemoveData(key []byte, cacheId string)
	RemoveSetOfDataFromPool(keys [][]byte, cacheId string)
	RemoveDataFromAllShards(key []byte)
	MergeShardStores(sourceCacheID, destCacheID string)
	Clear()
	ClearShardStore(cacheId string)
	CreateShardStore(cacheId string)
}

ShardedDataCacherNotifier defines what a sharded-data structure can perform

type StorageService

type StorageService interface {
	// GetStorer returns the storer from the chain map
	GetStorer(unitType UnitType) storage.Storer
	// AddStorer will add a new storer to the chain map
	AddStorer(key UnitType, s storage.Storer)
	// Has returns true if the key is found in the selected Unit or false otherwise
	Has(unitType UnitType, key []byte) error
	// Get returns the value for the given key if found in the selected storage unit, nil otherwise
	Get(unitType UnitType, key []byte) ([]byte, error)
	// Put stores the key, value pair in the selected storage unit
	Put(unitType UnitType, key []byte, value []byte) error
	// GetAll gets all the elements with keys in the keys array, from the selected storage unit
	// If there is a missing key in the unit, it returns an error
	GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)
	// Destroy removes the underlying files/resources used by the storage service
	Destroy() error
	//CloseAll will close all the units
	CloseAll() error
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

StorageService is the interface for data storage unit provided services

type StorageType

type StorageType uint8

StorageType defines the storage levels on a node

type TopicHandler

type TopicHandler interface {
	HasTopic(name string) bool
	CreateTopic(name string, createChannelForTopic bool) error
	RegisterMessageProcessor(topic string, handler p2p.MessageProcessor) error
}

TopicHandler defines the functionality needed by structs to manage topics and message processors

type TopicMessageHandler

type TopicMessageHandler interface {
	MessageHandler
	TopicHandler
}

TopicMessageHandler defines the functionality needed by structs to manage topics, message processors and to send data to other peers

type TopicResolverSender

type TopicResolverSender interface {
	SendOnRequestTopic(rd *RequestData) error
	Send(buff []byte, peer p2p.PeerID) error
	TopicRequestSuffix() string
	TargetShardID() uint32
	IsInterfaceNil() bool
}

TopicResolverSender defines what sending operations are allowed for a topic resolver

type TransactionCacher

type TransactionCacher interface {
	Clean()
	GetTx(txHash []byte) (data.TransactionHandler, error)
	AddTx(txHash []byte, tx data.TransactionHandler)
	IsInterfaceNil() bool
}

TransactionCacher defines the methods for the local cacher, info for current round

type TrieDataGetter

type TrieDataGetter interface {
	GetSerializedNodes([]byte, uint64) ([][]byte, error)
	IsInterfaceNil() bool
}

TrieDataGetter returns requested data from the trie

type UnitType

type UnitType uint8

UnitType is the type for Storage unit identifiers

const (
	// TransactionUnit is the transactions storage unit identifier
	TransactionUnit UnitType = 0
	// MiniBlockUnit is the transaction block body storage unit identifier
	MiniBlockUnit UnitType = 1
	// PeerChangesUnit is the peer change block body storage unit identifier
	PeerChangesUnit UnitType = 2
	// BlockHeaderUnit is the Block Headers Storage unit identifier
	BlockHeaderUnit UnitType = 3
	// MetaBlockUnit is the metachain blocks storage unit identifier
	MetaBlockUnit UnitType = 4
	// UnsignedTransactionUnit is the unsigned transaction unit identifier
	UnsignedTransactionUnit UnitType = 5
	// RewardTransactionUnit is the reward transaction unit identifier
	RewardTransactionUnit UnitType = 6
	// MetaHdrNonceHashDataUnit is the meta header nonce-hash pair data unit identifier
	MetaHdrNonceHashDataUnit UnitType = 7
	// HeartbeatUnit is the heartbeat storage unit identifier
	HeartbeatUnit UnitType = 8
	// MiniBlockHeaderUnit is the miniblock header data unit identifier
	MiniBlockHeaderUnit UnitType = 9
	// BootstrapUnit is the bootstrap storage unit identifier
	BootstrapUnit UnitType = 10
	//StatusMetricsUnit is the status metrics storage unit identifier
	StatusMetricsUnit UnitType = 11

	// ShardHdrNonceHashDataUnit is the header nonce-hash pair data unit identifier
	//TODO: Add only unit types lower than 100
	ShardHdrNonceHashDataUnit UnitType = 100
)