dataRetriever

Documentation

Index

• Variables
• type ChainStorer
  • func NewChainStorer() *ChainStorer
  • func (bc *ChainStorer) AddStorer(key UnitType, s storage.Storer)
  • func (bc *ChainStorer) Destroy() error
  • func (bc *ChainStorer) Get(unitType UnitType, key []byte) ([]byte, error)
  • func (bc *ChainStorer) GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)
  • func (bc *ChainStorer) GetStorer(unitType UnitType) storage.Storer
  • func (bc *ChainStorer) Has(unitType UnitType, key []byte) error
  • func (bc *ChainStorer) Put(unitType UnitType, key []byte, value []byte) error
• type DataPacker
• type DataRetriever
• type HeaderResolver
• type IntRandomizer
• type MessageHandler
• type MetaPoolsHolder
• type MiniBlocksResolver
• type Notifier
• type PoolsHolder
• type RequestData
  • func (rd *RequestData) Unmarshal(marshalizer marshal.Marshalizer, message p2p.MessageP2P) error
• type RequestDataType
  • func (rdt RequestDataType) String() string
• type Resolver
• type ResolversContainer
• type ResolversContainerFactory
• type ResolversFinder
• type ShardedDataCacherNotifier
• type StorageService
• type StorageType
• type TopicHandler
• type TopicMessageHandler
• type TopicResolverSender
• type Uint64Cacher
• type UnitType

Variables

var ErrContainerKeyAlreadyExists = errors.New("provided key already exists in container")

ErrContainerKeyAlreadyExists signals that an element was already set in the container's map

var ErrEmptyHeaderRequestTopic = errors.New("empty header request topic")

ErrEmptyHeaderRequestTopic signals that an empty transaction topic has been provided

var ErrEmptyMiniBlockRequestTopic = errors.New("empty miniblock request topic")

ErrEmptyMiniBlockRequestTopic signals that an empty miniblock topic has been provided

var ErrEmptyScrRequestTopic = errors.New("empty smart contract result request topic")

ErrEmptyScrRequestTopic signals that an empty smart contract result topic has been provided

var ErrEmptyTxRequestTopic = errors.New("empty transaction request topic")

ErrEmptyTxRequestTopic signals that an empty transaction topic has been provided

var ErrInvalidContainerKey = errors.New("element does not exist in container")

ErrInvalidContainerKey signals that an element does not exist in the container's map

var ErrInvalidMaxTxRequest = errors.New("max tx request number is invalid")

ErrInvalidMaxTxRequest signals that max tx request is too small

var ErrInvalidNonceByteSlice = errors.New("invalid nonce byte slice")

ErrInvalidNonceByteSlice signals that an invalid byte slice has been provided and an uint64 can not be decoded from that byte slice

var ErrInvalidRequestType = errors.New("invalid request type")

ErrInvalidRequestType signals that a request on a topic sends an invalid type

var ErrInvalidShardId = errors.New("invalid shard id")

ErrInvalidShardId signals that the shard id is invalid

var ErrLenMismatch = errors.New("lengths mismatch")

ErrLenMismatch signals that 2 or more slices have different lengths

var ErrNilAccountsAdapter = errors.New("nil AccountsAdapter")

ErrNilAccountsAdapter defines the error when trying to use a nil AccountsAddapter

var ErrNilAddressContainer = errors.New("nil AddressContainer")

ErrNilAddressContainer signals that an operation has been attempted to or with a nil AddressContainer implementation

var ErrNilAddressConverter = errors.New("nil AddressConverter")

ErrNilAddressConverter signals that an operation has been attempted to or with a nil AddressConverter implementation

var ErrNilBlockBodyPool = errors.New("nil block body pool")

ErrNilBlockBodyPool signals that a nil block body pool has been provided

var ErrNilBlockBodyStorage = errors.New("nil block body storage")

ErrNilBlockBodyStorage signals that a nil block body storage has been provided

var ErrNilBlockChain = errors.New("nil block chain")

ErrNilBlockChain signals that an operation has been attempted to or with a nil blockchain

var ErrNilBlockHeader = errors.New("nil block header")

ErrNilBlockHeader signals that an operation has been attempted to or with a nil block header

var ErrNilCacher = errors.New("nil cacher")

ErrNilCacher signals that a nil cache has been provided

var ErrNilContainerElement = errors.New("element cannot be nil")

ErrNilContainerElement signals when trying to add a nil element in the container

var ErrNilDataPacker = errors.New("nil data packer provided")

ErrNilDataPacker signals that a nil data packer has been provided

var ErrNilDataPoolHolder = errors.New("nil data pool holder")

ErrNilDataPoolHolder signals that the data pool holder is nil

var ErrNilDataToProcess = errors.New("nil data to process")

ErrNilDataToProcess signals that nil data was provided

var ErrNilHasher = errors.New("nil Hasher")

ErrNilHasher signals that an operation has been attempted to or with a nil hasher implementation

var ErrNilHeadersDataPool = errors.New("nil headers data pool")

ErrNilHeadersDataPool signals that a nil header pool has been provided

var ErrNilHeadersNoncesDataPool = errors.New("nil headers nonces cache")

ErrNilHeadersNoncesDataPool signals that a nil header - nonce cache

var ErrNilHeadersNoncesStorage = errors.New("nil headers nonces storage")

ErrNilHeadersNoncesStorage signals that a nil header-nonce storage has been provided

var ErrNilHeadersStorage = errors.New("nil headers storage")

ErrNilHeadersStorage signals that a nil header storage has been provided

var ErrNilMarshalizer = errors.New("nil Marshalizer")

ErrNilMarshalizer signals that an operation has been attempted to or with a nil Marshalizer implementation

var ErrNilMessage = errors.New("nil message")

ErrNilMessage signals that a nil message has been received

var ErrNilMessenger = errors.New("nil Messenger")

ErrNilMessenger signals that a nil Messenger object was provided

var ErrNilMetaBlockNoncesPool = errors.New("nil meta block nonces data pool")

ErrNilMetaBlockNoncesPool signals that a nil meta block data pool was provided

var ErrNilMetaBlockPool = errors.New("nil meta block data pool")

ErrNilMetaBlockPool signals that a nil meta block data pool was provided

var ErrNilMiniBlockHashesPool = errors.New("nil meta block mini block hashes data pool")

ErrNilMiniBlockHashesPool signals that a nil meta block data pool was provided

var ErrNilMiniBlocks = errors.New("nil mini blocks")

ErrNilMiniBlocks signals that an operation has been attempted with a nil mini-block

var ErrNilNonceConverter = errors.New("nil nonce converter")

ErrNilNonceConverter signals that a nil nonce converter has been provided

var ErrNilPeerChangeBlockDataPool = errors.New("nil peer change block data pool")

ErrNilPeerChangeBlockDataPool signals that a nil peer change pool has been provided

var ErrNilPublicKey = errors.New("nil public key")

ErrNilPublicKey signals that a operation has been attempted with a nil public key

var ErrNilRandomizer = errors.New("nil randomizer")

ErrNilRandomizer signals that a nil randomizer has been provided

var ErrNilResolverContainer = errors.New("nil resolver container")

ErrNilResolverContainer signals that a nil resolver container was provided

var ErrNilResolverFinder = errors.New("nil resolvers finder")

ErrNilResolverFinder signals that a nil resolver finder has been provided

var ErrNilResolverSender = errors.New("nil resolver sender")

ErrNilResolverSender signals that a nil resolver sender object has been provided

var ErrNilRounder = errors.New("nil Rounder")

ErrNilRounder signals that an operation has been attempted to or with a nil Rounder implementation

var ErrNilShardCoordinator = errors.New("nil shard coordinator")

ErrNilShardCoordinator signals that an operation has been attempted to or with a nil shard coordinator

var ErrNilShardHeaderPool = errors.New("nil meta block shard header data pool")

ErrNilShardHeaderPool signals that a nil meta block data pool was provided

var ErrNilSignature = errors.New("nil signature")

ErrNilSignature signals that a operation has been attempted with a nil signature

var ErrNilSingleSigner = errors.New("nil single signer")

ErrNilSingleSigner signals that a nil single signer is used

var ErrNilSmartContractResultsPool = errors.New("nil smart contract results data pool")

ErrNilSmartContractResultsPool signals that a nil smart contract results pool has been provided

var ErrNilStore = errors.New("nil data storage service")

ErrNilStore signals that the provided storage service is nil

var ErrNilTxBlockBody = errors.New("nil block body")

ErrNilTxBlockBody signals that an operation has been attempted to or with a nil block body

var ErrNilTxBlockDataPool = errors.New("nil tx block data pool")

ErrNilTxBlockDataPool signals that a nil tx block body pool has been provided

var ErrNilTxDataPool = errors.New("nil transaction data pool")

ErrNilTxDataPool signals that a nil transaction pool has been provided

var ErrNilTxStorage = errors.New("nil transaction storage")

ErrNilTxStorage signals that a nil transaction storage has been provided

var ErrNilUint64ByteSliceConverter = errors.New("nil byte slice converter")

ErrNilUint64ByteSliceConverter signals that a nil byte slice converter was provided

var ErrNilValue = errors.New("nil value")

ErrNilValue signals the value is nil

var ErrNoConnectedPeerToSendRequest = errors.New("connected peers list is empty. Can not send request")

ErrNoConnectedPeerToSendRequest signals that the connected peers list is empty and can not send request

var ErrNoSuchStorageUnit = errors.New("no such unit type")

ErrNoSuchStorageUnit defines the error for using an invalid storage unit

var ErrRequestTypeNotImplemented = errors.New("request type is not implemented")

ErrRequestTypeNotImplemented signals that a not implemented type of request has been received

var ErrResolveTypeUnknown = errors.New("unknown resolve type")

ErrResolveTypeUnknown signals that an unknown resolve type was provided

var ErrUnmarshalMBHashes = errors.New("could not unmarshal miniblock hashes")

ErrUnmarshalMBHashes signals the value is nil

var ErrWrongTypeInContainer = errors.New("wrong type of object inside container")

ErrWrongTypeInContainer signals that a wrong type of object was found in container

Types

type ChainStorer

type ChainStorer struct {
	// contains filtered or unexported fields
}

ChainStorer is a StorageService implementation that can hold multiple storages

grouped by storage unit type

func NewChainStorer

func NewChainStorer() *ChainStorer

NewChainStorer returns a new initialised ChainStorer

func (*ChainStorer) AddStorer

func (bc *ChainStorer) AddStorer(key UnitType, s storage.Storer)

AddStorer will add a new storer to the chain map

func (*ChainStorer) Destroy

func (bc *ChainStorer) Destroy() error

Destroy removes the underlying files/resources used by the storage service

func (*ChainStorer) Get

func (bc *ChainStorer) Get(unitType UnitType, key []byte) ([]byte, error)

Get returns the value for the given key if found in the selected storage unit, nil otherwise. It can return an error if the provided unit type is not supported or if the storage unit underlying implementation reports an error

func (*ChainStorer) GetAll

func (bc *ChainStorer) GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)

GetAll gets all the elements with keys in the keys array, from the selected storage unit It can report an error if the provided unit type is not supported, if there is a missing key in the unit, or if the underlying implementation of the storage unit reports an error.

func (*ChainStorer) GetStorer

func (bc *ChainStorer) GetStorer(unitType UnitType) storage.Storer

GetStorer returns the storer from the chain map or nil if the storer was not found

func (*ChainStorer) Has

func (bc *ChainStorer) Has(unitType UnitType, key []byte) error

Has returns true if the key is found in the selected Unit or false otherwise It can return an error if the provided unit type is not supported or if the underlying implementation of the storage unit reports an error.

func (*ChainStorer) Put

func (bc *ChainStorer) Put(unitType UnitType, key []byte, value []byte) error

Put stores the key, value pair in the selected storage unit It can return an error if the provided unit type is not supported or if the storage unit underlying implementation reports an error

type DataPacker

type DataPacker interface {
	PackDataInChunks(data [][]byte, limit int) ([][]byte, error)
}

DataPacker can split a large slice of byte slices in smaller packets

type DataRetriever

type DataRetriever interface {
	// Get methods searches for data in storage units and returns results, it is a blocking function
	Get(keys [][]byte, identifier string, lowestLevel StorageType, haveTime func() time.Duration) (map[string]interface{}, [][]byte, error)
	// Has searches for a value identifier by a key in storage
	Has(key []byte, identifier string, level StorageType) (StorageType, error)
	// HasOrAdd searches and adds a value if not exist in storage
	HasOrAdd(key []byte, value interface{}, identifier string, level StorageType)
	// Remove deletes an element from storage level
	Remove(key []byte, identifier string, lowestLevel StorageType) error
	// Put saves a key-value pair into storage
	Put(key []byte, value interface{}, identifier string, level StorageType) error
	// Keys returns all the keys from an identifier and storage type
	Keys(identifier string, level StorageType)
	// Request searches for data in specified storage level, if not present launches threads to search in network
	Request(keys [][]byte, identifier string, level StorageType, haveTime func() time.Duration, callbackHandler func(key []byte)) (map[string]interface{}, [][]byte, error)
}

DataRetriever interface provides functionality over high level data request component

type HeaderResolver

type HeaderResolver interface {
	Resolver
	RequestDataFromNonce(nonce uint64) error
}

HeaderResolver defines what a block header resolver should do

type IntRandomizer

type IntRandomizer interface {
	Intn(n int) (int, error)
}

IntRandomizer interface provides functionality over generating integer numbers

type MessageHandler

type MessageHandler interface {
	ConnectedPeersOnTopic(topic string) []p2p.PeerID
	SendToConnectedPeer(topic string, buff []byte, peerID p2p.PeerID) error
}

MessageHandler defines the functionality needed by structs to send data to other peers

type MetaPoolsHolder

type MetaPoolsHolder interface {
	MetaChainBlocks() storage.Cacher
	MiniBlockHashes() ShardedDataCacherNotifier
	ShardHeaders() storage.Cacher
	MetaBlockNonces() Uint64Cacher
	ShardHeadersNonces() Uint64Cacher
}

MetaPoolsHolder defines getter for data pools for metachain

type MiniBlocksResolver

type MiniBlocksResolver interface {
	Resolver
	RequestDataFromHashArray(hashes [][]byte) error

	// TODO miniblockresolver should not know about miniblockslice
	GetMiniBlocks(hashes [][]byte) block.MiniBlockSlice
}

MiniBlocksResolver defines what a mini blocks resolver should do

type Notifier

type Notifier interface {
	RegisterHandler(func(key []byte))
}

Notifier defines a way to register funcs that get called when something useful happens

type PoolsHolder

type PoolsHolder interface {
	Transactions() ShardedDataCacherNotifier
	SmartContractResults() ShardedDataCacherNotifier
	Headers() storage.Cacher
	HeadersNonces() Uint64Cacher
	MiniBlocks() storage.Cacher
	PeerChangesBlocks() storage.Cacher
	MetaBlocks() storage.Cacher
	MetaHeadersNonces() Uint64Cacher
}

PoolsHolder defines getters for data pools

type RequestData

type RequestData struct {
	Type  RequestDataType
	Value []byte
}

RequestData holds the requested data This struct will be serialized and sent to the other peers

func (*RequestData) Unmarshal

func (rd *RequestData) Unmarshal(marshalizer marshal.Marshalizer, message p2p.MessageP2P) error

Unmarshal sets the fields according to p2p.MessageP2P.Data() contents Errors if something went wrong

type RequestDataType

type RequestDataType byte

RequestDataType represents the data type for the requested data

const (
	// HashType indicates that the request data object is of type hash
	HashType RequestDataType = iota + 1
	// HashArrayType that the request data object contains a serialised array of hashes
	HashArrayType
	// NonceType indicates that the request data object is of type nonce (uint64)
	NonceType
)

func (RequestDataType) String

func (rdt RequestDataType) String() string

type Resolver

type Resolver interface {
	RequestDataFromHash(hash []byte) error
	ProcessReceivedMessage(message p2p.MessageP2P) error
}

Resolver defines what a data resolver should do

type ResolversContainer

type ResolversContainer interface {
	Get(key string) (Resolver, error)
	Add(key string, val Resolver) error
	AddMultiple(keys []string, resolvers []Resolver) error
	Replace(key string, val Resolver) error
	Remove(key string)
	Len() int
}

ResolversContainer defines a resolvers holder data type with basic functionality

type ResolversContainerFactory

type ResolversContainerFactory interface {
	Create() (ResolversContainer, error)
}

ResolversContainerFactory defines the functionality to create a resolvers container

type ResolversFinder

type ResolversFinder interface {
	ResolversContainer
	IntraShardResolver(baseTopic string) (Resolver, error)
	MetaChainResolver(baseTopic string) (Resolver, error)
	CrossShardResolver(baseTopic string, crossShard uint32) (Resolver, error)
}

ResolversFinder extends a container resolver and have 2 additional functionality

type ShardedDataCacherNotifier

type ShardedDataCacherNotifier interface {
	Notifier

	ShardDataStore(cacheId string) (c storage.Cacher)
	AddData(key []byte, data interface{}, cacheId string)
	SearchFirstData(key []byte) (value interface{}, ok bool)
	RemoveData(key []byte, cacheId string)
	RemoveSetOfDataFromPool(keys [][]byte, cacheId string)
	RemoveDataFromAllShards(key []byte)
	MergeShardStores(sourceCacheID, destCacheID string)
	MoveData(sourceCacheID, destCacheID string, key [][]byte)
	Clear()
	ClearShardStore(cacheId string)
	CreateShardStore(cacheId string)
}

ShardedDataCacherNotifier defines what a sharded-data structure can perform

type StorageService

type StorageService interface {
	// GetStorer returns the storer from the chain map
	GetStorer(unitType UnitType) storage.Storer
	// AddStorer will add a new storer to the chain map
	AddStorer(key UnitType, s storage.Storer)
	// Has returns true if the key is found in the selected Unit or false otherwise
	Has(unitType UnitType, key []byte) error
	// Get returns the value for the given key if found in the selected storage unit, nil otherwise
	Get(unitType UnitType, key []byte) ([]byte, error)
	// Put stores the key, value pair in the selected storage unit
	Put(unitType UnitType, key []byte, value []byte) error
	// GetAll gets all the elements with keys in the keys array, from the selected storage unit
	// If there is a missing key in the unit, it returns an error
	GetAll(unitType UnitType, keys [][]byte) (map[string][]byte, error)
	// Destroy removes the underlying files/resources used by the storage service
	Destroy() error
}

StorageService is the interface for data storage unit provided services

type StorageType

type StorageType uint8

const (
	CACHE     StorageType = 0
	LOCALDISC StorageType = 1
	NETWORK   StorageType = 2
)

type TopicHandler

type TopicHandler interface {
	HasTopic(name string) bool
	CreateTopic(name string, createChannelForTopic bool) error
	RegisterMessageProcessor(topic string, handler p2p.MessageProcessor) error
}

TopicHandler defines the functionality needed by structs to manage topics and message processors

type TopicMessageHandler

type TopicMessageHandler interface {
	MessageHandler
	TopicHandler
}

TopicMessageHandler defines the functionality needed by structs to manage topics, message processors and to send data to other peers

type TopicResolverSender

type TopicResolverSender interface {
	SendOnRequestTopic(rd *RequestData) error
	Send(buff []byte, peer p2p.PeerID) error
	TopicRequestSuffix() string
}

TopicResolverSender defines what sending operations are allowed for a topic resolver

type Uint64Cacher

type Uint64Cacher interface {
	Clear()
	Put(uint64, interface{}) bool
	Get(uint64) (interface{}, bool)
	Has(uint64) bool
	Peek(uint64) (interface{}, bool)
	HasOrAdd(uint64, interface{}) (bool, bool)
	Remove(uint64)
	RemoveOldest()
	Keys() []uint64
	Len() int
	RegisterHandler(handler func(nonce uint64))
}

Uint64Cacher defines a cacher-type struct that uses uint64 keys and []byte values (usually hashes)

type UnitType

type UnitType uint8

UnitType is the type for Storage unit identifiers

const (
	// TransactionUnit is the transactions storage unit identifier
	TransactionUnit UnitType = 0
	// MiniBlockUnit is the transaction block body storage unit identifier
	MiniBlockUnit UnitType = 1
	// PeerChangesUnit is the peer change block body storage unit identifier
	PeerChangesUnit UnitType = 2
	// BlockHeaderUnit is the Block Headers Storage unit identifier
	BlockHeaderUnit UnitType = 3
	// MetaBlockUnit is the metachain blocks storage unit identifier
	MetaBlockUnit UnitType = 4
	// MetaShardDataUnit is the metachain shard data unit identifier
	MetaShardDataUnit UnitType = 5
	// MetaPeerDataUnit is the metachain peer data unit identifier
	MetaPeerDataUnit UnitType = 6
	// SmartContractResultUnit is the smart contract results unit identifier
	SmartContractResultUnit UnitType = 7
	// MetaHdrNonceHashDataUnit is the meta header nonce-hash pair data unit identifier
	MetaHdrNonceHashDataUnit UnitType = 8

	//TODO: Add only unit types lower than 100
	// ShardHdrNonceHashDataUnit is the header nonce-hash pair data unit identifier
	ShardHdrNonceHashDataUnit UnitType = 100
)