storage

Documentation

Index

• Variables
• func IsNotFoundInStorageErr(err error) bool
• func MonitorNewCache(tag string, sizeInBytes uint64)
• type AdaptedSizedLRUCache
• type Batcher
• type Cacher
• type CustomDatabaseRemoverHandler
• type DirectoryReaderHandler
• type EpochStartNotifier
• type ForEachItem
• type LRUCacheHandler
• type LatestDataFromStorage
• type LatestStorageDataProviderHandler
• type PathManagerHandler
• type Persister
• type PersisterFactory
• type SerializedStoredData
• type ShardCoordinator
• type SizedLRUCacheHandler
• type StoredDataFactory
• type Storer
• type StorerWithPutInEpoch
• type TimeCacher
• type UnitOpenerHandler

Variables

var ErrBootstrapDataNotFoundInStorage = errors.New("didn't find any bootstrap data in storage")

ErrBootstrapDataNotFoundInStorage signals that no BootstrapData was find in the storage

var ErrCacheCapacityInvalid = errors.New("cache capacity is less than 1")

ErrCacheCapacityInvalid signals that capacity of cache is less than 1

var ErrCacheSizeInvalid = errors.New("cache size is less than 1")

ErrCacheSizeInvalid signals that size of cache is less than 1

var ErrCacheSizeIsLowerThanBatchSize = errors.New("cache size is lower than batch size")

ErrCacheSizeIsLowerThanBatchSize signals that size of cache is lower than size of batch

var ErrCannotComputeStorageOldestEpoch = errors.New("could not compute the oldest epoch for storage")

ErrCannotComputeStorageOldestEpoch signals an issue when computing the oldest epoch for storage

var ErrClosingPersisters = errors.New("cannot close all the persisters")

ErrClosingPersisters signals that not all persisters were closed

var ErrDestroyingUnit = errors.New("destroy unit didn't remove all the persisters")

ErrDestroyingUnit signals that the destroy unit method did not manage to destroy all the persisters in a pruning storer

var ErrEmptyKey = errors.New("key is empty")

ErrEmptyKey is raised when a key is empty

var ErrEmptyPruningPathTemplate = errors.New("empty path template for pruning storers")

ErrEmptyPruningPathTemplate signals that an empty path template for pruning storers has been provided

var ErrEmptyStaticPathTemplate = errors.New("empty path template for static storers")

ErrEmptyStaticPathTemplate signals that an empty path template for static storers has been provided

var ErrFailedCacheEviction = errors.New("failed eviction within cache")

ErrFailedCacheEviction signals a failed eviction within a cache

var ErrImmuneItemsCapacityReached = errors.New("capacity reached for immune items")

ErrImmuneItemsCapacityReached signals that capacity for immune items is reached

var ErrInvalidBatch = errors.New("batch is invalid")

ErrInvalidBatch is raised when the used batch is invalid

var ErrInvalidCacheExpiry = errors.New("invalid cache expiry")

ErrInvalidCacheExpiry signals that an invalid cache expiry was provided

var ErrInvalidConfig = errors.New("invalid config")

ErrInvalidConfig signals an invalid config

var ErrInvalidDatabasePath = errors.New("invalid database path")

ErrInvalidDatabasePath signals that an invalid database path has been provided

var ErrInvalidDefaultSpan = errors.New("invalid default span")

ErrInvalidDefaultSpan signals that an invalid default span was provided

var ErrInvalidNumOpenFiles = errors.New("maxOpenFiles is invalid")

ErrInvalidNumOpenFiles is raised when the max num of open files is less than 1

var ErrInvalidNumberOfActivePersisters = errors.New("invalid number of active persisters")

ErrInvalidNumberOfActivePersisters signals that an invalid number of active persisters has been provided

var ErrInvalidNumberOfEpochsToSave = errors.New("invalid number of epochs to save")

ErrInvalidNumberOfEpochsToSave signals that an invalid number of epochs to save has been provided

var ErrInvalidNumberOfOldPersisters = errors.New("invalid number of old active persisters")

ErrInvalidNumberOfOldPersisters signals that an invalid number of old persisters has been provided

var ErrInvalidNumberOfPersisters = errors.New("invalid number of active persisters")

ErrInvalidNumberOfPersisters signals that an invalid number of persisters has been provided

var ErrInvalidPruningPathTemplate = errors.New("invalid path template for pruning storers")

ErrInvalidPruningPathTemplate signals that an invalid path template for pruning storers has been provided

var ErrInvalidStaticPathTemplate = errors.New("invalid path template for static storers")

ErrInvalidStaticPathTemplate signals that an invalid path template for static storers has been provided

var ErrItemAlreadyInCache = errors.New("item already in cache")

ErrItemAlreadyInCache signals that an item is already in cache

var ErrKeyNotFound = errors.New("key not found")

ErrKeyNotFound is raised when a key is not found

var ErrLRUCacheInvalidSize = errors.New("wrong size in bytes value for LRU cache")

ErrLRUCacheInvalidSize signals that the provided size in bytes value for LRU cache is invalid

var ErrLRUCacheWithProvidedSize = errors.New("LRU cache does not support size in bytes")

ErrLRUCacheWithProvidedSize signals that a simple LRU cache is wanted but the user provided a positive size in bytes value

var ErrNegativeSizeInBytes = errors.New("negative size in bytes")

ErrNegativeSizeInBytes signals that the provided size in bytes value is negative

var ErrNilCacher = errors.New("expected not nil cacher")

ErrNilCacher is raised when a nil cacher is provided

var ErrNilCustomDatabaseRemover = errors.New("custom database remover")

ErrNilCustomDatabaseRemover signals that a nil custom database remover has been provided

var ErrNilEpochStartNotifier = errors.New("nil epoch start notifier")

ErrNilEpochStartNotifier signals that a nil epoch start notifier has been provided

var ErrNilMarshalizer = errors.New("nil marshalizer")

ErrNilMarshalizer signals that a nil marshalizer has been provided

var ErrNilNodeTypeProvider = errors.New("nil node type provider")

ErrNilNodeTypeProvider signals that a nil node type provider has been provided

var ErrNilOldDataCleanerProvider = errors.New("nil old data cleaner provider")

ErrNilOldDataCleanerProvider signals that a nil old data cleaner provider has been provided

var ErrNilPathManager = errors.New("nil path manager")

ErrNilPathManager signals that a nil path manager has been provided

var ErrNilPersister = errors.New("expected not nil persister")

ErrNilPersister is raised when a nil persister is provided

var ErrNilPersisterFactory = errors.New("nil persister factory")

ErrNilPersisterFactory signals that a nil persister factory has been provided

var ErrNilShardCoordinator = errors.New("nil shard coordinator")

ErrNilShardCoordinator signals that a nil shard coordinator has been provided

var ErrNilStatusHandler = errors.New("nil status handler")

ErrNilStatusHandler signals that a nil status handler has been provided

var ErrNilStorageListProvider = errors.New("nil storage list provider")

ErrNilStorageListProvider signals that a nil storage list provided has been provided

var ErrNilStoredDataFactory = errors.New("nil stored data factory")

ErrNilStoredDataFactory signals that a nil stored data factory has been provided

var ErrNilTimeCache = errors.New("nil time cache")

ErrNilTimeCache signals that a nil time cache has been provided

var ErrNilTxGasHandler = errors.New("nil tx gas handler")

ErrNilTxGasHandler signals that a nil tx gas handler was provided

var ErrNotSupportedCacheType = errors.New("not supported cache type")

ErrNotSupportedCacheType is raised when an unsupported cache type is provided

var ErrNotSupportedDBType = errors.New("not supported db type")

ErrNotSupportedDBType is raised when an unsupported database type is provided

var ErrNotSupportedHashType = errors.New("hash type not supported")

ErrNotSupportedHashType is raised when an unsupported hasher is provided

var ErrOldestEpochNotAvailable = errors.New("oldest epoch not available")

ErrOldestEpochNotAvailable signals that fetching the oldest epoch is not available

var ErrWrongTypeAssertion = errors.New("wrong type assertion")

ErrWrongTypeAssertion is thrown when a wrong type assertion is spotted

Functions

func IsNotFoundInStorageErr

func IsNotFoundInStorageErr(err error) bool

IsNotFoundInStorageErr returns whether an error is a "not found in storage" error. Currently, "item not found" storage errors are untyped (thus not distinguishable from others). E.g. see "pruningStorer.go". As a workaround, we test the error message for a match.

func MonitorNewCache

func MonitorNewCache(tag string, sizeInBytes uint64)

MonitorNewCache adds the size in the global cumulated size variable

Types

type AdaptedSizedLRUCache

type AdaptedSizedLRUCache interface {
	SizedLRUCacheHandler
	AddSizedAndReturnEvicted(key, value interface{}, sizeInBytes int64) map[interface{}]interface{}
	IsInterfaceNil() bool
}

AdaptedSizedLRUCache defines a cache that returns the evicted value

type Batcher

type Batcher interface {
	// Put inserts one entry - key, value pair - into the batch
	Put(key []byte, val []byte) error
	// Get returns the value from the batch
	Get(key []byte) []byte
	// Delete deletes the batch
	Delete(key []byte) error
	// Reset clears the contents of the batch
	Reset()
	// IsRemoved returns true if the provided key is marked for deletion
	IsRemoved(key []byte) bool
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

Batcher allows to batch the data first then write the batch to the persister in one go

type Cacher

type Cacher interface {
	// Clear is used to completely clear the cache.
	Clear()
	// Put adds a value to the cache.  Returns true if an eviction occurred.
	Put(key []byte, value interface{}, sizeInBytes int) (evicted bool)
	// Get looks up a key's value from the cache.
	Get(key []byte) (value interface{}, ok bool)
	// Has checks if a key is in the cache, without updating the
	// recent-ness or deleting it for being stale.
	Has(key []byte) bool
	// Peek returns the key value (or undefined if not found) without updating
	// the "recently used"-ness of the key.
	Peek(key []byte) (value interface{}, ok bool)
	// HasOrAdd checks if a key is in the cache without updating the
	// recent-ness or deleting it for being stale, and if not adds the value.
	HasOrAdd(key []byte, value interface{}, sizeInBytes int) (has, added bool)
	// Remove removes the provided key from the cache.
	Remove(key []byte)
	// Keys returns a slice of the keys in the cache, from oldest to newest.
	Keys() [][]byte
	// Len returns the number of items in the cache.
	Len() int
	// SizeInBytesContained returns the size in bytes of all contained elements
	SizeInBytesContained() uint64
	// MaxSize returns the maximum number of items which can be stored in the cache.
	MaxSize() int
	// RegisterHandler registers a new handler to be called when a new data is added
	RegisterHandler(handler func(key []byte, value interface{}), id string)
	// UnRegisterHandler deletes the handler from the list
	UnRegisterHandler(id string)
	// Close closes the underlying temporary db if the cacher implementation has one,
	// otherwise it does nothing
	Close() error
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

Cacher provides caching services

type CustomDatabaseRemoverHandler

type CustomDatabaseRemoverHandler interface {
	ShouldRemove(dbIdentifier string, epoch uint32) bool
	IsInterfaceNil() bool
}

CustomDatabaseRemoverHandler defines the behaviour of a component that should tell if a database is removable or not

type DirectoryReaderHandler

type DirectoryReaderHandler interface {
	ListFilesAsString(directoryPath string) ([]string, error)
	ListDirectoriesAsString(directoryPath string) ([]string, error)
	ListAllAsString(directoryPath string) ([]string, error)
	IsInterfaceNil() bool
}

DirectoryReaderHandler defines which actions should be done by a directory reader

type EpochStartNotifier

type EpochStartNotifier interface {
	RegisterHandler(handler epochStart.ActionHandler)
	IsInterfaceNil() bool
}

EpochStartNotifier defines which actions should be done for handling new epoch's events

type ForEachItem

type ForEachItem func(key []byte, value interface{})

ForEachItem is an iterator callback

type LRUCacheHandler

type LRUCacheHandler interface {
	Add(key, value interface{}) bool
	Get(key interface{}) (value interface{}, ok bool)
	Contains(key interface{}) (ok bool)
	ContainsOrAdd(key, value interface{}) (ok, evicted bool)
	Peek(key interface{}) (value interface{}, ok bool)
	Remove(key interface{}) bool
	Keys() []interface{}
	Len() int
	Purge()
}

LRUCacheHandler is the interface for LRU cache.

type LatestDataFromStorage

type LatestDataFromStorage struct {
	Epoch           uint32
	ShardID         uint32
	LastRound       int64
	EpochStartRound uint64
}

LatestDataFromStorage represents the DTO structure to return from storage

type LatestStorageDataProviderHandler

type LatestStorageDataProviderHandler interface {
	GetParentDirectory() string
	GetParentDirAndLastEpoch() (string, uint32, error)
	Get() (LatestDataFromStorage, error)
	GetShardsFromDirectory(path string) ([]string, error)
	IsInterfaceNil() bool
}

LatestStorageDataProviderHandler defines which actions be done by a component who fetches latest data from storage

type PathManagerHandler

type PathManagerHandler interface {
	PathForEpoch(shardId string, epoch uint32, identifier string) string
	PathForStatic(shardId string, identifier string) string
	DatabasePath() string
	IsInterfaceNil() bool
}

PathManagerHandler defines which actions should be done for generating paths for databases directories

type Persister

type Persister interface {
	// Put add the value to the (key, val) persistence medium
	Put(key, val []byte) error
	// Get gets the value associated to the key
	Get(key []byte) ([]byte, error)
	// Has returns true if the given key is present in the persistence medium
	Has(key []byte) error
	// Close closes the files/resources associated to the persistence medium
	Close() error
	// Remove removes the data associated to the given key
	Remove(key []byte) error
	// Destroy removes the persistence medium stored data
	Destroy() error
	// DestroyClosed removes the already closed persistence medium stored data
	DestroyClosed() error
	RangeKeys(handler func(key []byte, val []byte) bool)
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

Persister provides storage of data services in a database like construct

type PersisterFactory

type PersisterFactory interface {
	Create(path string) (Persister, error)
	IsInterfaceNil() bool
}

PersisterFactory defines which actions should be done for creating a persister

type SerializedStoredData

type SerializedStoredData interface {
	GetSerialized() []byte
	SetSerialized([]byte)
}

SerializedStoredData defines a data type that has the serialized data as a field

type ShardCoordinator

type ShardCoordinator interface {
	NumberOfShards() uint32
	ComputeId(address []byte) uint32
	SelfId() uint32
	SameShard(firstAddress, secondAddress []byte) bool
	CommunicationIdentifier(destShardID uint32) string
	IsInterfaceNil() bool
}

ShardCoordinator defines what a shard state coordinator should hold

type SizedLRUCacheHandler

type SizedLRUCacheHandler interface {
	AddSized(key, value interface{}, sizeInBytes int64) bool
	Get(key interface{}) (value interface{}, ok bool)
	Contains(key interface{}) (ok bool)
	AddSizedIfMissing(key, value interface{}, sizeInBytes int64) (ok, evicted bool)
	Peek(key interface{}) (value interface{}, ok bool)
	Remove(key interface{}) bool
	Keys() []interface{}
	Len() int
	SizeInBytesContained() uint64
	Purge()
}

SizedLRUCacheHandler is the interface for size capable LRU cache.

type StoredDataFactory

type StoredDataFactory interface {
	CreateEmpty() interface{}
	IsInterfaceNil() bool
}

StoredDataFactory creates empty objects of the stored data type

type Storer

type Storer interface {
	Put(key, data []byte) error
	PutInEpoch(key, data []byte, epoch uint32) error
	Get(key []byte) ([]byte, error)
	Has(key []byte) error
	SearchFirst(key []byte) ([]byte, error)
	RemoveFromCurrentEpoch(key []byte) error
	Remove(key []byte) error
	ClearCache()
	DestroyUnit() error
	GetFromEpoch(key []byte, epoch uint32) ([]byte, error)
	GetBulkFromEpoch(keys [][]byte, epoch uint32) ([]storage.KeyValuePair, error)
	GetOldestEpoch() (uint32, error)
	RangeKeys(handler func(key []byte, val []byte) bool)
	Close() error
	IsInterfaceNil() bool
}

Storer provides storage services in a two layered storage construct, where the first layer is represented by a cache and second layer by a persitent storage (DB-like)

type StorerWithPutInEpoch

type StorerWithPutInEpoch interface {
	Storer
	SetEpochForPutOperation(epoch uint32)
}

StorerWithPutInEpoch is an extended storer with the ability to set the epoch which will be used for put operations

type TimeCacher

type TimeCacher interface {
	Add(key string) error
	Upsert(key string, span time.Duration) error
	Has(key string) bool
	Sweep()
	IsInterfaceNil() bool
}

TimeCacher defines the cache that can keep a record for a bounded time

type UnitOpenerHandler

type UnitOpenerHandler interface {
	OpenDB(dbConfig config.DBConfig, shardID uint32, epoch uint32) (Storer, error)
	GetMostRecentStorageUnit(config config.DBConfig) (Storer, error)
	IsInterfaceNil() bool
}

UnitOpenerHandler defines which actions should be done for opening storage units