storage

Documentation

Index

• Constants
• Variables
• func IsNotFoundInStorageErr(err error) bool
• type AdaptedSizedLRUCache
• type Batcher
• type Cacher
• type CustomDatabaseRemoverHandler
• type DBConfigHandler
• type DirectoryReaderHandler
• type LatestDataFromStorage
• type LatestStorageDataProviderHandler
• type ManagedPeersHolder
• type PathManagerHandler
• type Persister
• type PersisterCreator
• type PersisterFactory
• type PersisterFactoryHandler
• type ShardCoordinator
• type ShardIDProvider
• type SizedLRUCacheHandler
• type StateStatsHandler
• type StoredDataFactory
• type Storer
• type StorerWithPutInEpoch
• type TimeCacher
• type UnitOpenerHandler

Constants

const DefaultDBPath = "db"

DefaultDBPath is the default path for nodes databases

const DefaultEpochString = "Epoch"

DefaultEpochString is the default folder root name for node per epoch databases

const DefaultShardString = "Shard"

DefaultShardString is the default folder root name for per shard databases

const DefaultStaticDbString = "Static"

DefaultStaticDbString is the default name for the static databases (not changing with epoch)

const MaxRetriesToCreateDB = common.MaxRetriesToCreateDB

MaxRetriesToCreateDB represents the maximum number of times to try to create DB if it failed

const PathEpochPlaceholder = "[E]"

PathEpochPlaceholder represents the placeholder for the epoch number in paths

const PathIdentifierPlaceholder = "[I]"

PathIdentifierPlaceholder represents the placeholder for the identifier in paths

const PathShardPlaceholder = "[S]"

PathShardPlaceholder represents the placeholder for the shard ID in paths

const ShardedDataNumItemsToPreemptivelyEvict = uint32(1000)

ShardedDataNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm

const SleepTimeBetweenCreateDBRetries = common.SleepTimeBetweenCreateDBRetries

SleepTimeBetweenCreateDBRetries represents the number of seconds to sleep between DB creates

const TxPoolDestinationMeNumItemsToPreemptivelyEvict = uint32(1000)

TxPoolDestinationMeNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm

const TxPoolSourceMeNumItemsToPreemptivelyEvict = uint32(50000)

TxPoolSourceMeNumItemsToPreemptivelyEvict is a configuration of the eviction algorithm

Variables

var ErrBootstrapDataNotFoundInStorage = errors.New("didn't find any bootstrap data in storage")

ErrBootstrapDataNotFoundInStorage signals that no BootstrapData was find in the storage

var ErrCacheSizeInvalid = storageErrors.ErrCacheSizeInvalid

ErrCacheSizeInvalid signals that size of cache is less than 1

var ErrCacheSizeIsLowerThanBatchSize = errors.New("cache size is lower than batch size")

ErrCacheSizeIsLowerThanBatchSize signals that size of cache is lower than size of batch

var ErrCannotComputeStorageOldestEpoch = errors.New("could not compute the oldest epoch for storage")

ErrCannotComputeStorageOldestEpoch signals an issue when computing the oldest epoch for storage

var ErrClosingPersisters = errors.New("cannot close all the persisters")

ErrClosingPersisters signals that not all persisters were closed

var ErrDBIsClosed = storageErrors.ErrDBIsClosed

ErrDBIsClosed is raised when the DB is closed

var ErrDestroyingUnit = errors.New("destroy unit didn't remove all the persisters")

ErrDestroyingUnit signals that the destroy unit method did not manage to destroy all the persisters in a pruning storer

var ErrEpochKeepIsLowerThanNumActive = errors.New("num epochs to keep is lower than num active epochs")

ErrEpochKeepIsLowerThanNumActive signals that num epochs to keep is lower than num active epochs

var ErrInvalidConfig = storageErrors.ErrInvalidConfig

ErrInvalidConfig signals an invalid config

var ErrInvalidFilePath = errors.New("invalid file path")

ErrInvalidFilePath signals that an invalid file path has been provided

var ErrInvalidNumberOfActivePersisters = errors.New("invalid number of active persisters")

ErrInvalidNumberOfActivePersisters signals that an invalid number of active persisters has been provided

var ErrInvalidNumberOfEpochsToSave = errors.New("invalid number of epochs to save")

ErrInvalidNumberOfEpochsToSave signals that an invalid number of epochs to save has been provided

var ErrInvalidNumberOfOldPersisters = errors.New("invalid number of old active persisters")

ErrInvalidNumberOfOldPersisters signals that an invalid number of old persisters has been provided

var ErrInvalidNumberOfPersisters = errors.New("invalid number of active persisters")

ErrInvalidNumberOfPersisters signals that an invalid number of persisters has been provided

var ErrKeyNotFound = storageErrors.ErrKeyNotFound

ErrKeyNotFound is raised when a key is not found

var ErrNilBootstrapDataProvider = errors.New("nil bootstrap data provider")

ErrNilBootstrapDataProvider signals that a nil bootstrap data provider has been provided

var ErrNilCustomDatabaseRemover = errors.New("custom database remover")

ErrNilCustomDatabaseRemover signals that a nil custom database remover has been provided

var ErrNilDBConfigHandler = errors.New("nil db config handler")

ErrNilDBConfigHandler signals that a nil db config handler has been provided

var ErrNilDirectoryReader = errors.New("nil directory reader")

ErrNilDirectoryReader signals that a nil directory reader has been provided

var ErrNilEpochStartNotifier = errors.New("nil epoch start notifier")

ErrNilEpochStartNotifier signals that a nil epoch start notifier has been provided

var ErrNilLatestStorageDataProvider = errors.New("nil latest storage data provider")

ErrNilLatestStorageDataProvider signals that a nil latest storage data provider has been provided

var ErrNilManagedPeersHolder = errors.New("nil managed peers holder")

ErrNilManagedPeersHolder signals that a nil managed peers holder has been provided

var ErrNilMarshalizer = errors.New("nil marshalizer")

ErrNilMarshalizer signals that a nil marshalizer has been provided

var ErrNilNodeTypeProvider = errors.New("nil node type provider")

ErrNilNodeTypeProvider signals that a nil node type provider has been provided

var ErrNilOldDataCleanerProvider = errors.New("nil old data cleaner provider")

ErrNilOldDataCleanerProvider signals that a nil old data cleaner provider has been provided

var ErrNilPathManager = errors.New("nil path manager")

ErrNilPathManager signals that a nil path manager has been provided

var ErrNilPersisterFactory = errors.New("nil persister factory")

ErrNilPersisterFactory signals that a nil persister factory has been provided

var ErrNilPersistersTracker = errors.New("nil persisters tracker provided")

ErrNilPersistersTracker signals that a nil persisters tracker has been provided

var ErrNilShardCoordinator = errors.New("nil shard coordinator")

ErrNilShardCoordinator signals that a nil shard coordinator has been provided

var ErrNilShardIDProvider = errors.New("nil shard id provider")

ErrNilShardIDProvider signals that a nil shard id provider has been provided

var ErrNilStatsCollector = errors.New("nil stats collector provided")

ErrNilStatsCollector signals that a nil stats collector has been provided

var ErrNilStorageListProvider = errors.New("nil storage list provider")

ErrNilStorageListProvider signals that a nil storage list provided has been provided

var ErrNotSupportedCacheType = storageErrors.ErrNotSupportedCacheType

ErrNotSupportedCacheType is raised when an unsupported cache type is provided

var ErrNotSupportedDBType = storageErrors.ErrNotSupportedDBType

ErrNotSupportedDBType is raised when an unsupported database type is provided

var ErrNotSupportedShardIDProviderType = errors.New("invalid shard id provider type has been provided")

ErrNotSupportedShardIDProviderType is raised when an unsupported shard id provider type is provided

var ErrWrongTypeAssertion = errors.New("wrong type assertion")

ErrWrongTypeAssertion is thrown when a wrong type assertion is spotted

Functions

func IsNotFoundInStorageErr

func IsNotFoundInStorageErr(err error) bool

IsNotFoundInStorageErr returns whether an error is a "not found in storage" error. Currently, "item not found" storage errors are untyped (thus not distinguishable from others). E.g. see "pruningStorer.go". As a workaround, we test the error message for a match.

Types

type AdaptedSizedLRUCache

type AdaptedSizedLRUCache interface {
	SizedLRUCacheHandler
	AddSizedAndReturnEvicted(key, value interface{}, sizeInBytes int64) map[interface{}]interface{}
	IsInterfaceNil() bool
}

AdaptedSizedLRUCache defines a cache that returns the evicted value

type Batcher

type Batcher interface {
	// Put inserts one entry - key, value pair - into the batch
	Put(key []byte, val []byte) error
	// Get returns the value from the batch
	Get(key []byte) []byte
	// Delete deletes the batch
	Delete(key []byte) error
	// Reset clears the contents of the batch
	Reset()
	// IsRemoved returns true if the provided key is marked for deletion
	IsRemoved(key []byte) bool
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

Batcher allows to batch the data first then write the batch to the persister in one go

type Cacher

type Cacher interface {
	// Clear is used to completely clear the cache.
	Clear()
	// Put adds a value to the cache.  Returns true if an eviction occurred.
	Put(key []byte, value interface{}, sizeInBytes int) (evicted bool)
	// Get looks up a key's value from the cache.
	Get(key []byte) (value interface{}, ok bool)
	// Has checks if a key is in the cache, without updating the
	// recent-ness or deleting it for being stale.
	Has(key []byte) bool
	// Peek returns the key value (or undefined if not found) without updating
	// the "recently used"-ness of the key.
	Peek(key []byte) (value interface{}, ok bool)
	// HasOrAdd checks if a key is in the cache without updating the
	// recent-ness or deleting it for being stale, and if not adds the value.
	HasOrAdd(key []byte, value interface{}, sizeInBytes int) (has, added bool)
	// Remove removes the provided key from the cache.
	Remove(key []byte)
	// Keys returns a slice of the keys in the cache, from oldest to newest.
	Keys() [][]byte
	// Len returns the number of items in the cache.
	Len() int
	// SizeInBytesContained returns the size in bytes of all contained elements
	SizeInBytesContained() uint64
	// MaxSize returns the maximum number of items which can be stored in the cache.
	MaxSize() int
	// RegisterHandler registers a new handler to be called when a new data is added
	RegisterHandler(handler func(key []byte, value interface{}), id string)
	// UnRegisterHandler deletes the handler from the list
	UnRegisterHandler(id string)
	// Close closes the underlying temporary db if the cacher implementation has one,
	// otherwise it does nothing
	Close() error
	// IsInterfaceNil returns true if there is no value under the interface
	IsInterfaceNil() bool
}

Cacher provides caching services

type CustomDatabaseRemoverHandler

type CustomDatabaseRemoverHandler interface {
	ShouldRemove(dbIdentifier string, epoch uint32) bool
	IsInterfaceNil() bool
}

CustomDatabaseRemoverHandler defines the behaviour of a component that should tell if a database is removable or not

type DBConfigHandler

type DBConfigHandler interface {
	GetDBConfig(path string) (*config.DBConfig, error)
	SaveDBConfigToFilePath(path string, dbConfig *config.DBConfig) error
	IsInterfaceNil() bool
}

DBConfigHandler defines the behaviour of a component that will handle db config

type DirectoryReaderHandler

type DirectoryReaderHandler interface {
	ListFilesAsString(directoryPath string) ([]string, error)
	ListDirectoriesAsString(directoryPath string) ([]string, error)
	ListAllAsString(directoryPath string) ([]string, error)
	IsInterfaceNil() bool
}

DirectoryReaderHandler defines which actions should be done by a directory reader

type LatestDataFromStorage

type LatestDataFromStorage = types.LatestDataFromStorage

LatestDataFromStorage represents the DTO structure to return from storage

type LatestStorageDataProviderHandler

type LatestStorageDataProviderHandler interface {
	GetParentDirectory() string
	GetParentDirAndLastEpoch() (string, uint32, error)
	Get() (LatestDataFromStorage, error)
	GetShardsFromDirectory(path string) ([]string, error)
	IsInterfaceNil() bool
}

LatestStorageDataProviderHandler defines which actions be done by a component who fetches the latest data from storage

type ManagedPeersHolder

type ManagedPeersHolder interface {
	IsMultiKeyMode() bool
	IsInterfaceNil() bool
}

ManagedPeersHolder defines the operations of an entity that holds managed identities for a node

type PathManagerHandler

type PathManagerHandler interface {
	PathForEpoch(shardId string, epoch uint32, identifier string) string
	PathForStatic(shardId string, identifier string) string
	DatabasePath() string
	IsInterfaceNil() bool
}

PathManagerHandler defines which actions should be done for generating paths for databases directories

type Persister

type Persister = types.Persister

Persister provides storage of data services in a database like construct

type PersisterCreator

type PersisterCreator = types.PersisterCreator

PersisterCreator defines the behavour of a component which is able to create a persister

type PersisterFactory

type PersisterFactory interface {
	Create(path string) (Persister, error)
	IsInterfaceNil() bool
}

PersisterFactory defines which actions should be done for creating a persister

type PersisterFactoryHandler

type PersisterFactoryHandler interface {
	Create(path string) (Persister, error)
	CreateWithRetries(path string) (Persister, error)
	IsInterfaceNil() bool
}

PersisterFactoryHandler defines the behaviour of a component which is able to create persisters

type ShardCoordinator

type ShardCoordinator interface {
	NumberOfShards() uint32
	ComputeId(address []byte) uint32
	SelfId() uint32
	SameShard(firstAddress, secondAddress []byte) bool
	CommunicationIdentifier(destShardID uint32) string
	IsInterfaceNil() bool
}

ShardCoordinator defines what a shard state coordinator should hold

type ShardIDProvider

type ShardIDProvider interface {
	ComputeId(key []byte) uint32
	NumberOfShards() uint32
	GetShardIDs() []uint32
	IsInterfaceNil() bool
}

ShardIDProvider defines what a component which is able to provide persister id per key should do

type SizedLRUCacheHandler

type SizedLRUCacheHandler interface {
	AddSized(key, value interface{}, sizeInBytes int64) bool
	Get(key interface{}) (value interface{}, ok bool)
	Contains(key interface{}) (ok bool)
	AddSizedIfMissing(key, value interface{}, sizeInBytes int64) (ok, evicted bool)
	Peek(key interface{}) (value interface{}, ok bool)
	Remove(key interface{}) bool
	Keys() []interface{}
	Len() int
	SizeInBytesContained() uint64
	Purge()
}

SizedLRUCacheHandler is the interface for size capable LRU cache.

type StateStatsHandler

type StateStatsHandler interface {
	IncrementCache()
	IncrementSnapshotCache()
	IncrementPersister(epoch uint32)
	IncrementSnapshotPersister(epoch uint32)
}

StateStatsHandler defines the behaviour needed to handler storage statistics

type StoredDataFactory

type StoredDataFactory interface {
	CreateEmpty() interface{}
	IsInterfaceNil() bool
}

StoredDataFactory creates empty objects of the stored data type

type Storer

type Storer interface {
	Put(key, data []byte) error
	PutInEpoch(key, data []byte, epoch uint32) error
	Get(key []byte) ([]byte, error)
	Has(key []byte) error
	SearchFirst(key []byte) ([]byte, error)
	RemoveFromCurrentEpoch(key []byte) error
	Remove(key []byte) error
	ClearCache()
	DestroyUnit() error
	GetFromEpoch(key []byte, epoch uint32) ([]byte, error)
	GetBulkFromEpoch(keys [][]byte, epoch uint32) ([]data.KeyValuePair, error)
	GetOldestEpoch() (uint32, error)
	RangeKeys(handler func(key []byte, val []byte) bool)
	Close() error
	IsInterfaceNil() bool
}

Storer provides storage services in a two layered storage construct, where the first layer is represented by a cache and second layer by a persistent storage (DB-like)

type StorerWithPutInEpoch

type StorerWithPutInEpoch interface {
	Storer
	SetEpochForPutOperation(epoch uint32)
}

StorerWithPutInEpoch is an extended storer with the ability to set the epoch which will be used for put operations

type TimeCacher

type TimeCacher interface {
	Add(key string) error
	Upsert(key string, span time.Duration) error
	Has(key string) bool
	Sweep()
	IsInterfaceNil() bool
}

TimeCacher defines the cache that can keep a record for a bounded time

type UnitOpenerHandler

type UnitOpenerHandler interface {
	OpenDB(dbConfig config.DBConfig, shardID uint32, epoch uint32) (Storer, error)
	GetMostRecentStorageUnit(config config.DBConfig) (Storer, error)
	IsInterfaceNil() bool
}

UnitOpenerHandler defines which actions should be done for opening storage units