storage

Documentation

Index

• Variables
• type Batch
• type BatchedStore
• type Batcher
• type ChunkGetterDeleter
• type ChunkLocker
• type ChunkState
• type ChunkStore
• type Cloner
• type Deleter
• type Descriptor
  • func (d *Descriptor) String() string
• type Filter
• type Getter
• type GetterFunc
  • func (f GetterFunc) Get(ctx context.Context, address swarm.Address) (swarm.Chunk, error)
• type Hasser
• type InMemTxRevertOpStore
  • func NewInMemTxRevertOpStore[K, V any](revOpsFn map[TxOpCode]TxRevertFn[K, V]) *InMemTxRevertOpStore[K, V]
  • func (s *InMemTxRevertOpStore[K, V]) Append(ops ...*TxRevertOp[K, V]) error
  • func (s *InMemTxRevertOpStore[K, V]) Clean() error
  • func (s *InMemTxRevertOpStore[K, V]) Revert() error
• type Item
• type IterateChunkFn
• type IterateFn
• type Key
• type Marshaler
• type NoOpTxRevertOpStore
  • func (s *NoOpTxRevertOpStore[K, V]) Append(...*TxRevertOp[K, V]) error
  • func (s *NoOpTxRevertOpStore[K, V]) Clean() error
  • func (s *NoOpTxRevertOpStore[K, V]) Revert() error
• type Order
• type PullSubscriber
• type PushReporter
• type PushSubscriber
• type Putter
• type PutterFunc
  • func (f PutterFunc) Put(ctx context.Context, chunk swarm.Chunk) error
• type Query
  • func (q Query) Validate() error
• type QueryItemProperty
• type ReadOnlyChunkStore
• type Reader
• type Recoverer
• type Repository
  • func NewRepository(txIndexStore TxStore, txChunkStore TxChunkStore, locker ChunkLocker) Repository
• type Result
• type SizeReporter
• type StateIterFunc
• type StateStorer
• type StateStorerCleaner
• type Store
• type Tx
• type TxChunkStore
• type TxChunkStoreBase
  • func (s *TxChunkStoreBase) Close() error
  • func (s *TxChunkStoreBase) Delete(ctx context.Context, address swarm.Address) error
  • func (s *TxChunkStoreBase) Get(ctx context.Context, address swarm.Address) (swarm.Chunk, error)
  • func (s *TxChunkStoreBase) Has(ctx context.Context, address swarm.Address) (bool, error)
  • func (s *TxChunkStoreBase) Iterate(ctx context.Context, fn IterateChunkFn) error
  • func (s *TxChunkStoreBase) Put(ctx context.Context, chunk swarm.Chunk) error
  • func (s *TxChunkStoreBase) Rollback() error
• type TxOpCode
• type TxRevertFn
• type TxRevertOp
• type TxRevertOpStore
• type TxState
  • func NewTxState(ctx context.Context) *TxState
  • func (tx *TxState) AwaitDone() <-chan struct{}
  • func (tx *TxState) Done() error
  • func (tx *TxState) IsDone() error
• type TxStore
• type TxStoreBase
  • func (s *TxStoreBase) Batch(ctx context.Context) (Batch, error)
  • func (s *TxStoreBase) Close() error
  • func (s *TxStoreBase) Count(key Key) (int, error)
  • func (s *TxStoreBase) Delete(item Item) error
  • func (s *TxStoreBase) Get(item Item) error
  • func (s *TxStoreBase) GetSize(key Key) (int, error)
  • func (s *TxStoreBase) Has(key Key) (bool, error)
  • func (s *TxStoreBase) Iterate(query Query, fn IterateFn) error
  • func (s *TxStoreBase) Put(item Item) error
  • func (s *TxStoreBase) Rollback() error
• type Unmarshaler
• type Writer

Variables

var (
	ErrOverwriteNewerChunk       = errors.New("overwriting chunk with newer timestamp")
	ErrOverwriteOfImmutableBatch = errors.New("overwrite of existing immutable batch")
)

var (
	ErrInvalidQuery    = errors.New("storage: invalid query")
	ErrNotFound        = errors.New("storage: not found")
	ErrReferenceLength = errors.New("storage: invalid reference length")
	ErrInvalidChunk    = errors.New("storage: invalid chunk")
)

ErrInvalidQuery indicates that the query is not a valid query.

var ErrBatchCommitted = errors.New("storage: batch has already been committed")

ErrBatchCommitted is returned by Batch.Commit call when a batch has already been committed.

var (
	// ErrNoStampsForChunk is returned when chunk was found but there were no associated stamps.
	ErrNoStampsForChunk = fmt.Errorf("no stamp for existing chunk: %w", ErrNotFound)
)

var ErrTxDone = errors.New("storage: transaction has already been committed or rolled back")

ErrTxDone is returned by any operation that is performed on a transaction that has already been committed or rolled back.

Types

type Batch

type Batch interface {
	// Put adds a new item to the batch.
	Put(Item) error

	// Delete adds a new delete operation to the batch.
	Delete(Item) error

	// Commit commits the batch.
	Commit() error
}

Batch provides set of operations that are batched.

type BatchedStore

type BatchedStore interface {
	Store
	Batcher
}

BatchedStore is a store that supports batching of Writer method calls.

type Batcher

type Batcher interface {
	// Batch returns a new Batch.
	Batch(context.Context) (Batch, error)
}

Batcher specifies a constructor for creating new batches.

type ChunkGetterDeleter

type ChunkGetterDeleter interface {
	Getter
	Deleter
}

ChunkGetterDeleter is a storage that provides only read and delete operations for chunks.

type ChunkLocker

type ChunkLocker func(chunk swarm.Address) func()

type ChunkState

type ChunkState = int

const (
	// ChunkSent is used by the pusher component to notify about successful push of chunk from
	// the node. A chunk could be retried on failure so, this sent count is maintained to
	// understand how many attempts were made by the node while pushing. The attempts are
	// registered only when an actual request was sent from this node.
	ChunkSent ChunkState = iota
	// ChunkStored is used by the pusher component to notify that the uploader node is
	// the closest node and has stored the chunk.
	ChunkStored
	// ChunkSynced is used by the pusher component to notify that the chunk is synced to the
	// network. This is reported when a valid receipt was received after the chunk was
	// pushed.
	ChunkSynced
	ChunkCouldNotSync
)

type ChunkStore

type ChunkStore interface {
	io.Closer
	Getter
	Putter
	Deleter
	Hasser

	// Iterate over chunks in no particular order.
	Iterate(context.Context, IterateChunkFn) error
}

type Cloner

type Cloner interface {
	Clone() Item
}

Cloner makes a deep copy of the Item.

type Deleter

type Deleter interface {
	// Delete a chunk by the given swarm.Address.
	Delete(context.Context, swarm.Address) error
}

Deleter is the interface that wraps the basic Delete method.

type Descriptor

type Descriptor struct {
	Address swarm.Address
	BinID   uint64
}

Descriptor holds information required for Pull syncing. This struct is provided by subscribing to pull index.

func (*Descriptor) String

func (d *Descriptor) String() string

type Filter

type Filter func(string, []byte) bool

Filter subtracts entries from the iteration. Filters would not construct the Item from the serialized bytes. Instead, users can add logic to check the entries directly in byte format or partially or fully unmarshal the data and check.

type Getter

type Getter interface {
	// Get a chunk by its swarm.Address. Returns the chunk associated with
	// the address alongside with its postage stamp, or a storage.ErrNotFound
	// if the chunk is not found.
	// If the chunk has multiple stamps, then the first stamp is returned in this
	// query. In order to deterministically get the stamp, use the GetterWithStamp
	// interface. If the chunk is not found storage.ErrNotFound will be returned.
	Get(context.Context, swarm.Address) (swarm.Chunk, error)
}

Getter is the interface that wraps the basic Get method.

type GetterFunc

type GetterFunc func(context.Context, swarm.Address) (swarm.Chunk, error)

func (GetterFunc) Get

func (f GetterFunc) Get(ctx context.Context, address swarm.Address) (swarm.Chunk, error)

type Hasser

type Hasser interface {
	// Has checks whether a chunk exists in the store.
	Has(context.Context, swarm.Address) (bool, error)
}

Hasser is the interface that wraps the basic Has method.

type InMemTxRevertOpStore

type InMemTxRevertOpStore[K, V any] struct {
	// contains filtered or unexported fields
}

InMemTxRevertOpStore is an in-memory implementation of TxRevertOpStore.

func NewInMemTxRevertOpStore

func NewInMemTxRevertOpStore[K, V any](revOpsFn map[TxOpCode]TxRevertFn[K, V]) *InMemTxRevertOpStore[K, V]

NewInMemTxRevertOpStore is a convenient constructor for creating instances of InMemTxRevertOpStore. The revOpsFn map is used to look up the revert function for a given TxOpCode.

func (*InMemTxRevertOpStore[K, V]) Append

func (s *InMemTxRevertOpStore[K, V]) Append(ops ...*TxRevertOp[K, V]) error

Append implements TxRevertOpStore.

func (*InMemTxRevertOpStore[K, V]) Clean

func (s *InMemTxRevertOpStore[K, V]) Clean() error

Clean implements TxRevertOpStore.

func (*InMemTxRevertOpStore[K, V]) Revert

func (s *InMemTxRevertOpStore[K, V]) Revert() error

Revert implements TxRevertOpStore.

type Item

type Item interface {
	Key
	Marshaler
	Unmarshaler
	Cloner
	fmt.Stringer
}

Item represents an item which can be used in the Store.

type IterateChunkFn

type IterateChunkFn func(swarm.Chunk) (stop bool, err error)

type IterateFn

type IterateFn func(Result) (bool, error)

IterateFn iterates through the Items of the store in the Key.Namespace. The function returns a boolean to indicate if the iteration should stop.

type Key

type Key interface {
	// ID is the unique identifier of Item.
	ID() string

	// Namespace is used to separate similar items.
	// E.g.: can be seen as a table construct.
	Namespace() string
}

Key represents the item identifiers.

type Marshaler

type Marshaler interface {
	Marshal() ([]byte, error)
}

Marshaler is the interface implemented by types that can marshal themselves into valid Item.

type NoOpTxRevertOpStore

type NoOpTxRevertOpStore[K, V any] struct{}

NoOpTxRevertOpStore is a no-op implementation of TxRevertOpStore.

func (*NoOpTxRevertOpStore[K, V]) Append

func (s *NoOpTxRevertOpStore[K, V]) Append(...*TxRevertOp[K, V]) error

func (*NoOpTxRevertOpStore[K, V]) Clean

func (s *NoOpTxRevertOpStore[K, V]) Clean() error

func (*NoOpTxRevertOpStore[K, V]) Revert

func (s *NoOpTxRevertOpStore[K, V]) Revert() error

type Order

type Order int

Order represents order of the iteration

const (
	// KeyAscendingOrder indicates a forward iteration based on ordering of keys.
	KeyAscendingOrder Order = iota

	// KeyDescendingOrder denotes the backward iteration based on ordering of keys.
	KeyDescendingOrder
)

type PullSubscriber

type PullSubscriber interface {
	SubscribePull(ctx context.Context, bin uint8, since, until uint64) (c <-chan Descriptor, closed <-chan struct{}, stop func())
}

type PushReporter

type PushReporter interface {
	Report(context.Context, swarm.Chunk, ChunkState) error
}

PushReporter is used to report chunk state.

type PushSubscriber

type PushSubscriber interface {
	SubscribePush(ctx context.Context) (c <-chan swarm.Chunk, stop func())
}

type Putter

type Putter interface {
	// Put a chunk into the store alongside with its postage stamp.
	Put(context.Context, swarm.Chunk) error
}

Putter is the interface that wraps the basic Put method.

type PutterFunc

type PutterFunc func(context.Context, swarm.Chunk) error

PutterFunc type is an adapter to allow the use of ChunkStore as Putter interface. If f is a function with the appropriate signature, PutterFunc(f) is a Putter that calls f.

func (PutterFunc) Put

func (f PutterFunc) Put(ctx context.Context, chunk swarm.Chunk) error

Put calls f(ctx, chunk).

type Query

type Query struct {
	// Factory is a constructor passed by client
	// to construct new object for the result.
	Factory func() Item

	// Prefix indicates interest in an item
	// that contains this prefix in its ID.
	Prefix string

	// PrefixAtStart indicates that the
	// iteration should start at the prefix.
	PrefixAtStart bool

	// SkipFirst skips the first element in the iteration.
	SkipFirst bool

	// ItemProperty indicates a specific interest of an Item property.
	ItemProperty QueryItemProperty

	// Order denotes the order of iteration.
	Order Order

	// Filters represent further constraints on the iteration.
	Filters []Filter
}

Query denotes the iteration attributes.

func (Query) Validate

func (q Query) Validate() error

Validate checks if the query is a valid query.

type QueryItemProperty

type QueryItemProperty int

QueryItemProperty tells the Query which Item property should be loaded from the store to the result.

const (
	// QueryItem indicates interest in the whole Item.
	QueryItem QueryItemProperty = iota

	// QueryItemID indicates interest in the Result.ID.
	// No data will be unmarshalled.
	QueryItemID

	// QueryItemSize indicates interest in the Result.Size.
	// No data will be unmarshalled.
	QueryItemSize
)

type ReadOnlyChunkStore

type ReadOnlyChunkStore interface {
	Getter
	Hasser
}

type Reader

type Reader interface {
	// Get unmarshalls object with the given Item.Key.ID into the given Item.
	Get(Item) error

	// Has reports whether the Item with the given Key.ID exists in the store.
	Has(Key) (bool, error)

	// GetSize returns the size of Item with the given Key.ID.
	GetSize(Key) (int, error)

	// Iterate executes the given IterateFn on this store.
	// The Result of the iteration will be affected by the given Query.
	Iterate(Query, IterateFn) error

	// Count returns the count of items in the
	// store that are in the same Key.Namespace.
	Count(Key) (int, error)
}

Reader groups methods that read from the store.

type Recoverer

type Recoverer interface {
	Recover() error
}

Recoverer allows store to recover from a failure when the transaction was not committed or rolled back.

type Repository

type Repository interface {
	IndexStore() BatchedStore
	ChunkStore() ChunkStore

	NewTx(context.Context) (repo Repository, commit func() error, rollback func() error)
}

Repository is a collection of stores that provides a unified interface to access them. Access to all stores can be guarded by a transaction.

func NewRepository

func NewRepository(
	txIndexStore TxStore,
	txChunkStore TxChunkStore,
	locker ChunkLocker,
) Repository

NewRepository returns a new Repository instance.

type Result

type Result struct {
	// ID is the Key.ID of the result Item.
	ID string

	// Size is the size of the result Item.
	Size int

	// Entry in the result Item.
	Entry Item
}

Result represents the item returned by the read operation, which returns the item as the result. Or Key and/or Size in case the whole Item is not needed.

type SizeReporter

type SizeReporter interface {
	Size() (uint64, error)
	Capacity() uint64
}

type StateIterFunc

type StateIterFunc func(key, val []byte) (stop bool, err error)

StateIterFunc is used when iterating through StateStorer key/value pairs

type StateStorer

type StateStorer interface {
	io.Closer

	// Get unmarshalls object with the given key into the given obj.
	Get(key string, obj interface{}) error

	// Put inserts or updates the given obj stored under the given key.
	Put(key string, obj interface{}) error

	// Delete removes object form the store stored under the given key.
	Delete(key string) error

	// Iterate iterates over all keys with the given prefix and calls iterFunc.
	Iterate(prefix string, iterFunc StateIterFunc) error
}

StateStorer is a storage interface for storing and retrieving key/value pairs.

type StateStorerCleaner

type StateStorerCleaner interface {
	// Nuke the store so that only the bare essential entries are left.
	Nuke() error
}

StateStorerCleaner is the interface for cleaning the store.

type Store

type Store interface {
	io.Closer

	Reader
	Writer
}

Store contains the methods required for the Data Abstraction Layer.

type Tx

type Tx interface {
	// Commit commits the transaction.
	Commit() error

	// Rollback aborts the transaction.
	Rollback() error
}

Tx represents an in-progress Store transaction. A transaction must end with a call to Commit or Rollback.

type TxChunkStore

type TxChunkStore interface {
	Tx
	ChunkStore

	NewTx(*TxState) TxChunkStore
}

TxChunkStore represents a Tx ChunkStore where all operations are completed in a transaction.

type TxChunkStoreBase

type TxChunkStoreBase struct {
	*TxState
	ChunkStore
	// contains filtered or unexported fields
}

TxChunkStoreBase implements the ChunkStore interface where the operations are guarded by a transaction.

func (*TxChunkStoreBase) Close

func (s *TxChunkStoreBase) Close() error

Close implements the ChunkStore interface. The operation is blocked until the transaction is not done.

func (*TxChunkStoreBase) Delete

func (s *TxChunkStoreBase) Delete(ctx context.Context, address swarm.Address) error

Delete implements the ChunkStore interface.

func (*TxChunkStoreBase) Get

func (s *TxChunkStoreBase) Get(ctx context.Context, address swarm.Address) (swarm.Chunk, error)

Get implements the ChunkStore interface.

func (*TxChunkStoreBase) Has

func (s *TxChunkStoreBase) Has(ctx context.Context, address swarm.Address) (bool, error)

Has implements the ChunkStore interface.

func (*TxChunkStoreBase) Iterate

func (s *TxChunkStoreBase) Iterate(ctx context.Context, fn IterateChunkFn) error

Iterate implements the ChunkStore interface.

func (*TxChunkStoreBase) Put

func (s *TxChunkStoreBase) Put(ctx context.Context, chunk swarm.Chunk) error

Put implements the ChunkStore interface.

func (*TxChunkStoreBase) Rollback

func (s *TxChunkStoreBase) Rollback() error

Rollback implements the TxChunkStore interface.

type TxOpCode

type TxOpCode string

TxOpCode represents code for tx operations.

const (
	PutOp       TxOpCode = "put"
	PutCreateOp TxOpCode = "putCreate"
	PutUpdateOp TxOpCode = "putUpdate"
	DeleteOp    TxOpCode = "delete"
)

type TxRevertFn

type TxRevertFn[K, V any] func(K, V) error

TxRevertFn represents a function that can be invoked to reverse the operation that was performed by the corresponding TxOpCode.

type TxRevertOp

type TxRevertOp[K, V any] struct {
	Origin   TxOpCode
	ObjectID string

	Key K
	Val V
}

TxRevertOp represents a reverse operation.

type TxRevertOpStore

type TxRevertOpStore[K, V any] interface {
	// Append appends a Revert operation to the store.
	Append(...*TxRevertOp[K, V]) error
	// Revert executes all the revere operations
	// in the store in reverse order.
	Revert() error
	// Clean cleans the store.
	Clean() error
}

TxRevertOpStore represents a store for TxRevertOp.

type TxState

type TxState struct {
	// contains filtered or unexported fields
}

TxState is a mix-in for Tx. It provides basic functionality for transaction state lifecycle.

func NewTxState

func NewTxState(ctx context.Context) *TxState

NewTxState is a convenient constructor for creating instances of TxState.

func (*TxState) AwaitDone

func (tx *TxState) AwaitDone() <-chan struct{}

AwaitDone returns a channel that blocks until the context in TxState is canceled or the transaction is done.

func (*TxState) Done

func (tx *TxState) Done() error

Done marks this transaction as complete. It returns ErrTxDone if the transaction has already been committed or rolled back or if the transaction was in progress and the context was canceled, it returns the context.Canceled error.

func (*TxState) IsDone

func (tx *TxState) IsDone() error

IsDone returns ErrTxDone if the transaction has already been committed or rolled back. If the transaction was in progress and the context was canceled, it returns the context.Canceled error.

type TxStore

type TxStore interface {
	Tx
	Store
	Batcher

	NewTx(*TxState) TxStore
}

TxStore represents a Tx Store where all operations are completed in a transaction.

type TxStoreBase

type TxStoreBase struct {
	*TxState
	BatchedStore
	// contains filtered or unexported fields
}

TxStoreBase implements the Store interface where the operations are guarded by a transaction.

func (*TxStoreBase) Batch

func (s *TxStoreBase) Batch(ctx context.Context) (Batch, error)

func (*TxStoreBase) Close

func (s *TxStoreBase) Close() error

Close implements the Store interface. The operation is blocked until the transaction is not done.

func (*TxStoreBase) Count

func (s *TxStoreBase) Count(key Key) (int, error)

Count implements the Store interface.

func (*TxStoreBase) Delete

func (s *TxStoreBase) Delete(item Item) error

Delete implements the Store interface.

func (*TxStoreBase) Get

func (s *TxStoreBase) Get(item Item) error

Get implements the Store interface.

func (*TxStoreBase) GetSize

func (s *TxStoreBase) GetSize(key Key) (int, error)

GetSize implements the Store interface.

func (*TxStoreBase) Has

func (s *TxStoreBase) Has(key Key) (bool, error)

Has implements the Store interface.

func (*TxStoreBase) Iterate

func (s *TxStoreBase) Iterate(query Query, fn IterateFn) error

Iterate implements the Store interface.

func (*TxStoreBase) Put

func (s *TxStoreBase) Put(item Item) error

Put implements the Store interface.

func (*TxStoreBase) Rollback

func (s *TxStoreBase) Rollback() error

Rollback implements the TxStore interface.

type Unmarshaler

type Unmarshaler interface {
	Unmarshal([]byte) error
}

Unmarshaler is the interface implemented by types that can unmarshal a JSON description of themselves. The input can be assumed to be a valid encoding of a Item value.

type Writer

type Writer interface {
	// Put inserts or updates the given Item identified by its Key.ID.
	Put(Item) error

	// Delete removes the given Item form the store.
	// It will not return error if the key doesn't exist.
	Delete(Item) error
}

Writer groups methods that change the state of the store.