api

package
v0.2103.10 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 23, 2022 License: Apache-2.0 Imports: 22 Imported by: 2

Documentation

Overview

Package api implements the storage backend API.

Index

Constants

View Source 
const (
	// ModuleName is the storage module name.
	ModuleName = "storage"

	// WriteLogIteratorChunkSize defines the chunk size of write log entries
	// for the GetDiff method.
	WriteLogIteratorChunkSize = 10
)
View Source 
const (
	// RootTypeInvalid is an invalid/uninitialized root type.
	RootTypeInvalid = mkvsNode.RootTypeInvalid
	// RootTypeState is the type for state storage roots.
	RootTypeState = mkvsNode.RootTypeState
	// RootTypeIO is the type for IO storage roots.
	RootTypeIO = mkvsNode.RootTypeIO
	// RootTypeMax is the number of different root types.
	RootTypeMax = mkvsNode.RootTypeMax
)

Variables

View Source 
var (
	// ErrCantProve is the error returned when the backend is incapable
	// of generating proofs (unsupported, no key, etc).
	ErrCantProve = errors.New(ModuleName, 1, "storage: unable to provide proofs")
	// ErrNoRoots is the error returned when the generated receipt would
	// not contain any roots.
	ErrNoRoots = errors.New(ModuleName, 2, "storage: no roots to generate receipt for")
	// ErrExpectedRootMismatch is the error returned when the expected root
	// does not match the computed root.
	ErrExpectedRootMismatch = errors.New(ModuleName, 3, "storage: expected root mismatch")
	// ErrUnsupported is the error returned when the called method is not
	// supported by the given backend.
	ErrUnsupported = errors.New(ModuleName, 4, "storage: method not supported by backend")
	// ErrLimitReached means that a configured limit has been reached.
	ErrLimitReached = errors.New(ModuleName, 5, "storage: limit reached")

	// ErrNodeNotFound indicates that a node with the specified hash couldn't be found
	// in the database.
	ErrNodeNotFound = nodedb.ErrNodeNotFound
	// ErrWriteLogNotFound indicates that a write log for the specified storage hashes
	// couldn't be found.
	ErrWriteLogNotFound = nodedb.ErrWriteLogNotFound
	// ErrNotFinalized indicates that the operation requires a version to be finalized
	// but the version is not yet finalized.
	ErrNotFinalized = nodedb.ErrNotFinalized
	// ErrAlreadyFinalized indicates that the given version has already been finalized.
	ErrAlreadyFinalized = nodedb.ErrAlreadyFinalized
	// ErrVersionNotFound indicates that the given version cannot be found.
	ErrVersionNotFound = nodedb.ErrVersionNotFound
	// ErrPreviousVersionMismatch indicates that the version given for the old root does
	// not match the previous version.
	ErrPreviousVersionMismatch = nodedb.ErrPreviousVersionMismatch
	// ErrVersionWentBackwards indicates that the new version is earlier than an already
	// inserted version.
	ErrVersionWentBackwards = nodedb.ErrVersionWentBackwards
	// ErrRootNotFound indicates that the given root cannot be found.
	ErrRootNotFound = nodedb.ErrRootNotFound
	// ErrRootMustFollowOld indicates that the passed new root does not follow old root.
	ErrRootMustFollowOld = nodedb.ErrRootMustFollowOld
	// ErrReadOnly indicates that the storage backend is read-only.
	ErrReadOnly = nodedb.ErrReadOnly

	// ReceiptSignatureContext is the signature context used for verifying MKVS receipts.
	ReceiptSignatureContext = signature.NewContext("oasis-core/storage: receipt", signature.WithChainSeparation())
)
View Source 
var (

	// ServiceName is the gRPC service name.
	ServiceName = cmnGrpc.NewServiceName("Storage")

	// MethodSyncGet is the SyncGet method.
	MethodSyncGet = ServiceName.NewMethod("SyncGet", GetRequest{}).
					WithNamespaceExtractor(func(ctx context.Context, req interface{}) (common.Namespace, error) {
			r, ok := req.(*GetRequest)
			if !ok {
				return common.Namespace{}, errInvalidRequestType
			}
			return r.Tree.Root.Namespace, nil
		}).
		WithAccessControl(cmnGrpc.AccessControlAlways)
	// MethodSyncGetPrefixes is the SyncGetPrefixes method.
	MethodSyncGetPrefixes = ServiceName.NewMethod("SyncGetPrefixes", GetPrefixesRequest{}).
							WithNamespaceExtractor(func(ctx context.Context, req interface{}) (common.Namespace, error) {
			r, ok := req.(*GetPrefixesRequest)
			if !ok {
				return common.Namespace{}, errInvalidRequestType
			}
			return r.Tree.Root.Namespace, nil
		}).
		WithAccessControl(cmnGrpc.AccessControlAlways)
	// MethodSyncIterate is the SyncIterate method.
	MethodSyncIterate = ServiceName.NewMethod("SyncIterate", IterateRequest{}).
						WithNamespaceExtractor(func(ctx context.Context, req interface{}) (common.Namespace, error) {
			r, ok := req.(*IterateRequest)
			if !ok {
				return common.Namespace{}, errInvalidRequestType
			}
			return r.Tree.Root.Namespace, nil
		}).
		WithAccessControl(cmnGrpc.AccessControlAlways)
	// MethodApply is the Apply method.
	MethodApply = ServiceName.NewMethod("Apply", ApplyRequest{}).
				WithNamespaceExtractor(func(ctx context.Context, req interface{}) (common.Namespace, error) {
			r, ok := req.(*ApplyRequest)
			if !ok {
				return common.Namespace{}, errInvalidRequestType
			}
			return r.Namespace, nil
		}).
		WithAccessControl(cmnGrpc.AccessControlAlways)

	// MethodApplyBatch is the ApplyBatch method.
	MethodApplyBatch = ServiceName.NewMethod("ApplyBatch", ApplyBatchRequest{}).
						WithNamespaceExtractor(func(ctx context.Context, req interface{}) (common.Namespace, error) {
			r, ok := req.(*ApplyBatchRequest)
			if !ok {
				return common.Namespace{}, errInvalidRequestType
			}
			return r.Namespace, nil
		}).
		WithAccessControl(cmnGrpc.AccessControlAlways)

	// MethodGetDiff is the GetDiff method.
	MethodGetDiff = ServiceName.NewMethod("GetDiff", GetDiffRequest{})

	// MethodGetCheckpoints is the GetCheckpoints method.
	MethodGetCheckpoints = ServiceName.NewMethod("GetCheckpoints", checkpoint.GetCheckpointsRequest{})

	// MethodGetCheckpointChunk is the GetCheckpointChunk method.
	MethodGetCheckpointChunk = ServiceName.NewMethod("GetCheckpointChunk", checkpoint.ChunkMetadata{})
)
View Source 
var ErrMuxDontContinue = errors.New("dontcontinue")

ErrMuxDontContinue is the error that should be returned by the MuxController function when an operation was successful, but the muxer shouldn't continue with other backends and should return an overall success.

Functions

func BlacklistAddNode added in v0.2101.0 

func BlacklistAddNode(ctx context.Context, node *node.Node)

BlacklistAddNode adds a node to the blacklist associated with the context. If there's no associated blacklist, the function does nothing.

func IsNodeBlacklistedInContext added in v0.2101.0 

func IsNodeBlacklistedInContext(ctx context.Context, node *node.Node) bool

IsNodeBlacklistedInContext checks to see if the node is blacklisted in this context. If the context doesn't have an associated blacklist, then no node is considered blacklisted.

func MuxPassthrough added in v0.2102.0 

func MuxPassthrough(i int, backend Backend, meth string, resp interface{}, err error) (interface{}, error)

MuxPassthrough is a mux controller that just returns the response and error it gets. Normally, this will cause the muxer to stop on error and continue on a response.

func NodePriorityHintFromContext added in v0.2012.3 

func NodePriorityHintFromContext(ctx context.Context) []signature.PublicKey

NodePriorityHintFromContext returns the storage node priority hint or nil if none is set.

func RegisterService 

func RegisterService(server *grpc.Server, service Backend)

RegisterService registers a new sentry service with the given gRPC server.

func WithNodeBlacklist added in v0.2101.0 

func WithNodeBlacklist(ctx context.Context) context.Context

WithNodeBlacklist sets a storage blacklist key for any storage requests using this context. The blacklist is initially empty (i.e. all nodes are acceptable).

func WithNodePriorityHint added in v0.2012.3 

func WithNodePriorityHint(ctx context.Context, nodes []signature.PublicKey) context.Context

WithNodePriorityHint sets a storage node priority hint for any storage read requests using this context. Only storage nodes that overlap with the configured committee will be used.

func WithNodePriorityHintFromMap added in v0.2012.3 

func WithNodePriorityHintFromMap(ctx context.Context, nodes map[signature.PublicKey]bool) context.Context

WithNodePriorityHintFromMap sets a storage node priority hint for any storage read requests using this context. Only storage nodes that overlap with the configured committee will be used.

func WithNodePriorityHintFromSignatures added in v0.2012.3 

func WithNodePriorityHintFromSignatures(ctx context.Context, sigs []signature.Signature) context.Context

WithNodePriorityHintFromSignatures sets a storage node priority hint for any storage read requests using this context. Only storage nodes that overlap with the configured committee will be used.

func WithNodeSelectionCallback added in v0.2101.0 

func WithNodeSelectionCallback(ctx context.Context, cb NodeSelectionCallback) context.Context

WithNodeSelectionCallback sets a callback that will be called by the storage client on every read request with the node that was eventually used to perform a successful request. If there was no success, the callback isn't called.

Types

type ApplyBatchRequest 

type ApplyBatchRequest struct {
	Namespace common.Namespace `json:"namespace"`
	DstRound  uint64           `json:"dst_round"`
	Ops       []ApplyOp        `json:"ops"`
}

ApplyBatchRequest is an ApplyBatch request.

type ApplyOp 

type ApplyOp struct {
	// RootType is the type of root this operation is for.
	RootType RootType `json:"root_type"`
	// SrcRound is the source root round.
	SrcRound uint64 `json:"src_round"`
	// SrcRoot is the merkle root to apply the operations against. It may
	// refer to a nil node (empty hash) in which case a new root will be
	// created.
	SrcRoot hash.Hash `json:"src_root"`
	// DstRoot is the expected merkle root after applying the write log.
	DstRoot hash.Hash `json:"dst_root"`
	// WriteLog is a write log of operations to apply.
	WriteLog WriteLog `json:"writelog"`
}

ApplyOp is an apply operation within a batch of apply operations.

type ApplyRequest 

type ApplyRequest struct {
	Namespace common.Namespace `json:"namespace"`
	RootType  RootType         `json:"root_type"`
	SrcRound  uint64           `json:"src_round"`
	SrcRoot   hash.Hash        `json:"src_root"`
	DstRound  uint64           `json:"dst_round"`
	DstRoot   hash.Hash        `json:"dst_root"`
	WriteLog  WriteLog         `json:"writelog"`
}

ApplyRequest is an Apply request.

type Backend 

type Backend interface {
	syncer.ReadSyncer
	checkpoint.ChunkProvider

	// Apply applies a set of operations against the MKVS.  The root may refer
	// to a nil node, in which case a new root will be created.
	// The expected new root is used to check if the new root after all the
	// operations are applied already exists in the local DB.  If it does, the
	// Apply is ignored.
	Apply(ctx context.Context, request *ApplyRequest) ([]*Receipt, error)

	// ApplyBatch applies multiple sets of operations against the MKVS and
	// returns a single receipt covering all applied roots.
	//
	// See Apply for more details.
	ApplyBatch(ctx context.Context, request *ApplyBatchRequest) ([]*Receipt, error)

	// GetDiff returns an iterator of write log entries that must be applied
	// to get from the first given root to the second one.
	GetDiff(ctx context.Context, request *GetDiffRequest) (WriteLogIterator, error)

	// Cleanup closes/cleans up the storage backend.
	Cleanup()

	// Initialized returns a channel that will be closed when the
	// backend is initialized and ready to service requests.
	Initialized() <-chan struct{}
}

Backend is a storage backend implementation.

func NewMetricsWrapper added in v0.2012.3 

func NewMetricsWrapper(base Backend) Backend

func NewStorageClient 

func NewStorageClient(c *grpc.ClientConn) Backend

NewStorageClient creates a new gRPC storage client service.

func NewStorageMux added in v0.2102.0 

func NewStorageMux(controller MuxController, backends ...Backend) Backend

NewStorageMux constructs a multiplexer for multiple storage backends. Requests are sent to all of them. It is the controller's job to determine on each step if the muxer should continue with further backends or not.

Normally, the return values are the last non-nil return of any backend and the last non-nil error of any backend, so client code should take care to take into account the otherwise unusual situation where both the response and error are valid non-nil values.

type ClientBackend 

type ClientBackend interface {
	Backend

	// GetConnectedNodes returns currently connected storage nodes.
	GetConnectedNodes() []*node.Node

	// EnsureCommitteeVersion waits for the storage committee client to be fully
	// synced to the given version.
	//
	// This method will error in case the storage-client is not configured to
	// track a specific committee.
	EnsureCommitteeVersion(ctx context.Context, version int64) error
}

ClientBackend is a storage client backend implementation.

type Config 

type Config struct {
	// Backend is the database backend.
	Backend string

	// DB is the path to the database.
	DB string

	// Signer is the signing key to use for generating recipts.
	Signer signature.Signer

	// ApplyLockLRUSlots is the number of LRU slots to use for Apply call locks.
	ApplyLockLRUSlots uint64

	// InsecureSkipChecks bypasses the known root checks.
	InsecureSkipChecks bool

	// Namespace is the namespace contained within the database.
	Namespace common.Namespace

	// MaxCacheSize is the maximum in-memory cache size for the database.
	MaxCacheSize int64

	// DiscardWriteLogs will cause all write logs to be discarded.
	DiscardWriteLogs bool

	// NoFsync will disable fsync() where possible.
	NoFsync bool

	// MemoryOnly will make the storage memory-only (if the backend supports it).
	MemoryOnly bool

	// ReadOnly will make the storage read-only.
	ReadOnly bool
}

Config is the storage backend configuration.

func (*Config) ToNodeDB 

func (cfg *Config) ToNodeDB() *nodedb.Config

ToNodeDB converts from a Config to a node DB Config.

type Depth 

type Depth = mkvsNode.Depth

Depth determines the node's (bit) depth in the tree. It is also used for storing the Key length in bits.

type GetDiffRequest 

type GetDiffRequest struct {
	StartRoot Root        `json:"start_root"`
	EndRoot   Root        `json:"end_root"`
	Options   SyncOptions `json:"options"`
}

GetDiffRequest is a GetDiff request.

type GetPrefixesRequest 

type GetPrefixesRequest = syncer.GetPrefixesRequest

GetPrefixesRequest is a request for the SyncGetPrefixes operation.

type GetRequest 

type GetRequest = syncer.GetRequest

GetRequest is a request for the SyncGet operation.

type InternalNode 

type InternalNode = mkvsNode.InternalNode

InternalNode is an internal node with two children.

type IterateRequest 

type IterateRequest = syncer.IterateRequest

IterateRequest is a request for the SyncIterate operation.

type Key 

type Key = mkvsNode.Key

Key is a node's key spelled out from the root to the node.

type LeafNode 

type LeafNode = mkvsNode.LeafNode

LeafNode is a leaf node containing a key/value pair.

type LocalBackend 

type LocalBackend interface {
	Backend

	// Checkpointer returns the checkpoint creator/restorer for this storage backend.
	Checkpointer() checkpoint.CreateRestorer

	// NodeDB returns the underlying node database.
	NodeDB() nodedb.NodeDB
}

LocalBackend is a storage implementation with a local backing store.

type LogEntry 

type LogEntry = writelog.LogEntry

LogEntry is a write log entry.

type MuxContinueWithError added in v0.2102.0 

type MuxContinueWithError struct {
	// contains filtered or unexported fields
}

MuxContinueWithError is an error type that can be returned by the MuxController function when an operation failed, but the muxer should continue anyway. The error returned will be last one returned by any failed operation.

NOTE: If one backend in the muxer fails and another succeeds, then using this may cause the abnormal situation of the muxer returning both a response and an error for the operation.

func (MuxContinueWithError) Error added in v0.2102.0 

func (e MuxContinueWithError) Error() string

func (MuxContinueWithError) Unwrap added in v0.2102.0 

func (e MuxContinueWithError) Unwrap() error

type MuxController added in v0.2102.0 

type MuxController func(i int, backend Backend, meth string, resp interface{}, err error) (interface{}, error)

MuxController controls how a mux storage shim steps through its backend list.

If the controller returns an error, the muxer will normally stop iterating through the backend list and return the error (with special handling for errors of type MuxContinueWithError and ErrMuxDontContinue).

func MuxIterateIgnoringLocalErrors added in v0.2102.7 

func MuxIterateIgnoringLocalErrors() MuxController

MuxIterateIgnoringLocalErrors creates a controller that tells the muxer to continue iterating through its backends even if a local one returns an error.

func MuxReadOpFinishEarly added in v0.2102.0 

func MuxReadOpFinishEarly(next MuxController) MuxController

MuxReadOpFinishEarly is a chainable controller that stops the muxer as soon as a readonly operation is successful. It passes through all other operation transparently to the next controller.

type Node 

type Node = mkvsNode.Node

Node is either an InternalNode or a LeafNode.

type NodeDB 

type NodeDB = nodedb.NodeDB

NodeDB is a node database.

type NodeSelectionCallback added in v0.2101.0 

type NodeSelectionCallback = func(*node.Node)

NodeSelectionCallback is a callback used by the storage client to report connections used for read requests.

func NodeSelectionCallbackFromContext added in v0.2101.0 

func NodeSelectionCallbackFromContext(ctx context.Context) NodeSelectionCallback

NodeSelectionCallbackFromContext returns the node selection callback associated with this context or nil if none is set.

type Pointer 

type Pointer = mkvsNode.Pointer

Pointer is a pointer to another node.

type Proof 

type Proof = syncer.Proof

Proof is a Merkle proof for a subtree.

type ProofResponse 

type ProofResponse = syncer.ProofResponse

ProofResponse is a response for requests that produce proofs.

type Receipt 

type Receipt struct {
	signature.Signed
}

Receipt is a signed ReceiptBody.

func SignReceipt 

func SignReceipt(signer signature.Signer, ns common.Namespace, round uint64, rootTypes []RootType, roots []hash.Hash) (*Receipt, error)

SignReceipt signs a storage receipt for the given roots.

func (*Receipt) Open 

func (s *Receipt) Open(receipt *ReceiptBody) error

Open first verifies the blob signature then unmarshals the blob.

type ReceiptBody 

type ReceiptBody struct {
	// Version is the storage data structure version.
	Version uint16 `json:"version"`
	// Namespace is the chain namespace under which the root(s) are stored.
	Namespace common.Namespace `json:"ns"`
	// Round is the chain round in which the root(s) are stored.
	Round uint64 `json:"round"`
	// RootTypes are the storage types of the merkle roots in Roots.
	RootTypes []RootType `json:"root_types"`
	// Roots are the merkle roots of the merklized data structure that the
	// storage node is certifying to store.
	Roots []hash.Hash `json:"roots"`
}

ReceiptBody is the body of a receipt.

type Root 

type Root = mkvsNode.Root

Root is a storage root.

type RootCache 

type RootCache struct {
	// contains filtered or unexported fields
}

RootCache is a LRU based tree cache.

func NewRootCache 

func NewRootCache(
	localDB nodedb.NodeDB,
	remoteSyncer syncer.ReadSyncer,
	applyLockLRUSlots uint64,
	insecureSkipChecks bool,
) (*RootCache, error)

func (*RootCache) Apply 

func (rc *RootCache) Apply(
	ctx context.Context,
	root Root,
	expectedNewRoot Root,
	writeLog WriteLog,
) (*hash.Hash, error)

Apply applies the write log, bypassing the apply operation iff the new root already is in the node database.

func (*RootCache) GetTree 

func (rc *RootCache) GetTree(ctx context.Context, root Root) (mkvs.Tree, error)

GetTree gets a tree entry from the cache by the root iff present, or creates a new tree with the specified root in the node database.

func (*RootCache) HasRoot 

func (rc *RootCache) HasRoot(root Root) bool

type RootType added in v0.2100.0 

type RootType = mkvsNode.RootType

RootType is a storage root type.

type SyncChunk 

type SyncChunk struct {
	Final    bool     `json:"final"`
	WriteLog WriteLog `json:"writelog"`
}

SyncChunk is a chunk of write log entries sent during GetDiff operation.

type SyncOptions 

type SyncOptions struct {
	OffsetKey []byte `json:"offset_key"`
	Limit     uint64 `json:"limit"`
}

SyncOptions are the sync options.

type TreeID 

type TreeID = syncer.TreeID

TreeID identifies a specific tree and a position within that tree.

type WrappedLocalBackend added in v0.2102.6 

type WrappedLocalBackend interface {
	// Unwrap returns the underlying local storage backend.
	Unwrap() LocalBackend
}

WrappedLocalBackend is an interface implemented by storage backends that wrap a local storage backend in order to support unwrapping.

type WriteLog 

type WriteLog = writelog.WriteLog

WriteLog is a write log.

The keys in the write log must be unique.

type WriteLogIterator 

type WriteLogIterator = writelog.Iterator

WriteLogIterator iterates over write log entries.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.