blockstore

package
v1.16.0-rc1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 10, 2024 License: Apache-2.0, MIT Imports: 31 Imported by: 9

Documentation

Overview

blockstore contains all the basic blockstore constructors used by lotus. Any blockstore not ultimately constructed out of the building blocks in this package may not work properly.

  • This package correctly wraps blockstores with the IdBlockstore. This blockstore:
  • Filters out all puts for blocks with CIDs using the "identity" hash function.
  • Extracts inlined blocks from CIDs using the identity hash function and returns them on get/has, ignoring the contents of the blockstore.
  • In the future, this package may enforce additional restrictions on block sizes, CID validity, etc.

To make auditing for misuse of blockstores tractable, this package re-exports parts of the go-ipfs-blockstore package such that no other package needs to import it directly.

Index

Constants

View Source
const (
	// FileIO is equivalent to badger/options.FileIO.
	FileIO = options.FileIO
	// MemoryMap is equivalent to badger/options.MemoryMap.
	MemoryMap = options.MemoryMap
	// LoadToRAM is equivalent to badger/options.LoadToRAM.
	LoadToRAM = options.LoadToRAM
)

aliases to mask badger dependencies.

Variables

View Source
var (
	NewGCLocker     = blockstore.NewGCLocker
	NewGCBlockstore = blockstore.NewGCBlockstore
)
View Source
var ErrBlockstoreClosed = fmt.Errorf("badger blockstore closed")

ErrBlockstoreClosed is returned from blockstore operations after the blockstore has been closed.

Functions

func CopyBlockstore

func CopyBlockstore(ctx context.Context, from, to Blockstore) error

func CopyParticial

func CopyParticial(ctx context.Context, from, to Blockstore, root cid.Cid) error

Types

type AutobatchBlockstore

type AutobatchBlockstore struct {
	// contains filtered or unexported fields
}

func NewAutobatch

func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore

func (*AutobatchBlockstore) AllKeysChan

func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*AutobatchBlockstore) DeleteBlock

func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error

func (*AutobatchBlockstore) DeleteMany

func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*AutobatchBlockstore) Flush

func (bs *AutobatchBlockstore) Flush(ctx context.Context) error

caller must NOT hold stateLock

func (*AutobatchBlockstore) Get

func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error)

func (*AutobatchBlockstore) GetSize

func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*AutobatchBlockstore) Has

func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)

func (*AutobatchBlockstore) HashOnRead

func (bs *AutobatchBlockstore) HashOnRead(enabled bool)

func (*AutobatchBlockstore) Put

func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error

func (*AutobatchBlockstore) PutMany

func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error

func (*AutobatchBlockstore) Shutdown

func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error

func (*AutobatchBlockstore) View

func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error

type BadgerBlockstore

type BadgerBlockstore struct {
	DB *badger.DB
	// contains filtered or unexported fields
}

blockstore is a badger-backed IPLD blockstore.

NOTE: once Close() is called, methods will try their best to return ErrBlockstoreClosed. This will guaranteed to happen for all subsequent operation calls after Close() has returned, but it may not happen for operations in progress. Those are likely to fail with a different error.

func Open

func Open(opts Options) (*BadgerBlockstore, error)

Open creates a new badger-backed blockstore, with the supplied options.

func (*BadgerBlockstore) AllKeysChan

func (b *BadgerBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

AllKeysChan implements blockstore.AllKeysChan.

func (*BadgerBlockstore) Close

func (b *BadgerBlockstore) Close() error

Close closes the store. If the store has already been closed, this noops and returns an error, even if the first closure resulted in error.

func (*BadgerBlockstore) ConvertKey

func (b *BadgerBlockstore) ConvertKey(cid cid.Cid) datastore.Key

func (*BadgerBlockstore) DeleteBlock

func (b *BadgerBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error

DeleteBlock implements blockstore.DeleteBlock.

func (*BadgerBlockstore) DeleteMany

func (b *BadgerBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*BadgerBlockstore) Flush added in v1.14.0

func (*BadgerBlockstore) Get

func (b *BadgerBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)

Get implements blockstore.Get.

func (*BadgerBlockstore) GetSize

func (b *BadgerBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error)

GetSize implements blockstore.GetSize.

func (*BadgerBlockstore) Has

func (b *BadgerBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error)

Has implements blockstore.Has.

func (*BadgerBlockstore) HashOnRead

func (b *BadgerBlockstore) HashOnRead(_ bool)

HashOnRead implements blockstore.HashOnRead. It is not supported by this blockstore.

func (*BadgerBlockstore) Put

func (b *BadgerBlockstore) Put(ctx context.Context, block blocks.Block) error

Put implements blockstore.Put.

func (*BadgerBlockstore) PutMany

func (b *BadgerBlockstore) PutMany(ctx context.Context, blks []blocks.Block) error

PutMany implements blockstore.PutMany.

func (*BadgerBlockstore) ReadonlyDatastore

func (b *BadgerBlockstore) ReadonlyDatastore() *TxBlockstore

func (*BadgerBlockstore) View

func (b *BadgerBlockstore) View(ctx context.Context, cid cid.Cid, fn func([]byte) error) error

View implements blockstore.Viewer, which leverages zero-copy read-only access to values.

type BasicBlockstore

type BasicBlockstore = blockstore.Blockstore

type BatchDeleter

type BatchDeleter interface {
	DeleteMany(ctx context.Context, cids []cid.Cid) error
}

type Blockstore

type Blockstore interface {
	blockstore.Blockstore
	blockstore.Viewer
	BatchDeleter
	Flusher
}

Blockstore is the blockstore interface used by Lotus. It is the union of the basic go-ipfs blockstore, with other capabilities required by Lotus, e.g. View or Sync.

func Adapt

Adapt adapts a standard blockstore to a Lotus blockstore by enriching it with the extra methods that Lotus requires (e.g. View, Sync).

View proxies over to Get and calls the callback with the value supplied by Get. Sync noops.

func FromDatastore

func FromDatastore(dstore ds.Batching) Blockstore

FromDatastore creates a new blockstore backed by the given datastore.

func NewAPIBlockstore

func NewAPIBlockstore(cio ChainIO) Blockstore

func NewBlockstore

func NewBlockstore(dstore ds.Batching) Blockstore

NewBlockstore creates a new blockstore wrapped by the given datastore.

func WrapIDStore

func WrapIDStore(bstore blockstore.Blockstore) Blockstore

WrapIDStore wraps the underlying blockstore in an "identity" blockstore.

type BlockstoreGC

type BlockstoreGC interface {
	CollectGarbage(options ...BlockstoreGCOption) error
}

BlockstoreGC is a trait for blockstores that support online garbage collection consider

type BlockstoreGCOption

type BlockstoreGCOption = func(*BlockstoreGCOptions) error // nolint

BlockstoreGCOption is a functional interface for controlling blockstore GC options

func WithFullGC

func WithFullGC(fullgc bool) BlockstoreGCOption

type BlockstoreGCOptions

type BlockstoreGCOptions struct {
	FullGC bool
}

BlockstoreGCOptions is a struct with GC options

type BufferedBS

type BufferedBS struct {
	// contains filtered or unexported fields
}

func NewBufferedBstore

func NewBufferedBstore(base Blockstore) *BufferedBS

func NewTieredBstore

func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBS

func (*BufferedBS) AllKeysChan

func (bs *BufferedBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*BufferedBS) DeleteBlock

func (bs *BufferedBS) DeleteBlock(ctx context.Context, c cid.Cid) error

func (*BufferedBS) DeleteMany

func (bs *BufferedBS) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*BufferedBS) Flush added in v1.14.0

func (bs *BufferedBS) Flush(ctx context.Context) error

func (*BufferedBS) Get

func (bs *BufferedBS) Get(ctx context.Context, c cid.Cid) (blocks.Block, error)

func (*BufferedBS) GetSize

func (bs *BufferedBS) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*BufferedBS) Has

func (bs *BufferedBS) Has(ctx context.Context, c cid.Cid) (bool, error)

func (*BufferedBS) HashOnRead

func (bs *BufferedBS) HashOnRead(hor bool)

func (*BufferedBS) Put

func (bs *BufferedBS) Put(ctx context.Context, blk blocks.Block) error

func (*BufferedBS) PutMany

func (bs *BufferedBS) PutMany(ctx context.Context, blks []blocks.Block) error

func (*BufferedBS) Read

func (bs *BufferedBS) Read() Blockstore

func (*BufferedBS) View

func (bs *BufferedBS) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error

func (*BufferedBS) Write

func (bs *BufferedBS) Write() Blockstore

type CacheOpts

type CacheOpts = blockstore.CacheOpts

Alias so other packages don't have to import go-ipfs-blockstore type Blockstore = blockstore.Blockstore

type ChainIO

type ChainIO interface {
	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
	ChainHasObj(context.Context, cid.Cid) (bool, error)
}

type Flusher added in v1.14.0

type Flusher interface {
	Flush(context.Context) error
}

type GCBlockstore

type GCBlockstore = blockstore.GCBlockstore

Alias so other packages don't have to import go-ipfs-blockstore type Blockstore = blockstore.Blockstore

type GCLocker

type GCLocker = blockstore.GCLocker

Alias so other packages don't have to import go-ipfs-blockstore type Blockstore = blockstore.Blockstore

type IBlockCache

type IBlockCache interface {
	Get(key string) (value interface{}, ok bool)
	Remove(key string)
	Add(key string, value interface{})
	AddWithExpire(key string, value interface{}, dur time.Duration)
}

type LruCache

type LruCache struct {
	// contains filtered or unexported fields
}

func NewLruCache

func NewLruCache(size int) *LruCache

func (LruCache) Add

func (l LruCache) Add(key string, value interface{})

func (LruCache) AddWithExpire

func (l LruCache) AddWithExpire(key string, value interface{}, dur time.Duration)

func (LruCache) Get

func (l LruCache) Get(key string) (interface{}, bool)

func (LruCache) Remove

func (l LruCache) Remove(key string)

type MemBlockstore

type MemBlockstore map[string]blocks.Block

MemBlockstore is a terminal blockstore that keeps blocks in memory.

func NewMemory

func NewMemory() MemBlockstore

NewMemory returns a temporary memory-backed blockstore.

func NewTemporary

func NewTemporary() MemBlockstore

NewTemporary returns a temporary blockstore.

func (MemBlockstore) AllKeysChan

func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

AllKeysChan returns a channel from which the CIDs in the Blockstore can be read. It should respect the given context, closing the channel if it becomes Done.

func (MemBlockstore) DeleteBlock

func (m MemBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error

func (MemBlockstore) DeleteMany

func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error

func (MemBlockstore) Flush added in v1.14.0

func (MemBlockstore) Get

func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error)

func (MemBlockstore) GetSize

func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error)

GetSize returns the CIDs mapped BlockSize

func (MemBlockstore) Has

func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error)

func (MemBlockstore) HashOnRead

func (m MemBlockstore) HashOnRead(enabled bool)

HashOnRead specifies if every read block should be rehashed to make sure it matches its CID.

func (MemBlockstore) Put

Put puts a given block to the underlying datastore

func (MemBlockstore) PutMany

func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error

PutMany puts a slice of blocks at the same time using batching capabilities of the underlying datastore whenever possible.

func (MemBlockstore) View

func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error

type NetRPCErr added in v1.9.0

type NetRPCErr struct {
	Type NetRPCErrType

	Msg string

	// in case of NRpcErrNotFound
	Cid *cid.Cid
}

func (*NetRPCErr) MarshalCBOR added in v1.9.0

func (t *NetRPCErr) MarshalCBOR(w io.Writer) error

func (*NetRPCErr) UnmarshalCBOR added in v1.9.0

func (t *NetRPCErr) UnmarshalCBOR(r io.Reader) (err error)

type NetRPCErrType added in v1.9.0

type NetRPCErrType byte
const (
	NRpcErrGeneric NetRPCErrType = iota
	NRpcErrNotFound
)

type NetRPCReq added in v1.9.0

type NetRPCReq struct {
	Type NetRPCReqType
	ID   uint64

	Cid  []cid.Cid // todo maxsize?
	Data [][]byte  // todo maxsize?
}

func (*NetRPCReq) MarshalCBOR added in v1.9.0

func (t *NetRPCReq) MarshalCBOR(w io.Writer) error

func (*NetRPCReq) UnmarshalCBOR added in v1.9.0

func (t *NetRPCReq) UnmarshalCBOR(r io.Reader) (err error)

type NetRPCReqType added in v1.9.0

type NetRPCReqType byte
const (
	NRpcHas NetRPCReqType = iota
	NRpcGet
	NRpcGetSize
	NRpcPut
	NRpcDelete
)

type NetRPCResp added in v1.9.0

type NetRPCResp struct {
	Type NetRPCRespType
	ID   uint64

	// error or cids in allkeys
	Data []byte // todo maxsize?
	// contains filtered or unexported fields
}

func (*NetRPCResp) MarshalCBOR added in v1.9.0

func (t *NetRPCResp) MarshalCBOR(w io.Writer) error

func (*NetRPCResp) UnmarshalCBOR added in v1.9.0

func (t *NetRPCResp) UnmarshalCBOR(r io.Reader) (err error)

type NetRPCRespType added in v1.9.0

type NetRPCRespType byte
const (
	NRpcOK NetRPCRespType = iota
	NRpcErr
	NRpcMore
)

type NetworkStore added in v1.9.0

type NetworkStore struct {
	// contains filtered or unexported fields
}

func NewNetworkStore added in v1.9.0

func NewNetworkStore(mss msgio.ReadWriteCloser) *NetworkStore

func NewNetworkStoreWS added in v1.9.0

func NewNetworkStoreWS(wc *websocket.Conn) *NetworkStore

func (*NetworkStore) AllKeysChan added in v1.9.0

func (n *NetworkStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*NetworkStore) DeleteBlock added in v1.9.0

func (n *NetworkStore) DeleteBlock(ctx context.Context, c cid.Cid) error

func (*NetworkStore) DeleteMany added in v1.9.0

func (n *NetworkStore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*NetworkStore) Flush added in v1.14.0

func (n *NetworkStore) Flush(context.Context) error

func (*NetworkStore) Get added in v1.9.0

func (n *NetworkStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error)

func (*NetworkStore) GetSize added in v1.9.0

func (n *NetworkStore) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*NetworkStore) Has added in v1.9.0

func (n *NetworkStore) Has(ctx context.Context, c cid.Cid) (bool, error)

func (*NetworkStore) HashOnRead added in v1.9.0

func (n *NetworkStore) HashOnRead(enabled bool)

func (*NetworkStore) OnClose added in v1.9.0

func (n *NetworkStore) OnClose(cb func())

func (*NetworkStore) Put added in v1.9.0

func (n *NetworkStore) Put(ctx context.Context, block blocks.Block) error

func (*NetworkStore) PutMany added in v1.9.0

func (n *NetworkStore) PutMany(ctx context.Context, blocks []blocks.Block) error

func (*NetworkStore) Stop added in v1.9.0

func (n *NetworkStore) Stop(ctx context.Context) error

func (*NetworkStore) View added in v1.9.0

func (n *NetworkStore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error

type NetworkStoreHandler added in v1.9.0

type NetworkStoreHandler struct {
	// contains filtered or unexported fields
}

func HandleNetBstoreStream added in v1.9.0

func HandleNetBstoreStream(ctx context.Context, bs Blockstore, mss msgio.ReadWriteCloser) *NetworkStoreHandler

NOTE: This code isn't yet hardened to accept untrusted input. See TODOs here and in net.go

func HandleNetBstoreWS added in v1.9.0

func HandleNetBstoreWS(ctx context.Context, bs Blockstore, wc *websocket.Conn) *NetworkStoreHandler

type Options

type Options struct {
	badger.Options

	// Prefix is an optional prefix to prepend to keys. Default: "".
	Prefix string
}

Options embeds the badger options themselves, and augments them with blockstore-specific options.

func BadgerBlockstoreOptions

func BadgerBlockstoreOptions(path string, readonly bool) (Options, error)

BadgerBlockstoreOptions returns the badger options to apply for the provided domain.

func DefaultOptions

func DefaultOptions(path string) Options

type Set

type Set struct {
	// contains filtered or unexported fields
}

Set is a implementation of a set of Cids, that is, a structure to which holds a single copy of every Cids that is added to it.

func NewSet

func NewSet() *Set

NewSet initializes and returns a new Set.

func (*Set) Add

func (s *Set) Add(c cid.Cid)

Add puts a Cid in the Set.

func (*Set) ForEach

func (s *Set) ForEach(f func(c cid.Cid) error) error

ForEach allows to run a custom function on each Cid in the set.

func (*Set) Has

func (s *Set) Has(c cid.Cid) bool

Has returns if the Set contains a given Cid.

func (*Set) Keys

func (s *Set) Keys() []cid.Cid

Keys returns the Cids in the set.

func (*Set) Len

func (s *Set) Len() int

Len returns how many elements the Set has.

func (*Set) Remove

func (s *Set) Remove(c cid.Cid)

Remove deletes a Cid from the Set.

func (*Set) Visit

func (s *Set) Visit(c cid.Cid) bool

Visit adds a Cid to the set only if it is not in it already.

type SyncStore

type SyncStore struct {
	// contains filtered or unexported fields
}

func NewTemporarySync

func NewTemporarySync() *SyncStore

NewTemporarySync returns a thread-safe temporary blockstore.

func (*SyncStore) AllKeysChan

func (m *SyncStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

AllKeysChan returns a channel from which the CIDs in the blockstore can be read. It should respect the given context, closing the channel if it becomes Done.

func (*SyncStore) DeleteBlock

func (m *SyncStore) DeleteBlock(ctx context.Context, k cid.Cid) error

func (*SyncStore) DeleteMany

func (m *SyncStore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*SyncStore) Flush added in v1.14.0

func (m *SyncStore) Flush(context.Context) error

func (*SyncStore) Get

func (m *SyncStore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error)

func (*SyncStore) GetSize

func (m *SyncStore) GetSize(ctx context.Context, k cid.Cid) (int, error)

GetSize returns the CIDs mapped BlockSize

func (*SyncStore) Has

func (m *SyncStore) Has(ctx context.Context, k cid.Cid) (bool, error)

func (*SyncStore) HashOnRead

func (m *SyncStore) HashOnRead(enabled bool)

HashOnRead specifies if every read block should be rehashed to make sure it matches its CID.

func (*SyncStore) Put

func (m *SyncStore) Put(ctx context.Context, b blocks.Block) error

Put puts a given block to the underlying datastore

func (*SyncStore) PutMany

func (m *SyncStore) PutMany(ctx context.Context, bs []blocks.Block) error

PutMany puts a slice of blocks at the same time using batching capabilities of the underlying datastore whenever possible.

func (*SyncStore) View

func (m *SyncStore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error

type TimeCache

type TimeCache struct {
	// contains filtered or unexported fields
}

func NewTimeCache

func NewTimeCache(expireTime, cleanTime time.Duration) *TimeCache

func (TimeCache) Add

func (timeCache TimeCache) Add(key string, value interface{})

func (TimeCache) AddWithExpire

func (timeCache TimeCache) AddWithExpire(key string, value interface{}, dur time.Duration)

func (TimeCache) Get

func (timeCache TimeCache) Get(key string) (interface{}, bool)

func (TimeCache) Remove

func (timeCache TimeCache) Remove(key string)

type TxBlockstore

type TxBlockstore struct {
	// contains filtered or unexported fields
}

func (*TxBlockstore) AllKeysChan

func (txBlockstore *TxBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*TxBlockstore) ConvertKey

func (txBlockstore *TxBlockstore) ConvertKey(cid cid.Cid) datastore.Key

func (*TxBlockstore) DeleteBlock

func (txBlockstore *TxBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error

func (*TxBlockstore) DeleteMany

func (txBlockstore *TxBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*TxBlockstore) Flush added in v1.14.0

func (txBlockstore *TxBlockstore) Flush(context.Context) error

func (*TxBlockstore) Get

func (txBlockstore *TxBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)

func (*TxBlockstore) GetSize

func (txBlockstore *TxBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error)

func (*TxBlockstore) Has

func (txBlockstore *TxBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error)

func (*TxBlockstore) HashOnRead

func (txBlockstore *TxBlockstore) HashOnRead(enabled bool)

func (*TxBlockstore) Put

func (txBlockstore *TxBlockstore) Put(ctx context.Context, block blocks.Block) error

func (*TxBlockstore) PutMany

func (txBlockstore *TxBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error

func (*TxBlockstore) View

func (txBlockstore *TxBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error

type Viewer

type Viewer = blockstore.Viewer

Alias so other packages don't have to import go-ipfs-blockstore type Blockstore = blockstore.Blockstore

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL