blockstore

package
v1.16.0-rc2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 22, 2022 License: Apache-2.0, MIT Imports: 25 Imported by: 88

Documentation

Overview

Package blockstore and subpackages contain most of the blockstore implementations used by Lotus.

Blockstores not ultimately constructed out of the building blocks in this package may not work properly.

This package re-exports parts of the go-ipfs-blockstore package such that no other package needs to import it directly, for ergonomics and traceability.

Index

Constants

This section is empty.

Variables

View Source 
var CacheMeasures = struct {
	HitRatio       *stats.Float64Measure
	Hits           *stats.Int64Measure
	Misses         *stats.Int64Measure
	Entries        *stats.Int64Measure
	QueriesServed  *stats.Int64Measure
	Adds           *stats.Int64Measure
	Updates        *stats.Int64Measure
	Evictions      *stats.Int64Measure
	CostAdded      *stats.Int64Measure
	CostEvicted    *stats.Int64Measure
	SetsDropped    *stats.Int64Measure
	SetsRejected   *stats.Int64Measure
	QueriesDropped *stats.Int64Measure
}{
	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
}

CacheMeasures groups all metrics emitted by the blockstore caches.

View Source 
var CacheMetricsEmitInterval = 5 * time.Second

CacheMetricsEmitInterval is the interval at which metrics are emitted onto OpenCensus.

View Source 
var (
	CacheName, _ = tag.NewKey("cache_name")
)
View Source 
var CacheViews = struct {
	HitRatio       *view.View
	Hits           *view.View
	Misses         *view.View
	Entries        *view.View
	QueriesServed  *view.View
	Adds           *view.View
	Updates        *view.View
	Evictions      *view.View
	CostAdded      *view.View
	CostEvicted    *view.View
	SetsDropped    *view.View
	SetsRejected   *view.View
	QueriesDropped *view.View
}{
	HitRatio: &view.View{
		Measure:     CacheMeasures.HitRatio,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Hits: &view.View{
		Measure:     CacheMeasures.Hits,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Misses: &view.View{
		Measure:     CacheMeasures.Misses,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Entries: &view.View{
		Measure:     CacheMeasures.Entries,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	QueriesServed: &view.View{
		Measure:     CacheMeasures.QueriesServed,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Adds: &view.View{
		Measure:     CacheMeasures.Adds,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Updates: &view.View{
		Measure:     CacheMeasures.Updates,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Evictions: &view.View{
		Measure:     CacheMeasures.Evictions,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	CostAdded: &view.View{
		Measure:     CacheMeasures.CostAdded,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	CostEvicted: &view.View{
		Measure:     CacheMeasures.CostEvicted,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	SetsDropped: &view.View{
		Measure:     CacheMeasures.SetsDropped,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	SetsRejected: &view.View{
		Measure:     CacheMeasures.SetsRejected,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	QueriesDropped: &view.View{
		Measure:     CacheMeasures.QueriesDropped,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
}

CacheViews groups all cache-related default views.

View Source 
var DefaultViews = []*view.View{
	CacheViews.HitRatio,
	CacheViews.Hits,
	CacheViews.Misses,
	CacheViews.Entries,
	CacheViews.QueriesServed,
	CacheViews.Adds,
	CacheViews.Updates,
	CacheViews.Evictions,
	CacheViews.CostAdded,
	CacheViews.CostEvicted,
	CacheViews.SetsDropped,
	CacheViews.SetsRejected,
	CacheViews.QueriesDropped,
}

DefaultViews exports all default views for this package.

View Source 
var ErrNotFound = blockstore.ErrNotFound

Functions

func IsHotView added in v1.14.0 

func IsHotView(ctx context.Context) bool

IsHotView returns true if the hot view option is set in the context

func WithHotView added in v1.14.0 

func WithHotView(ctx context.Context) context.Context

WithHotView constructs a new context with an option that provides a hint to the blockstore (e.g. the splitstore) that the object (and its ipld references) should be kept hot.

Types

type AutobatchBlockstore added in v1.14.0 

type AutobatchBlockstore struct {
	// contains filtered or unexported fields
}

func NewAutobatch added in v1.14.0 

func NewAutobatch(ctx context.Context, backingBs Blockstore, bufferCapacity int) *AutobatchBlockstore

func (*AutobatchBlockstore) AllKeysChan added in v1.14.0 

func (bs *AutobatchBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*AutobatchBlockstore) DeleteBlock added in v1.14.0 

func (bs *AutobatchBlockstore) DeleteBlock(context.Context, cid.Cid) error

func (*AutobatchBlockstore) DeleteMany added in v1.14.0 

func (bs *AutobatchBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*AutobatchBlockstore) Flush added in v1.14.0 

func (bs *AutobatchBlockstore) Flush(ctx context.Context) error

caller must NOT hold stateLock

func (*AutobatchBlockstore) Get added in v1.14.0 

func (bs *AutobatchBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error)

func (*AutobatchBlockstore) GetSize added in v1.14.0 

func (bs *AutobatchBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*AutobatchBlockstore) Has added in v1.14.0 

func (bs *AutobatchBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)

func (*AutobatchBlockstore) HashOnRead added in v1.14.0 

func (bs *AutobatchBlockstore) HashOnRead(enabled bool)

func (*AutobatchBlockstore) Put added in v1.14.0 

func (bs *AutobatchBlockstore) Put(ctx context.Context, blk block.Block) error

func (*AutobatchBlockstore) PutMany added in v1.14.0 

func (bs *AutobatchBlockstore) PutMany(ctx context.Context, blks []block.Block) error

func (*AutobatchBlockstore) Shutdown added in v1.14.0 

func (bs *AutobatchBlockstore) Shutdown(ctx context.Context) error

func (*AutobatchBlockstore) View added in v1.14.0 

func (bs *AutobatchBlockstore) View(ctx context.Context, cid cid.Cid, callback func([]byte) error) error

type BasicBlockstore 

type BasicBlockstore = blockstore.Blockstore

BasicBlockstore is an alias to the original IPFS Blockstore.

type BatchDeleter 

type BatchDeleter interface {
	DeleteMany(ctx context.Context, cids []cid.Cid) error
}

type Blockstore 

type Blockstore interface {
	blockstore.Blockstore
	blockstore.Viewer
	BatchDeleter
}

Blockstore is the blockstore interface used by Lotus. It is the union of the basic go-ipfs blockstore, with other capabilities required by Lotus, e.g. View or Sync.

func Adapt 

func Adapt(bs blockstore.Blockstore) Blockstore

Adapt adapts a standard blockstore to a Lotus blockstore by enriching it with the extra methods that Lotus requires (e.g. View, Sync).

View proxies over to Get and calls the callback with the value supplied by Get. Sync noops.

func FromDatastore 

func FromDatastore(dstore ds.Batching) Blockstore

FromDatastore creates a new blockstore backed by the given datastore.

func NewAPIBlockstore 

func NewAPIBlockstore(cio ChainIO) Blockstore

func NewDiscardStore added in v1.11.1 

func NewDiscardStore(bs Blockstore) Blockstore

func NewIDStore 

func NewIDStore(bs Blockstore) Blockstore

func NewLocalIPFSBlockstore 

func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error)

func NewRemoteIPFSBlockstore 

func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error)

func Union 

func Union(stores ...Blockstore) Blockstore

Union returns an unioned blockstore.

  • Reads return from the first blockstore that has the value, querying in the supplied order.
  • Writes (puts and deltes) are broadcast to all stores.

func UnwrapFallbackStore 

func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool)

UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore if it was a FallbackStore. Otherwise, it just returns the supplied store unmodified.

func WrapIDStore 

func WrapIDStore(bstore blockstore.Blockstore) Blockstore

WrapIDStore wraps the underlying blockstore in an "identity" blockstore. The ID store filters out all puts for blocks with CIDs using the "identity" hash function. It also extracts inlined blocks from CIDs using the identity hash function and returns them on get/has, ignoring the contents of the blockstore.

type BlockstoreGC added in v1.11.1 

type BlockstoreGC interface {
	CollectGarbage(options ...BlockstoreGCOption) error
}

BlockstoreGC is a trait for blockstores that support online garbage collection

type BlockstoreGCOption added in v1.11.1 

type BlockstoreGCOption = func(*BlockstoreGCOptions) error

BlockstoreGCOption is a functional interface for controlling blockstore GC options

func WithFullGC added in v1.11.1 

func WithFullGC(fullgc bool) BlockstoreGCOption

type BlockstoreGCOptions added in v1.11.1 

type BlockstoreGCOptions struct {
	FullGC bool
}

BlockstoreGCOptions is a struct with GC options

type BlockstoreIterator added in v1.11.1 

type BlockstoreIterator interface {
	ForEachKey(func(cid.Cid) error) error
}

BlockstoreIterator is a trait for efficient iteration

type BlockstoreSize added in v1.11.1 

type BlockstoreSize interface {
	Size() (int64, error)
}

BlockstoreSize is a trait for on-disk blockstores that can report their size

type BufferedBlockstore 

type BufferedBlockstore struct {
	// contains filtered or unexported fields
}

func NewBuffered 

func NewBuffered(base Blockstore) *BufferedBlockstore

func NewTieredBstore 

func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore

func (*BufferedBlockstore) AllKeysChan 

func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*BufferedBlockstore) DeleteBlock 

func (bs *BufferedBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error

func (*BufferedBlockstore) DeleteMany 

func (bs *BufferedBlockstore) DeleteMany(ctx context.Context, cids []cid.Cid) error

func (*BufferedBlockstore) Get 

func (bs *BufferedBlockstore) Get(ctx context.Context, c cid.Cid) (block.Block, error)

func (*BufferedBlockstore) GetSize 

func (bs *BufferedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*BufferedBlockstore) Has 

func (bs *BufferedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error)

func (*BufferedBlockstore) HashOnRead 

func (bs *BufferedBlockstore) HashOnRead(hor bool)

func (*BufferedBlockstore) Put 

func (bs *BufferedBlockstore) Put(ctx context.Context, blk block.Block) error

func (*BufferedBlockstore) PutMany 

func (bs *BufferedBlockstore) PutMany(ctx context.Context, blks []block.Block) error

func (*BufferedBlockstore) Read 

func (bs *BufferedBlockstore) Read() Blockstore

func (*BufferedBlockstore) View 

func (bs *BufferedBlockstore) View(ctx context.Context, c cid.Cid, callback func([]byte) error) error

type ChainIO 

type ChainIO interface {
	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
	ChainHasObj(context.Context, cid.Cid) (bool, error)
	ChainPutObj(context.Context, blocks.Block) error
}

type FallbackStore 

type FallbackStore struct {
	Blockstore
	// contains filtered or unexported fields
}

FallbackStore is a read-through store that queries another (potentially remote) source if the block is not found locally. If the block is found during the fallback, it stores it in the local store.

func (*FallbackStore) Get 

func (fbs *FallbackStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error)

func (*FallbackStore) GetSize 

func (fbs *FallbackStore) GetSize(ctx context.Context, c cid.Cid) (int, error)

func (*FallbackStore) SetFallback 

func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error))

type IPFSBlockstore 

type IPFSBlockstore struct {
	// contains filtered or unexported fields
}

func (*IPFSBlockstore) AllKeysChan 

func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*IPFSBlockstore) DeleteBlock 

func (i *IPFSBlockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error

func (*IPFSBlockstore) Get 

func (i *IPFSBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error)

func (*IPFSBlockstore) GetSize 

func (i *IPFSBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error)

func (*IPFSBlockstore) Has 

func (i *IPFSBlockstore) Has(ctx context.Context, cid cid.Cid) (bool, error)

func (*IPFSBlockstore) HashOnRead 

func (i *IPFSBlockstore) HashOnRead(enabled bool)

func (*IPFSBlockstore) Put 

func (i *IPFSBlockstore) Put(ctx context.Context, block blocks.Block) error

func (*IPFSBlockstore) PutMany 

func (i *IPFSBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error

type MemBlockstore 

type MemBlockstore map[cid.Cid]blocks.Block

MemBlockstore is a terminal blockstore that keeps blocks in memory.

func NewMemory 

func NewMemory() MemBlockstore

NewMemory returns a temporary memory-backed blockstore.

func (MemBlockstore) AllKeysChan 

func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

AllKeysChan returns a channel from which the CIDs in the Blockstore can be read. It should respect the given context, closing the channel if it becomes Done.

func (MemBlockstore) DeleteBlock 

func (m MemBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error

func (MemBlockstore) DeleteMany 

func (m MemBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error

func (MemBlockstore) Get 

func (m MemBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error)

func (MemBlockstore) GetSize 

func (m MemBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error)

GetSize returns the CIDs mapped BlockSize

func (MemBlockstore) Has 

func (m MemBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error)

func (MemBlockstore) HashOnRead 

func (m MemBlockstore) HashOnRead(enabled bool)

HashOnRead specifies if every read block should be rehashed to make sure it matches its CID.

func (MemBlockstore) Put 

func (m MemBlockstore) Put(ctx context.Context, b blocks.Block) error

Put puts a given block to the underlying datastore

func (MemBlockstore) PutMany 

func (m MemBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error

PutMany puts a slice of blocks at the same time using batching capabilities of the underlying datastore whenever possible.

func (MemBlockstore) View 

func (m MemBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error

type SyncBlockstore 

type SyncBlockstore struct {
	// contains filtered or unexported fields
}

SyncBlockstore is a terminal blockstore that is a synchronized version of MemBlockstore.

func NewMemorySync 

func NewMemorySync() *SyncBlockstore

NewMemorySync returns a thread-safe in-memory blockstore.

func (*SyncBlockstore) AllKeysChan 

func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*SyncBlockstore) DeleteBlock 

func (m *SyncBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error

func (*SyncBlockstore) DeleteMany 

func (m *SyncBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error

func (*SyncBlockstore) Get 

func (m *SyncBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error)

func (*SyncBlockstore) GetSize 

func (m *SyncBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error)

func (*SyncBlockstore) Has 

func (m *SyncBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error)

func (*SyncBlockstore) HashOnRead 

func (m *SyncBlockstore) HashOnRead(enabled bool)

func (*SyncBlockstore) Put 

func (m *SyncBlockstore) Put(ctx context.Context, b blocks.Block) error

func (*SyncBlockstore) PutMany 

func (m *SyncBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error

func (*SyncBlockstore) View 

func (m *SyncBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error

type TimedCacheBlockstore 

type TimedCacheBlockstore struct {
	// contains filtered or unexported fields
}

TimedCacheBlockstore is a blockstore that keeps blocks for at least the specified caching interval before discarding them. Garbage collection must be started and stopped by calling Start/Stop.

Under the covers, it's implemented with an active and an inactive blockstore that are rotated every cache time interval. This means all blocks will be stored at most 2x the cache interval.

Create a new instance by calling the NewTimedCacheBlockstore constructor.

func NewTimedCacheBlockstore 

func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore

func (*TimedCacheBlockstore) AllKeysChan 

func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error)

func (*TimedCacheBlockstore) DeleteBlock 

func (t *TimedCacheBlockstore) DeleteBlock(ctx context.Context, k cid.Cid) error

func (*TimedCacheBlockstore) DeleteMany 

func (t *TimedCacheBlockstore) DeleteMany(ctx context.Context, ks []cid.Cid) error

func (*TimedCacheBlockstore) Get 

func (t *TimedCacheBlockstore) Get(ctx context.Context, k cid.Cid) (blocks.Block, error)

func (*TimedCacheBlockstore) GetSize 

func (t *TimedCacheBlockstore) GetSize(ctx context.Context, k cid.Cid) (int, error)

func (*TimedCacheBlockstore) Has 

func (t *TimedCacheBlockstore) Has(ctx context.Context, k cid.Cid) (bool, error)

func (*TimedCacheBlockstore) HashOnRead 

func (t *TimedCacheBlockstore) HashOnRead(_ bool)

func (*TimedCacheBlockstore) Put 

func (t *TimedCacheBlockstore) Put(ctx context.Context, b blocks.Block) error

func (*TimedCacheBlockstore) PutMany 

func (t *TimedCacheBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error

func (*TimedCacheBlockstore) Start 

func (t *TimedCacheBlockstore) Start(_ context.Context) error

func (*TimedCacheBlockstore) Stop 

func (t *TimedCacheBlockstore) Stop(_ context.Context) error

func (*TimedCacheBlockstore) View 

func (t *TimedCacheBlockstore) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error

type Viewer 

type Viewer = blockstore.Viewer

Source Files

View all Source files

Directories

Path Synopsis
stm: #unit
 stm: #unit

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.