blockstore

package
v1.11.3-rc1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 8, 2021 License: Apache-2.0, MIT Imports: 25 Imported by: 87

Documentation

Overview

Package blockstore and subpackages contain most of the blockstore implementations used by Lotus.

Blockstores not ultimately constructed out of the building blocks in this package may not work properly.

This package re-exports parts of the go-ipfs-blockstore package such that no other package needs to import it directly, for ergonomics and traceability.

Index

Constants

This section is empty.

Variables

View Source
var CacheMeasures = struct {
	HitRatio       *stats.Float64Measure
	Hits           *stats.Int64Measure
	Misses         *stats.Int64Measure
	Entries        *stats.Int64Measure
	QueriesServed  *stats.Int64Measure
	Adds           *stats.Int64Measure
	Updates        *stats.Int64Measure
	Evictions      *stats.Int64Measure
	CostAdded      *stats.Int64Measure
	CostEvicted    *stats.Int64Measure
	SetsDropped    *stats.Int64Measure
	SetsRejected   *stats.Int64Measure
	QueriesDropped *stats.Int64Measure
}{
	HitRatio:       stats.Float64("blockstore/cache/hit_ratio", "Hit ratio of blockstore cache", stats.UnitDimensionless),
	Hits:           stats.Int64("blockstore/cache/hits", "Total number of hits at blockstore cache", stats.UnitDimensionless),
	Misses:         stats.Int64("blockstore/cache/misses", "Total number of misses at blockstore cache", stats.UnitDimensionless),
	Entries:        stats.Int64("blockstore/cache/entry_count", "Total number of entries currently in the blockstore cache", stats.UnitDimensionless),
	QueriesServed:  stats.Int64("blockstore/cache/queries_served", "Total number of queries served by the blockstore cache", stats.UnitDimensionless),
	Adds:           stats.Int64("blockstore/cache/adds", "Total number of adds to blockstore cache", stats.UnitDimensionless),
	Updates:        stats.Int64("blockstore/cache/updates", "Total number of updates in blockstore cache", stats.UnitDimensionless),
	Evictions:      stats.Int64("blockstore/cache/evictions", "Total number of evictions from blockstore cache", stats.UnitDimensionless),
	CostAdded:      stats.Int64("blockstore/cache/cost_added", "Total cost (byte size) of entries added into blockstore cache", stats.UnitBytes),
	CostEvicted:    stats.Int64("blockstore/cache/cost_evicted", "Total cost (byte size) of entries evicted by blockstore cache", stats.UnitBytes),
	SetsDropped:    stats.Int64("blockstore/cache/sets_dropped", "Total number of sets dropped by blockstore cache", stats.UnitDimensionless),
	SetsRejected:   stats.Int64("blockstore/cache/sets_rejected", "Total number of sets rejected by blockstore cache", stats.UnitDimensionless),
	QueriesDropped: stats.Int64("blockstore/cache/queries_dropped", "Total number of queries dropped by blockstore cache", stats.UnitDimensionless),
}

CacheMeasures groups all metrics emitted by the blockstore caches.

View Source
var CacheMetricsEmitInterval = 5 * time.Second

CacheMetricsEmitInterval is the interval at which metrics are emitted onto OpenCensus.

View Source
var (
	CacheName, _ = tag.NewKey("cache_name")
)
View Source
var CacheViews = struct {
	HitRatio       *view.View
	Hits           *view.View
	Misses         *view.View
	Entries        *view.View
	QueriesServed  *view.View
	Adds           *view.View
	Updates        *view.View
	Evictions      *view.View
	CostAdded      *view.View
	CostEvicted    *view.View
	SetsDropped    *view.View
	SetsRejected   *view.View
	QueriesDropped *view.View
}{
	HitRatio: &view.View{
		Measure:     CacheMeasures.HitRatio,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Hits: &view.View{
		Measure:     CacheMeasures.Hits,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Misses: &view.View{
		Measure:     CacheMeasures.Misses,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Entries: &view.View{
		Measure:     CacheMeasures.Entries,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	QueriesServed: &view.View{
		Measure:     CacheMeasures.QueriesServed,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Adds: &view.View{
		Measure:     CacheMeasures.Adds,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Updates: &view.View{
		Measure:     CacheMeasures.Updates,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	Evictions: &view.View{
		Measure:     CacheMeasures.Evictions,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	CostAdded: &view.View{
		Measure:     CacheMeasures.CostAdded,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	CostEvicted: &view.View{
		Measure:     CacheMeasures.CostEvicted,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	SetsDropped: &view.View{
		Measure:     CacheMeasures.SetsDropped,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	SetsRejected: &view.View{
		Measure:     CacheMeasures.SetsRejected,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
	QueriesDropped: &view.View{
		Measure:     CacheMeasures.QueriesDropped,
		Aggregation: view.LastValue(),
		TagKeys:     []tag.Key{CacheName},
	},
}

CacheViews groups all cache-related default views.

View Source
var DefaultViews = []*view.View{
	CacheViews.HitRatio,
	CacheViews.Hits,
	CacheViews.Misses,
	CacheViews.Entries,
	CacheViews.QueriesServed,
	CacheViews.Adds,
	CacheViews.Updates,
	CacheViews.Evictions,
	CacheViews.CostAdded,
	CacheViews.CostEvicted,
	CacheViews.SetsDropped,
	CacheViews.SetsRejected,
	CacheViews.QueriesDropped,
}

DefaultViews exports all default views for this package.

View Source
var ErrNotFound = blockstore.ErrNotFound

Functions

This section is empty.

Types

type BasicBlockstore

type BasicBlockstore = blockstore.Blockstore

BasicBlockstore is an alias to the original IPFS Blockstore.

type BatchDeleter

type BatchDeleter interface {
	DeleteMany(cids []cid.Cid) error
}

type Blockstore

type Blockstore interface {
	blockstore.Blockstore
	blockstore.Viewer
	BatchDeleter
}

Blockstore is the blockstore interface used by Lotus. It is the union of the basic go-ipfs blockstore, with other capabilities required by Lotus, e.g. View or Sync.

func Adapt

Adapt adapts a standard blockstore to a Lotus blockstore by enriching it with the extra methods that Lotus requires (e.g. View, Sync).

View proxies over to Get and calls the callback with the value supplied by Get. Sync noops.

func FromDatastore

func FromDatastore(dstore ds.Batching) Blockstore

FromDatastore creates a new blockstore backed by the given datastore.

func NewAPIBlockstore

func NewAPIBlockstore(cio ChainIO) Blockstore

func NewDiscardStore added in v1.11.1

func NewDiscardStore(bs Blockstore) Blockstore

func NewIDStore

func NewIDStore(bs Blockstore) Blockstore

func NewLocalIPFSBlockstore

func NewLocalIPFSBlockstore(ctx context.Context, onlineMode bool) (Blockstore, error)

func NewRemoteIPFSBlockstore

func NewRemoteIPFSBlockstore(ctx context.Context, maddr multiaddr.Multiaddr, onlineMode bool) (Blockstore, error)

func Union

func Union(stores ...Blockstore) Blockstore

Union returns an unioned blockstore.

  • Reads return from the first blockstore that has the value, querying in the supplied order.
  • Writes (puts and deltes) are broadcast to all stores.

func UnwrapFallbackStore

func UnwrapFallbackStore(bs Blockstore) (Blockstore, bool)

UnwrapFallbackStore takes a blockstore, and returns the underlying blockstore if it was a FallbackStore. Otherwise, it just returns the supplied store unmodified.

func WrapIDStore

func WrapIDStore(bstore blockstore.Blockstore) Blockstore

WrapIDStore wraps the underlying blockstore in an "identity" blockstore. The ID store filters out all puts for blocks with CIDs using the "identity" hash function. It also extracts inlined blocks from CIDs using the identity hash function and returns them on get/has, ignoring the contents of the blockstore.

type BlockstoreGC added in v1.11.1

type BlockstoreGC interface {
	CollectGarbage(options ...BlockstoreGCOption) error
}

BlockstoreGC is a trait for blockstores that support online garbage collection

type BlockstoreGCOption added in v1.11.1

type BlockstoreGCOption = func(*BlockstoreGCOptions) error

BlockstoreGCOption is a functional interface for controlling blockstore GC options

func WithFullGC added in v1.11.1

func WithFullGC(fullgc bool) BlockstoreGCOption

type BlockstoreGCOptions added in v1.11.1

type BlockstoreGCOptions struct {
	FullGC bool
}

BlockstoreGCOptions is a struct with GC options

type BlockstoreIterator added in v1.11.1

type BlockstoreIterator interface {
	ForEachKey(func(cid.Cid) error) error
}

BlockstoreIterator is a trait for efficient iteration

type BlockstoreSize added in v1.11.1

type BlockstoreSize interface {
	Size() (int64, error)
}

BlockstoreSize is a trait for on-disk blockstores that can report their size

type BufferedBlockstore

type BufferedBlockstore struct {
	// contains filtered or unexported fields
}

func NewBuffered

func NewBuffered(base Blockstore) *BufferedBlockstore

func NewTieredBstore

func NewTieredBstore(r Blockstore, w Blockstore) *BufferedBlockstore

func (*BufferedBlockstore) AllKeysChan

func (bs *BufferedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*BufferedBlockstore) DeleteBlock

func (bs *BufferedBlockstore) DeleteBlock(c cid.Cid) error

func (*BufferedBlockstore) DeleteMany

func (bs *BufferedBlockstore) DeleteMany(cids []cid.Cid) error

func (*BufferedBlockstore) Get

func (bs *BufferedBlockstore) Get(c cid.Cid) (block.Block, error)

func (*BufferedBlockstore) GetSize

func (bs *BufferedBlockstore) GetSize(c cid.Cid) (int, error)

func (*BufferedBlockstore) Has

func (bs *BufferedBlockstore) Has(c cid.Cid) (bool, error)

func (*BufferedBlockstore) HashOnRead

func (bs *BufferedBlockstore) HashOnRead(hor bool)

func (*BufferedBlockstore) Put

func (bs *BufferedBlockstore) Put(blk block.Block) error

func (*BufferedBlockstore) PutMany

func (bs *BufferedBlockstore) PutMany(blks []block.Block) error

func (*BufferedBlockstore) Read

func (bs *BufferedBlockstore) Read() Blockstore

func (*BufferedBlockstore) View

func (bs *BufferedBlockstore) View(c cid.Cid, callback func([]byte) error) error

type ChainIO

type ChainIO interface {
	ChainReadObj(context.Context, cid.Cid) ([]byte, error)
	ChainHasObj(context.Context, cid.Cid) (bool, error)
}

type FallbackStore

type FallbackStore struct {
	Blockstore
	// contains filtered or unexported fields
}

FallbackStore is a read-through store that queries another (potentially remote) source if the block is not found locally. If the block is found during the fallback, it stores it in the local store.

func (*FallbackStore) Get

func (fbs *FallbackStore) Get(c cid.Cid) (blocks.Block, error)

func (*FallbackStore) GetSize

func (fbs *FallbackStore) GetSize(c cid.Cid) (int, error)

func (*FallbackStore) SetFallback

func (fbs *FallbackStore) SetFallback(missFn func(context.Context, cid.Cid) (blocks.Block, error))

type IPFSBlockstore

type IPFSBlockstore struct {
	// contains filtered or unexported fields
}

func (*IPFSBlockstore) AllKeysChan

func (i *IPFSBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*IPFSBlockstore) DeleteBlock

func (i *IPFSBlockstore) DeleteBlock(cid cid.Cid) error

func (*IPFSBlockstore) Get

func (i *IPFSBlockstore) Get(cid cid.Cid) (blocks.Block, error)

func (*IPFSBlockstore) GetSize

func (i *IPFSBlockstore) GetSize(cid cid.Cid) (int, error)

func (*IPFSBlockstore) Has

func (i *IPFSBlockstore) Has(cid cid.Cid) (bool, error)

func (*IPFSBlockstore) HashOnRead

func (i *IPFSBlockstore) HashOnRead(enabled bool)

func (*IPFSBlockstore) Put

func (i *IPFSBlockstore) Put(block blocks.Block) error

func (*IPFSBlockstore) PutMany

func (i *IPFSBlockstore) PutMany(blocks []blocks.Block) error

type MemBlockstore

type MemBlockstore map[cid.Cid]blocks.Block

MemBlockstore is a terminal blockstore that keeps blocks in memory.

func NewMemory

func NewMemory() MemBlockstore

NewMemory returns a temporary memory-backed blockstore.

func (MemBlockstore) AllKeysChan

func (m MemBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

AllKeysChan returns a channel from which the CIDs in the Blockstore can be read. It should respect the given context, closing the channel if it becomes Done.

func (MemBlockstore) DeleteBlock

func (m MemBlockstore) DeleteBlock(k cid.Cid) error

func (MemBlockstore) DeleteMany

func (m MemBlockstore) DeleteMany(ks []cid.Cid) error

func (MemBlockstore) Get

func (m MemBlockstore) Get(k cid.Cid) (blocks.Block, error)

func (MemBlockstore) GetSize

func (m MemBlockstore) GetSize(k cid.Cid) (int, error)

GetSize returns the CIDs mapped BlockSize

func (MemBlockstore) Has

func (m MemBlockstore) Has(k cid.Cid) (bool, error)

func (MemBlockstore) HashOnRead

func (m MemBlockstore) HashOnRead(enabled bool)

HashOnRead specifies if every read block should be rehashed to make sure it matches its CID.

func (MemBlockstore) Put

func (m MemBlockstore) Put(b blocks.Block) error

Put puts a given block to the underlying datastore

func (MemBlockstore) PutMany

func (m MemBlockstore) PutMany(bs []blocks.Block) error

PutMany puts a slice of blocks at the same time using batching capabilities of the underlying datastore whenever possible.

func (MemBlockstore) View

func (m MemBlockstore) View(k cid.Cid, callback func([]byte) error) error

type SyncBlockstore

type SyncBlockstore struct {
	// contains filtered or unexported fields
}

SyncBlockstore is a terminal blockstore that is a synchronized version of MemBlockstore.

func NewMemorySync

func NewMemorySync() *SyncBlockstore

NewMemorySync returns a thread-safe in-memory blockstore.

func (*SyncBlockstore) AllKeysChan

func (m *SyncBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*SyncBlockstore) DeleteBlock

func (m *SyncBlockstore) DeleteBlock(k cid.Cid) error

func (*SyncBlockstore) DeleteMany

func (m *SyncBlockstore) DeleteMany(ks []cid.Cid) error

func (*SyncBlockstore) Get

func (m *SyncBlockstore) Get(k cid.Cid) (blocks.Block, error)

func (*SyncBlockstore) GetSize

func (m *SyncBlockstore) GetSize(k cid.Cid) (int, error)

func (*SyncBlockstore) Has

func (m *SyncBlockstore) Has(k cid.Cid) (bool, error)

func (*SyncBlockstore) HashOnRead

func (m *SyncBlockstore) HashOnRead(enabled bool)

func (*SyncBlockstore) Put

func (m *SyncBlockstore) Put(b blocks.Block) error

func (*SyncBlockstore) PutMany

func (m *SyncBlockstore) PutMany(bs []blocks.Block) error

func (*SyncBlockstore) View

func (m *SyncBlockstore) View(k cid.Cid, callback func([]byte) error) error

type TimedCacheBlockstore

type TimedCacheBlockstore struct {
	// contains filtered or unexported fields
}

TimedCacheBlockstore is a blockstore that keeps blocks for at least the specified caching interval before discarding them. Garbage collection must be started and stopped by calling Start/Stop.

Under the covers, it's implemented with an active and an inactive blockstore that are rotated every cache time interval. This means all blocks will be stored at most 2x the cache interval.

Create a new instance by calling the NewTimedCacheBlockstore constructor.

func NewTimedCacheBlockstore

func NewTimedCacheBlockstore(interval time.Duration) *TimedCacheBlockstore

func (*TimedCacheBlockstore) AllKeysChan

func (t *TimedCacheBlockstore) AllKeysChan(_ context.Context) (<-chan cid.Cid, error)

func (*TimedCacheBlockstore) DeleteBlock

func (t *TimedCacheBlockstore) DeleteBlock(k cid.Cid) error

func (*TimedCacheBlockstore) DeleteMany

func (t *TimedCacheBlockstore) DeleteMany(ks []cid.Cid) error

func (*TimedCacheBlockstore) Get

func (t *TimedCacheBlockstore) Get(k cid.Cid) (blocks.Block, error)

func (*TimedCacheBlockstore) GetSize

func (t *TimedCacheBlockstore) GetSize(k cid.Cid) (int, error)

func (*TimedCacheBlockstore) Has

func (t *TimedCacheBlockstore) Has(k cid.Cid) (bool, error)

func (*TimedCacheBlockstore) HashOnRead

func (t *TimedCacheBlockstore) HashOnRead(_ bool)

func (*TimedCacheBlockstore) Put

func (*TimedCacheBlockstore) PutMany

func (t *TimedCacheBlockstore) PutMany(bs []blocks.Block) error

func (*TimedCacheBlockstore) Start

func (*TimedCacheBlockstore) Stop

func (*TimedCacheBlockstore) View

func (t *TimedCacheBlockstore) View(k cid.Cid, callback func([]byte) error) error

type Viewer

type Viewer = blockstore.Viewer

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL