splitstore

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 15, 2021 License: Apache-2.0, MIT Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BloomFilterMinSize     = 10_000_000
	BloomFilterProbability = 0.01
)

Variables

View Source 
var (
	// CompactionThreshold is the number of epochs that need to have elapsed
	// from the previously compacted epoch to trigger a new compaction.
	//
	//        |················· CompactionThreshold ··················|
	//        |                                                        |
	// =======‖≡≡≡≡≡≡≡‖-----------------------|------------------------»
	//        |       |                       |   chain -->             ↑__ current epoch
	//        |·······|                       |
	//            ↑________ CompactionCold    ↑________ CompactionBoundary
	//
	// === :: cold (already archived)
	// ≡≡≡ :: to be archived in this compaction
	// --- :: hot
	CompactionThreshold = 5 * build.Finality

	// CompactionCold is the number of epochs that will be archived to the
	// cold store on compaction. See diagram on CompactionThreshold for a
	// better sense.
	CompactionCold = build.Finality

	// CompactionBoundary is the number of epochs from the current epoch at which
	// we will walk the chain for live objects
	CompactionBoundary = 2 * build.Finality
)

Functions

This section is empty.

Types

type BloomMarkSet 

type BloomMarkSet struct {
	// contains filtered or unexported fields
}

func (*BloomMarkSet) Close 

func (s *BloomMarkSet) Close() error

func (*BloomMarkSet) Has 

func (s *BloomMarkSet) Has(cid cid.Cid) (bool, error)

func (*BloomMarkSet) Mark 

func (s *BloomMarkSet) Mark(cid cid.Cid) error

type BloomMarkSetEnv 

type BloomMarkSetEnv struct{}

func NewBloomMarkSetEnv 

func NewBloomMarkSetEnv() (*BloomMarkSetEnv, error)

func (*BloomMarkSetEnv) Close 

func (e *BloomMarkSetEnv) Close() error

func (*BloomMarkSetEnv) Create 

func (e *BloomMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error)

type BoltMarkSet 

type BoltMarkSet struct {
	// contains filtered or unexported fields
}

func (*BoltMarkSet) Close 

func (s *BoltMarkSet) Close() error

func (*BoltMarkSet) Has 

func (s *BoltMarkSet) Has(cid cid.Cid) (result bool, err error)

func (*BoltMarkSet) Mark 

func (s *BoltMarkSet) Mark(cid cid.Cid) error

type BoltMarkSetEnv 

type BoltMarkSetEnv struct {
	// contains filtered or unexported fields
}

func NewBoltMarkSetEnv 

func NewBoltMarkSetEnv(path string) (*BoltMarkSetEnv, error)

func (*BoltMarkSetEnv) Close 

func (e *BoltMarkSetEnv) Close() error

func (*BoltMarkSetEnv) Create 

func (e *BoltMarkSetEnv) Create(name string, hint int64) (MarkSet, error)

type BoltTrackingStore 

type BoltTrackingStore struct {
	// contains filtered or unexported fields
}

func OpenBoltTrackingStore 

func OpenBoltTrackingStore(path string) (*BoltTrackingStore, error)

func (*BoltTrackingStore) Close 

func (s *BoltTrackingStore) Close() error

func (*BoltTrackingStore) Delete 

func (s *BoltTrackingStore) Delete(cid cid.Cid) error

func (*BoltTrackingStore) DeleteBatch 

func (s *BoltTrackingStore) DeleteBatch(cids []cid.Cid) error

func (*BoltTrackingStore) ForEach 

func (s *BoltTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error

func (*BoltTrackingStore) Get 

func (s *BoltTrackingStore) Get(cid cid.Cid) (epoch abi.ChainEpoch, err error)

func (*BoltTrackingStore) Put 

func (s *BoltTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error

func (*BoltTrackingStore) PutBatch 

func (s *BoltTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error

func (*BoltTrackingStore) Sync 

func (s *BoltTrackingStore) Sync() error

type ChainAccessor 

type ChainAccessor interface {
	GetTipsetByHeight(context.Context, abi.ChainEpoch, *types.TipSet, bool) (*types.TipSet, error)
	GetHeaviestTipSet() *types.TipSet
	SubscribeHeadChanges(change func(revert []*types.TipSet, apply []*types.TipSet) error)
	WalkSnapshot(context.Context, *types.TipSet, abi.ChainEpoch, bool, bool, func(cid.Cid) error) error
}

ChainAccessor allows the Splitstore to access the chain. It will most likely be a ChainStore at runtime.

type Config 

type Config struct {
	// TrackingStore is the type of tracking store to use.
	//
	// Supported values are: "bolt" (default if omitted), "mem" (for tests and readonly access).
	TrackingStoreType string

	// MarkSetType is the type of mark set to use.
	//
	// Supported values are: "bloom" (default if omitted), "bolt".
	MarkSetType string
	// DisableCompaction is the switch of splitstore compaction,
	// You should enable this option if you plan to disable move data from hot to cold.
	DisableCompaction bool
	// perform full reachability analysis (expensive) for compaction
	// You should enable this option if you plan to use the splitstore without a backing coldstore
	EnableFullCompaction bool
	// EXPERIMENTAL enable pruning of unreachable objects.
	// This has not been sufficiently tested yet; only enable if you know what you are doing.
	// Only applies if you enable full compaction.
	EnableGC bool
	// full archival nodes should enable this if EnableFullCompaction is enabled
	// do NOT enable this if you synced from a snapshot.
	// Only applies if you enabled full compaction
	Archival bool

	// drop the cold store, the node only save the hot data.
	EnableColdDrop bool
}

type MarkSet 

type MarkSet interface {
	Mark(cid.Cid) error
	Has(cid.Cid) (bool, error)
	Close() error
}

MarkSet is a utility to keep track of seen CID, and later query for them.

* If the expected dataset is large, it can be backed by a datastore (e.g. bbolt). * If a probabilistic result is acceptable, it can be backed by a bloom filter (default).

type MarkSetEnv 

type MarkSetEnv interface {
	Create(name string, sizeHint int64) (MarkSet, error)
	Close() error
}

func OpenMarkSetEnv 

func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error)

type MemTrackingStore 

type MemTrackingStore struct {
	sync.Mutex
	// contains filtered or unexported fields
}

MemTrackingStore is a simple in-memory tracking store

func NewMemTrackingStore 

func NewMemTrackingStore() *MemTrackingStore

NewMemTrackingStore creates an in-memory tracking store. This is only useful for test or situations where you don't want to open the real tracking store (eg concurrent read only access on a node's datastore)

func (*MemTrackingStore) Close 

func (s *MemTrackingStore) Close() error

func (*MemTrackingStore) Delete 

func (s *MemTrackingStore) Delete(cid cid.Cid) error

func (*MemTrackingStore) DeleteBatch 

func (s *MemTrackingStore) DeleteBatch(cids []cid.Cid) error

func (*MemTrackingStore) ForEach 

func (s *MemTrackingStore) ForEach(f func(cid.Cid, abi.ChainEpoch) error) error

func (*MemTrackingStore) Get 

func (s *MemTrackingStore) Get(cid cid.Cid) (abi.ChainEpoch, error)

func (*MemTrackingStore) Put 

func (s *MemTrackingStore) Put(cid cid.Cid, epoch abi.ChainEpoch) error

func (*MemTrackingStore) PutBatch 

func (s *MemTrackingStore) PutBatch(cids []cid.Cid, epoch abi.ChainEpoch) error

func (*MemTrackingStore) Sync 

func (s *MemTrackingStore) Sync() error

type SplitStore 

type SplitStore struct {
	// contains filtered or unexported fields
}

func Open 

func Open(path string, us bstore.Blockstore, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Config) (*SplitStore, error)

Open opens an existing splistore, or creates a new splitstore. The splitstore is backed by the provided hot and cold stores. The returned SplitStore MUST be attached to the ChainStore with Start in order to trigger compaction.

func (*SplitStore) AllKeysChan 

func (s *SplitStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)

func (*SplitStore) Close 

func (s *SplitStore) Close() error

func (*SplitStore) CollectGarbage 

func (s *SplitStore) CollectGarbage() error

func (*SplitStore) DeleteBlock 

func (s *SplitStore) DeleteBlock(_ cid.Cid) error

Blockstore interface

func (*SplitStore) DeleteMany 

func (s *SplitStore) DeleteMany(_ []cid.Cid) error

func (*SplitStore) Get 

func (s *SplitStore) Get(cid cid.Cid) (blocks.Block, error)

func (*SplitStore) GetSize 

func (s *SplitStore) GetSize(cid cid.Cid) (int, error)

func (*SplitStore) Has 

func (s *SplitStore) Has(cid cid.Cid) (bool, error)

func (*SplitStore) HashOnRead 

func (s *SplitStore) HashOnRead(enabled bool)

func (*SplitStore) HeadChange 

func (s *SplitStore) HeadChange(_, apply []*types.TipSet) error

func (*SplitStore) Put 

func (s *SplitStore) Put(blk blocks.Block) error

func (*SplitStore) PutMany 

func (s *SplitStore) PutMany(blks []blocks.Block) error

func (*SplitStore) Start 

func (s *SplitStore) Start(chain ChainAccessor) error

State tracking

func (*SplitStore) View 

func (s *SplitStore) View(cid cid.Cid, cb func([]byte) error) error

type TrackingStore 

type TrackingStore interface {
	Put(cid.Cid, abi.ChainEpoch) error
	PutBatch([]cid.Cid, abi.ChainEpoch) error
	Get(cid.Cid) (abi.ChainEpoch, error)
	Delete(cid.Cid) error
	DeleteBatch([]cid.Cid) error
	ForEach(func(cid.Cid, abi.ChainEpoch) error) error
	Sync() error
	Close() error
}

TrackingStore is a persistent store that tracks blocks that are added to the hotstore, tracking the epoch at which they are written.

func OpenTrackingStore 

func OpenTrackingStore(path string, ttype string) (TrackingStore, error)

OpenTrackingStore opens a tracking store of the specified type in the specified path.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.