stores

package
v1.13.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 19, 2021 License: Apache-2.0, MIT Imports: 37 Imported by: 0

Documentation

Index

Constants

View Source 
const MetaFile = "sectorstore.json"

Variables

View Source 
var CopyBuf = 1 << 20
View Source 
var FetchTempSubdir = "fetching"
View Source 
var HeartbeatInterval = 10 * time.Second
View Source 
var SkippedHeartbeatThresh = HeartbeatInterval * 5

Functions

This section is empty.

Types

type Decl 

type Decl struct {
	abi.SectorID
	storiface.SectorFileType
}

type DefaultPartialFileHandler added in v1.11.0 

type DefaultPartialFileHandler struct{}

DefaultPartialFileHandler is the default implementation of the partialFileHandler interface. This is probably the only implementation we'll ever use because the purpose of the interface to is to mock out partial file related functionality during testing.

func (*DefaultPartialFileHandler) Close added in v1.11.0 

func (d *DefaultPartialFileHandler) Close(pf *partialfile.PartialFile) error

Close closes the partial file

func (*DefaultPartialFileHandler) HasAllocated added in v1.11.0 

func (d *DefaultPartialFileHandler) HasAllocated(pf *partialfile.PartialFile, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)

func (*DefaultPartialFileHandler) OpenPartialFile added in v1.11.0 

func (d *DefaultPartialFileHandler) OpenPartialFile(maxPieceSize abi.PaddedPieceSize, path string) (*partialfile.PartialFile, error)

func (*DefaultPartialFileHandler) Reader added in v1.11.0 

func (d *DefaultPartialFileHandler) Reader(pf *partialfile.PartialFile, offset storiface.PaddedByteIndex, size abi.PaddedPieceSize) (*os.File, error)

type FetchHandler 

type FetchHandler struct {
	Local     Store
	PfHandler partialFileHandler
}

func (*FetchHandler) ServeHTTP 

func (handler *FetchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type HealthReport 

type HealthReport struct {
	Stat fsutil.FsStat
	Err  string
}

type ID 

type ID string

ID identifies sector storage by UUID. One sector storage should map to one 

filesystem, local or networked / shared by multiple machines

type Index 

type Index struct {
	// contains filtered or unexported fields
}

func NewIndex 

func NewIndex() *Index

func (*Index) FindSector 

func (i *Index) FindSector(id abi.SectorID, typ storiface.SectorFileType) ([]ID, error)

func (*Index) StorageAttach 

func (i *Index) StorageAttach(ctx context.Context, si StorageInfo, st fsutil.FsStat) error

func (*Index) StorageBestAlloc 

func (i *Index) StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]StorageInfo, error)

func (*Index) StorageDeclareSector 

func (i *Index) StorageDeclareSector(ctx context.Context, storageID ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error

func (*Index) StorageDropSector 

func (i *Index) StorageDropSector(ctx context.Context, storageID ID, s abi.SectorID, ft storiface.SectorFileType) error

func (*Index) StorageFindSector 

func (i *Index) StorageFindSector(ctx context.Context, s abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]SectorStorageInfo, error)

func (*Index) StorageInfo 

func (i *Index) StorageInfo(ctx context.Context, id ID) (StorageInfo, error)

func (*Index) StorageList 

func (i *Index) StorageList(ctx context.Context) (map[ID][]Decl, error)

func (Index) StorageLock 

func (i Index) StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error

func (*Index) StorageReportHealth 

func (i *Index) StorageReportHealth(ctx context.Context, id ID, report HealthReport) error

func (Index) StorageTryLock 

func (i Index) StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)

type Local 

type Local struct {
	// contains filtered or unexported fields
}

func NewLocal 

func NewLocal(ctx context.Context, ls LocalStorage, index SectorIndex, urls []string) (*Local, error)

func (*Local) AcquireSector 

func (st *Local) AcquireSector(ctx context.Context, sid storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, pathType storiface.PathType, op storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error)

func (*Local) FsStat 

func (st *Local) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error)

func (*Local) Local 

func (st *Local) Local(ctx context.Context) ([]StoragePath, error)

func (*Local) MoveStorage 

func (st *Local) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error

func (*Local) OpenPath 

func (st *Local) OpenPath(ctx context.Context, p string) error

func (*Local) Redeclare added in v1.1.3 

func (st *Local) Redeclare(ctx context.Context) error

func (*Local) Remove 

func (st *Local) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error

func (*Local) RemoveCopies 

func (st *Local) RemoveCopies(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType) error

func (*Local) Reserve 

func (st *Local) Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)

type LocalPath 

type LocalPath struct {
	Path string
}

type LocalStorage 

type LocalStorage interface {
	GetStorage() (StorageConfig, error)
	SetStorage(func(*StorageConfig)) error

	Stat(path string) (fsutil.FsStat, error)

	// returns real disk usage for a file/directory
	// os.ErrNotExit when file doesn't exist
	DiskUsage(path string) (int64, error)
}

type LocalStorageMeta 

type LocalStorageMeta struct {
	ID ID

	// A high weight means data is more likely to be stored in this path
	Weight uint64 // 0 = readonly

	// Intermediate data for the sealing process will be stored here
	CanSeal bool

	// Finalized sectors that will be proved over time will be stored here
	CanStore bool

	// MaxStorage specifies the maximum number of bytes to use for sector storage
	// (0 = unlimited)
	MaxStorage uint64
}

LocalStorageMeta path/sectorstore.json

type Remote 

type Remote struct {
	// contains filtered or unexported fields
}

func NewRemote 

func NewRemote(local Store, index SectorIndex, auth http.Header, fetchLimit int, pfHandler partialFileHandler) *Remote

func (*Remote) AcquireSector 

func (r *Remote) AcquireSector(ctx context.Context, s storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, pathType storiface.PathType, op storiface.AcquireMode) (storiface.SectorPaths, storiface.SectorPaths, error)

func (*Remote) CheckIsUnsealed added in v1.11.0 

func (r *Remote) CheckIsUnsealed(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize) (bool, error)

CheckIsUnsealed checks if we have an unsealed piece at the given offset in an already unsealed sector file for the given piece either locally or on any of the workers. Returns true if we have the unsealed piece, false otherwise.

func (*Remote) FsStat 

func (r *Remote) FsStat(ctx context.Context, id ID) (fsutil.FsStat, error)

func (*Remote) MoveStorage 

func (r *Remote) MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error

func (*Remote) Reader added in v1.11.0 

func (r *Remote) Reader(ctx context.Context, s storage.SectorRef, offset, size abi.PaddedPieceSize) (io.ReadCloser, error)

Reader returns a reader for an unsealed piece at the given offset in the given sector. If the Miner has the unsealed piece locally, it will return a reader that reads from the local copy. If the Miner does NOT have the unsealed piece locally, it will query all workers that have the unsealed sector file to know if they have the unsealed piece and will then read the unsealed piece data from a worker that has it.

Returns a nil reader if : 1. no worker(local worker included) has an unsealed file for the given sector OR 2. no worker(local worker included) has the unsealed piece in their unsealed sector file. Will return a nil reader and a nil error in such a case.

func (*Remote) Remove 

func (r *Remote) Remove(ctx context.Context, sid abi.SectorID, typ storiface.SectorFileType, force bool) error

func (*Remote) RemoveCopies 

func (r *Remote) RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error

func (*Remote) Reserve added in v1.11.0 

func (r *Remote) Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)

type SectorIndex 

type SectorIndex interface {
	StorageAttach(context.Context, StorageInfo, fsutil.FsStat) error
	StorageInfo(context.Context, ID) (StorageInfo, error)
	StorageReportHealth(context.Context, ID, HealthReport) error

	StorageDeclareSector(ctx context.Context, storageID ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error
	StorageDropSector(ctx context.Context, storageID ID, s abi.SectorID, ft storiface.SectorFileType) error
	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]SectorStorageInfo, error)

	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]StorageInfo, error)

	// atomically acquire locks on all sector file types. close ctx to unlock
	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error
	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)

	StorageList(ctx context.Context) (map[ID][]Decl, error)
}

type SectorStorageInfo 

type SectorStorageInfo struct {
	ID     ID
	URLs   []string // TODO: Support non-http transports
	Weight uint64

	CanSeal  bool
	CanStore bool

	Primary bool
}

type StorageConfig 

type StorageConfig struct {
	StoragePaths []LocalPath
}

StorageConfig .lotusstorage/storage.json

type StorageInfo 

type StorageInfo struct {
	ID         ID
	URLs       []string // TODO: Support non-http transports
	Weight     uint64
	MaxStorage uint64

	CanSeal  bool
	CanStore bool
}

type StoragePath 

type StoragePath struct {
	ID     ID
	Weight uint64

	LocalPath string

	CanSeal  bool
	CanStore bool
}

type Store 

type Store interface {
	AcquireSector(ctx context.Context, s storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, sealing storiface.PathType, op storiface.AcquireMode) (paths storiface.SectorPaths, stores storiface.SectorPaths, err error)
	Remove(ctx context.Context, s abi.SectorID, types storiface.SectorFileType, force bool) error

	// like remove, but doesn't remove the primary sector copy, nor the last
	// non-primary copy if there no primary copies
	RemoveCopies(ctx context.Context, s abi.SectorID, types storiface.SectorFileType) error

	// move sectors into storage
	MoveStorage(ctx context.Context, s storage.SectorRef, types storiface.SectorFileType) error

	FsStat(ctx context.Context, id ID) (fsutil.FsStat, error)

	Reserve(ctx context.Context, sid storage.SectorRef, ft storiface.SectorFileType, storageIDs storiface.SectorPaths, overheadTab map[storiface.SectorFileType]int) (func(), error)
}

type URLs added in v1.11.1 

type URLs []string

Source Files

View all Source files

Directories

Path Synopsis
Package mocks is a generated GoMock package.
 Package mocks is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.