Documentation ¶
Index ¶
- Constants
- Variables
- func WithPriority(ctx context.Context, priority int) context.Context
- type Call
- type CallState
- type EnvFunc
- type ExecutorFunc
- type FaultTracker
- type LocalWorker
- func (l *LocalWorker) AddPiece(ctx context.Context, sector storage.SectorRef, epcs []abi.UnpaddedPieceSize, ...) (storiface.CallID, error)
- func (l *LocalWorker) Close() error
- func (l *LocalWorker) Fetch(ctx context.Context, sector storage.SectorRef, ...) (storiface.CallID, error)
- func (l *LocalWorker) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)
- func (l *LocalWorker) FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) (storiface.CallID, error)
- func (l *LocalWorker) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) (storiface.CallID, error)
- func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error)
- func (l *LocalWorker) MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)
- func (l *LocalWorker) NewSector(ctx context.Context, sector storage.SectorRef) error
- func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error)
- func (l *LocalWorker) ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, ...) (storiface.CallID, error)
- func (l *LocalWorker) ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, ...) (storiface.CallID, error)
- func (l *LocalWorker) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) (storiface.CallID, error)
- func (l *LocalWorker) Remove(ctx context.Context, sector abi.SectorID) error
- func (l *LocalWorker) ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (storiface.CallID, error)
- func (l *LocalWorker) SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, ...) (storiface.CallID, error)
- func (l *LocalWorker) SealCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.Commit1Out) (storiface.CallID, error)
- func (l *LocalWorker) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, ...) (storiface.CallID, error)
- func (l *LocalWorker) SealPreCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.PreCommit1Out) (storiface.CallID, error)
- func (l *LocalWorker) Session(ctx context.Context) (uuid.UUID, error)
- func (l *LocalWorker) TaskDisable(ctx context.Context, tt sealtasks.TaskType) error
- func (l *LocalWorker) TaskEnable(ctx context.Context, tt sealtasks.TaskType) error
- func (l *LocalWorker) TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error)
- func (l *LocalWorker) UnsealPiece(ctx context.Context, sector storage.SectorRef, ...) (storiface.CallID, error)
- func (l *LocalWorker) WaitQuiet()
- type Manager
- func (m *Manager) Abort(ctx context.Context, call storiface.CallID) error
- func (m *Manager) AddLocalStorage(ctx context.Context, path string) error
- func (m *Manager) AddPiece(ctx context.Context, sector storage.SectorRef, ...) (abi.PieceInfo, error)
- func (m *Manager) AddWorker(ctx context.Context, w Worker) error
- func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, ...) (map[abi.SectorID]string, error)
- func (m *Manager) Close(ctx context.Context) error
- func (m *Manager) FinalizeReplicaUpdate(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error
- func (m *Manager) FinalizeSector(ctx context.Context, sector storage.SectorRef, keepUnsealed []storage.Range) error
- func (m *Manager) FsStat(ctx context.Context, id stores.ID) (fsutil.FsStat, error)
- func (m *Manager) GenerateSectorKeyFromData(ctx context.Context, sector storage.SectorRef, commD cid.Cid) error
- func (m *Manager) NewSector(ctx context.Context, sector storage.SectorRef) error
- func (m *Manager) ProveReplicaUpdate1(ctx context.Context, sector storage.SectorRef, ...) (out storage.ReplicaVanillaProofs, err error)
- func (m *Manager) ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, ...) (out storage.ReplicaUpdateProof, err error)
- func (m *Manager) ReleaseReplicaUpgrade(ctx context.Context, sector storage.SectorRef) error
- func (m *Manager) ReleaseSectorKey(ctx context.Context, sector storage.SectorRef) error
- func (m *Manager) ReleaseUnsealed(ctx context.Context, sector storage.SectorRef, safeToFree []storage.Range) error
- func (m *Manager) Remove(ctx context.Context, sector storage.SectorRef) error
- func (m *Manager) ReplicaUpdate(ctx context.Context, sector storage.SectorRef, pieces []abi.PieceInfo) (out storage.ReplicaUpdateOut, err error)
- func (m *Manager) ReturnAddPiece(ctx context.Context, callID storiface.CallID, pi abi.PieceInfo, ...) error
- func (m *Manager) ReturnFetch(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnFinalizeReplicaUpdate(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnFinalizeSector(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnGenerateSectorKeyFromData(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnMoveStorage(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnProveReplicaUpdate1(ctx context.Context, callID storiface.CallID, out storage.ReplicaVanillaProofs, ...) error
- func (m *Manager) ReturnProveReplicaUpdate2(ctx context.Context, callID storiface.CallID, proof storage.ReplicaUpdateProof, ...) error
- func (m *Manager) ReturnReadPiece(ctx context.Context, callID storiface.CallID, ok bool, ...) error
- func (m *Manager) ReturnReleaseUnsealed(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) ReturnReplicaUpdate(ctx context.Context, callID storiface.CallID, out storage.ReplicaUpdateOut, ...) error
- func (m *Manager) ReturnSealCommit1(ctx context.Context, callID storiface.CallID, out storage.Commit1Out, ...) error
- func (m *Manager) ReturnSealCommit2(ctx context.Context, callID storiface.CallID, proof storage.Proof, ...) error
- func (m *Manager) ReturnSealPreCommit1(ctx context.Context, callID storiface.CallID, p1o storage.PreCommit1Out, ...) error
- func (m *Manager) ReturnSealPreCommit2(ctx context.Context, callID storiface.CallID, sealed storage.SectorCids, ...) error
- func (m *Manager) ReturnUnsealPiece(ctx context.Context, callID storiface.CallID, err *storiface.CallError) error
- func (m *Manager) SchedDiag(ctx context.Context, doSched bool) (interface{}, error)
- func (m *Manager) SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, ...) (out storage.Commit1Out, err error)
- func (m *Manager) SealCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.Commit1Out) (out storage.Proof, err error)
- func (m *Manager) SealPreCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, ...) (out storage.PreCommit1Out, err error)
- func (m *Manager) SealPreCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.PreCommit1Out) (out storage.SectorCids, err error)
- func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, ...) error
- func (m *Manager) ServeHTTP(w http.ResponseWriter, r *http.Request)
- func (m *Manager) StorageLocal(ctx context.Context) (map[stores.ID]string, error)
- func (m *Manager) WorkerJobs() map[uuid.UUID][]storiface.WorkerJob
- func (m *Manager) WorkerStats() map[uuid.UUID]storiface.WorkerStats
- type ManagerStateStore
- type ManyBytes
- type PieceProvider
- type ResourceFilteringStrategy
- type ReturnType
- type SchedDiagInfo
- type SchedDiagRequestInfo
- type SealerConfig
- type SectorManager
- type StorageAuth
- type Unsealer
- type WorkID
- type WorkState
- type WorkStatus
- type Worker
- type WorkerAction
- type WorkerConfig
- type WorkerSelector
- type WorkerStateStore
Constants ¶
const ( // ResourceFilteringHardware specifies that available hardware resources // should be evaluated when scheduling a task against the worker. ResourceFilteringHardware = ResourceFilteringStrategy("hardware") // ResourceFilteringDisabled disables resource filtering against this // worker. The scheduler may assign any task to this worker. ResourceFilteringDisabled = ResourceFilteringStrategy("disabled") )
Variables ¶
var ClosedWorkerID = uuid.UUID{}
var DefaultSchedPriority = 0
var ErrNoWorkers = errors.New("no suitable workers found")
var InitWait = 3 * time.Second
var MaxPieceReaderBurnBytes int64 = 1 << 20 // 1M
For small read skips, it's faster to "burn" some bytes than to setup new sector reader. Assuming 1ms stream seek latency, and 1G/s stream rate, we're willing to discard up to 1 MiB.
var ReadBuf = 128 * (127 * 8) // unpadded(128k)
var SchedPriorityKey schedPrioCtxKey
var (
SchedWindows = 2
)
var SelectorTimeout = 5 * time.Second
Functions ¶
Types ¶
type Call ¶ added in v1.1.3
type Call struct { ID storiface.CallID RetType ReturnType State CallState Result *ManyBytes // json bytes }
type ExecutorFunc ¶ added in v1.1.3
type ExecutorFunc func() (ffiwrapper.Storage, error)
used do provide custom proofs impl (mostly used in testing)
type FaultTracker ¶
type FaultTracker interface {
CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error)
}
FaultTracker TODO: Track things more actively
type LocalWorker ¶
type LocalWorker struct {
// contains filtered or unexported fields
}
func NewLocalWorker ¶
func NewLocalWorker(wcfg WorkerConfig, store stores.Store, local *stores.Local, sindex stores.SectorIndex, ret storiface.WorkerReturn, cst *statestore.StateStore) *LocalWorker
func (*LocalWorker) AddPiece ¶
func (l *LocalWorker) AddPiece(ctx context.Context, sector storage.SectorRef, epcs []abi.UnpaddedPieceSize, sz abi.UnpaddedPieceSize, r io.Reader) (storiface.CallID, error)
func (*LocalWorker) Close ¶
func (l *LocalWorker) Close() error
func (*LocalWorker) Fetch ¶
func (l *LocalWorker) Fetch(ctx context.Context, sector storage.SectorRef, fileType storiface.SectorFileType, ptype storiface.PathType, am storiface.AcquireMode) (storiface.CallID, error)
func (*LocalWorker) FinalizeReplicaUpdate ¶ added in v1.14.0
func (*LocalWorker) FinalizeSector ¶
func (*LocalWorker) GenerateSectorKeyFromData ¶ added in v1.14.0
func (*LocalWorker) Info ¶
func (l *LocalWorker) Info(context.Context) (storiface.WorkerInfo, error)
func (*LocalWorker) MoveStorage ¶
func (l *LocalWorker) MoveStorage(ctx context.Context, sector storage.SectorRef, types storiface.SectorFileType) (storiface.CallID, error)
func (*LocalWorker) Paths ¶
func (l *LocalWorker) Paths(ctx context.Context) ([]stores.StoragePath, error)
func (*LocalWorker) ProveReplicaUpdate1 ¶ added in v1.14.0
func (*LocalWorker) ProveReplicaUpdate2 ¶ added in v1.14.0
func (l *LocalWorker) ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (storiface.CallID, error)
func (*LocalWorker) ReleaseUnsealed ¶
func (*LocalWorker) ReplicaUpdate ¶ added in v1.14.0
func (*LocalWorker) SealCommit1 ¶
func (l *LocalWorker) SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (storiface.CallID, error)
func (*LocalWorker) SealCommit2 ¶
func (l *LocalWorker) SealCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.Commit1Out) (storiface.CallID, error)
func (*LocalWorker) SealPreCommit1 ¶
func (*LocalWorker) SealPreCommit2 ¶
func (l *LocalWorker) SealPreCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.PreCommit1Out) (storiface.CallID, error)
func (*LocalWorker) TaskDisable ¶ added in v1.2.2
func (*LocalWorker) TaskEnable ¶ added in v1.2.2
func (*LocalWorker) UnsealPiece ¶
func (l *LocalWorker) UnsealPiece(ctx context.Context, sector storage.SectorRef, index storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, cid cid.Cid) (storiface.CallID, error)
func (*LocalWorker) WaitQuiet ¶ added in v1.1.3
func (l *LocalWorker) WaitQuiet()
WaitQuiet blocks as long as there are tasks running
type Manager ¶
func New ¶
func New(ctx context.Context, lstor *stores.Local, stor *stores.Remote, ls stores.LocalStorage, si stores.SectorIndex, sc SealerConfig, wss WorkerStateStore, mss ManagerStateStore) (*Manager, error)
func (*Manager) AddLocalStorage ¶
func (*Manager) CheckProvable ¶
func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, rg storiface.RGetter) (map[abi.SectorID]string, error)
CheckProvable returns unprovable sectors
func (*Manager) FinalizeReplicaUpdate ¶ added in v1.14.0
func (*Manager) FinalizeSector ¶
func (*Manager) GenerateSectorKeyFromData ¶ added in v1.14.0
func (*Manager) ProveReplicaUpdate1 ¶ added in v1.14.0
func (*Manager) ProveReplicaUpdate2 ¶ added in v1.14.0
func (m *Manager) ProveReplicaUpdate2(ctx context.Context, sector storage.SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs storage.ReplicaVanillaProofs) (out storage.ReplicaUpdateProof, err error)
func (*Manager) ReleaseReplicaUpgrade ¶ added in v1.14.0
func (*Manager) ReleaseSectorKey ¶ added in v1.14.0
func (*Manager) ReleaseUnsealed ¶
func (*Manager) ReplicaUpdate ¶ added in v1.14.0
func (*Manager) ReturnAddPiece ¶ added in v1.1.3
func (*Manager) ReturnFetch ¶ added in v1.1.3
func (*Manager) ReturnFinalizeReplicaUpdate ¶ added in v1.14.0
func (*Manager) ReturnFinalizeSector ¶ added in v1.1.3
func (*Manager) ReturnGenerateSectorKeyFromData ¶ added in v1.14.0
func (*Manager) ReturnMoveStorage ¶ added in v1.1.3
func (*Manager) ReturnProveReplicaUpdate1 ¶ added in v1.14.0
func (*Manager) ReturnProveReplicaUpdate2 ¶ added in v1.14.0
func (*Manager) ReturnReadPiece ¶ added in v1.1.3
func (*Manager) ReturnReleaseUnsealed ¶ added in v1.1.3
func (*Manager) ReturnReplicaUpdate ¶ added in v1.14.0
func (*Manager) ReturnSealCommit1 ¶ added in v1.1.3
func (*Manager) ReturnSealCommit2 ¶ added in v1.1.3
func (*Manager) ReturnSealPreCommit1 ¶ added in v1.1.3
func (*Manager) ReturnSealPreCommit2 ¶ added in v1.1.3
func (*Manager) ReturnUnsealPiece ¶ added in v1.1.3
func (*Manager) SealCommit1 ¶
func (m *Manager) SealCommit1(ctx context.Context, sector storage.SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids storage.SectorCids) (out storage.Commit1Out, err error)
func (*Manager) SealCommit2 ¶
func (*Manager) SealPreCommit1 ¶
func (*Manager) SealPreCommit2 ¶
func (m *Manager) SealPreCommit2(ctx context.Context, sector storage.SectorRef, phase1Out storage.PreCommit1Out) (out storage.SectorCids, err error)
func (*Manager) SectorsUnsealPiece ¶ added in v1.11.0
func (m *Manager) SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed *cid.Cid) error
SectorsUnsealPiece will Unseal the Sealed sector file for the given sector. It will schedule the Unsealing task on a worker that either already has the sealed sector files or has space in one of it's sealing scratch spaces to store them after fetching them from another worker. If the chosen worker already has the Unsealed sector file, we will NOT Unseal the sealed sector file again.
func (*Manager) StorageLocal ¶
func (*Manager) WorkerStats ¶
func (m *Manager) WorkerStats() map[uuid.UUID]storiface.WorkerStats
type ManagerStateStore ¶ added in v1.1.3
type ManagerStateStore *statestore.StateStore
type ManyBytes ¶ added in v1.1.3
type ManyBytes struct {
// contains filtered or unexported fields
}
Ideally this would be a tag on the struct field telling cbor-gen to enforce higher max-len
type PieceProvider ¶ added in v1.11.0
type PieceProvider interface { // ReadPiece is used to read an Unsealed piece at the given offset and of the given size from a Sector // pieceOffset + pieceSize specify piece bounds for unsealing (note: with SDR the entire sector will be unsealed by // default in most cases, but this might matter with future PoRep) // startOffset is added to the pieceOffset to get the starting reader offset. // The number of bytes that can be read is pieceSize-startOffset ReadPiece(ctx context.Context, sector storage.SectorRef, pieceOffset storiface.UnpaddedByteIndex, pieceSize abi.UnpaddedPieceSize, ticket abi.SealRandomness, unsealed cid.Cid) (mount.Reader, bool, error) IsUnsealed(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error) }
func NewPieceProvider ¶ added in v1.11.0
func NewPieceProvider(storage *stores.Remote, index stores.SectorIndex, uns Unsealer) PieceProvider
type ResourceFilteringStrategy ¶ added in v1.11.0
type ResourceFilteringStrategy string
ResourceFilteringStrategy is an enum indicating the kinds of resource filtering strategies that can be configured for workers.
type ReturnType ¶ added in v1.1.3
type ReturnType string
const ( AddPiece ReturnType = "AddPiece" SealPreCommit1 ReturnType = "SealPreCommit1" SealPreCommit2 ReturnType = "SealPreCommit2" SealCommit1 ReturnType = "SealCommit1" SealCommit2 ReturnType = "SealCommit2" FinalizeSector ReturnType = "FinalizeSector" FinalizeReplicaUpdate ReturnType = "FinalizeReplicaUpdate" ReplicaUpdate ReturnType = "ReplicaUpdate" ProveReplicaUpdate1 ReturnType = "ProveReplicaUpdate1" ProveReplicaUpdate2 ReturnType = "ProveReplicaUpdate2" GenerateSectorKey ReturnType = "GenerateSectorKey" ReleaseUnsealed ReturnType = "ReleaseUnsealed" MoveStorage ReturnType = "MoveStorage" UnsealPiece ReturnType = "UnsealPiece" Fetch ReturnType = "Fetch" )
type SchedDiagInfo ¶
type SchedDiagInfo struct { Requests []SchedDiagRequestInfo OpenWindows []string }
type SchedDiagRequestInfo ¶
type SealerConfig ¶
type SealerConfig struct { ParallelFetchLimit int // Local worker config AllowAddPiece bool AllowPreCommit1 bool AllowPreCommit2 bool AllowCommit bool AllowUnseal bool AllowReplicaUpdate bool AllowProveReplicaUpdate2 bool AllowRegenSectorKey bool // ResourceFiltering instructs the system which resource filtering strategy // to use when evaluating tasks against this worker. An empty value defaults // to "hardware". ResourceFiltering ResourceFilteringStrategy }
type SectorManager ¶
type SectorManager interface { ffiwrapper.StorageSealer storage.Prover storiface.WorkerReturn FaultTracker }
type StorageAuth ¶
type Unsealer ¶ added in v1.11.0
type Unsealer interface { // SectorsUnsealPiece will Unseal a Sealed sector file for the given sector. SectorsUnsealPiece(ctx context.Context, sector storage.SectorRef, offset storiface.UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd *cid.Cid) error }
type WorkState ¶ added in v1.1.3
type WorkStatus ¶ added in v1.1.3
type WorkStatus string
type Worker ¶
type Worker interface { storiface.WorkerCalls TaskTypes(context.Context) (map[sealtasks.TaskType]struct{}, error) // Returns paths accessible to the worker Paths(context.Context) ([]stores.StoragePath, error) Info(context.Context) (storiface.WorkerInfo, error) Session(context.Context) (uuid.UUID, error) Close() error // TODO: do we need this? }
type WorkerConfig ¶
type WorkerSelector ¶
type WorkerSelector interface { Ok(ctx context.Context, task sealtasks.TaskType, spt abi.RegisteredSealProof, a *workerHandle) (bool, error) // true if worker is acceptable for performing a task Cmp(ctx context.Context, task sealtasks.TaskType, a, b *workerHandle) (bool, error) // true if a is preferred over b }
type WorkerStateStore ¶ added in v1.1.3
type WorkerStateStore *statestore.StateStore