storiface

package
v1.27.2 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 17, 2024 License: Apache-2.0, MIT Imports: 21 Imported by: 63

Documentation

Index

Constants

View Source
const (
	RWPrepared = 1
	RWRunning  = 0
	RWRetWait  = -1
	RWReturned = -2
	RWRetDone  = -3
)
View Source
const FSOverheadDen = 10

FSOverheadDen represents the constant value 10, which is used to calculate the overhead in various storage space utilization calculations.

View Source
const IDSep = "."

Variables

View Source
var ErrSectorNotFound = errors.New("sector not found")

FSOverheadSeal is a map that represents the overheads for different SectorFileType in sectors which are being sealed.

View Source
var FTAll = func() (out SectorFileType) {
	for _, pathType := range PathTypes {
		out |= pathType
	}
	return out
}()

FTAll represents the combination of all available sector file. It is a variable of type SectorFileType. The value of FTAll is calculated by iterating over the PathTypes slice and using the |= operator to perform a bitwise OR operation on each path type. The result is assigned to the variable out and returned. FTAll is immediately invoked as a function using the anonymous function syntax, so the result is returned as soon as it is calculated.

FsOverheadFinalized is a map that represents the finalized overhead for different types of SectorFileType. The keys in the map are the SectorFileType values, and the values are integers representing the overhead. It is used to calculate the storage space usage for different types of sectors, as shown in the example below: The overhead value is retrieved from FsOverheadFinalized by using the SectorFileType value as the key. If the overhead value is not found in the map, an error is returned indicating that there is no finalized overhead information for the given sector type.

View Source
var NoSectorRef = SectorRef{}
View Source
var ParallelDenom uint64 = 100
View Source
var ParallelNum uint64 = 92

Percent of threads to allocate to parallel tasks

12 * 0.92 = 11 16 * 0.92 = 14 24 * 0.92 = 22 32 * 0.92 = 29 64 * 0.92 = 58 128 * 0.92 = 117

PathTypes is a slice of SectorFileType that represents different types of sector file paths. It contains the following types of sector file paths: - FTUnsealed: represents regular unsealed sectors - FTSealed: represents sealed sectors - FTCache: represents cache sectors - FTUpdate: represents snap sectors - FTUpdateCache: represents snap cache sectors - FTPiece: represents Piece Park sectors

View Source
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
	sealtasks.TTAddPiece: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 8 << 30,
			MinMemory: 8 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 4 << 30,
			MinMemory: 4 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTPreCommit1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 128 << 30,
			MinMemory: 112 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 10 << 20,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 64 << 30,
			MinMemory: 56 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 10 << 20,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 768 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 20,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTPreCommit2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 30 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 15 << 30,
			MinMemory: 15 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: -1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: -1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: -1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTCommit1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTCommit2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 190 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 150 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTFetch: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
	},

	sealtasks.TTReplicaUpdate: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 8 << 30,
			MinMemory: 8 << 30,

			MaxParallelism:    1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 4 << 30,
			MinMemory: 4 << 30,

			MaxParallelism:    1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTProveReplicaUpdate1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTProveReplicaUpdate2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 190 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 150 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTGenerateWindowPoSt: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 120 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 96 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTGenerateWinningPoSt: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
}

Functions

func ParseResourceEnv

func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error)

func ParseSectorID

func ParseSectorID(baseName string) (abi.SectorID, error)

ParseSectorID parses a sector ID from a given base name. It expects the format "s-t0%d-%d", where the first %d represents the miner ID and the second %d represents the sector number.

Parameters: - baseName: The base name from which to parse the sector ID.

Returns: - abi.SectorID: The parsed sector ID. - error: An error if parsing fails.

Example usage:

id, err := ParseSectorID(baseName)
if err != nil {
  // handle error
}
// use id

func PathByType

func PathByType(sps SectorPaths, fileType SectorFileType) string

PathByType returns the path associated with the specified fileType in the given SectorPaths. It panics if the requested path type is unknown.

func SectorName

func SectorName(sid abi.SectorID) string

SectorName returns the name of a sector in the format "s-t0<Miner>-<Number>"

Parameters:

  • sid: The sector ID

Returns:

  • The name of the sector as a string

func SetPathByType

func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string)

Types

type AcquireMode

type AcquireMode string
const (
	AcquireMove AcquireMode = "move"
	AcquireCopy AcquireMode = "copy"
)

type AcquireOption added in v1.27.0

type AcquireOption func(*AcquireSettings)

func AcquireInto added in v1.27.0

func AcquireInto(pathIDs PathsWithIDs) AcquireOption

type AcquireSettings added in v1.27.0

type AcquireSettings struct {
	Into *PathsWithIDs
}

type CallError

type CallError struct {
	Code    ErrorCode
	Message string
	// contains filtered or unexported fields
}

func Err

func Err(code ErrorCode, sub error) *CallError

func (*CallError) ErrCode added in v1.25.0

func (c *CallError) ErrCode() ErrorCode

func (*CallError) Error

func (c *CallError) Error() string

func (*CallError) Unwrap

func (c *CallError) Unwrap() error

type CallID

type CallID struct {
	Sector abi.SectorID
	ID     uuid.UUID
}
var UndefCall CallID

func (*CallID) MarshalCBOR

func (t *CallID) MarshalCBOR(w io.Writer) error

func (CallID) String

func (c CallID) String() string

func (*CallID) UnmarshalCBOR

func (t *CallID) UnmarshalCBOR(r io.Reader) (err error)

type Commit1Out

type Commit1Out []byte

type Data

type Data = io.Reader

type Decl

type Decl struct {
	abi.SectorID
	SectorFileType
}

type ErrorCode

type ErrorCode int
const (
	// Temp Errors
	ErrTempUnknown ErrorCode = iota + 100
	ErrTempWorkerRestart
	ErrTempAllocateSpace
)
const (
	ErrUnknown ErrorCode = iota
)

type FallbackChallenges

type FallbackChallenges struct {
	Sectors    []abi.SectorNumber
	Challenges map[abi.SectorNumber][]uint64
}

type Group

type Group = string

type HealthReport

type HealthReport struct {
	Stat fsutil.FsStat
	Err  string
}

type ID

type ID string

ID identifies sector storage by UUID. One sector storage should map to one

filesystem, local or networked / shared by multiple machines

type IDList

type IDList []ID

func ParseIDList

func ParseIDList(s string) IDList

func (IDList) String

func (il IDList) String() string

type LocalPath added in v1.19.0

type LocalPath struct {
	Path string
}

type LocalStorageMeta added in v1.19.0

type LocalStorageMeta struct {
	ID ID

	// A high weight means data is more likely to be stored in this path
	Weight uint64 // 0 = readonly

	// Intermediate data for the sealing process will be stored here
	CanSeal bool

	// Finalized sectors that will be proved over time will be stored here
	CanStore bool

	// MaxStorage specifies the maximum number of bytes to use for sector storage
	// (0 = unlimited)
	MaxStorage uint64

	// List of storage groups this path belongs to
	Groups []string

	// List of storage groups to which data from this path can be moved. If none
	// are specified, allow to all
	AllowTo []string

	// AllowTypes lists sector file types which are allowed to be put into this
	// path. If empty, all file types are allowed.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	AllowTypes []string

	// DenyTypes lists sector file types which aren't allowed to be put into this
	// path.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	DenyTypes []string

	// AllowMiners lists miner IDs which are allowed to store their sector data into
	// this path. If empty, all miner IDs are allowed
	AllowMiners []string

	// DenyMiners lists miner IDs which are denied to store their sector data into
	// this path
	DenyMiners []string
}

LocalStorageMeta path/sectorstore.json

type PaddedByteIndex

type PaddedByteIndex uint64

type PathType

type PathType string
const (
	PathStorage PathType = "storage"
	PathSealing PathType = "sealing"
)

type PathsWithIDs added in v1.27.0

type PathsWithIDs struct {
	Paths SectorPaths
	IDs   SectorPaths
}

PathsWithIDs represents paths and IDs for sector files.

func (PathsWithIDs) HasAllSet added in v1.27.0

func (p PathsWithIDs) HasAllSet(ft SectorFileType) bool

HasAllSet checks if all paths and IDs in PathsWithIDs have a corresponding path set for the specified SectorFileType. It returns true if all paths and IDs are set, and false otherwise.

type PieceNumber added in v1.27.0

type PieceNumber uint64

PieceNumber is a reference to a piece in the storage system; mapping between pieces in the storage system and piece CIDs is maintained by the storage index

func (PieceNumber) Ref added in v1.27.0

func (pn PieceNumber) Ref() SectorRef

type PostSectorChallenge

type PostSectorChallenge struct {
	SealProof    abi.RegisteredSealProof
	SectorNumber abi.SectorNumber
	SealedCID    cid.Cid
	Challenge    []uint64
	Update       bool
}

type PreCommit1Out

type PreCommit1Out []byte

type Proof

type Proof []byte

type Prover

type Prover interface {
	AggregateSealProofs(aggregateInfo proof.AggregateSealVerifyProofAndInfos, proofs [][]byte) ([]byte, error)
}

Prover contains cheap proving-related methods

type ProverPoSt

type ProverPoSt interface {
	GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error)
	GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, ppt abi.RegisteredPoStProof, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) (proof []proof.PoStProof, skipped []abi.SectorID, err error)

	GenerateWinningPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte) ([]proof.PoStProof, error)
	GenerateWindowPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte, partitionIdx int) (proof.PoStProof, error)
}

type RGetter

type RGetter func(ctx context.Context, id abi.SectorID) (sealed cid.Cid, update bool, err error)

type Range

type Range struct {
	Offset abi.UnpaddedPieceSize
	Size   abi.UnpaddedPieceSize
}

type Reader added in v1.27.1

type Reader interface {
	io.Closer
	io.Reader
	io.ReaderAt
	io.Seeker
}

Reader is a fully-featured Reader. It is the union of the standard IO sequential access method (Read), with seeking ability (Seek), as well random access (ReadAt).

type ReplicaUpdateOut

type ReplicaUpdateOut struct {
	NewSealed   cid.Cid
	NewUnsealed cid.Cid
}

type ReplicaUpdateProof

type ReplicaUpdateProof []byte

type ReplicaVanillaProofs

type ReplicaVanillaProofs [][]byte

type Resources

type Resources struct {
	MinMemory uint64 `envname:"MIN_MEMORY"` // What Must be in RAM for decent perf
	MaxMemory uint64 `envname:"MAX_MEMORY"` // Memory required (swap + ram; peak memory usage during task execution)

	// GPUUtilization specifes the number of GPUs a task can use
	GPUUtilization float64 `envname:"GPU_UTILIZATION"`

	// MaxParallelism specifies the number of CPU cores when GPU is NOT in use
	MaxParallelism int `envname:"MAX_PARALLELISM"` // -1 = multithread

	// MaxParallelismGPU specifies the number of CPU cores when GPU is in use
	MaxParallelismGPU int `envname:"MAX_PARALLELISM_GPU"` // when 0, inherits MaxParallelism

	BaseMinMemory uint64 `envname:"BASE_MIN_MEMORY"` // What Must be in RAM for decent perf (shared between threads)

	MaxConcurrent int `envname:"MAX_CONCURRENT"` // Maximum number of tasks of this type that can be scheduled on a worker (0=default, no limit)
}

func (Resources) Threads

func (r Resources) Threads(wcpus uint64, gpus int) uint64

TODO: Take NUMA into account

type Sealer

type Sealer interface {
	NewSector(ctx context.Context, sector SectorRef) error
	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)
	AddPiece(ctx context.Context, sector SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)

	SealPreCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (PreCommit1Out, error)
	SealPreCommit2(ctx context.Context, sector SectorRef, pc1o PreCommit1Out) (SectorCids, error)

	SealCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids SectorCids) (Commit1Out, error)
	SealCommit2(ctx context.Context, sector SectorRef, c1o Commit1Out) (Proof, error)

	FinalizeSector(ctx context.Context, sector SectorRef) error

	// ReleaseUnsealed marks parts of the unsealed sector file as safe to drop
	//  (called by the fsm on restart, allows storage to keep no persistent
	//   state about unsealed fast-retrieval copies)
	ReleaseUnsealed(ctx context.Context, sector SectorRef, keepUnsealed []Range) error
	// ReleaseSectorKey removes `sealed` from all storage
	// called after successful sector upgrade
	ReleaseSectorKey(ctx context.Context, sector SectorRef) error
	// ReleaseReplicaUpgrade removes `update` / `update-cache` from all storage
	// called when aborting sector upgrade
	ReleaseReplicaUpgrade(ctx context.Context, sector SectorRef) error

	// Removes all data associated with the specified sector
	Remove(ctx context.Context, sector SectorRef) error

	// Generate snap deals replica update
	ReplicaUpdate(ctx context.Context, sector SectorRef, pieces []abi.PieceInfo) (ReplicaUpdateOut, error)

	// Prove that snap deals replica was done correctly
	ProveReplicaUpdate1(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (ReplicaVanillaProofs, error)
	ProveReplicaUpdate2(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs ReplicaVanillaProofs) (ReplicaUpdateProof, error)

	// GenerateSectorKeyFromData computes sector key given unsealed data and updated replica
	GenerateSectorKeyFromData(ctx context.Context, sector SectorRef, unsealed cid.Cid) error

	FinalizeReplicaUpdate(ctx context.Context, sector SectorRef) error

	DownloadSectorData(ctx context.Context, sector SectorRef, finalized bool, src map[SectorFileType]SectorLocation) error
}

type SecDataHttpHeader added in v1.17.2

type SecDataHttpHeader struct {
	Key   string
	Value string
}

note: we can't use http.Header as that's backed by a go map, which is all kinds of messy

func (*SecDataHttpHeader) MarshalCBOR added in v1.17.2

func (t *SecDataHttpHeader) MarshalCBOR(w io.Writer) error

func (*SecDataHttpHeader) UnmarshalCBOR added in v1.17.2

func (t *SecDataHttpHeader) UnmarshalCBOR(r io.Reader) (err error)

type SectorCids

type SectorCids struct {
	Unsealed cid.Cid
	Sealed   cid.Cid
}

type SectorFileType

type SectorFileType int

SectorFileType represents the type of a sector file TypeFromString converts a string to a SectorFileType

const (
	// "regular" sectors
	FTUnsealed SectorFileType = 1 << iota
	FTSealed
	FTCache

	// snap
	FTUpdate
	FTUpdateCache

	// Piece Park
	FTPiece

	FileTypes = iota
)

FTUnsealed represents an unsealed sector file. FTSealed represents a sealed sector file. FTCache represents a cache sector file. FTUpdate represents an update sector file. FTUpdateCache represents an update cache sector file. FTPiece represents a Piece Park sector file. FileTypes represents the total number of file.

The SectorFileType type is an integer type that represents different sector file. It has several methods to manipulate and query the file. The String method returns a string representation of the file. The Strings method returns a slice of string representations of all the file that are set in the receiver object. The AllSet method returns a slice of all the file that are set in the receiver object. The Has method checks whether a specific file type is set in the receiver object. The SealSpaceUse method calculates the space used by the receiver object in sealing a sector of a given size. The SubAllowed method removes selected file from the receiver object based on a list of allowed and denied file. The Unset method removes selected file from the receiver object. The AnyAllowed method checks whether any file in the receiver object are allowed based on a list of allowed and denied file. The Allowed method checks whether all file in the receiver object are allowed based on a list of allowed and denied file. The StoreSpaceUse method calculates the space used by the receiver object in storing a sector of a given size. The All method returns an array that represents which file are set in the receiver object. The IsNone method checks whether the receiver object represents no file.

const (
	FTNone SectorFileType = 0
)

FTNone represents a sector file type of none. This constant is used in the StorageLock method to specify that a sector should not have any file locked. Example usage: err := m.index.StorageLock(ctx, sector.ID, storiface.FTNone, storiface.FTSealed|storiface.FTUnsealed|storiface.FTCache)

func TypeFromString added in v1.17.1

func TypeFromString(s string) (SectorFileType, error)

TypeFromString converts a string representation of a SectorFileType to its corresponding value. It returns the SectorFileType and nil error if the string matches one of the existing types. If the string does not match any type, it returns 0 and an error.

func (SectorFileType) All

func (t SectorFileType) All() [FileTypes]bool

All returns an array indicating whether each FileTypes flag is set in the SectorFileType.

func (SectorFileType) AllSet added in v1.17.1

func (t SectorFileType) AllSet() []SectorFileType

AllSet returns a slice of SectorFileType values that are set in the SectorFileType receiver value

func (SectorFileType) Allowed added in v1.17.1

func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool

Allowed checks if the SectorFileType is allowed based on the given allowTypes and denyTypes. Returns true if the SectorFileType is allowed, otherwise false.

func (SectorFileType) AnyAllowed added in v1.17.1

func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool

AnyAllowed checks if the SectorFileType has any allowed types and no denied types.

func (SectorFileType) Has

func (t SectorFileType) Has(singleType SectorFileType) bool

Has checks if the SectorFileType has a specific singleType.

func (SectorFileType) IsNone added in v1.27.0

func (t SectorFileType) IsNone() bool

IsNone checks if the SectorFileType value is equal to zero. It returns true if the value is zero, indicating that the type is none.

func (SectorFileType) SealSpaceUse

func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error)

SealSpaceUse calculates the amount of space needed for sealing the sector based on the given sector size. It iterates over the different path types and calculates the space needed for each path type using the FSOverheadSeal map. The overhead value is multiplied by the sector size and divided by the FSOverheadDen constant. The total space needed is accumulated and returned. If there is no seal overhead information for a particular path type, an error is returned.

Example usage:

fileType := FTSealed | FTCache
sectorSize := abi.SectorSize(32 << 20) // 32 MiB
spaceNeeded, err := fileType.SealSpaceUse(sectorSize)

Parameters:

ssize: The size of the sector

Returns:

uint64: The amount of space needed for sealing the sector
error: If there is no seal overhead information for a path type

func (SectorFileType) StoreSpaceUse

func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error)

StoreSpaceUse calculates the space used for storing sectors of a specific file type. It takes the sector size as input and returns the total space needed in bytes and an error, if any. The calculation is based on the finalized overhead information for the file type. If the overhead information is not available for a particular file type, an error will be returned.

func (SectorFileType) String

func (t SectorFileType) String() string

String returns a string representation of the SectorFileType.

func (SectorFileType) Strings added in v1.17.1

func (t SectorFileType) Strings() []string

Strings returns a slice of strings representing the names of the SectorFileType values that are set in the receiver value. Example usage:

fileType := SectorFileType(FTSealed | FTCache)
names := fileType.Strings() // names = ["sealed", "cache"]
fmt.Println(names)

func (SectorFileType) SubAllowed added in v1.17.1

func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType

The method takes in two parameters: allowTypes and denyTypes, both of which are slices of strings. If allowTypes is not empty, the method sets a denyMask with all bits set to 1, and then iterates over each allowType, converting it to a SectorFileType using the TypeFromString function and unsetting the corresponding bit in the denyMask. If a string in allowTypes cannot be converted to a valid SectorFileType, it is ignored. After processing allowTypes, the method iterates over each denyType, converting it to a SectorFileType using the TypeFromString function and setting the corresponding bit in the denyMask. If a string in denyTypes cannot be converted to a valid SectorFileType, it is ignored. Finally, the method returns the bitwise AND of the original SectorFileType and the denyMask. The returned SectorFileType will only allow the types specified in allowTypes and exclude the types specified in denyTypes.`

func (SectorFileType) Unset added in v1.27.0

func (t SectorFileType) Unset(toUnset SectorFileType) SectorFileType

Unset removes the specified sector file type(s) from the current SectorFileType value. It performs a bitwise AND operation between the current value and the bitwise complement of the toUnset value. The result is returned as a new SectorFileType value. Any bits that are set in toUnset will be cleared in the result. Usage: result = value.Unset(typesToUnset)

type SectorLocation added in v1.17.2

type SectorLocation struct {
	// Local when set to true indicates to lotus that sector data is already
	// available locally; When set lotus will skip fetching sector data, and
	// only check that sector data exists in sector storage
	Local bool

	// URL to the sector data
	// For sealed/unsealed sector, lotus expects octet-stream
	// For cache, lotus expects a tar archive with cache files
	// Valid schemas:
	// - http:// / https://
	URL string

	// optional http headers to use when requesting sector data
	Headers []SecDataHttpHeader
}

func (*SectorLocation) HttpHeaders added in v1.17.2

func (sd *SectorLocation) HttpHeaders() http.Header

func (*SectorLocation) MarshalCBOR added in v1.17.2

func (t *SectorLocation) MarshalCBOR(w io.Writer) error

func (*SectorLocation) UnmarshalCBOR added in v1.17.2

func (t *SectorLocation) UnmarshalCBOR(r io.Reader) (err error)

type SectorLock

type SectorLock struct {
	Sector abi.SectorID
	Write  [FileTypes]uint
	Read   [FileTypes]uint
}

type SectorLocks

type SectorLocks struct {
	Locks []SectorLock
}

type SectorPaths

type SectorPaths struct {
	ID abi.SectorID

	Unsealed    string
	Sealed      string
	Cache       string
	Update      string
	UpdateCache string
	Piece       string
}

SectorPaths represents the paths for different sector files.

func (SectorPaths) HasAllSet added in v1.27.0

func (sp SectorPaths) HasAllSet(ft SectorFileType) bool

HasAllSet checks if all paths of a SectorPaths struct are set for a given SectorFileType.

func (SectorPaths) Subset added in v1.27.0

func (sp SectorPaths) Subset(filter SectorFileType) SectorPaths

Subset returns a new instance of SectorPaths that contains only the paths specified by the filter SectorFileType. It iterates over each fileType in the filter, retrieves the corresponding path from the original SectorPaths instance, and sets it in the new instance. Finally, it sets the ID field of the new instance to be the same as the original instance.

type SectorRef

type SectorRef struct {
	ID        abi.SectorID
	ProofType abi.RegisteredSealProof
}

type SectorStorageInfo

type SectorStorageInfo struct {
	ID       ID
	URLs     []string // TODO: Support non-http transports
	BaseURLs []string
	Weight   uint64

	CanSeal  bool
	CanStore bool

	Primary bool

	AllowTypes  []string
	DenyTypes   []string
	AllowMiners []string
	DenyMiners  []string
}

type Storage

type Storage interface {
	ProverPoSt
	Sealer
	Unsealer
}

type StorageConfig added in v1.19.0

type StorageConfig struct {
	StoragePaths []LocalPath
}

StorageConfig .lotusstorage/storage.json

type StorageInfo

type StorageInfo struct {
	// ID is the UUID of the storage path
	ID ID

	// URLs for remote access
	URLs []string // TODO: Support non-http transports

	// Storage path weight; higher number means that the path will be preferred more often
	Weight uint64

	// MaxStorage is the number of bytes allowed to be used by files in the
	// storage path
	MaxStorage uint64

	// CanStore is true when the path is allowed to be used for io-intensive
	// sealing operations
	CanSeal bool

	// CanStore is true when the path is allowed to be used for long-term storage
	CanStore bool

	// Groups is the list of path groups this path belongs to
	Groups []Group

	// AllowTo is the list of paths to which data from this path can be moved to
	AllowTo []Group

	// AllowTypes lists sector file types which are allowed to be put into this
	// path. If empty, all file types are allowed.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	AllowTypes []string

	// DenyTypes lists sector file types which aren't allowed to be put into this
	// path.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	DenyTypes []string

	// AllowMiners lists miner IDs which are allowed to store their sector data into
	// this path. If empty, all miner IDs are allowed
	AllowMiners []string

	// DenyMiners lists miner IDs which are denied to store their sector data into
	// this path
	DenyMiners []string
}

type StoragePath

type StoragePath struct {
	ID     ID
	Weight uint64

	LocalPath string

	CanSeal  bool
	CanStore bool
}

type UnpaddedByteIndex

type UnpaddedByteIndex uint64

func UnpaddedCeil added in v1.23.3

func UnpaddedCeil(n uint64) UnpaddedByteIndex

func UnpaddedFloor added in v1.23.3

func UnpaddedFloor(n uint64) UnpaddedByteIndex

func (UnpaddedByteIndex) Padded

func (UnpaddedByteIndex) Valid

func (i UnpaddedByteIndex) Valid() error

type Unsealer

type Unsealer interface {
	UnsealPiece(ctx context.Context, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error
	ReadPiece(ctx context.Context, writer io.Writer, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)
}

type Validator

type Validator interface {
	CanCommit(sector SectorPaths) (bool, error)
	CanProve(sector SectorPaths) (bool, error)
}

type Verifier

type Verifier interface {
	VerifySeal(proof.SealVerifyInfo) (bool, error)
	VerifyAggregateSeals(aggregate proof.AggregateSealVerifyProofAndInfos) (bool, error)
	VerifyReplicaUpdate(update proof.ReplicaUpdateInfo) (bool, error)
	VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error)
	VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error)

	GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error)
}

type WindowPoStResult

type WindowPoStResult struct {
	PoStProofs proof.PoStProof
	Skipped    []abi.SectorID
}

type WorkError added in v1.25.0

type WorkError interface {
	ErrCode() ErrorCode
}

type WorkerCalls

type WorkerCalls interface {
	// async
	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData Data) (CallID, error)
	AddPiece(ctx context.Context, sector SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData Data) (CallID, error)
	SealPreCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (CallID, error)
	SealPreCommit2(ctx context.Context, sector SectorRef, pc1o PreCommit1Out) (CallID, error)
	SealCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids SectorCids) (CallID, error)
	SealCommit2(ctx context.Context, sector SectorRef, c1o Commit1Out) (CallID, error)
	FinalizeSector(ctx context.Context, sector SectorRef) (CallID, error)
	FinalizeReplicaUpdate(ctx context.Context, sector SectorRef) (CallID, error)
	ReleaseUnsealed(ctx context.Context, sector SectorRef, safeToFree []Range) (CallID, error)
	ReplicaUpdate(ctx context.Context, sector SectorRef, pieces []abi.PieceInfo) (CallID, error)
	ProveReplicaUpdate1(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (CallID, error)
	ProveReplicaUpdate2(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs ReplicaVanillaProofs) (CallID, error)
	GenerateSectorKeyFromData(ctx context.Context, sector SectorRef, commD cid.Cid) (CallID, error)
	MoveStorage(ctx context.Context, sector SectorRef, types SectorFileType) (CallID, error)
	UnsealPiece(context.Context, SectorRef, UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (CallID, error)
	Fetch(context.Context, SectorRef, SectorFileType, PathType, AcquireMode) (CallID, error)
	DownloadSectorData(ctx context.Context, sector SectorRef, finalized bool, src map[SectorFileType]SectorLocation) (CallID, error)

	// sync
	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)
	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (WindowPoStResult, error)
}

type WorkerID

type WorkerID uuid.UUID // worker session UUID

func (WorkerID) String

func (w WorkerID) String() string

type WorkerInfo

type WorkerInfo struct {
	Hostname string

	// IgnoreResources indicates whether the worker's available resources should
	// be used ignored (true) or used (false) for the purposes of scheduling and
	// task assignment. Only supported on local workers. Used for testing.
	// Default should be false (zero value, i.e. resources taken into account).
	IgnoreResources bool
	Resources       WorkerResources
}

type WorkerJob

type WorkerJob struct {
	ID     CallID
	Sector abi.SectorID
	Task   sealtasks.TaskType

	// 2+ - assigned
	// 1  - prepared
	// 0  - running
	// -1 - ret-wait
	// -2 - returned
	// -3 - ret-done
	RunWait int
	Start   time.Time

	Hostname string `json:",omitempty"` // optional, set for ret-wait jobs
}

type WorkerResources

type WorkerResources struct {
	MemPhysical uint64
	MemUsed     uint64
	MemSwap     uint64
	MemSwapUsed uint64

	CPUs uint64 // Logical cores
	GPUs []string

	// if nil use the default resource table
	Resources map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
}

func (WorkerResources) PrepResourceSpec added in v1.23.0

func (wr WorkerResources) PrepResourceSpec(spt abi.RegisteredSealProof, tt, prepTT sealtasks.TaskType) Resources

PrepResourceSpec is like ResourceSpec, but meant for use limiting parallel preparing tasks.

func (WorkerResources) ResourceSpec

type WorkerReturn

type WorkerReturn interface {
	ReturnDataCid(ctx context.Context, callID CallID, pi abi.PieceInfo, err *CallError) error
	ReturnAddPiece(ctx context.Context, callID CallID, pi abi.PieceInfo, err *CallError) error
	ReturnSealPreCommit1(ctx context.Context, callID CallID, p1o PreCommit1Out, err *CallError) error
	ReturnSealPreCommit2(ctx context.Context, callID CallID, sealed SectorCids, err *CallError) error
	ReturnSealCommit1(ctx context.Context, callID CallID, out Commit1Out, err *CallError) error
	ReturnSealCommit2(ctx context.Context, callID CallID, proof Proof, err *CallError) error
	ReturnFinalizeSector(ctx context.Context, callID CallID, err *CallError) error
	ReturnReleaseUnsealed(ctx context.Context, callID CallID, err *CallError) error
	ReturnReplicaUpdate(ctx context.Context, callID CallID, out ReplicaUpdateOut, err *CallError) error
	ReturnProveReplicaUpdate1(ctx context.Context, callID CallID, proofs ReplicaVanillaProofs, err *CallError) error
	ReturnProveReplicaUpdate2(ctx context.Context, callID CallID, proof ReplicaUpdateProof, err *CallError) error
	ReturnGenerateSectorKeyFromData(ctx context.Context, callID CallID, err *CallError) error
	ReturnFinalizeReplicaUpdate(ctx context.Context, callID CallID, err *CallError) error
	ReturnMoveStorage(ctx context.Context, callID CallID, err *CallError) error
	ReturnUnsealPiece(ctx context.Context, callID CallID, err *CallError) error
	ReturnReadPiece(ctx context.Context, callID CallID, ok bool, err *CallError) error
	ReturnDownloadSector(ctx context.Context, callID CallID, err *CallError) error
	ReturnFetch(ctx context.Context, callID CallID, err *CallError) error
}

type WorkerStats

type WorkerStats struct {
	Info    WorkerInfo
	Tasks   []sealtasks.TaskType
	Enabled bool

	MemUsedMin uint64
	MemUsedMax uint64
	GpuUsed    float64 // nolint
	CpuUse     uint64  // nolint

	TaskCounts map[string]int
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL