storiface

package
v0.0.0-...-484cccf Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 16, 2023 License: Apache-2.0, MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	RWPrepared = 1
	RWRunning  = 0
	RWRetWait  = -1
	RWReturned = -2
	RWRetDone  = -3
)
View Source 
const FSOverheadDen = 10
View Source 
const IDSep = "."

Variables

View Source 
var ErrSectorNotFound = errors.New("sector not found")
View Source 
var FSOverheadSeal = map[SectorFileType]int{
	FTUnsealed:    FSOverheadDen,
	FTSealed:      FSOverheadDen,
	FTUpdate:      FSOverheadDen,
	FTUpdateCache: FSOverheadDen*2 + 1,
	FTCache:       141,
}
View Source 
var FTAll = func() (out SectorFileType) {
	for _, pathType := range PathTypes {
		out |= pathType
	}
	return out
}()
View Source 
var FsOverheadFinalized = map[SectorFileType]int{
	FTUnsealed:    FSOverheadDen,
	FTSealed:      FSOverheadDen,
	FTUpdate:      FSOverheadDen,
	FTUpdateCache: 1,
	FTCache:       1,
}
View Source 
var NoSectorRef = SectorRef{}
View Source 
var ParallelDenom uint64 = 100
View Source 
var ParallelNum uint64 = 92

Percent of threads to allocate to parallel tasks

12 * 0.92 = 11 16 * 0.92 = 14 24 * 0.92 = 22 32 * 0.92 = 29 64 * 0.92 = 58 128 * 0.92 = 117

View Source 
var PathTypes = []SectorFileType{FTUnsealed, FTSealed, FTCache, FTUpdate, FTUpdateCache}
View Source 
var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
	sealtasks.TTAddPiece: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 8 << 30,
			MinMemory: 8 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 4 << 30,
			MinMemory: 4 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTPreCommit1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 128 << 30,
			MinMemory: 112 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 10 << 20,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 64 << 30,
			MinMemory: 56 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 10 << 20,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 768 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 20,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTPreCommit2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 30 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 15 << 30,
			MinMemory: 15 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: -1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: -1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: -1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTCommit1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTCommit2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 190 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 150 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTFetch: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 1 << 20,
			MinMemory: 1 << 20,

			MaxParallelism: 0,
			GPUUtilization: 0,

			BaseMinMemory: 0,
		},
	},

	sealtasks.TTReplicaUpdate: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 8 << 30,
			MinMemory: 8 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 4 << 30,
			MinMemory: 4 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 1,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTProveReplicaUpdate1: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism: 0,

			BaseMinMemory: 1 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTProveReplicaUpdate2: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 190 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 150 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTGenerateWindowPoSt: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 120 << 30,
			MinMemory: 60 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 96 << 30,
			MinMemory: 30 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 3 << 29,
			MinMemory: 1 << 30,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
	sealtasks.TTGenerateWinningPoSt: {
		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 64 << 30,
		},
		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
			MaxMemory: 1 << 30,
			MinMemory: 1 << 30,

			MaxParallelism:    -1,
			MaxParallelismGPU: 6,
			GPUUtilization:    1.0,

			BaseMinMemory: 32 << 30,
		},
		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 10 << 30,
		},
		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
			MaxMemory: 2 << 10,
			MinMemory: 2 << 10,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 2 << 10,
		},
		abi.RegisteredSealProof_StackedDrg8MiBV1: Resources{
			MaxMemory: 8 << 20,
			MinMemory: 8 << 20,

			MaxParallelism: 1,
			GPUUtilization: 1.0,

			BaseMinMemory: 8 << 20,
		},
	},
}

Functions

func ParseResourceEnv 

func ParseResourceEnv(lookup func(key, def string) (string, bool)) (map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources, error)

func ParseSectorID 

func ParseSectorID(baseName string) (abi.SectorID, error)

func PathByType 

func PathByType(sps SectorPaths, fileType SectorFileType) string

func SectorName 

func SectorName(sid abi.SectorID) string

func SetPathByType 

func SetPathByType(sps *SectorPaths, fileType SectorFileType, p string)

Types

type AcquireMode 

type AcquireMode string
const (
	AcquireMove AcquireMode = "move"
	AcquireCopy AcquireMode = "copy"
)

type CallError 

type CallError struct {
	Code    ErrorCode
	Message string
	// contains filtered or unexported fields
}

func Err 

func Err(code ErrorCode, sub error) *CallError

func (*CallError) Error 

func (c *CallError) Error() string

func (*CallError) Unwrap 

func (c *CallError) Unwrap() error

type CallID 

type CallID struct {
	Sector abi.SectorID
	ID     uuid.UUID
}
var UndefCall CallID

func (*CallID) MarshalCBOR 

func (t *CallID) MarshalCBOR(w io.Writer) error

func (CallID) String 

func (c CallID) String() string

func (*CallID) UnmarshalCBOR 

func (t *CallID) UnmarshalCBOR(r io.Reader) (err error)

type Commit1Out 

type Commit1Out []byte

type Data 

type Data = io.Reader

type Decl 

type Decl struct {
	abi.SectorID
	SectorFileType
}

type ErrorCode 

type ErrorCode int
const (
	// Temp Errors
	ErrTempUnknown ErrorCode = iota + 100
	ErrTempWorkerRestart
	ErrTempAllocateSpace
)
const (
	ErrUnknown ErrorCode = iota
)

type FallbackChallenges 

type FallbackChallenges struct {
	Sectors    []abi.SectorNumber
	Challenges map[abi.SectorNumber][]uint64
}

type Group 

type Group = string

type HealthReport 

type HealthReport struct {
	Stat fsutil.FsStat
	Err  string
}

type ID 

type ID string

ID identifies sector storage by UUID. One sector storage should map to one 

filesystem, local or networked / shared by multiple machines

type IDList 

type IDList []ID

func ParseIDList 

func ParseIDList(s string) IDList

func (IDList) String 

func (il IDList) String() string

type LocalPath 

type LocalPath struct {
	Path string
}

type LocalStorageMeta 

type LocalStorageMeta struct {
	ID ID

	// A high weight means data is more likely to be stored in this path
	Weight uint64 // 0 = readonly

	// Intermediate data for the sealing process will be stored here
	CanSeal bool

	// Finalized sectors that will be proved over time will be stored here
	CanStore bool

	// MaxStorage specifies the maximum number of bytes to use for sector storage
	// (0 = unlimited)
	MaxStorage uint64

	// List of storage groups this path belongs to
	Groups []string

	// List of storage groups to which data from this path can be moved. If none
	// are specified, allow to all
	AllowTo []string

	// AllowTypes lists sector file types which are allowed to be put into this
	// path. If empty, all file types are allowed.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	AllowTypes []string

	// DenyTypes lists sector file types which aren't allowed to be put into this
	// path.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	DenyTypes []string
}

LocalStorageMeta path/sectorstore.json

type PaddedByteIndex 

type PaddedByteIndex uint64

type PathType 

type PathType string
const (
	PathStorage PathType = "storage"
	PathSealing PathType = "sealing"
)

type PostSectorChallenge 

type PostSectorChallenge struct {
	SealProof    abi.RegisteredSealProof
	SectorNumber abi.SectorNumber
	SealedCID    cid.Cid
	Challenge    []uint64
	Update       bool
}

type PreCommit1Out 

type PreCommit1Out []byte

type Proof 

type Proof []byte

type Prover 

type Prover interface {
	AggregateSealProofs(aggregateInfo proof.AggregateSealVerifyProofAndInfos, proofs [][]byte) ([]byte, error)
}

Prover contains cheap proving-related methods

type ProverPoSt 

type ProverPoSt interface {
	GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error)
	GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) (proof []proof.PoStProof, skipped []abi.SectorID, err error)

	GenerateWinningPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte) ([]proof.PoStProof, error)
	GenerateWindowPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte, partitionIdx int) (proof.PoStProof, error)
}

type RGetter 

type RGetter func(ctx context.Context, id abi.SectorID) (sealed cid.Cid, update bool, err error)

type Range 

type Range struct {
	Offset abi.UnpaddedPieceSize
	Size   abi.UnpaddedPieceSize
}

type Refs 

type Refs struct {
	RefCount [FileTypes]uint
}

type ReplicaUpdateOut 

type ReplicaUpdateOut struct {
	NewSealed   cid.Cid
	NewUnsealed cid.Cid
}

type ReplicaUpdateProof 

type ReplicaUpdateProof []byte

type ReplicaVanillaProofs 

type ReplicaVanillaProofs [][]byte

type Resources 

type Resources struct {
	MinMemory uint64 `envname:"MIN_MEMORY"` // What Must be in RAM for decent perf
	MaxMemory uint64 `envname:"MAX_MEMORY"` // Memory required (swap + ram; peak memory usage during task execution)

	// GPUUtilization specifes the number of GPUs a task can use
	GPUUtilization float64 `envname:"GPU_UTILIZATION"`

	// MaxParallelism specifies the number of CPU cores when GPU is NOT in use
	MaxParallelism int `envname:"MAX_PARALLELISM"` // -1 = multithread

	// MaxParallelismGPU specifies the number of CPU cores when GPU is in use
	MaxParallelismGPU int `envname:"MAX_PARALLELISM_GPU"` // when 0, inherits MaxParallelism

	BaseMinMemory uint64 `envname:"BASE_MIN_MEMORY"` // What Must be in RAM for decent perf (shared between threads)

	MaxConcurrent int `envname:"MAX_CONCURRENT"` // Maximum number of tasks of this type that can be scheduled on a worker (0=default, no limit)
}

func (Resources) Threads 

func (r Resources) Threads(wcpus uint64, gpus int) uint64

TODO: Take NUMA into account

type Sealer 

type Sealer interface {
	NewSector(ctx context.Context, sector SectorRef) error
	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)
	AddPiece(ctx context.Context, sector SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)

	SealPreCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (PreCommit1Out, error)
	SealPreCommit2(ctx context.Context, sector SectorRef, pc1o PreCommit1Out) (SectorCids, error)

	SealCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids SectorCids) (Commit1Out, error)
	SealCommit2(ctx context.Context, sector SectorRef, c1o Commit1Out) (Proof, error)

	FinalizeSector(ctx context.Context, sector SectorRef) error

	// ReleaseUnsealed marks parts of the unsealed sector file as safe to drop
	//  (called by the fsm on restart, allows storage to keep no persistent
	//   state about unsealed fast-retrieval copies)
	ReleaseUnsealed(ctx context.Context, sector SectorRef, keepUnsealed []Range) error
	// ReleaseSectorKey removes `sealed` from all storage
	// called after successful sector upgrade
	ReleaseSectorKey(ctx context.Context, sector SectorRef) error
	// ReleaseReplicaUpgrade removes `update` / `update-cache` from all storage
	// called when aborting sector upgrade
	ReleaseReplicaUpgrade(ctx context.Context, sector SectorRef) error

	// Removes all data associated with the specified sector
	Remove(ctx context.Context, sector SectorRef) error

	// Generate snap deals replica update
	ReplicaUpdate(ctx context.Context, sector SectorRef, pieces []abi.PieceInfo) (ReplicaUpdateOut, error)

	// Prove that snap deals replica was done correctly
	ProveReplicaUpdate1(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (ReplicaVanillaProofs, error)
	ProveReplicaUpdate2(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs ReplicaVanillaProofs) (ReplicaUpdateProof, error)

	// GenerateSectorKeyFromData computes sector key given unsealed data and updated replica
	GenerateSectorKeyFromData(ctx context.Context, sector SectorRef, unsealed cid.Cid) error

	FinalizeReplicaUpdate(ctx context.Context, sector SectorRef) error

	DownloadSectorData(ctx context.Context, sector SectorRef, finalized bool, src map[SectorFileType]SectorLocation) error
}

type SecDataHttpHeader 

type SecDataHttpHeader struct {
	Key   string
	Value string
}

note: we can't use http.Header as that's backed by a go map, which is all kinds of messy

func (*SecDataHttpHeader) MarshalCBOR 

func (t *SecDataHttpHeader) MarshalCBOR(w io.Writer) error

func (*SecDataHttpHeader) UnmarshalCBOR 

func (t *SecDataHttpHeader) UnmarshalCBOR(r io.Reader) (err error)

type SectorCids 

type SectorCids struct {
	Unsealed cid.Cid
	Sealed   cid.Cid
}

type SectorFileType 

type SectorFileType int
const (
	FTUnsealed SectorFileType = 1 << iota
	FTSealed
	FTCache
	FTUpdate
	FTUpdateCache

	FileTypes = iota
)
const (
	FTNone SectorFileType = 0
)

func TypeFromString 

func TypeFromString(s string) (SectorFileType, error)

func (SectorFileType) All 

func (t SectorFileType) All() [FileTypes]bool

func (SectorFileType) AllSet 

func (t SectorFileType) AllSet() []SectorFileType

func (SectorFileType) Allowed 

func (t SectorFileType) Allowed(allowTypes []string, denyTypes []string) bool

func (SectorFileType) AnyAllowed 

func (t SectorFileType) AnyAllowed(allowTypes []string, denyTypes []string) bool

func (SectorFileType) Has 

func (t SectorFileType) Has(singleType SectorFileType) bool

func (SectorFileType) SealSpaceUse 

func (t SectorFileType) SealSpaceUse(ssize abi.SectorSize) (uint64, error)

func (SectorFileType) StoreSpaceUse 

func (t SectorFileType) StoreSpaceUse(ssize abi.SectorSize) (uint64, error)

func (SectorFileType) String 

func (t SectorFileType) String() string

func (SectorFileType) Strings 

func (t SectorFileType) Strings() []string

func (SectorFileType) SubAllowed 

func (t SectorFileType) SubAllowed(allowTypes []string, denyTypes []string) SectorFileType

type SectorLocation 

type SectorLocation struct {
	// Local when set to true indicates to lotus that sector data is already
	// available locally; When set lotus will skip fetching sector data, and
	// only check that sector data exists in sector storage
	Local bool

	// URL to the sector data
	// For sealed/unsealed sector, lotus expects octet-stream
	// For cache, lotus expects a tar archive with cache files
	// Valid schemas:
	// - http:// / https://
	URL string

	// optional http headers to use when requesting sector data
	Headers []SecDataHttpHeader
}

func (*SectorLocation) HttpHeaders 

func (sd *SectorLocation) HttpHeaders() http.Header

func (*SectorLocation) MarshalCBOR 

func (t *SectorLocation) MarshalCBOR(w io.Writer) error

func (*SectorLocation) UnmarshalCBOR 

func (t *SectorLocation) UnmarshalCBOR(r io.Reader) (err error)

type SectorLock 

type SectorLock struct {
	Sector abi.SectorID
	Write  [FileTypes]uint
	Read   [FileTypes]uint
}

type SectorLocks 

type SectorLocks struct {
	Locks []SectorLock
}

type SectorPaths 

type SectorPaths struct {
	ID abi.SectorID

	Unsealed    string
	Sealed      string
	Cache       string
	Update      string
	UpdateCache string
}

type SectorRef 

type SectorRef struct {
	ID        abi.SectorID
	ProofType abi.RegisteredSealProof
}

type SectorStorageInfo 

type SectorStorageInfo struct {
	ID       ID
	URLs     []string // TODO: Support non-http transports
	BaseURLs []string
	Weight   uint64

	CanSeal  bool
	CanStore bool

	Primary bool

	AllowTypes []string
	DenyTypes  []string
}

type Storage 

type Storage interface {
	ProverPoSt
	Sealer
	Unsealer
}

type StorageConfig 

type StorageConfig struct {
	StoragePaths []LocalPath
}

StorageConfig .lotusstorage/storage.json

type StorageInfo 

type StorageInfo struct {
	// ID is the UUID of the storage path
	ID ID

	// URLs for remote access
	URLs []string // TODO: Support non-http transports

	// Storage path weight; higher number means that the path will be preferred more often
	Weight uint64

	// MaxStorage is the number of bytes allowed to be used by files in the
	// storage path
	MaxStorage uint64

	// CanStore is true when the path is allowed to be used for io-intensive
	// sealing operations
	CanSeal bool

	// CanStore is true when the path is allowed to be used for long-term storage
	CanStore bool

	// Groups is the list of path groups this path belongs to
	Groups []Group

	// AllowTo is the list of paths to which data from this path can be moved to
	AllowTo []Group

	// AllowTypes lists sector file types which are allowed to be put into this
	// path. If empty, all file types are allowed.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	AllowTypes []string

	// DenyTypes lists sector file types which aren't allowed to be put into this
	// path.
	//
	// Valid values:
	// - "unsealed"
	// - "sealed"
	// - "cache"
	// - "update"
	// - "update-cache"
	// Any other value will generate a warning and be ignored.
	DenyTypes []string
}

type StoragePath 

type StoragePath struct {
	ID     ID
	Weight uint64

	LocalPath string

	CanSeal  bool
	CanStore bool
}

type UnpaddedByteIndex 

type UnpaddedByteIndex uint64

func (UnpaddedByteIndex) Padded 

func (i UnpaddedByteIndex) Padded() PaddedByteIndex

func (UnpaddedByteIndex) Valid 

func (i UnpaddedByteIndex) Valid() error

type Unsealer 

type Unsealer interface {
	UnsealPiece(ctx context.Context, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error
	ReadPiece(ctx context.Context, writer io.Writer, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)
}

type Validator 

type Validator interface {
	CanCommit(sector SectorPaths) (bool, error)
	CanProve(sector SectorPaths) (bool, error)
}

type Verifier 

type Verifier interface {
	VerifySeal(proof.SealVerifyInfo) (bool, error)
	VerifyAggregateSeals(aggregate proof.AggregateSealVerifyProofAndInfos) (bool, error)
	VerifyReplicaUpdate(update proof.ReplicaUpdateInfo) (bool, error)
	VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error)
	VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error)

	GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error)
}

type WindowPoStResult 

type WindowPoStResult struct {
	PoStProofs proof.PoStProof
	Skipped    []abi.SectorID
}

type WorkerCalls 

type WorkerCalls interface {
	// async
	DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData Data) (CallID, error)
	AddPiece(ctx context.Context, sector SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData Data) (CallID, error)
	SealPreCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (CallID, error)
	SealPreCommit2(ctx context.Context, sector SectorRef, pc1o PreCommit1Out) (CallID, error)
	SealCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids SectorCids) (CallID, error)
	SealCommit2(ctx context.Context, sector SectorRef, c1o Commit1Out) (CallID, error)
	FinalizeSector(ctx context.Context, sector SectorRef) (CallID, error)
	FinalizeReplicaUpdate(ctx context.Context, sector SectorRef) (CallID, error)
	ReleaseUnsealed(ctx context.Context, sector SectorRef, safeToFree []Range) (CallID, error)
	ReplicaUpdate(ctx context.Context, sector SectorRef, pieces []abi.PieceInfo) (CallID, error)
	ProveReplicaUpdate1(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (CallID, error)
	ProveReplicaUpdate2(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs ReplicaVanillaProofs) (CallID, error)
	GenerateSectorKeyFromData(ctx context.Context, sector SectorRef, commD cid.Cid) (CallID, error)
	MoveStorage(ctx context.Context, sector SectorRef, types SectorFileType) (CallID, error)
	UnsealPiece(context.Context, SectorRef, UnpaddedByteIndex, abi.UnpaddedPieceSize, abi.SealRandomness, cid.Cid) (CallID, error)
	Fetch(context.Context, SectorRef, SectorFileType, PathType, AcquireMode) (CallID, error)
	DownloadSectorData(ctx context.Context, sector SectorRef, finalized bool, src map[SectorFileType]SectorLocation) (CallID, error)

	// sync
	GenerateWinningPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []PostSectorChallenge, randomness abi.PoStRandomness) ([]proof.PoStProof, error)
	GenerateWindowPoSt(ctx context.Context, ppt abi.RegisteredPoStProof, mid abi.ActorID, sectors []PostSectorChallenge, partitionIdx int, randomness abi.PoStRandomness) (WindowPoStResult, error)
}

type WorkerID 

type WorkerID uuid.UUID // worker session UUID

func (WorkerID) String 

func (w WorkerID) String() string

type WorkerInfo 

type WorkerInfo struct {
	Hostname string

	// IgnoreResources indicates whether the worker's available resources should
	// be used ignored (true) or used (false) for the purposes of scheduling and
	// task assignment. Only supported on local workers. Used for testing.
	// Default should be false (zero value, i.e. resources taken into account).
	IgnoreResources bool
	Resources       WorkerResources
}

type WorkerJob 

type WorkerJob struct {
	ID     CallID
	Sector abi.SectorID
	Task   sealtasks.TaskType

	// 2+ - assigned
	// 1  - prepared
	// 0  - running
	// -1 - ret-wait
	// -2 - returned
	// -3 - ret-done
	RunWait int
	Start   time.Time

	Hostname string `json:",omitempty"` // optional, set for ret-wait jobs
}

type WorkerResources 

type WorkerResources struct {
	MemPhysical uint64
	MemUsed     uint64
	MemSwap     uint64
	MemSwapUsed uint64

	CPUs uint64 // Logical cores
	GPUs []string

	// if nil use the default resource table
	Resources map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
}

func (WorkerResources) PrepResourceSpec 

func (wr WorkerResources) PrepResourceSpec(spt abi.RegisteredSealProof, tt, prepTT sealtasks.TaskType) Resources

PrepResourceSpec is like ResourceSpec, but meant for use limiting parallel preparing tasks.

func (WorkerResources) ResourceSpec 

func (wr WorkerResources) ResourceSpec(spt abi.RegisteredSealProof, tt sealtasks.TaskType) Resources

type WorkerReturn 

type WorkerReturn interface {
	ReturnDataCid(ctx context.Context, callID CallID, pi abi.PieceInfo, err *CallError) error
	ReturnAddPiece(ctx context.Context, callID CallID, pi abi.PieceInfo, err *CallError) error
	ReturnSealPreCommit1(ctx context.Context, callID CallID, p1o PreCommit1Out, err *CallError) error
	ReturnSealPreCommit2(ctx context.Context, callID CallID, sealed SectorCids, err *CallError) error
	ReturnSealCommit1(ctx context.Context, callID CallID, out Commit1Out, err *CallError) error
	ReturnSealCommit2(ctx context.Context, callID CallID, proof Proof, err *CallError) error
	ReturnFinalizeSector(ctx context.Context, callID CallID, err *CallError) error
	ReturnReleaseUnsealed(ctx context.Context, callID CallID, err *CallError) error
	ReturnReplicaUpdate(ctx context.Context, callID CallID, out ReplicaUpdateOut, err *CallError) error
	ReturnProveReplicaUpdate1(ctx context.Context, callID CallID, proofs ReplicaVanillaProofs, err *CallError) error
	ReturnProveReplicaUpdate2(ctx context.Context, callID CallID, proof ReplicaUpdateProof, err *CallError) error
	ReturnGenerateSectorKeyFromData(ctx context.Context, callID CallID, err *CallError) error
	ReturnFinalizeReplicaUpdate(ctx context.Context, callID CallID, err *CallError) error
	ReturnMoveStorage(ctx context.Context, callID CallID, err *CallError) error
	ReturnUnsealPiece(ctx context.Context, callID CallID, err *CallError) error
	ReturnReadPiece(ctx context.Context, callID CallID, ok bool, err *CallError) error
	ReturnDownloadSector(ctx context.Context, callID CallID, err *CallError) error
	ReturnFetch(ctx context.Context, callID CallID, err *CallError) error
}

type WorkerStats 

type WorkerStats struct {
	Info    WorkerInfo
	Tasks   []sealtasks.TaskType
	Enabled bool

	MemUsedMin uint64
	MemUsedMax uint64
	GpuUsed    float64 // nolint
	CpuUse     uint64  // nolint

	TaskCounts map[string]int
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.