storage

Documentation

Index

• func ParallelCleanStorage(ctx context.Context, provider StorageProvider, storages []PreparedStorage) error
• type FileSystemNodeType
  • func (i FileSystemNodeType) String() string
• type PreparedStorage
  • func ParallelPrepareStorage(ctx context.Context, provider StorageProvider, storageDirectory string, ...) ([]PreparedStorage, error)
• type Storage
• type StorageProvider
• type StorageVolume
• type StorageVolumeConnectorType
  • func (i StorageVolumeConnectorType) String() string

Functions

func ParallelCleanStorage

func ParallelCleanStorage(
	ctx context.Context,
	provider StorageProvider,
	storages []PreparedStorage,
) error

Types

type FileSystemNodeType

type FileSystemNodeType int

Used to distinguish files from directories

const (
	FileSystemNodeDirectory FileSystemNodeType
	FileSystemNodeFile
)

func (FileSystemNodeType) String

func (i FileSystemNodeType) String() string

type PreparedStorage

type PreparedStorage struct {
	InputSource models.InputSource
	Volume      StorageVolume
}

func ParallelPrepareStorage

func ParallelPrepareStorage(
	ctx context.Context,
	provider StorageProvider,
	storageDirectory string,
	specs ...*models.InputSource,
) ([]PreparedStorage, error)

ParallelPrepareStorage downloads all of the data necessary for the passed storage specs in parallel, and returns a map of specs to their download volume counterparts.

type Storage

type Storage interface {
	provider.Providable

	HasStorageLocally(context.Context, models.InputSource) (bool, error)

	// how big is the given volume in terms of resource consumption?
	GetVolumeSize(context.Context, models.InputSource) (uint64, error)

	// PrepareStorage is provided an output directory, and an InputSource and
	// is expected to retrieve the InputSource into the output directory.
	PrepareStorage(context.Context, string, models.InputSource) (StorageVolume, error)

	CleanupStorage(context.Context, models.InputSource, StorageVolume) error

	// given a local file path - "store" it and return a SpecConfig
	Upload(context.Context, string) (models.SpecConfig, error)
}

type StorageProvider

type StorageProvider = provider.Provider[Storage]

StorageProvider returns a storage that can be used by the job to store data.

type StorageVolume

type StorageVolume struct {
	Type     StorageVolumeConnectorType `json:"type"`
	ReadOnly bool                       `json:"readOnly"`
	Source   string                     `json:"source"`
	Target   string                     `json:"target"`
}

a storage entity that is consumed are produced by a job input storage specs are turned into storage volumes by drivers for example - the input storage spec might be ipfs cid XXX and a driver will turn that into a host path that can be consumed by a job another example - a wasm storage driver references the upstream ipfs cid (source) that can be streamed via a library call using the target name put simply - the nature of a storage volume depends on it's use by the executor engine

type StorageVolumeConnectorType

type StorageVolumeConnectorType int

StorageVolumeConnector is how an upstream storage source will present the volume to a job - examples are "bind" or "library"

const (
	StorageVolumeConnectorBind StorageVolumeConnectorType
)

func (StorageVolumeConnectorType) String

func (i StorageVolumeConnectorType) String() string