bacalhau

package
v0.0.0-...-3ac8fd0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 27, 2024 License: Apache-2.0 Imports: 8 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func EngineNames 

func EngineNames() []string

func IsValidEngine 

func IsValidEngine(e Engine) bool

func IsValidPublisher 

func IsValidPublisher(publisherType Publisher) bool

func IsValidStorageSourceType 

func IsValidStorageSourceType(sourceType StorageSourceType) bool

func PublisherNames 

func PublisherNames() []string

func StorageSourceNames 

func StorageSourceNames() []string

Types

type Deal 

type Deal struct {
	// Whether the job should be run on any matching node (false) or all
	// matching nodes (true). If true, other fields in this struct are ignored.
	TargetingMode TargetingMode `json:"TargetingMode,omitempty"`
	// The maximum number of concurrent compute node bids that will be
	// accepted by the requester node on behalf of the client.
	Concurrency int `json:"Concurrency,omitempty"`
}

type Engine 

type Engine int
const (
	EngineNoop Engine
	EngineDocker
	EngineWasm
)

func EngineTypes 

func EngineTypes() []Engine

func ParseEngine 

func ParseEngine(str string) Engine

ParseEngine will either return a valid engine type or `engineUnknown`

func (Engine) MarshalText 

func (e Engine) MarshalText() ([]byte, error)

func (Engine) String 

func (i Engine) String() string

func (*Engine) UnmarshalText 

func (e *Engine) UnmarshalText(text []byte) (err error)

type EngineSpec 

type EngineSpec struct {
	Type   string
	Params map[string]interface{}
}

type ExecutionDesiredState 

type ExecutionDesiredState int
const (
	ExecutionDesiredStatePending ExecutionDesiredState = iota
	ExecutionDesiredStateRunning
	ExecutionDesiredStateStopped
)

type ExecutionID 

type ExecutionID struct {
	JobID       string `json:"JobID,omitempty"`
	NodeID      string `json:"NodeID,omitempty"`
	ExecutionID string `json:"ExecutionID,omitempty"`
}

ExecutionID a globally unique identifier for an execution

func (ExecutionID) String 

func (e ExecutionID) String() string

String returns a string representation of the execution id

type ExecutionState 

type ExecutionState struct {
	// JobID the job id
	JobID string `json:"JobID"`
	// which node is running this execution
	NodeID string `json:"NodeId"`
	// Compute node reference for this job execution
	ComputeReference string `json:"ComputeReference"`
	// State is the current state of the execution
	State ExecutionStateType `json:"State"`
	// an arbitrary status message
	Status string `json:"Status,omitempty"`
	// DesiredState is the desired state of the execution
	DesiredState ExecutionDesiredState `json:"DesiredState,omitempty"`
	// the published results for this execution
	PublishedResult StorageSpec `json:"PublishedResults,omitempty"`

	// RunOutput of the job
	RunOutput *RunCommandResult `json:"RunOutput,omitempty"`
	// Version is the version of the job state. It is incremented every time the job state is updated.
	Version int `json:"Version"`
	// CreateTime is the time when the job was created.
	CreateTime time.Time `json:"CreateTime"`
	// UpdateTime is the time when the job state was last updated.
	UpdateTime time.Time `json:"UpdateTime"`
}

func (ExecutionState) ID 

func (e ExecutionState) ID() ExecutionID

ID returns the ID for this execution

func (ExecutionState) String 

func (e ExecutionState) String() string

String returns a string representation of the execution

type ExecutionStateType 

type ExecutionStateType int

ExecutionStateType The state of an execution. An execution represents a single attempt to execute a job on a node. A compute node can have multiple executions for the same job due to retries, but there can only be a single active execution per node at any given time. 

const (
	ExecutionStateUndefined ExecutionStateType = iota
	// ExecutionStateNew The execution has been created, but not pushed to a compute node yet.
	ExecutionStateNew ExecutionStateType = iota
	// ExecutionStateAskForBid A node has been selected to execute a job, and is being asked to bid on the job.
	ExecutionStateAskForBid
	// ExecutionStateAskForBidAccepted compute node has rejected the ask for bid.
	ExecutionStateAskForBidAccepted
	// ExecutionStateAskForBidRejected compute node has rejected the ask for bid.
	ExecutionStateAskForBidRejected
	// ExecutionStateBidAccepted requester has accepted the bid, and the execution is expected to be running on the compute node.
	ExecutionStateBidAccepted // aka running
	// ExecutionStateBidRejected requester has rejected the bid.
	ExecutionStateBidRejected
	// ExecutionStateCompleted The execution has been completed, and the result has been published.
	ExecutionStateCompleted
	// ExecutionStateFailed The execution has failed.
	ExecutionStateFailed
	// ExecutionStateCancelled The execution has been canceled by the user
	ExecutionStateCancelled
)

func ExecutionStateTypes 

func ExecutionStateTypes() []ExecutionStateType

func (ExecutionStateType) IsActive 

func (s ExecutionStateType) IsActive() bool

IsActive returns true if the execution is running or has completed

func (ExecutionStateType) IsDiscarded 

func (s ExecutionStateType) IsDiscarded() bool

IsDiscarded returns true if the execution has been discarded due to a failure, rejection or cancellation

func (ExecutionStateType) IsPending 

func (s ExecutionStateType) IsPending() bool

IsPending returns true if the execution is still pending approval and did not yet start running or has been discarded

func (ExecutionStateType) IsTerminal 

func (s ExecutionStateType) IsTerminal() bool

IsTerminal returns true if the execution is in a terminal state where no further state changes are possible

func (ExecutionStateType) IsUndefined 

func (s ExecutionStateType) IsUndefined() bool

IsUndefined returns true if the execution state is undefined

func (ExecutionStateType) MarshalText 

func (s ExecutionStateType) MarshalText() ([]byte, error)

func (ExecutionStateType) String 

func (i ExecutionStateType) String() string

func (*ExecutionStateType) UnmarshalText 

func (s *ExecutionStateType) UnmarshalText(text []byte) (err error)

type Job 

type Job struct {
	APIVersion string `json:"APIVersion" example:"V1beta1"`

	// TODO this doesn't seem like it should be a part of the job as it cannot be known by a client ahead of time.
	Metadata Metadata `json:"Metadata,omitempty"`

	// The specification of this job.
	Spec Spec `json:"Spec,omitempty"`
}

type JobRequester 

type JobRequester struct {
	// The ID of the requester node that owns this job.
	RequesterNodeID string `json:"RequesterNodeID,omitempty" example:"QmXaXu9N5GNetatsvwnTfQqNtSeKAD6uCmarbh3LMRYAcF"`

	// The public key of the Requester node that created this job
	// This can be used to encrypt messages back to the creator
	RequesterPublicKey PublicKey `json:"RequesterPublicKey,omitempty"`
}

type JobSpecDocker 

type JobSpecDocker struct {
	// this should be pullable by docker
	Image string `json:"Image,omitempty"`
	// optionally override the default entrypoint
	Entrypoint []string `json:"Entrypoint,omitempty"`
	// Parameters holds additional commandline arguments
	Parameters []string `json:"Parameters,omitempty"`
	// a map of env to run the container with
	EnvironmentVariables []string `json:"EnvironmentVariables,omitempty"`
	// working directory inside the container
	WorkingDirectory string `json:"WorkingDirectory,omitempty"`
}

for VM style executors

type JobSpecWasm 

type JobSpecWasm struct {
	// The module that contains the WASM code to start running.
	EntryModule StorageSpec `json:"EntryModule,omitempty"`

	// The name of the function in the EntryModule to call to run the job. For
	// WASI jobs, this will always be `_start`, but jobs can choose to call
	// other WASM functions instead. The EntryPoint must be a zero-parameter
	// zero-result function.
	EntryPoint string `json:"EntryPoint,omitempty"`

	// The arguments supplied to the program (i.e. as ARGV).
	Parameters []string `json:"Parameters,omitempty"`

	// The variables available in the environment of the running program.
	EnvironmentVariables map[string]string `json:"EnvironmentVariables,omitempty"`

	// TODO #880: Other WASM modules whose exports will be available as imports
	// to the EntryModule.
	ImportModules []StorageSpec `json:"ImportModules,omitempty"`
}

Describes a raw WASM job

type JobState 

type JobState struct {
	// JobID is the unique identifier for the job
	JobID string `json:"JobID"`
	// Executions is a list of executions of the job across the nodes.
	// A new execution is created when a node is selected to execute the job, and a node can have multiple executions for the same
	// job due to retries, but there can only be a single active execution per node at any given time.
	Executions []ExecutionState `json:"Executions"`
	// State is the current state of the job
	State JobStateType `json:"State"`
	// Version is the version of the job state. It is incremented every time the job state is updated.
	Version int `json:"Version"`
	// CreateTime is the time when the job was created.
	CreateTime time.Time `json:"CreateTime"`
	// UpdateTime is the time when the job state was last updated.
	UpdateTime time.Time `json:"UpdateTime"`
	// TimeoutAt is the time when the job will be timed out if it is not completed.
	TimeoutAt time.Time `json:"TimeoutAt,omitempty"`
}

JobState The state of a job across the whole network that represents an aggregate view across the executions and nodes.

func (*JobState) GroupExecutionsByState 

func (s *JobState) GroupExecutionsByState() map[ExecutionStateType][]ExecutionState

GroupExecutionsByState groups the executions by state

func (*JobState) NonTerminalExecutions 

func (s *JobState) NonTerminalExecutions() []*ExecutionState

NonTerminalExecutions returns the executions that are not in a terminal state.

type JobStateType 

type JobStateType int

JobStateType The state of a job across the whole network that represents an aggregate view across the executions and nodes. 

const (
	JobStateUndefined JobStateType = iota

	JobStateNew

	JobStateInProgress

	// Job is canceled by the user.
	JobStateCancelled

	// Job have failed
	JobStateError

	// Job completed successfully
	JobStateCompleted

	// Job is waiting to be scheduled.
	JobStateQueued
)

these are the states a job can be in against a single node

func JobStateTypes 

func JobStateTypes() []JobStateType

func (JobStateType) IsTerminal 

func (s JobStateType) IsTerminal() bool

IsTerminal returns true if the given job type signals the end of the lifecycle of that job and that no change in the state can be expected.

func (JobStateType) IsUndefined 

func (s JobStateType) IsUndefined() bool

IsUndefined returns true if the job state is undefined

func (JobStateType) MarshalText 

func (s JobStateType) MarshalText() ([]byte, error)

func (JobStateType) String 

func (i JobStateType) String() string

func (*JobStateType) UnmarshalText 

func (s *JobStateType) UnmarshalText(text []byte) (err error)

type JobWithInfo 

type JobWithInfo struct {
	// Job info
	Job Job `json:"Job"`
	// The current state of the job
	State JobState `json:"State"`
}

JobWithInfo is the job request + the result of attempting to run it on the network

type LabelSelectorRequirement 

type LabelSelectorRequirement struct {
	// key is the label key that the selector applies to.
	Key string `json:"Key"`
	// operator represents a key's relationship to a set of values.
	// Valid operators are In, NotIn, Exists and DoesNotExist.
	Operator selection.Operator `json:"Operator"`
	// values is an array of string values. If the operator is In or NotIn,
	// the values array must be non-empty. If the operator is Exists or DoesNotExist,
	// the values array must be empty. This array is replaced during a strategic
	Values []string `json:"Values,omitempty"`
}

type Metadata 

type Metadata struct {
	// The unique global ID of this job in the bacalhau network.
	ID string `json:"ID,omitempty" example:"92d5d4ee-3765-4f78-8353-623f5f26df08"`

	// Time the job was submitted to the bacalhau network.
	CreatedAt time.Time `json:"CreatedAt,omitempty" example:"2022-11-17T13:29:01.871140291Z"`

	// The ID of the client that created this job.
	ClientID string `json:"ClientID,omitempty" example:"ac13188e93c97a9c2e7cf8e86c7313156a73436036f30da1ececc2ce79f9ea51"`

	Requester JobRequester `json:"Requester,omitempty"`
}

type Network 

type Network int
const (
	// NetworkNone specifies that the job does not require networking.
	NetworkNone Network = iota

	// NetworkFull specifies that the job requires unfiltered raw IP networking.
	NetworkFull

	// NetworkHTTP specifies that the job requires HTTP networking to certain domains.
	//
	// The model is: the job specifier submits a job with the domain(s) it will
	// need to communicate with, the compute provider uses this to make some
	// decision about the risk of the job and bids accordingly, and then at run
	// time the traffic is limited to only the domain(s) specified.
	//
	// As a command, something like:
	//
	//  bacalhau docker run —network=http —domain=crates.io —domain=github.com -i ipfs://Qmy1234myd4t4,dst=/code rust/compile
	//
	// The “risk” for the compute provider is that the job does something that
	// violates its terms, the terms of its hosting provider or ISP, or even the
	// law in its jurisdiction (e.g. accessing and spreading illegal content,
	// performing cyberattacks). So the same sort of risk as operating a Tor
	// exit node.
	//
	// The risk for the job specifier is that we are operating in an environment
	// they are paying for, so there is an incentive to hijack that environment
	// (e.g. via a compromised package download that runs a crypto miner on
	// install, and uses up all the paid-for job time). Having the traffic
	// enforced to only domains specified makes those sorts of attacks much
	// trickier and less valuable.
	//
	// The compute provider might well enforce its limits by other means, but
	// having the domains specified up front allows it to skip bidding on jobs
	// it knows will fail in its executor. So this is hopefully a better UX for
	// job specifiers who can have their job picked up only by someone who will
	// run it successfully.
	NetworkHTTP
)

func ParseNetwork 

func ParseNetwork(s string) (Network, error)

func (Network) MarshalText 

func (n Network) MarshalText() ([]byte, error)

func (Network) String 

func (i Network) String() string

func (*Network) UnmarshalText 

func (n *Network) UnmarshalText(text []byte) (err error)

type NetworkConfig 

type NetworkConfig struct {
	Type    Network  `json:"Type"`
	Domains []string `json:"Domains,omitempty"`
}

type PublicKey 

type PublicKey []byte

func (PublicKey) MarshalText 

func (pk PublicKey) MarshalText() ([]byte, error)

func (*PublicKey) UnmarshalText 

func (pk *PublicKey) UnmarshalText(text []byte) error

type Publisher 

type Publisher int
const (
	PublisherNoop Publisher
	PublisherIpfs
	PublisherEstuary
	PublisherS3
)

func ParsePublisher 

func ParsePublisher(str string) (Publisher, error)

func PublisherTypes 

func PublisherTypes() []Publisher

func (Publisher) MarshalText 

func (p Publisher) MarshalText() ([]byte, error)

func (Publisher) String 

func (i Publisher) String() string

func (*Publisher) UnmarshalText 

func (p *Publisher) UnmarshalText(text []byte) (err error)

type PublisherSpec 

type PublisherSpec struct {
	Type   Publisher              `json:"Type,omitempty"`
	Params map[string]interface{} `json:"Params,omitempty"`
}

type ResourceUsageConfig 

type ResourceUsageConfig struct {
	// https://github.com/BTBurke/k8sresource string
	CPU string `json:"CPU,omitempty"`
	// github.com/c2h5oh/datasize string
	Memory string `json:"Memory,omitempty"`

	Disk string `json:"Disk,omitempty"`
	GPU  string `json:"GPU"` // unsigned integer string

}

type RunCommandResult 

type RunCommandResult struct {
	// stdout of the run. Yaml provided for `describe` output
	STDOUT string `json:"stdout"`

	// bool describing if stdout was truncated
	StdoutTruncated bool `json:"stdouttruncated"`

	// stderr of the run.
	STDERR string `json:"stderr"`

	// bool describing if stderr was truncated
	StderrTruncated bool `json:"stderrtruncated"`

	// exit code of the run.
	ExitCode int `json:"exitCode"`

	// Runner error
	ErrorMsg string `json:"runnerError"`
}

type S3StorageSpec 

type S3StorageSpec struct {
	Bucket         string `json:"Bucket,omitempty"`
	Key            string `json:"Key,omitempty"`
	ChecksumSHA256 string `json:"Checksum,omitempty"`
	VersionID      string `json:"VersionID,omitempty"`
	Endpoint       string `json:"Endpoint,omitempty"`
	Region         string `json:"Region,omitempty"`
}

type Spec 

type Spec struct {
	// Deprecated: use EngineSpec.
	Engine Engine `json:"Engine,omitempty"`

	EngineSpec EngineSpec `json:"EngineSpec,omitempty"`

	// Deprecated: use PublisherSpec instead
	Publisher     Publisher     `json:"Publisher,omitempty"`
	PublisherSpec PublisherSpec `json:"PublisherSpec,omitempty"`

	// Deprecated: use EngineSpec.
	Docker JobSpecDocker `json:"Docker,omitempty"`
	// Deprecated: use EngineSpec.
	Wasm JobSpecWasm `json:"Wasm,omitempty"`

	// the compute (cpu, ram) resources this job requires
	Resources ResourceUsageConfig `json:"Resources,omitempty"`

	// The type of networking access that the job needs
	Network NetworkConfig `json:"Network,omitempty"`

	// How long a job can run in seconds before it is killed.
	// This includes the time required to run, verify and publish results
	Timeout int64 `json:"Timeout,omitempty"`

	// the data volumes we will read in the job
	// for example "read this ipfs cid"
	Inputs []StorageSpec `json:"Inputs,omitempty"`

	// the data volumes we will write in the job
	// for example "write the results to ipfs"
	Outputs []StorageSpec `json:"Outputs,omitempty"`

	// Annotations on the job - could be user or machine assigned
	Annotations []string `json:"Annotations,omitempty"`

	// NodeSelectors is a selector which must be true for the compute node to run this job.
	NodeSelectors []LabelSelectorRequirement `json:"NodeSelectors,omitempty"`

	// Do not track specified by the client
	DoNotTrack bool `json:"DoNotTrack,omitempty"`

	// The deal the client has made, such as which job bids they have accepted.
	Deal Deal `json:"Deal,omitempty"`
}

Spec is a complete specification of a job that can be run on some execution provider.

type StorageSourceType 

type StorageSourceType int

StorageSourceType is somewhere we can get data from e.g. ipfs / S3 are storage sources there can be multiple drivers for the same source e.g. ipfs fuse vs ipfs api copy 

const (
	StorageSourceIPFS StorageSourceType
	StorageSourceRepoClone
	StorageSourceRepoCloneLFS
	StorageSourceURLDownload
	StorageSourceEstuary
	StorageSourceInline
	StorageSourceLocalDirectory
	StorageSourceS3
)

func ParseStorageSourceType 

func ParseStorageSourceType(str string) (StorageSourceType, error)

func StorageSourceTypes 

func StorageSourceTypes() []StorageSourceType

func (StorageSourceType) MarshalText 

func (ss StorageSourceType) MarshalText() ([]byte, error)

func (StorageSourceType) String 

func (i StorageSourceType) String() string

func (*StorageSourceType) UnmarshalText 

func (ss *StorageSourceType) UnmarshalText(text []byte) (err error)

type StorageSpec 

type StorageSpec struct {
	// StorageSource is the abstract source of the data. E.g. a storage source
	// might be a URL download, but doesn't specify how the execution engine
	// does the download or what it will do with the downloaded data.
	StorageSource StorageSourceType `json:"StorageSource,omitempty"`

	// Name of the spec's data, for reference.
	Name string `json:"Name,omitempty" example:"job-9304c616-291f-41ad-b862-54e133c0149e-host-QmdZQ7ZbhnvWY1J12XYKGHApJ6aufKyLNSvf8jZBrBaAVL"` //nolint:lll

	// The unique ID of the data, where it makes sense (for example, in an
	// IPFS storage spec this will be the data's CID).
	// NOTE: The below is capitalized to match IPFS & IPLD (even though it's out of golang fmt)
	CID string `json:"CID,omitempty" example:"QmTVmC7JBD2ES2qGPqBNVWnX1KeEPNrPGb7rJ8cpFgtefe"`

	// Source URL of the data
	URL string `json:"URL,omitempty"`

	S3 *S3StorageSpec `json:"S3,omitempty"`

	// URL of the git Repo to clone
	Repo string `json:"Repo,omitempty"`

	// The path of the host data if we are using local directory paths
	SourcePath string `json:"SourcePath,omitempty"`

	// Allow write access for locally mounted inputs
	ReadWrite bool `json:"ReadWrite,omitempty"`

	// The path that the spec's data should be mounted on, where it makes
	// sense (for example, in a Docker storage spec this will be a filesystem
	// path).
	Path string `json:"Path,omitempty"`

	// Additional properties specific to each driver
	Metadata map[string]string `json:"Metadata,omitempty"`
}

StorageSpec represents some data on a storage engine. Storage engines are specific to particular execution engines, as different execution engines will mount data in different ways.

type TargetingMode 

type TargetingMode bool
const (
	TargetAny TargetingMode = false
	TargetAll TargetingMode = true
)

func ParseTargetingMode 

func ParseTargetingMode(s string) (TargetingMode, error)

func (TargetingMode) String 

func (t TargetingMode) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.