types

Documentation

Index

• Constants
• func Set(cfg BacalhauConfig)
• func SetDefaults(cfg BacalhauConfig)
• type APIConfig
• type BacalhauConfig
• type CapacityConfig
• type ComputeConfig
• type Duration
  • func (dur Duration) MarshalText() ([]byte, error)
  • func (dur Duration) String() string
  • func (dur *Duration) UnmarshalText(text []byte) error
• type EvaluationBrokerConfig
• type FeatureConfig
• type IpfsConfig
  • func (cfg IpfsConfig) GetSwarmAddresses() []string
• type JobDefaults
• type JobStoreConfig
• type JobTimeoutConfig
• type Libp2pConfig
• type LoggingConfig
• type MetricsConfig
• type NodeConfig
• type QueueConfig
• type RequesterConfig
• type S3StorageProviderConfig
• type StorageProviderConfig
• type StorageType
  • func ParseStorageType(s string) (ret StorageType, err error)
  • func (j StorageType) MarshalYAML() (interface{}, error)
  • func (i StorageType) String() string
  • func (j *StorageType) UnmarshalText(text []byte) error
  • func (j *StorageType) UnmarshalYAML(value *yaml.Node) error
• type TLSConfiguration
• type UpdateConfig
• type UserConfig
• type WorkerConfig

Constants

const Metrics = "Metrics"

const MetricsEventTracerPath = "Metrics.EventTracerPath"

const MetricsLibp2pTracerPath = "Metrics.Libp2pTracerPath"

const Node = "Node"

const NodeAllowListedLocalPaths = "Node.AllowListedLocalPaths"

const NodeBootstrapAddresses = "Node.BootstrapAddresses"

const NodeClientAPI = "Node.ClientAPI"

const NodeClientAPIHost = "Node.ClientAPI.Host"

const NodeClientAPIPort = "Node.ClientAPI.Port"

const NodeClientAPITLS = "Node.ClientAPI.TLS"

const NodeClientAPITLSAutoCert = "Node.ClientAPI.TLS.AutoCert"

const NodeClientAPITLSAutoCertCachePath = "Node.ClientAPI.TLS.AutoCertCachePath"

const NodeClientAPITLSServerCertificate = "Node.ClientAPI.TLS.ServerCertificate"

const NodeClientAPITLSServerKey = "Node.ClientAPI.TLS.ServerKey"

const NodeCompute = "Node.Compute"

const NodeComputeCapacity = "Node.Compute.Capacity"

const NodeComputeCapacityDefaultJobResourceLimits = "Node.Compute.Capacity.DefaultJobResourceLimits"

const NodeComputeCapacityDefaultJobResourceLimitsCPU = "Node.Compute.Capacity.DefaultJobResourceLimits.CPU"

const NodeComputeCapacityDefaultJobResourceLimitsDisk = "Node.Compute.Capacity.DefaultJobResourceLimits.Disk"

const NodeComputeCapacityDefaultJobResourceLimitsGPU = "Node.Compute.Capacity.DefaultJobResourceLimits.GPU"

const NodeComputeCapacityDefaultJobResourceLimitsMemory = "Node.Compute.Capacity.DefaultJobResourceLimits.Memory"

const NodeComputeCapacityIgnorePhysicalResourceLimits = "Node.Compute.Capacity.IgnorePhysicalResourceLimits"

const NodeComputeCapacityJobResourceLimits = "Node.Compute.Capacity.JobResourceLimits"

const NodeComputeCapacityJobResourceLimitsCPU = "Node.Compute.Capacity.JobResourceLimits.CPU"

const NodeComputeCapacityJobResourceLimitsDisk = "Node.Compute.Capacity.JobResourceLimits.Disk"

const NodeComputeCapacityJobResourceLimitsGPU = "Node.Compute.Capacity.JobResourceLimits.GPU"

const NodeComputeCapacityJobResourceLimitsMemory = "Node.Compute.Capacity.JobResourceLimits.Memory"

const NodeComputeCapacityQueueResourceLimits = "Node.Compute.Capacity.QueueResourceLimits"

const NodeComputeCapacityQueueResourceLimitsCPU = "Node.Compute.Capacity.QueueResourceLimits.CPU"

const NodeComputeCapacityQueueResourceLimitsDisk = "Node.Compute.Capacity.QueueResourceLimits.Disk"

const NodeComputeCapacityQueueResourceLimitsGPU = "Node.Compute.Capacity.QueueResourceLimits.GPU"

const NodeComputeCapacityQueueResourceLimitsMemory = "Node.Compute.Capacity.QueueResourceLimits.Memory"

const NodeComputeCapacityTotalResourceLimits = "Node.Compute.Capacity.TotalResourceLimits"

const NodeComputeCapacityTotalResourceLimitsCPU = "Node.Compute.Capacity.TotalResourceLimits.CPU"

const NodeComputeCapacityTotalResourceLimitsDisk = "Node.Compute.Capacity.TotalResourceLimits.Disk"

const NodeComputeCapacityTotalResourceLimitsGPU = "Node.Compute.Capacity.TotalResourceLimits.GPU"

const NodeComputeCapacityTotalResourceLimitsMemory = "Node.Compute.Capacity.TotalResourceLimits.Memory"

const NodeComputeExecutionStore = "Node.Compute.ExecutionStore"

const NodeComputeExecutionStorePath = "Node.Compute.ExecutionStore.Path"

const NodeComputeExecutionStoreType = "Node.Compute.ExecutionStore.Type"

const NodeComputeJobSelection = "Node.Compute.JobSelection"

const NodeComputeJobSelectionAcceptNetworkedJobs = "Node.Compute.JobSelection.AcceptNetworkedJobs"

const NodeComputeJobSelectionLocality = "Node.Compute.JobSelection.Locality"

const NodeComputeJobSelectionProbeExec = "Node.Compute.JobSelection.ProbeExec"

const NodeComputeJobSelectionProbeHTTP = "Node.Compute.JobSelection.ProbeHTTP"

const NodeComputeJobSelectionRejectStatelessJobs = "Node.Compute.JobSelection.RejectStatelessJobs"

const NodeComputeJobTimeouts = "Node.Compute.JobTimeouts"

const NodeComputeJobTimeoutsDefaultJobExecutionTimeout = "Node.Compute.JobTimeouts.DefaultJobExecutionTimeout"

const NodeComputeJobTimeoutsJobExecutionTimeoutClientIDBypassList = "Node.Compute.JobTimeouts.JobExecutionTimeoutClientIDBypassList"

const NodeComputeJobTimeoutsJobNegotiationTimeout = "Node.Compute.JobTimeouts.JobNegotiationTimeout"

const NodeComputeJobTimeoutsMaxJobExecutionTimeout = "Node.Compute.JobTimeouts.MaxJobExecutionTimeout"

const NodeComputeJobTimeoutsMinJobExecutionTimeout = "Node.Compute.JobTimeouts.MinJobExecutionTimeout"

const NodeComputeLogging = "Node.Compute.Logging"

const NodeComputeLoggingLogRunningExecutionsInterval = "Node.Compute.Logging.LogRunningExecutionsInterval"

const NodeComputeQueue = "Node.Compute.Queue"

const NodeComputeStoragePath = "Node.ComputeStoragePath"

const NodeDisabledFeatures = "Node.DisabledFeatures"

const NodeDisabledFeaturesEngines = "Node.DisabledFeatures.Engines"

const NodeDisabledFeaturesPublishers = "Node.DisabledFeatures.Publishers"

const NodeDisabledFeaturesStorages = "Node.DisabledFeatures.Storages"

const NodeDownloadURLRequestRetries = "Node.DownloadURLRequestRetries"

const NodeDownloadURLRequestTimeout = "Node.DownloadURLRequestTimeout"

const NodeEstuaryAPIKey = "Node.EstuaryAPIKey"

const NodeExecutorPluginPath = "Node.ExecutorPluginPath"

const NodeIPFS = "Node.IPFS"

const NodeIPFSConnect = "Node.IPFS.Connect"

const NodeIPFSPrivateInternal = "Node.IPFS.PrivateInternal"

const NodeIPFSServePath = "Node.IPFS.ServePath"

const NodeIPFSSwarmAddresses = "Node.IPFS.SwarmAddresses"

const NodeIPFSSwarmKeyPath = "Node.IPFS.SwarmKeyPath"

const NodeLabels = "Node.Labels"

const NodeLibp2p = "Node.Libp2p"

const NodeLibp2pPeerConnect = "Node.Libp2p.PeerConnect"

const NodeLibp2pSwarmPort = "Node.Libp2p.SwarmPort"

const NodeLoggingMode = "Node.LoggingMode"

const NodeRequester = "Node.Requester"

const NodeRequesterEvaluationBroker = "Node.Requester.EvaluationBroker"

const NodeRequesterEvaluationBrokerEvalBrokerInitialRetryDelay = "Node.Requester.EvaluationBroker.EvalBrokerInitialRetryDelay"

const NodeRequesterEvaluationBrokerEvalBrokerMaxRetryCount = "Node.Requester.EvaluationBroker.EvalBrokerMaxRetryCount"

const NodeRequesterEvaluationBrokerEvalBrokerSubsequentRetryDelay = "Node.Requester.EvaluationBroker.EvalBrokerSubsequentRetryDelay"

const NodeRequesterEvaluationBrokerEvalBrokerVisibilityTimeout = "Node.Requester.EvaluationBroker.EvalBrokerVisibilityTimeout"

const NodeRequesterExternalVerifierHook = "Node.Requester.ExternalVerifierHook"

const NodeRequesterFailureInjectionConfig = "Node.Requester.FailureInjectionConfig"

const NodeRequesterFailureInjectionConfigIsBadActor = "Node.Requester.FailureInjectionConfig.IsBadActor"

const NodeRequesterHousekeepingBackgroundTaskInterval = "Node.Requester.HousekeepingBackgroundTaskInterval"

const NodeRequesterJobDefaults = "Node.Requester.JobDefaults"

const NodeRequesterJobDefaultsExecutionTimeout = "Node.Requester.JobDefaults.ExecutionTimeout"

const NodeRequesterJobSelectionPolicy = "Node.Requester.JobSelectionPolicy"

const NodeRequesterJobSelectionPolicyAcceptNetworkedJobs = "Node.Requester.JobSelectionPolicy.AcceptNetworkedJobs"

const NodeRequesterJobSelectionPolicyLocality = "Node.Requester.JobSelectionPolicy.Locality"

const NodeRequesterJobSelectionPolicyProbeExec = "Node.Requester.JobSelectionPolicy.ProbeExec"

const NodeRequesterJobSelectionPolicyProbeHTTP = "Node.Requester.JobSelectionPolicy.ProbeHTTP"

const NodeRequesterJobSelectionPolicyRejectStatelessJobs = "Node.Requester.JobSelectionPolicy.RejectStatelessJobs"

const NodeRequesterJobStore = "Node.Requester.JobStore"

const NodeRequesterJobStorePath = "Node.Requester.JobStore.Path"

const NodeRequesterJobStoreType = "Node.Requester.JobStore.Type"

const NodeRequesterNodeRankRandomnessRange = "Node.Requester.NodeRankRandomnessRange"

const NodeRequesterOverAskForBidsFactor = "Node.Requester.OverAskForBidsFactor"

const NodeRequesterStorageProvider = "Node.Requester.StorageProvider"

const NodeRequesterStorageProviderS3 = "Node.Requester.StorageProvider.S3"

const NodeRequesterStorageProviderS3PreSignedURLDisabled = "Node.Requester.StorageProvider.S3.PreSignedURLDisabled"

const NodeRequesterStorageProviderS3PreSignedURLExpiration = "Node.Requester.StorageProvider.S3.PreSignedURLExpiration"

const NodeRequesterWorker = "Node.Requester.Worker"

const NodeRequesterWorkerWorkerCount = "Node.Requester.Worker.WorkerCount"

const NodeRequesterWorkerWorkerEvalDequeueBaseBackoff = "Node.Requester.Worker.WorkerEvalDequeueBaseBackoff"

const NodeRequesterWorkerWorkerEvalDequeueMaxBackoff = "Node.Requester.Worker.WorkerEvalDequeueMaxBackoff"

const NodeRequesterWorkerWorkerEvalDequeueTimeout = "Node.Requester.Worker.WorkerEvalDequeueTimeout"

const NodeServerAPI = "Node.ServerAPI"

const NodeServerAPIHost = "Node.ServerAPI.Host"

const NodeServerAPIPort = "Node.ServerAPI.Port"

const NodeServerAPITLS = "Node.ServerAPI.TLS"

const NodeServerAPITLSAutoCert = "Node.ServerAPI.TLS.AutoCert"

const NodeServerAPITLSAutoCertCachePath = "Node.ServerAPI.TLS.AutoCertCachePath"

const NodeServerAPITLSServerCertificate = "Node.ServerAPI.TLS.ServerCertificate"

const NodeServerAPITLSServerKey = "Node.ServerAPI.TLS.ServerKey"

const NodeType = "Node.Type"

const NodeVolumeSizeRequestTimeout = "Node.VolumeSizeRequestTimeout"

const Update = "Update"

const UpdateCheckFrequency = "Update.CheckFrequency"

const UpdateCheckStatePath = "Update.CheckStatePath"

const UpdateSkipChecks = "Update.SkipChecks"

const User = "User"

const UserInstallationID = "User.InstallationID"

const UserKeyPath = "User.KeyPath"

const UserLibp2pKeyPath = "User.Libp2pKeyPath"

Functions

func Set

func Set(cfg BacalhauConfig)

func SetDefaults

func SetDefaults(cfg BacalhauConfig)

Types

type APIConfig

type APIConfig struct {
	// Host is the hostname of an environment's public API servers.
	Host string `yaml:"Host"`
	// Port is the port that an environment serves the public API on.
	Port int `yaml:"Port"`
	// TLS returns information about how TLS is configured for the public server
	TLS TLSConfiguration `yaml:"TLS"`
}

type BacalhauConfig

type BacalhauConfig struct {
	Node    NodeConfig    `yaml:"Node"`
	User    UserConfig    `yaml:"User"`
	Metrics MetricsConfig `yaml:"Metrics"`
	Update  UpdateConfig  `yaml:"UpdateConfig"`
}

type CapacityConfig

type CapacityConfig struct {
	IgnorePhysicalResourceLimits bool `yaml:"IgnorePhysicalResourceLimits"`
	// Total amount of resource the system can be using at one time in aggregate for all jobs.
	TotalResourceLimits model.ResourceUsageConfig `yaml:"TotalResourceLimits"`
	// Per job amount of resource the system can be using at one time.
	JobResourceLimits        model.ResourceUsageConfig `yaml:"JobResourceLimits"`
	DefaultJobResourceLimits model.ResourceUsageConfig `yaml:"DefaultJobResourceLimits"`
	QueueResourceLimits      model.ResourceUsageConfig `yaml:"QueueResourceLimits"`
}

type ComputeConfig

type ComputeConfig struct {
	Capacity       CapacityConfig           `yaml:"Capacity"`
	ExecutionStore JobStoreConfig           `yaml:"ExecutionStore"`
	JobTimeouts    JobTimeoutConfig         `yaml:"JobTimeouts"`
	JobSelection   model.JobSelectionPolicy `yaml:"JobSelection"`
	Queue          QueueConfig              `yaml:"Queue"`
	Logging        LoggingConfig            `yaml:"Logging"`
}

type Duration

type Duration time.Duration

Duration is a wrapper type for time.Duration for decoding and encoding from/to YAML, etc.

func (Duration) MarshalText

func (dur Duration) MarshalText() ([]byte, error)

func (Duration) String

func (dur Duration) String() string

func (*Duration) UnmarshalText

func (dur *Duration) UnmarshalText(text []byte) error

UnmarshalText implements interface for YAML decoding

type EvaluationBrokerConfig

type EvaluationBrokerConfig struct {
	EvalBrokerVisibilityTimeout    Duration `yaml:"EvalBrokerVisibilityTimeout"`
	EvalBrokerInitialRetryDelay    Duration `yaml:"EvalBrokerInitialRetryDelay"`
	EvalBrokerSubsequentRetryDelay Duration `yaml:"EvalBrokerSubsequentRetryDelay"`
	EvalBrokerMaxRetryCount        int      `yaml:"EvalBrokerMaxRetryCount"`
}

type FeatureConfig

type FeatureConfig struct {
	Engines    []string `yaml:"Engines"`
	Publishers []string `yaml:"Publishers"`
	Storages   []string `yaml:"Storages"`
}

type IpfsConfig

type IpfsConfig struct {
	// Connect is the multiaddress to connect to for IPFS.
	Connect string `yaml:"Connect"`
	// Whether the in-process IPFS should automatically discover other IPFS nodes
	PrivateInternal bool `yaml:"PrivateInternal"`
	// IPFS multiaddresses that the in-process IPFS should connect to
	SwarmAddresses []string `yaml:"SwarmAddresses"`
	// Optional IPFS swarm key required to connect to a private IPFS swarm
	SwarmKeyPath string `yaml:"SwarmKeyPath"`
	// Path of the IPFS repo
	ServePath string `yaml:"ServePath"`
}

func (IpfsConfig) GetSwarmAddresses

func (cfg IpfsConfig) GetSwarmAddresses() []string

Due to a bug in Viper (https://github.com/spf13/viper/issues/380), string slice values can be comma-separated as a command-line flag but not as an environment variable. This getter exists to handle the case where swarm addresses that are meant to be comma-separated end up in the first item.

type JobDefaults

type JobDefaults struct {
	ExecutionTimeout Duration `yaml:"ExecutionTimeout"`
}

type JobStoreConfig

type JobStoreConfig struct {
	Type StorageType `yaml:"Type"`
	Path string      `yaml:"Path"`
}

type JobTimeoutConfig

type JobTimeoutConfig struct {
	// JobExecutionTimeoutClientIDBypassList is the list of clients that are allowed to bypass the job execution timeout
	// check.
	JobExecutionTimeoutClientIDBypassList []string `yaml:"JobExecutionTimeoutClientIDBypassList"`
	// JobNegotiationTimeout default timeout value to hold a bid for a job
	JobNegotiationTimeout Duration `yaml:"JobNegotiationTimeout"`
	// MinJobExecutionTimeout default value for the minimum execution timeout this compute node supports. Jobs with
	// lower timeout requirements will not be bid on.
	MinJobExecutionTimeout Duration `yaml:"MinJobExecutionTimeout"`
	// MaxJobExecutionTimeout default value for the maximum execution timeout this compute node supports. Jobs with
	// higher timeout requirements will not be bid on.
	MaxJobExecutionTimeout Duration `yaml:"MaxJobExecutionTimeout"`
	// DefaultJobExecutionTimeout default value for the execution timeout this compute node will assign to jobs with
	// no timeout requirement defined.
	DefaultJobExecutionTimeout Duration `yaml:"DefaultJobExecutionTimeout"`
}

type Libp2pConfig

type Libp2pConfig struct {
	SwarmPort int `yaml:"SwarmPort"`
	// PeerConnect is the libp2p multiaddress to connect to.
	PeerConnect string `yaml:"PeerConnect"`
}

type LoggingConfig

type LoggingConfig struct {
	// logging running executions
	LogRunningExecutionsInterval Duration `yaml:"LogRunningExecutionsInterval"`
}

type MetricsConfig

type MetricsConfig struct {
	Libp2pTracerPath string `yaml:"Libp2PTracerPath"`
	EventTracerPath  string `yaml:"EventTracerPath"`
}

type NodeConfig

type NodeConfig struct {
	ClientAPI APIConfig    `yaml:"ClientAPI"`
	ServerAPI APIConfig    `yaml:"ServerAPI"`
	Libp2p    Libp2pConfig `yaml:"Libp2P"`
	IPFS      IpfsConfig   `yaml:"IPFS"`

	Compute   ComputeConfig   `yaml:"Compute"`
	Requester RequesterConfig `yaml:"Requester"`

	// BootstrapAddresses is a list of bacalhau addresses for bootstrapping new local nodes.
	BootstrapAddresses []string `yaml:"BootstrapAddresses"`

	DownloadURLRequestRetries int      `yaml:"DownloadURLRequestRetries"`
	DownloadURLRequestTimeout Duration `yaml:"DownloadURLRequestTimeout"`
	VolumeSizeRequestTimeout  Duration `yaml:"VolumeSizeRequestTimeout"`

	ExecutorPluginPath string `yaml:"ExecutorPluginPath"`

	ComputeStoragePath string `yaml:"ComputeStoragePath"`

	LoggingMode logger.LogMode `yaml:"LoggingMode"`
	// Type is "compute", "requester" or both
	Type []string `yaml:"Type"`
	// Deprecated: TODO(forrest) remove.
	EstuaryAPIKey string `yaml:"EstuaryAPIKey"`
	// Local paths that are allowed to be mounted into jobs
	AllowListedLocalPaths []string `yaml:"AllowListedLocalPaths"`
	// What features should not be enabled even if installed
	DisabledFeatures FeatureConfig `yaml:"DisabledFeatures"`
	// Labels to apply to the node that can be used for node selection and filtering
	Labels map[string]string `yaml:"Labels"`
}

type QueueConfig

type QueueConfig struct {
}

type RequesterConfig

type RequesterConfig struct {
	JobDefaults JobDefaults `yaml:"JobDefaults"`
	// URL where to send external verification requests to.
	ExternalVerifierHook string `yaml:"ExternalVerifierHook"`
	// How the node decides what jobs to run.
	JobSelectionPolicy model.JobSelectionPolicy `yaml:"JobSelectionPolicy"`
	JobStore           JobStoreConfig           `yaml:"JobStore"`

	HousekeepingBackgroundTaskInterval Duration                              `yaml:"HousekeepingBackgroundTaskInterval"`
	NodeRankRandomnessRange            int                                   `yaml:"NodeRankRandomnessRange"`
	OverAskForBidsFactor               uint                                  `yaml:"OverAskForBidsFactor"`
	FailureInjectionConfig             model.FailureInjectionRequesterConfig `yaml:"FailureInjectionConfig"`

	EvaluationBroker EvaluationBrokerConfig `yaml:"EvaluationBroker"`
	Worker           WorkerConfig           `yaml:"Worker"`
	StorageProvider  StorageProviderConfig  `yaml:"StorageProvider"`
}

type S3StorageProviderConfig

type S3StorageProviderConfig struct {
	PreSignedURLDisabled   bool     `yaml:"PreSignedURLDisabled"`
	PreSignedURLExpiration Duration `yaml:"PreSignedURLExpiration"`
}

type StorageProviderConfig

type StorageProviderConfig struct {
	S3 S3StorageProviderConfig `yaml:"S3"`
}

type StorageType

type StorageType int64

const (
	InMemory StorageType = 0
	BoltDB   StorageType = 1
)

func ParseStorageType

func ParseStorageType(s string) (ret StorageType, err error)

func (StorageType) MarshalYAML

func (j StorageType) MarshalYAML() (interface{}, error)

func (StorageType) String

func (i StorageType) String() string

func (*StorageType) UnmarshalText

func (j *StorageType) UnmarshalText(text []byte) error

func (*StorageType) UnmarshalYAML

func (j *StorageType) UnmarshalYAML(value *yaml.Node) error

type TLSConfiguration

type TLSConfiguration struct {
	// AutoCert specifies a hostname for a certificate to be obtained via ACME.
	// This is only used by the server, and only by the requester node when it
	// has a publicly resolvable domain name.
	AutoCert string `yaml:"AutoCert"`

	// AutoCertCachePath specifies the directory where the autocert process
	// will cache certificates to avoid rate limits.
	AutoCertCachePath string `yaml:"AutoCertCachePath"`

	// ServerCertificate specifies the location of a TLS certificate to be used
	// by the requester to serve TLS requests
	ServerCertificate string `yaml:"ServerCertificate"`

	// ServerKey is the TLS server key to match the certificate to allow the
	// requester to server TLS.
	ServerKey string `yaml:"ServerTLSKey"`
}

type UpdateConfig

type UpdateConfig struct {
	SkipChecks     bool     `yaml:"SkipChecks"`
	CheckStatePath string   `yaml:"StatePath"`
	CheckFrequency Duration `yaml:"CheckFrequency"`
}

type UserConfig

type UserConfig struct {
	KeyPath        string `yaml:"KeyPath"`
	Libp2pKeyPath  string `yaml:"Libp2PKeyPath"`
	InstallationID string `yaml:"InstallationID"`
}

type WorkerConfig

type WorkerConfig struct {
	WorkerCount                  int      `yaml:"WorkerCount"`
	WorkerEvalDequeueTimeout     Duration `yaml:"WorkerEvalDequeueTimeout"`
	WorkerEvalDequeueBaseBackoff Duration `yaml:"WorkerEvalDequeueBaseBackoff"`
	WorkerEvalDequeueMaxBackoff  Duration `yaml:"WorkerEvalDequeueMaxBackoff"`
}