types

package
v2.8.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 29, 2023 License: Apache-2.0 Imports: 21 Imported by: 4

Documentation

Index

Constants

View Source 
const (
	// OpenFaaSBackend string to identify the OpenFaaS Serverless Backend in the configuration
	OpenFaaSBackend = "openfaas"
	// KnativeBackend string to identify the Knative Serverless Backend in the configuration
	KnativeBackend = "knative"
)
View Source 
const (
	// ContainerName name of the service container
	ContainerName = "oscar-container"

	// VolumeName name of the volume for mounting the OSCAR PVC
	VolumeName = "oscar-volume"

	// VolumePath path to mount the OSCAR PVC
	VolumePath = "/oscar/bin"

	// AlpineDirectory name of the Alpine binary directory
	AlpineDirectory = "alpine"

	// ConfigVolumeName name of the volume for mounting the service configMap
	ConfigVolumeName = "oscar-config"

	// ConfigPath path to mount the service configMap
	ConfigPath = "/oscar/config"

	// FDLFileName name of the FDL file to be stored in the service's configMap
	FDLFileName = "function_config.yaml"

	// ScriptFileName name of the user script file to be stored in the service's configMap
	ScriptFileName = "script.sh"

	// PVCName name of the OSCAR PVC
	PVCName = "oscar-pvc"

	// WatchdogName name of the OpenFaaS watchdog binary
	WatchdogName = "fwatchdog"

	// WatchdogProcess name of the environment variable used by the watchdog to handle requests
	WatchdogProcess = "fprocess"

	// SupervisorName name of the FaaS Supervisor binary
	SupervisorName = "supervisor"

	// ServiceLabel label for deploying services in all backs
	ServiceLabel = "oscar_service"

	// EventVariable name used by the environment variable where events are stored
	EventVariable = "EVENT"

	// JobUUIDVariable name used by the environment variable of the job UUID
	JobUUIDVariable = "JOB_UUID"

	// OpenfaasZeroScalingLabel label to enable zero scaling in OpenFaaS functions
	OpenfaasZeroScalingLabel = "com.openfaas.scale.zero"

	// YunikornApplicationIDLabel label to define the Yunikorn's application ID
	YunikornApplicationIDLabel = "applicationId"

	// YunikornQueueLabel label to define the Yunikorn's queue
	YunikornQueueLabel = "queue"

	// YunikornOscarQueue name of the Yunikorn's queue used for OSCAR services
	YunikornOscarQueue = "oscar-queue"

	// YunikornRootQueue name of the root Yunikorn's queue
	YunikornRootQueue = "root"

	// YunikornDefaultPartition name of the default Yunikorn partition
	YunikornDefaultPartition = "default"

	// KnativeVisibilityLabel name of the knative visibility label
	KnativeVisibilityLabel = "networking.knative.dev/visibility"

	// KnativeClusterLocalValue cluster-local value for the visibility label
	KnativeClusterLocalValue = "cluster-local"

	// KnativeMinScaleAnnotation annotation key to set the minimum number of replicas for a Knative service
	KnativeMinScaleAnnotation = "autoscaling.knative.dev/min-scale"

	// KnativeMaxScaleAnnotation annotation key to set the maximum number of replicas for a Knative service
	KnativeMaxScaleAnnotation = "autoscaling.knative.dev/max-scale"

	// ReSchedulerLabelKey label key to enable/disable the ReScheduler
	ReSchedulerLabelKey = "oscar_rescheduler"
)
View Source 
const (
	// DefaultProvider string identifier for the default StorageProvider
	DefaultProvider = "default"

	// MinIOName string representing the MinIO provider name
	MinIOName = "minio"

	// S3Name string representing the S3 provider name
	S3Name = "s3"

	// OnedataName string representing the Onedata provider name
	OnedataName = "onedata"

	// WebDavName string representing a storage provider accessed via webdav
	WebDavName = "webdav"

	// ProviderSeparator separator character used to split provider's name and identifier
	ProviderSeparator = "."
)

Variables

View Source 
var YAMLMarshal = yaml.Marshal

YAMLMarshal package-level yaml marshal function

Functions

func ConvertEnvVars added in v2.7.0 

func ConvertEnvVars(vars map[string]string) []v1.EnvVar

func SetImagePullSecrets added in v2.6.2 

func SetImagePullSecrets(secrets []string) []v1.LocalObjectReference

func SetSecurityContext added in v2.7.2 

func SetSecurityContext(podSpec *v1.PodSpec)

Types

type Cluster added in v2.5.0 

type Cluster struct {
	// Endpoint endpoint of the OSCAR cluster API
	Endpoint string `json:"endpoint"`
	// AuthUser username to connect to the cluster (basic auth)
	AuthUser string `json:"auth_user"`
	// AuthPassword password to connect to the cluster (basic auth)
	AuthPassword string `json:"auth_password"`
	// SSLVerify parameter to enable or disable the verification of SSL certificates
	SSLVerify bool `json:"ssl_verify"`
}

Cluster struct to store cluster access data

type Config 

type Config struct {
	// MinIOProvider access info
	MinIOProvider *MinIOProvider `json:"minio_provider"`

	// Basic auth username
	Username string `json:"-"`

	// Basic auth password
	Password string `json:"-"`

	// Kubernetes name for the deployment and service (default: oscar)
	Name string `json:"name"`

	// Kubernetes namespace for the deployment and service (default: oscar)
	Namespace string `json:"namespace"`

	// Kubernetes namespace for services and jobs (default: oscar-svc)
	ServicesNamespace string `json:"services_namespace"`

	// Parameter used to check if the cluster have GPUs
	GPUAvailable bool `json:"gpu_available"`

	// Port used for the ClusterIP k8s service (default: 8080)
	ServicePort int `json:"-"`

	// Serverless framework used to deploy services (Openfaas | Knative)
	// If not defined only async invocations allowed (Using KubeBackend)
	ServerlessBackend string `json:"serverless_backend,omitempty"`

	// OpenfaasNamespace namespace where the OpenFaaS gateway is deployed
	OpenfaasNamespace string `json:"-"`

	// OpenfaasPort service port where the OpenFaaS gateway is exposed
	OpenfaasPort int `json:"-"`

	// OpenfaasBasicAuthSecret name of the secret used to store the OpenFaaS credentials
	OpenfaasBasicAuthSecret string `json:"-"`

	// OpenfaasPrometheusPort service port where the OpenFaaS' Prometheus is exposed
	OpenfaasPrometheusPort int `json:"-"`

	// OpenfaasScalerEnable option to enable the Openfaas scaler
	OpenfaasScalerEnable bool `json:"-"`

	// OpenfaasScalerInterval time interval to check if any function could be scaled
	OpenfaasScalerInterval string `json:"-"`

	// OpenfaasScalerInactivityDuration
	OpenfaasScalerInactivityDuration string `json:"-"`

	// WatchdogMaxInflight
	WatchdogMaxInflight int `json:"-"`

	// WatchdogWriteDebug
	WatchdogWriteDebug bool `json:"-"`

	// WatchdogExecTimeout
	WatchdogExecTimeout int `json:"-"`

	// WatchdogReadTimeout
	WatchdogReadTimeout int `json:"-"`

	// WatchdogWriteTimeout
	WatchdogWriteTimeout int `json:"-"`

	// WatchdogHealthCheckInterval
	WatchdogHealthCheckInterval int `json:"-"`

	// HTTP timeout for reading the payload (default: 300)
	ReadTimeout time.Duration `json:"-"`

	// HTTP timeout for writing the response (default: 300)
	WriteTimeout time.Duration `json:"-"`

	// YunikornEnable option to configure Apache Yunikorn
	YunikornEnable bool `json:"yunikorn_enable"`

	// YunikornNamespace
	YunikornNamespace string `json:"-"`

	// YunikornConfigMap
	YunikornConfigMap string `json:"-"`

	// YunikornConfigFileName
	YunikornConfigFileName string `json:"-"`

	// ResourceManagerEnable option to enable the Resource Manager to delegate jobs
	// when there are no available resources in the cluster (if the service has replicas)
	ResourceManagerEnable bool `json:"-"`

	// ResourceManagerInterval time interval (in seconds) to update the available resources in the cluster
	ResourceManagerInterval int `json:"-"`

	// ReSchedulerEnable option to enable the ReScheduler to delegate jobs to a replica
	// when a threshold is reached
	ReSchedulerEnable bool `json:"-"`

	// ReSchedulerInterval time interval (in seconds) to check if pending jobs
	ReSchedulerInterval int `json:"-"`

	// ReSchedulerThreshold default time (in seconds) that a job (with replicas) can be queued before delegating it
	ReSchedulerThreshold int `json:"-"`

	// OIDCEnable parameter to enable OIDC support
	OIDCEnable bool `json:"-"`

	// OIDCIssuer OpenID Connect issuer as returned in the "iss" field of the JWT payload
	OIDCIssuer string `json:"-"`

	// OIDCSubject OpenID Connect Subject (user identifier)
	OIDCSubject string `json:"-"`

	// OIDCGroups OpenID comma-separated group list to grant access in the cluster.
	// Groups defined in the "eduperson_entitlement" OIDC scope,
	// as described here: https://docs.egi.eu/providers/check-in/sp/#10-groups
	OIDCGroups []string `json:"-"`

	//
	IngressHost string `json:"-"`
}

Config stores the configuration for the OSCAR server

func ReadConfig 

func ReadConfig() (*Config, error)

ReadConfig reads environment variables to create the OSCAR server configuration

func (*Config) CheckAvailableGPUs added in v2.5.1 

func (cfg *Config) CheckAvailableGPUs(kubeClientset kubernetes.Interface)

CheckAvailableGPUs checks if there are "nvidia.com/gpu" resources in the cluster

type Info 

type Info struct {
	Version               string                 `json:"version"`
	GitCommit             string                 `json:"git_commit"`
	Architecture          string                 `json:"architecture"`
	KubeVersion           string                 `json:"kubernetes_version"`
	ServerlessBackendInfo *ServerlessBackendInfo `json:"serverless_backend,omitempty"`
}

Info represents the system information to be exposed

type JobInfo 

type JobInfo struct {
	Status       string       `json:"status"`
	CreationTime *metav1.Time `json:"creation_time,omitempty"`
	StartTime    *metav1.Time `json:"start_time,omitempty"`
	FinishTime   *metav1.Time `json:"finish_time,omitempty"`
}

JobInfo details the current status of a service's job

type MinIOProvider 

type MinIOProvider struct {
	Endpoint  string `json:"endpoint"`
	Verify    bool   `json:"verify"`
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	Region    string `json:"region"`
}

MinIOProvider stores the credentials of the MinIO storage provider

func (MinIOProvider) GetS3Client 

func (minIOProvider MinIOProvider) GetS3Client() *s3.S3

GetS3Client creates a new S3 Client from a MinIOProvider

type OnedataProvider 

type OnedataProvider struct {
	OneproviderHost string `json:"oneprovider_host"`
	Token           string `json:"token"`
	Space           string `json:"space"`
}

OnedataProvider stores the credentials of the Onedata storage provider

func (OnedataProvider) GetCDMIClient 

func (onedataProvider OnedataProvider) GetCDMIClient() *cdmi.Client

GetCDMIClient creates a new CDMI Client from a OnedataProvider

type Replica added in v2.5.0 

type Replica struct {
	// Type of the replica to re-send events (can be "oscar" or "endpoint")
	Type string `json:"type"`
	// ClusterID identifier of the cluster as defined in the "clusters" FDL field
	// Only used if Type is "oscar"
	ClusterID string `json:"cluster_id"`
	// ServiceName name of the service in the replica cluster.
	// Only used if Type is "oscar"
	ServiceName string `json:"service_name"`
	// URL url of the endpoint to re-send events (HTTP POST).
	// Only used if Type is "endpoint"
	URL string `json:"url"`
	// SSLVerify parameter to enable or disable the verification of SSL certificates.
	// Only used if Type is "endpoint"
	// Optional. (default: true)
	SSLVerify bool `json:"ssl_verify"`
	// Priority value to define delegation priority. Highest priority is defined as 0.
	// If a delegation fails, OSCAR will try to delegate to another replica with lower priority
	// Optional. (default: 0)
	Priority uint `json:"priority"`
	// Headers headers to send in delegation requests
	// Optional
	Headers map[string]string `json:"headers"`
}

Replica struct to define service's replicas in other clusters or endpoints

type ReplicaList added in v2.5.0 

type ReplicaList []Replica

ReplicaList list of replicas implementing sort.Interface

func (ReplicaList) Len added in v2.5.0 

func (rl ReplicaList) Len() int

Len method to implement sort.Interface

func (ReplicaList) Less added in v2.5.0 

func (rl ReplicaList) Less(i, j int) bool

Less method to implement sort.Interface ordering by Replica.Priority

func (ReplicaList) Swap added in v2.5.0 

func (rl ReplicaList) Swap(i, j int)

Swap method to implement sort.Interface

type S3Provider 

type S3Provider struct {
	AccessKey string `json:"access_key"`
	SecretKey string `json:"secret_key"`
	Region    string `json:"region"`
}

S3Provider stores the credentials of the AWS S3 storage provider

func (S3Provider) GetS3Client 

func (s3Provider S3Provider) GetS3Client() *s3.S3

GetS3Client creates a new S3 Client from a S3Provider

type ServerlessBackend 

type ServerlessBackend interface {
	GetInfo() *ServerlessBackendInfo
	ListServices() ([]*Service, error)
	CreateService(service Service) error
	ReadService(name string) (*Service, error)
	UpdateService(service Service) error
	DeleteService(name string) error
	GetKubeClientset() kubernetes.Interface
}

ServerlessBackend define an interface for OSCAR's backends

type ServerlessBackendInfo 

type ServerlessBackendInfo struct {
	Name    string `json:"name"`
	Version string `json:"version"`
}

ServerlessBackendInfo shows the name and version of the underlying serverless backend

type Service 

type Service struct {
	// Name the name of the service
	Name string `json:"name" binding:"required,max=39,min=1"`

	// ClusterID identifier for the current cluster, used to specify the cluster's StorageProvider in job delegations
	// Optional. (default: ""). OSCAR-CLI sets it using the ClusterID from the FDL
	ClusterID string `json:"cluster_id"`

	// Memory memory limit for the service following the kubernetes format
	// https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-memory
	// Optional. (default: 256Mi)
	Memory string `json:"memory"`

	// CPU cpu limit for the service following the kubernetes format
	// https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu
	// Optional. (default: 0.2)
	CPU string `json:"cpu"`

	// TotalMemory limit for the memory used by all the service's jobs running simultaneously
	// Apache YuniKorn scheduler is required to work
	// Same format as Memory, but internally translated to MB (integer)
	// Optional. (default: "")
	TotalMemory string `json:"total_memory"`

	// TotalCPU limit for the virtual CPUs used by all the service's jobs running simultaneously
	// Apache YuniKorn scheduler is required to work
	// Same format as CPU, but internally translated to millicores (integer)
	// Optional. (default: "")
	TotalCPU string `json:"total_cpu"`

	// EnableGPU parameter to request gpu usage in service's executions (synchronous and asynchronous)
	// Optional. (default: false)
	EnableGPU bool `json:"enable_gpu"`

	// EnableSGX parameter to use the SCONE k8s plugin
	// Optional. (default: false)
	EnableSGX bool `json:"enable_sgx"`

	// ImagePrefetch parameter to enable the image cache functionality
	// Optional. (default: false)
	ImagePrefetch bool `json:"image_prefetch"`

	// Synchronous struct to configure specific sync parameters
	// Only Knative ServerlessBackend applies this settings
	// Optional.
	Synchronous struct {
		// MinScale minimum number of active replicas (pods) for the service
		// Optional. (default: 0)
		MinScale int `json:"min_scale"`
		// MaxScale maximum number of active replicas (pods) for the service
		// Optional. (default: 0 [Unlimited])
		MaxScale int `json:"max_scale"`
	} `json:"synchronous"`

	// Replicas list of replicas to delegate jobs
	// Optional
	Replicas ReplicaList `json:"replicas,omitempty"`

	// ReSchedulerThreshold time (in seconds) that a job (with replicas) can be queued before delegating it
	// Optional
	ReSchedulerThreshold int `json:"rescheduler_threshold"`

	// LogLevel log level for the FaaS Supervisor
	// Optional. (default: INFO)
	LogLevel string `json:"log_level"`

	// Image Docker image for the service
	Image string `json:"image" binding:"required"`

	// Alpine parameter to set if image is based on Alpine
	// A custom release of faas-supervisor will be used
	// Optional. (default: false)
	Alpine bool `json:"alpine"`

	// Token token for sync and async invocations
	// Read only. This field is automatically generated by OSCAR
	Token string `json:"token"`

	// A parameter to disable the download of input files by the FaaS Supervisor
	// Optional. (default: false)
	FileStageIn bool `json:"file_stage_in"`

	// Input StorageIOConfig slice with the input service configuration
	// Optional
	Input []StorageIOConfig `json:"input"`

	// Output StorageIOConfig slice with the output service configuration
	// Optional
	Output []StorageIOConfig `json:"output"`

	// Script the user script to execute when the service is invoked
	Script string `json:"script,omitempty" binding:"required"`

	// ImagePullSecrets list of Kubernetes secrets to login to a private registry
	// Optional
	ImagePullSecrets []string `json:"image_pull_secrets,omitempty"`

	Expose struct {
		MinScale     int32 `json:"min_scale" default:"1"`
		MaxScale     int32 `json:"max_scale" default:"10"`
		Port         int   `json:"port" `
		CpuThreshold int32 `json:"cpu_threshold" default:"80" `
	} `json:"expose"`

	// The user-defined environment variables assigned to the service
	// Optional
	Environment struct {
		Vars map[string]string `json:"Variables"`
	} `json:"environment"`

	// Annotations user-defined Kubernetes annotations to be set in job's definition
	// https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
	// Optional
	Annotations map[string]string `json:"annotations"`

	// Parameter to specify the VO from the user creating the service
	// Optional
	VO string `json:"vo"`

	// Labels user-defined Kubernetes labels to be set in job's definition
	// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
	// Optional
	Labels map[string]string `json:"labels"`

	// StorageProviders configuration for the storage providers used by the service
	// Optional. (default: MinIOProvider["default"] with the server's config credentials)
	StorageProviders *StorageProviders `json:"storage_providers,omitempty"`

	// Clusters configuration for the OSCAR clusters that can be used as service's replicas
	// Optional
	Clusters map[string]Cluster `json:"clusters,omitempty"`
}

Service represents an OSCAR service following the SCAR Function Definition Language

func (*Service) GetMinIOWebhookARN 

func (service *Service) GetMinIOWebhookARN() string

GetMinIOWebhookARN returns the MinIO's notify_webhook ARN for the specified function

func (*Service) GetSupervisorPath added in v2.3.0 

func (service *Service) GetSupervisorPath() string

GetSupervisorPath returns the appropriate supervisor path

func (*Service) HasReplicas added in v2.5.0 

func (service *Service) HasReplicas() bool

HasReplicas checks if the service has replicas defined

func (*Service) ToPodSpec 

func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error)

ToPodSpec returns a k8s podSpec from the Service

func (Service) ToYAML 

func (service Service) ToYAML() (string, error)

ToYAML returns the service as a Function Definition Language YAML

type StorageIOConfig 

type StorageIOConfig struct {
	// Provider reference to the provider's name and identifier specified in StorageProviders
	// The provider's name is separated from the ID by a point (e.g. "minio.myidentifier")
	Provider string   `json:"storage_provider"`
	Path     string   `json:"path"`
	Suffix   []string `json:"suffix,omitempty"`
	Prefix   []string `json:"prefix,omitempty"`
}

StorageIOConfig provides the storage input/output configuration for services

type StorageProviders 

type StorageProviders struct {
	S3      map[string]*S3Provider      `json:"s3,omitempty"`
	MinIO   map[string]*MinIOProvider   `json:"minio,omitempty"`
	Onedata map[string]*OnedataProvider `json:"onedata,omitempty"`
	WebDav  map[string]*WebDavProvider  `json:"webdav,omitempty"`
}

StorageProviders stores the credentials of all supported storage providers

type SyncBackend 

type SyncBackend interface {
	ServerlessBackend
	GetProxyDirector(serviceName string) func(req *http.Request)
}

SyncBackend define an interface for serverless backends that allow sync invocations

type WebDavProvider added in v2.5.0 

type WebDavProvider struct {
	Hostname string `json:"hostname"`
	Login    string `json:"login"`
	Password string `json:"password"`
}

WebDavProvider stores the credentials of the a storage provider that can be accessed via webdav

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.