design

Documentation

Index

• Variables

Variables

var Application = Type("Application", func() {

	Field(1, "name", String, "The name of the Application", func() {
		Example("mlflow-seldon-predictor-01")
	})
	Field(2, "type", String, "The type of the Application", func() {
		Example("predictor")
	})
	Field(3, "description", String, "Application description", func() {
		Example("My MLFlow predictor deployed via Seldon")
	})
	Field(4, "url", String, "The public URL for accessing the Application", func() {
		Example("http://fuseml.example.org/mlflow-seldon-predictor-01/predict")
	})
	Field(5, "workflow", String, "Name of the Workflow used to create Application", func() {
		Example("mlflow-seldon-e2e")
	})
	Field(6, "k8s_resources", ArrayOf(KubernetesResource), "Kubernetes resources describing the Application")
	Field(7, "k8s_namespace", String, "Kubernetes namespace where the resources are located", func() {
		Example("fuseml-workloads")
	})

	Required("name", "type", "url", "workflow", "k8s_namespace")
})

Application describes the Application

var Codeset = Type("Codeset", func() {

	Field(1, "name", String, "The name of the Codeset", func() {
		Example("mlflow-app-01")
	})
	Field(2, "project", String, "The project this Codeset belongs to", func() {
		Example("mlflow-project-01")
	})
	Field(3, "description", String, "Codeset description", func() {
		Example("My first MLFlow application with FuseML")
		Default("")
	})
	Field(4, "labels", ArrayOf(String), "Additional Codeset labels that helps with identifying the type", func() {
		Example([]string{"mlflow", "playground"})
	})
	Field(5, "url", String, "Full URL to the Codeset", func() {
		Example("http://my-gitea.server/project/repository.git")
	})
	Required("name", "project")
})

Codeset describes the Codeset

var CodesetArgumentDesc = Type("CodesetArgumentDesc", func() {
	Field(1, "type", ArrayOf(String),
		"The type of information contained in the codeset",
		func() {
			Elem(func() {
				Enum("code", "configuration", "data", "artifact", "container", "other")
			})
			Example([]string{"code"})
		})
	Field(2, "function", ArrayOf(String),
		"The intended function of the codeset's contents",
		func() {
			Elem(func() {
				Enum("data-extraction", "data-transformation", "data-loading", "data-conversion", "data-labeling",
					"data-validation", "data-split", "data-classification", "model-definition", "model-loading",
					"model-training", "model-validation", "model-exporting", "model-conversion", "model-prediction",
					"model-explanation", "data-visualization", "model-monitoring", "container-building", "model", "runnable",
					"workflow", "dataset", "other")
			})
			Example([]string{"model-loading", "model-training"})
		})
	Field(3, "format", ArrayOf(String),
		"The format(s) used for the codeset's contents",
		func() {
			Example([]string{"MLProject", "conda"})
		})
	Field(4, "requirements", MapOf(String, String),
		"Software packages, modules, libraries, toolkits etc. and optional semantic version or version requirements",
		func() {
			Example(map[string]string{
				"mlflow":       ">=1.15",
				"scikit-learn": "0.22.*,!=0.22.5",
			})
			Example(map[string]string{
				"mlflow":       "1.15",
				"scikit-learn": "0.22.3",
			})
		})
})

CodesetArgumentDesc describes the contents of codesets either accepted as input or generated as output by a runnable

var DatasetArgumentDesc = Type("DatasetArgumentDesc", func() {
	Field(1, "type", ArrayOf(String),
		"The type of dataset",
		func() {
			Elem(func() {
				Enum("tabular", "columnar", "nested", "array", "hierarchical", "media", "text", "other")
			})
			Example([]string{"tabular"})
		})
	Field(2, "format", ArrayOf(String),
		"The dataset format",
		func() {
			Elem(func() {
				Enum("CSV", "XLS/XSLX", "numpy/array", "pandas/dataframe", "text", "parquet", "orc", "petastorm", "json", "xml",
					"yaml", "avro", "HDF5", "NetCDF", "other")
			})
			Example([]string{"tabular"})
		})
	Field(3, "compression", ArrayOf(String),
		"The compression used for the dataset",
		func() {
			Example([]string{"zip"})
		})
})

DatasetArgumentDesc describes the datasets either accepted as input or generated as output by a runnable

var Extension = Type("Extension", func() {
	tag := 1
	Field(tag, "id", String, "Uniquely identifies an extension in the registry", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
	})
	tag++
	Field(tag, "product", String,
		`Universal product identifier that can be used to group and identify extensions according to the product
they belong to. Product values can be used to identify installations of the same product registered
with the same or different FuseML servers`, func() {
			MaxLength(100)
			Example("mlflow")
		})
	tag++
	Field(tag, "version", String,
		`Extension version. To support semantic version operations, such as listing extensions
that match a semantic version constraint, it should be formatted as [v]MAJOR[.MINOR[.PATCH[-PRERELEASE][+BUILD]]]`, func() {
			MaxLength(100)
			Example("1.0")
			Example("v10.3.1-prealpha+b10022020")
		})
	tag++
	Field(tag, "description", String, "Extension description", func() {
		MaxLength(1000)
	})
	tag++
	Field(tag, "zone", String,
		`Extension zone identifier. Can be used to group and lookup extensions according to the infrastructure
location / zone / area / domain where they are installed (e.g. kubernetes cluster).
Is used to automatically select between local and external endpoints when
running queries that specify a zone filter.`, func() {
			MaxLength(100)
			Example("eu-central-01")
			Example("kube-cluster-dev-00126")
		})
	tag++
	Field(tag, "configuration", MapOf(String, String),
		`Configuration entries (e.g. configuration values required to configure all clients that connect to
this extension), expressed as set of key-value entries`, func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"authentication": "enabled",
				"api_version":    "v2",
			})
		})
	tag++
	Field(tag, "status", ExtensionStatus, "Extension status")
	tag++
	Field(tag, "services", ArrayOf(ExtensionService), "List of services provided by this extension")
})

Extension descriptor

var ExtensionCredentials = Type("ExtensionCredentials", func() {
	tag := 1
	Field(tag, "id", String, "Uniquely identifies a set of credentials within the scope of an extension service", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("dev-token-1353411")
	})
	tag++
	Field(tag, "extension_id", String, "Reference to the extension this set of credentials belongs to", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
	})
	tag++
	Field(tag, "service_id", String, "Reference to the service this set of credentials belongs to", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3")
	})
	tag++
	Field(tag, "default", Boolean,
		`Use as default credentials. Used to automatically select one of several credentials with the same
scope matching the same query.`, func() {
		})
	tag++
	Field(tag, "scope", String,
		`The scope associated with this set of credentials. Global scoped credentials can be used by any
user/project. Project scoped credentials can be used only in the context of one of the projects
supplied in the Projects list. User scoped credentials can only be used by the users in the Users
list and, optionally, in the context of the projects supplied in the Projects list`, func() {
			Enum("global", "project", "user")
			Example("global")
			Example("project")
		})
	tag++
	Field(tag, "projects", ArrayOf(String), "List of projects allowed to use these credentials", func() {
		Example([]string{"prj-prototype-001", "prj-core"})
	})
	tag++
	Field(tag, "users", ArrayOf(String), "List of users allowed to use these credentials", func() {
		Example([]string{"bobthemagicpeanut", "core-admin"})
	})
	tag++
	Field(tag, "configuration", MapOf(String, String),
		`Configuration entries (e.g. usernames, passwords, tokens, keys), expressed as set of key-value entries`, func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"username": "bobthemagicpeanut",
				"key":      "sEsnFT4#F03",
			})
		})
	tag++
	Field(tag, "status", ExtensionCredentialsStatus, "Credentials status")
})

Extension credentials descriptor

var ExtensionCredentialsStatus = Type("ExtensionCredentialsStatus", func() {
	tag := 1
	Field(tag, "created", String, "The time when the set of credentials was created", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
	tag++
	Field(tag, "updated", String, "The time when the set of credentials was last updated", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
})

Extension credentials status descriptor

var ExtensionEndpoint = Type("ExtensionEndpoint", func() {
	tag := 1
	Field(tag, "url", String,
		`Endpoint URL. In case of k8s controllers and operators, the URL points to the cluster API.
Also used to uniquely identifies an endpoint within the scope of a service`, func() {
			MaxLength(200)
			Example("https://mlflow.10.120.130.140.nip.io")
		})
	tag++
	Field(tag, "extension_id", String, "Reference to the extension this endpoint belongs to", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
	})
	tag++
	Field(tag, "service_id", String, "Reference to the service this endpoint belongs to", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3")
	})
	tag++
	Field(tag, "type", String,
		`Endpoint type - internal/external. An internal endpoint can only be accessed when the consumer
is located in the same zone as the extension service`, func() {
			Enum("internal", "external")
			Example("internal")
			Example("external")
		})
	tag++
	Field(tag, "configuration", MapOf(String, String),
		`Configuration entries (e.g. configuration values required to configure all clients that connect to
this endpoint), expressed as set of key-value entries`, func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"ca_cert":  "A7sCSdd7879sDFDSj872jkcis7",
				"insecure": "true",
			})
		})
	tag++
	Field(tag, "status", ExtensionEndpointStatus, "Endpoint status")
})

Extension endpoint descriptor

var ExtensionEndpointStatus = Type("ExtensionEndpointStatus", func() {

})

Extension endpoint status descriptor

var ExtensionQuery = Type("ExtensionQuery", func() {
	tag := 1
	Field(tag, "extension_id", String, "Query by explicit extension ID", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Default("")
		Example("s3-storage-axW45s")
	})
	tag++
	Field(tag, "product", String,
		`Match extensions by a universal product identifier. Product values can be used to identify
installations of the same product registered with the same or different FuseML servers`, func() {
			MaxLength(100)
			Default("")
			Example("mlflow")
		})
	tag++
	Field(tag, "version", String,
		`Match extensions by version or by semantic version constraints`, func() {
			MaxLength(100)
			Default("")
			Example("1.0")
			Example("v10.3.1-prealpha+b10022020")
			Example(">=v1.1,<v2.5")
		})
	tag++
	Field(tag, "zone", String,
		`Return only extensions installed in a particular the infrastructure location / zone / area / domain
(e.g. kubernetes cluster).`, func() {
			MaxLength(100)
			Default("")
			Example("eu-central-01")
			Example("kube-cluster-dev-00126")
		})
	tag++
	Field(tag, "service_id", String,
		"Match extensions that provide services identified by an explicit service ID", func() {
			Pattern(optionalIdentifierPattern)
			MaxLength(100)
			Default("")
			Example("s3")
		})
	tag++
	Field(tag, "service_resource", String,
		`A service resource identifier can be used to match only extensions providing a particular API or protocol
(e.g. s3, git, mlflow)`, func() {
			MaxLength(100)
			Default("")
			Example("s3")
			Example("git")
			Example("mlflow-tracker")
		})
	tag++
	Field(tag, "service_category", String,
		`A universal service category can be used to match extensions providing one of the well-known
categories of AI/ML services (e.g. model store, feature store, distributed training, serving)`, func() {
			MaxLength(100)
			Default("")
			Example("model-store")
			Example("serving-platform")
		})
	tag++
})

ExtensionQuery parameters

var ExtensionService = Type("ExtensionService", func() {
	tag := 1
	Field(tag, "id", String, "Uniquely identifies an extension service within the scope of an extension", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3")
	})
	tag++
	Field(tag, "extension_id", String, "Reference to the extension this service belongs to", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
	})
	tag++
	Field(tag, "resource", String,
		`Universal service identifier that can be used to identify a service in any FuseML installation.
This identifier uniquely identifies the API or protocol (e.g. s3, git, mlflow) that the service
provides`, func() {
			MaxLength(100)
			Example("s3")
			Example("git")
			Example("mlflow-tracker")
		})
	tag++
	Field(tag, "category", String,
		`Universal service category. Used to classify services into well-known categories of AI/ML services
(e.g. model store, feature store, distributed training, serving)`, func() {
			MaxLength(100)
			Example("model-store")
			Example("serving-platform")
		})
	tag++
	Field(tag, "auth_required", Boolean,
		`Marks a service for which authentication is required. If set, a set of credentials is required
to access the service; if none of the provided credentials match the scope of the consumer,
this service will be excluded from queries`, func() {
		})
	tag++
	Field(tag, "description", String, "Service description", func() {
		MaxLength(1000)
	})
	tag++
	Field(tag, "configuration", MapOf(String, String),
		`Configuration entries (e.g. configuration values required to configure all clients that connect to
this service), expressed as set of key-value entries`, func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"authentication": "enabled",
				"api_version":    "v2",
			})
		})
	tag++
	Field(tag, "status", ExtensionServiceStatus, "Service status")
	tag++
	tag++
	Field(tag, "endpoints", ArrayOf(ExtensionEndpoint), "List of endpoints through which this service can be accessed")
	tag++
	Field(tag, "credentials", ArrayOf(ExtensionCredentials), "List of credentials required to access this service")
})

Extension service descriptor

var ExtensionServiceStatus = Type("ExtensionServiceStatus", func() {
	tag := 1
	Field(tag, "registered", String, "The time when the service was registered", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
	tag++
	Field(tag, "updated", String, "The time when the service was last updated", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
})

Extension service status descriptor

var ExtensionStatus = Type("ExtensionStatus", func() {
	tag := 1
	Field(tag, "registered", String, "The time when the extension was registered", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
	tag++
	Field(tag, "updated", String, "The time when the extension was last updated", func() {
		Format(FormatDateTime)
		Default(time.Now().Format(time.RFC3339))
		Example(time.Now().Format(time.RFC3339))
	})
})

Extension status descriptor

var KubernetesResource = Type("KubernetesResource", func() {
	Field(1, "name", String, "The name of the Kubernetes resource", func() {
		Example("serving-pod-01")
	})
	Field(2, "kind", String, "The kind of Kubernetes resource", func() {
		Example("Pod")
	})
	Required("name", "kind")
})

KubernetesResource describes the Kubernetes resource

var ModelArgumentDesc = Type("ModelArgumentDesc", func() {
	Field(1, "format", ArrayOf(String),
		"The format used to package the model",
		func() {
			Elem(func() {
				Enum("PMML", "PFA", "ONNX", "SKLearn/PKL", "XGBoost/JSON", "XGBoost/PKL", "XGBoost/RDS",
					"MLeap", "TensorFlow/protobuf", "PyTorch/PKL", "PyTorch/PTH", "Keras/H5", "MLModel", "Spark/MLib",
					"other")
			})
			Example([]string{"SKLearn/PKL", "PyTorch/PKL"})
		})
	Field(2, "pretrained", Boolean,
		"Denotes a pre-trained model that is ready to use. Not applicable to all learning methods (unsupervised)", func() {
			Default(true)
		})
	Field(3, "method", String, "Learning method used to train the model",
		func() {
			Enum("supervised", "unsupervised", "reinforcement", "semi-supervised", "other")
			Example("supervised")
		})
	Field(4, "class", String, "Class of algorithm implemented by the model",
		func() {
			Enum("regression", "classification", "clustering", "dimension-reduction", "instance-based",
				"decision-tree", "bayesian", "association-rule-learning", "neutral-networks", "deep-learning",
				"ensemble", "other")
			Example("regression")
		})
	Field(5, "function", String, "The intended function for the model",
		func() {
			Example("object-detection")
			Example("sentiment-analysis")
			Example("text-to-image")
			Example("text-generation")
		})
	Field(6, "requirements", MapOf(String, String),
		"Software packages, modules, libraries, toolkits etc. and optional semantic version or version requirements",
		func() {
			Example(map[string]string{
				"pytorch":      ">=1.5",
				"scikit-learn": ">=0.22.3,<0.23",
			})
			Example(map[string]string{
				"pytorch": "1.5",
			})
		})
})

ModelArgumentDesc describes the machine learning models either accepted as input or generated as output by a runnable

var Project = Type("Project", func() {

	Field(1, "name", String, "The name of the Project", func() {
		Example("mlflow-project-01")
	})
	Field(2, "users", ArrayOf(User), "Users assigned to the Project")
	Field(3, "description", String, "Project description", func() {
		Example("Set of MLFlow applications")
		Default("")
	})
	Required("name")
})

Project describes the Project

var Runnable = Type("Runnable", func() {
	tag := 1
	Field(tag, "id", String, "The unique runnable identifier", func() {
		Pattern(identifierPattern)
		MinLength(1)
		MaxLength(100)
		Example("PyTorch-model-trainer__with_GPU_Acceleration_axW45s")
	})
	tag++
	Field(tag, "created", String, "The runnable creation time", func() {
		Format(FormatDateTime)
		Example("2021-04-09T06:17:25Z")
	})
	tag++
	Field(tag, "description", String, "Runnable description", func() {
		MaxLength(1000)
		Default("")
	})
	tag++
	Field(tag, "author", String, "Runnable author", func() {
		MaxLength(1000)
		Default("")
	})
	tag++
	Field(tag, "source", String, "URL for the sources used to build this runnable", func() {
		MaxLength(1000)
		Default("")
	})
	tag++
	Field(tag, "kind", String, "The kind of runnable (builder, trainer, predictor etc.)", func() {
		Enum("custom", "builder", "trainer", "predictor")
		Example("trainer")
		Default("custom")
	})
	tag++
	Field(tag, "container", RunnableContainer, "Runnable container implementation")
	tag++
	Field(tag, "input", RunnableInput, "Input (artifacts, parameters) accepted by this runnable, grouped by category")
	tag++
	Field(tag, "output", RunnableOutput, "Output (artifacts, parameters) generated by this runnable, grouped by category")
	tag++
	Field(tag, "defaultInputPath", String,
		"The default container path where the container expects values of inputs passed by value to be provided as files or directories",
		func() {
			Example("/opt/inputs")
		})
	tag++
	Field(tag, "defaultOutputPath", String,
		"The default container path where the container generates the values of outputs as files or directories",
		func() {
			Example("/opt/outputs")
		})
	tag++
	Field(tag, "labels", MapOf(String, String),
		"List of labels associated with the runnable.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"vendor":       "acme",
				"extension":    "prediction-engine",
				"acceleration": "GPU",
			})
		})
	tag++
	Required("id", "container")
})

Runnable description

var RunnableArgumentDesc = Type("RunnableArgumentDesc", func() {
	Field(1, "kind", String, "The kind of runnable (builder, trainer, predictor etc.)", func() {
		Enum("custom", "builder", "trainer", "predictor")
		Example("trainer")
	})
})

RunnableArgumentDesc describes other runnables either accepted as input or generated as output by a runnable

var RunnableArtifactKind = Type("RunnableArtifactKind", func() {
	Field(1, "codeset", CodesetArgumentDesc,
		"Input codeset attributes.")
	Field(2, "model", ModelArgumentDesc,
		"Input machine learning model attributes.")
	Field(3, "dataset", DatasetArgumentDesc,
		"Input dataset attributes.")
	Field(4, "runnable", RunnableArgumentDesc,
		"Input runnable attributes.")
})

RunnableArtifactKind encodes the types of artifact that can be used as a runnable input/output, as well as their type-specific attributes

var RunnableContainer = Type("RunnableContainer", func() {
	Field(1, "image", String, "Container image URI",
		func() {
			Example("myregistry.io/repo123/pytorch_trainer:v2.4")
		})
	Field(2, "entrypoint", String, "Container entrypoint. Expressions referencing inputs and outputs by name may be used to specify values.",
		func() {
			Example("/usr/local/bin/train-model.sh")
			Example("python {{inputs.ml-project}}/data_transform.py")
		})
	Field(3, "env", MapOf(String, String),
		"List of environment variables and their values. Expressions referencing inputs and outputs by name may be used to specify values.",
		func() {
			Example(map[string]string{
				"EXPERIMENT_NAME":   "first-experiment",
				"INPUT_CODE_PATH":   "{{inputs.ml-project}}",
				"OUTPUT_MODEL_PATH": "{{outputs.model-uri}}",
			})
		})
	Field(4, "args", ArrayOf(String),
		"List of command line arguments. Expressions referencing inputs and outputs by name may be used to specify values.",
		func() {
			Example([6]string{
				"--experiment",
				"first-experiment",
				"--code-path",
				"{{inputs.ml-project}}",
				"--model-out-file",
				"{{outputs.ml_model}}",
			})
		})
	Required("image")
})

RunnableContainer describes the container flavor of implementation of a runnable

var RunnableInput = Type("RunnableInput", func() {
	Field(1, "parameters", MapOf(String, RunnableInputParameter),
		"Input parameters indexed by name.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
		})
	Field(2, "artifacts", MapOf(String, RunnableInputArtifact),
		"Input artifacts indexed by name.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
		})
})

RunnableInput describes a runnable input

var RunnableInputArtifact = Type("RunnableInputArtifact", func() {
	Field(1, "description", String, "Artifact description", func() {
		MaxLength(1000)
		Default("")
		Example("Artifact description")
	})
	Field(2, "optional", Boolean, "Optional input artifact", func() {
		Default(false)
	})
	Field(3, "provider", ArrayOf(String),
		`Data passing mechanisms supported by the runnable implementation used to provide
the artifact's contents to the container. The order is significant: the framework will choose the
first mechanism in the supplied list that matches the concrete input artifact supplied at
runtime as well as other conditions such as the workflow where this runnable is referenced,
the way its inputs and outputs are connected to other runnables etc. 

One or more of the following values may be supplied:

 - local: the artifact's contents are provided using a local container path. Use this
 mechanism when the runnable implementation doesn't have the capability to interact
 directly with a storage backend service or an artifact store API and expects the
 artifact to be available locally, as a file or a directory.
 - inline: same as local, with the difference that the artifact's contents are
 passed inline, similar to a regular string parameter (subject to a content size
 limit). This mode can be used to pass the actual artifact contents to the runnable
 implementation directly as an environment variable or command line argument.
 - fuseml: use this mechanism to indicate that the runnable is able to consume artifacts
 that are tracked in one of the built-in FuseML artifact stores or one of the external
 artifact stores registered with FuseML.
 A FuseML artifact URL is provided to the container and the runnable implementation
 must be able to interact with the FuseML API to retrieve information about the artifact,
 such as the remote location where the artifact is stored in the backend storage service.
 If the artifact's contents are required by the runnable implementation, the runnable also
 needs to interact directly with the back-end storage service to download the artifact's
 contents. This needs to be explicitly specified by listing additional 'provider' values
 indicating the backend storage services and/or data transfer protocols the runnable is
 compatible with, otherwise the framework will assume the artifact contents are either
 not required by the runnable or can be retrieved without further aid from the framework
 or other workflow steps.
 - git, s3, gcs, azure, nfs, ftp, sftp, http, https, hdfs, oci, or another value representing
 a data transfer protocol, persistent storage service or artifact store: with one of these mechanisms,
 the location of the artifact's contents is provided as a URI of the form 'protocol://hostname/path'
 indicating the type and location of a remote storage service (e.g. git server, S3, GCS or Azure
 storage service, or a plain FTP, SFTP, HTTP or HTTPs server) where the artifact contents are stored.
 The runnable implementation must be able to directly interact with the remote service to retrieve
 the artifact contents. Depending on the storage service or protocol type, additional implicit
 input parameters will be provided to the runnable, describing the configuration required to access
 the resource (e.g. account names, access keys, access tokens, containers, buckets etc.). These
 input parameters are automatically provided as environment variables, but may also be explicitly
 referenced in the runnable definition using expressions. 
 
Depending on the concrete provider type decided at runtime, the expression {{ input.<input-name> }}
will be expanded differently:
- local: the expression is resolved to the local container path where contents are mounted
- inline: expression is expanded to the actual artifact contents
- fuseml: the FuseML codeset resource URL
- everything else: the remote URL where the artifact contents are stored`, func() {
			Example([1]string{
				"local",
			})
			Example([5]string{
				"git",
				"http",
				"https",
				"ftp",
				"sftp",
			})
			Default([]string{"local"})
		})
	Field(4, "kind", RunnableArtifactKind,
		`The kind of input artifact and its specific attributes. For generic resources and artifacts, this field is not set.
These attributes describe requirements concerning the contents of artifacts that the runnable is able to process as input.`)
	Field(5, "path", String,
		"Specify a custom container path where the artifact contents or the artifact URL(s) are provided to the container", func() {
			Example("/workspace/input/models")
		})
	Field(7, "dimension", String,
		"Number of artifacts encoded by this input. Use 'single' to encode a single artifact and 'array' to encode multiple",
		func() {
			Enum("single", "array")
			Example("single")
			Default("single")
		})
	Field(8, "labels", MapOf(String, String),
		`List of multi-purpose labels. Used to further filter the range of artifacts that can be supplied as input to this runnable.
		Label values may be supplied as regular expressions.`,
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"library":  "pytorch|sklearn",
				"function": "predict",
			})
		})
})

RunnableInputArtifact describes a runnable input artifact

var RunnableInputParameter = Type("RunnableInputParameter", func() {
	Field(1, "description", String, "Parameter description", func() {
		MaxLength(1000)
		Default("")
		Example("Input parameter description")
	})
	Field(2, "optional", Boolean, "Optional input parameter", func() {
		Default(false)
	})
	Field(3, "defaultValue", String, "Default value for optional input parameters")
	Field(4, "path", String,
		"Specify a custom container path where the input parameter value is provided to the container as a file",
		func() {
			Example("/workspace/input/configuration.txt")
		})
	Field(5, "labels", MapOf(String, String),
		"List of custom labels used to determine how to connect this input parameter to outputs of other runnables.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"library":  "pytorch",
				"function": "predict",
			})
		})
})

RunnableInputParameter describes a runnable input parameter

var RunnableOutput = Type("RunnableOutput", func() {
	Field(1, "parameters", MapOf(String, RunnableOutputParameter),
		"Output parameters indexed by name.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
		})
	Field(2, "artifacts", MapOf(String, RunnableOutputArtifact),
		"Output artifacts indexed by name.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
		})
})

RunnableOutput describes a runnable output

var RunnableOutputArtifact = Type("RunnableOutputArtifact", func() {
	Field(1, "description", String, "Artifact description", func() {
		MaxLength(1000)
		Default("")
		Example("Artifact description")
	})
	Field(2, "optional", Boolean, "Optional output artifact", func() {
		Default(false)
	})
	Field(3, "provider", ArrayOf(String),
		`Data passing mechanisms supported by the runnable implementation used to provide
the artifact's contents to the container. The order is significant: the framework will choose the
first mechanism in the supplied list that matches the concrete input artifact supplied at
runtime as well as other conditions such as the workflow where this runnable is referenced,
the way its inputs and outputs are connected to other runnables etc. 

One or more of the following values may be supplied:

 - local: the artifact's contents are provided using a local container path. Use this
 mechanism when the runnable implementation doesn't have the capability to interact
 directly with a storage backend service or an artifact store API and is only able
 to provide the artifact contents locally, as a file or a directory.
 - inline: same as local, with the difference that the artifact's contents can
 be passed inline, similar to an output string parameter (subject to a content size
 limit).
 - fuseml: use this mechanism to indicate that the runnable is able to register artifacts
 in one of the built-in FuseML artifact stores or one of the external artifact stores
 registered with FuseML.
 A FuseML artifact URL is provided by the container as output, pointing to the location
 of the registered FuseML artifact. The runnable implementation must be able to interact
 with the FuseML API to register information about the artifact, such as the remote
 location where the artifact is stored in the backend storage service.
 If the artifact's contents are also generated by the runnable implementation, the runnable
 also needs to interact directly with the back-end storage service to upload the artifact's
 contents. This must be explicitly specified by listing additional 'provider' values
 indicating the backend storage services and/or data transfer protocols the runnable is
 compatible with, otherwise the framework will assume the artifact contents are either
 not generated by the runnable or can be uploaded in persistent storage without further
 aid from the framework or other workflow steps.
 - git, s3, gcs, azure, nfs, ftp, sftp, http, https, hdfs, oci or another value representing
 a data transfer protocol, persistent storage service or artifact store: use one of these mechanisms
 when the runnable implementation is able to interact directly with a remote storage service to upload
 the artifact contents. A concrete persistent storage backend (e.g. git server, S3, GCS or
 Azure storage service, or a plain FTP, SFTP, HTTP or HTTPs server) is selected by the framework at
 runtime and the details describing the configuration required to upload the resource (e.g. 
 base URL, account names, access keys, access tokens, containers, buckets etc.) are provided
 to the runnable as additional implicit input parameters. These input parameters are automatically
 provided as environment variables, but may also be explicitly referenced in the runnable
 definition using expressions. 
 The runnable must provide an output URI of the form 'protocol://hostname/path' indicating
 the location where the artifact's contents are uploaded. 

Depending on the concrete provider type decided at runtime, the expression {{ output.<output-name> }}
will be expanded differently:
- local: the expression is resolved to the local container path where contents are expected by
the framework
- inline: expression is expanded to the actual artifact contents
- fuseml: the FuseML codeset resource URL
- everything else: the remote URL where the artifact contents are stored
The expression {{ output.<output-name>.path }} will be expanded to the container path where the
framework expects the artifact contents or URL value to be provided by the runnable implementation.
`, func() {
			Example([]string{
				"local",
			})
			Example([]string{
				"fuseml",
				"s3",
			})
			Example([]string{
				"git",
				"http",
				"https",
				"ftp",
				"sftp",
			})
			Default([]string{"local"})
		})
	Field(4, "kind", RunnableArtifactKind,
		`The kind of output artifact and its specific attributes. For general resources and artifacts, this field is not set.
These attributes describe the contents of artifacts generated by the runnable as output.`)
	Field(5, "path", String,
		"Specify a custom container path where the artifact contents or the artifact URL(s) are provided by the container", func() {
			Example("/workspace/output/models")
		})
	Field(7, "dimension", String,
		"Number of artifacts encoded by this output. Use 'single' to encode a single artifact and 'array' to encode multiple",
		func() {
			Enum("single", "array")
			Example("single")
			Default("single")
		})
	Field(8, "labels", MapOf(String, String),
		`List of multi-purpose labels. Used to further label the artifacts that are generated as output by this runnable.`,
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"library":  "pytorch",
				"function": "predict",
			})
		})
})

RunnableOutputArtifact describes a runnable output artifact

var RunnableOutputParameter = Type("RunnableOutputParameter", func() {
	Field(1, "description", String, "Parameter description", func() {
		MaxLength(1000)
		Default("")
		Example("Output parameter description")
	})
	Field(2, "optional", Boolean, "Optional output parameter", func() {
		Default(false)
	})
	Field(3, "defaultValue", String, "Default value for optional output parameters")
	Field(4, "path", String,
		"Specify a custom container path where the output parameter value is provided by the container as a file",
		func() {
			Example("/workspace/output/model-url.txt")
		})
	Field(5, "labels", MapOf(String, String),
		"List of custom labels used to determine how to connect this input parameter to outputs of other runnables.",
		func() {
			Key(func() {
				Pattern(identifierPattern)
			})
			Example(map[string]string{
				"library":  "pytorch",
				"function": "predict",
			})
		})
})

RunnableOutputParameter describes a runnable output parameter

var User = Type("User", func() {
	Field(1, "name", String, "User name", func() {
		Example("fuseml-mlflow-project-01")
	})
	Field(2, "email", String, "User email", func() {
		Example("fuseml-mlflow-project-01@fuseml.org")
	})
	Required("name", "email")
})

User describes the user assigned to the project

var VersionInfo = Type("VersionInfo", func() {

	Field(1, "version", String, "The server version", func() {
		Example("v1.0")
	})
	Field(2, "gitCommit", String, "The git commit corresponding to the running server version", func() {
		Example("4833d673")
	})
	Field(3, "buildDate", String, "The date the server binary was built", func() {
		Example("2021-06-02T10:21:03Z")
	})
	Field(4, "golangVersion", String, "The GO version used to build the binary", func() {
		Example("go1.16.0")
	})
	Field(5, "golangCompiler", String, "The GO compiler used to build the binary", func() {
		Example("gc")
	})
	Field(6, "platform", String, "The platform where the server is running", func() {
		Example("linux/amd64")
	})
})

VersionInfo describes server version information

var Workflow = Type("Workflow", func() {
	Field(1, "created", String, "The workflow creation time", func() {
		Format(FormatDateTime)
		Example("2021-04-09T06:17:25Z")
	})
	Field(2, "name", String, "Name of the workflow", func() {
		Example("TrainAndServe")
	})
	Field(3, "description", String, "Description for the workflow", func() {
		Example("This workflow is just trains a model and serve it")
	})
	Field(4, "inputs", ArrayOf(WorkflowInput), "Inputs for the workflow")
	Field(5, "outputs", ArrayOf(WorkflowOutput), "Outputs from the workflow")
	Field(6, "steps", ArrayOf(WorkflowStep), "Steps to be executed by the workflow")

	Required("name", "steps")
})

Workflow describes a FuseML workflow

var WorkflowAssignment = Type("WorkflowAssignment", func() {
	Field(1, "workflow", String, "Workflow assigned to the codeset")
	Field(2, "codesets", ArrayOf(Codeset), "Codesets assigned to the workflow")
	Field(3, "status", WorkflowAssignmentStatus, "The status of the assignment")

	Required("workflow", "codesets")
})

WorkflowAssignment describes the assignment between a workflow and codesets

var WorkflowAssignmentStatus = Type("WorkflowAssignmentStatus", func() {
	Field(1, "available", Boolean, "The state of the assignment")
	Field(2, "URL", String, "Dashboard URL to the resource responsible for the assignment")

	Required("available")
})

WorkflowAssignmentStatus describes the status of the resource responsible for the assignment between a workflow and codesets

var WorkflowInput = Type("WorkflowInput", func() {
	Field(1, "name", String, "Name of the input", func() {
		Example("mlflow-codeset")
	})
	Field(2, "description", String, "Description of the input", func() {
		Example("An MLFlow project codeset")
	})
	Field(3, "type", String, "The type of the input (codeset, string, ...)", func() {
		Example("codeset")
	})
	Field(4, "default", String, "Default value for the input", func() {
		Example("mlflow-example")
	})
	Field(5, "labels", ArrayOf(String), "Labels associated with the input", func() {
		Example([]string{"mlflow-project"})
	})

	Required("name")
})

WorkflowInput defines the input for a FuseML workflow

var WorkflowOutput = Type("WorkflowOutput", func() {
	Field(1, "name", String, "Name of the output", func() {
		Example("prediction-url")
	})
	Field(2, "description", String, "Description of the output", func() {
		Example("The URL where the exposed prediction service endpoint can be contacted to run predictions.")
	})
	Field(3, "type", String, "The data type of the output", func() {
		Example("string")
	})

	Required("name")
})

WorkflowOutput defines the output from a FuseML workflow

var WorkflowRun = Type("WorkflowRun", func() {
	Field(1, "name", String, "Name of the run")
	Field(2, "workflowRef", String, "Reference to the Workflow")
	Field(3, "inputs", ArrayOf(WorkflowRunInput), "Workflow run inputs")
	Field(4, "outputs", ArrayOf(WorkflowRunOutput), "Outputs from the workflow run")
	Field(5, "startTime", String, "The time when the workflow run started", func() {
		Format(FormatDateTime)
		Example("2021-04-09T06:17:25Z")
	})
	Field(6, "completionTime", String, "The time when the workflow run completed", func() {
		Format(FormatDateTime)
		Example("2021-04-09T06:20:35Z")
	})
	Field(7, "status", String, "The current status of the workflow run", func() {
		Example("Succeeded")
	})
	Field(8, "URL", String, "Dashboard URL to the workflow run")

	Required("name", "workflowRef", "startTime", "completionTime", "status")
})

WorkflowRun describes a workflow run returned when listed

var WorkflowRunInput = Type("WorkflowRunInput", func() {
	Field(1, "input", WorkflowInput, "The workflow input")
	Field(2, "value", String, "The input value set by the Workflow run")

	Required("input", "value")
})

WorkflowRunInput describes a input from a WorkflowRun including its value

var WorkflowRunOutput = Type("WorkflowRunOutput", func() {
	Field(1, "output", WorkflowOutput, "The workflow output")
	Field(2, "value", String, "The output value set by the Workflow run")

	Required("output", "value")
})

WorkflowRunInput describes the output from a WorkflowRun including its value

var WorkflowStep = Type("WorkflowStep", func() {
	Field(1, "name", String, "The name of the step", func() {
		Example("predictor")
	})
	Field(2, "image", String, "The image used to execute the step", func() {
		Example("ghcr.io/fuseml/kserve-predictor:1.0")
	})
	Field(3, "inputs", ArrayOf(WorkflowStepInput), "List of inputs for the step")
	Field(4, "outputs", ArrayOf(WorkflowStepOutput), "List of output from the step")
	Field(5, "extensions", ArrayOf(WorkflowStepExtension), "List of extension requirements")
	Field(6, "env", ArrayOf(WorkflowStepEnv), "List of environment variables available for the container running the step")
	Field(7, "resources", WorkflowStepResources, "Set the resources requests and limits for the step.")

	Required("name", "image")
})

WorkflowStep defines a step for a FuseML workflow

var WorkflowStepEnv = Type("WorkflowStepEnv", func() {
	Field(1, "name", String, "Name of the environment variable", func() {
		Example("PATH")
	})
	Field(2, "value", String, "Value to set for the environment variable", func() {
		Example("/project")
	})

	Required("name", "value")
})

WorkflowStepEnv defines the environment variables that are loaded inside the container running a FuseML workflow step

var WorkflowStepExtension = Type("WorkflowStepExtension", func() {
	tag := 1
	Field(1, "name", String, "Unique name used to reference this extension requirement", func() {
		Pattern(identifierPattern)
		MaxLength(100)
		Example("s3")
	})
	tag++
	Field(tag, "extension_id", String, "Reference extension explicitly by ID", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
		Default("")
	})
	tag++
	Field(tag, "service_id", String, "Reference service explicitly by ID", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3")
		Default("")
	})
	tag++
	Field(tag, "product", String,
		`Reference extension by product. Product values are used to identify installations of the same product registered
with the same or different FuseML servers`, func() {
			MaxLength(100)
			Example("mlflow")
			Default("")
		})
	tag++
	Field(tag, "version", String,
		`Filter extension by version. This field can be set to an explicit version value or a semantic
version constraint`, func() {
			MaxLength(100)
			Example("1.0")
			Example("v10.3.1-prealpha+b10022020")
			Example(">=v1.0,<v1.5")
			Default("")
		})
	tag++
	Field(tag, "zone", String,
		`Match only extensions installed in a given zone - the infrastructure
location / zone / area / domain where they are installed (e.g. kubernetes cluster).
The zone filter is also used to automatically select between internal and external endpoints.`, func() {
			MaxLength(100)
			Example("eu-central-01")
			Example("kube-cluster-dev-00126")
			Default("")
		})
	tag++
	Field(tag, "service_resource", String,
		`Filter extension services by resource type. This identifier uniquely identifies the API or protocol
(e.g. s3, git, mlflow) that the service provides`, func() {
			MaxLength(100)
			Example("s3")
			Example("git")
			Example("mlflow-tracker")
			Default("")
		})
	tag++
	Field(tag, "service_category", String,
		`Filter extension services by service category. Used to classify services into well-known categories of AI/ML services
(e.g. model store, feature store, distributed training, serving)`, func() {
			MaxLength(100)
			Example("model-store")
			Example("serving-platform")
			Default("")
		})
	tag++
	Field(tag, "status", WorkflowStepExtensionStatus, "Extension requirement status")
	tag++
	Required("name")
})

WorkflowStepExtension defines the extension requirements that a FuseML workflow step has

var WorkflowStepExtensionStatus = Type("WorkflowStepExtensionStatus", func() {
	tag := 1
	Field(tag, "extension_id", String, "The unique ID of the extension", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3-storage-axW45s")
		Default("")
	})
	tag++
	Field(tag, "service_id", String, "The unique ID of the service belonging to the extension", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("s3")
		Default("")
	})
	tag++
	Field(tag, "url", String,
		`The endpoint URL. In case of k8s controllers and operators, the URL points to the cluster API.
Also used to uniquely identifies an endpoint within the scope of a service`, func() {
			Format(FormatURI)
			MaxLength(200)
			Example("https://mlflow.10.120.130.140.nip.io")
			Default("")
		})
	tag++
	Field(tag, "credentials_id", String, "The ID of the set of credentials required to access the endpoint", func() {
		Pattern(optionalIdentifierPattern)
		MaxLength(100)
		Example("dev-token-1353411")
		Default("")
	})
	tag++
})

WorkflowStepExtensionStatus defines the extension endpoint and set of credentials that an extension requirement is currently resolved to

var WorkflowStepInput = Type("WorkflowStepInput", func() {
	Field(1, "name", String, "Name of the input", func() {
		Example("model-uri")
	})
	Field(2, "value", String, "Value of the input", func() {
		Example("s3://mlflow-artifacts/3/c7ae3b0e6fd44b4b96f7066c66672551/artifacts/model")
	})
	Field(3, "codeset", WorkflowStepInputCodeset, "Codeset associated with the input")

	Required("name")
})

WorkflowStepInput defines the input for a FuseML workflow step

var WorkflowStepInputCodeset = Type("WorkflowStepInputCodeset", func() {
	Field(1, "name", String, "Name or ID of the codeset", func() {
		Example("mlflow-project")
	})
	Field(2, "path", String, "Path where the codeset will be mounted inside the container running the step", func() {
		Example("/project")
	})

	Required("name")
})

WorkflowStepInputCodeset defines the Codeset type of input for a FuseML workflow step

var WorkflowStepOutput = Type("WorkflowStepOutput", func() {
	Field(1, "name", String, "Name of the variable to hold the step output value", func() {
		Example("model-uri")
	})
	Field(2, "image", WorkflowStepOutputImage, "If the step builds a container image as output it will be referenced as 'image'")

	Required("name")
})

WorkflowStepOutput defines the output from a FuseML workflow step

var WorkflowStepOutputImage = Type("WorkflowStepOutputImage", func() {
	Field(1, "dockerfile", String, "Path to the Dockerfile used to build the image", func() {
		Example("/project/.fuseml/Dockerfile")
	})
	Field(2, "name", String, "Name of the image, including the repository where the image will be stored", func() {
		Example("registry.fuseml-registry/mlflow-project/mlflow-codeset:0.1")
	})

	Required("name")
})

WorkflowStepOutputImage defines the output from a FuseML workflow when it builds a container image

var WorkflowStepResources = Type("WorkflowStepResources", func() {
	Field(1, "requests", MapOf(String, String), "Resource requests", func() {
		Example(map[string]string{"cpu": "1", "memory": "1Gi"})
	})
	Field(2, "limits", MapOf(String, String), "Resource limits", func() {
		Example(map[string]string{"cpu": "2", "memory": "2Gi", "nvidia.com/gpu": "1"})
	})
})