inference

Documentation

Overview

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Index

• Constants
• Variables
• func CreatePresetInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, revisionNum string, ...) (client.Object, error)
• func CreateTemplateInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, ...) (client.Object, error)
• func GetInferenceImageInfo(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, ...) (string, []corev1.LocalObjectReference)

Constants

const (
	DefaultNnodes       = "1"
	DefaultNprocPerNode = "1"
	DefaultNodeRank     = "0"
	DefaultMasterAddr   = "localhost"
	DefaultMasterPort   = "29500"
)

const (
	DefaultMaxRestarts  = "3"
	DefaultRdzvId       = "rdzv_id"
	DefaultRdzvBackend  = "c10d"            // Pytorch Native Distributed data store
	DefaultRdzvEndpoint = "localhost:29500" // e.g. llama-2-13b-chat-0.llama-headless.default.svc.cluster.local:29500
)

Torch Rendezvous Params

const (
	DefaultConfigFile   = "config.yaml"
	DefaultNumProcesses = "1"
	DefaultNumMachines  = "1"
	DefaultMachineRank  = "0"
	DefaultGPUIds       = "all"
)

const (
	ProbePath     = "/healthz"
	Port5000      = int32(5000)
	InferenceFile = "inference_api.py"
)

Variables

var (
	DefaultTorchRunParams = map[string]string{
		"nnodes":         DefaultNnodes,
		"nproc_per_node": DefaultNprocPerNode,
		"node_rank":      DefaultNodeRank,
		"master_addr":    DefaultMasterAddr,
		"master_port":    DefaultMasterPort,
	}

	DefaultTorchRunRdzvParams = map[string]string{
		"max_restarts":  DefaultMaxRestarts,
		"rdzv_id":       DefaultRdzvId,
		"rdzv_backend":  DefaultRdzvBackend,
		"rdzv_endpoint": DefaultRdzvEndpoint,
	}

	DefaultAccelerateParams = map[string]string{
		"num_processes": DefaultNumProcesses,
		"num_machines":  DefaultNumMachines,
		"machine_rank":  DefaultMachineRank,
		"gpu_ids":       DefaultGPUIds,
	}

	DefaultImagePullSecrets = []corev1.LocalObjectReference{}
)

TODO: remove the above local variables starting with lower cases.

Functions

func CreatePresetInference

func CreatePresetInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, revisionNum string,
	inferenceObj *model.PresetParam, supportDistributedInference bool, kubeClient client.Client) (client.Object, error)

func CreateTemplateInference

func CreateTemplateInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, kubeClient client.Client) (client.Object, error)

func GetInferenceImageInfo

func GetInferenceImageInfo(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, presetObj *model.PresetParam) (string, []corev1.LocalObjectReference)