modelSource

Documentation

Index

• Constants
• type ModelHubProvider
  • func (p *ModelHubProvider) InjectModelLoader(template *corev1.PodTemplateSpec, index int)
  • func (p *ModelHubProvider) ModelName() string
  • func (p *ModelHubProvider) ModelPath() string
• type ModelSourceProvider
  • func NewModelSourceProvider(model *coreapi.OpenModel) ModelSourceProvider
• type URIProvider
  • func (p *URIProvider) InjectModelLoader(template *corev1.PodTemplateSpec, index int)
  • func (p *URIProvider) ModelName() string
  • func (p *URIProvider) ModelPath() string

Constants

const (
	// model path
	CONTAINER_MODEL_PATH = "/workspace/models/"
	HOST_MODEL_BASE_PATH = "/mnt/models/"
	// TODO: we may need /mnt/models/namespace1/ path in the future for isolates.
	HOST_CLUSTER_MODEL_PATH = HOST_MODEL_BASE_PATH + "cluster/"

	// container & volume configs
	DEFAULT_BACKEND_PORT        = 8080
	MODEL_VOLUME_NAME           = "model-volume"
	MODEL_RUNNER_CONTAINER_NAME = "model-runner"
	MODEL_LOADER_CONTAINER_NAME = "model-loader"

	// model source type
	MODEL_SOURCE_MODELHUB        = "modelhub"
	MODEL_SOURCE_MODEL_OBJ_STORE = "objstore"

	// secrets
	MODELHUB_SECRET_NAME  = "modelhub-secret"
	HUGGINGFACE_TOKEN_KEY = "HF_TOKEN"

	OSS_ACCESS_SECRET_NAME = "oss-access-secret"
	OSS_ACCESS_KEY_ID      = "OSS_ACCESS_KEY_ID"
	OSS_ACCESS_KEY_SECRET  = "OSS_ACCESS_KEY_SECRET"
)

const (
	OSS = "OSS"
)

Types

type ModelHubProvider

type ModelHubProvider struct {
	// contains filtered or unexported fields
}

func (*ModelHubProvider) InjectModelLoader

func (p *ModelHubProvider) InjectModelLoader(template *corev1.PodTemplateSpec, index int)

func (*ModelHubProvider) ModelName

func (p *ModelHubProvider) ModelName() string

func (*ModelHubProvider) ModelPath

func (p *ModelHubProvider) ModelPath() string

ModelPath Example 1:

• modelID: facebook/opt-125m modelPath: /workspace/models/models--facebook--opt-125m

Example 2:

• modelID: Qwen/Qwen2-0.5B-Instruct-GGUF fileName: qwen2-0_5b-instruct-q5_k_m.gguf modelPath: /workspace/models/qwen2-0_5b-instruct-q5_k_m.gguf

type ModelSourceProvider

type ModelSourceProvider interface {
	ModelName() string
	ModelPath() string
	// InjectModelLoader will inject the model loader to the spec,
	// index refers to the suffix of the initContainer name, like model-loader, model-loader-1.
	InjectModelLoader(spec *corev1.PodTemplateSpec, index int)
}

func NewModelSourceProvider

func NewModelSourceProvider(model *coreapi.OpenModel) ModelSourceProvider

type URIProvider

type URIProvider struct {
	// contains filtered or unexported fields
}

func (*URIProvider) InjectModelLoader

func (p *URIProvider) InjectModelLoader(template *corev1.PodTemplateSpec, index int)

func (*URIProvider) ModelName

func (p *URIProvider) ModelName() string

func (*URIProvider) ModelPath

func (p *URIProvider) ModelPath() string

Example 1:

• uri: bucket.endpoint/modelPath/opt-125m modelPath: /workspace/models/models--opt-125m

Example 2:

• uri: bucket.endpoint/modelPath/model.gguf modelPath: /workspace/models/model.gguf