infprocessor

Documentation

Index

• type EngineStatus
• type P
  • func NewP(engineRouter engineRouter, logger logr.Logger) *P
  • func (p *P) AddOrUpdateEngineStatus(taskSender TaskSender, engineStatus *v1.EngineStatus, tenantID string, ...)
  • func (p *P) DumpStatus() *Status
  • func (p *P) LocalEngines() map[string][]*v1.EngineStatus
  • func (p *P) MaxInProgressTaskDuration() time.Duration
  • func (p *P) NumEnginesByTenantID() map[string]int
  • func (p *P) NumInProgressTasks() int
  • func (p *P) NumQueuedTasks() int32
  • func (p *P) ProcessTaskResult(taskResult *v1.TaskResult)
  • func (p *P) RemoveEngine(engineID string, tenantID string)
  • func (p *P) Run(ctx context.Context) error
  • func (p *P) SendAndProcessTask(ctx context.Context, origTask *v1.Task, tenantID string, ...) error
  • func (p *P) SendChatCompletionTask(ctx context.Context, tenantID string, req *v1.CreateChatCompletionRequest, ...) (*http.Response, error)
  • func (p *P) SendEmbeddingTask(ctx context.Context, tenantID string, req *v1.CreateEmbeddingRequest, ...) (*http.Response, error)
• type Status
• type TaskSender
• type TaskStatus
• type TenantStatus

Types

type EngineStatus

type EngineStatus struct {
	RegisteredModelIDs []string      `json:"registeredModelIds"`
	InProgressModelIDs []string      `json:"inProgressModelIds"`
	Tasks              []*TaskStatus `json:"tasks"`
	IsLocal            bool          `json:"isLocal"`
}

EngineStatus is the status of an engine.

type P

type P struct {
	// contains filtered or unexported fields
}

P processes inference tasks.

func NewP

func NewP(engineRouter engineRouter, logger logr.Logger) *P

NewP creates a new processor.

func (*P) AddOrUpdateEngineStatus

func (p *P) AddOrUpdateEngineStatus(
	taskSender TaskSender,
	engineStatus *v1.EngineStatus,
	tenantID string,
	isLocal bool,
)

AddOrUpdateEngineStatus adds or updates the engine status.

func (*P) DumpStatus

func (p *P) DumpStatus() *Status

DumpStatus dumps the status of the processor.

func (*P) LocalEngines

func (p *P) LocalEngines() map[string][]*v1.EngineStatus

LocalEngines returns the local engine statuses grouped by tenant ID.

func (*P) MaxInProgressTaskDuration

func (p *P) MaxInProgressTaskDuration() time.Duration

MaxInProgressTaskDuration returns the maximum duration of in-progress tasks.

func (*P) NumEnginesByTenantID

func (p *P) NumEnginesByTenantID() map[string]int

NumEnginesByTenantID returns the number of engines by tenant ID.

func (*P) NumInProgressTasks

func (p *P) NumInProgressTasks() int

NumInProgressTasks returns the number of in-progress tasks.

func (*P) NumQueuedTasks

func (p *P) NumQueuedTasks() int32

NumQueuedTasks returns the number of queued tasks.

func (*P) ProcessTaskResult

func (p *P) ProcessTaskResult(taskResult *v1.TaskResult)

ProcessTaskResult processes the task result.

func (*P) RemoveEngine

func (p *P) RemoveEngine(engineID string, tenantID string)

RemoveEngine removes the engine.

func (*P) Run

func (p *P) Run(ctx context.Context) error

Run runs the processor.

func (*P) SendAndProcessTask

func (p *P) SendAndProcessTask(
	ctx context.Context,
	origTask *v1.Task,
	tenantID string,
	processResult func(*v1.TaskResult) error,
) error

SendAndProcessTask sends a task and processes the results.

func (*P) SendChatCompletionTask

func (p *P) SendChatCompletionTask(
	ctx context.Context,
	tenantID string,
	req *v1.CreateChatCompletionRequest,
	header http.Header,
) (*http.Response, error)

SendChatCompletionTask sends a chat completion task.

func (*P) SendEmbeddingTask

func (p *P) SendEmbeddingTask(
	ctx context.Context,
	tenantID string,
	req *v1.CreateEmbeddingRequest,
	header http.Header,
) (*http.Response, error)

SendEmbeddingTask sends an embedding task.

type Status

type Status struct {
	Tenants map[string]*TenantStatus `json:"tenants"`
}

Status is the status of the processor.

type TaskSender

type TaskSender interface {
	Send(*v1.ProcessTasksResponse) error
}

TaskSender sends a new task to the engine.

type TaskStatus

type TaskStatus struct {
	ID      string `json:"id"`
	ModelID string `json:"modelId"`
}

TaskStatus is the status of a task.

type TenantStatus

type TenantStatus struct {
	Engines map[string]*EngineStatus `json:"engines"`
}

TenantStatus is the status of a tenant.