infprocessor

Documentation

Index

• type EngineStatus
• type P
  • func NewP(engineRouter engineRouter, logger logr.Logger) *P
  • func (p *P) AddOrUpdateEngineStatus(srv engineCommunicator, engineStatus *v1.EngineStatus, ...)
  • func (p *P) DumpStatus() *Status
  • func (p *P) MaxInProgressTaskDuration() time.Duration
  • func (p *P) NumEnginesByTenantID() map[string]int
  • func (p *P) NumInProgressTasks() int
  • func (p *P) NumQueuedTasks() int32
  • func (p *P) ProcessTaskResult(taskResult *v1.TaskResult, clusterInfo *auth.ClusterInfo) error
  • func (p *P) RemoveEngine(engineID string, clusterInfo *auth.ClusterInfo)
  • func (p *P) Run(ctx context.Context) error
  • func (p *P) SendChatCompletionTask(ctx context.Context, tenantID string, req *v1.CreateChatCompletionRequest, ...) (*http.Response, error)
  • func (p *P) SendEmbeddingTask(ctx context.Context, tenantID string, req *v1.CreateEmbeddingRequest, ...) (*http.Response, error)
• type Status
• type TaskStatus
• type TenantStatus

Types

type EngineStatus

type EngineStatus struct {
	RegisteredModelIDs []string      `json:"registeredModelIds"`
	InProgressModelIDs []string      `json:"inProgressModelIds"`
	Tasks              []*TaskStatus `json:"tasks"`
}

EngineStatus is the status of an engine.

type P

type P struct {
	// contains filtered or unexported fields
}

P processes inference tasks.

func NewP

func NewP(engineRouter engineRouter, logger logr.Logger) *P

NewP creates a new processor.

func (*P) AddOrUpdateEngineStatus

func (p *P) AddOrUpdateEngineStatus(
	srv engineCommunicator,
	engineStatus *v1.EngineStatus,
	clusterInfo *auth.ClusterInfo,
)

AddOrUpdateEngineStatus adds or updates the engine status.

func (*P) DumpStatus

func (p *P) DumpStatus() *Status

DumpStatus dumps the status of the processor.

func (*P) MaxInProgressTaskDuration

func (p *P) MaxInProgressTaskDuration() time.Duration

MaxInProgressTaskDuration returns the maximum duration of in-progress tasks.

func (*P) NumEnginesByTenantID

func (p *P) NumEnginesByTenantID() map[string]int

NumEnginesByTenantID returns the number of engines by tenant ID.

func (*P) NumInProgressTasks

func (p *P) NumInProgressTasks() int

NumInProgressTasks returns the number of in-progress tasks.

func (*P) NumQueuedTasks

func (p *P) NumQueuedTasks() int32

NumQueuedTasks returns the number of queued tasks.

func (*P) ProcessTaskResult

func (p *P) ProcessTaskResult(
	taskResult *v1.TaskResult,
	clusterInfo *auth.ClusterInfo,
) error

ProcessTaskResult processes the task result.

func (*P) RemoveEngine

func (p *P) RemoveEngine(engineID string, clusterInfo *auth.ClusterInfo)

RemoveEngine removes the engine.

func (*P) Run

func (p *P) Run(ctx context.Context) error

Run runs the processor.

func (*P) SendChatCompletionTask

func (p *P) SendChatCompletionTask(
	ctx context.Context,
	tenantID string,
	req *v1.CreateChatCompletionRequest,
	header http.Header,
) (*http.Response, error)

SendChatCompletionTask sends a chat completion task.

func (*P) SendEmbeddingTask

func (p *P) SendEmbeddingTask(
	ctx context.Context,
	tenantID string,
	req *v1.CreateEmbeddingRequest,
	header http.Header,
) (*http.Response, error)

SendEmbeddingTask sends an embedding task.

type Status

type Status struct {
	Tenants map[string]*TenantStatus `json:"tenants"`
}

Status is the status of the processor.

type TaskStatus

type TaskStatus struct {
	ID      string `json:"id"`
	ModelID string `json:"modelId"`
}

TaskStatus is the status of a task.

type TenantStatus

type TenantStatus struct {
	Engines map[string]*EngineStatus `json:"engines"`
}

TenantStatus is the status of a tenant.