server

Documentation

Index

• type ModelClient
• type NoopModelClient
  • func (c *NoopModelClient) GetModel(ctx context.Context, in *mv1.GetModelRequest, opts ...grpc.CallOption) (*mv1.Model, error)
• type NoopRewriter
  • func (r *NoopRewriter) ProcessMessages(ctx context.Context, vstore *vsv1.VectorStore, ...) ([]*v1.CreateChatCompletionRequest_Message, error)
• type NoopVectorStoreClient
  • func (c *NoopVectorStoreClient) GetVectorStoreByName(ctx context.Context, req *vsv1.GetVectorStoreByNameRequest, ...) (*vsv1.VectorStore, error)
• type Rewriter
• type S
  • func New(m metricsMonitoring, usage sender.UsageSetter, modelClient ModelClient, ...) *S
  • func (s *S) CreateChatCompletion(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
  • func (s *S) CreateCompletion(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
  • func (s *S) CreateEmbedding(w http.ResponseWriter, req *http.Request, pathParams map[string]string)
  • func (s *S) Run(ctx context.Context, port int, authConfig config.AuthConfig) error
  • func (s *S) Stop()
• type VectorStoreClient
• type WS
  • func NewWorkerServiceServer(infProcessor *infprocessor.P, logger logr.Logger) *WS
  • func (ws *WS) ProcessTasks(srv v1.InferenceWorkerService_ProcessTasksServer) error
  • func (ws *WS) Run(ctx context.Context, port int, authConfig config.AuthConfig, ...) error
  • func (ws *WS) Stop()

Types

type ModelClient

type ModelClient interface {
	GetModel(ctx context.Context, in *mv1.GetModelRequest, opts ...grpc.CallOption) (*mv1.Model, error)
}

ModelClient is an interface for a model client.

type NoopModelClient

type NoopModelClient struct {
}

NoopModelClient is a no-op model client.

func (*NoopModelClient) GetModel

func (c *NoopModelClient) GetModel(ctx context.Context, in *mv1.GetModelRequest, opts ...grpc.CallOption) (*mv1.Model, error)

GetModel is a no-op implementation of GetModel.

type NoopRewriter

type NoopRewriter struct {
}

NoopRewriter is a no-op rewriter.

func (*NoopRewriter) ProcessMessages

func (r *NoopRewriter) ProcessMessages(
	ctx context.Context,
	vstore *vsv1.VectorStore,
	messages []*v1.CreateChatCompletionRequest_Message,
) ([]*v1.CreateChatCompletionRequest_Message, error)

ProcessMessages is a no-op implementation of ProcessMessages.

type NoopVectorStoreClient

type NoopVectorStoreClient struct {
}

NoopVectorStoreClient is a no-op vector store client.

func (*NoopVectorStoreClient) GetVectorStoreByName

func (c *NoopVectorStoreClient) GetVectorStoreByName(ctx context.Context, req *vsv1.GetVectorStoreByNameRequest, opts ...grpc.CallOption) (*vsv1.VectorStore, error)

GetVectorStoreByName is a no-op implementation of GetVectorStoreByName.

type Rewriter

type Rewriter interface {
	ProcessMessages(
		ctx context.Context,
		vstore *vsv1.VectorStore,
		messages []*v1.CreateChatCompletionRequest_Message,
	) ([]*v1.CreateChatCompletionRequest_Message, error)
}

Rewriter is an interface for rag.

type S

type S struct {
	v1.UnimplementedChatServiceServer
	// contains filtered or unexported fields
}

S is a server.

func New

func New(
	m metricsMonitoring,
	usage sender.UsageSetter,
	modelClient ModelClient,
	vsClient VectorStoreClient,
	r Rewriter,
	taskSender taskSender,
	logger logr.Logger,
) *S

New creates a server.

func (*S) CreateChatCompletion

func (s *S) CreateChatCompletion(
	w http.ResponseWriter,
	req *http.Request,
	pathParams map[string]string,
)

CreateChatCompletion creates a chat completion.

func (*S) CreateCompletion

func (s *S) CreateCompletion(
	w http.ResponseWriter,
	req *http.Request,
	pathParams map[string]string,
)

CreateCompletion creates a (legacy) completion.

The implementation is similar to CreateChatCompletion, but this has extra logic for converting a legacy request to a non-legacy request (and vice versa for response).

TODO(kenji): Avoid code duplication CreateChatCompletion.

func (*S) CreateEmbedding

func (s *S) CreateEmbedding(
	w http.ResponseWriter,
	req *http.Request,
	pathParams map[string]string,
)

CreateEmbedding creates an embedding.

func (*S) Run

func (s *S) Run(ctx context.Context, port int, authConfig config.AuthConfig) error

Run starts the gRPC server.

func (*S) Stop

func (s *S) Stop()

Stop stops the gRPC server.

type VectorStoreClient

type VectorStoreClient interface {
	GetVectorStoreByName(ctx context.Context, req *vsv1.GetVectorStoreByNameRequest, opts ...grpc.CallOption) (*vsv1.VectorStore, error)
}

VectorStoreClient is an interface for a vector store client.

type WS

type WS struct {
	v1.UnimplementedInferenceWorkerServiceServer
	// contains filtered or unexported fields
}

WS is a server for worker services.

func NewWorkerServiceServer

func NewWorkerServiceServer(infProcessor *infprocessor.P, logger logr.Logger) *WS

NewWorkerServiceServer creates a new worker service server.

func (*WS) ProcessTasks

func (ws *WS) ProcessTasks(srv v1.InferenceWorkerService_ProcessTasksServer) error

ProcessTasks processes tasks.

func (*WS) Run

func (ws *WS) Run(ctx context.Context, port int, authConfig config.AuthConfig, tlsConfig *config.TLS) error

Run runs the worker service server.

func (*WS) Stop

func (ws *WS) Stop()

Stop stops the worker service server.