modelproxy

Documentation

Index

• Variables
• type EndpointResolver
• type Handler
  • func NewHandler(modelScaler ModelScaler, resolver EndpointResolver, maxRetries int, ...) *Handler
  • func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)
• type ModelScaler

Variables

var AdditionalProxyRewrite = func(*httputil.ProxyRequest) {}

AdditionalProxyRewrite is an injection point for modifying proxy requests. Used in tests.

var ErrRetry = errors.New("retry")

Types

type EndpointResolver

type EndpointResolver interface {
	AwaitBestAddress(ctx context.Context, model string) (string, func(), error)
}

type Handler

type Handler struct {
	// contains filtered or unexported fields
}

Handler serves http requests for end-clients. It is also responsible for triggering scale-from-zero.

func NewHandler

func NewHandler(
	modelScaler ModelScaler,
	resolver EndpointResolver,
	maxRetries int,
	retryCodes map[int]struct{},
) *Handler

func (*Handler) ServeHTTP

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type ModelScaler

type ModelScaler interface {
	LookupModel(ctx context.Context, model string, selectors []string) (bool, error)
	ScaleAtLeastOneReplica(ctx context.Context, model string) error
}