modelproxy

Documentation

Index

• Variables
• func MustRegister(r prometheus.Registerer)
• type Handler
  • func NewHandler(modelScaler ModelScaler, modelResolver ModelResolver, maxRetries int, ...) *Handler
  • func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)
• type ModelResolver
• type ModelScaler

Variables

var AdditionalProxyRewrite = func(*httputil.ProxyRequest) {}

AdditionalProxyRewrite is an injection point for modifying proxy requests. Used in tests.

var ErrRetry = errors.New("retry")

Functions

func MustRegister

func MustRegister(r prometheus.Registerer)

Types

type Handler

type Handler struct {
	// contains filtered or unexported fields
}

Handler serves http requests for end-clients. It is also responsible for triggering scale-from-zero.

func NewHandler

func NewHandler(
	modelScaler ModelScaler,
	modelResolver ModelResolver,
	maxRetries int,
	retryCodes map[int]struct{},
) *Handler

func (*Handler) ServeHTTP

func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request)

type ModelResolver

type ModelResolver interface {
	AwaitBestAddress(ctx context.Context, model string) (string, func(), error)
}

type ModelScaler

type ModelScaler interface {
	ModelExists(ctx context.Context, model string) (bool, error)
	ScaleAtLeastOneReplica(ctx context.Context, model string) error
}