server

Documentation

Overview

Copyright 2021 IBM Corporation

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Copyright 2021 IBM Corporation

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• type AdapterConfiguration
  • func GetAdapterConfigurationFromEnv(log logr.Logger) (*AdapterConfiguration, error)
• type TorchServeAdapterServer
  • func NewTorchServeAdapterServer(config *AdapterConfiguration, log logr.Logger) *TorchServeAdapterServer
  • func (s *TorchServeAdapterServer) LoadModel(ctx context.Context, req *mmesh.LoadModelRequest) (*mmesh.LoadModelResponse, error)
  • func (s *TorchServeAdapterServer) ModelSize(ctx context.Context, req *mmesh.ModelSizeRequest) (*mmesh.ModelSizeResponse, error)
  • func (s *TorchServeAdapterServer) RuntimeStatus(ctx context.Context, req *mmesh.RuntimeStatusRequest) (*mmesh.RuntimeStatusResponse, error)
  • func (s *TorchServeAdapterServer) UnloadModel(ctx context.Context, req *mmesh.UnloadModelRequest) (*mmesh.UnloadModelResponse, error)

Constants

const (
	KServeServiceName string = "inference.GRPCInferenceService"
)

Types

type AdapterConfiguration

type AdapterConfiguration struct {
	Port                           int
	TorchServeManagementPort       int
	TorchServeInferenceEndpoint    string
	TorchServeContainerMemReqBytes int
	TorchServeMemBufferBytes       int
	CapacityInBytes                int
	MaxLoadingConcurrency          int
	ModelLoadingTimeoutMS          int
	DefaultModelSizeInBytes        int
	ModelSizeMultiplier            float64
	RuntimeVersion                 string
	LimitModelConcurrency          int // 0 means no limit (default)
	ModelStoreDir                  string
	UseEmbeddedPuller              bool
	RequestBatchSize               int32
	MaxBatchDelaySecs              int32
}

func GetAdapterConfigurationFromEnv

func GetAdapterConfigurationFromEnv(log logr.Logger) (*AdapterConfiguration, error)

type TorchServeAdapterServer

type TorchServeAdapterServer struct {
	ManagementClient  torchserve.ManagementAPIsServiceClient
	ManagementConn    *grpc.ClientConn
	Puller            *puller.Puller
	AdapterConfig     *AdapterConfiguration
	Log               logr.Logger
	InferenceEndpoint string
	InferenceClient   torchserve.InferenceAPIsServiceClient
	InferenceConn     *grpc.ClientConn

	// embed generated Unimplemented type for forward-compatibility for gRPC
	mmesh.UnimplementedModelRuntimeServer
}

func NewTorchServeAdapterServer

func NewTorchServeAdapterServer(config *AdapterConfiguration, log logr.Logger) *TorchServeAdapterServer

func (*TorchServeAdapterServer) LoadModel

func (s *TorchServeAdapterServer) LoadModel(ctx context.Context, req *mmesh.LoadModelRequest) (*mmesh.LoadModelResponse, error)

func (*TorchServeAdapterServer) ModelSize

func (s *TorchServeAdapterServer) ModelSize(ctx context.Context, req *mmesh.ModelSizeRequest) (*mmesh.ModelSizeResponse, error)

func (*TorchServeAdapterServer) RuntimeStatus

func (s *TorchServeAdapterServer) RuntimeStatus(ctx context.Context, req *mmesh.RuntimeStatusRequest) (*mmesh.RuntimeStatusResponse, error)

func (*TorchServeAdapterServer) UnloadModel

func (s *TorchServeAdapterServer) UnloadModel(ctx context.Context, req *mmesh.UnloadModelRequest) (*mmesh.UnloadModelResponse, error)