base

Documentation

Index

• func GetDatatype(inputTensor any) string
• type BaseInferInput
  • func (input *BaseInferInput) GetData() []any
  • func (input *BaseInferInput) GetDatatype() string
  • func (input *BaseInferInput) GetName() string
  • func (input *BaseInferInput) GetParameters() map[string]any
  • func (input *BaseInferInput) GetRawData() []byte
  • func (input *BaseInferInput) GetShape() []int64
  • func (input *BaseInferInput) SetData(inputTensor any, binaryData bool) error
  • func (input *BaseInferInput) SetDatatype(datatype string)
  • func (input *BaseInferInput) SetShape(shape []int64)
• type BaseInferOutput
  • func (output *BaseInferOutput) GetData() []any
  • func (output *BaseInferOutput) GetDatatype() string
  • func (output *BaseInferOutput) GetName() string
  • func (output *BaseInferOutput) GetParameters() map[string]any
  • func (output *BaseInferOutput) GetShape() []int64
  • func (output *BaseInferOutput) GetTensor() any
  • func (output *BaseInferOutput) UnmarshalJSON(data []byte) error
• type BaseInferResult
  • func (r *BaseInferResult) GetOutput(name string) (InferOutput, error)
  • func (r *BaseInferResult) GetShape(name string) ([]int64, error)
• type Client
• type GrpcClient
  • func NewGrpcClient(url string, connectionTimeout float64, networkTimeout float64, ssl bool, ...) (GrpcClient, error)
• type HttpClient
  • func NewHttpClient(connectionTimeout float64, insecure bool, client *http.Client) HttpClient
• type InferInput
• type InferOutput
• type InferOutputs
• type InferResult
• type Marshaller
• type RequestWrapper
• type ResponseWrapper

Functions

func GetDatatype

func GetDatatype(inputTensor any) string

GetDatatype determines the data type of the input tensor based on its Go type.

Types

type BaseInferInput

type BaseInferInput struct {
	Name       string
	Shape      []int64
	Datatype   string
	Parameters map[string]any
	Data       []any
	RawData    []byte
}

BaseInferInput is a base struct that implements common functionality for InferInput.

func (*BaseInferInput) GetData

func (input *BaseInferInput) GetData() []any

func (*BaseInferInput) GetDatatype

func (input *BaseInferInput) GetDatatype() string

func (*BaseInferInput) GetName

func (input *BaseInferInput) GetName() string

func (*BaseInferInput) GetParameters

func (input *BaseInferInput) GetParameters() map[string]any

func (*BaseInferInput) GetRawData

func (input *BaseInferInput) GetRawData() []byte

func (*BaseInferInput) GetShape

func (input *BaseInferInput) GetShape() []int64

func (*BaseInferInput) SetData

func (input *BaseInferInput) SetData(inputTensor any, binaryData bool) error

func (*BaseInferInput) SetDatatype

func (input *BaseInferInput) SetDatatype(datatype string)

func (*BaseInferInput) SetShape

func (input *BaseInferInput) SetShape(shape []int64)

type BaseInferOutput

type BaseInferOutput struct {
	Name       string
	Shape      []int64
	Datatype   string
	Parameters map[string]any
	Data       []any
}

BaseInferOutput represents basic output properties.

func (*BaseInferOutput) GetData

func (output *BaseInferOutput) GetData() []any

func (*BaseInferOutput) GetDatatype

func (output *BaseInferOutput) GetDatatype() string

func (*BaseInferOutput) GetName

func (output *BaseInferOutput) GetName() string

func (*BaseInferOutput) GetParameters

func (output *BaseInferOutput) GetParameters() map[string]any

func (*BaseInferOutput) GetShape

func (output *BaseInferOutput) GetShape() []int64

func (*BaseInferOutput) GetTensor

func (output *BaseInferOutput) GetTensor() any

func (*BaseInferOutput) UnmarshalJSON

func (output *BaseInferOutput) UnmarshalJSON(data []byte) error

UnmarshalJSON customizes the JSON unmarshaling for InferOutput, handling data conversion for different datatypes such as FP32, INT32, BOOL, etc.

type BaseInferResult

type BaseInferResult struct {
	OutputsResponse       InferOutputs
	OutputNameToBufferMap map[string]int
	Buffer                []byte
}

BaseInferResult provides common fields and methods for InferResult implementations.

func (*BaseInferResult) GetOutput

func (r *BaseInferResult) GetOutput(name string) (InferOutput, error)

func (*BaseInferResult) GetShape

func (r *BaseInferResult) GetShape(name string) ([]int64, error)

type Client

type Client interface {
	// IsServerLive checks if the server is live.
	IsServerLive(ctx context.Context, options *options.Options) (bool, error)
	// IsServerReady checks if the server is ready.
	IsServerReady(ctx context.Context, options *options.Options) (bool, error)
	// IsModelReady checks if the specified model is ready.
	IsModelReady(ctx context.Context, modelName string, modelVersion string, options *options.Options) (bool, error)
	// GetServerMetadata retrieves server metadata.
	GetServerMetadata(ctx context.Context, options *options.Options) (*models.ServerMetadataResponse, error)
	// GetModelMetadata retrieves metadata for a specific model.
	GetModelMetadata(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.ModelMetadataResponse, error)
	// GetModelConfig retrieves the configuration for a specific model.
	GetModelConfig(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.ModelConfigResponse, error)
	// GetModelRepositoryIndex retrieves the index of the model repository.
	GetModelRepositoryIndex(ctx context.Context, options *options.Options) ([]models.ModelRepositoryIndexResponse, error)
	// LoadModel loads a model into the server.
	LoadModel(ctx context.Context, modelName string, config string, files map[string][]byte, options *options.Options) error
	// UnloadModel unloads a model from the server.
	UnloadModel(ctx context.Context, modelName string, unloadDependents bool, options *options.Options) error
	// GetInferenceStatistics retrieves inference statistics for a model.
	GetInferenceStatistics(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.InferenceStatisticsResponse, error)
	// GetTraceSettings retrieves trace settings for a model or the server.
	GetTraceSettings(ctx context.Context, modelName string, options *options.Options) (*models.TraceSettingsResponse, error)
	// UpdateLogSettings updates the log settings of the server.
	UpdateLogSettings(ctx context.Context, request models.LogSettingsRequest, options *options.Options) error
	// GetLogSettings retrieves the log settings of the server.
	GetLogSettings(ctx context.Context, options *options.Options) (*models.LogSettingsResponse, error)
	// GetSystemSharedMemoryStatus retrieves the status of the system shared memory.
	GetSystemSharedMemoryStatus(ctx context.Context, regionName string, options *options.Options) ([]models.SystemSharedMemoryStatusResponse, error)
	// RegisterSystemSharedMemory registers a region of system shared memory.
	RegisterSystemSharedMemory(ctx context.Context, name string, key string, byteSize int, offset int, options *options.Options) error
	// UnregisterSystemSharedMemory unregisters a region of system shared memory.
	UnregisterSystemSharedMemory(ctx context.Context, name string, options *options.Options) error
	// GetCUDASharedMemoryStatus retrieves the status of the CUDA shared memory.
	GetCUDASharedMemoryStatus(ctx context.Context, regionName string, options *options.Options) ([]models.CUDASharedMemoryStatusResponse, error)
	// RegisterCUDASharedMemory registers a region of CUDA shared memory.
	RegisterCUDASharedMemory(ctx context.Context, name string, rawHandle []byte, deviceID int, byteSize int, options *options.Options) error
	// UnregisterCUDASharedMemory unregisters a region of CUDA shared memory.
	UnregisterCUDASharedMemory(ctx context.Context, name string, options *options.Options) error
	// Infer sends an inference request to the server.
	Infer(
		ctx context.Context,
		modelName string,
		modelVersion string,
		inputs []InferInput,
		outputs []InferOutput,
		options *options.InferOptions,
	) (InferResult, error)
}

type GrpcClient

type GrpcClient interface {
	// GetConnection returns the gRPC connection instance stored in the grpcClient struct.
	GetConnection() *grpc.ClientConn
}

GrpcClient interface defines a method to retrieve a gRPC client connection.

func NewGrpcClient

func NewGrpcClient(url string, connectionTimeout float64, networkTimeout float64, ssl bool, insecureConnection bool) (GrpcClient, error)

NewGrpcClient creates a new gRPC client connection with the given parameters.

type HttpClient

type HttpClient interface {
	// Get sends a GET request to the specified requestURI with the provided headers and query parameters.
	// Returns the HTTP response and any error encountered.
	Get(baseURL, requestURI string, headers map[string]string, queryParams map[string]string) (*http.Response, error)
	// Post sends a POST request to the specified requestURI with the provided headers, query parameters, and request body.
	// Returns the HTTP response and any error encountered.
	Post(baseURL, requestURI string, requestBody string, headers map[string]string, queryParams map[string]string) (*http.Response, error)
	// PostWithBytes sends a POST request to the specified requestURI with the provided headers, query parameters, and request body as bytes.
	// Returns the HTTP response and any error encountered.
	PostWithBytes(baseURL, requestURI string, requestBody []byte, headers, queryParams map[string]string) (*http.Response, error)
	Do(request *http.Request) (*http.Response, error)
}

func NewHttpClient

func NewHttpClient(connectionTimeout float64, insecure bool, client *http.Client) HttpClient

type InferInput

type InferInput interface {
	// GetName returns the name of the input tensor.
	GetName() string
	// GetShape returns the shape of the input tensor.
	GetShape() []int64
	// GetDatatype returns the data type of the input tensor.
	GetDatatype() string
	// GetParameters returns the parameters associated with the input tensor.
	GetParameters() map[string]any
	// GetData returns the data of the input tensor.
	GetData() []any
	// GetRawData returns the raw binary data of the input tensor.
	GetRawData() []byte
	// GetTensor constructs the tensor representation suitable for the HTTP/gRPC inference request.
	GetTensor() any
	GetBinaryData() []byte
	// SetDatatype sets the datatype of the input tensor.
	SetDatatype(datatype string)
	// SetShape sets the shape of the input tensor.
	SetShape(shape []int64)
	// SetData sets the data for the input tensor.
	// If binaryData is true, it serializes the inputTensor and stores it as RawData.
	// If binaryData is false, it flattens the inputTensor and stores it as Data.
	SetData(inputTensor any, binaryData bool) error
}

InferInput is an interface that defines methods for model inference inputs.

type InferOutput

type InferOutput interface {
	// GetName returns the output name.
	GetName() string
	// GetShape returns the output shape.
	GetShape() []int64
	// GetDatatype returns the datatype of the output.
	GetDatatype() string
	// GetParameters returns the parameters of the output.
	GetParameters() map[string]any
	// GetData returns the data of the output.
	GetData() []any
	GetTensor() any
}

InferOutput interface defines methods for output data.

type InferOutputs

type InferOutputs struct {
	ModelName    string             `json:"model_name"`
	ModelVersion string             `json:"model_version"`
	Outputs      []*BaseInferOutput `json:"outputs"`
}

InferOutputs holds model name, version, and output details.

type InferResult

type InferResult interface {
	// GetOutput returns the output tensor corresponding to the specified name.
	GetOutput(name string) (InferOutput, error)
	// GetShape returns the shape of the output tensor identified by name.
	GetShape(name string) ([]int64, error)
	// AsInt8Slice returns the output data as a slice of int8 values.
	AsInt8Slice(name string) ([]int8, error)
	// AsInt16Slice returns the output data as a slice of int16 values.
	AsInt16Slice(name string) ([]int16, error)
	// AsInt32Slice returns the output data as a slice of int32 values.
	AsInt32Slice(name string) ([]int32, error)
	// AsInt64Slice returns the output data as a slice of int64 values.
	AsInt64Slice(name string) ([]int64, error)
	// AsUint8Slice returns the output data as a slice of uint8 values.
	AsUint8Slice(name string) ([]uint8, error)
	// AsUint16Slice returns the output data as a slice of uint16 values.
	AsUint16Slice(name string) ([]uint16, error)
	// AsUint32Slice returns the output data as a slice of uint32 values.
	AsUint32Slice(name string) ([]uint32, error)
	// AsUint64Slice returns the output data as a slice of uint64 values.
	AsUint64Slice(name string) ([]uint64, error)
	// AsFloat16Slice returns the output data as a slice of float16 values, converted to float64.
	AsFloat16Slice(name string) ([]float64, error)
	// AsFloat32Slice returns the output data as a slice of float32 values.
	AsFloat32Slice(name string) ([]float32, error)
	// AsFloat64Slice returns the output data as a slice of float64 values.
	AsFloat64Slice(name string) ([]float64, error)
	// AsBoolSlice returns the output data as a slice of boolean values.
	AsBoolSlice(name string) ([]bool, error)
	// AsByteSlice returns the output data as a slice of strings.
	AsByteSlice(name string) ([]string, error)
}

InferResult defines methods for retrieving various types of output data from an inference result.

type Marshaller

type Marshaller interface {
	Marshal(v any) ([]byte, error)
}

Marshaller interface defines a method for marshaling data into byte format.

type RequestWrapper

type RequestWrapper interface {
	PrepareRequest() (any, error)
}

RequestWrapper defines an interface for preparing inference requests.

type ResponseWrapper

type ResponseWrapper interface {
	GetHeader(key string) string
	GetBody() ([]byte, error)
	GetRawOutputContents() [][]byte
	GetResponse() any
}

ResponseWrapper is an interface that abstracts the differences between HTTP and gRPC responses.