base

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 3, 2025 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetDatatype added in v0.1.0

func GetDatatype(inputTensor any) string

GetDatatype determines the data type of the input tensor based on its Go type.

Types

type BaseInferInput

type BaseInferInput struct {
	Name       string
	Shape      []int64
	Datatype   string
	Parameters map[string]any
	Data       []any
	RawData    []byte
}

BaseInferInput is a base struct that implements common functionality for InferInput.

func (*BaseInferInput) GetData

func (input *BaseInferInput) GetData() []any

func (*BaseInferInput) GetDatatype

func (input *BaseInferInput) GetDatatype() string

func (*BaseInferInput) GetName

func (input *BaseInferInput) GetName() string

func (*BaseInferInput) GetParameters

func (input *BaseInferInput) GetParameters() map[string]any

func (*BaseInferInput) GetRawData

func (input *BaseInferInput) GetRawData() []byte

func (*BaseInferInput) GetShape

func (input *BaseInferInput) GetShape() []int64

func (*BaseInferInput) SetData

func (input *BaseInferInput) SetData(inputTensor any, binaryData bool) error

func (*BaseInferInput) SetDatatype

func (input *BaseInferInput) SetDatatype(datatype string)

func (*BaseInferInput) SetShape

func (input *BaseInferInput) SetShape(shape []int64)

type BaseInferOutput

type BaseInferOutput struct {
	Name       string
	Shape      []int64
	Datatype   string
	Parameters map[string]any
	Data       []any
}

BaseInferOutput represents basic output properties.

func (*BaseInferOutput) GetData

func (output *BaseInferOutput) GetData() []any

func (*BaseInferOutput) GetDatatype

func (output *BaseInferOutput) GetDatatype() string

func (*BaseInferOutput) GetName

func (output *BaseInferOutput) GetName() string

func (*BaseInferOutput) GetParameters

func (output *BaseInferOutput) GetParameters() map[string]any

func (*BaseInferOutput) GetShape

func (output *BaseInferOutput) GetShape() []int64

func (*BaseInferOutput) GetTensor

func (output *BaseInferOutput) GetTensor() any

func (*BaseInferOutput) UnmarshalJSON added in v0.1.0

func (output *BaseInferOutput) UnmarshalJSON(data []byte) error

UnmarshalJSON customizes the JSON unmarshaling for InferOutput, handling data conversion for different datatypes such as FP32, INT32, BOOL, etc.

type BaseInferResult

type BaseInferResult struct {
	OutputsResponse       InferOutputs
	OutputNameToBufferMap map[string]int
	Buffer                []byte
}

BaseInferResult provides common fields and methods for InferResult implementations.

func (*BaseInferResult) GetOutput

func (r *BaseInferResult) GetOutput(name string) (InferOutput, error)

func (*BaseInferResult) GetShape added in v0.1.0

func (r *BaseInferResult) GetShape(name string) ([]int64, error)

type Client

type Client interface {
	// IsServerLive checks if the server is live.
	IsServerLive(ctx context.Context, options *options.Options) (bool, error)
	// IsServerReady checks if the server is ready.
	IsServerReady(ctx context.Context, options *options.Options) (bool, error)
	// IsModelReady checks if the specified model is ready.
	IsModelReady(ctx context.Context, modelName string, modelVersion string, options *options.Options) (bool, error)
	// GetServerMetadata retrieves server metadata.
	GetServerMetadata(ctx context.Context, options *options.Options) (*models.ServerMetadataResponse, error)
	// GetModelMetadata retrieves metadata for a specific model.
	GetModelMetadata(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.ModelMetadataResponse, error)
	// GetModelConfig retrieves the configuration for a specific model.
	GetModelConfig(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.ModelConfigResponse, error)
	// GetModelRepositoryIndex retrieves the index of the model repository.
	GetModelRepositoryIndex(ctx context.Context, options *options.Options) ([]models.ModelRepositoryIndexResponse, error)
	// LoadModel loads a model into the server.
	LoadModel(ctx context.Context, modelName string, config string, files map[string][]byte, options *options.Options) error
	// UnloadModel unloads a model from the server.
	UnloadModel(ctx context.Context, modelName string, unloadDependents bool, options *options.Options) error
	// GetInferenceStatistics retrieves inference statistics for a model.
	GetInferenceStatistics(ctx context.Context, modelName string, modelVersion string, options *options.Options) (*models.InferenceStatisticsResponse, error)
	// GetTraceSettings retrieves trace settings for a model or the server.
	GetTraceSettings(ctx context.Context, modelName string, options *options.Options) (*models.TraceSettingsResponse, error)
	// UpdateLogSettings updates the log settings of the server.
	UpdateLogSettings(ctx context.Context, request models.LogSettingsRequest, options *options.Options) error
	// GetLogSettings retrieves the log settings of the server.
	GetLogSettings(ctx context.Context, options *options.Options) (*models.LogSettingsResponse, error)
	// GetSystemSharedMemoryStatus retrieves the status of the system shared memory.
	GetSystemSharedMemoryStatus(ctx context.Context, regionName string, options *options.Options) ([]models.SystemSharedMemoryStatusResponse, error)
	// RegisterSystemSharedMemory registers a region of system shared memory.
	RegisterSystemSharedMemory(ctx context.Context, name string, key string, byteSize int, offset int, options *options.Options) error
	// UnregisterSystemSharedMemory unregisters a region of system shared memory.
	UnregisterSystemSharedMemory(ctx context.Context, name string, options *options.Options) error
	// GetCUDASharedMemoryStatus retrieves the status of the CUDA shared memory.
	GetCUDASharedMemoryStatus(ctx context.Context, regionName string, options *options.Options) ([]models.CUDASharedMemoryStatusResponse, error)
	// RegisterCUDASharedMemory registers a region of CUDA shared memory.
	RegisterCUDASharedMemory(ctx context.Context, name string, rawHandle []byte, deviceID int, byteSize int, options *options.Options) error
	// UnregisterCUDASharedMemory unregisters a region of CUDA shared memory.
	UnregisterCUDASharedMemory(ctx context.Context, name string, options *options.Options) error
	// Infer sends an inference request to the server.
	Infer(
		ctx context.Context,
		modelName string,
		modelVersion string,
		inputs []InferInput,
		outputs []InferOutput,
		options *options.InferOptions,
	) (InferResult, error)
}

type GrpcClient

type GrpcClient interface {
	// GetConnection returns the gRPC connection instance stored in the grpcClient struct.
	GetConnection() *grpc.ClientConn
}

GrpcClient interface defines a method to retrieve a gRPC client connection.

func NewGrpcClient

func NewGrpcClient(url string, connectionTimeout float64, networkTimeout float64, ssl bool, insecureConnection bool) (GrpcClient, error)

NewGrpcClient creates a new gRPC client connection with the given parameters.

type HttpClient

type HttpClient interface {
	// Get sends a GET request to the specified requestURI with the provided headers and query parameters.
	// Returns the HTTP response and any error encountered.
	Get(baseURL, requestURI string, headers map[string]string, queryParams map[string]string) (*http.Response, error)
	// Post sends a POST request to the specified requestURI with the provided headers, query parameters, and request body.
	// Returns the HTTP response and any error encountered.
	Post(baseURL, requestURI string, requestBody string, headers map[string]string, queryParams map[string]string) (*http.Response, error)
	// PostWithBytes sends a POST request to the specified requestURI with the provided headers, query parameters, and request body as bytes.
	// Returns the HTTP response and any error encountered.
	PostWithBytes(baseURL, requestURI string, requestBody []byte, headers, queryParams map[string]string) (*http.Response, error)
	Do(request *http.Request) (*http.Response, error)
}

func NewHttpClient

func NewHttpClient(connectionTimeout float64, insecure bool, client *http.Client) HttpClient

type InferInput

type InferInput interface {
	// GetName returns the name of the input tensor.
	GetName() string
	// GetShape returns the shape of the input tensor.
	GetShape() []int64
	// GetDatatype returns the data type of the input tensor.
	GetDatatype() string
	// GetParameters returns the parameters associated with the input tensor.
	GetParameters() map[string]any
	// GetData returns the data of the input tensor.
	GetData() []any
	// GetRawData returns the raw binary data of the input tensor.
	GetRawData() []byte
	// GetTensor constructs the tensor representation suitable for the HTTP/gRPC inference request.
	GetTensor() any
	GetBinaryData() []byte
	// SetDatatype sets the datatype of the input tensor.
	SetDatatype(datatype string)
	// SetShape sets the shape of the input tensor.
	SetShape(shape []int64)
	// SetData sets the data for the input tensor.
	// If binaryData is true, it serializes the inputTensor and stores it as RawData.
	// If binaryData is false, it flattens the inputTensor and stores it as Data.
	SetData(inputTensor any, binaryData bool) error
}

InferInput is an interface that defines methods for model inference inputs.

type InferOutput

type InferOutput interface {
	// GetName returns the output name.
	GetName() string
	// GetShape returns the output shape.
	GetShape() []int64
	// GetDatatype returns the datatype of the output.
	GetDatatype() string
	// GetParameters returns the parameters of the output.
	GetParameters() map[string]any
	// GetData returns the data of the output.
	GetData() []any
	GetTensor() any
}

InferOutput interface defines methods for output data.

type InferOutputs

type InferOutputs struct {
	ModelName    string             `json:"model_name"`
	ModelVersion string             `json:"model_version"`
	Outputs      []*BaseInferOutput `json:"outputs"`
}

InferOutputs holds model name, version, and output details.

type InferResult

type InferResult interface {
	// GetOutput returns the output tensor corresponding to the specified name.
	GetOutput(name string) (InferOutput, error)
	// GetShape returns the shape of the output tensor identified by name.
	GetShape(name string) ([]int64, error)
	// AsInt8Slice returns the output data as a slice of int8 values.
	AsInt8Slice(name string) ([]int8, error)
	// AsInt16Slice returns the output data as a slice of int16 values.
	AsInt16Slice(name string) ([]int16, error)
	// AsInt32Slice returns the output data as a slice of int32 values.
	AsInt32Slice(name string) ([]int32, error)
	// AsInt64Slice returns the output data as a slice of int64 values.
	AsInt64Slice(name string) ([]int64, error)
	// AsUint8Slice returns the output data as a slice of uint8 values.
	AsUint8Slice(name string) ([]uint8, error)
	// AsUint16Slice returns the output data as a slice of uint16 values.
	AsUint16Slice(name string) ([]uint16, error)
	// AsUint32Slice returns the output data as a slice of uint32 values.
	AsUint32Slice(name string) ([]uint32, error)
	// AsUint64Slice returns the output data as a slice of uint64 values.
	AsUint64Slice(name string) ([]uint64, error)
	// AsFloat16Slice returns the output data as a slice of float16 values, converted to float64.
	AsFloat16Slice(name string) ([]float64, error)
	// AsFloat32Slice returns the output data as a slice of float32 values.
	AsFloat32Slice(name string) ([]float32, error)
	// AsFloat64Slice returns the output data as a slice of float64 values.
	AsFloat64Slice(name string) ([]float64, error)
	// AsBoolSlice returns the output data as a slice of boolean values.
	AsBoolSlice(name string) ([]bool, error)
	// AsByteSlice returns the output data as a slice of strings.
	AsByteSlice(name string) ([]string, error)
}

InferResult defines methods for retrieving various types of output data from an inference result.

type Marshaller

type Marshaller interface {
	Marshal(v any) ([]byte, error)
}

Marshaller interface defines a method for marshaling data into byte format.

type RequestWrapper

type RequestWrapper interface {
	PrepareRequest() (any, error)
}

RequestWrapper defines an interface for preparing inference requests.

type ResponseWrapper

type ResponseWrapper interface {
	GetHeader(key string) string
	GetBody() ([]byte, error)
	GetRawOutputContents() [][]byte
	GetResponse() any
}

ResponseWrapper is an interface that abstracts the differences between HTTP and gRPC responses.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL