triton

Documentation

Index

• func DeserializeBytesTensor(encodedTensor []byte, capacity int64) []string
• func DeserializeFloat32Tensor(encodedTensor []byte) []float32
• func DeserializeInt32Tensor(encodedTensor []byte) []int32
• func GetOutputFromInferResponse(name string, response *inferenceserver.ModelInferResponse) (*inferenceserver.ModelInferResponse_InferOutputTensor, []byte, error)
• func ParseModel(modelMetadata *inferenceserver.ModelMetadataResponse, ...) (int64, int64, int64)
• func ReadFloat32(fourBytes []byte) float32
• func ReadInt32(fourBytes []byte) int32
• func Reshape1DArrayFloat32To2D(array []float32, shape []int64) ([][]float32, error)
• func Reshape1DArrayFloat32To3D(array []float32, shape []int64) ([][][]float32, error)
• func Reshape1DArrayFloat32To4D(array []float32, shape []int64) ([][][][]float32, error)
• func Reshape1DArrayInt32To2D(array []int32, shape []int64) ([][]int32, error)
• func Reshape1DArrayInt32To3D(array []int32, shape []int64) ([][][]int32, error)
• func Reshape1DArrayStringTo2D(array []string, shape []int64) ([][]string, error)
• func SerializeBytesTensor(tensor [][]byte) []byte
• type BatchUnspecifiedTaskOutputs
• type DetectionOutput
• type ImageInput
• type InferInput
• type InstanceSegmentationOutput
• type KeypointOutput
• type OcrOutput
• type SemanticSegmentationOutput
• type SingleOutputUnspecifiedTaskOutput
• type TextGenerationInput
• type TextGenerationOutput
• type TextToImageInput
• type TextToImageOutput
• type Triton
  • func NewTriton() Triton
• type UnspecifiedTaskOutput

Functions

func DeserializeBytesTensor

func DeserializeBytesTensor(encodedTensor []byte, capacity int64) []string

func DeserializeFloat32Tensor

func DeserializeFloat32Tensor(encodedTensor []byte) []float32

func DeserializeInt32Tensor

func DeserializeInt32Tensor(encodedTensor []byte) []int32

func GetOutputFromInferResponse

func GetOutputFromInferResponse(name string, response *inferenceserver.ModelInferResponse) (*inferenceserver.ModelInferResponse_InferOutputTensor, []byte, error)

func ParseModel

func ParseModel(modelMetadata *inferenceserver.ModelMetadataResponse, modelConfig *inferenceserver.ModelConfigResponse) (int64, int64, int64)

func ReadFloat32

func ReadFloat32(fourBytes []byte) float32

func ReadInt32

func ReadInt32(fourBytes []byte) int32

func Reshape1DArrayFloat32To2D

func Reshape1DArrayFloat32To2D(array []float32, shape []int64) ([][]float32, error)

func Reshape1DArrayFloat32To3D

func Reshape1DArrayFloat32To3D(array []float32, shape []int64) ([][][]float32, error)

func Reshape1DArrayFloat32To4D

func Reshape1DArrayFloat32To4D(array []float32, shape []int64) ([][][][]float32, error)

func Reshape1DArrayInt32To2D

func Reshape1DArrayInt32To2D(array []int32, shape []int64) ([][]int32, error)

func Reshape1DArrayInt32To3D

func Reshape1DArrayInt32To3D(array []int32, shape []int64) ([][][]int32, error)

func Reshape1DArrayStringTo2D

func Reshape1DArrayStringTo2D(array []string, shape []int64) ([][]string, error)

TODO: generalise reshape functions by using interface{} arguments and returned values

func SerializeBytesTensor

func SerializeBytesTensor(tensor [][]byte) []byte

Types

type BatchUnspecifiedTaskOutputs

type BatchUnspecifiedTaskOutputs struct {
	Name              string
	Shape             []int64
	DataType          string
	SerializedOutputs []interface{} // batching output
}

type DetectionOutput

type DetectionOutput struct {
	Boxes  [][][]float32
	Labels [][]string
}

type ImageInput

type ImageInput struct {
	ImgUrl    string
	ImgBase64 string
}

type InferInput

type InferInput interface{}

type InstanceSegmentationOutput

type InstanceSegmentationOutput struct {
	Rles   [][]string
	Boxes  [][][]float32
	Scores [][]float32
	Labels [][]string
}

type KeypointOutput

type KeypointOutput struct {
	Keypoints [][][][]float32
	Boxes     [][][]float32
	Scores    [][]float32
}

type OcrOutput

type OcrOutput struct {
	Boxes  [][][]float32
	Texts  [][]string
	Scores [][]float32
}

type SemanticSegmentationOutput

type SemanticSegmentationOutput struct {
	Rles       [][]string
	Categories [][]string
}

type SingleOutputUnspecifiedTaskOutput

type SingleOutputUnspecifiedTaskOutput struct {
	Name     string
	Shape    []int64
	DataType string
	Data     interface{} // batching output
}

type TextGenerationInput

type TextGenerationInput struct {
	Prompt        string
	OutputLen     int64
	BadWordsList  string
	StopWordsList string
	TopK          int64
	Seed          int64
}

type TextGenerationOutput

type TextGenerationOutput struct {
	Text []string
}

type TextToImageInput

type TextToImageInput struct {
	Prompt   string
	Steps    int64
	CfgScale float32
	Seed     int64
	Samples  int64
}

type TextToImageOutput

type TextToImageOutput struct {
	Images [][]string
}

type Triton

type Triton interface {
	ServerLiveRequest() *inferenceserver.ServerLiveResponse
	ServerReadyRequest() *inferenceserver.ServerReadyResponse
	ModelReadyRequest(modelName string, modelInstance string) *inferenceserver.ModelReadyResponse
	ModelMetadataRequest(modelName string, modelInstance string) *inferenceserver.ModelMetadataResponse
	ModelConfigRequest(modelName string, modelInstance string) *inferenceserver.ModelConfigResponse
	ModelInferRequest(task modelPB.Model_Task, inferInput InferInput, modelName string, modelInstance string, modelMetadata *inferenceserver.ModelMetadataResponse, modelConfig *inferenceserver.ModelConfigResponse) (*inferenceserver.ModelInferResponse, error)
	PostProcess(inferResponse *inferenceserver.ModelInferResponse, modelMetadata *inferenceserver.ModelMetadataResponse, task modelPB.Model_Task) (interface{}, error)
	LoadModelRequest(modelName string) (*inferenceserver.RepositoryModelLoadResponse, error)
	UnloadModelRequest(modelName string) (*inferenceserver.RepositoryModelUnloadResponse, error)
	ListModelsRequest() *inferenceserver.RepositoryIndexResponse
	IsTritonServerReady() bool
	Init()
	Close()
}

func NewTriton

func NewTriton() Triton

type UnspecifiedTaskOutput

type UnspecifiedTaskOutput struct {
	RawOutput []SingleOutputUnspecifiedTaskOutput
}