onnxruntime

README

go-onnxruntime

Unofficial C binding for Onnxruntime in Golang. This is used to perform onnx model inference in Go.

Installation

Download and install go-onnxruntime :

go get -v github.com/ivansuteja96/go-onnxruntime

The binding requires Onnxruntime C++ and Go 1.14++.

Onnxruntime C++ Library

The C binding for Onnxruntime in Golang. This repository is built based on Onnxruntime v1.11.0.

To install Onnxruntime C++ on your system, you can go to onnxruntime and download the assets depends on your system (linux/mac).

The Onnxruntime C++ libraries are expected to be under /usr/local/lib.

The Onnxruntime C++ header files are expected to be under /usrl/local/include.

Configure Environmental Variables

Configure the linker environmental variables since the Onnxruntime C++ library is under a non-system directory. Place the following in either your ~/.bashrc or ~/.zshrc file :

Linux (.bashrc) / macOS (.zshrc)

export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib

After that please run either source ~/.bashrc(linux) or source ~/.zshrc(macOS).

How to Run

For the quick experience you can run go-onnxruntime using this command :

    docker build --platform linux/arm64/v8 -f dockerfile/Dockerfile_ubuntu_arm64_example -t go-onnxruntime .
    docker run --rm -it  go-onnxruntime:latest 

Note : Currently we only provide Dockerfile for ubuntu arm64 architecture.

Examples

Examples of using the Go Onnxruntime binding to do model inference are under examples.

Credits

Some of the logic of conversion is referenced from https://github.com/c3sr/go-onnxruntime.

Documentation

Index

• type CudaOptions
• type CudnnConvAlgoSearch
• type ORTEnv
  • func NewORTEnv(loggingLevel ORTLoggingLevel, logEnv string) (ortEnv *ORTEnv)
  • func (o *ORTEnv) Close() error
• type ORTLoggingLevel
• type ORTSession
  • func NewORTSession(ortEnv *ORTEnv, modelLocation string, sessionOptions *ORTSessionOptions) (ortSession *ORTSession, err error)
  • func (ortSession *ORTSession) Close() error
  • func (ortSession *ORTSession) Predict(inputTensorValues []TensorValue) (result []TensorValue, err error)
• type ORTSessionOptions
  • func NewORTSessionOptions() *ORTSessionOptions
  • func (so ORTSessionOptions) AppendExecutionProviderCUDA(cudaOptions CudaOptions)
  • func (so ORTSessionOptions) Close() error
• type ORTValues
• type OnnxTensorElementDataType
• type TensorValue

Types

type CudaOptions

type CudaOptions struct {
	DeviceID              int
	CudnnConvAlgoSearch   CudnnConvAlgoSearch
	GPUMemorylimit        int
	ArenaExtendStrategy   bool
	DoCopyInDefaultStream bool
	HasUserComputeStream  bool
}

type CudnnConvAlgoSearch

type CudnnConvAlgoSearch int

const (
	OrtCudnnConvAlgoSearchExhaustive CudnnConvAlgoSearch = iota // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx
	OrtCudnnConvAlgoSearchHeuristic                             // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7
	OrtCudnnConvAlgoSearchDefault                               // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM
)

type ORTEnv

type ORTEnv struct {
	// contains filtered or unexported fields
}

func NewORTEnv

func NewORTEnv(loggingLevel ORTLoggingLevel, logEnv string) (ortEnv *ORTEnv)

NewORTEnv Create onnxruntime environment

func (*ORTEnv) Close

func (o *ORTEnv) Close() error

type ORTLoggingLevel

type ORTLoggingLevel int

const (
	ORT_LOGGING_LEVEL_VERBOSE ORTLoggingLevel = iota // Verbose informational messages (least severe).
	ORT_LOGGING_LEVEL_INFO                           // Informational messages.
	ORT_LOGGING_LEVEL_WARNING                        // Warning messages.
	ORT_LOGGING_LEVEL_ERROR                          // Error messages.
	ORT_LOGGING_LEVEL_FATAL                          // Fatal error messages (most severe).
)

type ORTSession

type ORTSession struct {
	// contains filtered or unexported fields
}

func NewORTSession

func NewORTSession(ortEnv *ORTEnv, modelLocation string, sessionOptions *ORTSessionOptions) (ortSession *ORTSession, err error)

NewORTSession return new onnxruntime session

func (*ORTSession) Close

func (ortSession *ORTSession) Close() error

func (*ORTSession) Predict

func (ortSession *ORTSession) Predict(inputTensorValues []TensorValue) (result []TensorValue, err error)

Predict do prediction from input data

type ORTSessionOptions

type ORTSessionOptions struct {
	// contains filtered or unexported fields
}

func NewORTSessionOptions

func NewORTSessionOptions() *ORTSessionOptions

NewORTSessionOptions return empty onnxruntime session options.

func (ORTSessionOptions) AppendExecutionProviderCUDA

func (so ORTSessionOptions) AppendExecutionProviderCUDA(cudaOptions CudaOptions)

AppendExecutionProviderCUDA append cuda device to the session options.

func (ORTSessionOptions) Close

func (so ORTSessionOptions) Close() error

type ORTValues

type ORTValues struct {
	// contains filtered or unexported fields
}

type OnnxTensorElementDataType

type OnnxTensorElementDataType int

const (
	ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED OnnxTensorElementDataType = iota
	ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT
	ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8
	ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8
	ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16
	ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16
	ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32
	ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64
	ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL
	ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE
	ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32
	ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64
)

type TensorValue

type TensorValue struct {
	Value interface{}
	Shape []int64
}