gpu

Documentation

Index

• Constants
• Variables
• func AMDDetected() bool
• func AMDDriverVersion() (driverMajor, driverMinor int, err error)
• func AMDValidateLibDir() (string, error)
• func Cleanup()
• func FindGPULibs(baseLibName string, defaultPatterns []string) []string
• func GetCPUMem() (memInfo, error)
• func GetCPUVariant() string
• func GetSupportedGFX(libDir string) ([]string, error)
• func LoadCUDARTMgmt(cudartLibPaths []string) (int, *C.cudart_handle_t, string)
• func LoadNVCUDAMgmt(nvcudaLibPaths []string) (int, *C.nvcuda_handle_t, string)
• func PayloadsDir() (string, error)
• func UpdatePath(dir string)
• type ByFreeMemory
  • func (a ByFreeMemory) Len() int
  • func (a ByFreeMemory) Less(i, j int) bool
  • func (a ByFreeMemory) Swap(i, j int)
• type GpuInfo
  • func AMDGetGPUInfo() []GpuInfo
• type GpuInfoList
  • func GetGPUInfo() GpuInfoList
  • func (l GpuInfoList) ByLibrary() []GpuInfoList
  • func (l GpuInfoList) GetVisibleDevicesEnv() (string, string)
  • func (l GpuInfoList) LogDetails()

Constants

const (
	DriverVersionFile     = "/sys/module/amdgpu/version"
	AMDNodesSysfsDir      = "/sys/class/kfd/kfd/topology/nodes/"
	GPUPropertiesFileGlob = AMDNodesSysfsDir + "*/properties"

	// Prefix with the node dir
	GPUTotalMemoryFileGlob = "mem_banks/*/properties" // size_in_bytes line
	GPUUsedMemoryFileGlob  = "mem_banks/*/used_memory"
)

const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU

TODO find a better way to detect iGPU instead of minimum memory

Variables

var (
	// Used to validate if the given ROCm lib is usable
	ROCmLibGlobs          = []string{"libhipblas.so.2*", "rocblas"} // TODO - probably include more coverage of files here...
	RocmStandardLocations = []string{"/opt/rocm/lib", "/usr/lib64"}
)

var CudaComputeMin = [2]C.int{5, 0}

With our current CUDA compile flags, older than 5.0 will not work properly

var CudaTegra string = os.Getenv("JETSON_JETPACK")

Jetson devices have JETSON_JETPACK="x.y.z" factory set to the Jetpack version installed. Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.

var CudartLinuxGlobs = []string{
	"/usr/local/cuda/lib64/libcudart.so*",
	"/usr/lib/x86_64-linux-gnu/nvidia/current/libcudart.so*",
	"/usr/lib/x86_64-linux-gnu/libcudart.so*",
	"/usr/lib/wsl/lib/libcudart.so*",
	"/usr/lib/wsl/drivers/*/libcudart.so*",
	"/opt/cuda/lib64/libcudart.so*",
	"/usr/local/cuda*/targets/aarch64-linux/lib/libcudart.so*",
	"/usr/lib/aarch64-linux-gnu/nvidia/current/libcudart.so*",
	"/usr/lib/aarch64-linux-gnu/libcudart.so*",
	"/usr/local/cuda/lib*/libcudart.so*",
	"/usr/lib*/libcudart.so*",
	"/usr/local/lib*/libcudart.so*",
}

var CudartWindowsGlobs = []string{
	"c:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v*\\bin\\cudart64_*.dll",
}

var NvcudaLinuxGlobs = []string{
	"/usr/local/cuda*/targets/*/lib/libcuda.so*",
	"/usr/lib/*-linux-gnu/nvidia/current/libcuda.so*",
	"/usr/lib/*-linux-gnu/libcuda.so*",
	"/usr/lib/wsl/lib/libcuda.so*",
	"/usr/lib/wsl/drivers/*/libcuda.so*",
	"/opt/cuda/lib*/libcuda.so*",
	"/usr/local/cuda/lib*/libcuda.so*",
	"/usr/lib*/libcuda.so*",
	"/usr/local/lib*/libcuda.so*",
}

var NvcudaWindowsGlobs = []string{
	"c:\\windows\\system*\\nvcuda.dll",
}

var RocmComputeMin = 9

Functions

func AMDDetected

func AMDDetected() bool

Quick check for AMD driver so we can skip amdgpu discovery if not present

func AMDDriverVersion

func AMDDriverVersion() (driverMajor, driverMinor int, err error)

func AMDValidateLibDir

func AMDValidateLibDir() (string, error)

Prefer to use host installed ROCm, as long as it meets our minimum requirements failing that, tell the user how to download it on their own

func Cleanup

func Cleanup()

func FindGPULibs

func FindGPULibs(baseLibName string, defaultPatterns []string) []string

func GetCPUMem

func GetCPUMem() (memInfo, error)

func GetCPUVariant

func GetCPUVariant() string

func GetSupportedGFX

func GetSupportedGFX(libDir string) ([]string, error)

func LoadCUDARTMgmt

func LoadCUDARTMgmt(cudartLibPaths []string) (int, *C.cudart_handle_t, string)

func LoadNVCUDAMgmt

func LoadNVCUDAMgmt(nvcudaLibPaths []string) (int, *C.nvcuda_handle_t, string)

func PayloadsDir

func PayloadsDir() (string, error)

func UpdatePath

func UpdatePath(dir string)

Types

type ByFreeMemory

type ByFreeMemory []GpuInfo

Sort by Free Space

func (ByFreeMemory) Len

func (a ByFreeMemory) Len() int

func (ByFreeMemory) Less

func (a ByFreeMemory) Less(i, j int) bool

func (ByFreeMemory) Swap

func (a ByFreeMemory) Swap(i, j int)

type GpuInfo

type GpuInfo struct {
	Library string `json:"library,omitempty"`

	// Optional variant to select (e.g. versions, cpu feature flags)
	Variant string `json:"variant,omitempty"`

	// MinimumMemory represents the minimum memory required to use the GPU
	MinimumMemory uint64 `json:"-"`

	// Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properly
	DependencyPath string `json:"lib_path,omitempty"`

	// GPU information
	ID      string `json:"gpu_id"`  // string to use for selection of this specific GPU
	Name    string `json:"name"`    // user friendly name if available
	Compute string `json:"compute"` // Compute Capability or gfx

	// Driver Information - TODO no need to put this on each GPU
	DriverMajor int `json:"driver_major,omitempty"`
	DriverMinor int `json:"driver_minor,omitempty"`
	// contains filtered or unexported fields
}

Beginning of an `ollama info` command

func AMDGetGPUInfo

func AMDGetGPUInfo() []GpuInfo

Gather GPU information from the amdgpu driver if any supported GPUs are detected

type GpuInfoList

type GpuInfoList []GpuInfo

func GetGPUInfo

func GetGPUInfo() GpuInfoList

func (GpuInfoList) ByLibrary

func (l GpuInfoList) ByLibrary() []GpuInfoList

Split up the set of gpu info's by Library and variant

func (GpuInfoList) GetVisibleDevicesEnv

func (l GpuInfoList) GetVisibleDevicesEnv() (string, string)

Given the list of GPUs this instantiation is targeted for, figure out the visible devices environment variable

If different libraries are detected, the first one is what we use

func (GpuInfoList) LogDetails

func (l GpuInfoList) LogDetails()

Report the GPU information into the log an Info level