discover

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 1, 2025 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DriverVersionFile     = "/sys/module/amdgpu/version"
	AMDNodesSysfsDir      = "/sys/class/kfd/kfd/topology/nodes/"
	GPUPropertiesFileGlob = AMDNodesSysfsDir + "*/properties"

	// Prefix with the node dir
	GPUTotalMemoryFileGlob = "mem_banks/*/properties" // size_in_bytes line

	// Direct Rendering Manager sysfs location
	DRMDeviceDirGlob   = "/sys/class/drm/card*/device"
	DRMTotalMemoryFile = "mem_info_vram_total"
	DRMUsedMemoryFile  = "mem_info_vram_used"

	// In hex; properties file is in decimal
	DRMUniqueIDFile = "unique_id"
	DRMVendorFile   = "vendor"
	DRMDeviceFile   = "device"
)
View Source 
const CpuInfoFilename = "/proc/cpuinfo"
View Source 
const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU

TODO find a better way to detect iGPU instead of minimum memory

Variables

View Source 
var (
	// Used to validate if the given ROCm lib is usable
	ROCmLibGlobs          = []string{"libhipblas.so.2*", "rocblas"} // TODO - probably include more coverage of files here...
	RocmStandardLocations = []string{"/opt/rocm/lib", "/usr/lib64"}
)
View Source 
var (
	CudaComputeMajorMin = "5"
	CudaComputeMinorMin = "0"
)

With our current CUDA compile flags, older than 5.0 will not work properly (string values used to allow ldflags overrides at build time)

View Source 
var (
	CudartMgmtName = "libcudart.so*"
	NvcudaMgmtName = "libcuda.so*"
	NvmlMgmtName   = "" // not currently wired on linux
	OneapiMgmtName = "libze_intel_gpu.so*"
)
View Source 
var CudaTegra string = os.Getenv("JETSON_JETPACK")

Jetson devices have JETSON_JETPACK="x.y.z" factory set to the Jetpack version installed. Included to drive logic for reducing Ollama-allocated overhead on L4T/Jetson devices.

View Source 
var CudartGlobs = []string{
	"/usr/local/cuda/lib64/libcudart.so*",
	"/usr/lib/x86_64-linux-gnu/nvidia/current/libcudart.so*",
	"/usr/lib/x86_64-linux-gnu/libcudart.so*",
	"/usr/lib/wsl/lib/libcudart.so*",
	"/usr/lib/wsl/drivers/*/libcudart.so*",
	"/opt/cuda/lib64/libcudart.so*",
	"/usr/local/cuda*/targets/aarch64-linux/lib/libcudart.so*",
	"/usr/lib/aarch64-linux-gnu/nvidia/current/libcudart.so*",
	"/usr/lib/aarch64-linux-gnu/libcudart.so*",
	"/usr/local/cuda/lib*/libcudart.so*",
	"/usr/lib*/libcudart.so*",
	"/usr/local/lib*/libcudart.so*",
}
View Source 
var LibOllamaPath string = func() string {
	exe, err := os.Executable()
	if err != nil {
		return ""
	}

	exe, err = filepath.EvalSymlinks(exe)
	if err != nil {
		return ""
	}

	var libPath string
	switch runtime.GOOS {
	case "windows":
		libPath = filepath.Join(filepath.Dir(exe), "lib", "ollama")
	case "linux":
		libPath = filepath.Join(filepath.Dir(exe), "..", "lib", "ollama")
	case "darwin":
		libPath = filepath.Dir(exe)
	}

	cwd, err := os.Getwd()
	if err != nil {
		return ""
	}

	paths := []string{
		libPath,

		filepath.Join(filepath.Dir(exe), "build", "lib", "ollama"),
		filepath.Join(cwd, "build", "lib", "ollama"),
	}

	for _, p := range paths {
		if _, err := os.Stat(p); err == nil {
			return p
		}
	}

	return filepath.Dir(exe)
}()

LibPath is a path to lookup dynamic libraries in development it's usually 'build/lib/ollama' in distribution builds it's 'lib/ollama' on Windows '../lib/ollama' on Linux and the executable's directory on macOS note: distribution builds, additional GPU-specific libraries are found in subdirectories of the returned path, such as 'cuda_v11', 'cuda_v12', 'rocm', etc.

View Source 
var NvcudaGlobs = []string{
	"/usr/local/cuda*/targets/*/lib/libcuda.so*",
	"/usr/lib/*-linux-gnu/nvidia/current/libcuda.so*",
	"/usr/lib/*-linux-gnu/libcuda.so*",
	"/usr/lib/wsl/lib/libcuda.so*",
	"/usr/lib/wsl/drivers/*/libcuda.so*",
	"/opt/cuda/lib*/libcuda.so*",
	"/usr/local/cuda/lib*/libcuda.so*",
	"/usr/lib*/libcuda.so*",
	"/usr/local/lib*/libcuda.so*",
}
View Source 
var NvmlGlobs = []string{}
View Source 
var OneapiGlobs = []string{
	"/usr/lib/x86_64-linux-gnu/libze_intel_gpu.so*",
	"/usr/lib*/libze_intel_gpu.so*",
}
View Source 
var RocmComputeMajorMin = "9"

Functions

func AMDDetected 

func AMDDetected() bool

Quick check for AMD driver so we can skip amdgpu discovery if not present

func AMDDriverVersion 

func AMDDriverVersion() (driverMajor, driverMinor int, err error)

func AMDValidateLibDir 

func AMDValidateLibDir() (string, error)

Prefer to use host installed ROCm, as long as it meets our minimum requirements failing that, tell the user how to download it on their own

func FindGPULibs 

func FindGPULibs(baseLibName string, defaultPatterns []string) []string

func GetCPUMem 

func GetCPUMem() (memInfo, error)

func GetSupportedGFX 

func GetSupportedGFX(libDir string) ([]string, error)

func IsNUMA 

func IsNUMA() bool

Types

type ByFreeMemory 

type ByFreeMemory []GpuInfo

Sort by Free Space

func (ByFreeMemory) Len 

func (a ByFreeMemory) Len() int

func (ByFreeMemory) Less 

func (a ByFreeMemory) Less(i, j int) bool

func (ByFreeMemory) Swap 

func (a ByFreeMemory) Swap(i, j int)

type CPU 

type CPU struct {
	ID                  string `cpuinfo:"processor"`
	VendorID            string `cpuinfo:"vendor_id"`
	ModelName           string `cpuinfo:"model name"`
	CoreCount           int
	EfficiencyCoreCount int // Performance = CoreCount - Efficiency
	ThreadCount         int
}

CPU type represents a CPU Package occupying a socket

func GetCPUDetails 

func GetCPUDetails() ([]CPU, error)

type CPUInfo 

type CPUInfo struct {
	GpuInfo
	CPUs []CPU
}

type CudaGPUInfo 

type CudaGPUInfo struct {
	GpuInfo
	OSOverhead uint64 // Memory overhead between the driver library and management library
	// contains filtered or unexported fields
}

type CudaGPUInfoList 

type CudaGPUInfoList []CudaGPUInfo

type GpuInfo 

type GpuInfo struct {
	Library string `json:"library,omitempty"`

	// Optional variant to select (e.g. versions, cpu feature flags)
	Variant string `json:"variant"`

	// MinimumMemory represents the minimum memory required to use the GPU
	MinimumMemory uint64 `json:"-"`

	// Any extra PATH/LD_LIBRARY_PATH dependencies required for the Library to operate properly
	DependencyPath []string `json:"lib_path,omitempty"`

	// Extra environment variables specific to the GPU as list of [key,value]
	EnvWorkarounds [][2]string `json:"envs,omitempty"`

	// Set to true if we can NOT reliably discover FreeMemory.  A value of true indicates
	// the FreeMemory is best effort, and may over or under report actual memory usage
	// False indicates FreeMemory can generally be trusted on this GPU
	UnreliableFreeMemory bool

	// GPU information
	ID      string `json:"gpu_id"`  // string to use for selection of this specific GPU
	Name    string `json:"name"`    // user friendly name if available
	Compute string `json:"compute"` // Compute Capability or gfx

	// Driver Information - TODO no need to put this on each GPU
	DriverMajor int `json:"driver_major,omitempty"`
	DriverMinor int `json:"driver_minor,omitempty"`
	// contains filtered or unexported fields
}

Beginning of an `ollama info` command

func (GpuInfo) RunnerName 

func (gpu GpuInfo) RunnerName() string

type GpuInfoList 

type GpuInfoList []GpuInfo

func GetCPUInfo 

func GetCPUInfo() GpuInfoList

func GetGPUInfo 

func GetGPUInfo() GpuInfoList

func (GpuInfoList) ByLibrary 

func (l GpuInfoList) ByLibrary() []GpuInfoList

Split up the set of gpu info's by Library and variant

func (GpuInfoList) FlashAttentionSupported 

func (l GpuInfoList) FlashAttentionSupported() bool

For each GPU, check if it does NOT support flash attention

func (GpuInfoList) GetVisibleDevicesEnv 

func (l GpuInfoList) GetVisibleDevicesEnv() (string, string)

Given the list of GPUs this instantiation is targeted for, figure out the visible devices environment variable

If different libraries are detected, the first one is what we use

func (GpuInfoList) LogDetails 

func (l GpuInfoList) LogDetails()

Report the GPU information into the log an Info level

type OneapiGPUInfo 

type OneapiGPUInfo struct {
	GpuInfo
	// contains filtered or unexported fields
}

type OneapiGPUInfoList 

type OneapiGPUInfoList []OneapiGPUInfo

type RocmGPUInfo 

type RocmGPUInfo struct {
	GpuInfo
	// contains filtered or unexported fields
}

func AMDGetGPUInfo 

func AMDGetGPUInfo() ([]RocmGPUInfo, error)

Gather GPU information from the amdgpu driver if any supported GPUs are detected Only called once during bootstrap

type RocmGPUInfoList 

type RocmGPUInfoList []RocmGPUInfo

func (RocmGPUInfoList) RefreshFreeMemory 

func (gpus RocmGPUInfoList) RefreshFreeMemory() error

type SystemInfo 

type SystemInfo struct {
	System          CPUInfo              `json:"system"`
	GPUs            []GpuInfo            `json:"gpus"`
	UnsupportedGPUs []UnsupportedGPUInfo `json:"unsupported_gpus"`
	DiscoveryErrors []string             `json:"discovery_errors"`
}

func GetSystemInfo 

func GetSystemInfo() SystemInfo

func (SystemInfo) GetOptimalThreadCount 

func (si SystemInfo) GetOptimalThreadCount() int

Return the optimal number of threads to use for inference

type UnsupportedGPUInfo 

type UnsupportedGPUInfo struct {
	GpuInfo
	Reason string `json:"reason"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.