quantest

package module

Go to main page

Versions in this module

v0

v0.0.10

Aug 5, 2024

v0.0.9

Aug 5, 2024

Changes in this version

+ func GetOllamaQuantLevel(modelName string) (string, error)

v0.0.8

Aug 5, 2024

Changes in this version

type ModelConfig

+ QuantLevel string

v0.0.7

Aug 5, 2024

v0.0.6

Aug 5, 2024

Changes in this version

type ModelConfig

+ IsOllama bool

+ ModelName string

+ func GetHFModelConfig(modelID string) (ModelConfig, error)

+ func GetOllamaModelConfig(modelID string) (ModelConfig, error)

type VRAMEstimation

+ ModelConfig ModelConfig

v0.0.5

Aug 5, 2024

Changes in this version

+ const CUDASize

+ const DefaultContextSize

+ const DefaultQuantLevel

+ const DefaultVRAM

+ var EXL2Options []float64

+ var GGUFMapping = map[string]float64

+ var Version string

+ func CalculateContext(modelID string, memory, bpw float64, kvCacheQuant KVCacheQuantisation, ...) (int, error)

+ func CalculateVRAM(modelID string, bpw float64, context int, kvCacheQuant KVCacheQuantisation, ...) (float64, error)

+ func CalculateVRAMRaw(config ModelConfig, bpwValues BPWValues, context int, numGPUs int, gqa bool) float64

+ func DownloadFile(url, filePath string, headers map[string]string) error

+ func GetAvailableMemory() (float64, error)

+ func GetMaxContextSize(estimation *VRAMEstimation) int

+ func GetMaximumQuant(estimation *VRAMEstimation) string

+ func GetRecommendations(estimation *VRAMEstimation) map[int]string

+ func GetSystemRAM() (float64, error)

+ func ParseBPW(bpw string) float64

+ func ParseBPWOrQuant(input string) (float64, error)

+ func PrintEstimationResults(estimation *VRAMEstimation) string

+ func PrintFormattedTable(table QuantResultTable) string

+ type BPWValues struct

+ BPW float64

+ KVCacheBPW float64

+ LMHeadBPW float64

+ func GetBPWValues(bpw float64, kvCacheQuant KVCacheQuantisation) BPWValues

+ type ContextVRAM struct

+ VRAM float64

+ VRAMQ4_0 float64

+ VRAMQ8_0 float64

+ type KVCacheQuantisation string

+ const KVCacheFP16

+ const KVCacheQ4_0

+ const KVCacheQ8_0

+ type ModelConfig struct

+ HiddenSize int

+ IntermediateSize int

+ MaxPositionEmbeddings int

+ NumAttentionHeads int

+ NumHiddenLayers int

+ NumKeyValueHeads int

+ NumParams float64

+ VocabSize int

+ func GetModelConfig(modelID string) (ModelConfig, error)

+ type OllamaModelInfo struct

+ Details struct{ ... }

+ ModelInfo struct{ ... }

+ func FetchOllamaModelInfo(modelName string) (*OllamaModelInfo, error)

+ type QuantRecommendations struct

+ Recommendations map[int]string

+ UserContext int

+ func CalculateBPW(modelID string, memory float64, context int, kvCacheQuant KVCacheQuantisation, ...) (interface{}, QuantRecommendations, error)

+ type QuantResult struct

+ BPW float64

+ Contexts map[int]ContextVRAM

+ QuantType string

+ type QuantResultTable struct

+ FitsVRAM float64

+ ModelID string

+ Results []QuantResult

+ func GenerateQuantTable(modelID string, fitsVRAM float64, ollamaModelInfo *OllamaModelInfo) (QuantResultTable, error)

+ func GenerateQuantTableForModel(modelName string, vram float64) (QuantResultTable, error)

+ type VRAMEstimation struct

+ AvailableVRAM float64

+ ContextSize int

+ EstimatedVRAM float64

+ FitsAvailable bool

+ KVCacheQuant KVCacheQuantisation

+ MaxContextSize int

+ MaximumQuant string

+ ModelName string

+ QuantLevel string

+ Recommendations map[int]string

+ func EstimateVRAM(modelName *string, contextSize int, kvCacheQuant KVCacheQuantisation, ...) (*VRAMEstimation, error)

+ func EstimateVRAMForModel(modelName string, vram float64, contextSize int, quantLevel, kvQuant string) (*VRAMEstimation, error)

v0.0.4

Aug 5, 2024

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL