gguf_parser

Dec 9, 2024

Changes in this version

type GGMLType

+ const GGMLTypeIQ4_NL_4_8

+ const GGMLTypeIQ4_NL_8_8

type GGUFFileType

+ const GGUFFileTypeMostlyIQ4_NL_4_8

+ const GGUFFileTypeMostlyIQ4_NL_8_8

Dec 4, 2024

Changes in this version

+ func GuessFLUXDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSD1DiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSD2DiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSD35LargeDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSD35MediumDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSD3MediumDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSDXLDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

+ func GuessSDXLRefinerDiffusionModelMemoryUsage(width, height uint32, flashAttention bool) uint64

type GGMLType

+ const GGMLTypeIQ4_NL_4_4

type GGUFFileType

+ const GGUFFileTypeMostlyIQ4_NL_4_4

type GGUFRunEstimateOption

+ func WithStableDiffusionCppFreeComputeMemoryImmediately() GGUFRunEstimateOption

Nov 29, 2024

Changes in this version

type GGUFArchitecture

+ DiffusionTransformer bool

Nov 20, 2024

Changes in this version

type StableDiffusionCppRunEstimate

+ ImageOnly bool

type StableDiffusionCppRunEstimateSummary

+ ImageOnly bool

Nov 20, 2024

Changes in this version

type GGUFArchitecture

+ DiffusionArchitecture string

+ DiffusionAutoencoder *GGUFArchitectureDiffusionAutoencoder

+ DiffusionConditioners GGUFArchitectureDiffusionConditioners

+ func (ga GGUFArchitecture) DiffusionHasAutoencoder() bool

+ func (ga GGUFArchitecture) DiffusionHasConditioners() bool

+ type GGUFArchitectureDiffusionAutoencoder struct

+ Architecture string

+ FileType GGUFFileType

+ func (gaa GGUFArchitectureDiffusionAutoencoder) String() string

+ type GGUFArchitectureDiffusionConditioner struct

+ Architecture string

+ FileType GGUFFileType

+ func (gac GGUFArchitectureDiffusionConditioner) String() string

+ type GGUFArchitectureDiffusionConditioners []GGUFArchitectureDiffusionConditioner

+ func (gacs GGUFArchitectureDiffusionConditioners) String() string

type GGUFFile

+ func (gf *GGUFFile) EstimateStableDiffusionCppRun(opts ...GGUFRunEstimateOption) (e StableDiffusionCppRunEstimate)

type GGUFFileType

+ func GetFileType(cm map[GGMLType]int) GGUFFileType

type GGUFLayerTensorInfos

+ func (ltis GGUFLayerTensorInfos) GetFileType() GGUFFileType

+ func (ltis GGUFLayerTensorInfos) Match(nameRegex *regexp.Regexp) bool

+ type GGUFRunDeviceMetric struct

+ DownBandwidth BytesPerSecondScalar

+ FLOPS FLOPSScalar

+ UpBandwidth BytesPerSecondScalar

+ type GGUFRunEstimateOption func(*_GGUFRunEstimateOptions)

+ func WithLLaMACppAdapters(adp []LLaMACppRunEstimate) GGUFRunEstimateOption

+ func WithLLaMACppCacheKeyType(t GGMLType) GGUFRunEstimateOption

+ func WithLLaMACppCacheValueType(t GGMLType) GGUFRunEstimateOption

+ func WithLLaMACppContextSize(size int32) GGUFRunEstimateOption

+ func WithLLaMACppDrafter(dft *LLaMACppRunEstimate) GGUFRunEstimateOption

+ func WithLLaMACppLogicalBatchSize(size int32) GGUFRunEstimateOption

+ func WithLLaMACppOffloadLayers(layers uint64) GGUFRunEstimateOption

+ func WithLLaMACppPhysicalBatchSize(size int32) GGUFRunEstimateOption

+ func WithLLaMACppProjector(prj *LLaMACppRunEstimate) GGUFRunEstimateOption

+ func WithLLaMACppSplitMode(mode LLaMACppSplitMode) GGUFRunEstimateOption

+ func WithStableDiffusionCppAutoencoderTiling() GGUFRunEstimateOption

+ func WithStableDiffusionCppBatchCount(count int32) GGUFRunEstimateOption

+ func WithStableDiffusionCppControlNet(cn *StableDiffusionCppRunEstimate) GGUFRunEstimateOption

+ func WithStableDiffusionCppHeight(height uint32) GGUFRunEstimateOption

+ func WithStableDiffusionCppUpscaler(ups *StableDiffusionCppRunEstimate) GGUFRunEstimateOption

+ func WithStableDiffusionCppWidth(width uint32) GGUFRunEstimateOption

+ func WithinLLaMACppMaxContextSize() GGUFRunEstimateOption

+ func WithoutLLaMACppOffloadKVCache() GGUFRunEstimateOption

+ func WithoutStableDiffusionCppOffloadAutoencoder() GGUFRunEstimateOption

+ func WithoutStableDiffusionCppOffloadConditioner() GGUFRunEstimateOption

type GGUFTensorInfo

+ func (ti GGUFTensorInfo) GetFileType() GGUFFileType

+ func (ti GGUFTensorInfo) Match(nameRegex *regexp.Regexp) bool

type GGUFTensorInfos

+ func (tis GGUFTensorInfos) GetFileType() GGUFFileType

+ func (tis GGUFTensorInfos) Layers(ignores ...string) GGUFLayerTensorInfos

+ func (tis GGUFTensorInfos) Match(nameRegex *regexp.Regexp) bool

type IGGUFTensorInfos

+ GetFileType func() GGUFFileType

+ Match func(nameRegex *regexp.Regexp) bool

+ type StableDiffusionCppRunDeviceUsage struct

+ Computation GGUFBytesScalar

+ Footprint GGUFBytesScalar

+ Parameter GGUFParametersScalar

+ Position int

+ Remote bool

+ Weight GGUFBytesScalar

+ type StableDiffusionCppRunEstimate struct

+ Architecture string

+ Autoencoder *StableDiffusionCppRunEstimate

+ Conditioners []StableDiffusionCppRunEstimate

+ ControlNet *StableDiffusionCppRunEstimate

+ Devices []StableDiffusionCppRunDeviceUsage

+ Distributable bool

+ FlashAttention bool

+ FullOffloaded bool

+ NoMMap bool

+ Type string

+ Upscaler *StableDiffusionCppRunEstimate

+ func (e StableDiffusionCppRunEstimate) Summarize(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (es StableDiffusionCppRunEstimateSummary)

+ func (e StableDiffusionCppRunEstimate) SummarizeItem(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (emi StableDiffusionCppRunEstimateSummaryItem)

+ type StableDiffusionCppRunEstimateMemory struct

+ NonUMA GGUFBytesScalar

+ Position int

+ Remote bool

+ UMA GGUFBytesScalar

+ type StableDiffusionCppRunEstimateSummary struct

+ Architecture string

+ Distributable bool

+ FlashAttention bool

+ Items []StableDiffusionCppRunEstimateSummaryItem

+ NoMMap bool

+ Type string

+ type StableDiffusionCppRunEstimateSummaryItem struct

+ FullOffloaded bool

+ RAM StableDiffusionCppRunEstimateMemory

+ VRAMs []StableDiffusionCppRunEstimateMemory

Oct 17, 2024

Changes in this version

type LLaMACppRunEstimate

+ Reranking bool

type LLaMACppRunEstimateSummary

+ Reranking bool

Sep 11, 2024

Changes in this version

type GGMLType

+ const GGMLTypeTQ1_0

+ const GGMLTypeTQ2_0

type GGUFFileType

+ const GGUFFileTypeMostlyTQ1_0

+ const GGUFFileTypeMostlyTQ2_0

Sep 11, 2024

Changes in this version

type LLaMACppRunDeviceUsage

+ Position int

type LLaMACppRunEstimateMemory

+ Position int

Aug 31, 2024

Changes in this version

+ var GGUFBytesScalarStringInMiBytes bool

+ type BytesPerSecondScalar uint64

+ func ParseBytesPerSecondScalar(s string) (_ BytesPerSecondScalar, err error)

+ func (s BytesPerSecondScalar) String() string

+ type FLOPSScalar uint64

+ func ParseFLOPSScalar(s string) (_ FLOPSScalar, err error)

+ func (s FLOPSScalar) String() string

type GGUFBytesScalar

+ func ParseGGUFBytesScalar(s string) (_ GGUFBytesScalar, err error)

+ type GGUFTokensPerSecondScalar float64

+ func (s GGUFTokensPerSecondScalar) String() string

+ type LLaMACppParameterUsage struct

+ Compute GGUFParametersScalar

+ Input GGUFParametersScalar

+ KVCache GGUFParametersScalar

+ Output GGUFParametersScalar

+ type LLaMACppRunDeviceMetric struct

+ DownBandwidth BytesPerSecondScalar

+ FLOPS FLOPSScalar

+ UpBandwidth BytesPerSecondScalar

+ type LLaMACppRunDeviceUsage struct

+ Computation LLaMACppComputationMemoryUsage

+ Footprint GGUFBytesScalar

+ HandleLastLayer int

+ HandleLayers uint64

+ HandleOutputLayer bool

+ KVCache LLaMACppKVCacheMemoryUsage

+ Parameter LLaMACppParameterUsage

+ Remote bool

+ Weight LLaMACppWeightMemoryUsage

type LLaMACppRunEstimate

+ MaximumTokensPerSecond *GGUFTokensPerSecondScalar

+ func (e LLaMACppRunEstimate) SummarizeItem(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (emi LLaMACppRunEstimateSummaryItem)

+ type LLaMACppRunEstimateMemory struct

+ HandleLastLayer int

+ HandleLayers uint64

+ HandleOutputLayer bool

+ NonUMA GGUFBytesScalar

+ Remote bool

+ UMA GGUFBytesScalar

type LLaMACppRunEstimateOption

+ func WithDeviceMetrics(metrics []LLaMACppRunDeviceMetric) LLaMACppRunEstimateOption

+ type LLaMACppRunEstimateSummary struct

+ Architecture string

+ ContextSize uint64

+ Distributable bool

+ EmbeddingOnly bool

+ FlashAttention bool

+ Items []LLaMACppRunEstimateSummaryItem

+ LogicalBatchSize int32

+ NoMMap bool

+ PhysicalBatchSize int32

+ Type string

+ type LLaMACppRunEstimateSummaryItem struct

+ FullOffloaded bool

+ MaximumTokensPerSecond *GGUFTokensPerSecondScalar

+ OffloadLayers uint64

+ RAM LLaMACppRunEstimateMemory

+ VRAMs []LLaMACppRunEstimateMemory

+ type SizeScalar uint64

+ func ParseSizeScalar(s string) (_ SizeScalar, err error)

+ func (s SizeScalar) String() string

Aug 28, 2024

Changes in this version

type GGUFFile

+ func (gf *GGUFFile) EstimateLLaMACppRun(opts ...LLaMACppRunEstimateOption) (e LLaMACppRunEstimate)

+ type LLaMACppComputationMemoryUsage struct

+ Compute GGUFBytesScalar

+ Footprint GGUFBytesScalar

+ Input GGUFBytesScalar

+ Output GGUFBytesScalar

+ func (u LLaMACppComputationMemoryUsage) Sum() GGUFBytesScalar

+ type LLaMACppKVCacheMemoryUsage struct

+ Key GGUFBytesScalar

+ Value GGUFBytesScalar

+ func (u LLaMACppKVCacheMemoryUsage) Sum() GGUFBytesScalar

type LLaMACppMemoryUsage

+ HandleLastLayer int

+ HandleOutputLayer bool

+ type LLaMACppRunEstimate struct

+ Adapters []LLaMACppRunEstimate

+ Architecture string

+ ContextSize uint64

+ Devices []LLaMACppMemoryUsage

+ Distributable bool

+ Drafter *LLaMACppRunEstimate

+ EmbeddingOnly bool

+ FlashAttention bool

+ FullOffloaded bool

+ LogicalBatchSize int32

+ NoMMap bool

+ OffloadLayers uint64

+ PhysicalBatchSize int32

+ Projector *LLaMACppRunEstimate

+ Type string

+ func (e LLaMACppRunEstimate) Summarize(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (es LLaMACppUsageEstimateSummary)

+ func (e LLaMACppRunEstimate) SummarizeMemory(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (ems LLaMACppUsageEstimateMemorySummary)

+ type LLaMACppRunEstimateOption func(*_LLaMACppRunEstimateOptions)

type LLaMACppUsageEstimateMemoryDetail

+ HandleLastLayer int

+ HandleLayers uint64

+ HandleOutputLayer bool

+ Remote bool

+ type LLaMACppWeightMemoryUsage struct

+ Compute GGUFBytesScalar

+ Input GGUFBytesScalar

+ Output GGUFBytesScalar

+ func (u LLaMACppWeightMemoryUsage) Sum() GGUFBytesScalar

Aug 26, 2024

Changes in this version

+ type GGUFArchitecture struct

+ AdapterControlVectorLayerCount uint32

+ AdapterLoRAAlpha float32

+ AdapterType string

+ Architecture string

+ AttentionCausal bool

+ AttentionClampKQV float32

+ AttentionHeadCount uint64

+ AttentionHeadCountKV uint64

+ AttentionKeyLength uint32

+ AttentionLayerNormEpsilon float32

+ AttentionLayerNormRMSEpsilon float32

+ AttentionMaxALiBIBias float32

+ AttentionValueLength uint32

+ BlockCount uint64

+ ClipHasTextEncoder bool

+ ClipHasVisionEncoder bool

+ ClipProjectorType string

+ EmbeddingGQA uint64

+ EmbeddingKeyGQA uint64

+ EmbeddingLength uint64

+ EmbeddingValueGQA uint64

+ ExpertCount uint32

+ ExpertFeedForwardLength uint64

+ ExpertSharedFeedForwardLength uint64

+ ExpertUsedCount uint32

+ FeedForwardLength uint64

+ MaximumContextLength uint64

+ RoPEDimensionCount uint64

+ RoPEFrequencyBase float32

+ RoPEScalingFactor float32

+ RoPEScalingFinetuned bool

+ RoPEScalingOriginalContextLength uint64

+ RoPEScalingType string

+ SSMConvolutionKernel uint32

+ SSMInnerSize uint32

+ SSMStateSize uint32

+ SSMTimeStepRank uint32

+ Type string

+ VocabularyLength uint64

type GGUFFile

+ func (gf *GGUFFile) Metadata() (gm GGUFMetadata)

+ type GGUFMetadata struct

+ Alignment uint32

+ Architecture string

+ Author string

+ BitsPerWeight GGUFBitsPerWeightScalar

+ Description string

+ FileSize GGUFBytesScalar

+ FileType GGUFFileType

+ License string

+ LittleEndian bool

+ Name string

+ Parameters GGUFParametersScalar

+ QuantizationVersion uint32

+ Size GGUFBytesScalar

+ Type string

+ URL string

+ type GGUFTokenizer struct

+ AddedTokensLength uint64

+ BOSTokenID int64

+ EOMTokenID int64

+ EOSTokenID int64

+ EOTTokenID int64

+ MergesLength uint64

+ MergesSize int64

+ Model string

+ PaddingTokenID int64

+ SeparatorTokenID int64

+ TokensLength uint64

+ TokensSize int64

+ UnknownTokenID int64

type LLaMACppMemoryUsage

+ Remote bool

type LLaMACppUsageEstimate

+ Adapters []LLaMACppUsageEstimate

+ Projector *LLaMACppUsageEstimate

+ Type string

type LLaMACppUsageEstimateOption

+ func WithAdapters(adp []LLaMACppUsageEstimate) LLaMACppUsageEstimateOption

+ func WithProjector(prj *LLaMACppUsageEstimate) LLaMACppUsageEstimateOption

+ func WithRPCServers(srvs []string) LLaMACppUsageEstimateOption

type LLaMACppUsageEstimateSummary

+ Type string

Aug 23, 2024

Changes in this version

type LLaMACppUsageEstimate

+ Distributable bool

type LLaMACppUsageEstimateSummary

+ Distributable bool

Aug 21, 2024

Changes in this version

+ var ShardGGUFFilenameRegex = regexp.MustCompile(`^(?P<Prefix>.*)-(?:(?P<Shard>\d{5})-of-(?P<ShardTotal>\d{5}))\.gguf$`)

+ func CompleteShardGGUFFilename(name string) []string

+ func DefaultCachePath() string

+ func IsShardGGUFFilename(name string) bool

type GGUFFile

+ SplitModelSizes []GGUFBytesScalar

+ SplitPaddings []int64

+ SplitSizes []GGUFBytesScalar

+ SplitTensorDataStartOffsets []int64

type GGUFFilename

+ BaseName string

+ Encoding string

+ FineTune string

+ SizeLabel string

+ Type string

+ Version string

+ func (gn GGUFFilename) IsShard() bool

Aug 20, 2024

Changes in this version

type LLaMACppMemoryUsage

+ HandleLayers uint64

+ LastLayer int

+ type LLaMACppSplitMode uint

+ const LLaMACppSplitModeLayer

+ const LLaMACppSplitModeNone

+ const LLaMACppSplitModeRow

type LLaMACppUsageEstimate

+ Devices []LLaMACppMemoryUsage

+ type LLaMACppUsageEstimateMemoryDetail struct

+ NonUMA GGUFBytesScalar

+ UMA GGUFBytesScalar

type LLaMACppUsageEstimateMemorySummary

+ RAM LLaMACppUsageEstimateMemoryDetail

+ VRAMs []LLaMACppUsageEstimateMemoryDetail

type LLaMACppUsageEstimateOption

+ func WithMainGPUIndex(di int) LLaMACppUsageEstimateOption

+ func WithSplitMode(mode LLaMACppSplitMode) LLaMACppUsageEstimateOption

+ func WithTensorSplitFraction(fractions []float64) LLaMACppUsageEstimateOption

Aug 21, 2024

Changes in this version

+ type OllamaModelOption func(*_OllamaModelOptions)

+ func SetOllamaModelBaseURL(baseURL string) OllamaModelOption

+ func SetOllamaModelDefaultNamespace(namespace string) OllamaModelOption

+ func SetOllamaModelDefaultRegistry(registry string) OllamaModelOption

+ func SetOllamaModelDefaultScheme(scheme string) OllamaModelOption

+ func SetOllamaModelDefaultTag(tag string) OllamaModelOption

Aug 7, 2024

Changes in this version

+ func OllamaRegistryAuthorize(ctx context.Context, cli *http.Client, authnToken string) (string, error)

+ func OllamaRegistryAuthorizeRetry(cli *http.Client, resp *http.Response) bool

+ func OllamaSingKeyLoad() (ssh.Signer, error)

+ func OllamaUserAgent() string

Aug 6, 2024

Changes in this version

type LLaMACppUsageEstimate

+ LogicalBatchSize int32

+ PhysicalBatchSize int32

type LLaMACppUsageEstimateSummary

+ LogicalBatchSize int32

+ PhysicalBatchSize int32

Aug 6, 2024

Changes in this version

type LLaMACppUsageEstimateOption

+ func WithLogicalBatchSize(size int32) LLaMACppUsageEstimateOption

Aug 5, 2024

Changes in this version

type GGUFTokenizerMetadata

+ EOMTokenID int64

+ EOTTokenID int64

Jul 30, 2024

Changes in this version

type GGUFArchitectureMetadata

+ AttentionCausal bool

type LLaMACppUsageEstimate

+ EmbeddingOnly bool

type LLaMACppUsageEstimateSummary

+ EmbeddingOnly bool

Jul 25, 2024

Changes in this version

+ const GGMLComputationGraphNodesDefault

+ const GGMLComputationGraphNodesMaximum

+ const GGMLComputationGraphSize

+ const GGMLObjectSize

+ const GGMLTensorSize

+ const OllamaDefaultNamespace

+ const OllamaDefaultRegistry

+ const OllamaDefaultScheme

+ const OllamaDefaultTag

+ var ErrGGUFFileCacheCorrupted = errors.New("GGUF file cache corrupted")

+ var ErrGGUFFileCacheDisabled = errors.New("GGUF file cache disabled")

+ var ErrGGUFFileCacheMissed = errors.New("GGUF file cache missed")

+ var ErrGGUFFileInvalidFormat = errors.New("invalid GGUF format")

+ var ErrOllamaBaseLayerNotFound = errors.New("ollama base layer not found")

+ var ErrOllamaInvalidModel = errors.New("ollama invalid model")

+ var GGUFFilenameRegex = regexp.MustCompile(...)

+ var InMiBytes bool

+ func GGMLComputationGraphOverhead(nodes uint64, grads bool) uint64

+ func GGMLHashSize(base uint64) uint64

+ func GGMLMemoryPadding(size uint64) uint64

+ func GGMLPadding(size, align uint64) uint64

+ func GGMLTensorOverhead() uint64

+ func ValueNumeric[T constraints.Integer | constraints.Float](kv GGUFMetadataKV) T

+ func ValuesNumeric[T constraints.Integer | constraints.Float](av GGUFMetadataKVArrayValue) []T

+ type GGMLType uint32

+ const GGMLTypeBF16

+ const GGMLTypeF16

+ const GGMLTypeF32

+ const GGMLTypeF64

+ const GGMLTypeI16

+ const GGMLTypeI32

+ const GGMLTypeI64

+ const GGMLTypeI8

+ const GGMLTypeIQ1_M

+ const GGMLTypeIQ1_S

+ const GGMLTypeIQ2_S

+ const GGMLTypeIQ2_XS

+ const GGMLTypeIQ2_XXS

+ const GGMLTypeIQ3_S

+ const GGMLTypeIQ3_XXS

+ const GGMLTypeIQ4_NL

+ const GGMLTypeIQ4_XS

+ const GGMLTypeQ2_K

+ const GGMLTypeQ3_K

+ const GGMLTypeQ4_0

+ const GGMLTypeQ4_0_4_4

+ const GGMLTypeQ4_0_4_8

+ const GGMLTypeQ4_0_8_8

+ const GGMLTypeQ4_1

+ const GGMLTypeQ4_2

+ const GGMLTypeQ4_3

+ const GGMLTypeQ4_K

+ const GGMLTypeQ5_0

+ const GGMLTypeQ5_1

+ const GGMLTypeQ5_K

+ const GGMLTypeQ6_K

+ const GGMLTypeQ8_0

+ const GGMLTypeQ8_1

+ const GGMLTypeQ8_K

+ func (i GGMLType) String() string

+ func (t GGMLType) RowSizeOf(dimensions []uint64) uint64

+ func (t GGMLType) Trait() (GGMLTypeTrait, bool)

+ type GGMLTypeTrait struct

+ BlockSize uint64

+ Quantized bool

+ TypeSize uint64

+ type GGUFArchitectureMetadata struct

+ Architecture string

+ AttentionClampKQV float32

+ AttentionHeadCount uint64

+ AttentionHeadCountKV uint64

+ AttentionKeyLength uint32

+ AttentionLayerNormEpsilon float32

+ AttentionLayerNormRMSEpsilon float32

+ AttentionMaxALiBIBias float32

+ AttentionValueLength uint32

+ BlockCount uint64

+ ClipHasLLaVaProjector bool

+ ClipHasTextEncoder bool

+ ClipHasVisionEncoder bool

+ ClipProjectorType string

+ EmbeddingGQA uint64

+ EmbeddingKeyGQA uint64

+ EmbeddingLength uint64

+ EmbeddingValueGQA uint64

+ ExpertCount uint32

+ ExpertFeedForwardLength uint64

+ ExpertSharedFeedForwardLength uint64

+ ExpertUsedCount uint32

+ FeedForwardLength uint64

+ MaximumContextLength uint64

+ RoPEDimensionCount uint64

+ RoPEFrequencyBase float32

+ RoPEScalingFactor float32

+ RoPEScalingFinetuned bool

+ RoPEScalingOriginalContextLength uint64

+ RoPEScalingType string

+ SSMConvolutionKernel uint32

+ SSMInnerSize uint32

+ SSMStateSize uint32

+ SSMTimeStepRank uint32

+ VocabularyLength uint64

+ type GGUFBitsPerWeightScalar float64

+ func (s GGUFBitsPerWeightScalar) String() string

+ type GGUFBytesScalar uint64

+ func (s GGUFBytesScalar) String() string

+ type GGUFFile struct

+ Header GGUFHeader

+ ModelBitsPerWeight GGUFBitsPerWeightScalar

+ ModelParameters GGUFParametersScalar

+ ModelSize GGUFBytesScalar

+ Padding int64

+ Size GGUFBytesScalar

+ TensorDataStartOffset int64

+ TensorInfos GGUFTensorInfos

+ func ParseGGUFFile(path string, opts ...GGUFReadOption) (*GGUFFile, error)

+ func ParseGGUFFileFromHuggingFace(ctx context.Context, repo, file string, opts ...GGUFReadOption) (*GGUFFile, error)

+ func ParseGGUFFileFromModelScope(ctx context.Context, repo, file string, opts ...GGUFReadOption) (*GGUFFile, error)

+ func ParseGGUFFileFromOllama(ctx context.Context, model string, opts ...GGUFReadOption) (*GGUFFile, error)

+ func ParseGGUFFileFromOllamaModel(ctx context.Context, model *OllamaModel, opts ...GGUFReadOption) (gf *GGUFFile, err error)

+ func ParseGGUFFileRemote(ctx context.Context, url string, opts ...GGUFReadOption) (gf *GGUFFile, err error)

+ func (gf *GGUFFile) Architecture() (ga GGUFArchitectureMetadata)

+ func (gf *GGUFFile) EstimateLLaMACppUsage(opts ...LLaMACppUsageEstimateOption) (e LLaMACppUsageEstimate)

+ func (gf *GGUFFile) Layers(ignores ...string) GGUFLayerTensorInfos

+ func (gf *GGUFFile) Model() (gm GGUFModelMetadata)

+ func (gf *GGUFFile) Tokenizer() (gt GGUFTokenizerMetadata)

+ type GGUFFileCache string

+ func (c GGUFFileCache) Delete(key string) error

+ func (c GGUFFileCache) Get(key string, exp time.Duration) (*GGUFFile, error)

+ func (c GGUFFileCache) Put(key string, gf *GGUFFile) error

+ type GGUFFileType uint32

+ const GGUFFileTypeAllF32

+ const GGUFFileTypeMostlyBF16

+ const GGUFFileTypeMostlyF16

+ const GGUFFileTypeMostlyIQ1_M

+ const GGUFFileTypeMostlyIQ1_S

+ const GGUFFileTypeMostlyIQ2_S

+ const GGUFFileTypeMostlyIQ2_XS

+ const GGUFFileTypeMostlyIQ2_XXS

+ const GGUFFileTypeMostlyIQ3_S

+ const GGUFFileTypeMostlyIQ3_XXS

+ const GGUFFileTypeMostlyIQ4_NL

+ const GGUFFileTypeMostlyIQ4_XS

+ const GGUFFileTypeMostlyQ2_K

+ const GGUFFileTypeMostlyQ3_K

+ const GGUFFileTypeMostlyQ4_0

+ const GGUFFileTypeMostlyQ4_0_4_4

+ const GGUFFileTypeMostlyQ4_0_4_8

+ const GGUFFileTypeMostlyQ4_0_8_8

+ const GGUFFileTypeMostlyQ4_1

+ const GGUFFileTypeMostlyQ4_1_F16

+ const GGUFFileTypeMostlyQ4_2

+ const GGUFFileTypeMostlyQ4_3

+ const GGUFFileTypeMostlyQ4_K

+ const GGUFFileTypeMostlyQ5_0

+ const GGUFFileTypeMostlyQ5_1

+ const GGUFFileTypeMostlyQ5_K

+ const GGUFFileTypeMostlyQ6_K

+ const GGUFFileTypeMostlyQ8_0

+ func (i GGUFFileType) String() string

+ func (t GGUFFileType) GGMLType() GGMLType

+ type GGUFFilename struct

+ EncodingScheme string

+ ExpertsCount *int

+ Major *int

+ Minor *int

+ ModelName string

+ Parameters string

+ Shard *int

+ ShardTotal *int

+ func ParseGGUFFilename(name string) *GGUFFilename

+ func (gn GGUFFilename) IsPreRelease() bool

+ func (gn GGUFFilename) IsSharding() bool

+ func (gn GGUFFilename) String() string

+ type GGUFHeader struct

+ Magic GGUFMagic

+ MetadataKV GGUFMetadataKVs

+ MetadataKVCount uint64

+ TensorCount uint64

+ Version GGUFVersion

+ type GGUFLayerTensorInfos []IGGUFTensorInfos

+ func (ltis GGUFLayerTensorInfos) Bytes() uint64

+ func (ltis GGUFLayerTensorInfos) Count() uint64

+ func (ltis GGUFLayerTensorInfos) Cut(names []string) (before, after GGUFLayerTensorInfos, found bool)

+ func (ltis GGUFLayerTensorInfos) Elements() uint64

+ func (ltis GGUFLayerTensorInfos) Get(name string) (info GGUFTensorInfo, found bool)

+ func (ltis GGUFLayerTensorInfos) Index(names []string) (infos map[string]GGUFTensorInfo, found int)

+ func (ltis GGUFLayerTensorInfos) Search(nameRegex *regexp.Regexp) (infos []GGUFTensorInfo)

+ type GGUFMagic uint32

+ const GGUFMagicGGJT

+ const GGUFMagicGGMF

+ const GGUFMagicGGML

+ const GGUFMagicGGUFBe

+ const GGUFMagicGGUFLe

+ func (i GGUFMagic) String() string

+ type GGUFMetadataKV struct

+ Key string

+ Value any

+ ValueType GGUFMetadataValueType

+ func (kv GGUFMetadataKV) ValueArray() GGUFMetadataKVArrayValue

+ func (kv GGUFMetadataKV) ValueBool() bool

+ func (kv GGUFMetadataKV) ValueFloat32() float32

+ func (kv GGUFMetadataKV) ValueFloat64() float64

+ func (kv GGUFMetadataKV) ValueInt16() int16

+ func (kv GGUFMetadataKV) ValueInt32() int32

+ func (kv GGUFMetadataKV) ValueInt64() int64

+ func (kv GGUFMetadataKV) ValueInt8() int8

+ func (kv GGUFMetadataKV) ValueString() string

+ func (kv GGUFMetadataKV) ValueUint16() uint16

+ func (kv GGUFMetadataKV) ValueUint32() uint32

+ func (kv GGUFMetadataKV) ValueUint64() uint64

+ func (kv GGUFMetadataKV) ValueUint8() uint8

+ type GGUFMetadataKVArrayValue struct

+ Array []any

+ Len uint64

+ Size int64

+ StartOffset int64

+ Type GGUFMetadataValueType

+ func (av GGUFMetadataKVArrayValue) ValuesArray() []GGUFMetadataKVArrayValue

+ func (av GGUFMetadataKVArrayValue) ValuesBool() []bool

+ func (av GGUFMetadataKVArrayValue) ValuesFloat32() []float32

+ func (av GGUFMetadataKVArrayValue) ValuesFloat64() []float64

+ func (av GGUFMetadataKVArrayValue) ValuesInt16() []int16

+ func (av GGUFMetadataKVArrayValue) ValuesInt32() []int32

+ func (av GGUFMetadataKVArrayValue) ValuesInt64() []int64

+ func (av GGUFMetadataKVArrayValue) ValuesInt8() []int8

+ func (av GGUFMetadataKVArrayValue) ValuesString() []string

+ func (av GGUFMetadataKVArrayValue) ValuesUint16() []uint16

+ func (av GGUFMetadataKVArrayValue) ValuesUint32() []uint32

+ func (av GGUFMetadataKVArrayValue) ValuesUint64() []uint64

+ func (av GGUFMetadataKVArrayValue) ValuesUint8() []uint8

+ type GGUFMetadataKVs []GGUFMetadataKV

+ func (kvs GGUFMetadataKVs) Get(key string) (value GGUFMetadataKV, found bool)

+ func (kvs GGUFMetadataKVs) Index(keys []string) (values map[string]GGUFMetadataKV, found int)

+ func (kvs GGUFMetadataKVs) Search(keyRegex *regexp.Regexp) (values []GGUFMetadataKV)

+ type GGUFMetadataValueType uint32

+ const GGUFMetadataValueTypeArray

+ const GGUFMetadataValueTypeBool

+ const GGUFMetadataValueTypeFloat32

+ const GGUFMetadataValueTypeFloat64

+ const GGUFMetadataValueTypeInt16

+ const GGUFMetadataValueTypeInt32

+ const GGUFMetadataValueTypeInt64

+ const GGUFMetadataValueTypeInt8

+ const GGUFMetadataValueTypeString

+ const GGUFMetadataValueTypeUint16

+ const GGUFMetadataValueTypeUint32

+ const GGUFMetadataValueTypeUint64

+ const GGUFMetadataValueTypeUint8

+ func (i GGUFMetadataValueType) String() string

+ type GGUFModelMetadata struct

+ Alignment uint32

+ Architecture string

+ Author string

+ BitsPerWeight GGUFBitsPerWeightScalar

+ Description string

+ FileSize GGUFBytesScalar

+ FileType GGUFFileType

+ License string

+ LittleEndian bool

+ Name string

+ Parameters GGUFParametersScalar

+ QuantizationVersion uint32

+ Size GGUFBytesScalar

+ URL string

+ type GGUFNamedTensorInfos struct

+ Name string

+ type GGUFParametersScalar uint64

+ func (s GGUFParametersScalar) String() string

+ type GGUFReadOption func(o *_GGUFReadOptions)

+ func SkipCache() GGUFReadOption

+ func SkipDNSCache() GGUFReadOption

+ func SkipLargeMetadata() GGUFReadOption

+ func SkipProxy() GGUFReadOption

+ func SkipRangeDownloadDetection() GGUFReadOption

+ func SkipTLSVerification() GGUFReadOption

+ func UseBearerAuth(token string) GGUFReadOption

+ func UseBufferSize(size int) GGUFReadOption

+ func UseCache() GGUFReadOption

+ func UseCacheExpiration(expiration time.Duration) GGUFReadOption

+ func UseCachePath(path string) GGUFReadOption

+ func UseDebug() GGUFReadOption

+ func UseMMap() GGUFReadOption

+ func UseProxy(url *url.URL) GGUFReadOption

+ type GGUFTensorInfo struct

+ Dimensions []uint64

+ NDimensions uint32

+ Name string

+ Offset uint64

+ StartOffset int64

+ Type GGMLType

+ func (ti GGUFTensorInfo) Bytes() uint64

+ func (ti GGUFTensorInfo) Count() uint64

+ func (ti GGUFTensorInfo) Elements() uint64

+ func (ti GGUFTensorInfo) Get(name string) (info GGUFTensorInfo, found bool)

+ func (ti GGUFTensorInfo) Index(names []string) (infos map[string]GGUFTensorInfo, found int)

+ func (ti GGUFTensorInfo) Search(nameRegex *regexp.Regexp) (infos []GGUFTensorInfo)

+ type GGUFTensorInfos []GGUFTensorInfo

+ func (tis GGUFTensorInfos) Bytes() uint64

+ func (tis GGUFTensorInfos) Count() uint64

+ func (tis GGUFTensorInfos) Elements() uint64

+ func (tis GGUFTensorInfos) Get(name string) (info GGUFTensorInfo, found bool)

+ func (tis GGUFTensorInfos) Index(names []string) (infos map[string]GGUFTensorInfo, found int)

+ func (tis GGUFTensorInfos) Search(nameRegex *regexp.Regexp) (infos []GGUFTensorInfo)

+ type GGUFTokenizerMetadata struct

+ AddedTokensLength uint64

+ BOSTokenID int64

+ EOSTokenID int64

+ MergesLength uint64

+ MergesSize int64

+ Model string

+ PaddingTokenID int64

+ SeparatorTokenID int64

+ TokensLength uint64

+ TokensSize int64

+ UnknownTokenID int64

+ type GGUFVersion uint32

+ const GGUFVersionV1

+ const GGUFVersionV2

+ const GGUFVersionV3

+ func (i GGUFVersion) String() string

+ type IGGUFTensorInfos interface

+ Bytes func() uint64

+ Count func() uint64

+ Elements func() uint64

+ Get func(name string) (info GGUFTensorInfo, found bool)

+ Index func(names []string) (infos map[string]GGUFTensorInfo, found int)

+ Search func(nameRegex *regexp.Regexp) (infos []GGUFTensorInfo)

+ type LLaMACppComputationUsage struct

+ Compute GGUFBytesScalar

+ Footprint GGUFBytesScalar

+ Input GGUFBytesScalar

+ Output GGUFBytesScalar

+ func (u LLaMACppComputationUsage) Sum() GGUFBytesScalar

+ type LLaMACppKVCacheUsage struct

+ Key GGUFBytesScalar

+ Value GGUFBytesScalar

+ func (u LLaMACppKVCacheUsage) Sum() GGUFBytesScalar

+ type LLaMACppMemoryUsage struct

+ Computation LLaMACppComputationUsage

+ Footprint GGUFBytesScalar

+ KVCache LLaMACppKVCacheUsage

+ Weight LLaMACppWeightUsage

+ type LLaMACppUsageEstimate struct

+ Architecture string

+ ContextSize uint64

+ Drafter *LLaMACppUsageEstimate

+ FlashAttention bool

+ FullOffloaded bool

+ Load LLaMACppMemoryUsage

+ MultimodalProjector *LLaMACppUsageEstimate

+ NoMMap bool

+ Offload LLaMACppMemoryUsage

+ OffloadLayers uint64

+ func (e LLaMACppUsageEstimate) Summarize(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (es LLaMACppUsageEstimateSummary)

+ func (e LLaMACppUsageEstimate) SummarizeMemory(mmap bool, nonUMARamFootprint, nonUMAVramFootprint uint64) (ems LLaMACppUsageEstimateMemorySummary)

+ type LLaMACppUsageEstimateMemorySummary struct

+ FullOffloaded bool

+ NonUMA struct{ ... }

+ OffloadLayers uint64

+ UMA struct{ ... }

+ type LLaMACppUsageEstimateOption func(*_LLaMACppUsageEstimateOptions)

+ func WithArchitecture(arch GGUFArchitectureMetadata) LLaMACppUsageEstimateOption

+ func WithCacheKeyType(t GGMLType) LLaMACppUsageEstimateOption

+ func WithCacheValueType(t GGMLType) LLaMACppUsageEstimateOption

+ func WithContextSize(size int32) LLaMACppUsageEstimateOption

+ func WithDrafter(dft *LLaMACppUsageEstimate) LLaMACppUsageEstimateOption

+ func WithFlashAttention() LLaMACppUsageEstimateOption

+ func WithMultimodalProjector(mmp *LLaMACppUsageEstimate) LLaMACppUsageEstimateOption

+ func WithOffloadLayers(layers uint64) LLaMACppUsageEstimateOption

+ func WithParallelSize(size int32) LLaMACppUsageEstimateOption

+ func WithPhysicalBatchSize(size int32) LLaMACppUsageEstimateOption

+ func WithTokenizer(tokenizer GGUFTokenizerMetadata) LLaMACppUsageEstimateOption

+ func WithinMaxContextSize() LLaMACppUsageEstimateOption

+ func WithoutOffloadKVCache() LLaMACppUsageEstimateOption

+ type LLaMACppUsageEstimateSummary struct

+ Architecture string

+ ContextSize uint64

+ FlashAttention bool

+ Memory []LLaMACppUsageEstimateMemorySummary

+ NoMMap bool

+ type LLaMACppWeightUsage struct

+ Compute GGUFBytesScalar

+ Input GGUFBytesScalar

+ Output GGUFBytesScalar

+ func (u LLaMACppWeightUsage) Sum() GGUFBytesScalar

+ type OllamaModel struct

+ Client *http.Client

+ Config OllamaModelLayer

+ Layers []OllamaModelLayer

+ MediaType string

+ Namespace string

+ Registry string

+ Repository string

+ Schema string

+ SchemaVersion uint32

+ Tag string

+ func ParseOllamaModel(model string) *OllamaModel

+ func (om *OllamaModel) Complete(ctx context.Context, cli *http.Client) error

+ func (om *OllamaModel) GetLayer(mediaType string) (OllamaModelLayer, bool)

+ func (om *OllamaModel) License(ctx context.Context, cli *http.Client) ([]string, error)

+ func (om *OllamaModel) Messages(ctx context.Context, cli *http.Client) ([]json.RawMessage, error)

+ func (om *OllamaModel) Params(ctx context.Context, cli *http.Client) (map[string]any, error)

+ func (om *OllamaModel) SearchLayers(mediaTypeRegex *regexp.Regexp) []OllamaModelLayer

+ func (om *OllamaModel) String() string

+ func (om *OllamaModel) System(ctx context.Context, cli *http.Client) (string, error)

+ func (om *OllamaModel) Template(ctx context.Context, cli *http.Client) (string, error)

+ func (om *OllamaModel) WebPageURL() *url.URL

+ type OllamaModelLayer struct

+ Digest string

+ MediaType string

+ Root *OllamaModel

+ Size uint64

+ func (ol *OllamaModelLayer) BlobURL() *url.URL

+ func (ol *OllamaModelLayer) FetchBlob(ctx context.Context, cli *http.Client) ([]byte, error)

+ func (ol *OllamaModelLayer) FetchBlobFunc(ctx context.Context, cli *http.Client, process func(*http.Response) error) error