llm

package

Go to main page

Versions in this module

v0

v0.1.39

May 23, 2024

Changes in this version

type ServerStatusResp

+ Progress float32

v0.1.38

May 14, 2024

Changes in this version

type LlamaServer

+ EstimatedTotal func() uint64

v0.1.37

May 11, 2024

v0.1.36

May 11, 2024

v0.1.35

May 10, 2024

Changes in this version

+ func DetectGGMLType(b []byte) string

+ func ParseFileType(s string) (fileType, error)

type CompletionResponse

+ DoneReason string

v0.1.34

May 7, 2024

Changes in this version

type ServerStatus

+ const ServerStatusNoSlotsAvailable

v0.1.34-rc1

May 6, 2024

v0.1.33

May 2, 2024

Changes in this version

+ func EstimateGPULayers(gpus []gpu.GpuInfo, ggml *GGML, projectors []string, opts api.Options) (int, uint64)

+ func PredictServerFit(allGpus gpu.GpuInfoList, ggml *GGML, adapters, projectors []string, ...) (bool, uint64)

type GGML

+ func LoadModel(model string) (*GGML, error)

type LlamaServer

+ EstimatedVRAM func() uint64

+ WaitUntilRunning func(ctx context.Context) error

type ServerStatus

+ func (s ServerStatus) ToString() string

v0.1.33-rc7

May 1, 2024

v0.1.33-rc5

Apr 26, 2024

v0.1.33-rc2

Apr 26, 2024

v0.1.32

Apr 14, 2024

Changes in this version

+ func NewGGUFV3(bo binary.ByteOrder) *gguf

+ func Quantize(infile, outfile, filetype string) error

+ func SystemInfo() string

+ type CompletionRequest struct

+ Format string

+ Images []ImageData

+ Options api.Options

+ Prompt string

+ type CompletionResponse struct

+ Content string

+ Done bool

+ EvalCount int

+ EvalDuration time.Duration

+ PromptEvalCount int

+ PromptEvalDuration time.Duration

type GGML

+ func (llm GGML) GraphSize(context, batch uint64) (partialOffload, fullOffload uint64)

type KV

+ func (kv KV) Architecture() string

+ func (kv KV) BlockCount() uint64

+ func (kv KV) ContextLength() uint64

+ func (kv KV) EmbeddingLength() uint64

+ func (kv KV) FileType() string

+ func (kv KV) GQA() uint64

+ func (kv KV) HeadCount() uint64

+ func (kv KV) HeadCountKV() uint64

+ func (kv KV) ParameterCount() uint64

+ type Layer map[string]*Tensor

+ type LlamaServer struct

+ func NewLlamaServer(model string, adapters, projectors []string, opts api.Options) (*LlamaServer, error)

+ func (s *LlamaServer) Close() error

+ func (s *LlamaServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error

+ func (s *LlamaServer) Detokenize(ctx context.Context, tokens []int) (string, error)

+ func (s *LlamaServer) Embedding(ctx context.Context, prompt string) ([]float64, error)

+ func (s *LlamaServer) Ping(ctx context.Context) error

+ func (s *LlamaServer) Tokenize(ctx context.Context, content string) ([]int, error)

+ type ServerStatus int

+ const ServerStatusError

+ const ServerStatusLoadingModel

+ const ServerStatusNoSlotsAvaialble

+ const ServerStatusNotResponding

+ const ServerStatusReady

+ type ServerStatusResp struct

+ Error string

+ SlotsIdle int

+ SlotsProcessing int

+ Status string

+ type StatusWriter struct

+ LastErrMsg string

+ func NewStatusWriter(out *os.File) *StatusWriter

+ func (w *StatusWriter) Write(b []byte) (int, error)

+ type Tensors []*Tensor

+ func (ts Tensors) Layers() map[string]Layer

v0.1.32-rc1

Apr 4, 2024

v0.1.31

Apr 4, 2024

v0.1.31-rc3

Apr 4, 2024

v0.1.30

Mar 27, 2024

Changes in this version

+ const FILE_MAGIC_GGJT

+ const FILE_MAGIC_GGLA

+ const FILE_MAGIC_GGMF

+ const FILE_MAGIC_GGML

+ const FILE_MAGIC_GGUF_BE

+ const FILE_MAGIC_GGUF_LE

+ const GGUFTokenByte

+ const GGUFTokenControl

+ const GGUFTokenNormal

+ const GGUFTokenUnknown

+ const GGUFTokenUnused

+ const GGUFTokenUserDefined

+ const GGUFTypeArray

+ const GGUFTypeBool

+ const GGUFTypeFloat32

+ const GGUFTypeFloat64

+ const GGUFTypeInt16

+ const GGUFTypeInt32

+ const GGUFTypeInt64

+ const GGUFTypeInt8

+ const GGUFTypeString

+ const GGUFTypeUint16

+ const GGUFTypeUint32

+ const GGUFTypeUint64

+ const GGUFTypeUint8

+ var ErrUnsupportedFormat = errors.New("unsupported model format")

+ func Init() error

+ type ContainerGGLA struct

+ func (c *ContainerGGLA) Decode(rs io.ReadSeeker) (model, error)

+ func (c *ContainerGGLA) Name() string

+ type ContainerGGUF struct

+ ByteOrder binary.ByteOrder

+ V1 struct{ ... }

+ V2 struct{ ... }

+ V3 struct{ ... }

+ Version uint32

+ func (c *ContainerGGUF) Decode(rs io.ReadSeeker) (model, error)

+ func (c *ContainerGGUF) Name() string

+ type DetokenizeRequest struct

+ Tokens []int

+ type DetokenizeResponse struct

+ Content string

+ type EmbeddingRequest struct

+ Content string

+ type EmbeddingResponse struct

+ Embedding []float64

+ type GGML struct

+ Size int64

+ func DecodeGGML(rs io.ReadSeeker) (*GGML, error)

+ type GGUFModel struct

+ Tensors []Tensor

+ func NewGGUFModel(container *ContainerGGUF) *GGUFModel

+ func (llm *GGUFModel) Decode(rs io.ReadSeeker) error

+ func (llm *GGUFModel) Encode(f *os.File) error

+ func (llm *GGUFModel) FileType() string

+ func (llm *GGUFModel) ModelFamily() string

+ func (llm *GGUFModel) ModelType() string

+ func (llm *GGUFModel) NumCtx() uint32

+ func (llm *GGUFModel) NumEmbed() uint32

+ func (llm *GGUFModel) NumGQA() uint32

+ func (llm *GGUFModel) NumHead() uint32

+ func (llm *GGUFModel) NumHeadKv() uint32

+ func (llm *GGUFModel) NumKV() uint64

+ func (llm *GGUFModel) NumLayers() uint32

+ func (llm *GGUFModel) NumTensor() uint64

+ type ImageData struct

+ Data []byte

+ ID int

+ type KV map[string]any

+ type LLM interface

+ Close func()

+ Decode func(context.Context, []int) (string, error)

+ Embedding func(context.Context, string) ([]float64, error)

+ Encode func(context.Context, string) ([]int, error)

+ Predict func(context.Context, PredictOpts, func(PredictResult)) error

+ func New(model string, adapters, projectors []string, opts api.Options) (LLM, error)

+ type ModelGGLA struct

+ func (*ModelGGLA) FileType() string

+ func (*ModelGGLA) ModelFamily() string

+ func (*ModelGGLA) ModelType() string

+ func (*ModelGGLA) NumCtx() uint32

+ func (*ModelGGLA) NumEmbed() uint32

+ func (*ModelGGLA) NumGQA() uint32

+ func (*ModelGGLA) NumHead() uint32

+ func (*ModelGGLA) NumHeadKv() uint32

+ func (*ModelGGLA) NumLayers() uint32

+ func (m *ModelGGLA) KV() KV

+ func (m *ModelGGLA) Tensor() []Tensor

+ type PredictOpts struct

+ Format string

+ Images []ImageData

+ Options api.Options

+ Prompt string

+ type PredictResult struct

+ Content string

+ Done bool

+ EvalCount int

+ EvalDuration time.Duration

+ PromptEvalCount int

+ PromptEvalDuration time.Duration

+ type Tensor struct

+ FileName string

+ FileOffsets []uint64

+ Kind uint32

+ Name string

+ Offset uint64

+ OffsetPadding uint64

+ Shape []uint64

+ func (t Tensor) BlockSize() uint64

+ func (t Tensor) Parameters() uint64

+ func (t Tensor) Repack(data []uint16, heads int) ([]uint16, error)

+ func (t Tensor) Size() uint64

+ func (t Tensor) TypeSize() uint64

+ type TokenizeRequest struct

+ Content string

+ type TokenizeResponse struct

+ Tokens []int

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL