llm

Documentation

Index

• Constants
• func CheckVRAM() (int64, error)
• func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int
• type DetokenizeRequest
• type DetokenizeResponse
• type EmbeddingRequest
• type EmbeddingResponse
• type GGML
  • func DecodeGGML(r io.ReadSeeker) (*GGML, error)
• type LLM
  • func New(workDir, model string, adapters []string, opts api.Options) (LLM, error)
• type ModelRunner
• type Running
• type StatusWriter
  • func NewStatusWriter() *StatusWriter
  • func (w *StatusWriter) Write(b []byte) (int, error)
• type TokenizeRequest
• type TokenizeResponse

Constants

const (
	// Magic constant for `ggml` files (unversioned).
	FILE_MAGIC_GGML = 0x67676d6c
	// Magic constant for `ggml` files (versioned, ggmf).
	FILE_MAGIC_GGMF = 0x67676d66
	// Magic constant for `ggml` files (versioned, ggjt).
	FILE_MAGIC_GGJT = 0x67676a74
	// Magic constant for `ggla` files (LoRA adapter).
	FILE_MAGIC_GGLA = 0x67676C61
	// Magic constant for `gguf` files (versioned, gguf)
	FILE_MAGIC_GGUF_LE = 0x46554747
	FILE_MAGIC_GGUF_BE = 0x47475546
)

Functions

func CheckVRAM

func CheckVRAM() (int64, error)

CheckVRAM returns the free VRAM in bytes on Linux machines with NVIDIA GPUs

func NumGPU

func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int

Types

type DetokenizeRequest

type DetokenizeRequest struct {
	Tokens []int `json:"tokens"`
}

type DetokenizeResponse

type DetokenizeResponse struct {
	Content string `json:"content"`
}

type EmbeddingRequest

type EmbeddingRequest struct {
	Content string `json:"content"`
}

type EmbeddingResponse

type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

type GGML

type GGML struct {
	// contains filtered or unexported fields
}

func DecodeGGML

func DecodeGGML(r io.ReadSeeker) (*GGML, error)

type LLM

type LLM interface {
	Predict(context.Context, []int, string, func(api.GenerateResponse)) error
	Embedding(context.Context, string) ([]float64, error)
	Encode(context.Context, string) ([]int, error)
	Decode(context.Context, []int) (string, error)
	SetOptions(api.Options)
	Close()
	Ping(context.Context) error
}

func New

func New(workDir, model string, adapters []string, opts api.Options) (LLM, error)

type ModelRunner

type ModelRunner struct {
	Path        string // path to the model runner executable
	Accelerated bool
}

type Running

type Running struct {
	Port   int
	Cmd    *exec.Cmd
	Cancel context.CancelFunc

	*StatusWriter // captures error messages from the llama runner process
	// contains filtered or unexported fields
}

type StatusWriter

type StatusWriter struct {
	ErrCh      chan error
	LastErrMsg string
}

StatusWriter is a writer that captures error messages from the llama runner process

func NewStatusWriter

func NewStatusWriter() *StatusWriter

func (*StatusWriter) Write

func (w *StatusWriter) Write(b []byte) (int, error)

type TokenizeRequest

type TokenizeRequest struct {
	Content string `json:"content"`
}

type TokenizeResponse

type TokenizeResponse struct {
	Tokens []int `json:"tokens"`
}