llamacpp

Documentation

Index

• Constants
• Variables
• type LlamaCppClient
  • func (self *LlamaCppClient) GenerateCompletion(ctx context.Context, responseChannel chan LlamaCppCompletionToken, ...)
  • func (self *LlamaCppClient) GetHealth(ctx context.Context, responseChannel chan<- LlamaCppHealthStatus)
• type LlamaCppCompletionRequest
• type LlamaCppCompletionToken
• type LlamaCppConfiguration
  • func (self *LlamaCppConfiguration) String() string
• type LlamaCppHealthStatus
  • func (self *LlamaCppHealthStatus) IsOk() bool
• type LlamaCppHealthStatusCode

Constants

const (
	CompletionDataPrefix = "data: "
)

Variables

var (
	ErrorNon200Response = errors.New("Non-200 response from llama.cpp")
)

Types

type LlamaCppClient

type LlamaCppClient struct {
	HttpClient            *http.Client
	LlamaCppConfiguration *LlamaCppConfiguration
}

func (*LlamaCppClient) GenerateCompletion

func (self *LlamaCppClient) GenerateCompletion(
	ctx context.Context,
	responseChannel chan LlamaCppCompletionToken,
	llamaCppCompletionRequest LlamaCppCompletionRequest,
)

func (*LlamaCppClient) GetHealth

func (self *LlamaCppClient) GetHealth(
	ctx context.Context,
	responseChannel chan<- LlamaCppHealthStatus,
)

type LlamaCppCompletionRequest

type LlamaCppCompletionRequest struct {
	JsonSchema any    `json:"json_schema"`
	NPredict   int    `json:"n_predict"`
	Prompt     string `json:"prompt"`
	Stream     bool   `json:"stream"`
}

type LlamaCppCompletionToken

type LlamaCppCompletionToken struct {
	Content string `json:"content"`
	Error   error  `json:"-"`
	IsLast  bool   `json:"stop"`
	SlotId  uint   `json:"id_slot"`
}

type LlamaCppConfiguration

type LlamaCppConfiguration struct {
	HttpAddress *netcfg.HttpAddressConfiguration
}

func (*LlamaCppConfiguration) String

func (self *LlamaCppConfiguration) String() string

type LlamaCppHealthStatus

type LlamaCppHealthStatus struct {
	Error           error                    `json:"-"`
	ErrorMessage    string                   `json:"error_message,omitempty"`
	Status          LlamaCppHealthStatusCode `json:"status"`
	SlotsIdle       uint                     `json:"slots_idle"`
	SlotsProcessing uint                     `json:"slots_processing"`
}

func (*LlamaCppHealthStatus) IsOk

func (self *LlamaCppHealthStatus) IsOk() bool

type LlamaCppHealthStatusCode

type LlamaCppHealthStatusCode string

const (
	Error           LlamaCppHealthStatusCode = "error"
	LoadingModel    LlamaCppHealthStatusCode = "loading model"
	NoSlotAvailable LlamaCppHealthStatusCode = "no slot available"
	Ok              LlamaCppHealthStatusCode = "ok"
)