llamacpp

Documentation

Index

• Constants
• Variables
• type LlamaCppClient
  • func (self *LlamaCppClient) GetHealth(ctx context.Context, responseChannel chan<- LlamaCppHealthStatus)
  • func (self *LlamaCppClient) GetSlots(ctx context.Context, responseChannel chan<- LlamaCppSlotStatus)
  • func (self *LlamaCppClient) GetSlotsAggregatedStatus(ctx context.Context, responseChannel chan<- LlamaCppSlotsAggregatedStatus)
• type LlamaCppCompletionRequest
• type LlamaCppConfiguration
  • func (self *LlamaCppConfiguration) String() string
• type LlamaCppHealthStatus
• type LlamaCppHealthStatusCode
• type LlamaCppSlotStatus
• type LlamaCppSlotsAggregatedStatus
  • func (self *LlamaCppSlotsAggregatedStatus) CopyFrom(other *LlamaCppSlotsAggregatedStatus)

Constants

const (
	CompletionDataPrefix = "data: "
)

Variables

var (
	ErrorNon200Response = errors.New("Non-200 response from llama.cpp")
)

Types

type LlamaCppClient

type LlamaCppClient struct {
	HttpClient            *http.Client
	LlamaCppConfiguration *LlamaCppConfiguration
}

func (*LlamaCppClient) GetHealth

func (self *LlamaCppClient) GetHealth(
	ctx context.Context,
	responseChannel chan<- LlamaCppHealthStatus,
)

func (*LlamaCppClient) GetSlots

func (self *LlamaCppClient) GetSlots(
	ctx context.Context,
	responseChannel chan<- LlamaCppSlotStatus,
)

func (*LlamaCppClient) GetSlotsAggregatedStatus

func (self *LlamaCppClient) GetSlotsAggregatedStatus(
	ctx context.Context,
	responseChannel chan<- LlamaCppSlotsAggregatedStatus,
)

type LlamaCppCompletionRequest

type LlamaCppCompletionRequest struct {
	JsonSchema any    `json:"json_schema"`
	NPredict   int    `json:"n_predict"`
	Prompt     string `json:"prompt"`
	Stream     bool   `json:"stream"`
}

type LlamaCppConfiguration

type LlamaCppConfiguration struct {
	HttpAddress *netcfg.HttpAddressConfiguration `json:"http_address"`
	ApiKey      string
}

func (*LlamaCppConfiguration) String

func (self *LlamaCppConfiguration) String() string

type LlamaCppHealthStatus

type LlamaCppHealthStatus struct {
	Error        error                    `json:"-"`
	ErrorMessage string                   `json:"error_message,omitempty"`
	Status       LlamaCppHealthStatusCode `json:"status"`
}

type LlamaCppHealthStatusCode

type LlamaCppHealthStatusCode string

const (
	Error           LlamaCppHealthStatusCode = "error"
	LoadingModel    LlamaCppHealthStatusCode = "loading model"
	NoSlotAvailable LlamaCppHealthStatusCode = "no slot available"
	Ok              LlamaCppHealthStatusCode = "ok"
)

type LlamaCppSlotStatus

type LlamaCppSlotStatus struct {
	Error        error  `json:"-"`
	ErrorMessage string `json:"error_message,omitempty"`
	IsProcessing bool   `json:"is_processing"`
}

type LlamaCppSlotsAggregatedStatus

type LlamaCppSlotsAggregatedStatus struct {
	Error           error                    `json:"-"`
	ErrorMessage    string                   `json:"error_message,omitempty"`
	Status          LlamaCppHealthStatusCode `json:"status"`
	SlotsIdle       int                      `json:"slots_idle"`
	SlotsProcessing int                      `json:"slots_processing"`
}

func (*LlamaCppSlotsAggregatedStatus) CopyFrom

func (self *LlamaCppSlotsAggregatedStatus) CopyFrom(other *LlamaCppSlotsAggregatedStatus)