local

Documentation

Index

• Variables
• type LLM
  • func New(opts ...Option) (*LLM, error)
  • func (o *LLM) Call(ctx context.Context, prompt string, options ...llms.CallOption) (string, error)
  • func (o *LLM) Generate(ctx context.Context, prompts []string, options ...llms.CallOption) ([]*llms.Generation, error)
  • func (o *LLM) GeneratePrompt(ctx context.Context, prompts []schema.PromptValue, options ...llms.CallOption) (llms.LLMResult, error)
  • func (o *LLM) GetNumTokens(text string) int
• type Option
  • func WithArgs(args string) Option
  • func WithBin(bin string) Option
  • func WithGlobalAsArgs() Option

Variables

var (
	// ErrEmptyResponse is returned when the local LLM binary returns an empty response.
	ErrEmptyResponse = errors.New("no response")
	// ErrMissingBin is returned when the LOCAL_LLM_BIN environment variable is not set.
	ErrMissingBin = errors.New("missing the local LLM binary path, set the LOCAL_LLM_BIN environment variable")
)

Types

type LLM

type LLM struct {
	CallbacksHandler callbacks.Handler
	// contains filtered or unexported fields
}

LLM is a local LLM implementation.

func New

func New(opts ...Option) (*LLM, error)

New creates a new local LLM implementation.

func (*LLM) Call

func (o *LLM) Call(ctx context.Context, prompt string, options ...llms.CallOption) (string, error)

Call calls the local LLM binary with the given prompt.

func (*LLM) Generate

func (o *LLM) Generate(ctx context.Context, prompts []string, options ...llms.CallOption) ([]*llms.Generation, error)

Generate generates completions using the local LLM binary.

func (*LLM) GeneratePrompt

func (o *LLM) GeneratePrompt(
	ctx context.Context,
	prompts []schema.PromptValue,
	options ...llms.CallOption,
) (llms.LLMResult, error)

func (*LLM) GetNumTokens

func (o *LLM) GetNumTokens(text string) int

type Option

type Option func(*options)

func WithArgs

func WithArgs(args string) Option

WithArgs passes the CLI arguments to the local LLM binary. If not set, then will be used the LOCAL_LLM_ARGS environment variable.

func WithBin

func WithBin(bin string) Option

WithBin passes the path to the local LLM binary to the client. If not set, then will be used the LOCAL_LLM_BIN environment variable.

func WithGlobalAsArgs

func WithGlobalAsArgs() Option

WithGlobalAsArgs passes the CLI arguments to the local LLM binary formed from global llms.Options.