vllm

Documentation

Index

• type ChatRequest
• type CompletionRequest
• type CostEstimator
  • func NewCostEstimator(tc tokenCounter) *CostEstimator
  • func (ce *CostEstimator) EstimateChatCompletionPromptToken(r *ChatRequest) int
  • func (ce *CostEstimator) EstimateCompletionPromptToken(r *CompletionRequest) int
  • func (ce *CostEstimator) EstimateContentTokenCounts(model string, content string) int
• type FuntionCallProp
  • func (p *FuntionCallProp) GetDescription() string
  • func (p *FuntionCallProp) GetEnum() []string
  • func (p *FuntionCallProp) GetItems() (functionCallProp, error)
  • func (p *FuntionCallProp) GetProperties() (map[string]functionCallProp, error)
  • func (p *FuntionCallProp) GetRequired() []string
  • func (p *FuntionCallProp) GetType() string
• type TokenCounter
  • func NewTokenCounter() (*TokenCounter, error)
  • func (tc *TokenCounter) Count(model string, input string) int

Types

type ChatRequest

type ChatRequest struct {
	goopenai.ChatCompletionRequest
	BestOf                     int     `json:"best_of,omitempty"`
	UseBeamSearch              bool    `json:"use_beam_search,omitempty"`
	TopK                       int     `json:"top_k,omitempty"`
	MinP                       int     `json:"min_p,omitempty"`
	RepetitionPenalty          float64 `json:"repetition_penalty,omitempty"`
	LengthPenalty              float64 `json:"length_penalty,omitempty"`
	EarlyStopping              bool    `json:"early_stopping,omitempty"`
	IgnoreEos                  bool    `json:"ignore_eos,omitempty"`
	MinTokens                  int     `json:"min_tokens,omitempty"`
	StopTokenIds               []int   `json:"stop_token_ids,omitempty"`
	SkipSpecialTokens          bool    `json:"skip_special_tokens,omitempty"`
	SpacesBetweenSpecialTokens bool    `json:"spaces_between_special_tokens,omitempty"`
}

type CompletionRequest

type CompletionRequest struct {
	goopenai.CompletionRequest
	UseBeamSearch              bool    `json:"use_beam_search,omitempty"`
	TopK                       int     `json:"top_k,omitempty"`
	MinP                       int     `json:"min_p,omitempty"`
	RepetitionPenalty          float64 `json:"repetition_penalty,omitempty"`
	LengthPenalty              float64 `json:"length_penalty,omitempty"`
	EarlyStopping              bool    `json:"early_stopping,omitempty"`
	StopTokenIds               []int   `json:"stop_token_ids,omitempty"`
	IgnoreEos                  bool    `json:"ignore_eos,omitempty"`
	MinTokens                  int     `json:"min_tokens,omitempty"`
	SkipSpecialTokens          bool    `json:"skip_special_tokens,omitempty"`
	SpacesBetweenSpecialTokens bool    `json:"spaces_between_special_tokens,omitempty"`
}

type CostEstimator

type CostEstimator struct {
	// contains filtered or unexported fields
}

func NewCostEstimator

func NewCostEstimator(tc tokenCounter) *CostEstimator

func (*CostEstimator) EstimateChatCompletionPromptToken

func (ce *CostEstimator) EstimateChatCompletionPromptToken(r *ChatRequest) int

func (*CostEstimator) EstimateCompletionPromptToken

func (ce *CostEstimator) EstimateCompletionPromptToken(r *CompletionRequest) int

func (*CostEstimator) EstimateContentTokenCounts

func (ce *CostEstimator) EstimateContentTokenCounts(model string, content string) int

type FuntionCallProp

type FuntionCallProp struct {
	Description string                 `json:"description,omitempty"`
	PropType    string                 `json:"type,omitempty"`
	Enum        []string               `json:"enum,omitempty"`
	Items       interface{}            `json:"items,omitempty"`
	Required    []string               `json:"required,omitempty"`
	Properties  map[string]interface{} `json:"properties,omitempty"`
}

func (*FuntionCallProp) GetDescription

func (p *FuntionCallProp) GetDescription() string

func (*FuntionCallProp) GetEnum

func (p *FuntionCallProp) GetEnum() []string

func (*FuntionCallProp) GetItems

func (p *FuntionCallProp) GetItems() (functionCallProp, error)

func (*FuntionCallProp) GetProperties

func (p *FuntionCallProp) GetProperties() (map[string]functionCallProp, error)

func (*FuntionCallProp) GetRequired

func (p *FuntionCallProp) GetRequired() []string

func (*FuntionCallProp) GetType

func (p *FuntionCallProp) GetType() string

type TokenCounter

type TokenCounter struct {
	// contains filtered or unexported fields
}

func NewTokenCounter

func NewTokenCounter() (*TokenCounter, error)

func (*TokenCounter) Count

func (tc *TokenCounter) Count(model string, input string) int