tokenizer

Documentation

Index

• type OpenAI
  • func NewOpenAI(modelName string) *OpenAI
  • func (o *OpenAI) GetNumTokens(text string) (int, error)
  • func (o *OpenAI) GetNumTokensFromMessage(messages schema.ChatMessages) (int, error)
  • func (o *OpenAI) GetTokenIDs(text string) ([]int, error)
• type Simple
  • func NewSimple() *Simple
  • func (t *Simple) GetNumTokens(text string) (int, error)
  • func (t *Simple) GetNumTokensFromMessage(messages schema.ChatMessages) (int, error)
  • func (t *Simple) GetTokenIDs(text string) ([]int, error)

Types

type OpenAI

type OpenAI struct {
	// contains filtered or unexported fields
}

func NewOpenAI

func NewOpenAI(modelName string) *OpenAI

func (*OpenAI) GetNumTokens

func (o *OpenAI) GetNumTokens(text string) (int, error)

func (*OpenAI) GetNumTokensFromMessage

func (o *OpenAI) GetNumTokensFromMessage(messages schema.ChatMessages) (int, error)

func (*OpenAI) GetTokenIDs

func (o *OpenAI) GetTokenIDs(text string) ([]int, error)

type Simple

type Simple struct{}

func NewSimple

func NewSimple() *Simple

func (*Simple) GetNumTokens

func (t *Simple) GetNumTokens(text string) (int, error)

func (*Simple) GetNumTokensFromMessage

func (t *Simple) GetNumTokensFromMessage(messages schema.ChatMessages) (int, error)

func (*Simple) GetTokenIDs

func (t *Simple) GetTokenIDs(text string) ([]int, error)