tokenizer

Documentation

Overview

Package tokenizer provides utilities for tokenizing text.

Index

• type Claude
  • func NewClaude() (*Claude, error)
  • func (t *Claude) GetNumTokens(ctx context.Context, text string) (uint, error)
  • func (t *Claude) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)
  • func (t *Claude) GetTokenIDs(ctx context.Context, text string) ([]uint, error)
• type Cohere
  • func NewCohere(modelName string) (*Cohere, error)
  • func (t *Cohere) GetNumTokens(ctx context.Context, text string) (uint, error)
  • func (t *Cohere) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)
  • func (t *Cohere) GetTokenIDs(ctx context.Context, text string) ([]uint, error)
• type GPT2
  • func NewGPT2() (*GPT2, error)
  • func (t *GPT2) GetNumTokens(ctx context.Context, text string) (uint, error)
  • func (t *GPT2) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)
  • func (t *GPT2) GetTokenIDs(ctx context.Context, text string) ([]uint, error)
• type GoogleGenAI
  • func NewGoogleGenAI(client GoogleGenAIClient, model string) *GoogleGenAI
  • func (t *GoogleGenAI) GetNumTokens(ctx context.Context, text string) (uint, error)
  • func (t *GoogleGenAI) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)
• type GoogleGenAIClient
• type OpenAI
  • func NewOpenAI(modelName string) *OpenAI
  • func (t *OpenAI) GetNumTokens(ctx context.Context, text string) (uint, error)
  • func (t *OpenAI) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)
  • func (t *OpenAI) GetTokenIDs(ctx context.Context, text string) ([]uint, error)

Types

type Claude

type Claude struct {
	// contains filtered or unexported fields
}

func NewClaude

func NewClaude() (*Claude, error)

func (*Claude) GetNumTokens

func (t *Claude) GetNumTokens(ctx context.Context, text string) (uint, error)

GetNumTokens returns the number of tokens in the provided text.

func (*Claude) GetNumTokensFromMessage

func (t *Claude) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)

GetNumTokensFromMessage returns the number of tokens in the provided chat messages.

func (*Claude) GetTokenIDs

func (t *Claude) GetTokenIDs(ctx context.Context, text string) ([]uint, error)

GetTokenIDs returns the token IDs corresponding to the provided text.

type Cohere

type Cohere struct {
	// contains filtered or unexported fields
}

func NewCohere

func NewCohere(modelName string) (*Cohere, error)

func (*Cohere) GetNumTokens

func (t *Cohere) GetNumTokens(ctx context.Context, text string) (uint, error)

GetNumTokens returns the number of tokens in the provided text.

func (*Cohere) GetNumTokensFromMessage

func (t *Cohere) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)

GetNumTokensFromMessage returns the number of tokens in the provided chat messages.

func (*Cohere) GetTokenIDs

func (t *Cohere) GetTokenIDs(ctx context.Context, text string) ([]uint, error)

GetTokenIDs returns the token IDs corresponding to the provided text.

type GPT2

type GPT2 struct {
	// contains filtered or unexported fields
}

func NewGPT2

func NewGPT2() (*GPT2, error)

func (*GPT2) GetNumTokens

func (t *GPT2) GetNumTokens(ctx context.Context, text string) (uint, error)

GetNumTokens returns the number of tokens in the provided text.

func (*GPT2) GetNumTokensFromMessage

func (t *GPT2) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)

GetNumTokensFromMessage returns the number of tokens in the provided chat messages.

func (*GPT2) GetTokenIDs

func (t *GPT2) GetTokenIDs(ctx context.Context, text string) ([]uint, error)

GetTokenIDs returns the token IDs corresponding to the provided text.

type GoogleGenAI

type GoogleGenAI struct {
	// contains filtered or unexported fields
}

func NewGoogleGenAI

func NewGoogleGenAI(client GoogleGenAIClient, model string) *GoogleGenAI

func (*GoogleGenAI) GetNumTokens

func (t *GoogleGenAI) GetNumTokens(ctx context.Context, text string) (uint, error)

GetNumTokens returns the number of tokens in the provided text.

func (*GoogleGenAI) GetNumTokensFromMessage

func (t *GoogleGenAI) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)

GetNumTokensFromMessage returns the number of tokens in the provided chat messages.

type GoogleGenAIClient

type GoogleGenAIClient interface {
	CountTokens(context.Context, *generativelanguagepb.CountTokensRequest, ...gax.CallOption) (*generativelanguagepb.CountTokensResponse, error)
}

GoogleGenAIClient is an interface for the GoogleGenAI model client.

type OpenAI

type OpenAI struct {
	// contains filtered or unexported fields
}

func NewOpenAI

func NewOpenAI(modelName string) *OpenAI

func (*OpenAI) GetNumTokens

func (t *OpenAI) GetNumTokens(ctx context.Context, text string) (uint, error)

GetNumTokens returns the number of tokens in the provided text.

func (*OpenAI) GetNumTokensFromMessage

func (t *OpenAI) GetNumTokensFromMessage(ctx context.Context, messages schema.ChatMessages) (uint, error)

GetNumTokensFromMessage returns the number of tokens in the provided chat messages.

func (*OpenAI) GetTokenIDs

func (t *OpenAI) GetTokenIDs(ctx context.Context, text string) ([]uint, error)

GetTokenIDs returns the token IDs corresponding to the provided text.