embeddings

package
v0.4.35 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 15, 2025 License: MIT Imports: 22 Imported by: 2

Documentation

Index

Constants

View Source 
const EmbeddingsSlug = "embeddings"

Variables

This section is empty.

Functions

This section is empty.

Types

type CachedProvider added in v0.4.26 

type CachedProvider struct {
	// contains filtered or unexported fields
}

CachedProvider wraps an embedding provider with LRU caching capabilities

func NewCachedProvider added in v0.4.26 

func NewCachedProvider(provider Provider, maxSize int) *CachedProvider

NewCachedProvider creates a new cached wrapper around an embedding provider maxSize determines how many embeddings to keep in cache (default 1000)

func (*CachedProvider) ClearCache added in v0.4.26 

func (c *CachedProvider) ClearCache()

ClearCache removes all cached embeddings

func (*CachedProvider) GenerateEmbedding added in v0.4.26 

func (c *CachedProvider) GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

GenerateEmbedding returns cached embeddings if available, otherwise generates new ones

func (*CachedProvider) GetModel added in v0.4.26 

func (c *CachedProvider) GetModel() EmbeddingModel

GetModel delegates to the underlying provider

func (*CachedProvider) MaxSize added in v0.4.26 

func (c *CachedProvider) MaxSize() int

MaxSize returns the maximum number of embeddings that can be cached

func (*CachedProvider) Size added in v0.4.26 

func (c *CachedProvider) Size() int

Size returns the current number of cached embeddings

type DiskCacheEntry added in v0.4.30 

type DiskCacheEntry struct {
	Embedding  []float32 `json:"embedding"`
	TextPrefix string    `json:"text_prefix"` // First 100 chars
}

DiskCacheEntry represents a single cached embedding with metadata

type DiskCacheProvider added in v0.4.30 

type DiskCacheProvider struct {
	// contains filtered or unexported fields
}

DiskCacheProvider implements the Provider interface with disk persistence

func NewDiskCacheProvider added in v0.4.30 

func NewDiskCacheProvider(provider Provider, opts ...Option) (*DiskCacheProvider, error)

NewDiskCacheProvider creates a new disk-based cache provider

func (*DiskCacheProvider) ClearCache added in v0.4.30 

func (p *DiskCacheProvider) ClearCache() error

func (*DiskCacheProvider) GenerateEmbedding added in v0.4.30 

func (p *DiskCacheProvider) GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

func (*DiskCacheProvider) GetCachedEntry added in v0.4.30 

func (p *DiskCacheProvider) GetCachedEntry(text string) (*DiskCacheEntry, error)

func (*DiskCacheProvider) GetModel added in v0.4.30 

func (p *DiskCacheProvider) GetModel() EmbeddingModel

type EmbeddingModel 

type EmbeddingModel struct {
	Name       string
	Dimensions int
}

EmbeddingModel contains metadata about the embedding model

type EmbeddingsConfig added in v0.4.32 

type EmbeddingsConfig struct {
	// Type specifies the provider type (e.g. "openai", "ollama")
	Type string `glazed.parameter:"embeddings-type"`
	// Engine specifies the model to use (e.g. "text-embedding-ada-002" for OpenAI)
	Engine string `glazed.parameter:"embeddings-engine"`
	// Dimensions specifies the embedding dimensions (defaults to 1536 for OpenAI)
	Dimensions int `glazed.parameter:"embeddings-dimensions"`
	// APIKeys maps provider types to their API keys
	APIKeys map[settings.ApiType]string `yaml:"api_keys,omitempty" glazed.parameter:"*-api-key"`
	// BaseURLs maps provider types to their base URLs
	BaseURLs map[settings.ApiType]string `yaml:"base_urls,omitempty" glazed.parameter:"*-base-url"`

	// Caching settings
	CacheType       string `glazed.parameter:"embeddings-cache-type"`
	CacheMaxSize    int64  `glazed.parameter:"embeddings-cache-max-size"`
	CacheMaxEntries int    `glazed.parameter:"embeddings-cache-max-entries"`
	CacheDirectory  string `glazed.parameter:"embeddings-cache-directory"`
}

EmbeddingsConfig contains the minimal configuration needed for embeddings

func NewEmbeddingsConfig added in v0.4.32 

func NewEmbeddingsConfig() (*EmbeddingsConfig, error)

type EmbeddingsFlagsLayer added in v0.4.32 

type EmbeddingsFlagsLayer struct {
	*layers.ParameterLayerImpl `yaml:",inline"`
}

func NewEmbeddingsFlagsLayer added in v0.4.32 

func NewEmbeddingsFlagsLayer(options ...layers.ParameterLayerOptions) (*EmbeddingsFlagsLayer, error)

type OllamaProvider 

type OllamaProvider struct {
	// contains filtered or unexported fields
}

func NewOllamaProvider 

func NewOllamaProvider(baseURL string, model string, dimensions int) *OllamaProvider

func (*OllamaProvider) GenerateEmbedding 

func (p *OllamaProvider) GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

func (*OllamaProvider) GetModel 

func (p *OllamaProvider) GetModel() EmbeddingModel

type OpenAIProvider 

type OpenAIProvider struct {
	// contains filtered or unexported fields
}

func NewOpenAIProvider 

func NewOpenAIProvider(apiKey string, model openai.EmbeddingModel, dimensions int) *OpenAIProvider

func (*OpenAIProvider) GenerateEmbedding 

func (p *OpenAIProvider) GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

func (*OpenAIProvider) GetModel 

func (p *OpenAIProvider) GetModel() EmbeddingModel

type Option added in v0.4.30 

type Option func(*DiskCacheProvider)

func WithDirectory added in v0.4.30 

func WithDirectory(dir string) Option

func WithMaxEntries added in v0.4.30 

func WithMaxEntries(count int) Option

func WithMaxSize added in v0.4.30 

func WithMaxSize(size int64) Option

type Provider 

type Provider interface {
	// GenerateEmbedding creates an embedding vector for the given text
	GenerateEmbedding(ctx context.Context, text string) ([]float32, error)

	// GetModel returns information about the embedding model being used
	GetModel() EmbeddingModel
}

Provider defines the interface for generating embeddings

type ProviderOption added in v0.4.32 

type ProviderOption func(*providerOptions)

ProviderOption is a function that configures a provider

func WithAPIKey added in v0.4.32 

func WithAPIKey(key string) ProviderOption

WithAPIKey sets the API key

func WithBaseURL added in v0.4.32 

func WithBaseURL(url string) ProviderOption

WithBaseURL sets the base URL

func WithDimensions added in v0.4.32 

func WithDimensions(d int) ProviderOption

WithDimensions sets the embedding dimensions

func WithEngine added in v0.4.32 

func WithEngine(e string) ProviderOption

WithEngine sets the model engine

func WithType added in v0.4.32 

func WithType(t string) ProviderOption

WithType sets the provider type

type SettingsFactory 

type SettingsFactory struct {
	// contains filtered or unexported fields
}

SettingsFactory creates embedding providers based on configuration

func NewSettingsFactory 

func NewSettingsFactory(config *EmbeddingsConfig) *SettingsFactory

NewSettingsFactory creates a new factory that uses the provided configuration

func NewSettingsFactoryFromStepSettings added in v0.4.32 

func NewSettingsFactoryFromStepSettings(s *settings.StepSettings) *SettingsFactory

NewSettingsFactoryFromStepSettings creates a new factory from StepSettings for backwards compatibility

func (*SettingsFactory) GetEmbeddingFuncMap added in v0.4.32 

func (f *SettingsFactory) GetEmbeddingFuncMap() template.FuncMap

func (*SettingsFactory) GetEmbeddingTagFunc added in v0.4.32 

func (f *SettingsFactory) GetEmbeddingTagFunc() emrichen.TagFunc

func (*SettingsFactory) NewCachedProvider added in v0.4.26 

func (f *SettingsFactory) NewCachedProvider(maxSize int) (Provider, error)

NewCachedProvider creates a new cached embedding provider based on the configuration maxSize determines how many embeddings to keep in cache (default 1000)

func (*SettingsFactory) NewProvider 

func (f *SettingsFactory) NewProvider(opts ...ProviderOption) (Provider, error)

NewProvider creates a new embedding provider based on the configuration and options

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.