redisvector

package
v0.0.0-...-b505bc2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 4, 2024 License: MIT Imports: 20 Imported by: 1

Documentation

Overview

Package redisvector contains an implementation of the VectorStore interface using redisvector.

Index

Constants

View Source 
const (
	// IndexType enum values.
	JSONIndexType IndexType = "JSON"
	HASHIndexType IndexType = "HASH"

	// Distance Metric enums values.
	L2DistanceMetric     DistanceMetric = "L2"
	CosineDistanceMetric DistanceMetric = "COSINE"
	IPDistanceMetric     DistanceMetric = "IP"

	// Vector Algorithm enum values.
	FlatVectorAlgorithm VectorAlgorithm = "FLAT"
	HNSWVectorAlgorithm VectorAlgorithm = "HNSW"

	// Vector DataType enum values.
	FLOAT32VectorDataType VectorDataType = "FLOAT32"
	FLOAT64VectorDataType VectorDataType = "FLOAT64"

	// Phonetic Matchers enum values.
	PhoneticDoubleMetaphoneEnglish    PhoneticMatcherType = "dm:en"
	PhoneticDoubleMetaphoneFrench     PhoneticMatcherType = "dm:fr"
	PhoneticDoubleMetaphonePortuguese PhoneticMatcherType = "dm:pt"
	PhoneticDoubleMetaphoneSpanish    PhoneticMatcherType = "dm:es"
)

Variables

View Source 
var (
	ErrInvalidSchemaFormat = errors.New("invalid schema format")
	ErrEmptySchemaContent  = errors.New("empty schema content")
)
View Source 
var (
	ErrEmptyIndexName         = errors.New("empty redis index name")
	ErrNotExistedIndex        = errors.New("redis index name does not exist")
	ErrInvalidEmbeddingVector = errors.New("embedding vector error")
	ErrInvalidScoreThreshold  = errors.New("score threshold must be between 0 and 1")
	ErrInvalidFilters         = errors.New("invalid filters")
)
View Source 
var ErrInvalidOptions = errors.New("invalid options")

ErrInvalidOptions is returned when the options given are invalid.

Functions

func VectorString32 

func VectorString32(v []float32) string

convert []float32 into string.

func VectorString64 

func VectorString64(v []float64) string

convert []float64 into string.

Types

type DistanceMetric 

type DistanceMetric string

type IndexSchema 

type IndexSchema struct {
	Tag     []TagField     `json:"tag"     yaml:"tag"`
	Text    []TextField    `json:"text"    yaml:"text"`
	Numeric []NumericField `json:"numeric" yaml:"numeric"`
	Vector  []VectorField  `json:"vector"  yaml:"vector"`
}

func (*IndexSchema) AsCommand 

func (s *IndexSchema) AsCommand() []string

func (*IndexSchema) MetadataKeys 

func (s *IndexSchema) MetadataKeys() map[string]any

return names of field exclude vector key.

type IndexType 

type IndexType string

type IndexVectorSearch 

type IndexVectorSearch struct {
	// contains filtered or unexported fields
}

func NewIndexVectorSearch 

func NewIndexVectorSearch(index string, vector []float32, opts ...SearchOption) (*IndexVectorSearch, error)

func (IndexVectorSearch) AsCommand 

func (s IndexVectorSearch) AsCommand() []string

type NumericField 

type NumericField struct {
	Name     string
	As       string `json:"as,omitempty"       yaml:"as,omitempty"`
	NoIndex  bool   `json:"no_index,omitempty" yaml:"no_index,omitempty"`
	Sortable bool   `json:"sortable,omitempty" yaml:"sortable,omitempty"`
}

func (NumericField) AsCommand 

func (f NumericField) AsCommand() []string

type Option 

type Option func(s *Store)

Option is a function type that can be used to modify the client.

func WithConnectionURL 

func WithConnectionURL(connectionURL string) Option

WithConnectionURL is an option for specifying the Redis connection URL. Must be set. URL meets the official redis url format (https://github.com/redis/redis-specifications/blob/master/uri/redis.txt) Example:

redis://<user>:<password>@<host>:<port>/<db_number> redis://<user>:<password>@<host>:<port>?addr=<host2>:<port2>&addr=<host3>:<port3> unix://<user>:<password>@</path/to/redis.sock>?db=<db_number>

func WithEmbedder 

func WithEmbedder(e embeddings.Embedder) Option

WithEmbedder is an option for setting the embedder to use. Must be set.

func WithIndexName 

func WithIndexName(name string, createIndexIfNotExists bool) Option

WithIndexName is an option for specifying the index name. Must be set.

`createIndexIfNotExists`: 

if set false, will throw error when the index does not exist
if set true, will create index when the index does not exist

If the `WithIndexSchema` option is set, the index will be created with this index schema, otherwise, the index will be created with the generated schema with document metadata in `AddDocuments`.

func WithIndexSchema 

func WithIndexSchema(format SchemaFormat, schemaFilePath string, schemaBytes []byte) Option

WithIndexSchema is an option for specifying the index schema with file or bytes 

`format`: support YAML & JSON format
`schemaFilePath`: schema config file path
`schemaBytes`: schema string

throw error if schemaFilePath & schemaBytes are both empty the schemaBytes will overwrite the schemaFilePath if schemaFilePath & schemaBytes both set if index doesn't exist, the schema will be used to create index otherwise, it only control what fields the metadata maps to return in search result ref: https://python.langchain.com/docs/integrations/vectorstores/redis/#custom-metadata-indexing

type PhoneticMatcherType 

type PhoneticMatcherType string

type RedisClient 

type RedisClient interface {
	DropIndex(ctx context.Context, index string, deleteDocuments bool) error
	CheckIndexExists(ctx context.Context, index string) bool
	CreateIndexIfNotExists(ctx context.Context, index string, schema *IndexSchema) error
	AddDocWithHash(ctx context.Context, prefix string, doc schema.Document) (string, error)
	AddDocsWithHash(ctx context.Context, prefix string, docs []schema.Document) ([]string, error)
	// TODO AddDocsWithJSON
	Search(ctx context.Context, search IndexVectorSearch) (int64, []schema.Document, error)
}

RedisClient interface of redis client, easy to replace third redis client package use rueidis temporarily, go-redis has not yet implemented RedisJSON & RediSearch.

type RedisIndex 

type RedisIndex struct {
	// contains filtered or unexported fields
}

func NewIndex 

func NewIndex(name string, prefix []string, indexType IndexType, schema IndexSchema) *RedisIndex

func (*RedisIndex) AsCommand 

func (i *RedisIndex) AsCommand() ([]string, error)

type RedisIndexAlgorithm 

type RedisIndexAlgorithm string

type RedisIndexSchemaField 

type RedisIndexSchemaField interface {
	// AsCommand convert schema into redis command string
	AsCommand() []string
}

type RueidisClient 

type RueidisClient struct {
	// contains filtered or unexported fields
}

func NewRueidisClient 

func NewRueidisClient(url string) (*RueidisClient, error)

NewRueidisClient create rueidis redist client.

func (RueidisClient) AddDocWithHash 

func (c RueidisClient) AddDocWithHash(ctx context.Context, prefix string, doc schema.Document) (string, error)

func (RueidisClient) AddDocsWithHash 

func (c RueidisClient) AddDocsWithHash(ctx context.Context, prefix string, docs []schema.Document) ([]string, error)

func (RueidisClient) CheckIndexExists 

func (c RueidisClient) CheckIndexExists(ctx context.Context, index string) bool

func (RueidisClient) CreateIndexIfNotExists 

func (c RueidisClient) CreateIndexIfNotExists(ctx context.Context, index string, schema *IndexSchema) error

func (RueidisClient) DropIndex 

func (c RueidisClient) DropIndex(ctx context.Context, index string, deleteDocuments bool) error

func (RueidisClient) Search 

func (c RueidisClient) Search(ctx context.Context, search IndexVectorSearch) (int64, []schema.Document, error)

type SchemaFormat 

type SchemaFormat string

SchemaFormat JSONSchemaFormat or YAMLSchemaFormat. 

const (
	JSONSchemaFormat SchemaFormat = "JSON"
	YAMLSchemaFormat SchemaFormat = "YAML"
)

type SearchOption 

type SearchOption func(s *IndexVectorSearch)

func WithOffsetLimit 

func WithOffsetLimit(offset, limit int) SearchOption

func WithPreFilters 

func WithPreFilters(preFilters string) SearchOption

func WithReturns 

func WithReturns(returns []string) SearchOption

func WithScoreThreshold 

func WithScoreThreshold(scoreThreshold float32) SearchOption

type Store 

type Store struct {
	// contains filtered or unexported fields
}

Store is a wrapper around the redis client.

func New 

func New(ctx context.Context, opts ...Option) (*Store, error)

New creates a new Store with options.

func (*Store) AddDocuments 

func (s *Store) AddDocuments(ctx context.Context, docs []schema.Document, _ ...vectorstores.Option) ([]string, error)

AddDocuments adds the text and metadata from the documents to the redis associated with 'Store'. and returns the ids of the added documents. Note: currently save documents with Hset command return `docIDs` that prefix with `doc:{index_name}` 

if doc.metadata has `keys` or `ids` field, the docId will use `keys` or `ids` value
if not, the docId is uuid string

func (*Store) DropIndex 

func (s *Store) DropIndex(ctx context.Context, index string, deleteDocuments bool) error

func (*Store) SimilaritySearch 

func (s *Store) SimilaritySearch(ctx context.Context, query string, numDocuments int, options ...vectorstores.Option) ([]schema.Document, error)

SimilaritySearch similarity search docs with `ScoreThreshold` `Filters` `Embedder` Support options: 

WithScoreThreshold:
WithFilters: filter string should match redis search pre-filter query pattern.(eg: @title:Dune)
	ref: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#pre-filter-query-attributes-hybrid-approach
WithEmbedder: if set, it will embed query string with this embedder; otherwise embed with vector's embedder

ref: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#pre-filter-query-attributes-hybrid-approach

type TagField 

type TagField struct {
	Name          string
	As            string `json:"as,omitempty"             yaml:"as,omitempty"`
	Separator     string `json:"separator"                yaml:"separator"` // default=","
	NoIndex       bool   `json:"no_index,omitempty"       yaml:"no_index,omitempty"`
	Sortable      bool   `json:"sortable,omitempty"       yaml:"sortable,omitempty"`
	CaseSensitive bool   `json:"case_sensitive,omitempty" yaml:"case_sensitive,omitempty"`
}

func (TagField) AsCommand 

func (f TagField) AsCommand() []string

type TextField 

type TextField struct {
	Name            string
	As              string              `json:"as,omitempty" yaml:"as,omitempty"`
	Weight          float32             // default=1
	NoStem          bool                `json:"no_stem,omitempty"          yaml:"no_stem,omitempty"`
	WithSuffixtrie  bool                `json:"withsuffixtrie,omitempty"   yaml:"withsuffixtrie,omitempty"`
	PhoneticMatcher PhoneticMatcherType `json:"phonetic_matcher,omitempty" yaml:"phonetic_matcher,omitempty"`
	NoIndex         bool                `json:"no_index,omitempty"         yaml:"no_index,omitempty"`
	Sortable        bool                `json:"sortable,omitempty"         yaml:"sortable,omitempty"`
}

func (TextField) AsCommand 

func (f TextField) AsCommand() []string

type VectorAlgorithm 

type VectorAlgorithm string

type VectorDataType 

type VectorDataType string

type VectorField 

type VectorField struct {
	Name      string
	As        string          `json:"as,omitempty" yaml:"as,omitempty"`
	Algorithm VectorAlgorithm // default="FLAT"

	// mandatory attributes
	Dims           int            // int = Field(...)
	Datatype       VectorDataType // default="FLOAT32"
	DistanceMetric DistanceMetric `json:"distance_metric" yaml:"distance_metric"` // default="COSINE"

	// optional attributes
	InitialCap int `json:"initial_cap,omitempty" yaml:"initial_cap,omitempty"` // Optional[int] = None

	// only FLAT attributes
	BlockSize int `json:"block_size,omitempty" yaml:"block_size,omitempty"`

	// only HSNW attributes
	M              int     `json:"m,omitempty"               yaml:"m,omitempty"`               // m: int = Field(default=16)
	EfConstruction int     `json:"ef_construction,omitempty" yaml:"ef_construction,omitempty"` // ef_construction: int = Field(default=200)
	EfRuntime      int     `json:"ef_runtime,omitempty"      yaml:"ef_runtime,omitempty"`      // ef_runtime: int = Field(default=10)
	Epsilon        float32 `json:"epsilon,omitempty"         yaml:"epsilon,omitempty"`         // epsilon: float = Field(default=0.01)
}

func (VectorField) AsCommand 

func (f VectorField) AsCommand() []string

nolint: cyclop

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.