compressionhelpers

package
v1.24.23 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 20, 2024 License: BSD-3-Clause Imports: 21 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func Concurrently 

func Concurrently(n uint64, action Action)

func ExtractCode8 

func ExtractCode8(encoded []byte, index int) byte

Only made public for testing purposes... Not sure we need it outside

func PutCode8 

func PutCode8(code byte, buffer []byte, index int)

Only made public for testing purposes... Not sure we need it outside

Types

type Action 

type Action func(taskIndex uint64)

type BQDistancer 

type BQDistancer struct {
	// contains filtered or unexported fields
}

func (*BQDistancer) Distance 

func (d *BQDistancer) Distance(x []uint64) (float32, error)

func (*BQDistancer) DistanceToFloat 

func (d *BQDistancer) DistanceToFloat(x []float32) (float32, error)

type BinaryQuantizer 

type BinaryQuantizer struct {
	// contains filtered or unexported fields
}

func NewBinaryQuantizer 

func NewBinaryQuantizer(distancer distancer.Provider) BinaryQuantizer

func (*BinaryQuantizer) CompressedBytes 

func (bq *BinaryQuantizer) CompressedBytes(compressed []uint64) []byte

func (*BinaryQuantizer) DistanceBetweenCompressedAndUncompressedVectors 

func (bq *BinaryQuantizer) DistanceBetweenCompressedAndUncompressedVectors(x []float32, y []uint64) (float32, error)

func (BinaryQuantizer) DistanceBetweenCompressedVectors 

func (bq BinaryQuantizer) DistanceBetweenCompressedVectors(x, y []uint64) (float32, error)

func (BinaryQuantizer) Encode 

func (bq BinaryQuantizer) Encode(vec []float32) []uint64

func (*BinaryQuantizer) ExposeFields 

func (bq *BinaryQuantizer) ExposeFields() PQData

func (*BinaryQuantizer) FromCompressedBytes 

func (bq *BinaryQuantizer) FromCompressedBytes(compressed []byte) []uint64

func (*BinaryQuantizer) NewCompressedQuantizerDistancer 

func (bq *BinaryQuantizer) NewCompressedQuantizerDistancer(a []uint64) quantizerDistancer[uint64]

func (*BinaryQuantizer) NewDistancer 

func (bq *BinaryQuantizer) NewDistancer(a []float32) *BQDistancer

func (*BinaryQuantizer) NewQuantizerDistancer 

func (bq *BinaryQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[uint64]

func (*BinaryQuantizer) ReturnQuantizerDistancer 

func (bq *BinaryQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[uint64])

type Centroid 

type Centroid struct {
	Center     []float32
	Calculated atomic.Bool
}

type CompressionDistanceBag 

type CompressionDistanceBag interface {
	Load(ctx context.Context, id uint64) error
	Distance(x, y uint64) (float32, error)
}

type CompressorDistancer 

type CompressorDistancer interface {
	DistanceToNode(id uint64) (float32, error)
	DistanceToFloat(vec []float32) (float32, error)
}

type DLUTPool 

type DLUTPool struct {
	// contains filtered or unexported fields
}

func NewDLUTPool 

func NewDLUTPool() *DLUTPool

func (*DLUTPool) Get 

func (p *DLUTPool) Get(segments, centroids int, centers []float32) *DistanceLookUpTable

func (*DLUTPool) Return 

func (p *DLUTPool) Return(dlt *DistanceLookUpTable)

type DistanceLookUpTable 

type DistanceLookUpTable struct {
	// contains filtered or unexported fields
}

func NewDistanceLookUpTable 

func NewDistanceLookUpTable(segments int, centroids int, center []float32) *DistanceLookUpTable

func (*DistanceLookUpTable) LookUp 

func (lut *DistanceLookUpTable) LookUp(
	encoded []byte,
	pq *ProductQuantizer,
) float32

func (*DistanceLookUpTable) Reset 

func (lut *DistanceLookUpTable) Reset(segments int, centroids int, center []float32)

type Encoder 

type Encoder byte
const (
	UseTileEncoder   Encoder = 0
	UseKMeansEncoder Encoder = 1
)

type EncoderDistribution 

type EncoderDistribution byte
const (
	NormalEncoderDistribution    EncoderDistribution = 0
	LogNormalEncoderDistribution EncoderDistribution = 1
)

type FilterFunc 

type FilterFunc func([]float32) []float32

type KMeans 

type KMeans struct {
	K                  int     // How many centroids
	DeltaThreshold     float32 // Used to stop fitting if there are not too much changes in the centroids anymore
	IterationThreshold int     // Used to stop fitting after a certain amount of iterations
	Distance           distancer.Provider
	// contains filtered or unexported fields
}

func NewKMeans 

func NewKMeans(k int, dimensions int, segment int) *KMeans

func NewKMeansWithCenters 

func NewKMeansWithCenters(k int, dimensions int, segment int, centers [][]float32) *KMeans

func (*KMeans) Add 

func (m *KMeans) Add(x []float32)

func (*KMeans) Center 

func (m *KMeans) Center(point []float32) []float32

func (*KMeans) Centers 

func (m *KMeans) Centers() [][]float32

func (*KMeans) Centroid 

func (m *KMeans) Centroid(i byte) []float32

func (*KMeans) Encode 

func (m *KMeans) Encode(point []float32) byte

func (*KMeans) ExposeDataForRestore 

func (m *KMeans) ExposeDataForRestore() []byte

func (*KMeans) Fit 

func (m *KMeans) Fit(data [][]float32) error

func (*KMeans) NNearest 

func (m *KMeans) NNearest(point []float32, n int) []uint64

func (*KMeans) Nearest 

func (m *KMeans) Nearest(point []float32) uint64

func (*KMeans) String 

func (k *KMeans) String() string

String prints some minimal information about the encoder. This can be used for viability checks to see if the encoder was initialized correctly – for example after a restart.

type KMeansPartitionData 

type KMeansPartitionData struct {
	// contains filtered or unexported fields
}

type PQData 

type PQData struct {
	Ks                  uint16
	M                   uint16
	Dimensions          uint16
	EncoderType         Encoder
	EncoderDistribution byte
	Encoders            []PQEncoder
	UseBitsEncoding     bool
	TrainingLimit       int
}

type PQDistancer 

type PQDistancer struct {
	// contains filtered or unexported fields
}

func (*PQDistancer) Distance 

func (d *PQDistancer) Distance(x []byte) (float32, error)

func (*PQDistancer) DistanceToFloat 

func (d *PQDistancer) DistanceToFloat(x []float32) (float32, error)

type PQEncoder 

type PQEncoder interface {
	Encode(x []float32) byte
	Centroid(b byte) []float32
	Add(x []float32)
	Fit(data [][]float32) error
	ExposeDataForRestore() []byte
}

type ProductQuantizer 

type ProductQuantizer struct {
	// contains filtered or unexported fields
}

func NewProductQuantizer 

func NewProductQuantizer(cfg ent.PQConfig, distance distancer.Provider, dimensions int) (*ProductQuantizer, error)

func NewProductQuantizerWithEncoders 

func NewProductQuantizerWithEncoders(cfg ent.PQConfig, distance distancer.Provider, dimensions int, encoders []PQEncoder) (*ProductQuantizer, error)

func (*ProductQuantizer) CenterAt 

func (pq *ProductQuantizer) CenterAt(vec []float32) *DistanceLookUpTable

func (*ProductQuantizer) CompressedBytes 

func (pq *ProductQuantizer) CompressedBytes(compressed []byte) []byte

func (*ProductQuantizer) Decode 

func (pq *ProductQuantizer) Decode(code []byte) []float32

func (*ProductQuantizer) Distance 

func (pq *ProductQuantizer) Distance(encoded []byte, lut *DistanceLookUpTable) float32

func (*ProductQuantizer) DistanceBetweenCompressedAndUncompressedVectors 

func (pq *ProductQuantizer) DistanceBetweenCompressedAndUncompressedVectors(x []float32, encoded []byte) (float32, error)

func (*ProductQuantizer) DistanceBetweenCompressedVectors 

func (pq *ProductQuantizer) DistanceBetweenCompressedVectors(x, y []byte) (float32, error)

func (*ProductQuantizer) Encode 

func (pq *ProductQuantizer) Encode(vec []float32) []byte

func (*ProductQuantizer) ExposeFields 

func (pq *ProductQuantizer) ExposeFields() PQData

func (*ProductQuantizer) Fit 

func (pq *ProductQuantizer) Fit(data [][]float32) error

func (*ProductQuantizer) FromCompressedBytes 

func (pq *ProductQuantizer) FromCompressedBytes(compressed []byte) []byte

func (*ProductQuantizer) NewCompressedQuantizerDistancer 

func (pq *ProductQuantizer) NewCompressedQuantizerDistancer(a []byte) quantizerDistancer[byte]

func (*ProductQuantizer) NewDistancer 

func (pq *ProductQuantizer) NewDistancer(a []float32) *PQDistancer

func (*ProductQuantizer) NewQuantizerDistancer 

func (pq *ProductQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[byte]

func (*ProductQuantizer) ReturnDistancer 

func (pq *ProductQuantizer) ReturnDistancer(d *PQDistancer)

func (*ProductQuantizer) ReturnQuantizerDistancer 

func (pq *ProductQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[byte])

type ReturnDistancerFn 

type ReturnDistancerFn func()

type TileEncoder 

type TileEncoder struct {
	// contains filtered or unexported fields
}

func NewTileEncoder 

func NewTileEncoder(bits int, segment int, encoderDistribution EncoderDistribution) *TileEncoder

func RestoreTileEncoder 

func RestoreTileEncoder(bins float64, mean float64, stdDev float64, size float64, s1 float64, s2 float64, segment uint16, encoderDistribution byte) *TileEncoder

func (*TileEncoder) Add 

func (te *TileEncoder) Add(x []float32)

func (*TileEncoder) Centroid 

func (te *TileEncoder) Centroid(b byte) []float32

func (*TileEncoder) Encode 

func (te *TileEncoder) Encode(x []float32) byte

func (*TileEncoder) ExposeDataForRestore 

func (te *TileEncoder) ExposeDataForRestore() []byte

func (*TileEncoder) Fit 

func (te *TileEncoder) Fit(data [][]float32) error

type VectorCompressor 

type VectorCompressor interface {
	Drop() error
	GrowCache(size uint64)
	SetCacheMaxSize(size int64)
	GetCacheMaxSize() int64
	Delete(ctx context.Context, id uint64)
	Preload(id uint64, vector []float32)
	Prefetch(id uint64)
	PrefillCache()

	DistanceBetweenCompressedVectorsFromIDs(ctx context.Context, x, y uint64) (float32, error)
	DistanceBetweenCompressedAndUncompressedVectorsFromID(ctx context.Context, x uint64, y []float32) (float32, error)
	NewDistancer(vector []float32) (CompressorDistancer, ReturnDistancerFn)
	NewDistancerFromID(id uint64) (CompressorDistancer, error)
	NewBag() CompressionDistanceBag

	ExposeFields() PQData
}

func NewBQCompressor 

func NewBQCompressor(
	distance distancer.Provider,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func NewHNSWPQCompressor added in v1.24.1 

func NewHNSWPQCompressor(
	cfg hnsw.PQConfig,
	distance distancer.Provider,
	dimensions int,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	data [][]float32,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func RestoreHNSWPQCompressor added in v1.24.1 

func RestoreHNSWPQCompressor(
	cfg hnsw.PQConfig,
	distance distancer.Provider,
	dimensions int,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	encoders []PQEncoder,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.