compressionhelpers

package
v0.0.0-...-f09cf9b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 13, 2025 License: BSD-3-Clause Imports: 26 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func Concurrently 

func Concurrently(log logrus.FieldLogger, n uint64, action Action)

func ConcurrentlyWithError 

func ConcurrentlyWithError(log logrus.FieldLogger, n uint64, action func(taskIndex uint64) error) error

func ExtractCode8 

func ExtractCode8(encoded []byte, index int) byte

Only made public for testing purposes... Not sure we need it outside

func NewParallelIterator 

func NewParallelIterator[T byte | uint64](bucket *lsmkv.Bucket, parallel int, loadId func([]byte) uint64, fromCompressedBytes func(compressed []byte, buf *[]T) []T,
	logger logrus.FieldLogger,
) *parallelIterator[T]

func PutCode8 

func PutCode8(code byte, buffer []byte, index int)

Only made public for testing purposes... Not sure we need it outside

Types

type Action 

type Action func(taskIndex uint64)

type BQDistancer 

type BQDistancer struct {
	// contains filtered or unexported fields
}

func (*BQDistancer) Distance 

func (d *BQDistancer) Distance(x []uint64) (float32, error)

func (*BQDistancer) DistanceToFloat 

func (d *BQDistancer) DistanceToFloat(x []float32) (float32, error)

type BinaryQuantizer 

type BinaryQuantizer struct {
	// contains filtered or unexported fields
}

func NewBinaryQuantizer 

func NewBinaryQuantizer(distancer distancer.Provider) BinaryQuantizer

func (*BinaryQuantizer) CompressedBytes 

func (bq *BinaryQuantizer) CompressedBytes(compressed []uint64) []byte

func (BinaryQuantizer) DistanceBetweenCompressedVectors 

func (bq BinaryQuantizer) DistanceBetweenCompressedVectors(x, y []uint64) (float32, error)

func (BinaryQuantizer) Encode 

func (bq BinaryQuantizer) Encode(vec []float32) []uint64

func (*BinaryQuantizer) FromCompressedBytes 

func (bq *BinaryQuantizer) FromCompressedBytes(compressed []byte) []uint64

func (*BinaryQuantizer) FromCompressedBytesWithSubsliceBuffer 

func (bq *BinaryQuantizer) FromCompressedBytesWithSubsliceBuffer(compressed []byte, buffer *[]uint64) []uint64

FromCompressedBytesWithSubsliceBuffer is like FromCompressedBytes, but instead of allocating a new slice you can pass in a buffer to use. It will slice something off of that buffer. If the buffer is too small, it will allocate a new buffer.

func (*BinaryQuantizer) NewCompressedQuantizerDistancer 

func (bq *BinaryQuantizer) NewCompressedQuantizerDistancer(a []uint64) quantizerDistancer[uint64]

func (*BinaryQuantizer) NewDistancer 

func (bq *BinaryQuantizer) NewDistancer(a []float32) *BQDistancer

func (*BinaryQuantizer) NewQuantizerDistancer 

func (bq *BinaryQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[uint64]

func (*BinaryQuantizer) PersistCompression 

func (bq *BinaryQuantizer) PersistCompression(logger CommitLogger)

func (*BinaryQuantizer) ReturnQuantizerDistancer 

func (bq *BinaryQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[uint64])

type Centroid 

type Centroid struct {
	Center     []float32
	Calculated atomic.Bool
}

type CommitLogger 

type CommitLogger interface {
	AddPQCompression(PQData) error
	AddSQCompression(SQData) error
}

type CompressionDistanceBag 

type CompressionDistanceBag interface {
	Load(ctx context.Context, id uint64) error
	Distance(x, y uint64) (float32, error)
}

type CompressorDistancer 

type CompressorDistancer interface {
	DistanceToNode(id uint64) (float32, error)
	DistanceToFloat(vec []float32) (float32, error)
}

type DLUTPool 

type DLUTPool struct {
	// contains filtered or unexported fields
}

func NewDLUTPool 

func NewDLUTPool() *DLUTPool

func (*DLUTPool) Get 

func (p *DLUTPool) Get(segments, centroids int, centers []float32) *DistanceLookUpTable

func (*DLUTPool) Return 

func (p *DLUTPool) Return(dlt *DistanceLookUpTable)

type DistanceLookUpTable 

type DistanceLookUpTable struct {
	// contains filtered or unexported fields
}

func NewDistanceLookUpTable 

func NewDistanceLookUpTable(segments int, centroids int, center []float32) *DistanceLookUpTable

func (*DistanceLookUpTable) LookUp 

func (lut *DistanceLookUpTable) LookUp(
	encoded []byte,
	pq *ProductQuantizer,
) float32

func (*DistanceLookUpTable) Reset 

func (lut *DistanceLookUpTable) Reset(segments int, centroids int, center []float32)

type Encoder 

type Encoder byte
const (
	UseTileEncoder   Encoder = 0
	UseKMeansEncoder Encoder = 1
)

type EncoderDistribution 

type EncoderDistribution byte
const (
	NormalEncoderDistribution    EncoderDistribution = 0
	LogNormalEncoderDistribution EncoderDistribution = 1
)

type FilterFunc 

type FilterFunc func([]float32) []float32

type KMeans 

type KMeans struct {
	K                  int     // How many centroids
	DeltaThreshold     float32 // Used to stop fitting if there are not too much changes in the centroids anymore
	IterationThreshold int     // Used to stop fitting after a certain amount of iterations
	Distance           distancer.Provider
	// contains filtered or unexported fields
}

func NewKMeans 

func NewKMeans(k int, dimensions int, segment int) *KMeans

func NewKMeansWithCenters 

func NewKMeansWithCenters(k int, dimensions int, segment int, centers [][]float32) *KMeans

func (*KMeans) Add 

func (m *KMeans) Add(x []float32)

func (*KMeans) Center 

func (m *KMeans) Center(point []float32) []float32

func (*KMeans) Centers 

func (m *KMeans) Centers() [][]float32

func (*KMeans) Centroid 

func (m *KMeans) Centroid(i byte) []float32

func (*KMeans) Encode 

func (m *KMeans) Encode(point []float32) byte

func (*KMeans) ExposeDataForRestore 

func (m *KMeans) ExposeDataForRestore() []byte

func (*KMeans) Fit 

func (m *KMeans) Fit(data [][]float32) error

func (*KMeans) NNearest 

func (m *KMeans) NNearest(point []float32, n int) []uint64

func (*KMeans) Nearest 

func (m *KMeans) Nearest(point []float32) uint64

func (*KMeans) String 

func (k *KMeans) String() string

String prints some minimal information about the encoder. This can be used for viability checks to see if the encoder was initialized correctly – for example after a restart.

type KMeansPartitionData 

type KMeansPartitionData struct {
	// contains filtered or unexported fields
}

type PQData 

type PQData struct {
	Ks                  uint16
	M                   uint16
	Dimensions          uint16
	EncoderType         Encoder
	EncoderDistribution byte
	Encoders            []PQEncoder
	UseBitsEncoding     bool
	TrainingLimit       int
}

type PQDistancer 

type PQDistancer struct {
	// contains filtered or unexported fields
}

func (*PQDistancer) Distance 

func (d *PQDistancer) Distance(x []byte) (float32, error)

func (*PQDistancer) DistanceToFloat 

func (d *PQDistancer) DistanceToFloat(x []float32) (float32, error)

type PQEncoder 

type PQEncoder interface {
	Encode(x []float32) byte
	Centroid(b byte) []float32
	Add(x []float32)
	Fit(data [][]float32) error
	ExposeDataForRestore() []byte
}

type ProductQuantizer 

type ProductQuantizer struct {
	// contains filtered or unexported fields
}

func NewProductQuantizer 

func NewProductQuantizer(cfg ent.PQConfig, distance distancer.Provider, dimensions int, logger logrus.FieldLogger) (*ProductQuantizer, error)

func NewProductQuantizerWithEncoders 

func NewProductQuantizerWithEncoders(cfg ent.PQConfig, distance distancer.Provider, dimensions int, encoders []PQEncoder, logger logrus.FieldLogger) (*ProductQuantizer, error)

func (*ProductQuantizer) CenterAt 

func (pq *ProductQuantizer) CenterAt(vec []float32) *DistanceLookUpTable

func (*ProductQuantizer) CompressedBytes 

func (pq *ProductQuantizer) CompressedBytes(compressed []byte) []byte

func (*ProductQuantizer) Decode 

func (pq *ProductQuantizer) Decode(code []byte) []float32

func (*ProductQuantizer) Distance 

func (pq *ProductQuantizer) Distance(encoded []byte, lut *DistanceLookUpTable) float32

func (*ProductQuantizer) DistanceBetweenCompressedVectors 

func (pq *ProductQuantizer) DistanceBetweenCompressedVectors(x, y []byte) (float32, error)

func (*ProductQuantizer) Encode 

func (pq *ProductQuantizer) Encode(vec []float32) []byte

func (*ProductQuantizer) Fit 

func (pq *ProductQuantizer) Fit(data [][]float32) error

func (*ProductQuantizer) FromCompressedBytes 

func (pq *ProductQuantizer) FromCompressedBytes(compressed []byte) []byte

func (*ProductQuantizer) FromCompressedBytesWithSubsliceBuffer 

func (pq *ProductQuantizer) FromCompressedBytesWithSubsliceBuffer(compressed []byte, buffer *[]byte) []byte

func (*ProductQuantizer) NewCompressedQuantizerDistancer 

func (pq *ProductQuantizer) NewCompressedQuantizerDistancer(a []byte) quantizerDistancer[byte]

func (*ProductQuantizer) NewDistancer 

func (pq *ProductQuantizer) NewDistancer(a []float32) *PQDistancer

func (*ProductQuantizer) NewQuantizerDistancer 

func (pq *ProductQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[byte]

func (*ProductQuantizer) PersistCompression 

func (pq *ProductQuantizer) PersistCompression(logger CommitLogger)

func (*ProductQuantizer) ReturnDistancer 

func (pq *ProductQuantizer) ReturnDistancer(d *PQDistancer)

func (*ProductQuantizer) ReturnQuantizerDistancer 

func (pq *ProductQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[byte])

type ReturnDistancerFn 

type ReturnDistancerFn func()

type SQData 

type SQData struct {
	A          float32
	B          float32
	Dimensions uint16
}

type SQDistancer 

type SQDistancer struct {
	// contains filtered or unexported fields
}

func (*SQDistancer) Distance 

func (d *SQDistancer) Distance(x []byte) (float32, error)

func (*SQDistancer) DistanceToFloat 

func (d *SQDistancer) DistanceToFloat(x []float32) (float32, error)

type ScalarQuantizer 

type ScalarQuantizer struct {
	// contains filtered or unexported fields
}

func NewScalarQuantizer 

func NewScalarQuantizer(data [][]float32, distance distancer.Provider) *ScalarQuantizer

func RestoreScalarQuantizer 

func RestoreScalarQuantizer(a, b float32, dimensions uint16, distance distancer.Provider) (*ScalarQuantizer, error)

func (*ScalarQuantizer) CompressedBytes 

func (sq *ScalarQuantizer) CompressedBytes(compressed []byte) []byte

func (*ScalarQuantizer) DistanceBetweenCompressedVectors 

func (sq *ScalarQuantizer) DistanceBetweenCompressedVectors(x, y []byte) (float32, error)

func (*ScalarQuantizer) Encode 

func (sq *ScalarQuantizer) Encode(vec []float32) []byte

func (*ScalarQuantizer) FromCompressedBytes 

func (sq *ScalarQuantizer) FromCompressedBytes(compressed []byte) []byte

func (*ScalarQuantizer) FromCompressedBytesWithSubsliceBuffer 

func (pq *ScalarQuantizer) FromCompressedBytesWithSubsliceBuffer(compressed []byte, buffer *[]byte) []byte

func (*ScalarQuantizer) NewCompressedQuantizerDistancer 

func (sq *ScalarQuantizer) NewCompressedQuantizerDistancer(a []byte) quantizerDistancer[byte]

func (*ScalarQuantizer) NewDistancer 

func (sq *ScalarQuantizer) NewDistancer(a []float32) *SQDistancer

func (*ScalarQuantizer) NewQuantizerDistancer 

func (sq *ScalarQuantizer) NewQuantizerDistancer(a []float32) quantizerDistancer[byte]

func (*ScalarQuantizer) PersistCompression 

func (sq *ScalarQuantizer) PersistCompression(logger CommitLogger)

func (*ScalarQuantizer) ReturnQuantizerDistancer 

func (sq *ScalarQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[byte])

type TileEncoder 

type TileEncoder struct {
	// contains filtered or unexported fields
}

func NewTileEncoder 

func NewTileEncoder(bits int, segment int, encoderDistribution EncoderDistribution) *TileEncoder

func RestoreTileEncoder 

func RestoreTileEncoder(bins float64, mean float64, stdDev float64, size float64, s1 float64, s2 float64, segment uint16, encoderDistribution byte) *TileEncoder

func (*TileEncoder) Add 

func (te *TileEncoder) Add(x []float32)

func (*TileEncoder) Centroid 

func (te *TileEncoder) Centroid(b byte) []float32

func (*TileEncoder) Encode 

func (te *TileEncoder) Encode(x []float32) byte

func (*TileEncoder) ExposeDataForRestore 

func (te *TileEncoder) ExposeDataForRestore() []byte

func (*TileEncoder) Fit 

func (te *TileEncoder) Fit(data [][]float32) error

type VecAndID 

type VecAndID[T uint64 | byte] struct {
	Id  uint64
	Vec []T
}

type VectorCompressor 

type VectorCompressor interface {
	Drop() error
	GrowCache(size uint64)
	GrowMultiCache(id uint64)
	SetCacheMaxSize(size int64)
	GetCacheMaxSize() int64
	Delete(ctx context.Context, id uint64)
	Preload(id uint64, vector []float32)
	PreloadMulti(docID uint64, ids []uint64, vecs [][]float32)
	GetKeys(id uint64) (uint64, uint64)
	SetKeys(id uint64, docID uint64, relativeID uint64)
	Prefetch(id uint64)
	CountVectors() int64
	PrefillCache()

	DistanceBetweenCompressedVectorsFromIDs(ctx context.Context, x, y uint64) (float32, error)
	NewDistancer(vector []float32) (CompressorDistancer, ReturnDistancerFn)
	NewDistancerFromID(id uint64) (CompressorDistancer, error)
	NewBag() CompressionDistanceBag

	PersistCompression(CommitLogger)
}

func NewBQCompressor 

func NewBQCompressor(
	distance distancer.Provider,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func NewBQMultiCompressor 

func NewBQMultiCompressor(
	distance distancer.Provider,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func NewHNSWPQCompressor 

func NewHNSWPQCompressor(
	cfg hnsw.PQConfig,
	distance distancer.Provider,
	dimensions int,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	data [][]float32,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func NewHNSWSQCompressor 

func NewHNSWSQCompressor(
	distance distancer.Provider,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	data [][]float32,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func RestoreHNSWPQCompressor 

func RestoreHNSWPQCompressor(
	cfg hnsw.PQConfig,
	distance distancer.Provider,
	dimensions int,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	encoders []PQEncoder,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

func RestoreHNSWSQCompressor 

func RestoreHNSWSQCompressor(
	distance distancer.Provider,
	vectorCacheMaxObjects int,
	logger logrus.FieldLogger,
	a, b float32,
	dimensions uint16,
	store *lsmkv.Store,
	allocChecker memwatch.AllocChecker,
) (VectorCompressor, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.