helpers

Documentation

Index

• Constants
• Variables
• func AnnotateSlowQueryLog(ctx context.Context, key string, value any)
• func AnnotateSlowQueryLogAppend(ctx context.Context, key string, value any)
• func BucketFromPropNameLSM(propName string) string
• func BucketFromPropNameLengthLSM(propName string) string
• func BucketFromPropNameMetaCountLSM(propName string) string
• func BucketFromPropNameNullLSM(propName string) string
• func BucketRangeableFromPropNameLSM(propName string) string
• func BucketSearchableFromPropNameLSM(propName string) string
• func ExtractSlowQueryDetails(ctx context.Context) map[string]any
• func InitSlowQueryDetails(ctx context.Context) context.Context
• func MetaCountProp(propName string) string
• func PropLength(propName string) string
• func PropNull(propName string) string
• func TempBucketFromBucketName(bucketName string) string
• func Tokenize(tokenization string, in string) []string
• func TokenizeAndCountDuplicates(tokenization string, in string) ([]string, []int)
• func TokenizeWithWildcards(tokenization string, in string) []string
• type AllowList
  • func NewAllowList(ids ...uint64) AllowList
  • func NewAllowListFromBitmap(bm *sroar.Bitmap) AllowList
  • func NewAllowListFromBitmapDeepCopy(bm *sroar.Bitmap) AllowList
• type AllowListIterator
• type BaseSlowReporter
  • func NewSlowQueryReporter(threshold time.Duration, logger logrus.FieldLogger) *BaseSlowReporter
  • func (sq *BaseSlowReporter) LogIfSlow(ctx context.Context, startTime time.Time, fields map[string]any)
• type KagomeTokenizers
• type NoopSlowReporter
  • func (sq *NoopSlowReporter) LogIfSlow(ctx context.Context, startTime time.Time, fields map[string]any)
• type SlowQueryDetails
  • func NewSlowQueryDetails() *SlowQueryDetails
• type SlowQueryReporter
  • func NewSlowQueryReporterFromEnv(logger logrus.FieldLogger) SlowQueryReporter

Constants

const (
	ObjectsBucketLSMDocIDSecondaryIndex int = iota
	ObjectsBucketLSMTokenRangeSecondaryIndex
)

Variables

var (
	ObjectsBucket              = []byte("objects")
	ObjectsBucketLSM           = "objects"
	VectorsCompressedBucketLSM = "vectors_compressed"
	VectorsBucketLSM           = "vectors"
	DimensionsBucketLSM        = "dimensions"
)

var (
	UseGse          = false
	KagomeKrEnabled = false
	KagomeJaEnabled = false
)

var Tokenizations []string = []string{
	models.PropertyTokenizationWord,
	models.PropertyTokenizationLowercase,
	models.PropertyTokenizationWhitespace,
	models.PropertyTokenizationField,
	models.PropertyTokenizationTrigram,
}

Optional tokenizers can be enabled with an environment variable like: 'ENABLE_TOKENIZER_XXX', e.g. 'ENABLE_TOKENIZER_GSE', 'ENABLE_TOKENIZER_KAGOME_KR'

Functions

func AnnotateSlowQueryLog

func AnnotateSlowQueryLog(ctx context.Context, key string, value any)

func AnnotateSlowQueryLogAppend

func AnnotateSlowQueryLogAppend(ctx context.Context, key string, value any)

func BucketFromPropNameLSM

func BucketFromPropNameLSM(propName string) string

BucketFromPropNameLSM creates string used as the bucket name for a particular prop in the inverted index

func BucketFromPropNameLengthLSM

func BucketFromPropNameLengthLSM(propName string) string

func BucketFromPropNameMetaCountLSM

func BucketFromPropNameMetaCountLSM(propName string) string

func BucketFromPropNameNullLSM

func BucketFromPropNameNullLSM(propName string) string

func BucketRangeableFromPropNameLSM

func BucketRangeableFromPropNameLSM(propName string) string

func BucketSearchableFromPropNameLSM

func BucketSearchableFromPropNameLSM(propName string) string

func ExtractSlowQueryDetails

func ExtractSlowQueryDetails(ctx context.Context) map[string]any

func InitSlowQueryDetails

func InitSlowQueryDetails(ctx context.Context) context.Context

func MetaCountProp

func MetaCountProp(propName string) string

MetaCountProp helps create an internally used propName for meta props that don't explicitly exist in the user schema, but are required for proper indexing, such as the count of arrays.

func PropLength

func PropLength(propName string) string

func PropNull

func PropNull(propName string) string

func TempBucketFromBucketName

func TempBucketFromBucketName(bucketName string) string

func Tokenize

func Tokenize(tokenization string, in string) []string

func TokenizeAndCountDuplicates

func TokenizeAndCountDuplicates(tokenization string, in string) ([]string, []int)

func TokenizeWithWildcards

func TokenizeWithWildcards(tokenization string, in string) []string

Types

type AllowList

type AllowList interface {
	Insert(ids ...uint64)
	Contains(id uint64) bool
	DeepCopy() AllowList
	WrapOnWrite() AllowList
	Slice() []uint64

	IsEmpty() bool
	Len() int
	Min() uint64
	Max() uint64
	Size() uint64
	Truncate(uint64) AllowList

	Iterator() AllowListIterator
	LimitedIterator(limit int) AllowListIterator
}

func NewAllowList

func NewAllowList(ids ...uint64) AllowList

func NewAllowListFromBitmap

func NewAllowListFromBitmap(bm *sroar.Bitmap) AllowList

func NewAllowListFromBitmapDeepCopy

func NewAllowListFromBitmapDeepCopy(bm *sroar.Bitmap) AllowList

type AllowListIterator

type AllowListIterator interface {
	Next() (uint64, bool)
	Len() int
}

type BaseSlowReporter

type BaseSlowReporter struct {
	// contains filtered or unexported fields
}

func NewSlowQueryReporter

func NewSlowQueryReporter(threshold time.Duration, logger logrus.FieldLogger) *BaseSlowReporter

func (*BaseSlowReporter) LogIfSlow

func (sq *BaseSlowReporter) LogIfSlow(ctx context.Context, startTime time.Time, fields map[string]any)

LogIfSlow prints a warning log if the request takes longer than the threshold. Usage:

	startTime := time.Now()
	defer s.slowQueryReporter.LogIfSlow(startTime, map[string]any{
		"key": "value"
  })

TODO (sebneira): Consider providing fields out of the box (e.g. shard info). Right now we're limited because of circular dependencies.

type KagomeTokenizers

type KagomeTokenizers struct {
	Korean   *kagomeTokenizer.Tokenizer
	Japanese *kagomeTokenizer.Tokenizer
}

type NoopSlowReporter

type NoopSlowReporter struct{}

NoopSlowReporter is used when the reporter is disabled.

func (*NoopSlowReporter) LogIfSlow

func (sq *NoopSlowReporter) LogIfSlow(ctx context.Context, startTime time.Time, fields map[string]any)

type SlowQueryDetails

type SlowQueryDetails struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewSlowQueryDetails

func NewSlowQueryDetails() *SlowQueryDetails

type SlowQueryReporter

type SlowQueryReporter interface {
	LogIfSlow(context.Context, time.Time, map[string]any)
}

func NewSlowQueryReporterFromEnv

func NewSlowQueryReporterFromEnv(logger logrus.FieldLogger) SlowQueryReporter