index

package
v0.0.0-...-ac0f108 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 29, 2024 License: MIT Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const Bm25B = 0.75
View Source 
const Bm25K1 = 1.2
View Source 
const FieldLengthSize = 256

Variables

View Source 
var FieldLengthTable = [FieldLengthSize]uint64{}/* 256 elements not displayed */

Functions

func ToGlobalDocId 

func ToGlobalDocId(segmentId, localDocId uint32) uint64

func ToSegmentId 

func ToSegmentId(docId uint64) uint32

Types

type ArrayStoreReader 

type ArrayStoreReader struct {
	// contains filtered or unexported fields
}

func (*ArrayStoreReader) Get 

func (reader *ArrayStoreReader) Get(position uint32) []byte

type ArrayStoreWriter 

type ArrayStoreWriter struct {
	// contains filtered or unexported fields
}

func (*ArrayStoreWriter) Append 

func (writer *ArrayStoreWriter) Append(value []byte) error

type Commit 

type Commit struct {
	SegmentIds []uint32 `json:"segmentIds"`
	DeletedId  *uint32  `json:"deletedId,omitempty"`
}

type DeletedReader 

type DeletedReader interface {
	GetDeletedDocIdsForSegment(segmentId uint32) (*roaring.Bitmap, error)
}

type DeletedWriter 

type DeletedWriter struct {
	// contains filtered or unexported fields
}

func (*DeletedWriter) DeletedDocs 

func (writer *DeletedWriter) DeletedDocs(deletedDocIdsBySegment map[uint32]*roaring.Bitmap)

TODO: Need to revisit, weird API, since we do most of the work outside of DeletedWriter

func (*DeletedWriter) Write 

func (writer *DeletedWriter) Write(directory string, deletedId string) error

type DictionaryReader 

type DictionaryReader struct {
	// contains filtered or unexported fields
}

func (*DictionaryReader) Get 

func (reader *DictionaryReader) Get(term []byte) *TermInfo

type DictionaryWriter 

type DictionaryWriter struct {
	// contains filtered or unexported fields
}

func (*DictionaryWriter) Close 

func (writer *DictionaryWriter) Close() error

func (*DictionaryWriter) Write 

func (writer *DictionaryWriter) Write(term []byte, termInfo *TermInfo) error

type DocFieldLengthReader 

type DocFieldLengthReader struct {
	// contains filtered or unexported fields
}

func (*DocFieldLengthReader) FieldLengthReader 

func (reader *DocFieldLengthReader) FieldLengthReader(fieldName string) (*FieldLengthReader, error)

type Document 

type Document []Field

type DocumentId 

type DocumentId uint32

type Field 

type Field struct {
	FieldType FieldType
	Name      string
	Value     []byte
}

type FieldFreqsReader 

type FieldFreqsReader struct {
	// contains filtered or unexported fields
}

func (*FieldFreqsReader) TermFreqsIterator 

func (reader *FieldFreqsReader) TermFreqsIterator(termInfo *TermInfo) *TermFreqsIterator

type FieldFreqsWriter 

type FieldFreqsWriter struct {
	// contains filtered or unexported fields
}

func (*FieldFreqsWriter) Close 

func (w *FieldFreqsWriter) Close() error

func (*FieldFreqsWriter) WriteBlock 

func (writer *FieldFreqsWriter) WriteBlock(docIds []uint32, termFreqs []uint64, minFieldLengthId byte) (uint64, uint64, error)

type FieldLengthNorms 

type FieldLengthNorms struct {
	// contains filtered or unexported fields
}

For a specific doc

func NewFieldLengthNorms 

func NewFieldLengthNorms(fieldLengthReaders []*FieldLengthReader, precomputedLengthNorms [][]float32) *FieldLengthNorms

func (*FieldLengthNorms) Get 

func (ctx *FieldLengthNorms) Get(fieldIndex int) float32

func (*FieldLengthNorms) SetDocId 

func (ctx *FieldLengthNorms) SetDocId(newDocId DocumentId)

type FieldLengthReader 

type FieldLengthReader struct {
	// contains filtered or unexported fields
}

func (*FieldLengthReader) GetId 

func (reader *FieldLengthReader) GetId(docId DocumentId) (byte, error)

type FieldStatsReader 

type FieldStatsReader struct {
	// contains filtered or unexported fields
}

func (*FieldStatsReader) Read 

func (reader *FieldStatsReader) Read() (uint32, uint64, error)

type FieldStatsWriter 

type FieldStatsWriter struct {
	// contains filtered or unexported fields
}

func (*FieldStatsWriter) Write 

func (writer *FieldStatsWriter) Write(docCount uint32, sumTermFreq uint64) error

type FieldStoreReader 

type FieldStoreReader struct {
	// contains filtered or unexported fields
}

func (*FieldStoreReader) Value 

func (reader *FieldStoreReader) Value(docId DocumentId) []byte

type FieldType 

type FieldType int
const (
	TextFieldType FieldType = iota
	ByteFieldType
)

type FileDeletedReader 

type FileDeletedReader struct {
	// contains filtered or unexported fields
}

func (*FileDeletedReader) GetDeletedDocIdsForSegment 

func (reader *FileDeletedReader) GetDeletedDocIdsForSegment(segmentId uint32) (*roaring.Bitmap, error)

type FileReader 

type FileReader struct {
	// contains filtered or unexported fields
}

func (*FileReader) Slice 

func (reader *FileReader) Slice(start, end uint64) []byte

type IndexReader 

type IndexReader struct {
	SegmentReaders []*SegmentReader
}

func NewIndexReader 

func NewIndexReader(directory string) (*IndexReader, error)

func (*IndexReader) SearchByExactValues 

func (reader *IndexReader) SearchByExactValues(fieldName string, values [][]byte) ([]uint64, error)

func (*IndexReader) Value 

func (reader *IndexReader) Value(fieldName string, docId uint64) ([]byte, error)

type IndexWriter 

type IndexWriter struct {
	// contains filtered or unexported fields
}

func NewIndexWriter 

func NewIndexWriter(directory string) *IndexWriter

func (*IndexWriter) AddDocuments 

func (writer *IndexWriter) AddDocuments(docs []Document) error

func (*IndexWriter) DeleteDocuments 

func (writer *IndexWriter) DeleteDocuments(fieldName string, values [][]byte) error

type InvertedIndexWriter 

type InvertedIndexWriter struct {
	// contains filtered or unexported fields
}

func (*InvertedIndexWriter) Doc 

func (writer *InvertedIndexWriter) Doc(docId DocumentId)

func (*InvertedIndexWriter) EndField 

func (w *InvertedIndexWriter) EndField()

func (*InvertedIndexWriter) Field 

func (w *InvertedIndexWriter) Field(fieldName string, value []byte)

func (*InvertedIndexWriter) Term 

func (w *InvertedIndexWriter) Term(term []byte)

func (*InvertedIndexWriter) Write 

func (w *InvertedIndexWriter) Write(directory, segmentId string) error

type KVStoreReader 

type KVStoreReader struct {
	// contains filtered or unexported fields
}

func (*KVStoreReader) Close 

func (kv *KVStoreReader) Close() error

func (*KVStoreReader) Get 

func (kv *KVStoreReader) Get(key []byte) []byte

type KVStoreWriter 

type KVStoreWriter struct {
	// contains filtered or unexported fields
}

func (*KVStoreWriter) Append 

func (w *KVStoreWriter) Append(key []byte, values ...[]byte) error

Caller is responsible to check that keys are inserted in order

func (*KVStoreWriter) Close 

func (w *KVStoreWriter) Close() error

type NullDeletedReader 

type NullDeletedReader struct {
}

func (*NullDeletedReader) GetDeletedDocIdsForSegment 

func (reader *NullDeletedReader) GetDeletedDocIdsForSegment(segmentId uint32) (*roaring.Bitmap, error)

type Posting 

type Posting struct {
	// contains filtered or unexported fields
}

type SegmentComponentWriter 

type SegmentComponentWriter interface {
	Doc(docId DocumentId)
	Field(fieldName string, value []byte)
	EndField()
	Term(term []byte)
	Write(directory, segmentId string) error
}

Caller calls in order: - Doc() - Field() - Term() - Term() - ... - Field() - Term() - Term() - ... - Doc() - ... - Write()

type SegmentReader 

type SegmentReader struct {
	DeletedDocIds *roaring.Bitmap

	DocLengthReader *DocFieldLengthReader

	Id       uint32
	IdString string
	// contains filtered or unexported fields
}

func (*SegmentReader) DictionaryReader 

func (reader *SegmentReader) DictionaryReader(fieldName string) (*DictionaryReader, error)

func (*SegmentReader) DocCountAndSumTermFreqForField 

func (reader *SegmentReader) DocCountAndSumTermFreqForField(fieldName string) (uint32, uint64, error)

func (*SegmentReader) FieldFreqsReader 

func (reader *SegmentReader) FieldFreqsReader(fieldName string) (*FieldFreqsReader, error)

type StandardTokenizer 

type StandardTokenizer struct {
	// contains filtered or unexported fields
}

func NewStandardTokenizer 

func NewStandardTokenizer() *StandardTokenizer

func (*StandardTokenizer) NextToken 

func (t *StandardTokenizer) NextToken() (*Token, bool)

Token is valid until the next call to NextToken

func (*StandardTokenizer) Reset 

func (t *StandardTokenizer) Reset(input []byte)

type StoreReader 

type StoreReader struct {
	// contains filtered or unexported fields
}

func (*StoreReader) GetFieldStoreReader 

func (reader *StoreReader) GetFieldStoreReader(fieldName string) (*FieldStoreReader, error)

type StoreWriter 

type StoreWriter struct {
	// contains filtered or unexported fields
}

func (*StoreWriter) Doc 

func (writer *StoreWriter) Doc(docId DocumentId)

func (*StoreWriter) EndField 

func (writer *StoreWriter) EndField()

func (*StoreWriter) Field 

func (writer *StoreWriter) Field(fieldName string, value []byte)

func (*StoreWriter) Term 

func (writer *StoreWriter) Term(term []byte)

func (*StoreWriter) Write 

func (writer *StoreWriter) Write(directory, segmentId string) error

type TermFreqsIterator 

type TermFreqsIterator struct {
	LastDocId DocumentId
	// contains filtered or unexported fields
}

func (*TermFreqsIterator) BlockMaxFreqMinLengthId 

func (it *TermFreqsIterator) BlockMaxFreqMinLengthId() (uint64, byte)

func (*TermFreqsIterator) DocId 

func (it *TermFreqsIterator) DocId() DocumentId

func (*TermFreqsIterator) Next 

func (it *TermFreqsIterator) Next(docId DocumentId) bool

func (*TermFreqsIterator) NextShallow 

func (it *TermFreqsIterator) NextShallow(docId DocumentId) bool

func (*TermFreqsIterator) TermFreq 

func (it *TermFreqsIterator) TermFreq() uint64

type TermInfo 

type TermInfo struct {
	DocFreq              uint32
	FreqsFileStartOffset uint64
	FreqsFileEndOffset   uint64
}

type Token 

type Token struct {
	Text []byte
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.