index

Documentation

Index

• Variables
• func AnalysisWorker(q AnalysisQueue)
• type AnalysisQueue
  • func NewAnalysisQueue(numWorkers int) *AnalysisQueue
  • func (q *AnalysisQueue) Close()
  • func (q *AnalysisQueue) Queue(work *AnalysisWork)
• type AnalysisResult
• type AnalysisWork
  • func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork
• type Batch
  • func NewBatch() *Batch
  • func (b *Batch) Delete(id string)
  • func (b *Batch) DeleteInternal(key []byte)
  • func (b *Batch) Reset()
  • func (b *Batch) SetInternal(key, val []byte)
  • func (b *Batch) String() string
  • func (b *Batch) Update(doc *document.Document)
• type DictEntry
• type DocIDReader
• type DocumentFieldTermVisitor
• type FieldCache
  • func NewFieldCache() *FieldCache
  • func (f *FieldCache) AddExisting(field string, index uint16)
  • func (f *FieldCache) FieldIndexed(index uint16) (field string)
  • func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool)
• type FieldDict
• type FieldTerms
  • func (f FieldTerms) FieldsNotYetCached(fields []string) []string
  • func (f FieldTerms) Merge(other FieldTerms)
• type Index
• type IndexInternalID
  • func (id IndexInternalID) Compare(other IndexInternalID) int
  • func (id IndexInternalID) Equals(other IndexInternalID) bool
• type IndexReader
• type IndexRow
• type TermFieldDoc
  • func (tfd *TermFieldDoc) Reset() *TermFieldDoc
• type TermFieldReader
• type TermFieldVector

Variables

var ErrorUnknownStorageType = fmt.Errorf("unknown storage type")

Functions

func AnalysisWorker

func AnalysisWorker(q AnalysisQueue)

Types

type AnalysisQueue

type AnalysisQueue struct {
	// contains filtered or unexported fields
}

func NewAnalysisQueue

func NewAnalysisQueue(numWorkers int) *AnalysisQueue

func (*AnalysisQueue) Close

func (q *AnalysisQueue) Close()

func (*AnalysisQueue) Queue

func (q *AnalysisQueue) Queue(work *AnalysisWork)

type AnalysisResult

type AnalysisResult struct {
	DocID string
	Rows  []IndexRow

	// scorch
	Document *document.Document
	Analyzed []analysis.TokenFrequencies
	Length   []int
}

type AnalysisWork

type AnalysisWork struct {
	// contains filtered or unexported fields
}

func NewAnalysisWork

func NewAnalysisWork(i Index, d *document.Document, rc chan *AnalysisResult) *AnalysisWork

type Batch

type Batch struct {
	IndexOps    map[string]*document.Document
	InternalOps map[string][]byte
}

func NewBatch

func NewBatch() *Batch

func (*Batch) Delete

func (b *Batch) Delete(id string)

func (*Batch) DeleteInternal

func (b *Batch) DeleteInternal(key []byte)

func (*Batch) Reset

func (b *Batch) Reset()

func (*Batch) SetInternal

func (b *Batch) SetInternal(key, val []byte)

func (*Batch) String

func (b *Batch) String() string

func (*Batch) Update

func (b *Batch) Update(doc *document.Document)

type DictEntry

type DictEntry struct {
	Term  string
	Count uint64
}

type DocIDReader

type DocIDReader interface {
	// Next returns the next document internal identifier in the natural
	// index order, nil when the end of the sequence is reached.
	Next() (IndexInternalID, error)

	// Advance resets the iteration to the first internal identifier greater than
	// or equal to ID. If ID is smaller than the start of the range, the iteration
	// will start there instead. If ID is greater than or equal to the end of
	// the range, Next() call will return io.EOF.
	Advance(ID IndexInternalID) (IndexInternalID, error)
	Close() error
}

DocIDReader is the interface exposing enumeration of documents identifiers. Close the reader to release associated resources.

type DocumentFieldTermVisitor

type DocumentFieldTermVisitor func(field string, term []byte)

type FieldCache

type FieldCache struct {
	// contains filtered or unexported fields
}

func NewFieldCache

func NewFieldCache() *FieldCache

func (*FieldCache) AddExisting

func (f *FieldCache) AddExisting(field string, index uint16)

func (*FieldCache) FieldIndexed

func (f *FieldCache) FieldIndexed(index uint16) (field string)

func (*FieldCache) FieldNamed

func (f *FieldCache) FieldNamed(field string, createIfMissing bool) (uint16, bool)

FieldNamed returns the index of the field, and whether or not it existed before this call. if createIfMissing is true, and new field index is assigned but the second return value will still be false

type FieldDict

type FieldDict interface {
	Next() (*DictEntry, error)
	Close() error
}

type FieldTerms

type FieldTerms map[string][]string

FieldTerms contains the terms used by a document, keyed by field

func (FieldTerms) FieldsNotYetCached

func (f FieldTerms) FieldsNotYetCached(fields []string) []string

FieldsNotYetCached returns a list of fields not yet cached out of a larger list of fields

func (FieldTerms) Merge

func (f FieldTerms) Merge(other FieldTerms)

Merge will combine two FieldTerms it assumes that the terms lists are complete (thus do not need to be merged) field terms from the other list always replace the ones in the receiver

type Index

type Index interface {
	Open() error
	Close() error

	Update(doc *document.Document) error
	Delete(id string) error
	Batch(batch *Batch) error

	SetInternal(key, val []byte) error
	DeleteInternal(key []byte) error

	// Reader returns a low-level accessor on the index data. Close it to
	// release associated resources.
	Reader() (IndexReader, error)

	Stats() json.Marshaler
	StatsMap() map[string]interface{}

	Analyze(d *document.Document) *AnalysisResult

	Advanced() (store.KVStore, error)
}

type IndexInternalID

type IndexInternalID []byte

IndexInternalID is an opaque document identifier interal to the index impl

func (IndexInternalID) Compare

func (id IndexInternalID) Compare(other IndexInternalID) int

func (IndexInternalID) Equals

func (id IndexInternalID) Equals(other IndexInternalID) bool

type IndexReader

type IndexReader interface {
	TermFieldReader(term []byte, field string, includeFreq, includeNorm, includeTermVectors bool) (TermFieldReader, error)

	// DocIDReader returns an iterator over all doc ids
	// The caller must close returned instance to release associated resources.
	DocIDReaderAll() (DocIDReader, error)

	DocIDReaderOnly(ids []string) (DocIDReader, error)

	FieldDict(field string) (FieldDict, error)

	// FieldDictRange is currently defined to include the start and end terms
	FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)

	Document(id string) (*document.Document, error)
	DocumentVisitFieldTerms(id IndexInternalID, fields []string, visitor DocumentFieldTermVisitor) error

	Fields() ([]string, error)

	GetInternal(key []byte) ([]byte, error)

	DocCount() (uint64, error)

	ExternalID(id IndexInternalID) (string, error)
	InternalID(id string) (IndexInternalID, error)

	DumpAll() chan interface{}
	DumpDoc(id string) chan interface{}
	DumpFields() chan interface{}

	Close() error
}

type IndexRow

type IndexRow interface {
	KeySize() int
	KeyTo([]byte) (int, error)
	Key() []byte

	ValueSize() int
	ValueTo([]byte) (int, error)
	Value() []byte
}

type TermFieldDoc

type TermFieldDoc struct {
	Term    string
	ID      IndexInternalID
	Freq    uint64
	Norm    float64
	Vectors []*TermFieldVector
}

func (*TermFieldDoc) Reset

func (tfd *TermFieldDoc) Reset() *TermFieldDoc

Reset allows an already allocated TermFieldDoc to be reused

type TermFieldReader

type TermFieldReader interface {
	// Next returns the next document containing the term in this field, or nil
	// when it reaches the end of the enumeration.  The preAlloced TermFieldDoc
	// is optional, and when non-nil, will be used instead of allocating memory.
	Next(preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Advance resets the enumeration at specified document or its immediate
	// follower.
	Advance(ID IndexInternalID, preAlloced *TermFieldDoc) (*TermFieldDoc, error)

	// Count returns the number of documents contains the term in this field.
	Count() uint64
	Close() error
}

TermFieldReader is the interface exposing the enumeration of documents containing a given term in a given field. Documents are returned in byte lexicographic order over their identifiers.

type TermFieldVector

type TermFieldVector struct {
	Field          string
	ArrayPositions []uint64
	Pos            uint64
	Start          uint64
	End            uint64
}