ingest

Documentation

Index

• func ReadFile(fileName string, r DocumentReader, idx index.Index, opts interface{}, ...) error
• func ReadTerms(fileName string, r DocumentReader, idx index.Index, chunk int, ...) (finalTerms []string, err error)
• type DocumentReader
• type PmcReader
  • func (rr *PmcReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error
• type RedditReader
  • func (rr *RedditReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error
• type Stats
• type WikipediaAbstractsReader
  • func NewWikipediaAbstractsReader() *WikipediaAbstractsReader
  • func (r *WikipediaAbstractsReader) LoadScores(fileName string) error
  • func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

Functions

func ReadFile

func ReadFile(fileName string, r DocumentReader, idx index.Index, opts interface{}, chunk int, maxDocsToRead int64, indexingWorkers int) error

IngestDocuments ingests documents into an index using a DocumentReader

func ReadTerms

func ReadTerms(fileName string, r DocumentReader, idx index.Index, chunk int, maxDocsToRead int, maxTermsToProduce int, propertyName string, termStopWords []string) (finalTerms []string, err error)

IngestDocuments ingests documents into an index using a DocumentReader

Types

type DocumentReader

type DocumentReader interface {
	Read(io.Reader, chan index.Document, int, index.Index) error
}

DocumentReader implements parsing a data source and yielding documents

type PmcReader

type PmcReader struct{}

func (*PmcReader) Read

func (rr *PmcReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

type RedditReader

type RedditReader struct{}

func (*RedditReader) Read

func (rr *RedditReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error

type Stats

type Stats struct {
	TotalDocs             int64
	CurrentWindowDocs     int
	CurrentWindowDuration time.Duration
	CurrentWindowRate     float64
	CurrentWindowLatency  time.Duration
}

type WikipediaAbstractsReader

type WikipediaAbstractsReader struct {
	// contains filtered or unexported fields
}

func NewWikipediaAbstractsReader

func NewWikipediaAbstractsReader() *WikipediaAbstractsReader

func (*WikipediaAbstractsReader) LoadScores

func (r *WikipediaAbstractsReader) LoadScores(fileName string) error

func (*WikipediaAbstractsReader) Read

func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error