rindex

README

rindex

This is a repository inspired by rubiojr/rindex. My goal of this project is to build something similar to the rubiojr/rindex with a goal to learn about indexing using blugeindex for restic.

WARNING

DO NOT USE THIS FOR PRODUCTION. It's a hobby project.

Documentation

Index

• Variables
• type FieldVisitor
• type Filter
• type IndexOptions
• type IndexStats
  • func NewStats() IndexStats
  • func (s *IndexStats) AlreadyIndexedInc()
  • func (s *IndexStats) CurrentSnapshotFilesInc()
  • func (s *IndexStats) ErrorsAdd(err error)
  • func (s *IndexStats) IndexedFilesInc()
  • func (s *IndexStats) MismatchInc()
  • func (s *IndexStats) ScannedFilesInc()
  • func (s *IndexStats) ScannedNodesInc()
  • func (s *IndexStats) ScannedSnapshotsInc()
  • func (s *IndexStats) SetCurrentSnapshotTotalFiles(count uint64)
  • func (s *IndexStats) SetMissingSnapshots(count uint64)
  • func (s *IndexStats) SetSnapshotFiles(id string, count uint64)
  • func (s *IndexStats) SetTotalSnapshots(count uint64)
• type Indexer
  • func New(indexPath string, repo, pass string) (Indexer, error)
  • func NewOffline(indexPath string, repo, pass string) (Indexer, error)
  • func (i *Indexer) Close()
  • func (i *Indexer) Has(restic.ID) bool
  • func (i *Indexer) Index(ctx context.Context, opts IndexOptions, indexStats chan IndexStats) (IndexStats, error)
  • func (i *Indexer) MissingSnapshots(ctx context.Context) ([]string, error)
  • func (i *Indexer) SQLiteFileIndex(ctx context.Context, indexStats chan IndexStats) (IndexStats, error)
  • func (i *Indexer) Search(query string, fVisitor FieldVisitor, srVisitor SearchResultVisitor) (uint64, error)
• type MatchAllFilter
  • func (m *MatchAllFilter) ShouldIndex(path string) bool
• type SearchResultVisitor

Variables

var DefaultIndexOptions = IndexOptions{
	Filter:          &MatchAllFilter{},
	BatchSize:       2000,
	DocumentBuilder: blugeindex.FileDocumentBuilder{},
	// contains filtered or unexported fields
}

DefaultIndexOptions will: * Index all the files found * With batching disabled * Adding basic file metadata to every file being indexed * Using the default document builder

var ErrIndexLocked = errors.New("there's an Indexer instance running")

var ErrSearchNotReady = errors.New("not ready for search")

Types

type FieldVisitor

type FieldVisitor = func(field string, value []byte) bool

type Filter

type Filter interface {
	ShouldIndex(path string) bool
}

Filter determines if a file should be indexed or not

type IndexOptions

type IndexOptions struct {
	// The Filter decides if the file is indexed or not
	Filter Filter
	// Batching improves indexing speed at the cost of using
	// some more memory. Dramatically improves indexing speed
	// for large number of files.
	// 0 or 1 disables batching. Defaults to 1 (no batching) if not set.
	BatchSize uint
	// If set to true, all the repository snapshots and files will be scanned and re-indexed.
	Reindex bool
	// DocumentBuilder is responsible of creating the Bluge document that will be indexed
	DocumentBuilder blugeindex.DocumentBuilder

	Debug bool
	// contains filtered or unexported fields
}

IndexOptions to be passed to Index

type IndexStats

type IndexStats struct {
	// Files that did not pass the filter (not indexed)
	Mismatch uint64
	// Number of nodes (files or directories) scanned
	ScannedNodes uint64
	// Number of files indexed
	IndexedFiles uint64
	// Number of snapshots visited for indexing
	ScannedSnapshots uint64
	// Number of files previously indexed
	AlreadyIndexed uint64
	// Number of files scanned
	ScannedFiles uint64
	// Errors found while scanning or indexing files
	Errors []error
	// Last file indexed
	LastMatch string
	// Snapshots that will be scanned
	MissingSnapshots uint64

	// List of files in every snapshot
	SnapshotFiles map[string]uint64

	// Number of files scanned in the current snapshot
	CurrentSnapshotFiles uint64
	// Total number of files in the snapshot being scanned
	CurrentSnapshotTotalFiles uint64

	// Total number of snapshots
	TotalSnapshots uint64
	// contains filtered or unexported fields
}

IndexStats is returned every time an new document is indexed or when the indexing process finishes.

func NewStats

func NewStats() IndexStats

func (*IndexStats) AlreadyIndexedInc

func (s *IndexStats) AlreadyIndexedInc()

func (*IndexStats) CurrentSnapshotFilesInc

func (s *IndexStats) CurrentSnapshotFilesInc()

func (*IndexStats) ErrorsAdd

func (s *IndexStats) ErrorsAdd(err error)

func (*IndexStats) IndexedFilesInc

func (s *IndexStats) IndexedFilesInc()

func (*IndexStats) MismatchInc

func (s *IndexStats) MismatchInc()

func (*IndexStats) ScannedFilesInc

func (s *IndexStats) ScannedFilesInc()

func (*IndexStats) ScannedNodesInc

func (s *IndexStats) ScannedNodesInc()

func (*IndexStats) ScannedSnapshotsInc

func (s *IndexStats) ScannedSnapshotsInc()

func (*IndexStats) SetCurrentSnapshotTotalFiles

func (s *IndexStats) SetCurrentSnapshotTotalFiles(count uint64)

func (*IndexStats) SetMissingSnapshots

func (s *IndexStats) SetMissingSnapshots(count uint64)

func (*IndexStats) SetSnapshotFiles

func (s *IndexStats) SetSnapshotFiles(id string, count uint64)

func (*IndexStats) SetTotalSnapshots

func (s *IndexStats) SetTotalSnapshots(count uint64)

type Indexer

type Indexer struct {
	// The Restic repository location (RESTIC_REPOSITORY)
	RepositoryLocation string
	// The Restic repository password (RESTIC_PASSWORD)
	RepositoryPassword string
	// The path to the directory that will hold the index
	IndexPath string
	// IndexingEngine being used (Bluge is the only one supported right now)
	IndexEngine *blugeindex.BlugeIndex
	// contains filtered or unexported fields
}

func New

func New(indexPath string, repo, pass string) (Indexer, error)

New creates a new Indexer. indexPath is the path to the directory that will contain the index files.

func NewOffline

func NewOffline(indexPath string, repo, pass string) (Indexer, error)

func (*Indexer) Close

func (i *Indexer) Close()

func (*Indexer) Has

func (i *Indexer) Has(restic.ID) bool

func (*Indexer) Index

func (i *Indexer) Index(ctx context.Context, opts IndexOptions, indexStats chan IndexStats) (IndexStats, error)

Index will start indexing the repository.

A channel can be passed to follow the indexing process in real time. IndexStats is sent to the channel every time a new file is indexed.

func (*Indexer) MissingSnapshots

func (i *Indexer) MissingSnapshots(ctx context.Context) ([]string, error)

FIXME: this can't be called when indexing

func (*Indexer) SQLiteFileIndex

func (i *Indexer) SQLiteFileIndex(ctx context.Context, indexStats chan IndexStats) (IndexStats, error)

func (*Indexer) Search

func (i *Indexer) Search(query string, fVisitor FieldVisitor, srVisitor SearchResultVisitor) (uint64, error)

Search searches the index and calls srVisitor for every result obtained and fVisitor for every field in that search result.

type MatchAllFilter

type MatchAllFilter struct{}

MatchAllFilter will index all files found

func (*MatchAllFilter) ShouldIndex

func (m *MatchAllFilter) ShouldIndex(path string) bool

ShouldIndex returns true for every file foud

type SearchResultVisitor

type SearchResultVisitor = func() bool