pages

Documentation

Index

• func CleanPages(cleaned chan<- protos.ID) jobs.Page
• func FilterTitles(titles []string) func(chan<- protos.ID) jobs.Page
• func Run(ctx context.Context, source Source, parallel int, ...) error
• type Source
  • func StreamDB(inDBPath string, parallel int) Source
  • func StreamDBKeys(inDBPath string, parallel int, keys []uint) Source

Functions

func CleanPages

func CleanPages(cleaned chan<- protos.ID) jobs.Page

func FilterTitles

func FilterTitles(titles []string) func(chan<- protos.ID) jobs.Page

FilterTitles filters a source channel of pages to only include those with the specified Title.

func Run

func Run(ctx context.Context, source Source, parallel int, job func(chan<- protos.ID) jobs.Page, sink protos.Sink) error

Types

type Source

type Source func(ctx context.Context, errs chan<- error) (<-chan *documents.Page, error)

func StreamDB

func StreamDB(inDBPath string, parallel int) Source

func StreamDBKeys

func StreamDBKeys(inDBPath string, parallel int, keys []uint) Source