pipeline

Documentation

Overview

pipeline is a package used by the bookpipeline command, which handles the core functionality, using channels heavily to coordinate jobs. Note that it is considered an "internal" package, not intended for external use, and no guarantee is made of the stability of any interfaces provided.

Index

• Constants
• func Analyse(conn Pipeliner) func(chan string, chan string, chan error, *log.Logger)
• func CheckImages(dir string) error
• func DetectQueueType(dir string, conn Pipeliner) string
• func DownloadAll(name string, conn Pipeliner) error
• func DownloadAnalyses(name string, conn Pipeliner) error
• func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error
• func DownloadPdfs(name string, conn Pipeliner) error
• func GetMailSettings() (mailSettings, error)
• func Ocr(training string, tesscmd string) func(chan string, chan string, chan error, *log.Logger)
• func OcrPage(msg bookpipeline.Qmsg, conn Pipeliner, ...) error
• func Preprocess(thresholds []float64) func(chan string, chan string, chan error, *log.Logger)
• func ProcessBook(msg bookpipeline.Qmsg, conn Pipeliner, ...) error
• func SaveLogs(conn Pipeliner, starttime int64, hostname string) error
• func UploadImages(dir string, bookname string, conn Pipeliner) error
• func Wipe(towipe chan string, up chan string, errc chan error, logger *log.Logger)
• type Clouder
• type MinPipeliner
• type NullWriter
  • func (w NullWriter) Write(p []byte) (n int, err error)
• type Pipeliner

Constants

const HeartbeatSeconds = 60

Functions

func Analyse

func Analyse(conn Pipeliner) func(chan string, chan string, chan error, *log.Logger)

func CheckImages

func CheckImages(dir string) error

func DetectQueueType

func DetectQueueType(dir string, conn Pipeliner) string

func DownloadAll

func DownloadAll(name string, conn Pipeliner) error

func DownloadAnalyses

func DownloadAnalyses(name string, conn Pipeliner) error

func DownloadBestPages

func DownloadBestPages(name string, conn Pipeliner, pluspngs bool) error

func DownloadPdfs

func DownloadPdfs(name string, conn Pipeliner) error

func GetMailSettings

func GetMailSettings() (mailSettings, error)

func Ocr

func Ocr(training string, tesscmd string) func(chan string, chan string, chan error, *log.Logger)

func OcrPage

func OcrPage(msg bookpipeline.Qmsg, conn Pipeliner, process func(chan string, chan string, chan error, *log.Logger), fromQueue string, toQueue string) error

OcrPage OCRs a page based on a message. It may make sense to roll this back into processBook (on which it is based) once working well.

func Preprocess

func Preprocess(thresholds []float64) func(chan string, chan string, chan error, *log.Logger)

func ProcessBook

func ProcessBook(msg bookpipeline.Qmsg, conn Pipeliner, process func(chan string, chan string, chan error, *log.Logger), match *regexp.Regexp, fromQueue string, toQueue string) error

func SaveLogs

func SaveLogs(conn Pipeliner, starttime int64, hostname string) error

func UploadImages

func UploadImages(dir string, bookname string, conn Pipeliner) error

func Wipe

func Wipe(towipe chan string, up chan string, errc chan error, logger *log.Logger)

Types

type Clouder

type Clouder interface {
	Init() error
	ListObjects(bucket string, prefix string) ([]string, error)
	Download(bucket string, key string, fn string) error
	Upload(bucket string, key string, path string) error
	CheckQueue(url string, timeout int64) (bookpipeline.Qmsg, error)
	AddToQueue(url string, msg string) error
	DelFromQueue(url string, handle string) error
	QueueHeartbeat(msg bookpipeline.Qmsg, qurl string, duration int64) (bookpipeline.Qmsg, error)
}

type MinPipeliner

type MinPipeliner interface {
	Pipeliner
	MinimalInit() error
}

type NullWriter

type NullWriter bool

null writer to enable non-verbose logging to be discarded

func (NullWriter) Write

func (w NullWriter) Write(p []byte) (n int, err error)

type Pipeliner

type Pipeliner interface {
	Clouder
	PreQueueId() string
	WipeQueueId() string
	OCRPageQueueId() string
	AnalyseQueueId() string
	WIPStorageId() string
	GetLogger() *log.Logger
	Log(v ...interface{})
}