worker

Documentation

Overview

Package worker provides the workers that do the actual loading & processing of pages.

Index

• func Mangle(basename string) []string
• type HTMLWorker
  • func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker
  • func (*HTMLWorker) Eligible(resp *http.Response) bool
  • func (*HTMLWorker) GetLinks(body io.Reader) []string
  • func (w *HTMLWorker) Handle(t *task.Task, body io.Reader, result *results.Result)
• type PageWorker
• type Stoppable
• type Worker
  • func NewWorker(settings *ss.ScanSettings, factory client.ClientFactory, src <-chan *task.Task, ...) *Worker
  • func StartWorkers(settings *ss.ScanSettings, factory client.ClientFactory, src <-chan *task.Task, ...) []*Worker
  • func (w *Worker) HandleTask(t *task.Task)
  • func (w *Worker) KeepSpidering(code int) bool
  • func (w *Worker) ResultForError(t *task.Task, resp *http.Response, err error) *results.Result
  • func (w *Worker) ResultForResponse(t *task.Task, resp *http.Response) *results.Result
  • func (w *Worker) Run()
  • func (w *Worker) RunInBackground()
  • func (w *Worker) SetPageWorker(pw PageWorker)
  • func (w *Worker) Sleep()
  • func (w *Worker) Stop()
  • func (w *Worker) TryMangleTask(t *task.Task)
  • func (w *Worker) TryTask(t *task.Task) int
  • func (w *Worker) Wait()

Functions

func Mangle

func Mangle(basename string) []string

Mangle a basename

Types

type HTMLWorker

type HTMLWorker struct {
	// contains filtered or unexported fields
}

func NewHTMLWorker

func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker

func (*HTMLWorker) Eligible

func (*HTMLWorker) Eligible(resp *http.Response) bool

Check if this response can be handled by this worker

func (*HTMLWorker) GetLinks

func (*HTMLWorker) GetLinks(body io.Reader) []string

Get the links for the body.

func (*HTMLWorker) Handle

func (w *HTMLWorker) Handle(t *task.Task, body io.Reader, result *results.Result)

Work on this response

type PageWorker

type PageWorker interface {
	Eligible(*http.Response) bool
	Handle(*task.Task, io.Reader, *results.Result)
}

type Stoppable

type Stoppable interface {
	Stop()
}

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

Workers do the work of connecting to the server, issuing the request, and then optionally parsing the response. Normally a pool of several workers will be used due to network latency.

func NewWorker

func NewWorker(settings *ss.ScanSettings,
	factory client.ClientFactory,
	src <-chan *task.Task,
	adder workqueue.QueueAddFunc,
	done workqueue.QueueDoneFunc,
	rchan chan<- *results.Result) *Worker

Construct a worker with given settings.

func StartWorkers

func StartWorkers(settings *ss.ScanSettings,
	factory client.ClientFactory,
	src <-chan *task.Task,
	adder workqueue.QueueAddFunc,
	done workqueue.QueueDoneFunc,
	rchan chan<- *results.Result) []*Worker

Starts a batch of workers based on the relevant settings.

func (*Worker) HandleTask

func (w *Worker) HandleTask(t *task.Task)

func (*Worker) KeepSpidering

func (w *Worker) KeepSpidering(code int) bool

Should we keep spidering from this code?

func (*Worker) ResultForError

func (w *Worker) ResultForError(t *task.Task, resp *http.Response, err error) *results.Result

func (*Worker) ResultForResponse

func (w *Worker) ResultForResponse(t *task.Task, resp *http.Response) *results.Result

func (*Worker) Run

func (w *Worker) Run()

Run the worker, processing input from a channel until either signalled to stop or the input channel is closed.

func (*Worker) RunInBackground

func (w *Worker) RunInBackground()

func (*Worker) SetPageWorker

func (w *Worker) SetPageWorker(pw PageWorker)

func (*Worker) Sleep

func (w *Worker) Sleep()

func (*Worker) Stop

func (w *Worker) Stop()

func (*Worker) TryMangleTask

func (w *Worker) TryMangleTask(t *task.Task)

func (*Worker) TryTask

func (w *Worker) TryTask(t *task.Task) int

func (*Worker) Wait

func (w *Worker) Wait()