middleware

Documentation

Index

• type AllowedDomains
  • func (a *AllowedDomains) ProcessRequest(r *client.Request)
• type DuplicateRequests
  • func (a *DuplicateRequests) ProcessRequest(r *client.Request)
• type Headers
  • func (a *Headers) ProcessRequest(r *client.Request)
• type LogStats
  • func (p *LogStats) ProcessResponse(r *client.Response)
• type Metrics
  • func (a *Metrics) ProcessRequest(r *client.Request)
  • func (a *Metrics) ProcessResponse(r *client.Response)
• type ParseHTML
  • func (p *ParseHTML) ProcessResponse(r *client.Response)
• type RequestProcessor
  • func NewDelay(requestDelayRandomize bool, requestDelay time.Duration) RequestProcessor
  • func NewRobotsTxt(client *client.Client, metrics *metrics.Metrics, robotsDisabled bool) RequestProcessor
• type RequestResponseProcessor
• type ResponseProcessor
• type RobotsTxt
  • func (m *RobotsTxt) ProcessRequest(r *client.Request)

Types

type AllowedDomains

type AllowedDomains struct {
	AllowedDomains []string
	// contains filtered or unexported fields
}

AllowedDomains checks for request host if it exists in AllowedDomains

func (*AllowedDomains) ProcessRequest

func (a *AllowedDomains) ProcessRequest(r *client.Request)

type DuplicateRequests

type DuplicateRequests struct {
	RevisitEnabled bool
	// contains filtered or unexported fields
}

DuplicateRequests checks for already visited URLs

func (*DuplicateRequests) ProcessRequest

func (a *DuplicateRequests) ProcessRequest(r *client.Request)

type Headers

type Headers struct {
	UserAgent string
}

Headers sets default request headers

func (*Headers) ProcessRequest

func (a *Headers) ProcessRequest(r *client.Request)

type LogStats

type LogStats struct {
	LogDisabled bool
}

LogStats logs responses

func (*LogStats) ProcessResponse

func (p *LogStats) ProcessResponse(r *client.Response)

type Metrics

type Metrics struct {
	Metrics *metrics.Metrics
}

Metrics sets stats for request and responses

func (*Metrics) ProcessRequest

func (a *Metrics) ProcessRequest(r *client.Request)

func (*Metrics) ProcessResponse

func (a *Metrics) ProcessResponse(r *client.Response)

type ParseHTML

type ParseHTML struct {
	ParseHTMLDisabled bool
}

ParseHTML parses response if response is HTML

func (*ParseHTML) ProcessResponse

func (p *ParseHTML) ProcessResponse(r *client.Response)

type RequestProcessor

type RequestProcessor interface {
	ProcessRequest(r *client.Request)
}

RequestProcessor called before requests made. Set request.Cancelled = true to cancel request

func NewDelay

func NewDelay(requestDelayRandomize bool, requestDelay time.Duration) RequestProcessor

func NewRobotsTxt

func NewRobotsTxt(client *client.Client, metrics *metrics.Metrics, robotsDisabled bool) RequestProcessor

type RequestResponseProcessor

type RequestResponseProcessor interface {
	RequestProcessor
	ResponseProcessor
}

RequestResponseProcessor interface is for middlewares that needs to process both requests and responses

type ResponseProcessor

type ResponseProcessor interface {
	ProcessResponse(r *client.Response)
}

ResponseProcessor called after request response receive

type RobotsTxt

type RobotsTxt struct {
	// contains filtered or unexported fields
}

RobotsTxt middleware filters out requests forbidden by the robots.txt exclusion standard.

func (*RobotsTxt) ProcessRequest

func (m *RobotsTxt) ProcessRequest(r *client.Request)