requests

Documentation

Index

• type Request
• type RequestRegistry
  • func NewRegistry(done *syncx.WaitGroup) RequestRegistry

Types

type Request

type Request struct {
	Method         string
	Url            string
	RequestId      string
	NetworkId      string
	CrawlLog       *logV1.CrawlLog
	GotNew         bool
	GotComplete    bool
	Initiator      string
	ResourceType   string
	Referrer       string
	RedirectParent *Request
	FromCache      bool
	// contains filtered or unexported fields
}

type RequestRegistry

type RequestRegistry interface {
	NotifyLoadStart()
	NotifyLoadFinished()
	AddRequest(req *Request)
	GetByNetworkId(id string) *Request
	GetByRequestId(id string) *Request
	GetByUrl(url string, onlyNew bool) *Request
	MatchCrawlLogs() bool
	Walk(w func(*Request))
	InitialRequest() *Request
	RootRequest() *Request
	FinalizeResponses(requestedUrl *frontier.QueuedUri)
}

func NewRegistry

func NewRegistry(done *syncx.WaitGroup) RequestRegistry