tegenaria

package module

Go to main page

Versions in this module

v0

v0.5.0

Mar 23, 2023

Changes in this version

+ var ErrNoMaterNodeLive error = errors.New("no any master node is active")

+ func AbsFilePathTest(t *testing.T, path string) string

+ func DefaultWatcher(ch chan EventType, hooker EventHooksInterface) error

+ func GetEngineID() string

+ func GetMachineIP() (string, error)

+ func GoRunner(wg *conc.WaitGroup, funcs ...GoFunc) <-chan error

+ func Interface2Uint(value interface{}) uint

+ func MD5(s string) string

+ func NewTestProxyServer() *httptest.Server

+ func NewTestServer() *httptest.Server

type CacheInterface

+ Close func() error

+ Dequeue func() (interface{}, error)

+ Enqueue func(ctx *Context) error

+ GetSize func() uint64

+ IsEmpty func() bool

+ SetCurrentSpider func(spider SpiderInterface)

+ type ComponentInterface interface

+ CheckWorkersStop func() bool

+ GetDupefilter func() RFPDupeFilterInterface

+ GetEventHooks func() EventHooksInterface

+ GetLimiter func() LimitInterface

+ GetQueue func() CacheInterface

+ GetStats func() StatisticInterface

+ SetCurrentSpider func(spider SpiderInterface)

+ SpiderBeforeStart func(engine *CrawlEngine, spider SpiderInterface) error

type Context

+ CtxID string

+ func NewTestRequest(spider SpiderInterface, opts ...RequestOption) *Context

+ func (c Context) GetCtxID() string

type ContextOption

+ func WithContextID(ctxID string) ContextOption

type CrawlEngine

+ func NewTestEngine(spiderName string, opts ...EngineOption) *CrawlEngine

+ func (e *CrawlEngine) GetComponents() ComponentInterface

+ func (e *CrawlEngine) GetCurrentSpider() SpiderInterface

+ func (e *CrawlEngine) GetRuntimeStatus() *RuntimeStatus

+ func (e *CrawlEngine) GetStatic() StatisticInterface

+ type DefaultComponents struct

+ func NewDefaultComponents(opts ...DefaultComponentsOption) *DefaultComponents

+ func (d *DefaultComponents) CheckWorkersStop() bool

+ func (d *DefaultComponents) GetDupefilter() RFPDupeFilterInterface

+ func (d *DefaultComponents) GetEventHooks() EventHooksInterface

+ func (d *DefaultComponents) GetLimiter() LimitInterface

+ func (d *DefaultComponents) GetQueue() CacheInterface

+ func (d *DefaultComponents) GetStats() StatisticInterface

+ func (d *DefaultComponents) SetCurrentSpider(spider SpiderInterface)

+ func (d *DefaultComponents) SpiderBeforeStart(engine *CrawlEngine, spider SpiderInterface) error

+ type DefaultComponentsOption func(d *DefaultComponents)

+ func DefaultComponentsWithDefaultHooks(events *DefaultHooks) DefaultComponentsOption

+ func DefaultComponentsWithDefaultLimiter(limiter *DefaultLimiter) DefaultComponentsOption

+ func DefaultComponentsWithDefaultQueue(queue *DefaultQueue) DefaultComponentsOption

+ func DefaultComponentsWithDefaultStatistic(statistic *DefaultStatistic) DefaultComponentsOption

+ func DefaultComponentsWithDupefilter(dupefilter *DefaultRFPDupeFilter) DefaultComponentsOption

type DefaultHooks

+ func (d *DefaultHooks) Pause(params ...interface{}) error

+ func (d *DefaultHooks) SetCurrentSpider(spider SpiderInterface)

+ type DefaultLimiter struct

+ func (d *DefaultLimiter) CheckAndWaitLimiterPass() error

+ func (d *DefaultLimiter) SetCurrentSpider(spider SpiderInterface)

+ type DefaultQueue struct

+ func NewDefaultQueue(size int) *DefaultQueue

+ func (c *DefaultQueue) Close() error

+ func (c *DefaultQueue) Dequeue() (interface{}, error)

+ func (c *DefaultQueue) Enqueue(ctx *Context) error

+ func (c *DefaultQueue) GetSize() uint64

+ func (c *DefaultQueue) IsEmpty() bool

+ func (c *DefaultQueue) SetCurrentSpider(spider SpiderInterface)

+ type DefaultRFPDupeFilter struct

+ func (f *DefaultRFPDupeFilter) DoDupeFilter(ctx *Context) (bool, error)

+ func (f *DefaultRFPDupeFilter) Fingerprint(ctx *Context) ([]byte, error)

+ func (f *DefaultRFPDupeFilter) SetCurrentSpider(spider SpiderInterface)

+ type DefaultStatistic struct

+ Metrics map[string]*uint64

+ func NewDefaultStatistic() *DefaultStatistic

+ func (s *DefaultStatistic) Get(metric string) uint64

+ func (s *DefaultStatistic) GetAllStats() map[string]uint64

+ func (s *DefaultStatistic) Incr(metrics string)

+ func (s *DefaultStatistic) SetCurrentSpider(spider SpiderInterface)

type DistributedWorkerInterface

+ GetWorkerID func() string

+ IsMaster func() bool

+ PauseNode func() error

+ SetCurrentSpider func(spider SpiderInterface)

type DownloaderOption

+ func DownloadWithTLSConfig(tls *tls.Config) DownloaderOption

type EngineOption

+ func EngineWithComponents(components ComponentInterface) EngineOption

+ func EngineWithReqChannelSize(size int) EngineOption

type EventHooksInterface

+ Pause func(params ...interface{}) error

+ SetCurrentSpider func(spider SpiderInterface)

type EventType

+ const PAUSE

type HandleError

+ CtxID string

type ItemMeta

+ CtxID string

type LimitInterface

+ CheckAndWaitLimiterPass func() error

+ SetCurrentSpider func(spider SpiderInterface)

type RFPDupeFilterInterface

+ SetCurrentSpider func(spider SpiderInterface)

type Request

+ Headers map[string]string

type RequestOption

+ func RequestWithBodyReader(body io.Reader) RequestOption

+ func RequestWithPostForm(payload url.Values) RequestOption

type Response

+ Body io.ReadCloser

+ Headers map[string][]string

+ func (r *Response) WriteTo(writer io.Writer) (int64, error)

+ type RuntimeStatus struct

+ Duration float64

+ RestartAt int64

+ StartAt int64

+ StatusOn StatusType

+ StopAt int64

+ func NewRuntimeStatus() *RuntimeStatus

+ func (r *RuntimeStatus) GetDuration() float64

+ func (r *RuntimeStatus) GetRestartAt() int64

+ func (r *RuntimeStatus) GetStartAt() int64

+ func (r *RuntimeStatus) GetStatusOn() StatusType

+ func (r *RuntimeStatus) GetStopAt() int64

+ func (r *RuntimeStatus) SetDuration(duration float64)

+ func (r *RuntimeStatus) SetRestartAt(startAt int64)

+ func (r *RuntimeStatus) SetStartAt(startAt int64)

+ func (r *RuntimeStatus) SetStatus(status StatusType)

+ func (r *RuntimeStatus) SetStopAt(stopAt int64)

type Spiders

+ func (s *Spiders) GetAllSpidersName() []string

type StatisticInterface

+ Get func(metric string) uint64

+ GetAllStats func() map[string]uint64

+ Incr func(metric string)

+ SetCurrentSpider func(spider SpiderInterface)

+ type StatusType uint

+ const ON_PAUSE

+ const ON_START

+ const ON_STOP

+ func (p StatusType) GetTypeName() string

+ type TestDownloadMiddler struct

+ Name string

+ Priority int

+ func (m TestDownloadMiddler) GetName() string

+ func (m TestDownloadMiddler) GetPriority() int

+ func (m TestDownloadMiddler) ProcessRequest(ctx *Context) error

+ func (m TestDownloadMiddler) ProcessResponse(ctx *Context, req chan<- *Context) error

+ type TestDownloadMiddler2 struct

+ Name string

+ Priority int

+ func (m TestDownloadMiddler2) GetName() string

+ func (m TestDownloadMiddler2) GetPriority() int

+ func (m TestDownloadMiddler2) ProcessRequest(ctx *Context) error

+ func (m TestDownloadMiddler2) ProcessResponse(ctx *Context, req chan<- *Context) error

+ type TestItemPipeline struct

+ Priority int

+ func (p *TestItemPipeline) GetPriority() int

+ func (p *TestItemPipeline) ProcessItem(spider SpiderInterface, item *ItemMeta) error

+ type TestItemPipeline2 struct

+ Priority int

+ func (p *TestItemPipeline2) GetPriority() int

+ func (p *TestItemPipeline2) ProcessItem(spider SpiderInterface, item *ItemMeta) error

+ type TestItemPipeline3 struct

+ Priority int

+ func (p *TestItemPipeline3) GetPriority() int

+ func (p *TestItemPipeline3) ProcessItem(spider SpiderInterface, item *ItemMeta) error

+ type TestItemPipeline4 struct

+ Priority int

+ func (p *TestItemPipeline4) GetPriority() int

+ func (p *TestItemPipeline4) ProcessItem(spider SpiderInterface, item *ItemMeta) error

+ type TestSpider struct

+ func (s *TestSpider) ErrorHandler(err *Context, req chan<- *Context)

+ func (s *TestSpider) GetFeedUrls() []string

+ func (s *TestSpider) GetName() string

+ func (s *TestSpider) Parser(resp *Context, req chan<- *Context) error

+ func (s *TestSpider) StartRequest(req chan<- *Context)

v0.4.6

Jan 27, 2023

Changes in this version

type EngineOption

+ func EngineWithInterval(interval time.Duration) EngineOption

type Request

+ DoNotFilter bool

type RequestOption

+ func RequestWithDoNotFilter(doNotFilter bool) RequestOption

v0.4.5

Jan 21, 2023

Changes in this version

+ type DefaultHooks struct

+ func NewDefaultHooks() *DefaultHooks

+ func (d *DefaultHooks) Error(params ...interface{}) error

+ func (d *DefaultHooks) EventsWatcher(ch chan EventType) error

+ func (d *DefaultHooks) Exit(params ...interface{}) error

+ func (d *DefaultHooks) Heartbeat(params ...interface{}) error

+ func (d *DefaultHooks) Start(params ...interface{}) error

+ func (d *DefaultHooks) Stop(params ...interface{}) error

v0.4.4

Jan 21, 2023

v0.4.3

Jan 21, 2023

v0.4.2

Jan 21, 2023

v0.4.1

Jan 21, 2023

Changes in this version

+ func AddGo(wg *sync.WaitGroup, funcs ...GoFunc) <-chan error

+ func DefaultventsWatcher(ch chan EventType, hooker EventHooksInterface) error

+ func ExecuteCmd(engine *CrawlEngine)

+ func GetAllParserMethod(spider SpiderInterface) map[string]Parser

+ func GetFunctionName(fn Parser) string

+ func GetMachineIp() string

+ func Map2String(m interface{}) string

+ func NewDefaultLimiter(limitRate int) *defaultLimiter

+ func NewLeakyBucketLimiterWithRdb(safetyLevel int, rdb redis.Cmdable, keyFunc GetRDBKey) *leakyBucketLimiterWithRdb

+ func NewRdbClient(config *DistributedWorkerConfig) *redis.Client

+ func NewRdbClusterCLient(config *WorkerConfigWithRdbCluster) *redis.ClusterClient

+ func NewRdbConfig(config *DistributedWorkerConfig) *redis.Options

+ func OptimalNumOfBits(n int64, p float64) int64

+ func OptimalNumOfHashFunctions(n int64, m int64) int64

+ type CheckMasterLive func() (bool, error)

type Configuration

+ func (c *Configuration) GetValue(key string) (interface{}, error)

type Context

+ Items chan *ItemMeta

+ Ref int64

+ Response *Response

+ Spider SpiderInterface

+ func (c *Context) Close()

+ type ContextInterface interface

+ IsDone func() bool

type ContextOption

+ func WithContextId(ctxId string) ContextOption

+ func WithItemChannelSize(size int) ContextOption

+ type CrawlEngine struct

+ func NewEngine(opts ...EngineOption) *CrawlEngine

+ func (e *CrawlEngine) Close()

+ func (e *CrawlEngine) EventsWatcherRunner() error

+ func (e *CrawlEngine) Execute()

+ func (e *CrawlEngine) GetSpiders() *Spiders

+ func (e *CrawlEngine) RegisterDownloadMiddlewares(middlewares MiddlewaresInterface)

+ func (e *CrawlEngine) RegisterPipelines(pipeline PipelinesInterface)

+ func (e *CrawlEngine) RegisterSpiders(spider SpiderInterface)

+ func (e *CrawlEngine) Scheduler() error

+ type DefualtHooks struct

+ func NewDefualtHooks() *DefualtHooks

+ func (d *DefualtHooks) Error(params ...interface{}) error

+ func (d *DefualtHooks) EventsWatcher(ch chan EventType) error

+ func (d *DefualtHooks) Exit(params ...interface{}) error

+ func (d *DefualtHooks) Heartbeat(params ...interface{}) error

+ func (d *DefualtHooks) Start(params ...interface{}) error

+ func (d *DefualtHooks) Stop(params ...interface{}) error

+ type DistributeOptions func(w *DistributedWorkerConfig)

+ func DistributedWithBloomN(bloomN uint) DistributeOptions

+ func DistributedWithBloomP(bloomP float64) DistributeOptions

+ func DistributedWithConnectionsSize(size int) DistributeOptions

+ func DistributedWithGetBFKey(keyFunc GetRDBKey) DistributeOptions

+ func DistributedWithGetLimitKey(keyFunc GetRDBKey) DistributeOptions

+ func DistributedWithGetqueueKey(keyFunc GetRDBKey) DistributeOptions

+ func DistributedWithLimiterRate(rate int) DistributeOptions

+ func DistributedWithRdbMaxRetry(retry int) DistributeOptions

+ func DistributedWithRdbTimeout(timeout time.Duration) DistributeOptions

+ type DistributeStatistic struct

+ func NewDistributeStatistic(statsPrefixKey string, rdb redis.Cmdable, wg *sync.WaitGroup, ...) *DistributeStatistic

+ func (s *DistributeStatistic) GetDownloadFail() uint64

+ func (s *DistributeStatistic) GetErrorCount() uint64

+ func (s *DistributeStatistic) GetItemScraped() uint64

+ func (s *DistributeStatistic) GetRequestSent() uint64

+ func (s *DistributeStatistic) GetStatsField(field StatsFieldType) uint64

+ func (s *DistributeStatistic) IncrDownloadFail()

+ func (s *DistributeStatistic) IncrErrorCount()

+ func (s *DistributeStatistic) IncrItemScraped()

+ func (s *DistributeStatistic) IncrRequestSent()

+ func (s *DistributeStatistic) IncrStats(field StatsFieldType)

+ func (s *DistributeStatistic) OutputStats() map[string]uint64

+ func (s *DistributeStatistic) Reset() error

+ type DistributeStatisticOption func(d *DistributeStatistic)

+ func DistributeStatisticAfterResetTTL(ttl time.Duration) DistributeStatisticOption

+ type DistributedHooks struct

+ func NewDistributedHooks(worker DistributedWorkerInterface) *DistributedHooks

+ func (d *DistributedHooks) Error(params ...interface{}) error

+ func (d *DistributedHooks) EventsWatcher(ch chan EventType) error

+ func (d *DistributedHooks) Exit(params ...interface{}) error

+ func (d *DistributedHooks) Heartbeat(params ...interface{}) error

+ func (d *DistributedHooks) Start(params ...interface{}) error

+ func (d *DistributedHooks) Stop(params ...interface{}) error

+ type DistributedWorker struct

+ func NewDistributedWorker(addr string, config *DistributedWorkerConfig) *DistributedWorker

+ func NewWorkerWithRdbCluster(config *WorkerConfigWithRdbCluster) *DistributedWorker

+ func (w *DistributedWorker) Add(fingerprint []byte) error

+ func (w *DistributedWorker) AddNode() error

+ func (w *DistributedWorker) CheckAllNodesStop() (bool, error)

+ func (w *DistributedWorker) CheckMasterLive() (bool, error)

+ func (w *DistributedWorker) DelNode() error

+ func (w *DistributedWorker) DoDupeFilter(ctx *Context) (bool, error)

+ func (w *DistributedWorker) Fingerprint(ctx *Context) ([]byte, error)

+ func (w *DistributedWorker) GetLimter() LimitInterface

+ func (w *DistributedWorker) Heartbeat() error

+ func (w *DistributedWorker) SetMaster(flag bool)

+ func (w *DistributedWorker) SetSpiders(spiders *Spiders)

+ func (w *DistributedWorker) StopNode() error

+ func (w *DistributedWorker) TestOrAdd(fingerprint []byte) (bool, error)

+ type DistributedWorkerConfig struct

+ BloomN uint

+ BloomP float64

+ GetBFKey GetRDBKey

+ GetqueueKey GetRDBKey

+ LimiterRate int

+ RdbConnectionsSize uint64

+ RdbMaxRetry int

+ RdbTimeout time.Duration

+ RedisAddr string

+ RedisDB uint32

+ RedisPasswd string

+ RedisUsername string

+ func NewDistributedWorkerConfig(username string, passwd string, db uint32, opts ...DistributeOptions) *DistributedWorkerConfig

+ type DistributedWorkerInterface interface

+ AddNode func() error

+ CheckAllNodesStop func() (bool, error)

+ CheckMasterLive func() (bool, error)

+ DelNode func() error

+ GetLimter func() LimitInterface

+ Heartbeat func() error

+ SetMaster func(flag bool)

+ SetSpiders func(spiders *Spiders)

+ StopNode func() error

type DownloaderOption

+ func DownloadWithH2(h2 bool) DownloaderOption

type EngineOption

+ func EngineWithCache(cache CacheInterface) EngineOption

+ func EngineWithDistributedWorker(woker DistributedWorkerInterface) EngineOption

+ func EngineWithFilter(filter RFPDupeFilterInterface) EngineOption

+ func EngineWithLimiter(limiter LimitInterface) EngineOption

type ErrorOption

+ func ErrorWithExtras(extras map[string]interface{}) ErrorOption

+ type EventHooksInterface interface

+ Error func(params ...interface{}) error

+ EventsWatcher func(ch chan EventType) error

+ Exit func(params ...interface{}) error

+ Heartbeat func(params ...interface{}) error

+ Start func(params ...interface{}) error

+ Stop func(params ...interface{}) error

+ type EventType int

+ const ERROR

+ const EXIT

+ const HEARTBEAT

+ const START

+ const STOP

+ type EventsWatcher func(ch chan EventType) error

+ type GetRDBKey func(params ...interface{}) (string, time.Duration)

+ type GoFunc func() error

type HandleError

+ Extras map[string]interface{}

+ type Hook func(params ...interface{}) error

+ type LimitInterface interface

type Parser

+ func GetParserByName(spider SpiderInterface, name string) Parser

+ type RdbNodes []string

type Request

+ AllowStatusCode []uint64

+ MaxConnsPerHost int

+ Parser Parser

+ Timeout time.Duration

+ func RequestFromMap(src map[string]interface{}, opts ...RequestOption) *Request

+ func (r *Request) ToMap() (map[string]interface{}, error)

+ type RequestMethod string

+ type RequestOption func(r *Request)

+ func RequestWithAllowedStatusCode(allowStatusCode []uint64) RequestOption

+ func RequestWithParser(parser Parser) RequestOption

+ func RequestWithRequestBytesBody(body []byte) RequestOption

+ func RequestWithTimeout(timeout time.Duration) RequestOption

type SpiderInterface

+ GetFeedUrls func() []string

type Spiders

+ Parsers map[string]Parser

+ type Statistic struct

+ DownloadFail uint64

+ ErrorCount uint64

+ ItemScraped uint64

+ RequestSent uint64

+ func NewStatistic() *Statistic

+ func (s *Statistic) GetDownloadFail() uint64

+ func (s *Statistic) GetErrorCount() uint64

+ func (s *Statistic) GetItemScraped() uint64

+ func (s *Statistic) GetRequestSent() uint64

+ func (s *Statistic) IncrDownloadFail()

+ func (s *Statistic) IncrErrorCount()

+ func (s *Statistic) IncrItemScraped()

+ func (s *Statistic) IncrRequestSent()

+ func (s *Statistic) OutputStats() map[string]uint64

+ func (s *Statistic) Reset() error

+ type StatisticInterface interface

+ GetDownloadFail func() uint64

+ GetErrorCount func() uint64

+ GetItemScraped func() uint64

+ GetRequestSent func() uint64

+ IncrDownloadFail func()

+ IncrErrorCount func()

+ IncrItemScraped func()

+ IncrRequestSent func()

+ OutputStats func() map[string]uint64

+ Reset func() error

+ type StatsFieldType string

+ const DownloadFailStats

+ const ErrorStats

+ const ItemsStats

+ const RequestStats

+ type WorkerConfigWithRdbCluster struct

+ func NewWorkerConfigWithRdbCluster(config *DistributedWorkerConfig, nodes RdbNodes) *WorkerConfigWithRdbCluster

v0.4.0

Jan 20, 2023

v0.3.0

Feb 15, 2022

v0.2.6

Jan 29, 2022

retracted

v0.2.5

Jan 29, 2022

retracted

v0.2.4

Jan 28, 2022

retracted

Changes in this version

type Context

+ Ctx context.Context

+ Item ItemInterface

type ContextOption

+ func ContextWithItem(item ItemInterface) ContextOption

type Logger

+ func (l *Logger) GetValue(key string) (string, error)

v0.2.3

Jan 28, 2022

retracted

v0.2.2

Jan 27, 2022

retracted

v0.2.1

Jan 26, 2022

retracted

v0.2.0

Jan 24, 2022

retracted

v0.1.4

Feb 15, 2022

Changes in this version

type Context

+ Cancel context.CancelFunc

v0.1.3

Jan 23, 2022

retracted

v0.1.2-fix

Feb 5, 2022

v0.1.0

Jan 29, 2022

Changes in this version

+ const DELETE

+ const GET

+ const HEAD

+ const OPTIONS

+ const POST

+ const PUT

+ var ErrDuplicateSpiderName error = errors.New("register a duplicate spider name error")

+ var ErrEmptySpiderName error = errors.New("register a empty spider name error")

+ var ErrGetCacheItem error = errors.New("getting item from cache error")

+ var ErrGetHttpProxy error = errors.New("getting http proxy ")

+ var ErrGetHttpsProxy error = errors.New("getting https proxy ")

+ var ErrNotAllowStatusCode error = errors.New("not allow handle status code")

+ var ErrParseSocksProxy error = errors.New("parse socks proxy ")

+ var ErrResponseParse error = errors.New("parse response error")

+ var ErrResponseRead error = errors.New("read response to buffer error")

+ var ErrSpiderCrawls error = errors.New("handle spider crawl error")

+ var ErrSpiderMiddleware error = errors.New("handle spider middleware error")

+ var ErrSpiderNotExist error = errors.New("not found spider")

+ var ProcessId string = uuid.New().String()

+ func DefaultErrorHandler(spider SpiderInterface, err *HandleError)

+ func GetLogger(Name string) *logrus.Entry

+ func GetUUID() string

+ func NewRequestCache() *requestCache

+ type BaseSpider struct

+ FeedUrls []string

+ Name string

+ func NewBaseSpider(name string, feedUrls []string) *BaseSpider

+ func (s *BaseSpider) ErrorHandler(err *HandleError)

+ func (s *BaseSpider) Parser(resp *Context, item chan<- *ItemMeta, req chan<- *Context) error

+ func (s *BaseSpider) StartRequest(req chan<- *Context)

+ type CacheInterface interface

+ type Configuration struct

+ Log *Logger

+ var Config *Configuration = &Configuration{ ... }

+ type Context struct

+ CtxId string

+ DownloadResult *RequestResult

+ Error error

+ Request *Request

+ func NewContext(request *Request, opts ...ContextOption) *Context

+ func (c *Context) Deadline() (deadline time.Time, ok bool)

+ func (c *Context) Done() <-chan struct{}

+ func (c *Context) Err() error

+ func (c *Context) Value(key interface{}) interface{}

+ func (c Context) GetCtxId() string

+ type ContextOption func(c *Context)

+ func WithContext(ctx context.Context) ContextOption

+ type DefaultFieldHook struct

+ func (hook *DefaultFieldHook) Fire(entry *logrus.Entry) error

+ func (hook *DefaultFieldHook) Levels() []logrus.Level

+ type Downloader interface

+ CheckStatus func(statusCode uint64, allowStatus []uint64) bool

+ Download func(ctx *Context, result chan<- *Context)

+ func NewDownloader(opts ...DownloaderOption) Downloader

+ type DownloaderOption func(d *SpiderDownloader)

+ func DownloadWithClient(client http.Client) DownloaderOption

+ func DownloadWithTimeout(timeout time.Duration) DownloaderOption

+ func DownloadWithTlsConfig(tls *tls.Config) DownloaderOption

+ func DownloaderWithStreamThreshold(streamThreshold uint64) DownloaderOption

+ func DownloaderWithtransport(transport *http.Transport) DownloaderOption

+ type EngineOption func(r *SpiderEngine)

+ func EngineWithAllowStatusCode(allowStatusCode []uint64) EngineOption

+ func EngineWithContext(ctx context.Context) EngineOption

+ func EngineWithDownloader(downloader Downloader) EngineOption

+ func EngineWithReadCacheNum(cacheReadNum uint) EngineOption

+ func EngineWithRequestNum(requestNum uint) EngineOption

+ func EngineWithSchedulerNum(schedulerNum uint) EngineOption

+ func EngineWithTimeout(timeout time.Duration) EngineOption

+ func EngineWithUniqueReq(uniqueReq bool) EngineOption

+ type ErrorHandler func(spider SpiderInterface, err *HandleError)

+ type ErrorOption func(e *HandleError)

+ func ErrorWithItem(item *ItemMeta) ErrorOption

+ func ErrorWithRequest(request *Request) ErrorOption

+ func ErrorWithResponse(response *Response) ErrorOption

+ type HandleError struct

+ CtxId string

+ Err error

+ Item *ItemMeta

+ Request *Request

+ Response *Response

+ func NewError(ctxId string, err error, opts ...ErrorOption) *HandleError

+ func (e *HandleError) Error() string

+ type ItemInterface interface

+ type ItemMeta struct

+ Ctx context.Context

+ CtxId string

+ Item ItemInterface

+ func NewItem(ctx *Context, item ItemInterface) *ItemMeta

+ type ItemPipelines []PipelinesInterface

+ func (p ItemPipelines) Len() int

+ func (p ItemPipelines) Less(i, j int) bool

+ func (p ItemPipelines) Swap(i, j int)

+ type Logger struct

+ Level string

+ Path string

+ type Middlewares []MiddlewaresInterface

+ func (p Middlewares) Len() int

+ func (p Middlewares) Less(i, j int) bool

+ func (p Middlewares) Swap(i, j int)

+ type MiddlewaresBase struct

+ Priority int

+ type MiddlewaresInterface interface

+ GetName func() string

+ GetPriority func() int

+ ProcessRequest func(ctx *Context) error

+ ProcessResponse func(ctx *Context, req chan<- *Context) error

+ type Option func(r *Request)

+ func RequestWithAllowRedirects(allowRedirects bool) Option

+ func RequestWithMaxConnsPerHost(maxConnsPerHost int) Option

+ func RequestWithMaxRedirects(maxRedirects int) Option

+ func RequestWithRequestBody(body map[string]interface{}) Option

+ func RequestWithRequestCookies(cookies map[string]string) Option

+ func RequestWithRequestHeader(header map[string]string) Option

+ func RequestWithRequestMeta(meta map[string]interface{}) Option

+ func RequestWithRequestParams(params map[string]string) Option

+ func RequestWithRequestProxy(proxy Proxy) Option

+ func RequestWithResponseWriter(write io.Writer) Option

+ type Parser func(resp *Context, item chan<- *ItemMeta, req chan<- *Context) error

+ type PipelinesBase struct

+ Priority int

+ type PipelinesInterface interface

+ GetPriority func() int

+ ProcessItem func(spider SpiderInterface, item *ItemMeta) error

+ type ProcessResponse func(ctx *Context) error

+ type Proxy struct

+ ProxyUrl string

+ type RFPDupeFilter struct

+ func NewRFPDupeFilter(bloomM uint, bloomK uint) *RFPDupeFilter

+ func (f *RFPDupeFilter) DoDupeFilter(request *Request) (bool, error)

+ func (f *RFPDupeFilter) Fingerprint(request *Request) ([]byte, error)

+ type RFPDupeFilterInterface interface

+ DoDupeFilter func(request *Request) (bool, error)

+ Fingerprint func(request *Request) ([]byte, error)

+ type RedirectError struct

+ RedirectNum int

+ func (e *RedirectError) Error() string

+ type Request struct

+ AllowRedirects bool

+ Body []byte

+ BodyReader io.Reader

+ Cookies map[string]string

+ Header map[string]string

+ MaxRedirects int

+ Meta map[string]interface{}

+ Method string

+ Params map[string]string

+ Proxy *Proxy

+ ResponseWriter io.Writer

+ Url string

+ func NewRequest(url string, method string, parser Parser, opts ...Option) *Request

+ type RequestResult struct

+ Error *HandleError

+ Response *Response

+ func NewDownloadResult() *RequestResult

+ type Response struct

+ Buffer *bytes.Buffer

+ ContentLength int

+ Delay float64

+ Header map[string][]string

+ Status int

+ URL string

+ func NewResponse() *Response

+ func (r *Response) Json() (map[string]interface{}, error)

+ func (r *Response) String() string

+ type Settings interface

+ GetValue func(key string) (error, string)

+ type SpiderDownloader struct

+ ProxyFunc func(req *http.Request) (*url.URL, error)

+ StreamThreshold uint64

+ func (d *SpiderDownloader) CheckStatus(statusCode uint64, allowStatus []uint64) bool

+ func (d *SpiderDownloader) Download(ctx *Context, result chan<- *Context)

+ type SpiderEngine struct

+ Ctx context.Context

+ DownloadTimeout time.Duration

+ ErrorHandler ErrorHandler

+ RFPDupeFilter RFPDupeFilterInterface

+ Stats *SpiderStats

+ var Engine *SpiderEngine

+ func NewSpiderEngine(opts ...EngineOption) *SpiderEngine

+ func (e *SpiderEngine) Close()

+ func (e *SpiderEngine) RegisterDownloadMiddlewares(middlewares MiddlewaresInterface)

+ func (e *SpiderEngine) RegisterPipelines(pipeline PipelinesInterface)

+ func (e *SpiderEngine) RegisterSpider(spider SpiderInterface)

+ func (e *SpiderEngine) SetAllowedStatus(allowedStatusCode []uint64)

+ func (e *SpiderEngine) SetDownloadTimeout(timeout time.Duration)

+ func (e *SpiderEngine) Start(spiderName string)

+ type SpiderInterface interface

+ ErrorHandler func(err *HandleError, req chan<- *Context)

+ GetName func() string

+ Parser func(resp *Context, item chan<- *ItemMeta, req chan<- *Context) error

+ StartRequest func(req chan<- *Context)

+ type SpiderStats struct

+ ErrorCount uint64

+ ItemScraped uint64

+ NetworkTraffic int64

+ RequestDownloaded uint64

+ type Spiders struct

+ SpidersModules map[string]SpiderInterface

+ var SpidersList *Spiders

+ func NewSpiders() *Spiders

+ func (s *Spiders) GetSpider(name string) (SpiderInterface, error)

+ func (s *Spiders) Register(spider SpiderInterface) error

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL