crawler

README

crawler

example

// 初始化爬虫
crawler = crawler.NewColly(
           task,
           url,
           // db pipeline回调函数(可选)
           crawler.WithPipeline(pipeline),
           // 爬虫回调函数(可选)
           crawler.WithCrawlCallback(callback),
           // ...
        )

// 执行爬虫
crawler.Run(
    // db pipeline回调函数(可选)
    crawler.WithPipeline(pipeline),
    // 爬虫回调函数(可选)
    crawler.WithCrawlCallback(callback),
)

Documentation

Index

• Constants
• type Callback
• type Colly
  • func NewColly(task, url string, options ...Option) *Colly
  • func (c *Colly) Run(options ...Option) error
• type Option
  • func WithCrawlCallback(cb Callback) Option
  • func WithPipeline(pipeline Pipeline) Option
  • func WithPostForm(form map[string]string) Option
  • func WithPostRaw(raw []byte) Option
  • func WithReqType(t ReqType) Option
• type Pipeline
• type ReqType

Constants

const (
	Get = iota
	Post
	PostRaw
)

const (
	Success = "success"
)

Types

type Callback

type Callback func()

type Colly

type Colly struct {
	Task     string
	Url      string
	Callback []Callback
	Pip      []Pipeline
	Counter  *prometheus.Counter

	Crawler *colly.Collector
	// contains filtered or unexported fields
}

func NewColly

func NewColly(task, url string, options ...Option) *Colly

func (*Colly) Run

func (c *Colly) Run(options ...Option) error

type Option

type Option func(c *Colly)

func WithCrawlCallback

func WithCrawlCallback(cb Callback) Option

func WithPipeline

func WithPipeline(pipeline Pipeline) Option

func WithPostForm

func WithPostForm(form map[string]string) Option

func WithPostRaw

func WithPostRaw(raw []byte) Option

func WithReqType

func WithReqType(t ReqType) Option

type Pipeline

type Pipeline func() error

type ReqType

type ReqType uint32