crawler

Documentation

Index

• func PowerfulFind(s *goquery.Selection, q string) *goquery.Selection
• type Crawler
  • func New() *Crawler
  • func (s *Crawler) Init(rules ...Rule) error
  • func (s *Crawler) Run()
  • func (s *Crawler) Shutdown()
• type Fun
  • func ParseFun(sel *goquery.Selection, str string) *Fun
  • func (f *Fun) Append(s string) (*Fun, *Fun)
  • func (f *Fun) InitSelector() error
  • func (f *Fun) Invoke() (string, error)
• type Result
• type Rule
  • func (r Rule) Run() []map[string]string

Functions

func PowerfulFind

func PowerfulFind(s *goquery.Selection, q string) *goquery.Selection

Types

type Crawler

type Crawler struct {
	Send func(id, name string, out []map[string]string)
	// contains filtered or unexported fields
}

func New

func New() *Crawler

func (*Crawler) Init

func (s *Crawler) Init(rules ...Rule) error

func (*Crawler) Run

func (s *Crawler) Run()

func (*Crawler) Shutdown

func (s *Crawler) Shutdown()

type Fun

type Fun struct {
	Name   string
	Raw    string
	Params []string

	Document  *goquery.Document
	Selection *goquery.Selection
	Result    string

	PrevFun *Fun
	NextFun *Fun
}

func ParseFun

func ParseFun(sel *goquery.Selection, str string) *Fun

func (*Fun) Append

func (f *Fun) Append(s string) (*Fun, *Fun)

func (*Fun) InitSelector

func (f *Fun) InitSelector() error

func (*Fun) Invoke

func (f *Fun) Invoke() (string, error)

type Result

type Result struct {
	Name   string
	ID     string
	Mode   string
	Result []map[string]string
}

type Rule

type Rule struct {
	Name   string
	Enable bool
	Id     string
	When   string
	Mode   string
	Page   *struct {
		URL  string
		List string
		Item map[string]string
	} `json:"page,omitempty"`
	Json *struct {
		URL  string
		List string
		Item map[string]string
	} `json:"json,omitempty"`
	Feed *struct {
		URL  string
		Item map[string]string
	} `json:"feed,omitempty"`
}

func (Rule) Run

func (r Rule) Run() []map[string]string