spider

Documentation

Index

• func RegisterEngine(engine SearchEngine)
• type NewsItem
• type SearchEngine
• type Spider
  • func (s *Spider) FetchNews(engineName, keyword string) error
  • func (s *Spider) Work(job *SubscribeJob) error
• type SubscribeJob

Functions

func RegisterEngine

func RegisterEngine(engine SearchEngine)

注册spider

Types

type NewsItem

type NewsItem struct {
	Title       string    `json:title`
	Description string    `json:description`
	Source      string    `json:source`
	Url         string    `json:url`
	CreatedAt   time.Time `json:created_at`
}

type SearchEngine

type SearchEngine interface {
	EngineName() string
	UrlFormat() string
	Keyword() string
	ParsePage(document *goquery.Document) ([]NewsItem, error)
}

spider engine 需要实现是的方法

type Spider

type Spider struct {
	Engines map[string]SearchEngine
	// contains filtered or unexported fields
}

func (*Spider) FetchNews

func (s *Spider) FetchNews(engineName, keyword string) error

抓取新闻

func (*Spider) Work

func (s *Spider) Work(job *SubscribeJob) error

type SubscribeJob

type SubscribeJob struct {
	SpiderFrequency    time.Duration
	SubscribeFrequency time.Duration
	Keyword            string
	SelectedEngine     []string
}