controller

Documentation

Index

• Variables
• type Controller
  • func (self *Controller) CloseCrawler(name string) error
  • func (self *Controller) DelCrawler(name string) error
  • func (self *Controller) Finish()
  • func (self *Controller) Init(dir string, wc int) error
  • func (self *Controller) Run()
  • func (self *Controller) UpdateCrawler(item *types.CrawlerItem, isNew bool) error
• type CrawlerScheduler
  • func (self *CrawlerScheduler) Init()
  • func (self *CrawlerScheduler) Insert(item Item) error
  • func (self *CrawlerScheduler) Remove(name string) error
  • func (self *CrawlerScheduler) WeightedChoice() (string, error)
• type Item

Variables

var (
	ErrNotInited      = errors.New("controller/controller.go call Controller.Init first")
	ErrWorkerCount    = errors.New("controller/controller.go worker count must between 0 and 1000")
	ErrNilCrawlerItem = errors.New("controller/controller.go CrawlerItem is nil")
	ErrNamesNotSame   = errors.New("controller/controller.go CrawlerNames are not the same")
	ErrNameTaken      = errors.New("controller/controller.go CrawlerName already taken")
	ErrNoName         = errors.New("controller/controller.go no CrawlerName")
)

var (
	ErrNilList      = errors.New("SortedList is nil")
	ErrTotalWeight  = errors.New("SortedList totalWeight 0")
	ErrNeverHappen  = errors.New("weighted random selection error")
	ErrNameNotFound = errors.New("crawler name not found")
)

var StoreNames = []string{"crawler", "seed", "running", "crontab"}

Types

type Controller

type Controller struct {
	Crawlers    map[string]crawler.Crawler
	Schduler    CrawlerScheduler
	Stores      map[string]*store.LevelStore
	WorkerCount int
	// contains filtered or unexported fields
}

func (*Controller) CloseCrawler

func (self *Controller) CloseCrawler(name string) error

func (*Controller) DelCrawler

func (self *Controller) DelCrawler(name string) error

func (*Controller) Finish

func (self *Controller) Finish()

func (*Controller) Init

func (self *Controller) Init(dir string, wc int) error

func (*Controller) Run

func (self *Controller) Run()

func (*Controller) UpdateCrawler

func (self *Controller) UpdateCrawler(item *types.CrawlerItem, isNew bool) error

type CrawlerScheduler

type CrawlerScheduler struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*CrawlerScheduler) Init

func (self *CrawlerScheduler) Init()

func (*CrawlerScheduler) Insert

func (self *CrawlerScheduler) Insert(item Item) error

func (*CrawlerScheduler) Remove

func (self *CrawlerScheduler) Remove(name string) error

func (*CrawlerScheduler) WeightedChoice

func (self *CrawlerScheduler) WeightedChoice() (string, error)

type Item

type Item struct {
	CrawlerName string
	Weight      int
}