exec

Documentation

Index

• Variables
• func UniqueID() (string, error)
• type Fether
• type Recv
  • func (c *Recv) Index() int
  • func (c *Recv) IsEmpty() bool
  • func (c *Recv) Parent() *Recv
  • func (c *Recv) ParentResult() interface{}
  • func (c *Recv) Parents(lasts ...int) *Recv
  • func (c *Recv) ParentsResult(lasts ...int) interface{}
  • func (c *Recv) Result(args ...int) interface{}
  • func (c *Recv) Rule() *Rule
  • func (c *Recv) Title() string
  • func (c *Recv) URL() string
  • func (_ *Recv) UniqueID() (string, error)
• type Result
  • func (c *Result) ElapsedString(lang string) string
  • func (c *Result) IsMap() bool
  • func (c *Result) IsSlice() bool
• type Rule
  • func (c *Rule) Collect(parentID uint64, fetch Fether, index int, extra []*Rule, ...) ([]Result, error)
  • func (c *Rule) IsExited() bool
  • func (c *Rule) ParseTmplContent(tmplContent string) (string, error)
  • func (c *Rule) Result() *Recv
• type Rules
  • func NewRules() *Rules
  • func (c *Rules) Collect(debug bool, noticeSender sender.Notice, progress *notice.Progress) ([]Result, error)
  • func (c *Rules) SetExitedFn(exitedFn func() bool) *Rules
  • func (c *Rules) SetExportFn(exportFn func(pageID uint, lastResult *Recv, collected echo.Store, ...) error) *Rules

Variables

var ErrForcedExit = errors.New(`Forced exit`)

var RegexpTitle = regexp.MustCompile(`(?i)<title[\s]*>([^<]+)</title[\s]*>`)

var SonyFlake *sonyflake.Sonyflake

Functions

func UniqueID

func UniqueID() (string, error)

Types

type Fether

type Fether func(pageURL string, charset string) (body []byte, transcoded bool, err error)

type Recv

type Recv struct {
	// contains filtered or unexported fields
}

func (*Recv) Index

func (c *Recv) Index() int

func (*Recv) IsEmpty

func (c *Recv) IsEmpty() bool

func (*Recv) Parent

func (c *Recv) Parent() *Recv

func (*Recv) ParentResult

func (c *Recv) ParentResult() interface{}

func (*Recv) Parents

func (c *Recv) Parents(lasts ...int) *Recv

func (*Recv) ParentsResult

func (c *Recv) ParentsResult(lasts ...int) interface{}

func (*Recv) Result

func (c *Recv) Result(args ...int) interface{}

func (*Recv) Rule

func (c *Recv) Rule() *Rule

func (*Recv) Title

func (c *Recv) Title() string

func (*Recv) URL

func (c *Recv) URL() string

func (*Recv) UniqueID

func (_ *Recv) UniqueID() (string, error)

type Result

type Result struct {
	Title     string
	URL       string
	Result    interface{}
	Type      string //map/slice
	StartTime time.Time
	EndTime   time.Time
	Elapsed   time.Duration
}

func (*Result) ElapsedString

func (c *Result) ElapsedString(lang string) string

func (*Result) IsMap

func (c *Result) IsMap() bool

func (*Result) IsSlice

func (c *Result) IsSlice() bool

type Rule

type Rule struct {
	*dbschema.CollectorPage
	RuleList []*dbschema.CollectorRule
	// contains filtered or unexported fields
}

Rule 页面规则

func (*Rule) Collect

func (c *Rule) Collect(parentID uint64,
	fetch Fether, index int, extra []*Rule,
	noticeSender sender.Notice,
	progress *notice.Progress) ([]Result, error)

func (*Rule) IsExited

func (c *Rule) IsExited() bool

func (*Rule) ParseTmplContent

func (c *Rule) ParseTmplContent(tmplContent string) (string, error)

func (*Rule) Result

func (c *Rule) Result() *Recv

type Rules

type Rules struct {
	*Rule         //主页面规则
	Extra []*Rule //扩展页面规则
	// contains filtered or unexported fields
}

Rules 完整规则

func NewRules

func NewRules() *Rules

func (*Rules) Collect

func (c *Rules) Collect(debug bool, noticeSender sender.Notice, progress *notice.Progress) ([]Result, error)

func (*Rules) SetExitedFn

func (c *Rules) SetExitedFn(exitedFn func() bool) *Rules

func (*Rules) SetExportFn

func (c *Rules) SetExportFn(exportFn func(pageID uint, lastResult *Recv, collected echo.Store, noticeSender sender.Notice) error) *Rules