exec

Documentation

Index

• Variables
• type Fether
• type Recv
  • func (c *Recv) Parent() *Recv
  • func (c *Recv) ParentItem() interface{}
  • func (c *Recv) ParentResult() interface{}
  • func (c *Recv) Parents(lasts ...int) *Recv
  • func (c *Recv) ParentsItem(lasts ...int) interface{}
  • func (c *Recv) ParentsResult(lasts ...int) interface{}
  • func (c *Recv) ResultByIndex(index int) interface{}
  • func (c *Recv) String() string
  • func (_ *Recv) UniqueID() (string, error)
• type Result
  • func (c *Result) ElapsedString(lang string) string
  • func (c *Result) IsMap() bool
  • func (c *Result) IsSlice() bool
• type Rule
  • func (c *Rule) Collect(parentID uint64, parentURL string, recv *Recv, fetch Fether, extra []*Rule, ...) ([]Result, error)
  • func (c *Rule) CollectOne(recv *Recv, urlIndex int, parentID uint64, parentURL string, pageURL string, ...) (collection interface{}, result []Result, ignore bool, err error)
  • func (c *Rule) IsExited() bool
  • func (c *Rule) ParseTmplContent(tmplContent string, result *Recv) (string, error)
• type Rules
  • func NewRules() *Rules
  • func (c *Rules) Collect(debug bool, noticeSender sender.Notice, progress *notice.Progress) (rs []Result, err error)
  • func (c *Rules) SetExitedFn(exitedFn func() bool) *Rules
  • func (c *Rules) SetExportFn(exportFn func(pageID uint, lastResult *Recv, collected echo.Store, ...) error) *Rules

Variables

var ErrForcedExit = errors.New(`Forced exit`)

var RegexpTitle = regexp.MustCompile(`(?i)<title[\s]*>([^<]+)</title[\s]*>`)

Types

type Fether

type Fether func(pageURL string, charset string) (body []byte, transcoded bool, err error)

type Recv

type Recv struct {
	Index      int
	IsEmpty    bool        //是否为空结果
	LevelIndex int         //层级索引
	URLIndex   int         //网址列表索引
	Result     interface{} //采集结果数据
	//rule       *Rule       //页面规则
	Title string //页面标题
	URL   string //网址
	// contains filtered or unexported fields
}

func (*Recv) Parent

func (c *Recv) Parent() *Recv

func (*Recv) ParentItem

func (c *Recv) ParentItem() interface{}

func (*Recv) ParentResult

func (c *Recv) ParentResult() interface{}

func (*Recv) Parents

func (c *Recv) Parents(lasts ...int) *Recv

func (*Recv) ParentsItem

func (c *Recv) ParentsItem(lasts ...int) interface{}

func (*Recv) ParentsResult

func (c *Recv) ParentsResult(lasts ...int) interface{}

func (*Recv) ResultByIndex

func (c *Recv) ResultByIndex(index int) interface{}

func (*Recv) String

func (c *Recv) String() string

func (*Recv) UniqueID

func (_ *Recv) UniqueID() (string, error)

type Result

type Result struct {
	Title     string
	URL       string
	Result    interface{}
	Type      string //map/slice
	StartTime time.Time
	EndTime   time.Time
	Elapsed   time.Duration
}

func (*Result) ElapsedString

func (c *Result) ElapsedString(lang string) string

func (*Result) IsMap

func (c *Result) IsMap() bool

func (*Result) IsSlice

func (c *Result) IsSlice() bool

type Rule

type Rule struct {
	*dbschema.NgingCollectorPage                                // 页面配置
	RuleList                     []*dbschema.NgingCollectorRule // 采集规则列表
	// contains filtered or unexported fields
}

Rule 页面规则

func (*Rule) Collect

func (c *Rule) Collect(parentID uint64, parentURL string, recv *Recv,
	fetch Fether, extra []*Rule,
	noticeSender sender.Notice,
	progress *notice.Progress) ([]Result, error)

func (*Rule) CollectOne

func (c *Rule) CollectOne(recv *Recv, urlIndex int,
	parentID uint64, parentURL string, pageURL string,
	fetch Fether, extra []*Rule,
	noticeSender sender.Notice,
	progress *notice.Progress, historyMdl *dbschema.NgingCollectorHistory) (collection interface{}, result []Result, ignore bool, err error)

func (*Rule) IsExited

func (c *Rule) IsExited() bool

func (*Rule) ParseTmplContent

func (c *Rule) ParseTmplContent(tmplContent string, result *Recv) (string, error)

type Rules

type Rules struct {
	*Rule         //主页面配置以及采集规则列表
	Extra []*Rule //扩展页面配置以及采集规则列表
	// contains filtered or unexported fields
}

Rules 完整规则

func NewRules

func NewRules() *Rules

func (*Rules) Collect

func (c *Rules) Collect(debug bool, noticeSender sender.Notice, progress *notice.Progress) (rs []Result, err error)

func (*Rules) SetExitedFn

func (c *Rules) SetExitedFn(exitedFn func() bool) *Rules

func (*Rules) SetExportFn

func (c *Rules) SetExportFn(exportFn func(pageID uint, lastResult *Recv, collected echo.Store, noticeSender sender.Notice) error) *Rules