collector

Documentation

Overview

结果收集与输出

Index

• Variables
• type Collector
  • func NewCollector(sp *spider.Spider) *Collector
  • func (self *Collector) CollectData(dataCell data.DataCell) error
  • func (self *Collector) CollectFile(fileCell data.FileCell) error
  • func (self *Collector) Logger() logs.Logs
  • func (self *Collector) Report()
  • func (self *Collector) Run()
  • func (self *Collector) SetLogger(logger logs.Logs)
  • func (self *Collector) Start()
  • func (self *Collector) Stop()

Variables

var (
	// 全局支持的输出方式
	DataOutput = make(map[string]func(self *Collector) error)
	FileOutput = make(map[string]func(self *Collector, file data.FileCell) (string, int64, error))

	// 全局支持的文本数据输出方式名称列表
	DataOutputLib []string
)

Types

type Collector

type Collector struct {
	*spider.Spider                    //绑定的采集规则
	DataChan       chan data.DataCell //文本数据收集通道
	FileChan       chan data.FileCell //文件收集通道

	FileOutPath string //文件输出路径
	// contains filtered or unexported fields
}

结果收集与输出

func NewCollector

func NewCollector(sp *spider.Spider) *Collector

func (*Collector) CollectData

func (self *Collector) CollectData(dataCell data.DataCell) error

func (*Collector) CollectFile

func (self *Collector) CollectFile(fileCell data.FileCell) error

func (*Collector) Logger

func (self *Collector) Logger() logs.Logs

func (*Collector) Report

func (self *Collector) Report()

返回报告

func (*Collector) Run

func (self *Collector) Run()

启动数据收集/输出管道

func (*Collector) SetLogger

func (self *Collector) SetLogger(logger logs.Logs)

func (*Collector) Start

func (self *Collector) Start()

启动数据收集/输出管道

func (*Collector) Stop

func (self *Collector) Stop()

停止