Documentation
¶
Index ¶
- Constants
- func StartRPCServer(spider *Spider, listen string) error
- type Data
- func (this Data) GetBytes(key string) ([]byte, bool)
- func (this Data) GetInt(key string) (int, bool)
- func (this Data) GetString(key string) (string, bool)
- func (this Data) Has(key string) bool
- func (this Data) MustGet(key string) interface{}
- func (this Data) MustGetBytes(key string) []byte
- func (this Data) MustGetInt(key string) int
- func (this Data) MustGetString(key string) string
- type Listener
- type Pipe
- type RPC
- func (rpc *RPC) Add(uriList []string, ack *bool) error
- func (rpc *RPC) Info(skip bool, info *SpiderInfo) error
- func (rpc *RPC) Pause(skip bool, ack *bool) error
- func (rpc *RPC) Pong(skip bool, message *string) error
- func (rpc *RPC) Resume(skip bool, ack *bool) error
- func (rpc *RPC) Stop(skip bool, ack *bool) error
- type RPCClient
- type Spider
- func (this *Spider) AddTask(task *Task) *Spider
- func (this *Spider) AddUri(uris ...string) *Spider
- func (this *Spider) DoneTask(task *Task)
- func (this *Spider) FailTask(task *Task, reason interface{})
- func (this *Spider) IgnoreTask(task *Task, reason interface{})
- func (this *Spider) IsFinished() bool
- func (this *Spider) On(e int, f Listener) *Spider
- func (this *Spider) Pause()
- func (this *Spider) Pipe(pipe Pipe) *Spider
- func (this *Spider) Resume()
- func (this *Spider) Run()
- func (this *Spider) RunAndServe(listen string) error
- func (this *Spider) RunForever(quit chan bool)
- func (this *Spider) StartTask(task *Task)
- func (this *Spider) Stop()
- func (this *Spider) Trigger(e int, t *Task)
- type SpiderInfo
- type Status
- type Task
Constants ¶
View Source
const ( ON_START = iota ON_STOP = iota )
Variables ¶
This section is empty.
Functions ¶
func StartRPCServer ¶
Types ¶
type Data ¶
type Data map[string]interface{}
func (Data) MustGetBytes ¶
func (Data) MustGetInt ¶
func (Data) MustGetString ¶
type RPCClient ¶
type RPCClient struct {
// contains filtered or unexported fields
}
func (*RPCClient) Info ¶
func (client *RPCClient) Info() (SpiderInfo, error)
type Spider ¶
type Spider struct { Concurrency int Stats map[Status]uint64 IsPaused bool IsStopped bool IsDebug bool // contains filtered or unexported fields }
func (*Spider) IgnoreTask ¶
Mark a task as ignored.
func (*Spider) IsFinished ¶
Check if all tasks have been processed.
func (*Spider) RunAndServe ¶
Run spider and start a RPC server
func (*Spider) RunForever ¶
Run spider forever, and accept a quit channel to close it.
Loop through the task list and run each of them with the help of a buffered channel.
type SpiderInfo ¶
Click to show internal directories.
Click to hide internal directories.