README
¶
ziva
一个go实现的多任务、多线程爬虫库
一个demo
package main
import (
"fmt"
"github.com/zituocn/ziva"
"net/http"
)
func main() {
job := ziva.NewJob("article", ziva.Options{
CreateQueue: func() ziva.TodoQueue {
ids := []int{3263, 3262, 3261, 3260, 3259}
queue := ziva.NewMemQueue()
header := &http.Header{}
header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
for _, item := range ids {
queue.Add(&ziva.Task{
Url: fmt.Sprintf("%s%d", "https://22v.net/article/", item),
Method: "GET",
Header: header,
})
}
return queue
},
SucceedFunc: func(ctx *ziva.Context) {
fmt.Println("成功的回调")
fmt.Println("返回信息 :", ctx.Response.Status)
},
FailedFunc: func(ctx *ziva.Context) {
fmt.Println("失败的回调")
fmt.Println("返回状态 :", ctx.Response.StatusCode)
},
SheepTime: 3000,
Num: 1,
})
job.Do()
}
使用redis queue
package main
import (
"fmt"
"net/http"
"github.com/zituocn/ziva"
"github.com/zituocn/ziva/goredis"
)
func main() {
job := ziva.NewJob("article", ziva.Options{
CreateQueue: func() ziva.TodoQueue {
ids := []int{3263, 3262, 3261, 3260, 3259}
queue := ziva.NewRedisQueue("article", &goredis.RedisConfig{
Name: "127.0.0.1",
Host: "127.0.0.1",
Port: 6379,
Password: "123456",
DB: 1,
})
header := &http.Header{}
header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36")
for _, item := range ids {
queue.Add(&ziva.Task{
Url: fmt.Sprintf("%s%d", "https://22v.net/article/", item),
Method: "GET",
Header: header,
})
}
return queue
},
SucceedFunc: func(ctx *ziva.Context) {
fmt.Println("成功的回调")
fmt.Println("返回信息 :", ctx.Response.Status)
},
FailedFunc: func(ctx *ziva.Context) {
fmt.Println("失败的回调")
fmt.Println("返回状态 :", ctx.Response.StatusCode)
},
SheepTime: 3000,
IsDebug: true,
Num: 1,
})
job.Do()
}
Documentation
¶
Index ¶
- func BuilderHeader(s string) *http.Header
- func GetFuncName(i interface{}) string
- func GetRandSleepTime(min, max int) int
- func GetStatusByCode(code int) string
- func Str2Int64(s string) int64
- type CallbackFunc
- type Context
- func DoRequest(url, method string, vs ...interface{}) *Context
- func DoTask(task *Task) *Context
- func HttpGet(url string, vs ...interface{}) *Context
- func HttpPost(url string, vs ...interface{}) *Context
- func HttpPut(url string, vs ...interface{}) *Context
- func NewContext(req *http.Request, vs ...interface{}) *Context
- func NewRequest(urlStr, method string, vs ...interface{}) (*Context, error)
- func (c *Context) Do()
- func (c *Context) SetProxy(httpProxy string) *Context
- func (c *Context) SetProxyLib(lib *ProxyLib) *Context
- func (c *Context) SetTransport(f func() *http.Transport) *Context
- func (c *Context) ToByte() []byte
- func (c *Context) ToFile(writer io.Writer) error
- func (c *Context) ToHTML() string
- func (c *Context) ToJSON(v interface{}) error
- func (c *Context) ToSection(path string) string
- func (c *Context) ToString() string
- type Cookie
- type CreateQueue
- type FormData
- type Job
- type MemQueue
- type Options
- type ProxyIP
- type ProxyLib
- type RedisQueue
- type StatusCode
- type Task
- type TodoQueue
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func BuilderHeader ¶
BuilderHeader convert string headers to http.Header
func GetFuncName ¶
func GetFuncName(i interface{}) string
GetFuncName return the function name by reflection
func GetStatusByCode ¶
Types ¶
type CallbackFunc ¶
type CallbackFunc func(ctx *Context)
type Context ¶
type Context struct {
// Request *http.Request
Request *http.Request
// Response *http.Response
Response *http.Response
Err error
// RespBody http.Response.Body
RespBody []byte
Task *Task
Data map[string]interface{}
Options Options
// contains filtered or unexported fields
}
func NewContext ¶
func NewRequest ¶
func (*Context) SetProxyLib ¶ added in v0.0.3
type CreateQueue ¶
type CreateQueue func() TodoQueue
type FormData ¶
func BuilderFormData ¶
BuilderFormData convert string FormData to esme.FormData
page=1&limit=15&id=&nick_name=&mobile=&source_type=-100 to FormData
type Options ¶
type Options struct {
// goroutine num
Num int
// Queue all task
Queue TodoQueue
CreateQueue CreateQueue
StartFunc CallbackFunc
SucceedFunc CallbackFunc
RetryFunc CallbackFunc
FailedFunc CallbackFunc
CompleteFunc CallbackFunc
// ProxyIP http or https proxy ip
ProxyIP string
// ProxyLib proxy ips
ProxyLib *ProxyLib
SheepTime int
TimeOut int
IsDebug bool
}
type ProxyLib ¶ added in v0.0.3
type ProxyLib struct {
// contains filtered or unexported fields
}
type RedisQueue ¶
type RedisQueue struct {
// contains filtered or unexported fields
}
RedisQueue task queue in redis
func (*RedisQueue) AddTasks ¶
func (q *RedisQueue) AddTasks(list []*Task)
AddTasks add multiple tasks to the queue
func (*RedisQueue) IsEmpty ¶
func (q *RedisQueue) IsEmpty() bool
IsEmpty returns whether the queue is empty
func (*RedisQueue) Pop ¶
func (q *RedisQueue) Pop() *Task
Pop get a task while removing it from the queue
from left
func (*RedisQueue) Print ¶
func (q *RedisQueue) Print()
type StatusCode ¶
type TodoQueue ¶
type TodoQueue interface {
Add(tas *Task)
AddTasks(list []*Task)
Pop() *Task
Clear() bool
Size() int
IsEmpty() bool
Print()
}
func NewMemQueue ¶
func NewMemQueue() TodoQueue
func NewRedisQueue ¶
func NewRedisQueue(key string, rc *goredis.RedisConfig) TodoQueue
NewRedisQueue use redis configuration
Source Files
¶
Directories
Click to show internal directories.
Click to hide internal directories.