cachenet

README

cachenet

From github.com/godcong/excavator/net

Documentation

Index

• func CacheDataQuery(base_url string, url_data string) (*html.Node, error)
• func CacheQuery(req_url string) (*html.Node, error)
• func CacheQueryOneXpath(req_url string, xpath_str string) *html.Node
• func CacheQueryString(req_url string) (string, error)
• func CacheQueryXml(req_url string) (*xmlquery.Node, error)
• func CacheQueryXpath(req_url string, xpath_str string) []*html.Node
• func GetRequest(req_url string) (io.ReadCloser, error)
• func Hash(url string) string
• func PostRequest(req_url string, url_data string) (io.ReadCloser, error)
• func PreparePath(path string) (string, string)
• func RegisterProxy(addr string, args ...ProxyArgs) (e error)
• func Request(r *http.Request) (*http.Response, error)
• func UrlMerge(base_url string, wd_url string) string
• type Cache
  • func NewCache(tmp string) *Cache
  • func (c *Cache) Cache(closer io.ReadCloser, req_url string) (io.ReadCloser, error)
  • func (c *Cache) CacheData(closer io.ReadCloser, req_url string, url_data string) (io.ReadCloser, error)
  • func (c *Cache) Get(req_url string) (e error)
  • func (c *Cache) PrepareDataPath(req_url string, url_data string) string
  • func (c *Cache) PrepareGetPath(req_url string) string
  • func (c *Cache) Reader(req_url string) (reader io.ReadCloser, e error)
  • func (c *Cache) ReaderData(req_url string, url_data string) (reader io.ReadCloser, e error)
  • func (c *Cache) Save(req_url string, to string) (written int64, e error)
• type ProxyArgs
  • func TimeOut(sec int) ProxyArgs

Functions

func CacheDataQuery

func CacheDataQuery(base_url string, url_data string) (*html.Node, error)

由编码后的url.Values获取cache

data := url.Values{
	"Tid": {"10"},
	"wd":  {wd},
}

dataStr := data.Encode()

func CacheQuery

func CacheQuery(req_url string) (*html.Node, error)

由URL获取cache

func CacheQueryOneXpath

func CacheQueryOneXpath(req_url string, xpath_str string) *html.Node

func CacheQueryString

func CacheQueryString(req_url string) (string, error)

由URL获取cache

func CacheQueryXml

func CacheQueryXml(req_url string) (*xmlquery.Node, error)

由URL获取cache

func CacheQueryXpath

func CacheQueryXpath(req_url string, xpath_str string) []*html.Node

func GetRequest

func GetRequest(req_url string) (io.ReadCloser, error)

由字的url获取页面

func Hash

func Hash(url string) string

func PostRequest

func PostRequest(req_url string, url_data string) (io.ReadCloser, error)

Generated by curl-to-Go: https://mholt.github.io/curl-to-go

data := url.Values{
	"Tid": {"10"},
	"wd":  {wd},
}

dataStr := data.Encode()

func PreparePath

func PreparePath(path string) (string, string)

为目标路径创建文件夹，返回目标路径的绝对路径和文件名

func RegisterProxy

func RegisterProxy(addr string, args ...ProxyArgs) (e error)

RegisterProxy ...

func Request

func Request(r *http.Request) (*http.Response, error)

func UrlMerge

func UrlMerge(base_url string, wd_url string) string

URL拼接

Types

type Cache

type Cache struct {
	Tmp string
}

Cache ...

func NewCache

func NewCache(tmp string) *Cache

NewCache ...

func (*Cache) Cache

func (c *Cache) Cache(closer io.ReadCloser, req_url string) (io.ReadCloser, error)

从URL和http响应内容句柄获得cache文件句柄

func (*Cache) CacheData

func (c *Cache) CacheData(closer io.ReadCloser, req_url string, url_data string) (io.ReadCloser, error)

从Unicode和http响应内容句柄获得cache文件句柄

func (*Cache) Get

func (c *Cache) Get(req_url string) (e error)

URL存为cache文件

func (*Cache) PrepareDataPath

func (c *Cache) PrepareDataPath(req_url string, url_data string) string

为目标路径创建文件夹，返回目标路径的绝对路径，url_data为url.Values的encode()过的形态

func (*Cache) PrepareGetPath

func (c *Cache) PrepareGetPath(req_url string) string

为目标路径创建文件夹，返回目标路径的绝对路径

func (*Cache) Reader

func (c *Cache) Reader(req_url string) (reader io.ReadCloser, e error)

从URL获得cache文件句柄

func (*Cache) ReaderData

func (c *Cache) ReaderData(req_url string, url_data string) (reader io.ReadCloser, e error)

从unicode获得cache文件句柄

func (*Cache) Save

func (c *Cache) Save(req_url string, to string) (written int64, e error)

URL对应的缓存文件转存成指定文件

type ProxyArgs

type ProxyArgs func(cli *http.Client)

func TimeOut

func TimeOut(sec int) ProxyArgs