miner

package
v1.0.9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 30, 2021 License: Apache-2.0 Imports: 17 Imported by: 18

Documentation

Overview 

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

Package miner is the core of this project, use to request for http api.

Example: 

package main

import (
	"fmt"
	"github.com/hunterhug/marmot/miner"
)

func main() {
	miner.SetLogLevel(miner.DEBUG)

	// Use Default Worker, You can Also New One:
	//worker, _ := miner.New(nil)
	//worker = miner.NewWorkerWithNoProxy()
	//worker = miner.NewAPI()
	//worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080")
	worker := miner.Clone()
	_, err := worker.SetUrl("https://www.gov.cn").Go()
	if err != nil {
		fmt.Println(err.Error())
	} else {
		fmt.Println(worker.ToString())
	}
}

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

All right reserved https://github.com/hunterhug/marmot at 2016-2021
Attribution-NonCommercial-NoDerivatives 4.0 International
Notice: The following code's copyright by hunterhug, Please do not spread and modify.
You can use it for education only but can't make profits for any companies and individuals!

Index

Constants

View Source 
const (
	VERSION = "1.0.9"

	// GET HTTP method
	GET      = "GET"
	POST     = "POST"
	POSTJSON = "POSTJSON"
	POSTXML  = "POSTXML"
	POSTFILE = "POSTFILE"
	PUT      = "PUT"
	PUTJSON  = "PUTJSON"
	PUTXML   = "PUTXML"
	PUTFILE  = "PUTFILE"
	DELETE   = "DELETE"
	OTHER    = "OTHER" // this stand for you can use other method this lib not own.

	// HTTPFORMContentType HTTP content type
	HTTPFORMContentType = "application/x-www-form-urlencoded"
	HTTPJSONContentType = "application/json"
	HTTPXMLContentType  = "text/xml"
	HTTPFILEContentType = "multipart/form-data"

	// CRITICAL Log mark
	CRITICAL = "CRITICAL"
	ERROR    = "ERROR"
	WARNING  = "WARNING"
	NOTICE   = "NOTICE"
	INFO     = "INFO"
	DEBUG    = "DEBUG"
)

Variables

View Source 
var (
	// Client Save Cookie
	Client = &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			Logger.Debugf("[GoWorker] Redirect:%v", req.URL)
			return nil
		},
		Jar:     NewJar(),
		Timeout: util.Second(DefaultTimeOut),
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}

	// NoCookieClient Not Save Cookie
	NoCookieClient = &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			Logger.Debugf("[GoWorker] Redirect:%v", req.URL)
			return nil
		},
		Timeout: util.Second(DefaultTimeOut),
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}
)

Default Client

View Source 
var (
	DefaultHeader = map[string][]string{
		"User-Agent": {
			ourLoveUa,
		},
	}

	// DefaultTimeOut http get and post No timeout
	DefaultTimeOut = 0
)
View Source 
var (
	Logger = logging.MustGetLogger("Marmot")

	// LevelNames Level name you can refer
	LevelNames = []string{
		"CRITICAL",
		"ERROR",
		"WARNING",
		"NOTICE",
		"INFO",
		"DEBUG",
	}
)

Global logger config for debug

View Source 
var Pool = &_Workers{ws: make(map[string]*Worker)}

Pool for many Worker, every Worker can only serial execution

View Source 
var Ua = map[int]string{}

Ua Global User-Agent provide

Functions

func CloneHeader 

func CloneHeader(h map[string][]string) map[string][]string

CloneHeader Clone a header, If not exist Ua, Set our Ua!

func CopyM 

func CopyM(h http.Header) http.Header

CopyM Header map[string][]string ,can use to copy a http header, so that they are not effect each other

func Delete 

func Delete() (body []byte, e error)

func Get 

func Get() (body []byte, e error)

func GetCookies 

func GetCookies() []*http.Cookie

func GetResponseStatusCode 

func GetResponseStatusCode() int

func Go 

func Go() (body []byte, e error)

func GoByMethod 

func GoByMethod(method string) (body []byte, e error)

func JsonToString 

func JsonToString() (string, error)

JsonToString This make effect only your Worker exec serial! Attention! Change Your JSON like Raw data to string

func Log 

func Log() *logging.Logger

Log Return global log

func MergeCookie 

func MergeCookie(before []*http.Cookie, after []*http.Cookie) []*http.Cookie

MergeCookie Merge Cookie, not use

func NewClient 

func NewClient() *http.Client

NewClient New a client, diff from proxy client

func NewJar 

func NewJar() *cookiejar.Jar

NewJar Cookie record Jar

func NewProxyClient 

func NewProxyClient(proxyString string) (*http.Client, error)

NewProxyClient New a Proxy client, Default save cookie, Can timeout We should support some proxy way such as http(s) or socks

func OtherGo 

func OtherGo(method, contentType string) (body []byte, e error)

func OutputMaps 

func OutputMaps(info string, args map[string][]string)

OutputMaps Just debug a map

func Post 

func Post() (body []byte, e error)

func PostFILE 

func PostFILE() (body []byte, e error)

func PostJSON 

func PostJSON() (body []byte, e error)

func PostXML 

func PostXML() (body []byte, e error)

func Put 

func Put() (body []byte, e error)

func PutFILE 

func PutFILE() (body []byte, e error)

func PutJSON 

func PutJSON() (body []byte, e error)

func PutXML 

func PutXML() (body []byte, e error)

func RandomUa 

func RandomUa() string

RandomUa back random User-Agent

func SetGlobalTimeout 

func SetGlobalTimeout(num int)

SetGlobalTimeout Set global timeout, it can only by this way!

func SetLogLevel 

func SetLogLevel(level string)

SetLogLevel Set log level

func ToString 

func ToString() string

ToString This make effect only your Worker exec serial! Attention! Change Your Raw data To string

func TooSortSizes 

func TooSortSizes(data []byte, sizes float64) error

TooSortSizes if a file size small than sizes(KB) ,it will be throw a error

func UaInit 

func UaInit()

UaInit User-Agent init

func Wait 

func Wait(waitTime int)

Wait some second

Types

type Request 

type Request struct {
	Data         url.Values    // Sent by form data
	FileName     string        // FileName which sent to remote
	FileFormName string        // File Form Name which sent to remote
	BData        []byte        // Sent by binary data, can together with File
	Request      *http.Request // Debug
}

type Response 

type Response struct {
	Response           *http.Response // Debug
	Raw                []byte         // Raw data we get
	ResponseStatusCode int            // The last url response code, such as 404
}

type Worker 

type Worker struct {
	// In order fast chain func call I put the basic config below
	*Request
	*Response

	// Which url we want
	Url string

	// Get,Post method
	Method string

	// Our Client
	Client *http.Client

	// Wait Sleep Time
	Wait int

	// Worker proxy ip, just for user to record their proxy ip, default: localhost
	Ip string

	// AOP like Java
	Ctx          context.Context
	BeforeAction func(context.Context, *Worker)
	AfterAction  func(context.Context, *Worker)

	// Http header
	Header http.Header
	// contains filtered or unexported fields
}

Worker is the main object to sent http request and return result of response 

var DefaultWorker *Worker

DefaultWorker Global Worker

func Clear 

func Clear() *Worker

func ClearAll 

func ClearAll() *Worker

func ClearCookie 

func ClearCookie() *Worker

func Clone added in v1.0.7 

func Clone() *Worker

func New 

func New(ipString interface{}) (*Worker, error)

New Alias Name for NewWorker

func NewAPI 

func NewAPI() *Worker

NewAPI New API Worker, No Cookie Keep, share the same http client

func NewWorker 

func NewWorker(proxyIpString interface{}) (*Worker, error)

NewWorker New a worker, if ipString is a proxy address, New a proxy client. evey time gen a new http client! Proxy address such as: 

http://[user]:[password@]ip:port, [] stand it can choose or not. case: socks5://127.0.0.1:1080

func NewWorkerByClient 

func NewWorkerByClient(client *http.Client) *Worker

NewWorkerByClient New Worker by Your Client

func NewWorkerWithNoProxy added in v1.0.7 

func NewWorkerWithNoProxy() *Worker

NewWorkerWithNoProxy Alias func

func NewWorkerWithProxy added in v1.0.7 

func NewWorkerWithProxy(proxyIpString interface{}) (*Worker, error)

NewWorkerWithProxy Alias func

func SetAfterAction 

func SetAfterAction(fc func(context.Context, *Worker)) *Worker

func SetBData 

func SetBData(data []byte) *Worker

func SetBeforeAction 

func SetBeforeAction(fc func(context.Context, *Worker)) *Worker

func SetContext 

func SetContext(ctx context.Context) *Worker

func SetCookie 

func SetCookie(v string) *Worker

func SetCookieByFile 

func SetCookieByFile(file string) (*Worker, error)

func SetFileInfo 

func SetFileInfo(fileName, fileFormName string) *Worker

func SetForm 

func SetForm(form url.Values) *Worker

func SetFormParam added in v1.0.9 

func SetFormParam(k, v string) *Worker

func SetHeader 

func SetHeader(header http.Header) *Worker

SetHeader Default Worker SetHeader!

func SetHeaderParam added in v1.0.9 

func SetHeaderParam(k, v string) *Worker

func SetMethod 

func SetMethod(method string) *Worker

func SetRefer 

func SetRefer(refer string) *Worker

func SetUa 

func SetUa(ua string) *Worker

func SetUrl 

func SetUrl(url string) *Worker

func SetWaitTime 

func SetWaitTime(num int) *Worker

func (*Worker) Clear 

func (worker *Worker) Clear() *Worker

Clear data we sent I suggest use Clone() to avoid clear

func (*Worker) ClearAll 

func (worker *Worker) ClearAll() *Worker

ClearAll All clear include header

func (*Worker) ClearCookie 

func (worker *Worker) ClearCookie() *Worker

ClearCookie Clear Cookie

func (*Worker) Clone added in v1.0.7 

func (worker *Worker) Clone() *Worker

Clone Should Clone a new worker if you want to use repeat

func (*Worker) Delete 

func (worker *Worker) Delete() (body []byte, e error)

func (*Worker) Get 

func (worker *Worker) Get() (body []byte, e error)

Get method

func (*Worker) GetCookies 

func (worker *Worker) GetCookies() []*http.Cookie

GetCookies Get Cookies

func (*Worker) GetResponseStatusCode 

func (worker *Worker) GetResponseStatusCode() int

GetResponseStatusCode Get ResponseStatusCode

func (*Worker) Go 

func (worker *Worker) Go() (body []byte, e error)

Go Auto decide which method, Default Get.

func (*Worker) GoByMethod 

func (worker *Worker) GoByMethod(method string) (body []byte, e error)

func (*Worker) JsonToString 

func (worker *Worker) JsonToString() (string, error)

JsonToString This make effect only your worker exec serial! Attention! Change Your JSON'like Raw data to string

func (*Worker) OtherGo 

func (worker *Worker) OtherGo(method, contentType string) (body []byte, e error)

OtherGo Method 

  Method         = "OPTIONS"                ; Section 9.2
                 | "GET"                    ; Section 9.3
                 | "HEAD"                   ; Section 9.4
                 | "POST"                   ; Section 9.5
                 | "PUT"                    ; Section 9.6
                 | "DELETE"                 ; Section 9.7
                 | "TRACE"                  ; Section 9.8
                 | "CONNECT"                ; Section 9.9
                 | extension-method
extension-method = token
  token          = 1*<any CHAR except CTLs or separators>

Content Type 

"application/x-www-form-urlencoded"
"application/json"
"text/xml"
"multipart/form-data"

func (*Worker) OtherGoBinary 

func (worker *Worker) OtherGoBinary(method, contentType string) (body []byte, e error)

func (*Worker) Post 

func (worker *Worker) Post() (body []byte, e error)

Post Almost include bellow: 

"application/x-www-form-urlencoded"
"application/json"
"text/xml"
"multipart/form-data"

func (*Worker) PostFILE 

func (worker *Worker) PostFILE() (body []byte, e error)

func (*Worker) PostJSON 

func (worker *Worker) PostJSON() (body []byte, e error)

func (*Worker) PostXML 

func (worker *Worker) PostXML() (body []byte, e error)

func (*Worker) Put 

func (worker *Worker) Put() (body []byte, e error)

func (*Worker) PutFILE 

func (worker *Worker) PutFILE() (body []byte, e error)

func (*Worker) PutJSON 

func (worker *Worker) PutJSON() (body []byte, e error)

func (*Worker) PutXML 

func (worker *Worker) PutXML() (body []byte, e error)

func (*Worker) SetAfterAction 

func (worker *Worker) SetAfterAction(fc func(context.Context, *Worker)) *Worker

func (*Worker) SetBData 

func (worker *Worker) SetBData(data []byte) *Worker

func (*Worker) SetBeforeAction 

func (worker *Worker) SetBeforeAction(fc func(context.Context, *Worker)) *Worker

func (*Worker) SetContext 

func (worker *Worker) SetContext(ctx context.Context) *Worker

SetContext Set Context so Action can soft

func (*Worker) SetCookie 

func (worker *Worker) SetCookie(v string) *Worker

func (*Worker) SetCookieByFile 

func (worker *Worker) SetCookieByFile(file string) (*Worker, error)

SetCookieByFile Set Cookie by file.

func (*Worker) SetFileInfo 

func (worker *Worker) SetFileInfo(fileName, fileFormName string) *Worker

func (*Worker) SetForm 

func (worker *Worker) SetForm(form url.Values) *Worker

func (*Worker) SetFormParam added in v1.0.9 

func (worker *Worker) SetFormParam(k, v string) *Worker

func (*Worker) SetHeader 

func (worker *Worker) SetHeader(header http.Header) *Worker

SetHeader Java Bean Chain pattern

func (*Worker) SetHeaderParam added in v1.0.9 

func (worker *Worker) SetHeaderParam(k, v string) *Worker

func (*Worker) SetHost 

func (worker *Worker) SetHost(host string) *Worker

func (*Worker) SetMethod 

func (worker *Worker) SetMethod(method string) *Worker

func (*Worker) SetRefer 

func (worker *Worker) SetRefer(refer string) *Worker

func (*Worker) SetUa 

func (worker *Worker) SetUa(ua string) *Worker

func (*Worker) SetUrl 

func (worker *Worker) SetUrl(url string) *Worker

SetUrl at the same time SetHost

func (*Worker) SetWaitTime 

func (worker *Worker) SetWaitTime(num int) *Worker

func (*Worker) ToString 

func (worker *Worker) ToString() string

ToString This make effect only your worker exec serial! Attention! Change Your Raw data To string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.