tools

package
v1.1.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 13, 2023 License: GPL-3.0 Imports: 29 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var UserAgentList = map[string][]string{
	"windows": {
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.61 Safari/537.36",
		"Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.3729.169 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.53 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.5195.52 Safari/537.36",
	},
	"darwin": {
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36",
	},
	"linux": {
		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.40 Safari/537.36",
		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36",
	},
}

Functions

func ActionFuncSetCookies 

func ActionFuncSetCookies(domain *string, cookies *types.Dict) chromedp.ActionFunc

func ConvertGbkToUtf8 

func ConvertGbkToUtf8(str string) string

func ConvertStructToDict 

func ConvertStructToDict(obj any) types.Dict

ConvertStructToDict 需要结构体字段Tag设置为 `dict:"目标键名"` 或 `dict:"目标键名,omitempty"` 如 Text string `dict:"text,omitempty"`

func ConvertUtf8ToGbk 

func ConvertUtf8ToGbk(str string) string

func ConvertValueToString 

func ConvertValueToString(obj reflect.Value) string

func CovertStructToJson 

func CovertStructToJson(obj any) map[string]any

CovertStructToJson 需要结构体字段Tag设置为 `json:"目标键名"` 或 `json:"目标键名,omitempty"` 如 Text string `json:"text,omitempty"`

func CustomDownloader 

func CustomDownloader(response *requests.Response, fileTypes []string, savePath string, flags []chromedp.ExecAllocatorOption, actions ...chromedp.Action) string

func Download 

func Download(session *requests.Session, rawUrl *string, savePath *string)

func EventListenerConsoleAPICalled 

func EventListenerConsoleAPICalled(result *[]ConsoleLog) func(ev interface{})

func EventListenerNetwork 

func EventListenerNetwork(networkLogs *map[string]NetworkLog) func(ev interface{})

func EventListenerParserToDownload 

func EventListenerParserToDownload(session *requests.Session, typeDict *map[string]bool, savePath *string) func(ev interface{})

func FileDownload 

func FileDownload(response *requests.Response, fileTypes []string, savePath string) string

FileDownload 文件下载器，参考 https://github.com/chromedp/examples/blob/master/download_image/main.go

func HmacSha256 

func HmacSha256(data []byte, secret []byte) []byte

func HmacSha256Base64Encode 

func HmacSha256Base64Encode(data []byte, secret []byte) string

func HookClientJA3Func 

func HookClientJA3Func(fingerprint string) types.Hook

HookClientJA3Func 传入JA3 Client指纹 例如 "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53-10,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-21,29-23-24,0"

func HookClientMitmFunc 

func HookClientMitmFunc(fingerprint string) types.Hook

HookClientMitmFunc 传入MitmEngine指纹，自动转换 例如 "303:4,5,a,13,2f,32,33,35,38,39,c009,c00a,c013,c014:0,a,b,17,ff01:17,18:0" 目前不是很优雅，封装了JA3的函数，后面自己写个TP生成器

func HookCloudFlareWorkerFunc 

func HookCloudFlareWorkerFunc(workerHost string, headers types.Dict) types.Hook

func HookRandomUserAgent 

func HookRandomUserAgent(request any) (error, any)

func HookResponseGbkToUtf8 

func HookResponseGbkToUtf8(response any) (error, any)

func HookResponseUtf8ToGbk 

func HookResponseUtf8ToGbk(response any) (error, any)

func Md5 

func Md5(data []byte) string

func RandomIPv4 

func RandomIPv4() string

RandomIPv4 不完全合法，包含了 0/24 127/8 等地址，可能造成被WAF拦截

func RandomIPv6 

func RandomIPv6() string

RandomIPv6 不确定合法程度，没有细看相关私有地址等内容，可能造成被WAF拦截

func RandomUserAgent 

func RandomUserAgent() string

func ReportHTTPResponse 

func ReportHTTPResponse(response *requests.Response, savePath *string) string

Types

type ConsoleLog 

type ConsoleLog struct {
	Type  string
	Value string
}

type NetworkLog 

type NetworkLog struct {
	RequestID  string
	Time       time.Time
	URL        string
	Type       string
	StatusCode int64
	EndURL     string
	IP         string
	Error      string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.