CUSTOM_GO_MODULE

Documentation

Index

• Variables
• func ADD_RESP_ERROR(newerror string)
• func PROXY_SCRAPE(url string, ALL_PARAMS ...interface{}) (bool, *goquery.Document, string)
• func SCRAPE_TOOL(URL string, EXTRA_ARGS ...string) (bool, *goquery.Document, string, string)
• func SCRAPE_from_FILE(filename string) (bool, *goquery.Document)

Variables

var API_KEY = "f26e62fe199a53b7746d712da5c2a5fde4320703"

var COUNTRY = "" // Check zenrows if you need another country  .. otherise select 'us'  ..or leave blank!

var DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36"

MULTI-PURPOSE SCREEN SCRAPE TOOL Params: URL, UserAgent Returns: bool, GOQUERY_DOC, Text of Response

var RESP_ERROR_MATRIX = []string{
	"Could not get content",
	"returned a 404",
	"The concurrency limit was reached",
}

var S_RETRY_MAX = 10

var S_RETRY_SLEEP = 15

var TIMEOUT = 120 //60

Functions

func ADD_RESP_ERROR

func ADD_RESP_ERROR(newerror string)

adds errors to the RESP_ERROR_MATRIX

func PROXY_SCRAPE

func PROXY_SCRAPE(url string, ALL_PARAMS ...interface{}) (bool, *goquery.Document, string)

This scrapes by way of using a proxy (Zenrows)... you need an account..

func SCRAPE_TOOL

func SCRAPE_TOOL(URL string, EXTRA_ARGS ...string) (bool, *goquery.Document, string, string)

func SCRAPE_from_FILE

func SCRAPE_from_FILE(filename string) (bool, *goquery.Document)