hawkgo

package module
v0.0.0-...-a7781ad Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 30, 2021 License: MIT Imports: 16 Imported by: 0

README

hawk-go

Port of HawkAPI's cloudscraper

Everything is exposed to give access to modification, this is subject to change.

Installation: go get github.com/juiced-aio/hawk-go

Example:

import (
    "github.com/useflyent/fhttp/cookiejar"
    hawk "github.com/juiced-aio/hawk-go"
    http "github.com/useflyent/fhttp"
)

// Client has to be from fhttp and up to CloudFlare's standards, this can include ja3 fingerprint/http2 settings.

// Client also will need a cookie jar.
cookieJar, _: = cookiejar.New(nil)
client.Jar = cookieJar
scraper: = hawk.CFInit(client, "YOUR_KEY_HERE", true)

// You will have to create your own function if you want to solve captchas.
scraper.CaptchaFunction = func(originalURL string, siteKey string) (string, error) {
  // CaptchaFunction should return the token as a string.
  return "", nil
}

req, _ := http.NewRequest("GET", "https://www.nakedcph.com/en/product/9468/nike-sportswear-dunk-low-disrupt-ck6654-001", nil)

req.Header = http.Header{
  "sec-ch-ua":                 {`"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"`},
  "sec-ch-ua-mobile":          {`?0`},
  "upgrade-insecure-requests": {`1`},
  "user-agent":                {`Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36`},
  "accept":                    {`text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9`},
  "sec-fetch-site":            {`none`},
  "sec-fetch-mode":            {`navigate`},
  "sec-fetch-user":            {`?1`},
  "sec-fetch-dest":            {`document`},
  "accept-encoding":           {`gzip, deflate`},
  "accept-language":           {`en-US,en;q=0.9`},
  http.HeaderOrderKey:         {"sec-ch-ua", "sec-ch-ua-mobile", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"},
  http.PHeaderOrderKey:        {":method", ":authority", ":scheme", ":path"},
}

resp, err := scraper.Do(req)

All of the logic is based off of HawkAPIs cloudscraper.

Thanks to zMrKrabz for fhttp

For most questions I'd make a ticket in HawkAPI's discord server but if needed my discord is Humphreyyyy#0088.

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var EP_MAPPING = map[int]string{
	1: px_1_ep,
	2: px_2_ep,
	3: px_cap_15_ep,
	4: px_cap_hold_ep,
	5: px_cap_google_ep,
}
View Source 
var SITE_IDS = map[string]string{
	"www.hibbett.com": "PXAJDckzHD",
	"www.solebox.com": "PXuR63h57Z",
	"www.snipes.com":  "PX6XNN2xkk",
	"www.onygo.com":   "PXJ1N025xg",
	"www.revolve.com": "PX78VMO82C",
	"www.walmart.com": "PXu6b0qd2S",
	"www.ssense.com":  "PX58Asv359",
}

Functions

func CheckForCaptcha 

func CheckForCaptcha(body string) bool

func CreateParams 

func CreateParams(paramsLong map[string]string) string

func IsFingerprintChallenge 

func IsFingerprintChallenge(response *fhttp.Response) bool

func IsNewCaptchaChallenge 

func IsNewCaptchaChallenge(response *fhttp.Response) bool

func IsNewIUAMChallenge 

func IsNewIUAMChallenge(response *fhttp.Response) bool

func ReadAndCloseBody 

func ReadAndCloseBody(respBody io.ReadCloser) ([]byte, error)

func ReadAndCopyBody 

func ReadAndCopyBody(r interface{}) ([]byte, error)

func ReadAndUnmarshalBody 

func ReadAndUnmarshalBody(respBody io.ReadCloser, x interface{}) error

Types

type PX 

type PX struct {
	Domain         string
	PXID           string
	Client         http.Client
	Captcha        bool
	Gcap           bool
	CurrentPayload string
	Delay          int64
	PXResponse     PXResponse
	CaptchaSuccess bool
	PXEp           string
	URL            string
	PxHeaders      http.Header
	USER_AGENT     string
	Params         map[string]string
	MetaPayload    map[string]interface{}
	URLBase        string
	GetCaptcha     func(domain string) (string, error)
}

func PXInit 

func PXInit(client http.Client, domain, key string) (PX, error)

func (*PX) GetPayload 

func (px *PX) GetPayload(endpoint int, token ...string) error

func (*PX) GetUserAgent 

func (px *PX) GetUserAgent() (string, error)

func (*PX) PX15Solve 

func (px *PX) PX15Solve() error

func (*PX) PX1Solve 

func (px *PX) PX1Solve() error

func (*PX) PX2Solve 

func (px *PX) PX2Solve() error

func (*PX) PXGoogleSolve 

func (px *PX) PXGoogleSolve() error

func (*PX) PXHoldSolve 

func (px *PX) PXHoldSolve() error

func (*PX) ParseGoogleSiteKey 

func (px *PX) ParseGoogleSiteKey() (string, error)

func (*PX) ParsePXResponse 

func (px *PX) ParsePXResponse(captcha ...bool) map[string]string

func (*PX) PostPayloadToPX 

func (px *PX) PostPayloadToPX(isGet ...bool) error

func (*PX) Reset 

func (px *PX) Reset()

func (*PX) SolveGoogle 

func (px *PX) SolveGoogle(url string) (SolveResponse, error)

func (*PX) SolveHold 

func (px *PX) SolveHold(url string) (SolveResponse, error)

func (*PX) SolveNormal 

func (px *PX) SolveNormal(url string) (SolveResponse, error)

func (*PX) UpdateUserAgent 

func (px *PX) UpdateUserAgent(ua string)

type PXResponse 

type PXResponse struct {
	Do []string `json:"do"`
}

type Scraper 

type Scraper struct {
	Client                         fhttp.Client
	CaptchaFunction                func(originalURL, siteKey string) (string, error)
	FingerprintChallenge           bool
	Script                         string
	InitScript                     *fhttp.Response
	ChallengePayload               *fhttp.Response
	MainPayloadResponse            *fhttp.Response
	InitURL                        string
	RequestURL                     string
	CaptchaScript                  string
	ApiDomain                      string
	TimeOut                        int
	ErrorDelay                     int
	InitHeaders                    fhttp.Header
	ChallengeHeaders               fhttp.Header
	SubmitHeaders                  fhttp.Header
	OriginalRequest                *fhttp.Response
	Domain                         string
	Debug                          bool
	Key                            string
	AuthParams                     map[string]string
	Md                             string
	Captcha                        bool
	StartTime                      time.Time
	SolveRetries                   int
	SolveMaxRetries                int
	Result                         string
	Name                           string
	BaseObj                        string
	RequestPass                    string
	RequestR                       string
	TS                             int
	TargetURL                      string
	InitPayloadRetries             int
	InitPayloadMaxRetries          int
	KeyStrUriSafe                  string
	InitChallengeRetries           int
	InitChallengeMaxRetries        int
	FetchingChallengeRetries       int
	FetchingChallengeMaxRetries    int
	SubmitChallengeRetries         int
	SubmitChallengeMaxRetries      int
	ChallengeResultRetries         int
	ChallengeResultMaxRetries      int
	FinalApi                       apiResponse
	SubmitFinalChallengeRetries    int
	SubmitFinalChallengeMaxRetries int
	RerunRetries                   int
	RerunMaxRetries                int
	CaptchaRetries                 int
	CaptchaMaxRetries              int
	FirstCaptchaResult             apiResponse
	CaptchaResponseAPI             apiResponse
	SubmitCaptchaRetries           int
	SubmitCaptchaMaxRetries        int
}

Cloudflare

func CFInit 

func CFInit(client http.Client, key string, debug bool) (scraper Scraper)

func (*Scraper) ChallengeInitiationPayload 

func (scraper *Scraper) ChallengeInitiationPayload() (*http.Response, error)

func (*Scraper) Do 

func (scraper *Scraper) Do(request *http.Request) (*http.Response, error)

func (*Scraper) Get 

func (scraper *Scraper) Get(url string) (resp *http.Response, err error)

func (*Scraper) GetChallengeResult 

func (scraper *Scraper) GetChallengeResult() (*http.Response, error)

func (*Scraper) GetPage 

func (scraper *Scraper) GetPage() (*http.Response, error)

func (*Scraper) GetPayloadFromAPI 

func (scraper *Scraper) GetPayloadFromAPI() (*http.Response, error)

func (*Scraper) HandleCaptcha 

func (scraper *Scraper) HandleCaptcha() (*http.Response, error)

In Progress

func (*Scraper) HandleFinalApi 

func (scraper *Scraper) HandleFinalApi() (*http.Response, error)

func (*Scraper) HandleLoopError 

func (scraper *Scraper) HandleLoopError(errFormat string, err error)

func (*Scraper) HandleRerun 

func (scraper *Scraper) HandleRerun() (*http.Response, error)

func (*Scraper) InitiateCloudflare 

func (scraper *Scraper) InitiateCloudflare() (*http.Response, error)

func (*Scraper) InitiateScript 

func (scraper *Scraper) InitiateScript() (*http.Response, error)

func (*Scraper) Injection 

func (scraper *Scraper) Injection(response *http.Response, err error) (*http.Response, error)

func (*Scraper) Post 

func (scraper *Scraper) Post(url string, contentType string, body io.Reader) (resp *http.Response, err error)

func (*Scraper) SendMainPayload 

func (scraper *Scraper) SendMainPayload() (*http.Response, error)

func (*Scraper) Solve 

func (scraper *Scraper) Solve() (*http.Response, error)

func (*Scraper) SolvePayload 

func (scraper *Scraper) SolvePayload() (*http.Response, error)

func (*Scraper) SubmitCaptcha 

func (scraper *Scraper) SubmitCaptcha() (*http.Response, error)

func (*Scraper) SubmitChallenge 

func (scraper *Scraper) SubmitChallenge() (*http.Response, error)

func (*Scraper) SubmitFingerprintChallenge 

func (scraper *Scraper) SubmitFingerprintChallenge() (*http.Response, error)

type SolveResponse 

type SolveResponse struct {
	Result         map[string]string `json:"result"`
	CaptchaSuccess interface{}       `json:"captchaSuccess"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.