urlscan

Documentation

Index

• Variables
• func String(s string) *string
• func Uint64(u uint64) *uint64
• type Client
  • func NewClient(apiKey string) Client
  • func (x *Client) Search(args SearchArguments) (SearchResponse, error)
  • func (x *Client) Submit(args SubmitArguments) (Task, error)
• type ScanData
• type ScanGeo
• type ScanLists
• type ScanMeta
• type ScanPage
• type ScanResult
• type ScanStats
• type ScanStatsDetail
• type ScanTask
• type SearchArguments
• type SearchResponse
• type SubmitArguments
• type Task
  • func (x *Task) Wait() error

Examples

• Client.Search
• Client.Submit

Variables

var Logger = logrus.New()

Logger is a logrus logger. You can replace the logger with yours or change setting if you need.

Functions

func String

func String(s string) *string

String converts string variable and literal to pointer

func Uint64

func Uint64(u uint64) *uint64

Uint64 converts uint64 variable and literal to pointer

Types

type Client

type Client struct {
	BaseURL string
	// contains filtered or unexported fields
}

Client is main structure of the library, a requester to urlscan.io.

func NewClient

func NewClient(apiKey string) Client

NewClient is a constructor of Client

func (*Client) Search

func (x *Client) Search(args SearchArguments) (SearchResponse, error)

Search sends query to search existing scan results with query

Example

package main

import (
	"fmt"
	"log"

	"github.com/m-mizutani/urlscan-go/urlscan"
)

func main() {
	client := urlscan.NewClient("YOUR-API-KEY")

	resp, err := client.Search(urlscan.SearchArguments{
		Query:  urlscan.String("ip:1.2.3.x"),
		Size:   urlscan.Uint64(1),
		Offset: urlscan.Uint64(0),
	})

	if err != nil {
		log.Fatal(err)
	}

	for _, result := range resp.Results {
		fmt.Printf("Related URL: %s\n", result.Page.URL)
	}
}

func (*Client) Submit

func (x *Client) Submit(args SubmitArguments) (Task, error)

Submit sends a request of sandbox execution for specified URL.

Example

package main

import (
	"fmt"
	"log"

	"github.com/m-mizutani/urlscan-go/urlscan"
)

func main() {
	client := urlscan.NewClient("YOUR-API-KEY")
	task, err := client.Submit(urlscan.SubmitArguments{URL: "https://golang.org"})
	if err != nil {
		log.Fatal(err)
	}

	err = task.Wait()
	if err != nil {
		log.Fatal(err)
	}

	for _, cookie := range task.Result.Data.Cookies {
		fmt.Printf("Cookie: %s = %s\n", cookie.Name, cookie.Value)
	}
}

type ScanData

type ScanData struct {
	Console []interface{} `json:"console"`

	Cookies []struct {
		Domain   string  `json:"domain"`
		Expires  float64 `json:"expires"`
		HTTPOnly bool    `json:"httpOnly"`
		Name     string  `json:"name"`
		Path     string  `json:"path"`
		Secure   bool    `json:"secure"`
		Session  bool    `json:"session"`
		Size     int64   `json:"size"`
		Value    string  `json:"value"`
	} `json:"cookies"`

	Globals []struct {
		Prop string `json:"prop"`
		Type string `json:"type"`
	} `json:"globals"`
	Links []struct {
		Href string `json:"href"`
		Text string `json:"text"`
	} `json:"links"`

	Requests []struct {
		InitiatorInfo struct {
			Host string `json:"host"`
			Type string `json:"type"`
			URL  string `json:"url"`
		} `json:"initiatorInfo"`

		Request struct {
			DocumentURL    string `json:"documentURL"`
			FrameID        string `json:"frameId"`
			HasUserGesture bool   `json:"hasUserGesture"`
			Initiator      struct {
				LineNumber int64 `json:"lineNumber"`
				Stack      struct {
					CallFrames []struct {
						ColumnNumber int64  `json:"columnNumber"`
						FunctionName string `json:"functionName"`
						LineNumber   int64  `json:"lineNumber"`
						ScriptID     string `json:"scriptId"`
						URL          string `json:"url"`
					} `json:"callFrames"`
				} `json:"stack"`
				Type string `json:"type"`
				URL  string `json:"url"`
			} `json:"initiator"`
			LoaderID string `json:"loaderId"`
			Request  struct {
				HasPostData bool `json:"hasPostData"`
				Headers     struct {
					ContentType             string `json:"Content-Type"`
					Origin                  string `json:"Origin"`
					Referer                 string `json:"Referer"`
					UpgradeInsecureRequests string `json:"Upgrade-Insecure-Requests"`
					UserAgent               string `json:"User-Agent"`
				} `json:"headers"`
				InitialPriority  string `json:"initialPriority"`
				Method           string `json:"method"`
				MixedContentType string `json:"mixedContentType"`
				PostData         string `json:"postData"`
				ReferrerPolicy   string `json:"referrerPolicy"`
				URL              string `json:"url"`
			} `json:"request"`
			RequestID string  `json:"requestId"`
			Timestamp float64 `json:"timestamp"`
			Type      string  `json:"type"`
			WallTime  float64 `json:"wallTime"`
		} `json:"request"`

		Response struct {
			Abp struct {
				Source string `json:"source"`
				Type   string `json:"type"`
				URL    string `json:"url"`
			} `json:"abp"`
			Asn struct {
				Asn         string `json:"asn"`
				Country     string `json:"country"`
				Date        string `json:"date"`
				Description string `json:"description"`
				IP          string `json:"ip"`
				Name        string `json:"name"`
				Registrar   string `json:"registrar"`
				Route       string `json:"route"`
			} `json:"asn"`
			DataLength        int64         `json:"dataLength"`
			EncodedDataLength int64         `json:"encodedDataLength"`
			Geoip             ScanGeo       `json:"geoip"`
			Hash              string        `json:"hash"`
			Hashmatches       []interface{} `json:"hashmatches"`
			Rdns              struct {
				IP  string `json:"ip"`
				Ptr string `json:"ptr"`
			} `json:"rdns"`
			RequestID string `json:"requestId"`

			Response struct {
				EncodedDataLength int64             `json:"encodedDataLength"`
				Headers           map[string]string `json:"headers"`
				MimeType          string            `json:"mimeType"`
				Protocol          string            `json:"protocol"`
				RemoteIPAddress   string            `json:"remoteIPAddress"`
				RemotePort        int64             `json:"remotePort"`
				RequestHeaders    map[string]string `json:"requestHeaders"`
				SecurityDetails   struct {
					CertificateID                     int64         `json:"certificateId"`
					CertificateTransparencyCompliance string        `json:"certificateTransparencyCompliance"`
					Cipher                            string        `json:"cipher"`
					Issuer                            string        `json:"issuer"`
					KeyExchange                       string        `json:"keyExchange"`
					KeyExchangeGroup                  string        `json:"keyExchangeGroup"`
					Protocol                          string        `json:"protocol"`
					SanList                           []string      `json:"sanList"`
					SignedCertificateTimestampList    []interface{} `json:"signedCertificateTimestampList"`
					SubjectName                       string        `json:"subjectName"`
					ValidFrom                         int64         `json:"validFrom"`
					ValidTo                           int64         `json:"validTo"`
				} `json:"securityDetails"`
				SecurityHeaders []struct {
					Name  string `json:"name"`
					Value string `json:"value"`
				} `json:"securityHeaders"`
				SecurityState string `json:"securityState"`
				Status        int64  `json:"status"`
				StatusText    string `json:"statusText"`
				Timing        struct {
					ConnectEnd        float64 `json:"connectEnd"`
					ConnectStart      float64 `json:"connectStart"`
					DNSEnd            float64 `json:"dnsEnd"`
					DNSStart          float64 `json:"dnsStart"`
					ProxyEnd          int64   `json:"proxyEnd"`
					ProxyStart        int64   `json:"proxyStart"`
					PushEnd           int64   `json:"pushEnd"`
					PushStart         int64   `json:"pushStart"`
					ReceiveHeadersEnd float64 `json:"receiveHeadersEnd"`
					RequestTime       float64 `json:"requestTime"`
					SendEnd           float64 `json:"sendEnd"`
					SendStart         float64 `json:"sendStart"`
					SslEnd            float64 `json:"sslEnd"`
					SslStart          float64 `json:"sslStart"`
					WorkerReady       int64   `json:"workerReady"`
					WorkerStart       int64   `json:"workerStart"`
				} `json:"timing"`
				URL string `json:"url"`
			} `json:"response"`
			Size int64  `json:"size"`
			Type string `json:"type"`
		} `json:"response"`
	} `json:"requests"`
	Timing struct {
		BeginNavigation      string `json:"beginNavigation"`
		DomContentEventFired string `json:"domContentEventFired"`
		FrameNavigated       string `json:"frameNavigated"`
		FrameStartedLoading  string `json:"frameStartedLoading"`
		FrameStoppedLoading  string `json:"frameStoppedLoading"`
		LoadEventFired       string `json:"loadEventFired"`
	} `json:"timing"`
}

ScanData presents main result of the scan

type ScanGeo

type ScanGeo struct {
	City        string      `json:"city"`
	Country     string      `json:"country"`
	CountryName string      `json:"country_name"`
	LL          []float64   `json:"ll"`
	Metro       int64       `json:"metro"`
	Range       interface{} `json:"range"`
	Region      string      `json:"region"`
	Zip         int64       `json:"zip"`
}

ScanGeo presents GeoLocation information

type ScanLists

type ScanLists struct {
	Asns         []string `json:"asns"`
	Certificates []struct {
		Issuer      string `json:"issuer"`
		SubjectName string `json:"subjectName"`
		ValidFrom   int64  `json:"validFrom"`
		ValidTo     int64  `json:"validTo"`
	} `json:"certificates"`
	Countries   []string      `json:"countries"`
	Domains     []string      `json:"domains"`
	Hashes      []interface{} `json:"hashes"`
	Ips         []string      `json:"ips"`
	LinkDomains []string      `json:"linkDomains"`
	Servers     []string      `json:"servers"`
	Urls        []string      `json:"urls"`
}

ScanLists shows lists

type ScanMeta

type ScanMeta struct {
	Processors struct {
		Abp struct {
			Data []struct {
				Source string `json:"source"`
				Type   string `json:"type"`
				URL    string `json:"url"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"abp"`
		Asn struct {
			Data []struct {
				Asn         string `json:"asn"`
				Country     string `json:"country"`
				Date        string `json:"date"`
				Description string `json:"description"`
				IP          string `json:"ip"`
				Name        string `json:"name"`
				Registrar   string `json:"registrar"`
				Route       string `json:"route"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"asn"`
		Cdnjs struct {
			Data  []interface{} `json:"data"`
			State string        `json:"state"`
		} `json:"cdnjs"`
		Done struct {
			Data struct {
				State string `json:"state"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"done"`
		Download struct {
			Data  []interface{} `json:"data"`
			State string        `json:"state"`
		} `json:"download"`
		Geoip struct {
			Data []struct {
				Geoip ScanGeo `json:"geoip"`
				IP    string  `json:"ip"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"geoip"`
		Gsb struct {
			Data  struct{} `json:"data"`
			State string   `json:"state"`
		} `json:"gsb"`
		Rdns struct {
			Data []struct {
				IP  string `json:"ip"`
				Ptr string `json:"ptr"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"rdns"`
		Wappa struct {
			Data []struct {
				App        string `json:"app"`
				Categories []struct {
					Name     string `json:"name"`
					Priority string `json:"priority"`
				} `json:"categories"`
				Confidence []struct {
					Confidence interface{} `json:"confidence"`
					Pattern    string      `json:"pattern"`
				} `json:"confidence"`
				ConfidenceTotal int64  `json:"confidenceTotal"`
				Icon            string `json:"icon"`
				Website         string `json:"website"`
			} `json:"data"`
			State string `json:"state"`
		} `json:"wappa"`
	} `json:"processors"`
}

ScanMeta presents scan meta data

type ScanPage

type ScanPage struct {
	Asn     string `json:"asn"`
	Asnname string `json:"asnname"`
	City    string `json:"city"`
	Country string `json:"country"`
	Domain  string `json:"domain"`
	IP      string `json:"ip"`
	Ptr     string `json:"ptr"`
	Server  string `json:"server"`
	URL     string `json:"url"`
}

ScanPage shows page information

type ScanResult

type ScanResult struct {
	Data  ScanData  `json:"data"`
	Lists ScanLists `json:"lists"`
	Meta  ScanMeta  `json:"meta"`
	Page  ScanPage  `json:"page"`
	Stats ScanStats `json:"stats"`
	Task  ScanTask  `json:"task"`
}

ScanResult of a root data structure of scan result.

type ScanStats

type ScanStats struct {
	IPv6Percentage int64             `json:"IPv6Percentage"`
	AdBlocked      int64             `json:"adBlocked"`
	DomainStats    []ScanStatsDetail `json:"domainStats"`
	IPStats        []struct {
		Asn struct {
			Asn         string `json:"asn"`
			Country     string `json:"country"`
			Date        string `json:"date"`
			Description string `json:"description"`
			IP          string `json:"ip"`
			Name        string `json:"name"`
			Registrar   string `json:"registrar"`
			Route       string `json:"route"`
		} `json:"asn"`
		Count       interface{} `json:"count"`
		Countries   []string    `json:"countries"`
		DNS         struct{}    `json:"dns"`
		Domains     []string    `json:"domains"`
		EncodedSize int64       `json:"encodedSize"`
		Geoip       ScanGeo     `json:"geoip"`
		Index       int64       `json:"index"`
		IP          string      `json:"ip"`
		Ipv6        bool        `json:"ipv6"`
		Rdns        struct {
			IP  string `json:"ip"`
			Ptr string `json:"ptr"`
		} `json:"rdns"`
		Redirects int64 `json:"redirects"`
		Requests  int64 `json:"requests"`
		Size      int64 `json:"size"`
	} `json:"ipStats"`
	Malicious        int64             `json:"malicious"`
	ProtocolStats    []ScanStatsDetail `json:"protocolStats"`
	RegDomainStats   []ScanStatsDetail `json:"regDomainStats"`
	ResourceStats    []ScanStatsDetail `json:"resourceStats"`
	SecurePercentage int64             `json:"securePercentage"`
	SecureRequests   int64             `json:"secureRequests"`
	ServerStats      []ScanStatsDetail `json:"serverStats"`
	TLSStats         []ScanStatsDetail `json:"tlsStats"`
	TotalLinks       int64             `json:"totalLinks"`
	UniqCountries    int64             `json:"uniqCountries"`
}

ScanStats is a like summary of scan.

type ScanStatsDetail

type ScanStatsDetail struct {
	Compression   string      `json:"compression"`
	Count         int64       `json:"count"`
	Countries     []string    `json:"countries"`
	Domain        string      `json:"domain"`
	EncodedSize   int64       `json:"encodedSize"`
	Index         int64       `json:"index"`
	Initiators    []string    `json:"initiators"`
	Ips           []string    `json:"ips"`
	Latency       int64       `json:"latency"`
	Percentage    interface{} `json:"percentage"`
	Protocol      string      `json:"protocol"`
	Protocols     interface{} `json:"protocols"`
	Redirects     int64       `json:"redirects"`
	RegDomain     string      `json:"regDomain"`
	SecurityState interface{} `json:"securityState"`
	Server        string      `json:"server"`
	Size          int64       `json:"size"`
	Type          string      `json:"type"`
	SubDomains    []struct {
		Domain string `json:"domain"`
		Failed bool   `json:"failed"`
	} `json:"subDomains"`
}

ScanStatsDetail is a detail of scan

type ScanTask

type ScanTask struct {
	DomURL  string `json:"domURL"`
	Method  string `json:"method"`
	Options struct {
		Useragent string `json:"useragent"`
	} `json:"options"`
	ReportURL     string `json:"reportURL"`
	ScreenshotURL string `json:"screenshotURL"`
	Source        string `json:"source"`
	Time          string `json:"time"`
	URL           string `json:"url"`
	UserAgent     string `json:"userAgent"`
	UUID          string `json:"uuid"`
	Visibility    string `json:"visibility"`
}

ScanTask presents submitted task of scan

type SearchArguments

type SearchArguments struct {
	// Optional. urlscan.io search query.
	// See Help & Example of https://urlscan.io/search/ for more detail
	Query *string `json:"query"`
	// Optional. Page size
	Size *uint64 `json:"size"`
	// Optional. Offset of the result
	Offset *uint64 `json:"offset"`
	// Optional. specificied via $sort_field:$sort_order. Default: _score
	Sort *string `json:"sort"`
}

SearchArguments is input data structure of Search()

type SearchResponse

type SearchResponse struct {
	Results []struct {
		ID   string `json:"_id"`
		Page struct {
			Asn     string `json:"asn"`
			Asnname string `json:"asnname"`
			City    string `json:"city"`
			Country string `json:"country"`
			Domain  string `json:"domain"`
			IP      string `json:"ip"`
			Ptr     string `json:"ptr"`
			Server  string `json:"server"`
			URL     string `json:"url"`
		} `json:"page"`
		Result string `json:"result"`
		Stats  struct {
			ConsoleMsgs       int64 `json:"consoleMsgs"`
			DataLength        int64 `json:"dataLength"`
			EncodedDataLength int64 `json:"encodedDataLength"`
			Requests          int64 `json:"requests"`
			UniqIPs           int64 `json:"uniqIPs"`
		} `json:"stats"`
		Task struct {
			Method     string `json:"method"`
			Source     string `json:"source"`
			Time       string `json:"time"`
			URL        string `json:"url"`
			Visibility string `json:"visibility"`
		} `json:"task"`
		UniqCountries int64 `json:"uniq_countries"`
	} `json:"results"`
	Total int64 `json:"total"`
}

SearchResponse is returned by Search() and including existing scan results.

type SubmitArguments

type SubmitArguments struct {
	// URL is Required.
	URL string `json:"url"`
	// CustomAgent is optional. You can set own UserAgent as you like it.
	CustomAgent *string `json:"customagent"`
	// Referer is optional. You can set any Referer.
	Referer *string `json:"referer"`
	// Public is optional. Default is "off" that means "private". You need to set it as "on" if you want to make the result public.
	Public *string `json:"public"`
}

SubmitArguments is input argument of Submit()

type Task

type Task struct {
	Result ScanResult
	// contains filtered or unexported fields
}

Task is returned by Submit() and you can fetch a result of the submitted scan from the Task.

func (*Task) Wait

func (x *Task) Wait() error

Wait tries to retrieve a result. If scan is not still completed, it retries up to 30 times