http_utils

Documentation

Index

• Constants
• func BuildRequestFromHistoryItem(historyItem *db.History) (*http.Request, error)
• func ClassifyHTTPResponseHeader(headerKey string) string
• func CreateHistoryFromHttpResponse(response *http.Response, responseData FullResponseData, ...) (*db.History, error)
• func CreateHttp2Client() *http.Client
• func CreateHttp2Transport() *http2.Transport
• func CreateHttp3Client() *http.Client
• func CreateHttp3Transport() *http3.RoundTripper
• func CreateHttpClient() *http.Client
• func CreateHttpTransport() *http.Transport
• func HeadersToString(headersMap map[string][]string) string
• func JoinCookies(cookies []*http.Cookie) string
• func ParseCookies(cookieStr string) []*http.Cookie
• func ReadHttpResponseAndCreateHistory(response *http.Response, options HistoryCreationOptions) (*db.History, error)
• func ReadResponseBodyData(response *http.Response) (body []byte, size int, err error)
• func SendRequest(client *http.Client, req *http.Request) (*http.Response, error)
• func SetRequestHeadersFromHistoryItem(request *http.Request, historyItem *db.History) error
• type FullResponseData
  • func ReadFullResponse(response *http.Response, createNewBodyStream bool) (FullResponseData, io.ReadCloser, error)
• type HistoryCreationOptions
• type HostRateLimiter
  • func NewHostRateLimiter(hostName string, rate float64, maxTokens float64) *HostRateLimiter
  • func (h *HostRateLimiter) AddRequest(request *http.Request) <-chan *ResponseWrapper
  • func (h *HostRateLimiter) GetNextQueuedRequest() *QueuedRequest
  • func (h *HostRateLimiter) ProcessQueue(client RequestExecutor)
  • func (h *HostRateLimiter) RecordResponseTime(responseTime float64)
• type MimeType
  • func GetApplicationMimeTypes() (mimeTypes []MimeType)
  • func GetAudioMimeTypes() (mimeTypes []MimeType)
  • func GetFontMimeTypes() (mimeTypes []MimeType)
  • func GetImageMimeTypes() (mimeTypes []MimeType)
  • func GetMimeTypes() (mimeTypes []MimeType)
  • func GetTextMimeTypes() (mimeTypes []MimeType)
  • func GetVideoMimeTypes() (mimeTypes []MimeType)
• type MimeTypeChecker
  • func NewMimeTypeChecker() MimeTypeChecker
  • func (c *MimeTypeChecker) GetMimeTypeFromContentTypeString(contentType string) MimeType
  • func (c *MimeTypeChecker) GetMimeTypeFromURL(url string) MimeType
• type QueuedRequest
• type RequestExecutor
• type ResponseBodyData
• type ResponseWrapper
• type SiteBehavior
  • func CheckSiteBehavior(options SiteBehaviourCheckOptions) (*SiteBehavior, error)
  • func (b *SiteBehavior) IsNotFound(history *db.History) bool
• type SiteBehaviourCheckOptions
  • func (o *SiteBehaviourCheckOptions) Validate() error
• type TokenBucket
  • func NewTokenBucket(rate float64, maxTokens float64, minRate float64) *TokenBucket
  • func (tb *TokenBucket) AdjustRate(newRate float64)
  • func (tb *TokenBucket) HasToken() bool

Constants

const (
	MimeTypeImgGroup         = "image"
	MimeTypeTextGroup        = "text"
	MimeTypeApplicationGroup = "application"
	MimeTypeModelGroup       = "model"
	MimeTypeVideoGroup       = "video"
	MimeTypeFontGroup        = "font"
	MimeTypeAudioGroup       = "audio"
)

var MimeTypes sync.Map

const (
	UPPER_THRESHOLD = 2   // 2 seconds
	LOWER_THRESHOLD = 0.3 // 300 milliseconds
	MIN_RATE        = 3   // 3 requests per second
)

const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"

Functions

func BuildRequestFromHistoryItem

func BuildRequestFromHistoryItem(historyItem *db.History) (*http.Request, error)

BuildRequestFromHistoryItem gets a history item and returns an http.Request with the same data

func ClassifyHTTPResponseHeader

func ClassifyHTTPResponseHeader(headerKey string) string

ClassifyHTTPResponseHeader classifies a given HTTP response header key by its purpose.

func CreateHistoryFromHttpResponse

func CreateHistoryFromHttpResponse(response *http.Response, responseData FullResponseData, options HistoryCreationOptions) (*db.History, error)

func CreateHttp2Client

func CreateHttp2Client() *http.Client

CreateHttp2Client creates an HTTP/2 client.

func CreateHttp2Transport

func CreateHttp2Transport() *http2.Transport

CreateHttp2Transport creates an HTTP/2 transport.

func CreateHttp3Client

func CreateHttp3Client() *http.Client

CreateHttp3Client creates an HTTP/3 client.

func CreateHttp3Transport

func CreateHttp3Transport() *http3.RoundTripper

CreateHttp3Transport creates an HTTP/3 transport.

func CreateHttpClient

func CreateHttpClient() *http.Client

CreateHttpClient creates a regular HTTP client.

func CreateHttpTransport

func CreateHttpTransport() *http.Transport

CreateHttpTransport creates an HTTP transport with no pre-defined http version.

func HeadersToString

func HeadersToString(headersMap map[string][]string) string

func JoinCookies

func JoinCookies(cookies []*http.Cookie) string

JoinCookies is a helper function to join cookies into a string

func ParseCookies

func ParseCookies(cookieStr string) []*http.Cookie

ParseCookies is a helper function to parse multiple cookies from a string

func ReadHttpResponseAndCreateHistory

func ReadHttpResponseAndCreateHistory(response *http.Response, options HistoryCreationOptions) (*db.History, error)

func ReadResponseBodyData

func ReadResponseBodyData(response *http.Response) (body []byte, size int, err error)

ReadResponseBodyData reads an http response body and returns it as string + its length as bytes

func SendRequest

func SendRequest(client *http.Client, req *http.Request) (*http.Response, error)

SendRequest sends an http request and returns the response ensuring that the Request body is still readable so we can dump it

func SetRequestHeadersFromHistoryItem

func SetRequestHeadersFromHistoryItem(request *http.Request, historyItem *db.History) error

Types

type FullResponseData

type FullResponseData struct {
	Body      []byte
	BodySize  int
	Raw       []byte
	RawString string
	RawSize   int
	// contains filtered or unexported fields
}

func ReadFullResponse

func ReadFullResponse(response *http.Response, createNewBodyStream bool) (FullResponseData, io.ReadCloser, error)

ReadResponseBodyData should be replaced by this

type HistoryCreationOptions

type HistoryCreationOptions struct {
	Source              string
	WorkspaceID         uint
	TaskID              uint
	CreateNewBodyStream bool
	PlaygroundSessionID uint
	TaskJobID           uint
}

type HostRateLimiter

type HostRateLimiter struct {
	// contains filtered or unexported fields
}

func NewHostRateLimiter

func NewHostRateLimiter(hostName string, rate float64, maxTokens float64) *HostRateLimiter

func (*HostRateLimiter) AddRequest

func (h *HostRateLimiter) AddRequest(request *http.Request) <-chan *ResponseWrapper

func (*HostRateLimiter) GetNextQueuedRequest

func (h *HostRateLimiter) GetNextQueuedRequest() *QueuedRequest

func (*HostRateLimiter) ProcessQueue

func (h *HostRateLimiter) ProcessQueue(client RequestExecutor)

func (*HostRateLimiter) RecordResponseTime

func (h *HostRateLimiter) RecordResponseTime(responseTime float64)

type MimeType

type MimeType struct {
	Extensions             []string
	MimeTypes              []string
	Group                  string
	Name                   string
	Code                   string
	CreateIssueOnDetection bool
	PassiveScanCallbacks   []interface{}
}

MimeType holds information about a specific mime type

func GetApplicationMimeTypes

func GetApplicationMimeTypes() (mimeTypes []MimeType)

func GetAudioMimeTypes

func GetAudioMimeTypes() (mimeTypes []MimeType)

func GetFontMimeTypes

func GetFontMimeTypes() (mimeTypes []MimeType)

func GetImageMimeTypes

func GetImageMimeTypes() (mimeTypes []MimeType)

func GetMimeTypes

func GetMimeTypes() (mimeTypes []MimeType)

func GetTextMimeTypes

func GetTextMimeTypes() (mimeTypes []MimeType)

func GetVideoMimeTypes

func GetVideoMimeTypes() (mimeTypes []MimeType)

type MimeTypeChecker

type MimeTypeChecker struct {
	Available []MimeType
}

MimeTypeChecker exposes few functions to detect mime type

func NewMimeTypeChecker

func NewMimeTypeChecker() MimeTypeChecker

func (*MimeTypeChecker) GetMimeTypeFromContentTypeString

func (c *MimeTypeChecker) GetMimeTypeFromContentTypeString(contentType string) MimeType

func (*MimeTypeChecker) GetMimeTypeFromURL

func (c *MimeTypeChecker) GetMimeTypeFromURL(url string) MimeType

type QueuedRequest

type QueuedRequest struct {
	Request   *http.Request
	Response  chan *ResponseWrapper
	QueueTime time.Time
}

type RequestExecutor

type RequestExecutor interface {
	Do(req *http.Request) (*http.Response, error)
}

type ResponseBodyData

type ResponseBodyData struct {
	Content string
	Size    int
	// contains filtered or unexported fields
}

type ResponseWrapper

type ResponseWrapper struct {
	Response  *http.Response
	Request   *http.Request
	QueueTime time.Time
	SentTime  time.Time
}

type SiteBehavior

type SiteBehavior struct {
	NotFoundReturns404 bool          `json:"not_found_returns_404"`
	NotFoundChanges    bool          `json:"not_found_changes"`
	NotFoundSamples    []*db.History `json:"not_found_samples"`
	BaseURLSample      *db.History   `json:"base_url_sample"`
	CommonHash         string        `json:"common_hash"`
}

func CheckSiteBehavior

func CheckSiteBehavior(options SiteBehaviourCheckOptions) (*SiteBehavior, error)

func (*SiteBehavior) IsNotFound

func (b *SiteBehavior) IsNotFound(history *db.History) bool

type SiteBehaviourCheckOptions

type SiteBehaviourCheckOptions struct {
	Concurrency            int
	BaseURL                string
	HistoryCreationOptions HistoryCreationOptions
	Client                 *http.Client
	Headers                map[string][]string `json:"headers" validate:"omitempty"`
}

func (*SiteBehaviourCheckOptions) Validate

func (o *SiteBehaviourCheckOptions) Validate() error

type TokenBucket

type TokenBucket struct {
	// contains filtered or unexported fields
}

func NewTokenBucket

func NewTokenBucket(rate float64, maxTokens float64, minRate float64) *TokenBucket

func (*TokenBucket) AdjustRate

func (tb *TokenBucket) AdjustRate(newRate float64)

func (*TokenBucket) HasToken

func (tb *TokenBucket) HasToken() bool