gocrawlhq

README

gocrawlhq

Go client for Crawl HQ v3

Documentation

Index

• Variables
• type AddPayload
• type Client
  • func Init(key, secret, project, address, identifier string) (c *Client, err error)
  • func (c *Client) Add(ctx context.Context, URLs []URL, bypassSeencheck bool) (err error)
  • func (c *Client) Delete(ctx context.Context, URLs []URL, localCrawls int) (err error)
  • func (c *Client) Get(ctx context.Context, size int, strategy string) (URLs []URL, err error)
  • func (c *Client) GetProject(ctx context.Context) (p *Project, err error)
  • func (c *Client) Identify(msg *IdentifyMessage) (err error)
  • func (c *Client) InitWebsocketConn() (err error)
  • func (c *Client) Reset(ctx context.Context) (err error)
  • func (c *Client) ResetURL(ctx context.Context, ID string) (err error)
  • func (c *Client) Seencheck(ctx context.Context, URLs []URL) (outputURLs []URL, err error)
• type DeletePayload
• type GetResponse
• type IdentifyMessage
• type Project
• type URL

Variables

var (
	Version = "1.2.15"
)

Types

type AddPayload

type AddPayload struct {
	URLs            []URL `json:"urls"`
	BypassSeencheck bool  `json:"bypassSeencheck"`
}

type Client

type Client struct {
	Key               string
	Secret            string
	Project           string
	HQAddress         string
	Identifier        string
	URLsEndpoint      *url.URL
	SeencheckEndpoint *url.URL
	ResetEndpoint     *url.URL
	ProjectEndpoint   *url.URL
	HTTPClient        *http.Client
	WebsocketConn     *net.Conn
}

func Init

func Init(key, secret, project, address, identifier string) (c *Client, err error)

func (*Client) Add

func (c *Client) Add(ctx context.Context, URLs []URL, bypassSeencheck bool) (err error)

func (*Client) Delete

func (c *Client) Delete(ctx context.Context, URLs []URL, localCrawls int) (err error)

func (*Client) Get

func (c *Client) Get(ctx context.Context, size int, strategy string) (URLs []URL, err error)

func (*Client) GetProject

func (c *Client) GetProject(ctx context.Context) (p *Project, err error)

func (*Client) Identify

func (c *Client) Identify(msg *IdentifyMessage) (err error)

func (*Client) InitWebsocketConn

func (c *Client) InitWebsocketConn() (err error)

func (*Client) Reset

func (c *Client) Reset(ctx context.Context) (err error)

func (*Client) ResetURL

func (c *Client) ResetURL(ctx context.Context, ID string) (err error)

func (*Client) Seencheck

func (c *Client) Seencheck(ctx context.Context, URLs []URL) (outputURLs []URL, err error)

type DeletePayload

type DeletePayload struct {
	LocalCrawls int   `json:"localCrawls"`
	URLs        []URL `json:"urls"`
}

type GetResponse

type GetResponse struct {
	Project string `json:"project"`
	URLs    []URL  `json:"urls"`
}

type IdentifyMessage

type IdentifyMessage struct {
	Project    string `json:"project"`
	Job        string `json:"job"`
	IP         string `json:"ip"`
	Hostname   string `json:"hostname"`
	Identifier string `json:"identifier"`
	Timestamp  int64  `json:"timestamp"`
	GoVersion  string `json:"goVersion"`
}

type Project

type Project struct {
	Paused           bool     `json:"paused"`
	Name             string   `json:"name"`
	Exclusions       []string `json:"exclusions"`
	SeencheckEnabled bool     `json:"seencheck_enabled"`
	SeencheckTTL     int      `json:"seencheck_ttl"`
	Stats            struct {
		Pending         int `json:"pending"`
		Processing      int `json:"processing"`
		CompletedSeeds  int `json:"completed_seeds"`
		CompletedAssets int `json:"completed_assets"`
	} `json:"stats"`
}

type URL

type URL struct {
	ID        string `json:"id" db:"id"`
	Value     string `json:"value" db:"value"`
	Via       string `json:"via,omitempty" db:"via"`
	Host      string `json:"host,omitempty" db:"host"`
	Path      string `json:"path,omitempty" db:"path"`
	Type      string `json:"type,omitempty" db:"type"`
	Crawler   string `json:"crawler,omitempty" db:"crawler"`
	Status    string `json:"status" db:"status"`
	LiftOff   int64  `json:"lift_off" db:"lift_off"`
	Timestamp int64  `json:"timestamp" db:"timestamp"`
}