bbcrawler

README

bbcrowler

Bug Bounty Crawler

Functionality of crawler

• Fetching hackerone.com new programs using JSON parsing
• Fetching new programs from bugcrowd.com using HTML parsing
• Fetching new hacktivity events from hackerone.com using JSON parsing

History

• This crawler created special for using inside any type of information bot and adapted for usage at Heroku

Architecture

• In-memory data saving using standard Golang maps
• For synchronisation purposes used Firebase storage

Example of usage

• See example/main.go

MIT License

Copyright (c) 2016 Maddevs

Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Documentation

Index

• Constants
• type BugCrowdCrawler
  • func NewBugCrowdCrawler(config *HackerOneCrawlerConfig) *BugCrowdCrawler
  • func (h BugCrowdCrawler) ClearNewRecords()
  • func (b *BugCrowdCrawler) Crawl()
  • func (h BugCrowdCrawler) GetNewRecords() interface{}
• type BugCrowdNewProgramsRecord
• type BugCrowdStore
  • func (h *BugCrowdStore) Clear()
  • func (h BugCrowdStore) GetNewRecords() interface{}
  • func (h *BugCrowdStore) IsEmpty() (bool, error)
  • func (h *BugCrowdStore) Store(data interface{}) error
• type Fetcher
• type FireBaseSync
  • func (f FireBaseSync) Read(data interface{}, path string) error
  • func (f FireBaseSync) Write(data interface{}, path string) error
• type FireBaseSyncer
• type H1HacktivityCrawler
  • func NewH1HacktivityCrowler(config *HackerOneCrawlerConfig) *H1HacktivityCrawler
  • func (h H1HacktivityCrawler) ClearNewRecords()
  • func (h *H1HacktivityCrawler) Crawl()
  • func (h H1HacktivityCrawler) GetNewRecords() interface{}
• type H1HacktivityReporter
• type H1HacktivityStore
  • func (h *H1HacktivityStore) Clear()
  • func (h H1HacktivityStore) GetNewRecords() interface{}
  • func (h *H1HacktivityStore) IsEmpty() (bool, error)
  • func (h *H1HacktivityStore) Store(data interface{}) error
• type H1HacktivityTeam
• type H1HacktivityTeamProfile
• type H1HacktivityTeamProfilePict
• type H1HactivityRecord
• type H1HactivityResponse
• type HackerOneBounty
• type HackerOneCrawler
  • func NewHackerOneCrowler(config *HackerOneCrawlerConfig) *HackerOneCrawler
  • func (h HackerOneCrawler) ClearNewRecords()
  • func (h *HackerOneCrawler) Crawl()
  • func (h HackerOneCrawler) GetNewRecords() interface{}
• type HackerOneCrawlerConfig
• type HackerOneRecord
• type HackerOneResponse
• type HackerOneStore
  • func (h *HackerOneStore) Clear()
  • func (h HackerOneStore) GetNewRecords() interface{}
  • func (h *HackerOneStore) IsEmpty() (bool, error)
  • func (h *HackerOneStore) Store(data interface{}) error
• type Reader
• type Storer
• type UniversalFetcher

Constants

const (
	FIREBASE_BUGCROWD_NEW_DATABASE = "bugcrowd_new"
)

const (
	FIREBASE_H1_HACKTIVITY_DATABASE = "h1hacktivity"
)

Types

type BugCrowdCrawler

type BugCrowdCrawler struct {
	sync.RWMutex

	Done chan bool
	// contains filtered or unexported fields
}

func NewBugCrowdCrawler

func NewBugCrowdCrawler(config *HackerOneCrawlerConfig) *BugCrowdCrawler

func (BugCrowdCrawler) ClearNewRecords

func (h BugCrowdCrawler) ClearNewRecords()

func (*BugCrowdCrawler) Crawl

func (b *BugCrowdCrawler) Crawl()

func (BugCrowdCrawler) GetNewRecords

func (h BugCrowdCrawler) GetNewRecords() interface{}

type BugCrowdNewProgramsRecord

type BugCrowdNewProgramsRecord struct {
	Name string `json:"name"`
	Link string `json:"link"`
}

type BugCrowdStore

type BugCrowdStore struct {
	PathToDb string

	sync.RWMutex
	// contains filtered or unexported fields
}

func (*BugCrowdStore) Clear

func (h *BugCrowdStore) Clear()

func (BugCrowdStore) GetNewRecords

func (h BugCrowdStore) GetNewRecords() interface{}

func (*BugCrowdStore) IsEmpty

func (h *BugCrowdStore) IsEmpty() (bool, error)

func (*BugCrowdStore) Store

func (h *BugCrowdStore) Store(data interface{}) error

type Fetcher

type Fetcher interface {
	Fetch(url string, params map[string]string) ([]byte, error)
}

type FireBaseSync

type FireBaseSync struct {
	Token   string
	BaseUrl string
}

func (FireBaseSync) Read

func (f FireBaseSync) Read(data interface{}, path string) error

func (FireBaseSync) Write

func (f FireBaseSync) Write(data interface{}, path string) error

type FireBaseSyncer

type FireBaseSyncer interface {
	Write(data interface{}, path string) error
	Read(data interface{}, path string) error
}

type H1HacktivityCrawler

type H1HacktivityCrawler struct {
	sync.RWMutex

	Done chan bool
	// contains filtered or unexported fields
}

func NewH1HacktivityCrowler

func NewH1HacktivityCrowler(config *HackerOneCrawlerConfig) *H1HacktivityCrawler

func (H1HacktivityCrawler) ClearNewRecords

func (h H1HacktivityCrawler) ClearNewRecords()

func (*H1HacktivityCrawler) Crawl

func (h *H1HacktivityCrawler) Crawl()

func (H1HacktivityCrawler) GetNewRecords

func (h H1HacktivityCrawler) GetNewRecords() interface{}

type H1HacktivityReporter

type H1HacktivityReporter struct {
	Id       int    `json:"id"`
	Url      string `json:"url"`
	Username string `json:"username"`
}

type H1HacktivityStore

type H1HacktivityStore struct {
	PathToDb string

	sync.RWMutex
	// contains filtered or unexported fields
}

func (*H1HacktivityStore) Clear

func (h *H1HacktivityStore) Clear()

func (H1HacktivityStore) GetNewRecords

func (h H1HacktivityStore) GetNewRecords() interface{}

func (*H1HacktivityStore) IsEmpty

func (h *H1HacktivityStore) IsEmpty() (bool, error)

func (*H1HacktivityStore) Store

func (h *H1HacktivityStore) Store(data interface{}) error

type H1HacktivityTeam

type H1HacktivityTeam struct {
	Handle             string                      `json:"handle"`
	Url                string                      `json:"url"`
	Profile            H1HacktivityTeamProfile     `json:"profile"`
	ProfilePictureUrls H1HacktivityTeamProfilePict `json:"profile_picture_urls"`
}

type H1HacktivityTeamProfile

type H1HacktivityTeamProfile struct {
	Name string `json:"name"`
}

type H1HacktivityTeamProfilePict

type H1HacktivityTeamProfilePict struct {
	Medium string `json:"medium"`
	Small  string `json:"small"`
}

type H1HactivityRecord

type H1HactivityRecord struct {
	BountyDisclosed             bool                 `json:"bounty_disclosed"`
	FormattedBounty             string               `json:"formatted_bounty"`
	Id                          int                  `json:"id"`
	LatestDisclosableAction     string               `json:"latest_disclosable_action"`
	LatestDisclosableActivityAt string               `json:"latest_disclosable_activity_at"`
	ReadableSubstate            string               `json:"readable_substate"`
	Reporter                    H1HacktivityReporter `json:"reporter"`
	Substate                    string               `json:"substate"`
	Swag                        bool                 `json:"swag"`
	Team                        H1HacktivityTeam     `json:"team"`
	Title                       string               `json:"title"`
	Url                         string               `json:"url"`
	VoteCount                   int                  `json:"vote_count"`
	Voters                      []string             `json:"voters"`
}

type H1HactivityResponse

type H1HactivityResponse struct {
	Count   int                 `json:"count"`
	Reports []H1HactivityRecord `json:"reports"`
}

type HackerOneBounty

type HackerOneBounty struct {
	BugCount      int `json:"bug_count"`
	MinimumBounty int `json:"minimum_bounty"`
}

type HackerOneCrawler

type HackerOneCrawler struct {
	sync.RWMutex

	Done chan bool
	// contains filtered or unexported fields
}

func NewHackerOneCrowler

func NewHackerOneCrowler(config *HackerOneCrawlerConfig) *HackerOneCrawler

func (HackerOneCrawler) ClearNewRecords

func (h HackerOneCrawler) ClearNewRecords()

func (*HackerOneCrawler) Crawl

func (h *HackerOneCrawler) Crawl()

func (HackerOneCrawler) GetNewRecords

func (h HackerOneCrawler) GetNewRecords() interface{}

type HackerOneCrawlerConfig

type HackerOneCrawlerConfig struct {
	SearchUrl           string
	PathToLocalDb       string
	FireBaseUrl         string
	FireBaseToken       string
	HacktivitySearchUrl string
	BugCrowdProgramsUrl string
}

type HackerOneRecord

type HackerOneRecord struct {
	Id                int             `json:"id"`
	Url               string          `json:"url"`
	Name              string          `json:"name"`
	Meta              HackerOneBounty `json:"meta"`
	About             string          `json:"about"`
	StrippedPolicy    string          `json:"stripped_policy"`
	Handle            string          `json:"handle"`
	ProfilePicture    string          `json:"profile_picture"`
	InternetBugBounty bool            `json:"internet_bug_bounty"`
}

type HackerOneResponse

type HackerOneResponse struct {
	Limit   int `json:"limit"`
	Total   int `json:"total"`
	Results []HackerOneRecord
}

type HackerOneStore

type HackerOneStore struct {
	PathToDb string

	sync.RWMutex
	// contains filtered or unexported fields
}

func (*HackerOneStore) Clear

func (h *HackerOneStore) Clear()

func (HackerOneStore) GetNewRecords

func (h HackerOneStore) GetNewRecords() interface{}

func (*HackerOneStore) IsEmpty

func (h *HackerOneStore) IsEmpty() (bool, error)

func (*HackerOneStore) Store

func (h *HackerOneStore) Store(data interface{}) error

type Reader

type Reader interface {
	Read(data []byte) (interface{}, error)
}

type Storer

type Storer interface {
	Store(data interface{}) error
	GetNewRecords() interface{}
	IsEmpty() (bool, error)
	Clear()
}

type UniversalFetcher

type UniversalFetcher interface {
	Fetch(fetchUrl string) (interface{}, error)
}