console

Documentation

Overview

This file contains the web-facing handlers.

This file contains functionality related to rendering templates

Index

• Constants
• Variables
• func AddLinkIndexController(w http.ResponseWriter, req *http.Request)
• func BuildRender()
• func FindDomainController(w http.ResponseWriter, req *http.Request)
• func FindLinksController(w http.ResponseWriter, req *http.Request)
• func HomeController(w http.ResponseWriter, req *http.Request)
• func LinksController(w http.ResponseWriter, req *http.Request)
• func LinksHistoricalController(w http.ResponseWriter, req *http.Request)
• func ListDomainsController(w http.ResponseWriter, req *http.Request)
• func Run()
• func SpoofData()
• func Start()
• func Stop()
• type CqlModel
  • func NewCqlModel() (*CqlModel, error)
  • func (ds *CqlModel) Close()
  • func (ds *CqlModel) FindDomain(domain string) (*DomainInfo, error)
  • func (ds *CqlModel) FindLink(link string) (*LinkInfo, error)
  • func (ds *CqlModel) InsertLinks(links []string) []error
  • func (ds *CqlModel) ListDomains(seed string, limit int) ([]DomainInfo, error)
  • func (ds *CqlModel) ListLinkHistorical(linkUrl string, seedIndex int, limit int) ([]LinkInfo, int, error)
  • func (ds *CqlModel) ListLinks(domain string, seedUrl string, limit int) ([]LinkInfo, error)
  • func (ds *CqlModel) ListWorkingDomains(seedDomain string, limit int) ([]DomainInfo, error)
• type DomainInfo
• type LinkInfo
• type Model
• type Route
  • func Routes() []Route

Constants

const DontSeedDomain = ""

DataStore represents all the interaction the application has with the datastore.

const DontSeedIndex = 0

const DontSeedUrl = ""

const PageWindowLength = 15

Variables

var Render *render.Render

Functions

func AddLinkIndexController

func AddLinkIndexController(w http.ResponseWriter, req *http.Request)

TODO: I think that we should have a confirm page after you add the links. But thats an advanced feature.

func BuildRender

func BuildRender()

func FindDomainController

func FindDomainController(w http.ResponseWriter, req *http.Request)

func FindLinksController

func FindLinksController(w http.ResponseWriter, req *http.Request)

func HomeController

func HomeController(w http.ResponseWriter, req *http.Request)

func LinksController

func LinksController(w http.ResponseWriter, req *http.Request)

func LinksHistoricalController

func LinksHistoricalController(w http.ResponseWriter, req *http.Request)

func ListDomainsController

func ListDomainsController(w http.ResponseWriter, req *http.Request)

func Run

func Run()

Run will run console until SIGINT is caught

func SpoofData

func SpoofData()

func Start

func Start()

Start the console. NOTE: we only support a single instance of console at a time. You must match all your Start() calls with Stop() calls or else bad things happen.

func Stop

func Stop()

Stop will stop the console from running. Currently unused, but I'm leaving it here for now, as it seems like something one might want to be able to do.

Types

type CqlModel

type CqlModel struct {
	Cluster *gocql.ClusterConfig
	Db      *gocql.Session
}

Cassandra DataSTore

func NewCqlModel

func NewCqlModel() (*CqlModel, error)

func (*CqlModel) Close

func (ds *CqlModel) Close()

func (*CqlModel) FindDomain

func (ds *CqlModel) FindDomain(domain string) (*DomainInfo, error)

itr = db.Query("SELECT domain, claim_tok, claim_time FROM domain_info WHERE dispatched = true AND TOKEN(domain) > TOKEN(?) LIMIT ?", seed, limit).Iter()

func (*CqlModel) FindLink

func (ds *CqlModel) FindLink(link string) (*LinkInfo, error)

func (*CqlModel) InsertLinks

func (ds *CqlModel) InsertLinks(links []string) []error

NOTE: InsertLinks should try to insert as much information as possible return errors for things it can't handle

func (*CqlModel) ListDomains

func (ds *CqlModel) ListDomains(seed string, limit int) ([]DomainInfo, error)

func (*CqlModel) ListLinkHistorical

func (ds *CqlModel) ListLinkHistorical(linkUrl string, seedIndex int, limit int) ([]LinkInfo, int, error)

func (*CqlModel) ListLinks

func (ds *CqlModel) ListLinks(domain string, seedUrl string, limit int) ([]LinkInfo, error)

func (*CqlModel) ListWorkingDomains

func (ds *CqlModel) ListWorkingDomains(seedDomain string, limit int) ([]DomainInfo, error)

type DomainInfo

type DomainInfo struct {
	//TLD+1
	Domain string

	//Why did this domain get excluded, or empty if not excluded
	ExcludeReason string

	//When did this domain last get queued to be crawled. Or TimeQueed.IsZero() if not crawled
	TimeQueued time.Time

	//What was the UUID of the crawler that last crawled the domain
	UuidOfQueued gocql.UUID

	//Number of (unique) links found in this domain
	NumberLinksTotal int

	//Number of (unique) links queued to be processed for this domain
	NumberLinksQueued int
}

type LinkInfo

type LinkInfo struct {
	//URL of the link
	Url string

	//Status of the GET
	Status int

	//Any error reported during the get
	Error string

	//Was this excluded by robots
	RobotsExcluded bool

	//When did this link get crawled
	CrawlTime time.Time
}

type Model

type Model interface {

	// Close the data store after you're done with it
	Close()

	// InsertLinks queues a set of URLS to be crawled
	InsertLinks(links []string) []error

	// Find a specific domain
	FindDomain(domain string) (*DomainInfo, error)

	// List domains
	ListDomains(seedDomain string, limit int) ([]DomainInfo, error)

	// Same as ListDomains, but only lists the domains that are currently queued
	ListWorkingDomains(seedDomain string, limit int) ([]DomainInfo, error)

	// List links from the given domain
	ListLinks(domain string, seedUrl string, limit int) ([]LinkInfo, error)

	// For a given linkUrl, return the entire crawl history
	ListLinkHistorical(linkUrl string, seedIndex int, limit int) ([]LinkInfo, int, error)

	// Find a link
	FindLink(links string) (*LinkInfo, error)
}

var DS Model

type Route

type Route struct {
	Path       string
	Controller func(w http.ResponseWriter, req *http.Request)
}

func Routes

func Routes() []Route