webl

Documentation

Index

• Variables
• func AddDomain(domain *Resource)
• func Crawl(input string, saveDir string) bool
• func DeleteAllDomains()
• func DeleteDomain(domainName string)
• func FailOnError(err error)
• func InitLogging(isQuiet bool, isVerbose bool, isTimestamped bool, ws *websocket.Conn)
• func IsWebpage(contentType string) bool
• func NewPool(server, password string) *redis.Pool
• func ToFriendlyName(raw_url string) (name string)
• func ToFriendlyStatus(status string, code int) string
• func ToFriendlyType(raw_type string) string
• func Version() string
• type Edge
• type Graph
  • func CreateGraph(domain *Resource) Graph
• type Resource
  • func ListDomains() (domains []*Resource)
  • func LoadDomain(domain string, isBasic bool) *Resource
  • func LoadResource(domain string, isBasic bool) *Resource
  • func (resource *Resource) FriendlyName() string
  • func (resource *Resource) FriendlyStatus() string
  • func (resource *Resource) FriendlyType() string
• type Sitemap
  • func GenerateSitemap(domain *Resource, onlyValidUrls bool) *Sitemap
  • func InitSitemap() Sitemap
  • func WriteSitemap(domain *Resource, filename string) (sitemap *Sitemap)
• type UrlItem
  • func (item *UrlItem) Assets() []*Resource
  • func (item *UrlItem) FriendlyName() string
  • func (item *UrlItem) Links() []*Resource

Variables

var (
	TRACE *log.Logger
	INFO  *log.Logger
	WARN  *log.Logger
	ERROR *log.Logger
)

var (
	Pool *redis.Pool
)

Functions

func AddDomain

func AddDomain(domain *Resource)

func Crawl

func Crawl(input string, saveDir string) bool

func DeleteAllDomains

func DeleteAllDomains()

func DeleteDomain

func DeleteDomain(domainName string)

func FailOnError

func FailOnError(err error)

func InitLogging

func InitLogging(isQuiet bool, isVerbose bool, isTimestamped bool, ws *websocket.Conn)

func IsWebpage

func IsWebpage(contentType string) bool

func NewPool

func NewPool(server, password string) *redis.Pool

func ToFriendlyName

func ToFriendlyName(raw_url string) (name string)

func ToFriendlyStatus

func ToFriendlyStatus(status string, code int) string

func ToFriendlyType

func ToFriendlyType(raw_type string) string

func Version

func Version() string

Types

type Edge

type Edge struct {
	FromName string
	ToName   string
}

type Graph

type Graph struct {
	Edges []Edge
}

func CreateGraph

func CreateGraph(domain *Resource) Graph

type Resource

type Resource struct {
	Name         string
	LastAnalyzed string
	Url          string
	Status       string
	StatusCode   int
	LastModified string
	Type         string
	Links        []*Resource
	Assets       []*Resource
}

func ListDomains

func ListDomains() (domains []*Resource)

func LoadDomain

func LoadDomain(domain string, isBasic bool) *Resource

func LoadResource

func LoadResource(domain string, isBasic bool) *Resource

func (*Resource) FriendlyName

func (resource *Resource) FriendlyName() string

func (*Resource) FriendlyStatus

func (resource *Resource) FriendlyStatus() string

func (*Resource) FriendlyType

func (resource *Resource) FriendlyType() string

type Sitemap

type Sitemap struct {
	XMLName        xml.Name `xml:"urlset"`
	Namespace      string   `xml:"xmlns,attr"`
	Schema         string   `xml:"xmlns:xsi,attr"`
	SchemaLocation string   `xml:"xsi:schemaLocation,attr"`
	Urls           []UrlItem
}

func GenerateSitemap

func GenerateSitemap(domain *Resource, onlyValidUrls bool) *Sitemap

func InitSitemap

func InitSitemap() Sitemap

func WriteSitemap

func WriteSitemap(domain *Resource, filename string) (sitemap *Sitemap)

type UrlItem

type UrlItem struct {
	XMLName    xml.Name `xml:"url"`
	Loc        string   `xml:"loc"`
	StatusCode int      `xml:"-"`
	LastMod    string   `xml:"lastmod,omitempty"`
	Priority   float32  `xml:"priority,omitempty"`
	ChangeFreq string   `xml:"changefreq,omitempty"`
}

func (*UrlItem) Assets

func (item *UrlItem) Assets() []*Resource

func (*UrlItem) FriendlyName

func (item *UrlItem) FriendlyName() string

func (*UrlItem) Links

func (item *UrlItem) Links() []*Resource