core

Documentation

Index

• Constants
• Variables
• func CanonicalPage(doc *html.Node) (string, error)
• func HtmlMarkdown(src string) template.HTML
• func StableVersion(n *html.Node) (string, string)
• func TextMarkdown(src string) string
• func Title(page string) string
• func Titles(pages []string) []string
• func WikiBasePage(href string) string
• type Curated
  • func (c *Curated) DefaultTitle() string
  • func (c *Curated) Title() string
• type DB
• type ErrNoVersion
  • func (e ErrNoVersion) Error() string
• type ErrNotFound
  • func (e ErrNotFound) Error() string
• type ErrRedirect
  • func (e ErrRedirect) Error() string
• type Link
  • func FindHeadLink(doc *html.Node) ([]Link, error)
• type Memory
  • func NewMemory() *Memory
  • func (m *Memory) CreateCurated() (string, error)
  • func (m *Memory) CuratedSetPages(id string, pages []string) error
  • func (m *Memory) CuratedSetTitle(id string, title string) error
  • func (m *Memory) CuratedSetUsed(id string) error
  • func (m *Memory) Current(limit int, order SortBy) ([]Page, error)
  • func (m *Memory) CurrentIn(pages ...string) ([]Page, error)
  • func (m *Memory) History(pages ...string) ([]Page, error)
  • func (m *Memory) Known() ([]string, error)
  • func (m *Memory) Last(page string) (*Page, error)
  • func (m *Memory) LoadCurated(id string) (*Curated, error)
  • func (m *Memory) Store(p Page) error
• type Page
• type Postgres
  • func NewPostgres(url string) (*Postgres, error)
  • func (p *Postgres) CreateCurated() (string, error)
  • func (p *Postgres) CuratedSetPages(id string, pages []string) error
  • func (p *Postgres) CuratedSetTitle(id, title string) error
  • func (p *Postgres) CuratedSetUsed(id string) error
  • func (p *Postgres) Current(limit int, order SortBy) ([]Page, error)
  • func (p *Postgres) CurrentIn(pages ...string) ([]Page, error)
  • func (p *Postgres) History(pages ...string) ([]Page, error)
  • func (p *Postgres) Known() ([]string, error)
  • func (p *Postgres) Last(page string) (*Page, error)
  • func (p *Postgres) LoadCurated(id string) (*Curated, error)
  • func (p *Postgres) Store(e Page) error
  • func (p *Postgres) StoreCurated(cur Curated) error
• type SortBy
  • func (s SortBy) OrderBy() string
• type Spider
• type SpiderCache
  • func NewSpiderCache(spider Spider) *SpiderCache
  • func (u *SpiderCache) Spider(page string) (*Page, error)
• type Table
  • func FindTables(doc *html.Node) ([]Table, error)
• type WikipediaSpider
  • func NewWikipediaSpider(url func(string) string) *WikipediaSpider
  • func (spider *WikipediaSpider) Spider(page string) (*Page, error)

Constants

const (
	DefaultUserAgent = "verssion_bot/1.0 (https://verssion.one)"
	MaxConcurrency   = 5
)

const (
	CacheOK = 6 * time.Hour
)

const DBURL = "postgresql:///w"

Variables

var ErrCuratedNotFound = errors.New("curated ID not found")

Functions

func CanonicalPage

func CanonicalPage(doc *html.Node) (string, error)

func HtmlMarkdown

func HtmlMarkdown(src string) template.HTML

markdown to html (properly escaped)

func StableVersion

func StableVersion(n *html.Node) (string, string)

func TextMarkdown

func TextMarkdown(src string) string

markdown to plain text

func Title

func Title(page string) string

title version of a wikipage path

func Titles

func Titles(pages []string) []string

func WikiBasePage

func WikiBasePage(href string) string

given a full wikipedia url returns the page name also works with parts of the wikipedia url

Types

type Curated

type Curated struct {
	CustomTitle string
	Created     time.Time
	LastUsed    time.Time
	LastUpdated time.Time
	Pages       []string
}

func (*Curated) DefaultTitle

func (c *Curated) DefaultTitle() string

func (*Curated) Title

func (c *Curated) Title() string

type DB

type DB interface {
	Last(string) (*Page, error) // Last spider
	Current(limit int, order SortBy) ([]Page, error)
	CurrentIn(...string) ([]Page, error)
	History(...string) ([]Page, error) // Newest first
	Store(Page) error
	Known() ([]string, error)

	CreateCurated() (string, error)
	LoadCurated(string) (*Curated, error) // will return (nil, nil) on not found
	CuratedSetPages(string, []string) error
	CuratedSetUsed(string) error
	CuratedSetTitle(string, string) error
}

type ErrNoVersion

type ErrNoVersion struct {
	Page string
}

func (ErrNoVersion) Error

func (e ErrNoVersion) Error() string

type ErrNotFound

type ErrNotFound struct {
	Page string
}

func (ErrNotFound) Error

func (e ErrNotFound) Error() string

type ErrRedirect

type ErrRedirect struct {
	Page, To string
}

func (ErrRedirect) Error

func (e ErrRedirect) Error() string

type Link

type Link struct {
	Rel   string
	Href  string
	Title string
}

func FindHeadLink

func FindHeadLink(doc *html.Node) ([]Link, error)

FindHeadLink returns all <link> nodes

type Memory

type Memory struct {
	// contains filtered or unexported fields
}

func NewMemory

func NewMemory() *Memory

func (*Memory) CreateCurated

func (m *Memory) CreateCurated() (string, error)

func (*Memory) CuratedSetPages

func (m *Memory) CuratedSetPages(id string, pages []string) error

func (*Memory) CuratedSetTitle

func (m *Memory) CuratedSetTitle(id string, title string) error

func (*Memory) CuratedSetUsed

func (m *Memory) CuratedSetUsed(id string) error

func (*Memory) Current

func (m *Memory) Current(limit int, order SortBy) ([]Page, error)

func (*Memory) CurrentIn

func (m *Memory) CurrentIn(pages ...string) ([]Page, error)

func (*Memory) History

func (m *Memory) History(pages ...string) ([]Page, error)

func (*Memory) Known

func (m *Memory) Known() ([]string, error)

func (*Memory) Last

func (m *Memory) Last(page string) (*Page, error)

func (*Memory) LoadCurated

func (m *Memory) LoadCurated(id string) (*Curated, error)

func (*Memory) Store

func (m *Memory) Store(p Page) error

type Page

type Page struct {
	Page          string
	T             time.Time
	StableVersion string
	Homepage      string
}

type Postgres

type Postgres struct {
	// contains filtered or unexported fields
}

func NewPostgres

func NewPostgres(url string) (*Postgres, error)

func (*Postgres) CreateCurated

func (p *Postgres) CreateCurated() (string, error)

func (*Postgres) CuratedSetPages

func (p *Postgres) CuratedSetPages(id string, pages []string) error

pages must be unique

func (*Postgres) CuratedSetTitle

func (p *Postgres) CuratedSetTitle(id, title string) error

func (*Postgres) CuratedSetUsed

func (p *Postgres) CuratedSetUsed(id string) error

func (*Postgres) Current

func (p *Postgres) Current(limit int, order SortBy) ([]Page, error)

func (*Postgres) CurrentIn

func (p *Postgres) CurrentIn(pages ...string) ([]Page, error)

func (*Postgres) History

func (p *Postgres) History(pages ...string) ([]Page, error)

History of a list of page. Newest first.

func (*Postgres) Known

func (p *Postgres) Known() ([]string, error)

func (*Postgres) Last

func (p *Postgres) Last(page string) (*Page, error)

func (*Postgres) LoadCurated

func (p *Postgres) LoadCurated(id string) (*Curated, error)

func (*Postgres) Store

func (p *Postgres) Store(e Page) error

func (*Postgres) StoreCurated

func (p *Postgres) StoreCurated(cur Curated) error

type SortBy

type SortBy int

const (
	SpiderT SortBy = iota
	Alphabet
)

func (SortBy) OrderBy

func (s SortBy) OrderBy() string

type Spider

type Spider interface {
	// Get the named page
	Spider(page string) (*Page, error)
}

type SpiderCache

type SpiderCache struct {
	// contains filtered or unexported fields
}

SpiderCache wraps a Spider implementation

func NewSpiderCache

func NewSpiderCache(spider Spider) *SpiderCache

func (*SpiderCache) Spider

func (u *SpiderCache) Spider(page string) (*Page, error)

type Table

type Table struct {
	Rows [][]string
}

func FindTables

func FindTables(doc *html.Node) ([]Table, error)

FindTables returns all top-level tables

type WikipediaSpider

type WikipediaSpider struct {
	URL       func(string) string
	UserAgent string
	// contains filtered or unexported fields
}

func NewWikipediaSpider

func NewWikipediaSpider(url func(string) string) *WikipediaSpider

NewWikipediaSpider makes a spider. url makes a URL from a page.

func (*WikipediaSpider) Spider

func (spider *WikipediaSpider) Spider(page string) (*Page, error)

Spider downloads and parses given wikipage