mongo

Documentation

Index

• type Storage
  • func (s *Storage) Cookies(u *url.URL) string
  • func (s *Storage) Init() error
  • func (s *Storage) IsVisited(requestID uint64) (bool, error)
  • func (s *Storage) SavePage(requestID uint64, u *url.URL, body []byte)
  • func (s *Storage) SetCookies(u *url.URL, cookies string)
  • func (s *Storage) Visited(requestID uint64) error

Types

type Storage

type Storage struct {
	Database          string
	URI               string
	VisitedCollection string "colly_visited"
	CookiesCollection string "colly_cookies"
	PagesCollection   string "colly_pages"
	// contains filtered or unexported fields
}

Storage implements a MongoDB storage backend for colly

func (*Storage) Cookies

func (s *Storage) Cookies(u *url.URL) string

Cookies implements colly/storage.Cookies()

func (*Storage) Init

func (s *Storage) Init() error

Init initializes the MongoDB storage

func (*Storage) IsVisited

func (s *Storage) IsVisited(requestID uint64) (bool, error)

IsVisited implements colly/storage.IsVisited()

func (*Storage) SavePage

func (s *Storage) SavePage(requestID uint64, u *url.URL, body []byte)

SavePage not part of colly

func (*Storage) SetCookies

func (s *Storage) SetCookies(u *url.URL, cookies string)

SetCookies implements colly/storage.SetCookies()

func (*Storage) Visited

func (s *Storage) Visited(requestID uint64) error

Visited implements colly/storage.Visited()