bolt

Documentation

Index

• type Storage
  • func NewStorage(db *bolt.DB) *Storage
  • func (s *Storage) AddRequest(request []byte) error
  • func (s *Storage) ClearVisited(siteID int) error
  • func (s *Storage) Cookies(u *url.URL) string
  • func (s *Storage) GetRequest() ([]byte, error)
  • func (s *Storage) Init() error
  • func (s *Storage) IsVisited(requestID string) (bool, error)
  • func (s *Storage) QueueSize() (int, error)
  • func (s *Storage) SetCookies(u *url.URL, cookies string)
  • func (s *Storage) Visited(requestID string) error

Types

type Storage

type Storage struct {
	// contains filtered or unexported fields
}

Storage is a implementation for colly/queue and colly/storage

func NewStorage

func NewStorage(db *bolt.DB) *Storage

func (*Storage) AddRequest

func (s *Storage) AddRequest(request []byte) error

AddRequest adds a serialized request to the queue

func (*Storage) ClearVisited

func (s *Storage) ClearVisited(siteID int) error

Clear visited for site id

func (*Storage) Cookies

func (s *Storage) Cookies(u *url.URL) string

Cookies retrieves stored cookies for a given host{}

func (*Storage) GetRequest

func (s *Storage) GetRequest() ([]byte, error)

GetRequest pops the next request from the queue or returns error if the queue is empty

func (*Storage) Init

func (s *Storage) Init() error

Init initializes the storage

func (*Storage) IsVisited

func (s *Storage) IsVisited(requestID string) (bool, error)

IsVisited returns true if the request was visited before IsVisited{} is called{}

func (*Storage) QueueSize

func (s *Storage) QueueSize() (int, error)

QueueSize returns with the size of the queue

func (*Storage) SetCookies

func (s *Storage) SetCookies(u *url.URL, cookies string)

SetCookies stores cookies for a given host{}

func (*Storage) Visited

func (s *Storage) Visited(requestID string) error

Visited receives and stores a request ID that is visited by the Collector{}