crawler

README

crawler

Documentation

Index

• Variables
• func ReaderToStoringFetchable(ctx context.Context, reader io.Reader, channelBufferSize int, store *Store, ...) <-chan fetcher.Fetchable
• type Store
  • func NewStore(dirname string) (*Store, error)
  • func (s *Store) Close()
  • func (s *Store) Exists(url string) (bool, error)
  • func (s *Store) Get(url string) (*crawled_url.CrawledUrl, io.Closer, error)
  • func (s *Store) LastCrawlTs(url string) (int64, error)
  • func (s *Store) Save(url string, resp *http.Response) error
• type StoringFetchable
  • func NewStoringFetchable(urlStr string, store *Store, minInterval time.Duration) *StoringFetchable
  • func (s *StoringFetchable) HandleResponse(resp *http.Response) error
  • func (s *StoringFetchable) Id() string
  • func (s *StoringFetchable) Request() (*http.Request, error)
  • func (s *StoringFetchable) Validate() error

Variables

var (
	ErrAlreadyCrawledRecently = errors.New("error: already crawled recently")
)

Functions

func ReaderToStoringFetchable

func ReaderToStoringFetchable(ctx context.Context, reader io.Reader, channelBufferSize int, store *Store, minInterval time.Duration) <-chan fetcher.Fetchable

Types

type Store

type Store struct {
	// contains filtered or unexported fields
}

func NewStore

func NewStore(dirname string) (*Store, error)

func (*Store) Close

func (s *Store) Close()

func (*Store) Exists

func (s *Store) Exists(url string) (bool, error)

func (*Store) Get

func (s *Store) Get(url string) (*crawled_url.CrawledUrl, io.Closer, error)

func (*Store) LastCrawlTs

func (s *Store) LastCrawlTs(url string) (int64, error)

func (*Store) Save

func (s *Store) Save(url string, resp *http.Response) error

type StoringFetchable

type StoringFetchable struct {
	// contains filtered or unexported fields
}

func NewStoringFetchable

func NewStoringFetchable(urlStr string, store *Store, minInterval time.Duration) *StoringFetchable

func (*StoringFetchable) HandleResponse

func (s *StoringFetchable) HandleResponse(resp *http.Response) error

func (*StoringFetchable) Id

func (s *StoringFetchable) Id() string

func (*StoringFetchable) Request

func (s *StoringFetchable) Request() (*http.Request, error)

func (*StoringFetchable) Validate

func (s *StoringFetchable) Validate() error