crawler

Documentation

Index

• type Crawler
  • func New(logger *zap.Logger) *Crawler
  • func (c *Crawler) Close()
  • func (c *Crawler) Detect() error
  • func (c *Crawler) FromAuto(useCycleTLS bool) error
  • func (c *Crawler) FromFile() error
  • func (c *Crawler) FromHTTP() error
  • func (c *Crawler) FromHTTPCycleTLS() error
  • func (c *Crawler) FromStdin() error
  • func (c *Crawler) GetContentType() string
  • func (c *Crawler) GetFeedLink() (string, string, error)
  • func (c *Crawler) GetFeedLinkFromHTML() (string, string, error)
  • func (c *Crawler) GetReadable(useCycleTLS bool) (ItemCrawled, error)
  • func (c *Crawler) GetSource() io.ReadCloser
  • func (c *Crawler) ParseFeed() (*gofeed.Feed, error)
  • func (c *Crawler) Reset()
  • func (c *Crawler) SetBasicAuth(username string, password string)
  • func (c *Crawler) SetLocation(sourceLocation string) error
• type ItemCrawled

Types

type Crawler

type Crawler struct {
	UserAgent string
	// contains filtered or unexported fields
}

func New

func New(logger *zap.Logger) *Crawler

func (*Crawler) Close

func (c *Crawler) Close()

func (*Crawler) Detect

func (c *Crawler) Detect() error

func (*Crawler) FromAuto

func (c *Crawler) FromAuto(useCycleTLS bool) error

func (*Crawler) FromFile

func (c *Crawler) FromFile() error

func (*Crawler) FromHTTP

func (c *Crawler) FromHTTP() error

func (*Crawler) FromHTTPCycleTLS

func (c *Crawler) FromHTTPCycleTLS() error

func (*Crawler) FromStdin

func (c *Crawler) FromStdin() error

func (*Crawler) GetContentType

func (c *Crawler) GetContentType() string

func (*Crawler) GetFeedLink

func (c *Crawler) GetFeedLink() (string, string, error)

func (*Crawler) GetFeedLinkFromHTML

func (c *Crawler) GetFeedLinkFromHTML() (string, string, error)

func (*Crawler) GetReadable

func (c *Crawler) GetReadable(useCycleTLS bool) (ItemCrawled, error)

func (*Crawler) GetSource

func (c *Crawler) GetSource() io.ReadCloser

func (*Crawler) ParseFeed

func (c *Crawler) ParseFeed() (*gofeed.Feed, error)

func (*Crawler) Reset

func (c *Crawler) Reset()

func (*Crawler) SetBasicAuth

func (c *Crawler) SetBasicAuth(username string, password string)

func (*Crawler) SetLocation

func (c *Crawler) SetLocation(sourceLocation string) error

type ItemCrawled

type ItemCrawled struct {
	Title       string
	Author      string
	Excerpt     string
	SiteName    string
	Image       string
	ContentHtml string
	ContentText string
}