crawler

Documentation

Index

• Variables
• type Engine
• type Error
  • func (e *Error) Error() string
• type Setting
  • func SettingFromFile(path string) (*Setting, error)
• type Spider
  • func NewSpider(setting *Setting) (*Spider, error)
  • func (s *Spider) Logger() error
  • func (s *Spider) Run()
• type Store
  • func NewStore() *Store
  • func (s *Store) Close()
  • func (s *Store) Delete(key string)
  • func (s *Store) Found(key string) bool
  • func (s *Store) Save(key string)
• type Writer

Variables

var (
	// Errors
	ErrBadUrl         = errors.New("bad url format")
	ErrUnsupportedTLD = errors.New("unsupported TLD")
	//
	OnTimeParse     = "OnTimeParse"
	OnProxyConnect  = "OnProxyConnect"
	OnDomainExtract = "OnDomainExtract"
	OnScrap         = "OnScrapping"
	OnWhois         = "OnWhois"
	OnDomain        = "OnDomain"
	OnMoz           = "OnMoz"
	OnTextExtract   = "OnTextExtract"
	//
	OnProcessor = "OnProcessor"
	OnStorage   = "OnStorage"
	OnPool      = "OnPool"
	OnBodyRead  = "OnBodyRead"
)

Types

type Engine

type Engine struct {
	Worker      int      `yaml:"worker"`
	Parallel    int      `yaml:"parallel"`
	Depth       int      `yaml:"depth"`
	URLPath     string   `yaml:"urls"`
	ProxyList   []string `yaml:"proxies,flow"`
	TLDs        []string `yaml:"tlds,flow"`
	Level       string   `yaml:"level"`
	RandomDelay string   `yaml:"random_delay"`
	Timeout     string   `yaml:"timeout"`
}

Engine struct

type Error

type Error struct {
	// contains filtered or unexported fields
}

Error struct

func (*Error) Error

func (e *Error) Error() string

Error method

type Setting

type Setting struct {
	Engine *Engine `yaml:"Engine"`
}

Setting struct

func SettingFromFile

func SettingFromFile(path string) (*Setting, error)

SettingFromFile func

type Spider

type Spider struct {
	// contains filtered or unexported fields
}

Spider struct

func NewSpider

func NewSpider(setting *Setting) (*Spider, error)

NewSpider with setting

func (*Spider) Logger

func (s *Spider) Logger() error

Logger func

func (*Spider) Run

func (s *Spider) Run()

Run func

type Store

type Store struct {
	// contains filtered or unexported fields
}

Store struct

func NewStore

func NewStore() *Store

NewStore func

func (*Store) Close

func (s *Store) Close()

Close

func (*Store) Delete

func (s *Store) Delete(key string)

Delete a value from store.

func (*Store) Found

func (s *Store) Found(key string) bool

Found checks if key exists on store.

func (*Store) Save

func (s *Store) Save(key string)

Save add value to store.

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer struct