proxypool

Documentation

Index

• Variables
• func AcquireProxy(addr string) error
• func CrawlProxy(exitCh chan bool, wg *sync.WaitGroup)
• func GetMySQLHandler() *gorm.DB
• func InsertProxy(proxy *Proxy) error
• func InsertProxyStr(p string) error
• func InvalidProxy(addr string) error
• func Run(exitCh chan bool, wg *sync.WaitGroup)
• func Validate(addr string) bool
• type Crawler
  • func NewCrawler(conf string) *Crawler
  • func (c *Crawler) Crawl() []string
• type Proxy
  • func ReadProxy() (records []*Proxy, err error)
  • func (Proxy) TableName() string
• type ProxyServer
  • func (s *ProxyServer) Get() string
  • func (s *ProxyServer) ReadProxy()
  • func (s *ProxyServer) Run(exitCh chan bool, wg *sync.WaitGroup)
  • func (s *ProxyServer) Web()

Variables

var (
	DiggerUrl        = "http://www.site-digger.com/html/articles/20110516/proxieslist.html"
	DiggerPattern    = `<td><script>document.write\(decrypt\("(.*)"\)\);</script></td>`
	ReDigger         = regexp.MustCompile(DiggerPattern)
	DiggerKeyPattern = `var baidu_union_id = "(.+)";`
	ReDiggerKey      = regexp.MustCompile(DiggerKeyPattern)
)

var NameFuncs = make(map[string]func() []string)

Functions

func AcquireProxy

func AcquireProxy(addr string) error

func CrawlProxy

func CrawlProxy(exitCh chan bool, wg *sync.WaitGroup)

func GetMySQLHandler

func GetMySQLHandler() *gorm.DB

func InsertProxy

func InsertProxy(proxy *Proxy) error

func InsertProxyStr

func InsertProxyStr(p string) error

func InvalidProxy

func InvalidProxy(addr string) error

func Run

func Run(exitCh chan bool, wg *sync.WaitGroup)

func Validate

func Validate(addr string) bool

Types

type Crawler

type Crawler struct {
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(conf string) *Crawler

func (*Crawler) Crawl

func (c *Crawler) Crawl() []string

type Proxy

type Proxy struct {
	Id           uint64    `json:"id" gorm:"id"`
	IpPort       string    `json:"ip_port" gorm:"ip_port"`
	UpdateTime   time.Time `json:"update_time" gorm:"update_time"`
	LastFailTime time.Time `json:"last_fail_time" gorm:"last_fail_time"`
	TotalCrawl   uint64    `json:"total_crawl" gorm:"total_crawl"`
	TotalFail    uint64    `json:"total_fail" gorm:"total_fail"`
}

func ReadProxy

func ReadProxy() (records []*Proxy, err error)

func (Proxy) TableName

func (Proxy) TableName() string

type ProxyServer

type ProxyServer struct {
	sync.RWMutex
	ProxyList []*Proxy
	N         int
}

func (*ProxyServer) Get

func (s *ProxyServer) Get() string

func (*ProxyServer) ReadProxy

func (s *ProxyServer) ReadProxy()

func (*ProxyServer) Run

func (s *ProxyServer) Run(exitCh chan bool, wg *sync.WaitGroup)

func (*ProxyServer) Web

func (s *ProxyServer) Web()