impl

Documentation

Index

• Constants
• func RunCrawlClient(ctx context.Context, broker gmq.Broker)
• func RunCrawlServer(ctx context.Context, broker gmq.Broker)
• type ErrCrawlIMDB
  • func (e *ErrCrawlIMDB) Error() string

Constants

const (
	IMDBPrefix       = "https://imdb.com"
	Source           = "IMDB"
	UniqueIn         = 60 * 60 * 24 // seconds ， 防止重复入队
	ProcessdInterval = 10           // seconds ， 处理间隔，避免IP被封
)

Functions

func RunCrawlClient

func RunCrawlClient(ctx context.Context, broker gmq.Broker)

func RunCrawlServer

func RunCrawlServer(ctx context.Context, broker gmq.Broker)

Types

type ErrCrawlIMDB

type ErrCrawlIMDB struct {
	TopIdx string
	Op     string
	Detail string
}

func (*ErrCrawlIMDB) Error

func (e *ErrCrawlIMDB) Error() string