downloader

Documentation

Index

• Constants
• type Downloader
• type RealtimeDownloader
  • func (s *RealtimeDownloader) Start()
  • func (s *RealtimeDownloader) Stop()
  • func (s *RealtimeDownloader) WaitCloser()
• type RedisDownloader
  • func (s *RedisDownloader) Start()
  • func (s *RedisDownloader) Stop()
  • func (s *RedisDownloader) WaitCloser()
• type Url

Constants

const (
	REALTIME = iota + 1
	REDIS
)

const (
	URL_KEY_PREFIX      = "LAOSJ:URLS"
	WAITTING_KEY_PREFIX = URL_KEY_PREFIX + ":WAITTING"
)

const (
	URL_CACHE_KEY = URL_KEY_PREFIX + ":DOWNLOADED" // Key for downloaded url cache
)

Types

type Downloader

type Downloader interface {
	Start()
	Stop()
	WaitCloser()
}

type RealtimeDownloader

type RealtimeDownloader struct {
	// exported
	ConcurrencyLimit int                      // max number of goroutines to download
	SourceQueue      string                   // url queue
	Store            rrstorage.StorageWrapper // for saving downloaded binary
	UrlChannelFactor int
	Urls             chan Url
	// contains filtered or unexported fields
}

func (*RealtimeDownloader) Start

func (s *RealtimeDownloader) Start()

func (*RealtimeDownloader) Stop

func (s *RealtimeDownloader) Stop()

Stop RealtimeDownloader

func (*RealtimeDownloader) WaitCloser

func (s *RealtimeDownloader) WaitCloser()

Wait all urls in redis queue be downloaded

type RedisDownloader

type RedisDownloader struct {
	// exported
	ConcurrencyLimit int                      // max number of goroutines to download
	RedisConnStr     string                   // redis connection string
	SourceQueue      string                   // url queue
	Store            rrstorage.StorageWrapper // for saving downloaded binary
	UrlChannelFactor int
	// contains filtered or unexported fields
}

RedisDownloader get urls from redis SourceQueue and download them concurrently then save downloaded binary to storage

func (*RedisDownloader) Start

func (s *RedisDownloader) Start()

Start RedisDownloader

func (*RedisDownloader) Stop

func (s *RedisDownloader) Stop()

Stop RedisDownloader

func (*RedisDownloader) WaitCloser

func (s *RedisDownloader) WaitCloser()

Wait all urls in redis queue be downloaded

type Url

type Url struct {
	V      string
	Header http.Header
}