downloader

Documentation

Index

• type DownloadWorker
  • func NewDownloader(registry registry.Interface, writer writers.Writer) (*DownloadWorker, error)
  • func (d *DownloadWorker) Download(ctx context.Context, source string, target string, document string) error
• type Interface
  • func New(workerCount int, failFast bool, wg *sync.WaitGroup, ...) (Interface, taskqueue.QueueController, error)

Types

type DownloadWorker

type DownloadWorker struct {
	// contains filtered or unexported fields
}

DownloadWorker is the structure that processes downloads

func NewDownloader

func NewDownloader(registry registry.Interface, writer writers.Writer) (*DownloadWorker, error)

NewDownloader creates new downloader

func (*DownloadWorker) Download

func (d *DownloadWorker) Download(ctx context.Context, source string, target string, document string) error

Download downloads source as target

type Interface

type Interface interface {
	// Schedule is a typesafe wrapper for enqueuing download tasks. An error is returned if scheduling fails.
	Schedule(source string, target string, document string) error
}

Interface encapsulates activities for asynchronous and parallel scheduling and download of resources

func New

func New(workerCount int, failFast bool, wg *sync.WaitGroup, registry registry.Interface, writer writers.Writer) (Interface, taskqueue.QueueController, error)

New create a DownloadScheduler to schedule download resources