utils

Documentation

Index

• func DownloadURLs(urls []url.URL, concurrencyLevel int, outputDirectory string) error
• func ReadLines(filename string) ([]string, error)
• func StringToURL(rawurls []string) ([]url.URL, error)
• type ProgressPrinter
  • func NewProgressPrinter(numWriters int) ProgressPrinter
  • func (pp *ProgressPrinter) PrintProgress()
• type ProgressWriter
  • func (pw *ProgressWriter) Write(p []byte) (int, error)
• type Semaphore
  • func NewSemaphore(capacity int) Semaphore

Functions

func DownloadURLs

func DownloadURLs(urls []url.URL, concurrencyLevel int, outputDirectory string) error

DownloadURLs downloads all provided URLs concurrently. It limits the number of concurrent downloads to the provided concurrencyLevel.

func ReadLines

func ReadLines(filename string) ([]string, error)

ReadLines attempts to open the file indicated by filename and read it. It returns a slice containing all lines in the file and any error encountered.

func StringToURL

func StringToURL(rawurls []string) ([]url.URL, error)

StringToURL accepts a slice of raw URL strings and attempts to convert them to URL structures. It returns an error if the URL scheme is not supported. It returns a slice of URL structures and any error encountered.

Types

type ProgressPrinter

type ProgressPrinter struct {
	// contains filtered or unexported fields
}

func NewProgressPrinter

func NewProgressPrinter(numWriters int) ProgressPrinter

func (*ProgressPrinter) PrintProgress

func (pp *ProgressPrinter) PrintProgress()

type ProgressWriter

type ProgressWriter struct {
	// contains filtered or unexported fields
}

func (*ProgressWriter) Write

func (pw *ProgressWriter) Write(p []byte) (int, error)

type Semaphore

type Semaphore interface {
	Down()
	Up()
}

func NewSemaphore

func NewSemaphore(capacity int) Semaphore