urlfetch

Documentation

Overview

Package urlfetch implements routines to fetch files given a URL.

urlfetch currently supports HTTP, TFTP, local files, and a retrying HTTP client.

Index

• Variables
• func Fetch(u *url.URL) (io.ReaderAt, error)
• func IsURLError(err error) bool
• func LazyFetch(u *url.URL) (io.ReaderAt, error)
• func RegisterScheme(scheme string, fs FileScheme)
• type FileScheme
  • func NewTFTPClient(opts ...tftp.ClientOpt) FileScheme
• type HTTPClient
  • func NewHTTPClient(c *http.Client) *HTTPClient
  • func (h HTTPClient) Fetch(u *url.URL) (io.ReaderAt, error)
• type HTTPClientWithRetries
  • func (h HTTPClientWithRetries) Fetch(u *url.URL) (io.ReaderAt, error)
• type LocalFileClient
  • func (lfs LocalFileClient) Fetch(u *url.URL) (io.ReaderAt, error)
• type MockScheme
  • func NewMockScheme(scheme string) *MockScheme
  • func (m *MockScheme) Add(host string, p string, content string)
  • func (m *MockScheme) Fetch(u *url.URL) (io.ReaderAt, error)
  • func (m *MockScheme) NumCalled(u *url.URL) uint
• type SchemeWithRetries
  • func (s *SchemeWithRetries) Fetch(u *url.URL) (io.ReaderAt, error)
• type Schemes
  • func (s Schemes) Fetch(u *url.URL) (io.ReaderAt, error)
  • func (s Schemes) LazyFetch(u *url.URL) (io.ReaderAt, error)
  • func (s Schemes) Register(scheme string, fs FileScheme)
• type TFTPClient
  • func (t *TFTPClient) Fetch(u *url.URL) (io.ReaderAt, error)
• type URLError
  • func (s *URLError) Error() string

Variables

var (
	// ErrWrongScheme means the wrong mocked scheme was used.
	ErrWrongScheme = errors.New("wrong scheme")
	// ErrNoSuchHost means there is no host record in the mock.
	ErrNoSuchHost = errors.New("no such host exists")
	// ErrNoSuchFile means there is no file record in the mock.
	ErrNoSuchFile = errors.New("no such file exists on this host")
)

var (
	// DefaultHTTPClient is the default HTTP FileScheme.
	//
	// It is not recommended to use this for HTTPS. We recommend creating an
	// http.Client that accepts only a private pool of certificates.
	DefaultHTTPClient = NewHTTPClient(http.DefaultClient)

	// DefaultTFTPClient is the default TFTP FileScheme.
	DefaultTFTPClient = NewTFTPClient()

	// DefaultSchemes are the schemes supported by default.
	DefaultSchemes = Schemes{
		"tftp": DefaultTFTPClient,
		"http": DefaultHTTPClient,
		"file": &LocalFileClient{},
	}
)

var (
	// ErrNoSuchScheme is returned by Schemes.Fetch and
	// Schemes.LazyFetch if there is no registered FileScheme
	// implementation for the given URL scheme.
	ErrNoSuchScheme = errors.New("no such scheme")
)

Functions

func Fetch

func Fetch(u *url.URL) (io.ReaderAt, error)

Fetch fetchs a file via DefaultSchemes.

func IsURLError

func IsURLError(err error) bool

IsURLError returns true iff err is a URLError.

func LazyFetch

func LazyFetch(u *url.URL) (io.ReaderAt, error)

LazyFetch calls LazyFetch on DefaultSchemes.

func RegisterScheme

func RegisterScheme(scheme string, fs FileScheme)

RegisterScheme calls DefaultSchemes.Register.

Types

type FileScheme

type FileScheme interface {
	// Fetch returns a reader that gives the contents of `u`.
	//
	// It may do so by fetching `u` and placing it in a buffer, or by
	// returning an io.ReaderAt that fetchs the file.
	Fetch(u *url.URL) (io.ReaderAt, error)
}

FileScheme represents the implementation of a URL scheme and gives access to fetching files of that scheme.

For example, an http FileScheme implementation would fetch files using the HTTP protocol.

func NewTFTPClient

func NewTFTPClient(opts ...tftp.ClientOpt) FileScheme

NewTFTPClient returns a new TFTP client based on the given tftp.ClientOpt.

type HTTPClient

type HTTPClient struct {
	// contains filtered or unexported fields
}

HTTPClient implements FileScheme for HTTP files.

func NewHTTPClient

func NewHTTPClient(c *http.Client) *HTTPClient

NewHTTPClient returns a new HTTP FileScheme based on the given http.Client.

func (HTTPClient) Fetch

func (h HTTPClient) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

type HTTPClientWithRetries

type HTTPClientWithRetries struct {
	Client  *http.Client
	BackOff backoff.BackOff
}

HTTPClientWithRetries implements FileScheme for HTTP files and automatically retries (with backoff) upon an error.

func (HTTPClientWithRetries) Fetch

func (h HTTPClientWithRetries) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

type LocalFileClient

type LocalFileClient struct{}

LocalFileClient implements FileScheme for files on disk.

func (LocalFileClient) Fetch

func (lfs LocalFileClient) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

type MockScheme

type MockScheme struct {
	// scheme is the scheme name.
	Scheme string
	// contains filtered or unexported fields
}

MockScheme is a Scheme mock for testing.

func NewMockScheme

func NewMockScheme(scheme string) *MockScheme

NewMockScheme creates a new MockScheme with the given scheme name.

func (*MockScheme) Add

func (m *MockScheme) Add(host string, p string, content string)

Add adds a file to the MockScheme

func (*MockScheme) Fetch

func (m *MockScheme) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

func (*MockScheme) NumCalled

func (m *MockScheme) NumCalled(u *url.URL) uint

NumCalled returns how many times a url has been looked up.

type SchemeWithRetries

type SchemeWithRetries struct {
	Scheme  FileScheme
	BackOff backoff.BackOff
}

SchemeWithRetries wraps a FileScheme and automatically retries (with backoff) when Fetch returns a non-nil err.

func (*SchemeWithRetries) Fetch

func (s *SchemeWithRetries) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

type Schemes

type Schemes map[string]FileScheme

Schemes is a map of URL scheme identifier -> implementation that can fetch a file for that scheme.

func (Schemes) Fetch

func (s Schemes) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch fetchs the file with the given `u`. `u.Scheme` is used to select the FileScheme via `s`.

If `s` does not contain a FileScheme for `u.Scheme`, ErrNoSuchScheme is returned.

func (Schemes) LazyFetch

func (s Schemes) LazyFetch(u *url.URL) (io.ReaderAt, error)

LazyFetch returns a reader that will Fetch the file given by `u` when Read is called, based on `u`s scheme. See Schemes.Fetch for more details.

func (Schemes) Register

func (s Schemes) Register(scheme string, fs FileScheme)

Register registers a scheme identified by `scheme` to be `fs`.

type TFTPClient

type TFTPClient struct {
	// contains filtered or unexported fields
}

TFTPClient implements FileScheme for TFTP files.

func (*TFTPClient) Fetch

func (t *TFTPClient) Fetch(u *url.URL) (io.ReaderAt, error)

Fetch implements FileScheme.Fetch.

type URLError

type URLError struct {
	URL *url.URL
	Err error
}

URLError is an error involving URLs.

func (*URLError) Error

func (s *URLError) Error() string

Error implements error.Error.