fetch

Documentation

Index

• Constants
• Variables
• type BatchOptions
• type BatchURLFetcher
• type Client
  • func MustClient(options ...ClientOption) Client
  • func NewClient(options ...ClientOption) (Client, error)
• type ClientOption
  • func WithFiles(path string) ClientOption
  • func WithHTTPClient(client *http.Client) ClientOption
  • func WithTimeout(timeout time.Duration) ClientOption
  • func WithTransport(transport http.RoundTripper) ClientOption
  • func WithUserAgent(ua string) ClientOption
• type Factory
• type FeedFetcher
• type HttpError
  • func NewHTTPError(resp *http.Response) HttpError
  • func (e HttpError) Error() string
  • func (e HttpError) Is(target error) bool
  • func (e HttpError) String() string
• type URLFetcher
• type UnsupportedContentTypeError
  • func NewUnsupportedContentTypeError(contentType string) *UnsupportedContentTypeError
  • func (e UnsupportedContentTypeError) Is(target error) bool

Constants

const (
	DefaultTimeout = 30 * time.Second
)

const (
	DefaultUserAgent = "Mozilla/5.0 (X11; Linux x86_64; rv:88.0) Gecko/20100101 Firefox/88.0"
)

Variables

var (
	ErrUnsupportedContentType = UnsupportedContentTypeError{
		HttpError{
			StatusCode: http.StatusUnsupportedMediaType,
			Status:     http.StatusText(http.StatusUnsupportedMediaType),
			Message:    "Unsupported content type",
		},
	}
)

Types

type BatchOptions

type BatchOptions struct {
}

type BatchURLFetcher

type BatchURLFetcher interface {
	Batch([]string, BatchOptions) <-chan *resource.WebPage
}

type Client

type Client interface {
	Get(url string, headers http.Header) (*http.Response, error)
	Identifier() resource.FetchClient
}

func MustClient

func MustClient(options ...ClientOption) Client

func NewClient

func NewClient(options ...ClientOption) (Client, error)

type ClientOption

type ClientOption func(*defaultClient) error

func WithFiles

func WithFiles(path string) ClientOption

func WithHTTPClient

func WithHTTPClient(client *http.Client) ClientOption

func WithTimeout

func WithTimeout(timeout time.Duration) ClientOption

func WithTransport

func WithTransport(transport http.RoundTripper) ClientOption

func WithUserAgent

func WithUserAgent(ua string) ClientOption

type Factory

type Factory func() (URLFetcher, error)

type FeedFetcher

type FeedFetcher interface {
	Open(context.Context) error
	Fetch(*nurl.URL) (*resource.Feed, error)
	Close() error
}

type HttpError

type HttpError struct {
	StatusCode int
	Status     string
	Message    string
}

func NewHTTPError

func NewHTTPError(resp *http.Response) HttpError

TODO: Resolve/make consistent the pointer/value semantics of this and other errors here.

func (HttpError) Error

func (e HttpError) Error() string

func (HttpError) Is

func (e HttpError) Is(target error) bool

We consider the Is test true if the target is an HttpError. For further resolution, check the StatusCode field of the error. TODO: The idea of providing generic and specific HTTPErors is well-intentioned, but the Is checks are a bit baroque (because to make sense these need to work bidrectionally). Consider simplifying.

func (HttpError) String

func (e HttpError) String() string

type URLFetcher

type URLFetcher interface {
	Open(context.Context) error
	Fetch(*nurl.URL) (*resource.WebPage, error)
	Close() error
}

type UnsupportedContentTypeError

type UnsupportedContentTypeError struct {
	HttpError
}

func NewUnsupportedContentTypeError

func NewUnsupportedContentTypeError(contentType string) *UnsupportedContentTypeError

func (UnsupportedContentTypeError) Is

func (e UnsupportedContentTypeError) Is(target error) bool

Makes errors.Is(err, ErrUnsupportedContentType) return true for any instance of UnsupportedContentTypeError or for an HttpError with a 415 status code.