gophetch

Dec 31, 2023

Changes in this version

type Gophetch

+ Logger *slog.Logger

+ func (g *Gophetch) SetLogger(logger *slog.Logger)

Dec 15, 2023

Changes in this version

type ImageInlinerOptions

+ RelativeURL *url.URL

Dec 15, 2023

Changes in this version

type ImageInlinerOptions

+ MediaProxyURL string

type InlineStrategy

+ const InlineMediaProxy

Dec 15, 2023

Changes in this version

type ImageInlinerOptions

+ PrefixURL string

type InlineStrategy

+ const InlinePrefixProxy

Dec 12, 2023

Changes in this version

+ func ExtractDomain(rawURL string) (string, error)

+ func ExtractSrcset(srcset string) ([]string, []string)

+ type Extractor struct

+ Errors []error

+ Rules map[string]rules.Rule

+ func NewExtractor() *Extractor

+ func (e *Extractor) ApplySiteSpecificRules(site sites.Site)

+ func (e *Extractor) ExtractMetadata(node *html.Node, targetURL *url.URL) (metadata.Metadata, error)

+ func (e *Extractor) ExtractRule(node *html.Node, targetURL *url.URL, rule rules.Rule) (rules.ExtractResult, error)

+ func (e *Extractor) ExtractRuleByKey(node *html.Node, targetURL *url.URL, key string) (rules.ExtractResult, error)

+ type Gophetch struct

+ Extractor *Extractor

+ Fetchers []fetchers.HTMLFetcher

+ Parser *Parser

+ SiteRegistry map[string]sites.Site

+ func New(fetchers ...fetchers.HTMLFetcher) *Gophetch

+ func (g *Gophetch) FetchAndParse(targetURL string) (Result, error)

+ func (g *Gophetch) ReadAndParse(r io.Reader, targetURL string) (Result, error)

+ func (g *Gophetch) RegisterSite(site sites.Site)

+ type Headers map[string][]string

+ type ImageFetcher interface

+ NewImageFromURL func(url string, maxSize int) (*image.Image, error)

+ type ImageInliner struct

+ ShouldInline ShouldInlineFunc

+ func NewImageInliner(opts ImageInlinerOptions) *ImageInliner

+ func (inliner *ImageInliner) InlineImages(readableHTML string) (string, error)

+ type ImageInlinerOptions struct

+ Fetcher ImageFetcher

+ InlineStrategy InlineStrategy

+ MaxContentSize int64

+ MaxHeight int

+ MaxInlinedSize int

+ MaxWidth int

+ ShouldInlineFunc ShouldInlineFunc

+ SrcsetStrategy SrcsetStrategy

+ UploadFunc UploadFunc

+ type InlineStrategy int

+ const InlineAll

+ const InlineHybrid

+ const InlineNone

+ type Parser struct

+ func NewParser() *Parser

+ func (p *Parser) Headers() Headers

+ func (p *Parser) IsHTML() bool

+ func (p *Parser) MimeType() string

+ func (p *Parser) Node() *html.Node

+ func (p *Parser) Parse(reader io.Reader, resp *http.Response, targetURL string) error

+ func (p *Parser) URL() *url.URL

+ type RealImageFetcher struct

+ func (r *RealImageFetcher) NewImageFromURL(url string, maxSize int) (*image.Image, error)

+ type Result struct

+ FetcherName string

+ HTMLNode *html.Node

+ Headers map[string][]string

+ IsHTML bool

+ Metadata metadata.Metadata

+ MimeType string

+ Response *http.Response

+ StatusCode int

+ type ShouldInlineFunc func(*image.Image) bool

+ type SrcsetStrategy int

+ const SrcsetAllImages

+ const SrcsetLargestImage

+ const SrcsetPreferredDescriptors

+ const SrcsetSmallestImage

+ type UploadFunc func(*image.Image) (string, error)