_sedefaults

Documentation

Index

• func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, ...) error
• func DoPostRequest(urll string, requestData io.Reader, colCtx *colly.Context, ...) error
• func FieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)
• func InitializeCollectors(ctx context.Context, engineName engines.Name, options engines.Options, ...) (*colly.Collector, *colly.Collector)
• func PageFromContext(ctx *colly.Context, seName engines.Name) int
• func Prepare(ctx context.Context, info engines.Info, support engines.SupportedSettings, ...) (context.Context, error)
• func RawFieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)
• func SanitizeDescription(descText string) string
• func SanitizeFields(linkText string, titleText string, descText string) (string, string, string)
• func SanitizeTitle(titleText string) string
• func SanitizeURL(urlText string) string

Functions

func DoGetRequest

func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name) error

func DoPostRequest

func DoPostRequest(urll string, requestData io.Reader, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name) error

func FieldsFromDOM

func FieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)

Fetches from DOM via dompaths and sanitizes. Returns (url, title, description).

func InitializeCollectors

func InitializeCollectors(ctx context.Context, engineName engines.Name, options engines.Options, settings config.Settings, timings config.CategoryTimings, relay *bucket.Relay) (*colly.Collector, *colly.Collector)

func PageFromContext

func PageFromContext(ctx *colly.Context, seName engines.Name) int

func Prepare

func Prepare(ctx context.Context, info engines.Info, support engines.SupportedSettings, options engines.Options, settings config.Settings) (context.Context, error)

func RawFieldsFromDOM

func RawFieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName engines.Name) (string, string, string)

Fetches from DOM via dompaths. Returns (url, title, description).

func SanitizeDescription

func SanitizeDescription(descText string) string

func SanitizeFields

func SanitizeFields(linkText string, titleText string, descText string) (string, string, string)

func SanitizeTitle

func SanitizeTitle(titleText string) string

func SanitizeURL

func SanitizeURL(urlText string) string