scraper

Documentation

Index

• Constants
• Variables
• func FindEntries[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)
• func FindEntriesJSON[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)
• func FindEntriesXHTML[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)
• type AttributesFinder
• type Element
• type PathFinder
• type PathFinderCategory
• type PathParserStrategy
• type SourceDefinition

Constants

const (
	XML  = "XML"
	HTML = "HTML"
)

Variables

var ErrCategoryNotAllowed = errors.New("category not allowed")

var ErrFailToCrateRequest = errors.New("fail to create request")

Functions

func FindEntries

func FindEntries[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)

func FindEntriesJSON

func FindEntriesJSON[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)

func FindEntriesXHTML

func FindEntriesXHTML[T model.IEntry](ctx context.Context, source SourceDefinition) ([]T, error)

Types

type AttributesFinder

type AttributesFinder struct {
	EntrySelector string             `yaml:"entry_selector"`
	Category      PathFinderCategory `yaml:"category"`
	Link          PathFinder         `yaml:"link"`
	Title         PathFinder         `yaml:"title"`
	Image         PathFinder         `yaml:"image"`
}

type Element

type Element interface {
	Attr(attribute string) string
	ChildAttr(selector, attribute string) string
	ChildText(selector string) string
}

type PathFinder

type PathFinder struct {
	Path          string             `yaml:"path"`
	Attribute     string             `yaml:"attribute"`
	ParseStrategy PathParserStrategy `yaml:"parse_strategy"`
}

type PathFinderCategory

type PathFinderCategory struct {
	PathFinder `yaml:"path_finder"`
	Alloweds   []string `yaml:"allows"`
}

type PathParserStrategy

type PathParserStrategy = string

const (
	ParserStrategyNone  PathParserStrategy = ""
	ParserStrategyStyle PathParserStrategy = "style"
)

type SourceDefinition

type SourceDefinition struct {
	Name           string           `yaml:"name"`
	Enabled        bool             `yaml:"enabled"`
	SupportStories bool             `yaml:"support_stories"`
	BaseURL        string           `yaml:"base_url"`
	Paths          []string         `yaml:"paths"`
	Limit          int              `yaml:"limit"`
	Parser         string           `yaml:"parser"`
	Attributes     AttributesFinder `yaml:"attributes"`
}