config

Documentation

Index

• func ApplyFilters(filter Filter, value string) string
• func ExtractLinks(data []LinkData) []string
• type DetailSchema
• type Field
  • func (field Field) CompileFlat(e *colly.HTMLElement) (interface{}, string, error)
  • func (field Field) GetRegex() []Regex
  • func (field Field) GetRemove() []string
  • func (field Field) GetReplace() map[string]string
  • func (field Field) IsFlat() bool
  • func (field Field) IsFloat() bool
  • func (field Field) IsInt() bool
  • func (field Field) IsList() bool
  • func (field Field) IsString() bool
  • func (field Field) SafeCompile(e *colly.HTMLElement) interface{}
  • func (field Field) SafeCompileDeep(e *colly.HTMLElement) interface{}
  • func (field Field) SafeCompileFlat(e *colly.HTMLElement) interface{}
• type FieldContains
• type Filter
• type LinkData
• type LinkField
  • func (field LinkField) GetRegex() []Regex
  • func (field LinkField) GetRemove() []string
  • func (field LinkField) GetReplace() map[string]string
  • func (ls *LinkField) SafeCompile(e *colly.HTMLElement) string
• type ListItem
  • func (li *ListItem) SafeCompile(e *colly.HTMLElement) interface{}
• type ListSchema
  • func (ls *ListSchema) SafeCompile(e *colly.HTMLElement) []LinkData
• type PageSchema
  • func ReadSchema(filename string) (*PageSchema, error)
• type Pagination
• type PaginationLink
  • func (pl *PaginationLink) GetValue(e *colly.HTMLElement) string
• type ParsedLinks
  • func ParseLinkData(data []LinkData) ParsedLinks
  • func (pl ParsedLinks) Extend(links ParsedLinks)
• type Regex

Functions

func ApplyFilters

func ApplyFilters(filter Filter, value string) string

func ExtractLinks

func ExtractLinks(data []LinkData) []string

Types

type DetailSchema

type DetailSchema struct {
	Fields []Field
}

type Field

type Field struct {
	Name     string
	Selector string
	Type     *string
	Attr     *string
	Contains *FieldContains
	Replace  map[string]string
	Remove   []string
	Regex    []Regex
	Item     *ListItem
}

func (Field) CompileFlat

func (field Field) CompileFlat(e *colly.HTMLElement) (interface{}, string, error)

func (Field) GetRegex

func (field Field) GetRegex() []Regex

func (Field) GetRemove

func (field Field) GetRemove() []string

func (Field) GetReplace

func (field Field) GetReplace() map[string]string

func (Field) IsFlat

func (field Field) IsFlat() bool

func (Field) IsFloat

func (field Field) IsFloat() bool

func (Field) IsInt

func (field Field) IsInt() bool

func (Field) IsList

func (field Field) IsList() bool

func (Field) IsString

func (field Field) IsString() bool

func (Field) SafeCompile

func (field Field) SafeCompile(e *colly.HTMLElement) interface{}

func (Field) SafeCompileDeep

func (field Field) SafeCompileDeep(e *colly.HTMLElement) interface{}

func (Field) SafeCompileFlat

func (field Field) SafeCompileFlat(e *colly.HTMLElement) interface{}

type FieldContains

type FieldContains struct {
	Raw    bool
	String string
}

type Filter

type Filter interface {
	GetRegex() []Regex
	GetRemove() []string
	GetReplace() map[string]string
}

type LinkData

type LinkData struct {
	A    string                 `json:"a"`
	Data map[string]interface{} `json:"data"`
}

type LinkField

type LinkField struct {
	Selector string `yaml:"selector"`
	Attr     string
	Regex    []Regex
	Remove   []string
	Replace  map[string]string
}

func (LinkField) GetRegex

func (field LinkField) GetRegex() []Regex

func (LinkField) GetRemove

func (field LinkField) GetRemove() []string

func (LinkField) GetReplace

func (field LinkField) GetReplace() map[string]string

func (*LinkField) SafeCompile

func (ls *LinkField) SafeCompile(e *colly.HTMLElement) string

type ListItem

type ListItem struct {
	Plain    bool
	Selector *string
	Field    Field
	Fields   []Field
}

func (*ListItem) SafeCompile

func (li *ListItem) SafeCompile(e *colly.HTMLElement) interface{}

type ListSchema

type ListSchema struct {
	ContainerSelector string `yaml:"container_selector"`
	ItemSelector      string `yaml:"item_selector"`
	Prefix            string `yaml:"prefix"`
	IncludePrefix     bool   `yaml:"include_prefix"`
	Pagination        Pagination
	Link              struct {
		A    LinkField
		Data []Field
	}
}

func (*ListSchema) SafeCompile

func (ls *ListSchema) SafeCompile(e *colly.HTMLElement) []LinkData

type PageSchema

type PageSchema struct {
	ID         string
	Version    string
	Cloudflare bool
	Detail     DetailSchema
	List       ListSchema
}

func ReadSchema

func ReadSchema(filename string) (*PageSchema, error)

type Pagination

type Pagination struct {
	Next     PaginationLink
	Previous PaginationLink
}

type PaginationLink

type PaginationLink struct {
	Selector string
	Attr     string
	Prefix   string
	Limit    int
}

func (*PaginationLink) GetValue

func (pl *PaginationLink) GetValue(e *colly.HTMLElement) string

type ParsedLinks

type ParsedLinks map[string]map[string]interface{}

func ParseLinkData

func ParseLinkData(data []LinkData) ParsedLinks

func (ParsedLinks) Extend

func (pl ParsedLinks) Extend(links ParsedLinks)

type Regex

type Regex struct {
	Pattern string
	Group   int
}