config

Documentation

Index

• type Config
  • func Init() (cfg Config)
• type Crawler
  • func (c *Crawler) GetPaginatorOrigin() string
• type Database
• type Header
• type Iterator
• type Paginator
• type Sleep

Types

type Config

type Config struct {
	Crawlers []Crawler `yaml:"crawlers"`
	Database Database  `yaml:"database"`
}

func Init

func Init() (cfg Config)

type Crawler

type Crawler struct {
	Name      string    `yaml:"name"`
	OnError   string    `yaml:"on_error"`
	Type      string    `yaml:"type"`
	Method    string    `yaml:"method"`
	Link      string    `yaml:"link"`
	Payload   string    `yaml:"payload"`
	Paginator Paginator `yaml:"paginator"`
	Iterator  Iterator  `yaml:"iterator"`
	Headers   []Header  `yaml:"headers"`
}

func (*Crawler) GetPaginatorOrigin

func (c *Crawler) GetPaginatorOrigin() string

type Database

type Database struct {
	Sslmode  string `yaml:"sslmode"`
	Host     string `yaml:"host"`
	Port     int    `yaml:"port"`
	Name     string `yaml:"name"`
	User     string `yaml:"user"`
	Password string `yaml:"password"`
}

type Header

type Header struct {
	Key   string `yaml:"key"`
	Value string `yaml:"value"`
}

type Iterator

type Iterator struct {
	Type              string `yaml:"type"`
	Regexp            string `yaml:"regex"`
	CollectionPath    string `yaml:"collection_path"`
	IdentificatorPath string `yaml:"identificator_path"`
	Separator         string `yaml:"separator"`
}

type Paginator

type Paginator struct {
	Start int      `yaml:"start"`
	End   int      `yaml:"end"`
	Type  string   `yaml:"type"`
	Sleep Sleep    `yaml:"sleep"`
	Key   string   `yaml:"key"`
	Items []string `yaml:"items"`
}

type Sleep

type Sleep struct {
	Min string `yaml:"min"`
	Max string `yaml:"max"`
}