internal

Documentation

Index

• Variables
• type Collector
  • func NewCollector(script *Script, db *bolt.DB) *Collector
  • func (c Collector) Run()
• type Exporter
  • func NewExporter(db *bolt.DB) *Exporter
  • func (e Exporter) ExportCSV(filename string) error
  • func (e Exporter) ExportJSON(filename string) error
• type PipelineItem
• type Script
• type ScriptParser
  • func NewScriptParser() *ScriptParser
  • func (p ScriptParser) Parse(fileName string) (*Script, error)

Variables

var DB = []byte("DB")

DB the root bucket name

var Entries = []byte("ENTRIES")

Entries the data bucket name

Types

type Collector

type Collector struct {
	*colly.Collector // extend colly.Collector
	// contains filtered or unexported fields
}

Collector data collector

func NewCollector

func NewCollector(script *Script, db *bolt.DB) *Collector

NewCollector return an extended colly.Collector configured just the way Skrapa likes it based on the given script

func (Collector) Run

func (c Collector) Run()

Run execute the data collection

type Exporter

type Exporter struct {
	// contains filtered or unexported fields
}

Exporter parse scripts

func NewExporter

func NewExporter(db *bolt.DB) *Exporter

NewExporter return a new script parser

func (Exporter) ExportCSV

func (e Exporter) ExportCSV(filename string) error

ExportCSV export database as CSV

func (Exporter) ExportJSON

func (e Exporter) ExportJSON(filename string) error

ExportJSON export database as JSON

type PipelineItem

type PipelineItem struct {
	Selector  string `toml:"selector"`
	Action    string `toml:"action"`
	Attribute string `toml:"attr"`
	Column    string `toml:"column,omitempty"`
	VisitOnce bool   `toml:"visit_once,omitempty"`
}

PipelineItem models a Skrapa pipeline item

type Script

type Script struct {
	Main struct {
		URL            string   `toml:"url"`
		File           string   `toml:"file"`
		Format         string   `toml:"format"`
		UserAgent      string   `toml:"user_agent,omitempty"`
		Delay          int      `toml:"delay,omitempty"`
		AllowedDomains []string `toml:"allowed_domains,omitempty"`
	} `toml:"main"`
	Pipeline []*PipelineItem `toml:"pipeline"`
}

Script models the Skrapa TOML config

type ScriptParser

type ScriptParser struct{}

ScriptParser parse scripts

func NewScriptParser

func NewScriptParser() *ScriptParser

NewScriptParser return a new script parser

func (ScriptParser) Parse

func (p ScriptParser) Parse(fileName string) (*Script, error)

Parse do the parsing of scripts