scrapers

Documentation

Index

• Constants
• func MergeStreets(dir string, mID int64) ([]*pb.Settlement, error)
• func SaveMunicipalities(ms []*pb.Municipality, dir string) error
• func SaveSettlements(ss []*pb.Settlement, dir string, mID int64) error
• func Scrape4Captchas(ctx context.Context, municipalities string, samples int) (chan *pb.Captcha, chan error)
• func Scrape5Captchas(ctx context.Context, municipalities string, samples int) (chan *pb.Captcha, chan error)
• func ScrapeCaptchas(ctx context.Context, typ pb.Captcha_Type, municipalities string, samples int) (cs chan *pb.Captcha, errs chan error)
• func ScrapeMunicipalities() ([]*pb.Municipality, error)
• func ScrapeStreets(dir string, mID int64) (chan *StreetSearchResults, chan error)
• type ScalarMunicipality
• type StreetSearchResults
  • func (sr *StreetSearchResults) Save(dir string, mID int64) error

Constants

const DirPerm = 0o755

DirPerm encodes new directory permissions.

Functions

func MergeStreets

func MergeStreets(dir string, mID int64) ([]*pb.Settlement, error)

func SaveMunicipalities

func SaveMunicipalities(ms []*pb.Municipality, dir string) error

SaveMunicipalities stores municipality data in the expected directory layout.

func SaveSettlements

func SaveSettlements(ss []*pb.Settlement, dir string, mID int64) error

func Scrape4Captchas

func Scrape4Captchas(ctx context.Context, municipalities string, samples int) (chan *pb.Captcha, chan error)

Scrape5Captchas scrapes 4-digit captchas.

func Scrape5Captchas

func Scrape5Captchas(ctx context.Context, municipalities string, samples int) (chan *pb.Captcha, chan error)

Scrape5Captchas scrapes 5-digit captchas.

func ScrapeCaptchas

func ScrapeCaptchas(ctx context.Context, typ pb.Captcha_Type, municipalities string, samples int) (cs chan *pb.Captcha, errs chan error)

ScrapeCaptchas fetches captchas of any type. It will keep generating captchas until the context is cancelled.

func ScrapeMunicipalities

func ScrapeMunicipalities() ([]*pb.Municipality, error)

ScrapeMunicipalities fetches all municipality data.

func ScrapeStreets

func ScrapeStreets(dir string, mID int64) (chan *StreetSearchResults, chan error)

ScrapeStreets fetches streets for a single municipality.

Types

type ScalarMunicipality

type ScalarMunicipality struct {
	pb.Municipality

	CadastralMunicipalities []int64 `json:"cadastral_municipalities"`
}

ScalarMunicipality is a Municipality with only scalar fields. Non-scalar (i.e. message) fields are turned into references (i.e. IDs).

type StreetSearchResults

type StreetSearchResults struct {
	Query     string       `json:"query"`
	Results   []*pb.Street `json:"results"`
	UpdatedAt time.Time    `json:"updated_at"`
}

func (*StreetSearchResults) Save

func (sr *StreetSearchResults) Save(dir string, mID int64) error

Save saves the results to disk.