rpa

README

Overview

A library of auxiliary tools for rod, the goal is to simplify the process of rod scraping web data through configurability. The final output can be changed by modifying the configuration file without having to recompile the program.

Documentation

Index

• func ElementVisible(page *rod.Page, selector string) bool
• func WaitElementHide(page *rod.Page, selector string, timeoutSeconds int) (err error)
• func WaitElementShow(page *rod.Page, selector string, timeoutSeconds int) (err error)
• type Crawler
  • func (c *Crawler) AttachDefaultBrowser() *rod.Browser
  • func (c *Crawler) CrawlPage(page *rod.Page, cfgFilePath string, autoDownload bool, closeTab bool) (*IResult, error)
  • func (c *Crawler) CrawlUrl(url string, cfgFilePath string, autoDownload bool, closeTab bool) (*IResult, *rod.Page, error)
  • func (c *Crawler) OpenPage(url string, sleep int64, selector string, sign WaitTarget) (page *rod.Page, err error)
• type DownloadType
• type IConfig
• type IConfigNode
• type IDownloadConfig
• type IDownloadResult
• type IExternalResult
• type IPageLoad
• type IResult
• type WaitTarget

Functions

func ElementVisible

func ElementVisible(page *rod.Page, selector string) bool

ElementVisible detects whether the selected element is existed and visible

func WaitElementHide

func WaitElementHide(page *rod.Page, selector string, timeoutSeconds int) (err error)

func WaitElementShow

func WaitElementShow(page *rod.Page, selector string, timeoutSeconds int) (err error)

Types

type Crawler

type Crawler struct {
	Browser    *rod.Browser
	CfgFetcher func(path string) (*IConfig, error)
}

func (*Crawler) AttachDefaultBrowser

func (c *Crawler) AttachDefaultBrowser() *rod.Browser

func (*Crawler) CrawlPage

func (c *Crawler) CrawlPage(page *rod.Page, cfgFilePath string, autoDownload bool, closeTab bool) (*IResult, error)

func (*Crawler) CrawlUrl

func (c *Crawler) CrawlUrl(url string, cfgFilePath string, autoDownload bool, closeTab bool) (*IResult, *rod.Page, error)

func (*Crawler) OpenPage

func (c *Crawler) OpenPage(url string, sleep int64, selector string, sign WaitTarget) (page *rod.Page, err error)

type DownloadType

type DownloadType string

const (
	DownloadUrl     DownloadType = "url"
	DownloadElement DownloadType = "element"
)

type IConfig

type IConfig struct {
	PageLoad        IPageLoad         `json:"pageLoad,omitempty"`
	DataSection     []map[string]any  `json:"dataSection"`
	SwitchSection   map[string]any    `json:"switchSection,omitempty"`
	DownloadRoot    string            `json:"downloadRoot,omitempty"`
	DownloadSection []IDownloadConfig `json:"downloadSection,omitempty"`
}

type IConfigNode

type IConfigNode struct {
	Selector string `json:"selector"`
	Label    string `json:"label"`
	ID       string `json:"id"`
}

type IDownloadConfig

type IDownloadConfig struct {
	IConfigNode
	SavePath   string       `json:"savePath,omitempty"`
	NameProper string       `json:"nameProper,omitempty"`
	Type       DownloadType `json:"type"`
}

type IDownloadResult

type IDownloadResult struct {
	Count     int      `json:"count"`
	Errors    []int    `json:"errors"`
	FileNames []string `json:"fileNames"`
	Links     []string `json:"links"`
}

IDownloadResult is a part of result section

type IExternalResult

type IExternalResult struct {
	Config  string `json:"config"`
	Connect string `json:"connect"`
	ID      string `json:"id"`
}

type IPageLoad

type IPageLoad struct {
	Wait     WaitTarget `json:"wait"`
	Selector string     `json:"selector,omitempty"`
	Sleep    int64      `json:"sleep,omitempty"`
}

type IResult

type IResult struct {
	Data            map[string]any             `json:"data"`
	DownloadRoot    string                     `json:"downloadRoot"`
	Downloads       map[string]IDownloadResult `json:"downloads"`
	ExternalSection map[string]IExternalResult `json:"externalSection"`
}

type WaitTarget

type WaitTarget string

const (
	WaitShow  WaitTarget = "show"
	WaitHide  WaitTarget = "hide"
	WaitDelay WaitTarget = "wait"
)