scrape

Documentation

Overview

Copyright by LunaSec (owned by Refinery Labs, Inc)

Licensed under the Business Source License v1.1 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

https://github.com/lunasec-io/lunasec/blob/master/licenses/BSL-LunaTrace.txt

See the License for the specific language governing permissions and limitations under the License.

Index

• Variables
• func NewScraper(config Config) *scraper
• func NormalizeContent(content string) (string, error)
• type Config
  • func NewConfig(provider config.Provider) (config Config, err error)
• type ProxyConfig
• type Response
• type Scraper

Variables

var ProviderSet = wire.NewSet(
	NewConfig,
	NewScraper,
	wire.Bind(new(Scraper), new(*scraper)),
)

Functions

func NewScraper

func NewScraper(config Config) *scraper

func NormalizeContent

func NormalizeContent(content string) (string, error)

Types

type Config

type Config struct {
	BrowserDomains string      `yaml:"browser_domains"`
	Workers        int         `yaml:"workers"`
	Proxy          ProxyConfig `yaml:"proxy"`
}

func NewConfig

func NewConfig(provider config.Provider) (config Config, err error)

type ProxyConfig

type ProxyConfig struct {
	Host     string `yaml:"host"`
	Username string `yaml:"username"`
	Password string `yaml:"password"`
}

type Response

type Response struct {
	Title       string
	Content     string
	ContentType string
}

type Scraper

type Scraper interface {
	ScrapeWithChrome(url string) (*Response, error)
}