goscraper

README

goscraper

Golang package to quickly return a preview of a webpage, you can get easily its title, description & images

Usage

func main() {
	s, err := goscraper.Scrape("https://www.w3.org/", 5)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("Title : %s\n", s.Preview.Title)
	fmt.Printf("Description : %s\n", s.Preview.Description)
	fmt.Printf("Image: %s\n", s.Preview.Images[0])
	fmt.Printf("Url : %s\n", s.Preview.Link)
}

output:

Title : World Wide Web Consortium (W3C)
Description : The World Wide Web Consortium (W3C) is an international community where Member organizations, a full-time staff, and the public work together to develop Web standards.
Image: https://www.w3.org/2008/site/images/logo-w3c-mobile-lg
Url : https://www.w3.org/

License

Goscraper is licensed under the MIT License.

Documentation

Index

• Variables
• type Document
  • func Scrape(uri string, maxRedirect int, language, authorization string) (*Document, error)
• type DocumentPreview
• type Scraper
  • func (scraper *Scraper) Scrape() (*Document, error)

Variables

var (
	EscapedFragment string = "_escaped_fragment_="
)

Types

type Document

type Document struct {
	Body    bytes.Buffer
	Preview DocumentPreview
}

func Scrape

func Scrape(uri string, maxRedirect int, language, authorization string) (*Document, error)

type DocumentPreview

type DocumentPreview struct {
	Title       string
	Description string
	Images      []string
	Link        string
	Name        string
	Icon        string
}

type Scraper

type Scraper struct {
	Url                *url.URL
	Target             *url.URL
	EscapedFragmentUrl *url.URL
	MaxRedirect        int
	Authorization      string
	Language           string
}

func (*Scraper) Scrape

func (scraper *Scraper) Scrape() (*Document, error)