goscraper

README

goscraper

Golang package to quickly return a preview of a webpage, you can get easily its title, description & images

Usage

func main() {
	s, err := goscraper.Scrape("https://www.w3.org/", 5)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("Icon : %s\n", s.Preview.Icon)
	fmt.Printf("Name : %s\n", s.Preview.Name)
	fmt.Printf("Title : %s\n", s.Preview.Title)
	fmt.Printf("Description : %s\n", s.Preview.Description)
	fmt.Printf("Image: %s\n", s.Preview.Images[0])
	fmt.Printf("Url : %s\n", s.Preview.Link)
}

output:

Icon : https://www.w3.org/favicon.ico
Name : www.w3.org
Title : World Wide Web Consortium (W3C)
Description : The World Wide Web Consortium (W3C) is an international community where Member organizations, a full-time staff, and the public work together to develop Web standards.
Image: https://www.w3.org/2008/site/images/logo-w3c-mobile-lg
Url : https://www.w3.org/

License

Goscraper is licensed under the MIT License.

Documentation

Index

• Variables
• type Document
  • func Scrape(uri string, maxRedirect int) (*Document, error)
• type DocumentPreview
• type Scraper
  • func (scraper *Scraper) Scrape() (*Document, error)

Variables

var (
	EscapedFragment string = "_escaped_fragment_="
)

Types

type Document

type Document struct {
	Body    bytes.Buffer
	Preview DocumentPreview
}

func Scrape

func Scrape(uri string, maxRedirect int) (*Document, error)

type DocumentPreview

type DocumentPreview struct {
	Icon        string   `json:"icon" bson:"icon"`
	Name        string   `json:"name" bson:"name"`
	Title       string   `json:"title" bson:"title"`
	Description string   `json:"description" bson:"description"`
	Images      []string `json:"images" bson:"images"`
	Link        string   `json:"link" bson:"link"`
}

type Scraper

type Scraper struct {
	Url                *url.URL
	EscapedFragmentUrl *url.URL
	MaxRedirect        int
}

func (*Scraper) Scrape

func (scraper *Scraper) Scrape() (*Document, error)