goelp

package module
v0.0.0-...-756bce9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 15, 2024 License: MIT Imports: 17 Imported by: 0

README

Yelp Information Extractor in Go

Overview

This project is an open-source tool developed in Golang for extracting product information from Yelp. It's designed to be fast, efficient, and easy to use, making it an ideal solution for developers looking for Yelp bussiness information.

Features

  • Full search support
  • Extracts detailed product information from Yelp
  • Implemented in Go for performance and efficiency
  • Easy to integrate with existing Go projects
  • The code is optimize to work on this format: https://www.yelp.com/biz/[yelp biz name]

IMPORTANT

Use proxies by default, if you don't, your IP will get banned for days when yelp detects it's a bot

Examples

Quick testing
    package main

    import (
        "encoding/json"
        "log"
        "os"

        "github.com/johnbalvin/goelp"
    )
    func main(){
        client := goelp.DefaulClient()
        client.Language = "en_US"
        client.Location = "San Francisco, CA"
        searchValue := "law"
        index:=30
        searchResult, err := client.Search(index, searchValue)
        if err != nil {
           log.Println("err: ", err)
          return
        }
        log.Printf("searchResult: %+v\n",searchResult)
        f, _ := os.Create("./searchResult.json")
        json.NewEncoder(f).Encode(searchResult)
    }

    package main

    import (
        "github.com/johnbalvin/goelp"
    )
    func main(){
        //you need to have write permissions, the result will be save inside folder "test"
        goelp.TestSaveOnDisk()
    }

    package main

    import (
	    "log"
        "github.com/johnbalvin/goelp"
    )
    func main(){
        //If you have write permissions errors with the project, try printing the data at least
        datas,err:=goelp.TestNoImages()
        if err!=nil{
            log.Println("err",err)
            return
        }
        log.Printf("allDatas: %+v\n",datas)
    }
Basic data
    package main

    import (
        "log"
        "github.com/johnbalvin/goelp"
    )
    func main(){
        yelpBizURL:="https://www.yelp.com.mx/biz/[yelp bizness name]"
        client := goelp.DefaulClient()
        data, err := client.GetFromYelpBizURL(yelpBizURL)
        if err != nil {
            log.Println("test:2 -> err: ", err)
            return
        }
        log.Printf("data: %+v\n",data)
    }
Basic data and images
    package main

    import (
        "encoding/json"
        "log"
        "os"
        "github.com/johnbalvin/goelp"
    )
    func main(){
        //you need to have write permissions, the result will be save inside folder "test"
        if err := os.MkdirAll("./test/images", 0644); err != nil {
            log.Println("test 1 -> err: ", err)
            return
        }
        yelpBizURL:="https://www.yelp.com.mx/biz/[yelp bizness name]"
        client := goelp.DefaulClient()
        data,  err := client.GetFromYelpBizURL(yelpBizURL)
        if err != nil {
            log.Println("test:2 -> err: ", err)
            return
        }
        if err := data.SetImages(client.ProxyURL); err != nil {
            log.Println("test:3 -> err: ", err)
            return
        }
        for j, img := range data.Images {
        	fname3 := fmt.Sprintf("./test/images/%d%s", j, img.Extension)
        	os.WriteFile(fname3, img.Content, 0644)
        }
        f, err := os.Create("./test/data.json")
        if err != nil {
            log.Println("test:4 -> err: ", err)
            return
        }
        json.NewEncoder(f).Encode(data)
    }
With proxy
    package main

    import (
        "encoding/json"
        "log"
        "os"
        "github.com/johnbalvin/goelp"
    )
    func main(){
        //you need to have write permissions, the result will be save inside folder "test"
        if err := os.MkdirAll("./test/images", 0644); err != nil {
            log.Println("test 1 -> err: ", err)
            return
        }
        proxyURL, err := goelp.ParseProxy("http://[IP | domain]:[port]", "username", "password")
        if err != nil {
            log.Println("test:1 -> err: ", err)
            return
        }
        client := goelp.NewClient("es_MX", "San Francisco, CA", goelp.SortHighestRate, proxyURL)
        yelpBizURL:="https://www.yelp.com.mx/biz/[yelp bizness name]"
        data,  err := client.GetFromYelpBizURL(yelpBizURL)
        if err != nil {
            log.Println("test:2 -> err: ", err)
            continue
        }
        if err := data.SetImages(client.ProxyURL); err != nil {
            log.Println("test:3 -> err: ", err)
            return
        }
        for j, img := range data.Images {
        	fname3 := fmt.Sprintf("./test/images/%d%s", j, img.Extension)
        	os.WriteFile(fname3, img.Content, 0644)
        }
        f, err := os.Create("./test/data.json")
        if err != nil {
            log.Println("test:4 -> err: ", err)
            return
        }
        json.NewEncoder(f).Encode(data)
    }

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	SortHighestRate = SortValue("rating")
	SortMostViewed  = SortValue("review_count")
	SortRecomended  = SortValue("recommended")
)

Functions

func GetImg 

func GetImg(imgURL string, proxyURL *url.URL) (string, []byte, error)

func ParseProxy 

func ParseProxy(urlToParse, userName, password string) (*url.URL, error)

func RemoveSpace 

func RemoveSpace(value string) string

func TestSaveOnDisk 

func TestSaveOnDisk()

Types

type Amenity 

type Amenity struct {
	Title    string
	IsActive bool
	Icon     string
}

type Answer 

type Answer struct {
	Answer        string
	AuthorName    string
	Date          string
	HelpfullCount int
}

type Client 

type Client struct {
	Language  string    //for example es_CL,es_MX
	SortValue SortValue //for example SortHighestRate,SortMostViewed,SortRecomended
	Location  string
	ProxyURL  *url.URL
}

func DefaulClient 

func DefaulClient() Client

func NewClient 

func NewClient(language, location string, sortValue SortValue, proxyURL *url.URL) Client

func (Client) GetFromYelpBizURL 

func (cl Client) GetFromYelpBizURL(yelpURL string) (Data, error)

func (Client) Search 

func (cl Client) Search(index int, searchValue string) (SearchOutput, error)

type Coordinates 

type Coordinates struct {
	Latitude  float64
	Longitude float64
}

type Data 

type Data struct {
	Language            string
	TotalQuestionNumber int
	Name                string
	Claimed             bool
	Telephone           string
	YelpURL             string
	BizURL              string
	MainImage           string
	Rate                Rate
	Location            Location
	Coordinates         Coordinates
	Images              []Img
	Amenities           []Amenity
	Qas                 []Qa
	Reviews             []Review
}

func GetFromYelpBizURL 

func GetFromYelpBizURL(yelpURL string, proxyURL *url.URL) (Data, error)

func ParseBodyDetails 

func ParseBodyDetails(body []byte) (Data, error)

func TestImages 

func TestImages() ([]Data, error)

func TestNoImages 

func TestNoImages() ([]Data, error)

func (*Data) SetImages 

func (data *Data) SetImages(proxyURL *url.URL) error

type Img 

type Img struct {
	URL         string
	ContentType string
	Extension   string
	Content     []byte `json:"-"`
}

func (*Img) SetImage 

func (img *Img) SetImage(proxyURL *url.URL) error

type License 

type License struct {
	Licensee     string
	Number       string
	IssuedBy     string
	Trade        string
	VerifiedDate string
	ExpiryDate   string
}

type Location 

type Location struct {
	Address    string
	City       string
	RegionCode string
	Zip        string
	Country    string
}

type ManualLocation 

type ManualLocation struct {
	Adress  string
	City    string
	State   string
	Zip     string
	Country string
}

type Qa 

type Qa struct {
	Question string
	Answer   Answer
}

type Rate 

type Rate struct {
	Value       float32
	ReviewCount int
	Star1       int
	Start2      int
	Start3      int
	Start4      int
	Start5      int
}

type Review 

type Review struct {
	Name  string
	Rate  int
	Date  string
	Value string
}

type SearchItem 

type SearchItem struct {
	ID               string
	IsAd             bool
	Name             string
	SubTitle         string
	ShortDescription string
	YelpURL          string
	BizURL           string
	Rating           float32
	ReviewCount      int
	//yelp, rounds ResponseTimeMin to the biggest integer with a zero to the right(on the duration category), so if minutes is 12 on the screen is 20 minutes, if minutes is 26 on the screen is 30 minutes, if minutes is 68 on the screen is 2 hours,if minutes is 125 on the screen is 3 hours
	ResponseTimeMin int
	ManualLocation  ManualLocation
	AddressLines    []string
	Neighborhoods   []string
	Categories      []string
	City            string
	Licenses        []License
	Coordinates     Coordinates
}

type SearchOutput 

type SearchOutput struct {
	Language    string
	SearchURL   string
	RedirectURL string
	Items       []SearchItem
}

func ParseBodySearch 

func ParseBodySearch(body []byte) (SearchOutput, error)
func Search(index int, searchValue, location, language string, sv SortValue, proxyURL *url.URL) (SearchOutput, error)

type SortValue 

type SortValue string

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.