googlesc

Documentation

Index

• Constants
• type Doc
  • func ProcessData(data []byte) (doc Doc, err error)
• type Googlesc
  • func New() Googlesc
  • func (gs *Googlesc) Fetch() (doc Doc, err error)
  • func (gs *Googlesc) FetchData() (data []byte, headers http.Header, status int, err error)
• type IPv4Entry
• type IPv6Entry
• type RawDoc
• type RawIPv4Entry
• type RawIPv6Entry

Constants

const (
	ShortName   = "googlesc"
	FullName    = "Google Special Crawlers"
	HostType    = "crawlers"
	SourceURL   = "https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot"
	DownloadURL = "https://developers.google.com/static/search/apis/ipranges/special-crawlers.json"
)

Types

type Doc

type Doc struct {
	CreationTime time.Time
	IPv4Prefixes []IPv4Entry
	IPv6Prefixes []IPv6Entry
}

func ProcessData

func ProcessData(data []byte) (doc Doc, err error)

type Googlesc

type Googlesc struct {
	Client      *retryablehttp.Client
	DownloadURL string
}

func New

func New() Googlesc

func (*Googlesc) Fetch

func (gs *Googlesc) Fetch() (doc Doc, err error)

func (*Googlesc) FetchData

func (gs *Googlesc) FetchData() (data []byte, headers http.Header, status int, err error)

type IPv4Entry

type IPv4Entry struct {
	IPv4Prefix netip.Prefix `json:"ipv4Prefix"`
}

type IPv6Entry

type IPv6Entry struct {
	IPv6Prefix netip.Prefix `json:"ipv6Prefix"`
}

type RawDoc

type RawDoc struct {
	CreationTime  string `json:"creationTime"`
	LastRequested time.Time
	Entries       []json.RawMessage `json:"prefixes"`
}

type RawIPv4Entry

type RawIPv4Entry struct {
	IPv4Prefix string `json:"ipv4Prefix"`
}

type RawIPv6Entry

type RawIPv6Entry struct {
	IPv6Prefix string `json:"ipv6Prefix"`
}