Versions in this module Expand all Collapse all v1 v1.0.0 Jan 4, 2024 Changes in this version + var WhitespaceChars = []rune + type Group struct + Agent string + CrawlDelay time.Duration + Rules []*Rule + func (g *Group) Test(path string) bool + type ParseError struct + Errs []error + func (e ParseError) Error() string + type RobotsData struct + AllowAll bool + DisallowAll bool + Groups map[string]*Group + Host string + Sitemaps []string + func FromBytes(body []byte) (r *RobotsData, err error) + func FromResponse(res *http.Response) (*RobotsData, error) + func FromStatusAndBytes(statusCode int, body []byte) (*RobotsData, error) + func FromStatusAndString(statusCode int, body string) (*RobotsData, error) + func FromString(body string) (r *RobotsData, err error) + func (r *RobotsData) FindGroup(agent string) (ret *Group) + func (r *RobotsData) TestAgent(path, agent string) bool + type Rule struct + Allow bool + Path string + Pattern *regexp.Regexp