url

README

url

The url package is a simple url standardizer that parses a url into constituant parts and will set url.IP or url.IDNA flag when detected. Parse generally confirms the hostname has the basic elements required for domain or an IPv4/6 address.

Convenience helper functions are provided for simple boolean tests along with simple extractions or manipulations. A convenience Parser method reads off an io.Reader source and populates the passed in *url.URL and unique key generator for standardizing to 64-bit or 256-bit based a stated kind format requested.

The safebrowsing package contains the urls.go package extracted from google/safebrowsing for safebrowsing standardization and validation to help with malicious spoofing attemts. It does not include the hasher package.

    // convienence helper example
    var u url.URL
    u.Parse("example.com/path/logo.jpg")
    if url.HasPage(u) {
        url.NoPage(u)
    }

    // convienence parser example
	r, _ := os.Open("my/file")
	defer r.Close()
    var u url.URL
	next := url.Parser(r)
	for next(&u) {
		fmt.Println(u.Host)
	}
    
    // unique key generator example
    u.Parse("sub.example.com/path")
    FPHex64(u,url.Apex) // 2883ba7dc9aa3289
    FPHex64(u,url.Host) // 41e8219220802dab
    FPHex64(u,url.Full) // 3d173d4e8fd04260

    // safebrowsing stanadardization example that converts
    // alternative ipv4 format to ipv4 dot notation
    url, _ := safebrowsing.ParseURL("991234565/path/page")
	fmt.Println(url) // http://59.21.10.5/path/page

Note: It may be necessary to recompile or refresh the golang/x/net/publicsuffix package periodically to keep it current since it uses an intenally compressed reference set.

Documentation

Index

• Constants
• func EffectiveTLDPlusOne(u *URL) (string, error)
• func FPByte256(u *URL, kind int) (key []byte, ok bool)
• func FPHex256(u *URL, kind int) (key string, ok bool)
• func FPHex64(u *URL, kind int) (key string, ok bool)
• func FPMByte256(u *URL, kind int) (fp struct{ ... })
• func FPMHex256(u *URL, kind int) (fp struct{ ... })
• func FPMHex64(u *URL) (fp struct{ ... })
• func FPMUint64(u *URL) (fp struct{ ... })
• func FPUint64(u *URL, kind int) (key uint64, ok bool)
• func HasLabel(u *URL) bool
• func HasPage(u *URL) bool
• func HasPath(u *URL) bool
• func HasPort(u *URL) bool
• func HasWWW(u *URL) bool
• func IsPrivate(ip interface{}) (ok bool)
• func NoLabel(u *URL)
• func NoPage(u *URL)
• func NoPath(u *URL)
• func NoPort(u *URL)
• func NoWWW(u *URL)
• func Parser(r io.Reader) func(u *URL) bool
• func Segmentizer(u *URL) (segment struct{ ... })
• type URL
  • func (u *URL) NoIDNA()
  • func (u *URL) Parse(url string) bool
  • func (u *URL) String() string

Constants

const (
	Apex = iota
	Host
	Full
	FullNoPage
)

hash generator kind defination

Functions

func EffectiveTLDPlusOne

func EffectiveTLDPlusOne(u *URL) (string, error)

EffectiveTLDPlusOne is a wrapper around public suffix version that will convert the u.Host to the eTLD+1 version

func FPByte256

func FPByte256(u *URL, kind int) (key []byte, ok bool)

FPByte256 is a sha256 fingerpint generator utility

func FPHex256

func FPHex256(u *URL, kind int) (key string, ok bool)

FPHex256 is a sha256 fingerprint generator utility

func FPHex64

func FPHex64(u *URL, kind int) (key string, ok bool)

FPHex64 generates a key based on the kind request

func FPMByte256

func FPMByte256(u *URL, kind int) (fp struct{ Apex, Host, Full, FullNoPage []byte })

FPMByte256 is a multiform sha256 fingerprint generator utility

func FPMHex256

func FPMHex256(u *URL, kind int) (fp struct{ Apex, Host, Full, FullNoPage string })

FPMHex256 is a multiform sha256 fingerprint generator utility

func FPMHex64

func FPMHex64(u *URL) (fp struct{ Apex, Host, Full, FullNoPage string })

FPMHex64 is a multiform xxhash64 fingerprint generator utility

func FPMUint64

func FPMUint64(u *URL) (fp struct{ Apex, Host, Full, FullNoPage uint64 })

FPMUint64 is a multiform xxhash64 fingerprint generator utility

func FPUint64

func FPUint64(u *URL, kind int) (key uint64, ok bool)

FPUint64 generates a key based on the kind request

func HasLabel

func HasLabel(u *URL) bool

HasLabel reports if a www label is present

func HasPage

func HasPage(u *URL) bool

HasPage reports is a page is present in the parser

func HasPath

func HasPath(u *URL) bool

HasPath reports is a path is present in the parser

func HasPort

func HasPort(u *URL) bool

HasPort reports if a port is present in the parser

func HasWWW

func HasWWW(u *URL) bool

HasWWW reports if a www label is present

func IsPrivate

func IsPrivate(ip interface{}) (ok bool)

IsPrivate vefifies that an ipv4/6 representation is not in a reserved range; supports url.URL, net.IP, and string types

func NoLabel

func NoLabel(u *URL)

NoLabel removes any label from u.Host when present convering to apex

func NoPage

func NoPage(u *URL)

NoPage removes the page from the parser

func NoPath

func NoPath(u *URL)

NoPath removes the path from the parser

func NoPort

func NoPort(u *URL)

NoPort removes the port from the parser

func NoWWW

func NoWWW(u *URL)

NoWWW removes the www label from u.Host when present

func Parser

func Parser(r io.Reader) func(u *URL) bool

Parser reads io.Reader and parses into *url.URL

func Segmentizer

func Segmentizer(u *URL) (segment struct {
	Apex  string
	Label []string
})

Segmentizer parses the host into the apex and labels

Types

type URL

type URL struct {
	Host, Port, Path, Page string
	IP, IDNA               bool
	// contains filtered or unexported fields
}

URL parse and validate url with type detection flags for IP|IDNA

func (*URL) NoIDNA

func (u *URL) NoIDNA()

NoIDNA flag toggle, turn off INDA transcoding; default:on

func (*URL) Parse

func (u *URL) Parse(url string) bool

Parse the url into consitituate parts. Set u.IP flag if hostname is IPv4|6 and u.IDNA flag when domain converted

func (*URL) String

func (u *URL) String() string

String representation of the *URL components rebuit that will and always ensure no trailing slash is present when no path is present