canonicalizer

Documentation

Index

• Constants
• Variables
• func New(opts ...url.ParserOption) url.Parser
• func WithDefaultScheme(scheme string) url.ParserOption
• func WithRemoveFragment() url.ParserOption
• func WithRemovePort() url.ParserOption
• func WithRemoveUserInfo() url.ParserOption
• func WithRepeatedPercentDecoding() url.ParserOption
• func WithSortQuery(sortType querySort) url.ParserOption

Examples

• New

Constants

const (
	// Do not sort query.
	NoSort querySort = iota
	// Stable sort on query parameter keys.
	SortKeys
	// Sort on entire query parameter.
	SortParameter
)

Variables

var GoogleSafeBrowsing = New(
	url.WithLaxHostParsing(),
	url.WithQueryPercentEncodeSet(LaxQueryPercentEncodeSet),
	url.WithCollapseConsecutiveSlashes(),
	url.WithAcceptInvalidCodepoints(),
	url.WithPercentEncodeSinglePercentSign(),
	url.WithPreParseHostFunc(func(u *url.Url, host string) string {
		host = strings.Trim(host, ".")
		var re = regexp.MustCompile(`\.\.+`)
		host = re.ReplaceAllString(host, ".")
		return host
	}),
	url.WithSkipEqualsForEmptySearchParamsValue(),
	WithRemovePort(),
	WithRemoveFragment(),
	WithRepeatedPercentDecoding(),
	WithDefaultScheme("http"),
)

GoogleSafeBrowsing is a profile that follows the canonicalization rules used by Google Safe Browsing.

var LaxPathPercentEncodeSet = url.PathPercentEncodeSet.Clear(0x2E, 0x3C, 0x3E)

var LaxQueryPercentEncodeSet = url.QueryPercentEncodeSet.Clear(0x22, 0x25, 0x2F, 0x3B, 0x3F, 0x7B)

var RepeatedQueryPercentDecodeSet = url.C0OrSpacePercentEncodeSet.Set('#', '%', '&', '=')

var Semantic = New(
	url.WithLaxHostParsing(),
	url.WithPathPercentEncodeSet(LaxPathPercentEncodeSet),
	url.WithQueryPercentEncodeSet(LaxQueryPercentEncodeSet),
	url.WithCollapseConsecutiveSlashes(),
	url.WithAcceptInvalidCodepoints(),
	url.WithPercentEncodeSinglePercentSign(),
	url.WithAllowSettingPathForNonBaseUrl(),
	url.WithEncodingOverride(charmap.ISO8859_1),
	url.WithPreParseHostFunc(func(u *url.Url, host string) string {
		if host != "" {
			host = strings.Trim(host, ".")
			var re = regexp.MustCompile(`\.\.+`)
			host = re.ReplaceAllString(host, ".")
			if host == "" {
				host = "0.0.0.0"
			}
		}
		return host
	}),
	url.WithSpecialSchemes(map[string]string{
		"ftp":    "21",
		"file":   "",
		"http":   "80",
		"https":  "443",
		"ws":     "80",
		"wss":    "443",
		"gopher": "70",
	}),
	WithRemoveUserInfo(),
	WithDefaultScheme("http"),
	WithSortQuery(SortKeys),
	WithRepeatedPercentDecoding(),
	WithRemoveFragment(),
)

var WhatWg = New()

WhatWg is a profile that follows the canonicalization rules used by WHATWG.

var WhatWgSortQuery = New(
	WithSortQuery(SortKeys),
)

WhatWgSortQuery is a profile that follows the canonicalization rules used by WHATWG, but sorts query parameters.

Functions

func New

func New(opts ...url.ParserOption) url.Parser

Example

package main

import (
	"fmt"
	"github.com/nlnwa/whatwg-url/canonicalizer"
)

func main() {
	c := canonicalizer.New(canonicalizer.WithRemoveUserInfo(), canonicalizer.WithRemoveFragment())
	u, err := c.Parse("http://user@example.com/a//d?b#c")
	if err == nil {
		fmt.Println(u)
	}
}

Output:

http://example.com/a//d?b

func WithDefaultScheme

func WithDefaultScheme(scheme string) url.ParserOption

WithDefaultScheme sets a scheme to add if url is missing scheme.

This API is EXPERIMENTAL.

func WithRemoveFragment

func WithRemoveFragment() url.ParserOption

WithRemoveFragment removes the fragment part of the url.

This API is EXPERIMENTAL.

func WithRemovePort

func WithRemovePort() url.ParserOption

WithRemovePort always removes port from url. Default is to remove port if port is default port for scheme.

This API is EXPERIMENTAL.

func WithRemoveUserInfo

func WithRemoveUserInfo() url.ParserOption

WithRemoveUserInfo removes username and password from url.

This API is EXPERIMENTAL.

func WithRepeatedPercentDecoding

func WithRepeatedPercentDecoding() url.ParserOption

WithRepeatedPercentDecoding.

This API is EXPERIMENTAL.

func WithSortQuery

func WithSortQuery(sortType querySort) url.ParserOption

WithSortQuery sets sort type for query parameters. if query should be sorted: 0 = no sort, 1 = sort keys, but leave repeated keys in same order, 2 = sort key,value

This API is EXPERIMENTAL.