utils

package
v0.1.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 4, 2023 License: AGPL-3.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	KB = 1000
	MB = 1000 * KB
	GB = 1000 * MB
	TB = 1000 * GB
	PB = 1000 * TB

	KiB = 1024
	MiB = 1024 * KiB
	GiB = 1024 * MiB
	TiB = 1024 * GiB
	PiB = 1024 * TiB
)

See: http://en.wikipedia.org/wiki/Binary_prefix

Variables

View Source 
var (
	// from https://scrapfly.io/web-scraping-tools/ja3-fingerprint
	// must generate it without the "TLS Session has been resurected" warning
	CHROME_JA3 = "" /* 140-byte string literal not displayed */
	// 最新稳定版 Chrome (en-US) 在 Windows 11 x64 环境下访问网页的默认请求 headers
	CHROME_HTTP_REQUEST_HEADERS = map[string](string){
		"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",

		"Accept-Language":           "en-US,en;q=0.9",
		"Cache-Control":             "max-age=0",
		"Connection":                "keep-alive",
		"sec-ch-ua":                 `"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"`,
		"sec-ch-ua-mobile":          "?0",
		"sec-ch-ua-platform":        `"Windows"`,
		"Sec-Fetch-Dest":            "document",
		"Sec-Fetch-Mode":            "navigate",
		"Sec-Fetch-Site":            "none",
		"Sec-Fetch-User":            "?1",
		"Upgrade-Insecure-Requests": "1",
		"User-Agent":                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
	}
)

Functions

func AppendUrlQueryString added in v0.1.1 

func AppendUrlQueryString(url string, qs string) string

func AppendUrlQueryStringDelimiter added in v0.1.1 

func AppendUrlQueryStringDelimiter(url string) string

append a proper ? or & to url

func Assign 

func Assign(dst any, src any, excludeFieldIndexes []int)

https://stackoverflow.com/questions/23350173 copy none-empty field values from src to dst. dst and src must be pointors of same type of plain struct

func BytesSize 

func BytesSize(size float64) string

BytesSize returns a human-readable size in bytes, kibibytes, mebibytes, gibibytes, or tebibytes (eg. "44kiB", "17MiB").

func Capitalize 

func Capitalize(str string) string

func ContainsI 

func ContainsI(str string, substr string) bool

func CopyMap 

func CopyMap[T1 comparable, T2 any](m map[T1](T2)) map[T1](T2)

func CopySlice 

func CopySlice[T any](src []T) []T

func CustomSize 

func CustomSize(format string, size float64, base float64, _map []string) string

CustomSize returns a human-readable approximation of a size using custom format.

func DomHtml 

func DomHtml(el *goquery.Selection) string

func DomRemovedSpecialCharsText 

func DomRemovedSpecialCharsText(node *goquery.Selection) string

func DomRemovedSpecialCharsTextPreservingTime added in v0.1.5 

func DomRemovedSpecialCharsTextPreservingTime(node *goquery.Selection) string

func DomSanitizedText 

func DomSanitizedText(el *goquery.Selection) string

func DomSelectorText 

func DomSelectorText(el *goquery.Selection, selector string) (text string)

* DIY 了几个选择器语法（附加在标准CSS选择器字符串末尾） * @text 用于选择某个 Element 里的第一个 TEXT_NODE * @after 用于选择某个 Element 后面的 TEXT_NODE

func DomTime 

func DomTime(s *goquery.Selection, location *time.Location) int64

try to extract absoulte time from DOM

func ExtractSizeStr added in v0.1.2 

func ExtractSizeStr(str string) (int64, error)

func ExtractTime 

func ExtractTime(str string, location *time.Location) (time int64, offset int64)

offset: if > 0, indicates the bytes offset of the end of found time string in original str

func FetchJson 

func FetchJson(url string, v any, client *http.Client,
	cookie string, ua string, otherHeaders map[string](string)) error

func FetchUrl 

func FetchUrl(url string, client *http.Client,
	cookie string, ua string, otherHeaders map[string](string)) (*http.Response, http.Header, error)

func Filter 

func Filter[T any](ss []T, test func(T) bool) (ret []T)

func FilterNot added in v0.1.2 

func FilterNot[T any](ss []T, test func(T) bool) (ret []T)

func FindInSlice 

func FindInSlice[T any](slice []T, checker func(T) bool) *T

func FormatDate 

func FormatDate(ts int64) string

func FormatDate2 

func FormatDate2(ts int64) string

func FormatDuration 

func FormatDuration(seconds int64) (str string)

func FormatTime 

func FormatTime(ts int64) string

func FromHumanSize 

func FromHumanSize(size string) (int64, error)

FromHumanSize returns an integer from a human-readable specification of a size using SI standard (eg. "44kB", "17MB").

func GetDurationString 

func GetDurationString(seconds int64) string

func GetNewFilename added in v0.1.1 

func GetNewFilename(filename string) string

return a none-existing filename

func GetStructFieldValue added in v0.1.6 

func GetStructFieldValue(obj any, field string, defaultValue any) any

func GetUrlDoc 

func GetUrlDoc(url string, client *http.Client,
	cookie string, ua string, otherHeaders map[string](string)) (*goquery.Document, *http.Response, error)

func GetUrlDomain added in v0.1.1 

func GetUrlDomain(url string) string

return (top-level) domain of a url. eg. https://www.google.com/ => google.com

func GetWildcardFilenames added in v0.1.1 

func GetWildcardFilenames(filestr string) []string

"*.torrent" => ["a.torrent", "b.torrent"...]. Windows cmd / powershell 均不支持命令行 *.torrent 参数扩展。必须应用自己实现。做个简易版的

func HumanSize 

func HumanSize(size float64) string

HumanSize returns a human-readable approximation of a size capped at 4 valid numbers (eg. "2.746 MB", "796 KB").

func HumanSizeWithPrecision 

func HumanSizeWithPrecision(size float64, precision int) string

HumanSizeWithPrecision allows the size to be in any precision, instead of 4 digit precision used in units.HumanSize.

func IsIntString added in v0.1.5 

func IsIntString(str string) bool

func IsUrl 

func IsUrl(str string) bool

func Map 

func Map[T1 any, T2 any](ss []T1, mapper func(T1) T2) (ret []T2)

func MapMaxElementKey 

func MapMaxElementKey[TK comparable, TV constraints.Ordered](m map[TK](TV)) TK

func MatchUrlWithHostOrUrl added in v0.1.5 

func MatchUrlWithHostOrUrl(urlStr string, hostOrUrl string) bool

func Max 

func Max[T constraints.Ordered](args ...T) T

func Min 

func Min[T constraints.Ordered](args ...T) T

func Now 

func Now() int64

func ParseArgs added in v0.1.6 

func ParseArgs(s string) []string

Splitting a string at Space, except inside quotation marks. from https://stackoverflow.com/questions/47489745/splitting-a-string-at-space-except-inside-quotation-marks

func ParseDuration 

func ParseDuration(s string) (time.Duration, error)

ParseDuration parses a duration string. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h", "d", "w".

func ParseFilenameArgs added in v0.1.1 

func ParseFilenameArgs(args ...string) []string

func ParseFutureTime 

func ParseFutureTime(str string) (int64, error)

func ParseInt 

func ParseInt(str string) int64

func ParseLocalDateTime 

func ParseLocalDateTime(str string) (int64, error)

func ParseTime 

func ParseTime(str string, location *time.Location) (int64, error)

parse time. Treat duration time as pasted

func ParseTimeDuration 

func ParseTimeDuration(str string) (int64, error)

func ParseUrlHostname 

func ParseUrlHostname(urlStr string) string

func PostUrlForJson 

func PostUrlForJson(url string, data url.Values, v any, client *http.Client) error

func PrintStringInWidth 

func PrintStringInWidth(str string, width int64, padRight bool)

func QuoteFilename added in v0.1.6 

func QuoteFilename(str string) string

func RAMInBytes 

func RAMInBytes(size string) (int64, error)

RAMInBytes parses a human-readable string representing an amount of RAM in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and returns the number of bytes, or -1 if the string is unparseable. Units are case-insensitive, and the 'b' suffix is optional.

func ResolvePointerValue added in v0.1.6 

func ResolvePointerValue(obj any) any

func SanitizeText 

func SanitizeText(text string) string

func SelfDir 

func SelfDir() string

https://stackoverflow.com/questions/18537257/how-to-get-the-directory-of-the-currently-running-file

func SetHttpRequestBrowserHeaders 

func SetHttpRequestBrowserHeaders(req *http.Request, ua string)

func SetStructFieldValue added in v0.1.6 

func SetStructFieldValue(obj any, field string, value any)

https://stackoverflow.com/questions/6395076/using-reflect-how-do-you-set-the-value-of-a-struct-field

func Sha1 

func Sha1(s []byte) string

func Sha1String 

func Sha1String(s string) string

func Sleep 

func Sleep(seconds int64)

func String2Any added in v0.1.6 

func String2Any(value string) (any, reflect.Kind)

func StringPrefixInWidth added in v0.1.6 

func StringPrefixInWidth(str string, width int64) (string, int64)

return prefix of string at most width and actual width. ASCII char has 1 width. CJK char has 2 width

func UniqueSlice 

func UniqueSlice[T comparable](slice []T) []T

func UniqueSliceFn 

func UniqueSliceFn[TS any, TK comparable](slice []TS, keyFunc func(TS) TK) []TS

return de-duplicated slice that every member has unique key

Types

This section is empty.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.