util

package
v0.0.0-...-45e1a9a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 15, 2024 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	Byte = 1 << (iota * 10)
	KiByte
	MiByte
	GiByte
	TiByte
	PiByte
	EiByte
)

IEC Sizes. kibis of bits

View Source 
const (
	IByte = 1
	KByte = IByte * 1000
	MByte = KByte * 1000
	GByte = MByte * 1000
	TByte = GByte * 1000
	PByte = TByte * 1000
	EByte = PByte * 1000
)

SI Sizes.

View Source 
const HttpStatusCancelled = 499
View Source 
const (
	VolumeSizeLimitGB = 30
)

Variables

View Source 
var (
	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.69)
	VERSION        = sizeLimit + " " + VERSION_NUMBER
	COMMIT         = ""
)
View Source 
var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
View Source 
var RetryWaitTime = 6 * time.Second
View Source 
var (
	Transport *http.Transport
)
View Source 
var (
	UnsupportedCompression = fmt.Errorf("unsupported compression")
)

Functions

func Base64Encode 

func Base64Encode(data []byte) string

func Base64Md5 

func Base64Md5(data []byte) string

func Base64Md5ToBytes 

func Base64Md5ToBytes(contentMd5 string) []byte

func BytesToHumanReadable 

func BytesToHumanReadable(b uint64) string

BytesToHumanReadable returns the converted human readable representation of the bytes.

func BytesToUint16 

func BytesToUint16(b []byte) (v uint16)

func BytesToUint32 

func BytesToUint32(b []byte) (v uint32)

func BytesToUint64 

func BytesToUint64(b []byte) (v uint64)

func CanonicalizeETag 

func CanonicalizeETag(etag string) string

func CheckFile 

func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)

func CloseRequest 

func CloseRequest(req *http.Request)

func CloseResponse 

func CloseResponse(resp *http.Response)

func DecompressData 

func DecompressData(input []byte) ([]byte, error)

func Decrypt 

func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)

func Delete 

func Delete(url string, jwt string) error

func DeleteProxied 

func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error)

func DetectedHostAddress 

func DetectedHostAddress() string

func Do 

func Do(req *http.Request) (resp *http.Response, err error)

func DownloadFile 

func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error)

func EmptyTo 

func EmptyTo(s, to string) string

EmptyTo returns to if s is empty.

func Encrypt 

func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)

func FileExists 

func FileExists(filename string) bool

func FileNameBase 

func FileNameBase(filename string) string

func FolderExists 

func FolderExists(folder string) bool

func Get 

func Get(url string) ([]byte, bool, error)

github.com/gateway-dao/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go may need increasing http.Client.Timeout

func GetAuthenticated 

func GetAuthenticated(url, jwt string) ([]byte, bool, error)

func GetBufferStream 

func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error

func GetFileSize 

func GetFileSize(file *os.File) (size int64, err error)

func GetFileUidGid 

func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)

func GetUrlStream 

func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error

func GunzipStream 

func GunzipStream(w io.Writer, r io.Reader) (int64, error)

func GzipData 

func GzipData(input []byte) ([]byte, error)

func GzipStream 

func GzipStream(w io.Writer, r io.Reader) (int64, error)

func HashStringToLong 

func HashStringToLong(dir string) (v int64)

returns a 64 bit big int

func HashToInt32 

func HashToInt32(data []byte) (v int32)
func Head(url string) (http.Header, error)

func HumanReadableInts 

func HumanReadableInts(ids ...int) string

HumanReadableInts joins a serials of inits into a smart one like 1-3 5 7-10 for human readable.

func HumanReadableIntsMax 

func HumanReadableIntsMax(max int, ids ...int) string

HumanReadableIntsMax joins a serials of inits into a smart one like 1-3 5 ... for human readable.

func IfElse 

func IfElse(b bool, this, that string) string

IfElse works like b ? this : that.

func IsCompressableFileType 

func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)

func IsEmpty 

func IsEmpty(given interface{}) bool

IsEmpty returns true if the given value has the zero value for its type.

func IsGzippedContent 

func IsGzippedContent(data []byte) bool

func IsNotEmpty 

func IsNotEmpty(given interface{}) bool

IsNotEmpty returns true if the given value is not zero or empty.

func Join 

func Join(names ...string) string

func JoinHostPort 

func JoinHostPort(host string, port int) string

func LoadConfiguration 

func LoadConfiguration(configFileName string, required bool) (loaded bool)

func MaybeDecompressData 

func MaybeDecompressData(input []byte) []byte

func MaybeGzipData 

func MaybeGzipData(input []byte) []byte

func Md5 

func Md5(data []byte) []byte

func Md5String 

func Md5String(data []byte) string

func MultiRetry 

func MultiRetry(name string, errList []string, job func() error) (err error)

func NewIpAndLocalListeners 

func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipListener net.Listener, localListener net.Listener, err error)

func NewListener 

func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error)

func NormalizeUrl 

func NormalizeUrl(url string) string

func Nvl 

func Nvl(values ...string) string

Nvl return the first non-empty string

func ParseBytes 

func ParseBytes(s string) (uint64, error)

ParseBytes parses a string representation of bytes into the number of bytes it represents.

See Also: Bytes, IBytes.

ParseBytes("42MB") -> 42000000, nil ParseBytes("42 MB") -> 42000000, nil ParseBytes("42 mib") -> 44040192, nil

func ParseFilerUrl 

func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)

func ParseHostPort 

func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)

func ParseInt 

func ParseInt(text string, defaultValue int) int

func ParseUint64 

func ParseUint64(text string, defaultValue uint64) uint64

func Post 

func Post(url string, values url.Values) ([]byte, error)

func RandomBytes 

func RandomBytes(byteCount int) []byte

func RandomInt32 

func RandomInt32() int32

func ReadUrl 

func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error)

func ReadUrlAsReaderCloser 

func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*http.Response, io.ReadCloser, error)

func ReadUrlAsStream 

func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error)

func ReadUrlAsStreamAuthenticated 

func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error)

func ResolvePath 

func ResolvePath(path string) string

func RetriedFetchChunkData 

func RetriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64) (n int, err error)

func Retry 

func Retry(name string, job func() error) (err error)

func RetryUntil 

func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool))

RetryUntil retries until the job returns no error or onErrFn returns false

func StringSplit 

func StringSplit(separatedValues string, sep string) []string

func TestFolderWritable 

func TestFolderWritable(folder string) (err error)

func ToShortFileName 

func ToShortFileName(path string) string

func Uint16toBytes 

func Uint16toBytes(b []byte, v uint16)

func Uint32toBytes 

func Uint32toBytes(b []byte, v uint32)

func Uint64toBytes 

func Uint64toBytes(b []byte, v uint64)

func Uint8toBytes 

func Uint8toBytes(b []byte, v uint8)

func Version 

func Version() string

func WriteFile 

func WriteFile(name string, data []byte, perm os.FileMode) error

Copied from os.WriteFile(), adding file sync. see https://github.com/golang/go/issues/20599

Types

type ActiveLock 

type ActiveLock struct {
	ID int64
	// contains filtered or unexported fields
}

type BytesReader 

type BytesReader struct {
	Bytes []byte
	*bytes.Reader
}

func NewBytesReader 

func NewBytesReader(b []byte) *BytesReader

type CipherKey 

type CipherKey []byte

func GenCipherKey 

func GenCipherKey() CipherKey

type ConcurrentReadMap 

type ConcurrentReadMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

A mostly for read map, which can thread-safely initialize the map entries.

func NewConcurrentReadMap 

func NewConcurrentReadMap() *ConcurrentReadMap

func (*ConcurrentReadMap) Delete 

func (m *ConcurrentReadMap) Delete(key string)

func (*ConcurrentReadMap) Find 

func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)

func (*ConcurrentReadMap) Get 

func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}

func (*ConcurrentReadMap) Items 

func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})

type Configuration 

type Configuration interface {
	GetString(key string) string
	GetBool(key string) bool
	GetInt(key string) int
	GetStringSlice(key string) []string
	SetDefault(key string, value interface{})
}

type Conn 

type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
	// contains filtered or unexported fields
}

Conn wraps a net.Conn, and sets a deadline for every read and write operation.

func (*Conn) Close 

func (c *Conn) Close() error

func (*Conn) Read 

func (c *Conn) Read(b []byte) (count int, e error)

func (*Conn) Write 

func (c *Conn) Write(b []byte) (count int, e error)

type CountingReader 

type CountingReader struct {
	BytesRead int
	// contains filtered or unexported fields
}

func (*CountingReader) Read 

func (r *CountingReader) Read(p []byte) (n int, err error)

type DirectoryValueType 

type DirectoryValueType string
var (
	ConfigurationFileDirectory DirectoryValueType
)

func (*DirectoryValueType) Set 

func (s *DirectoryValueType) Set(value string) error

func (*DirectoryValueType) String 

func (s *DirectoryValueType) String() string

type FullPath 

type FullPath string

func JoinPath 

func JoinPath(names ...string) FullPath

func NewFullPath 

func NewFullPath(dir, name string) FullPath

func (FullPath) AsInode 

func (fp FullPath) AsInode(unixTime int64) uint64

AsInode an in-memory only inode representation

func (FullPath) Child 

func (fp FullPath) Child(name string) FullPath

func (FullPath) DirAndName 

func (fp FullPath) DirAndName() (string, string)

func (FullPath) IsLongerFileName 

func (fp FullPath) IsLongerFileName(maxFilenameLength uint32) bool

func (FullPath) IsUnder 

func (fp FullPath) IsUnder(other FullPath) bool

func (FullPath) Name 

func (fp FullPath) Name() string

func (FullPath) Split 

func (fp FullPath) Split() []string

split, but skipping the root

type Future 

type Future interface {
	Await() interface{}
}

type LimitedAsyncExecutor 

type LimitedAsyncExecutor struct {
	// contains filtered or unexported fields
}

func NewLimitedAsyncExecutor 

func NewLimitedAsyncExecutor(limit int) *LimitedAsyncExecutor

func (*LimitedAsyncExecutor) Execute 

func (ae *LimitedAsyncExecutor) Execute(job func() interface{})

func (*LimitedAsyncExecutor) NextFuture 

func (ae *LimitedAsyncExecutor) NextFuture() Future

type LimitedConcurrentExecutor 

type LimitedConcurrentExecutor struct {
	// contains filtered or unexported fields
}

LimitedConcurrentExecutor object

func NewLimitedConcurrentExecutor 

func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor

func (*LimitedConcurrentExecutor) Execute 

func (c *LimitedConcurrentExecutor) Execute(job func())

Execute adds a function to the execution queue. if num of go routines allocated by this instance is < limit launch a new go routine to execute job else wait until a go routine becomes available

type Listener 

type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

Listener wraps a net.Listener, and gives a place to store the timeout parameters. On Accept, it will wrap the net.Conn with our own Conn for us.

func (*Listener) Accept 

func (l *Listener) Accept() (net.Conn, error)

type LockEntry 

type LockEntry struct {
	// contains filtered or unexported fields
}

type LockTable 

type LockTable[T comparable] struct {
	// contains filtered or unexported fields
}

LockTable is a table of locks that can be acquired. Locks are acquired in order of request.

func NewLockTable 

func NewLockTable[T comparable]() *LockTable[T]

func (*LockTable[T]) AcquireLock 

func (lt *LockTable[T]) AcquireLock(intention string, key T, lockType LockType) (lock *ActiveLock)

func (*LockTable[T]) NewActiveLock 

func (lt *LockTable[T]) NewActiveLock(intention string, lockType LockType) *ActiveLock

func (*LockTable[T]) ReleaseLock 

func (lt *LockTable[T]) ReleaseLock(key T, lock *ActiveLock)

type LockType 

type LockType int
const (
	SharedLock LockType = iota
	ExclusiveLock
)

type MinFreeSpace 

type MinFreeSpace struct {
	Type    MinFreeSpaceType
	Bytes   uint64
	Percent float32
	Raw     string
}

MinFreeSpace is type that defines the limit for the minimum free space.

func MustParseMinFreeSpace 

func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace)

MustParseMinFreeSpace parses comma-separated argument for min free space setting. minFreeSpace has the high priority than minFreeSpacePercent if it is set.

func ParseMinFreeSpace 

func ParseMinFreeSpace(s string) (*MinFreeSpace, error)

ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G

func (MinFreeSpace) IsLow 

func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string)

IsLow tells whether the free space is low or not.

func (MinFreeSpace) String 

func (s MinFreeSpace) String() string

String returns a string representation of MinFreeSpace.

type MinFreeSpaceType 

type MinFreeSpaceType int

MinFreeSpaceType is the type of MinFreeSpace. 

const (
	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
	AsPercent MinFreeSpaceType = iota
	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
	AsBytes
)

type Queue 

type Queue[T any] struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewQueue 

func NewQueue[T any]() *Queue[T]

func (*Queue[T]) Dequeue 

func (q *Queue[T]) Dequeue() (result T)

func (*Queue[T]) Enqueue 

func (q *Queue[T]) Enqueue(item T)

func (*Queue[T]) Len 

func (q *Queue[T]) Len() int

type UnboundedQueue 

type UnboundedQueue struct {
	// contains filtered or unexported fields
}

func NewUnboundedQueue 

func NewUnboundedQueue() *UnboundedQueue

func (*UnboundedQueue) Consume 

func (q *UnboundedQueue) Consume(fn func([]string))

func (*UnboundedQueue) EnQueue 

func (q *UnboundedQueue) EnQueue(items ...string)

type ViperProxy 

type ViperProxy struct {
	*viper.Viper
	sync.Mutex
}

func GetViper 

func GetViper() *ViperProxy

func (*ViperProxy) GetBool 

func (vp *ViperProxy) GetBool(key string) bool

func (*ViperProxy) GetInt 

func (vp *ViperProxy) GetInt(key string) int

func (*ViperProxy) GetString 

func (vp *ViperProxy) GetString(key string) string

func (*ViperProxy) GetStringSlice 

func (vp *ViperProxy) GetStringSlice(key string) []string

func (*ViperProxy) SetDefault 

func (vp *ViperProxy) SetDefault(key string, value interface{})

type WriteThrottler 

type WriteThrottler struct {
	// contains filtered or unexported fields
}

func NewWriteThrottler 

func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler

func (*WriteThrottler) MaybeSlowdown 

func (wt *WriteThrottler) MaybeSlowdown(delta int64)

Source Files

View all Source files

Directories

Path Synopsis
Package flag implements command-line flag parsing.
 Package flag implements command-line flag parsing.
Package httpdown provides http.ConnState enabled graceful termination of http.Server.
 Package httpdown provides http.ConnState enabled graceful termination of http.Server.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.