util

package
v0.0.0-...-45e1a9a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 15, 2024 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source
const (
	Byte = 1 << (iota * 10)
	KiByte
	MiByte
	GiByte
	TiByte
	PiByte
	EiByte
)

IEC Sizes. kibis of bits

View Source
const (
	IByte = 1
	KByte = IByte * 1000
	MByte = KByte * 1000
	GByte = MByte * 1000
	TByte = GByte * 1000
	PByte = TByte * 1000
	EByte = PByte * 1000
)

SI Sizes.

View Source
const HttpStatusCancelled = 499
View Source
const (
	VolumeSizeLimitGB = 30
)

Variables

View Source
var (
	VERSION_NUMBER = fmt.Sprintf("%.02f", 3.69)
	VERSION        = sizeLimit + " " + VERSION_NUMBER
	COMMIT         = ""
)
View Source
var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
View Source
var RetryWaitTime = 6 * time.Second
View Source
var (
	Transport *http.Transport
)
View Source
var (
	UnsupportedCompression = fmt.Errorf("unsupported compression")
)

Functions

func Base64Encode

func Base64Encode(data []byte) string

func Base64Md5

func Base64Md5(data []byte) string

func Base64Md5ToBytes

func Base64Md5ToBytes(contentMd5 string) []byte

func BytesToHumanReadable

func BytesToHumanReadable(b uint64) string

BytesToHumanReadable returns the converted human readable representation of the bytes.

func BytesToUint16

func BytesToUint16(b []byte) (v uint16)

func BytesToUint32

func BytesToUint32(b []byte) (v uint32)

func BytesToUint64

func BytesToUint64(b []byte) (v uint64)

func CanonicalizeETag

func CanonicalizeETag(etag string) string

func CheckFile

func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)

func CloseRequest

func CloseRequest(req *http.Request)

func CloseResponse

func CloseResponse(resp *http.Response)

func DecompressData

func DecompressData(input []byte) ([]byte, error)

func Decrypt

func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)

func Delete

func Delete(url string, jwt string) error

func DeleteProxied

func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error)

func DetectedHostAddress

func DetectedHostAddress() string

func Do

func Do(req *http.Request) (resp *http.Response, err error)

func DownloadFile

func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error)

func EmptyTo

func EmptyTo(s, to string) string

EmptyTo returns to if s is empty.

func Encrypt

func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)

func FileExists

func FileExists(filename string) bool

func FileNameBase

func FileNameBase(filename string) string

func FolderExists

func FolderExists(folder string) bool

func Get

func Get(url string) ([]byte, bool, error)

github.com/gateway-dao/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go may need increasing http.Client.Timeout

func GetAuthenticated

func GetAuthenticated(url, jwt string) ([]byte, bool, error)

func GetBufferStream

func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error

func GetFileSize

func GetFileSize(file *os.File) (size int64, err error)

func GetFileUidGid

func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)

func GetUrlStream

func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error

func GunzipStream

func GunzipStream(w io.Writer, r io.Reader) (int64, error)

func GzipData

func GzipData(input []byte) ([]byte, error)

func GzipStream

func GzipStream(w io.Writer, r io.Reader) (int64, error)

func HashStringToLong

func HashStringToLong(dir string) (v int64)

returns a 64 bit big int

func HashToInt32

func HashToInt32(data []byte) (v int32)
func Head(url string) (http.Header, error)

func HumanReadableInts

func HumanReadableInts(ids ...int) string

HumanReadableInts joins a serials of inits into a smart one like 1-3 5 7-10 for human readable.

func HumanReadableIntsMax

func HumanReadableIntsMax(max int, ids ...int) string

HumanReadableIntsMax joins a serials of inits into a smart one like 1-3 5 ... for human readable.

func IfElse

func IfElse(b bool, this, that string) string

IfElse works like b ? this : that.

func IsCompressableFileType

func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)

func IsEmpty

func IsEmpty(given interface{}) bool

IsEmpty returns true if the given value has the zero value for its type.

func IsGzippedContent

func IsGzippedContent(data []byte) bool

func IsNotEmpty

func IsNotEmpty(given interface{}) bool

IsNotEmpty returns true if the given value is not zero or empty.

func Join

func Join(names ...string) string

func JoinHostPort

func JoinHostPort(host string, port int) string

func LoadConfiguration

func LoadConfiguration(configFileName string, required bool) (loaded bool)

func MaybeDecompressData

func MaybeDecompressData(input []byte) []byte

func MaybeGzipData

func MaybeGzipData(input []byte) []byte

func Md5

func Md5(data []byte) []byte

func Md5String

func Md5String(data []byte) string

func MultiRetry

func MultiRetry(name string, errList []string, job func() error) (err error)

func NewIpAndLocalListeners

func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipListener net.Listener, localListener net.Listener, err error)

func NewListener

func NewListener(addr string, timeout time.Duration) (ipListener net.Listener, err error)

func NormalizeUrl

func NormalizeUrl(url string) string

func Nvl

func Nvl(values ...string) string

Nvl return the first non-empty string

func ParseBytes

func ParseBytes(s string) (uint64, error)

ParseBytes parses a string representation of bytes into the number of bytes it represents.

See Also: Bytes, IBytes.

ParseBytes("42MB") -> 42000000, nil ParseBytes("42 MB") -> 42000000, nil ParseBytes("42 mib") -> 44040192, nil

func ParseFilerUrl

func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)

func ParseHostPort

func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)

func ParseInt

func ParseInt(text string, defaultValue int) int

func ParseUint64

func ParseUint64(text string, defaultValue uint64) uint64

func Post

func Post(url string, values url.Values) ([]byte, error)

func RandomBytes

func RandomBytes(byteCount int) []byte

func RandomInt32

func RandomInt32() int32

func ReadUrl

func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error)

func ReadUrlAsReaderCloser

func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*http.Response, io.ReadCloser, error)

func ReadUrlAsStream

func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error)

func ReadUrlAsStreamAuthenticated

func ReadUrlAsStreamAuthenticated(fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error)

func ResolvePath

func ResolvePath(path string) string

func RetriedFetchChunkData

func RetriedFetchChunkData(buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64) (n int, err error)

func Retry

func Retry(name string, job func() error) (err error)

func RetryUntil

func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool))

RetryUntil retries until the job returns no error or onErrFn returns false

func StringSplit

func StringSplit(separatedValues string, sep string) []string

func TestFolderWritable

func TestFolderWritable(folder string) (err error)

func ToShortFileName

func ToShortFileName(path string) string

func Uint16toBytes

func Uint16toBytes(b []byte, v uint16)

func Uint32toBytes

func Uint32toBytes(b []byte, v uint32)

func Uint64toBytes

func Uint64toBytes(b []byte, v uint64)

func Uint8toBytes

func Uint8toBytes(b []byte, v uint8)

func Version

func Version() string

func WriteFile

func WriteFile(name string, data []byte, perm os.FileMode) error

Copied from os.WriteFile(), adding file sync. see https://github.com/golang/go/issues/20599

Types

type ActiveLock

type ActiveLock struct {
	ID int64
	// contains filtered or unexported fields
}

type BytesReader

type BytesReader struct {
	Bytes []byte
	*bytes.Reader
}

func NewBytesReader

func NewBytesReader(b []byte) *BytesReader

type CipherKey

type CipherKey []byte

func GenCipherKey

func GenCipherKey() CipherKey

type ConcurrentReadMap

type ConcurrentReadMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

A mostly for read map, which can thread-safely initialize the map entries.

func NewConcurrentReadMap

func NewConcurrentReadMap() *ConcurrentReadMap

func (*ConcurrentReadMap) Delete

func (m *ConcurrentReadMap) Delete(key string)

func (*ConcurrentReadMap) Find

func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)

func (*ConcurrentReadMap) Get

func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}

func (*ConcurrentReadMap) Items

func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})

type Configuration

type Configuration interface {
	GetString(key string) string
	GetBool(key string) bool
	GetInt(key string) int
	GetStringSlice(key string) []string
	SetDefault(key string, value interface{})
}

type Conn

type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
	// contains filtered or unexported fields
}

Conn wraps a net.Conn, and sets a deadline for every read and write operation.

func (*Conn) Close

func (c *Conn) Close() error

func (*Conn) Read

func (c *Conn) Read(b []byte) (count int, e error)

func (*Conn) Write

func (c *Conn) Write(b []byte) (count int, e error)

type CountingReader

type CountingReader struct {
	BytesRead int
	// contains filtered or unexported fields
}

func (*CountingReader) Read

func (r *CountingReader) Read(p []byte) (n int, err error)

type DirectoryValueType

type DirectoryValueType string
var (
	ConfigurationFileDirectory DirectoryValueType
)

func (*DirectoryValueType) Set

func (s *DirectoryValueType) Set(value string) error

func (*DirectoryValueType) String

func (s *DirectoryValueType) String() string

type FullPath

type FullPath string

func JoinPath

func JoinPath(names ...string) FullPath

func NewFullPath

func NewFullPath(dir, name string) FullPath

func (FullPath) AsInode

func (fp FullPath) AsInode(unixTime int64) uint64

AsInode an in-memory only inode representation

func (FullPath) Child

func (fp FullPath) Child(name string) FullPath

func (FullPath) DirAndName

func (fp FullPath) DirAndName() (string, string)

func (FullPath) IsLongerFileName

func (fp FullPath) IsLongerFileName(maxFilenameLength uint32) bool

func (FullPath) IsUnder

func (fp FullPath) IsUnder(other FullPath) bool

func (FullPath) Name

func (fp FullPath) Name() string

func (FullPath) Split

func (fp FullPath) Split() []string

split, but skipping the root

type Future

type Future interface {
	Await() interface{}
}

type LimitedAsyncExecutor

type LimitedAsyncExecutor struct {
	// contains filtered or unexported fields
}

func NewLimitedAsyncExecutor

func NewLimitedAsyncExecutor(limit int) *LimitedAsyncExecutor

func (*LimitedAsyncExecutor) Execute

func (ae *LimitedAsyncExecutor) Execute(job func() interface{})

func (*LimitedAsyncExecutor) NextFuture

func (ae *LimitedAsyncExecutor) NextFuture() Future

type LimitedConcurrentExecutor

type LimitedConcurrentExecutor struct {
	// contains filtered or unexported fields
}

LimitedConcurrentExecutor object

func NewLimitedConcurrentExecutor

func NewLimitedConcurrentExecutor(limit int) *LimitedConcurrentExecutor

func (*LimitedConcurrentExecutor) Execute

func (c *LimitedConcurrentExecutor) Execute(job func())

Execute adds a function to the execution queue. if num of go routines allocated by this instance is < limit launch a new go routine to execute job else wait until a go routine becomes available

type Listener

type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

Listener wraps a net.Listener, and gives a place to store the timeout parameters. On Accept, it will wrap the net.Conn with our own Conn for us.

func (*Listener) Accept

func (l *Listener) Accept() (net.Conn, error)

type LockEntry

type LockEntry struct {
	// contains filtered or unexported fields
}

type LockTable

type LockTable[T comparable] struct {
	// contains filtered or unexported fields
}

LockTable is a table of locks that can be acquired. Locks are acquired in order of request.

func NewLockTable

func NewLockTable[T comparable]() *LockTable[T]

func (*LockTable[T]) AcquireLock

func (lt *LockTable[T]) AcquireLock(intention string, key T, lockType LockType) (lock *ActiveLock)

func (*LockTable[T]) NewActiveLock

func (lt *LockTable[T]) NewActiveLock(intention string, lockType LockType) *ActiveLock

func (*LockTable[T]) ReleaseLock

func (lt *LockTable[T]) ReleaseLock(key T, lock *ActiveLock)

type LockType

type LockType int
const (
	SharedLock LockType = iota
	ExclusiveLock
)

type MinFreeSpace

type MinFreeSpace struct {
	Type    MinFreeSpaceType
	Bytes   uint64
	Percent float32
	Raw     string
}

MinFreeSpace is type that defines the limit for the minimum free space.

func MustParseMinFreeSpace

func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace)

MustParseMinFreeSpace parses comma-separated argument for min free space setting. minFreeSpace has the high priority than minFreeSpacePercent if it is set.

func ParseMinFreeSpace

func ParseMinFreeSpace(s string) (*MinFreeSpace, error)

ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G

func (MinFreeSpace) IsLow

func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string)

IsLow tells whether the free space is low or not.

func (MinFreeSpace) String

func (s MinFreeSpace) String() string

String returns a string representation of MinFreeSpace.

type MinFreeSpaceType

type MinFreeSpaceType int

MinFreeSpaceType is the type of MinFreeSpace.

const (
	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
	AsPercent MinFreeSpaceType = iota
	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
	AsBytes
)

type Queue

type Queue[T any] struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewQueue

func NewQueue[T any]() *Queue[T]

func (*Queue[T]) Dequeue

func (q *Queue[T]) Dequeue() (result T)

func (*Queue[T]) Enqueue

func (q *Queue[T]) Enqueue(item T)

func (*Queue[T]) Len

func (q *Queue[T]) Len() int

type UnboundedQueue

type UnboundedQueue struct {
	// contains filtered or unexported fields
}

func NewUnboundedQueue

func NewUnboundedQueue() *UnboundedQueue

func (*UnboundedQueue) Consume

func (q *UnboundedQueue) Consume(fn func([]string))

func (*UnboundedQueue) EnQueue

func (q *UnboundedQueue) EnQueue(items ...string)

type ViperProxy

type ViperProxy struct {
	*viper.Viper
	sync.Mutex
}

func GetViper

func GetViper() *ViperProxy

func (*ViperProxy) GetBool

func (vp *ViperProxy) GetBool(key string) bool

func (*ViperProxy) GetInt

func (vp *ViperProxy) GetInt(key string) int

func (*ViperProxy) GetString

func (vp *ViperProxy) GetString(key string) string

func (*ViperProxy) GetStringSlice

func (vp *ViperProxy) GetStringSlice(key string) []string

func (*ViperProxy) SetDefault

func (vp *ViperProxy) SetDefault(key string, value interface{})

type WriteThrottler

type WriteThrottler struct {
	// contains filtered or unexported fields
}

func NewWriteThrottler

func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler

func (*WriteThrottler) MaybeSlowdown

func (wt *WriteThrottler) MaybeSlowdown(delta int64)

Directories

Path Synopsis
Package flag implements command-line flag parsing.
Package flag implements command-line flag parsing.
Package httpdown provides http.ConnState enabled graceful termination of http.Server.
Package httpdown provides http.ConnState enabled graceful termination of http.Server.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL