util

Documentation

Index

• Constants
• Variables
• func Base64Encode(data []byte) string
• func Base64Md5(data []byte) string
• func Base64Md5ToBytes(contentMd5 string) []byte
• func BytesToHumanReadable(b uint64) string
• func BytesToUint16(b []byte) (v uint16)
• func BytesToUint32(b []byte) (v uint32)
• func BytesToUint64(b []byte) (v uint64)
• func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)
• func CloseResponse(resp *http.Response)
• func DecompressData(input []byte) ([]byte, error)
• func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)
• func Delete(url string, jwt string) error
• func DetectedHostAddress() string
• func Do(req *http.Request) (resp *http.Response, err error)
• func DownloadFile(fileUrl string) (filename string, header http.Header, resp *http.Response, e error)
• func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)
• func FileExists(filename string) bool
• func Get(url string) ([]byte, error)
• func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error
• func GetFileSize(file *os.File) (size int64, err error)
• func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)
• func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error
• func GetViper() *viper.Viper
• func GzipData(input []byte) ([]byte, error)
• func HashStringToLong(dir string) (v int64)
• func HashToInt32(data []byte) (v int32)
• func Head(url string) (http.Header, error)
• func HumanReadableInts(ids ...int) string
• func HumanReadableIntsMax(max int, ids ...int) string
• func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)
• func IsGzippedContent(data []byte) bool
• func IsZstdContent(data []byte) bool
• func Join(names ...string) string
• func LoadConfiguration(configFileName string, required bool) (loaded bool)
• func MaybeDecompressData(input []byte) []byte
• func MaybeGzipData(input []byte) []byte
• func Md5(data []byte) []byte
• func Md5String(data []byte) string
• func NewListener(addr string, timeout time.Duration) (net.Listener, error)
• func NormalizeUrl(url string) string
• func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)
• func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)
• func ParseInt(text string, defaultValue int) int
• func ParseUint64(text string, defaultValue uint64) uint64
• func Post(url string, values url.Values) ([]byte, error)
• func PostBytes(url string, body []byte) ([]byte, error)
• func RandomInt32() int32
• func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, ...) (int64, error)
• func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error)
• func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, ...) error
• func ResolvePath(path string) string
• func TestFolderWritable(folder string) (err error)
• func Uint16toBytes(b []byte, v uint16)
• func Uint32toBytes(b []byte, v uint32)
• func Uint64toBytes(b []byte, v uint64)
• func Uint8toBytes(b []byte, v uint8)
• func Version() string
• func ZstdData(input []byte) ([]byte, error)
• type CipherKey
  • func GenCipherKey() CipherKey
• type ConcurrentReadMap
  • func NewConcurrentReadMap() *ConcurrentReadMap
  • func (m *ConcurrentReadMap) Delete(key string)
  • func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)
  • func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}
  • func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})
• type Configuration
• type Conn
  • func (c *Conn) Close() error
  • func (c *Conn) Read(b []byte) (count int, e error)
  • func (c *Conn) Write(b []byte) (count int, e error)
• type FullPath
  • func JoinPath(names ...string) FullPath
  • func NewFullPath(dir, name string) FullPath
  • func (fp FullPath) AsInode() uint64
  • func (fp FullPath) Child(name string) FullPath
  • func (fp FullPath) DirAndName() (string, string)
  • func (fp FullPath) Name() string
  • func (fp FullPath) Split() []string
• type Listener
  • func (l *Listener) Accept() (net.Conn, error)
• type Queue
  • func NewQueue() *Queue
  • func (q *Queue) Dequeue() interface{}
  • func (q *Queue) Enqueue(item interface{})
  • func (q *Queue) Len() int
• type UnboundedQueue
  • func NewUnboundedQueue() *UnboundedQueue
  • func (q *UnboundedQueue) Consume(fn func([]string))
  • func (q *UnboundedQueue) EnQueue(items ...string)
• type WriteThrottler
  • func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler
  • func (wt *WriteThrottler) MaybeSlowdown(delta int64)

Constants

const (
	VolumeSizeLimitGB = 30
)

Variables

var (
	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 2, 00)
	COMMIT  = ""
)

var (
	Transport *http.Transport
)

var (
	UnsupportedCompression = fmt.Errorf("unsupported compression")
)

Functions

func Base64Encode

func Base64Encode(data []byte) string

func Base64Md5

func Base64Md5(data []byte) string

func Base64Md5ToBytes

func Base64Md5ToBytes(contentMd5 string) []byte

func BytesToHumanReadable

func BytesToHumanReadable(b uint64) string

BytesToHumanReadable returns the converted human readable representation of the bytes.

func BytesToUint16

func BytesToUint16(b []byte) (v uint16)

func BytesToUint32

func BytesToUint32(b []byte) (v uint32)

func BytesToUint64

func BytesToUint64(b []byte) (v uint64)

func CheckFile

func CheckFile(filename string) (exists, canRead, canWrite bool, modTime time.Time, fileSize int64)

func CloseResponse

func CloseResponse(resp *http.Response)

func DecompressData

func DecompressData(input []byte) ([]byte, error)

func Decrypt

func Decrypt(ciphertext []byte, key CipherKey) ([]byte, error)

func Delete

func Delete(url string, jwt string) error

func DetectedHostAddress

func DetectedHostAddress() string

func Do

func Do(req *http.Request) (resp *http.Response, err error)

func DownloadFile

func DownloadFile(fileUrl string) (filename string, header http.Header, resp *http.Response, e error)

func Encrypt

func Encrypt(plaintext []byte, key CipherKey) ([]byte, error)

func FileExists

func FileExists(filename string) bool

func Get

func Get(url string) ([]byte, error)

github.com/bary321/seaweedfs-1/unmaintained/repeated_vacuum/repeated_vacuum.go may need increasing http.Client.Timeout

func GetBufferStream

func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error

func GetFileSize

func GetFileSize(file *os.File) (size int64, err error)

func GetFileUidGid

func GetFileUidGid(fi os.FileInfo) (uid, gid uint32)

func GetUrlStream

func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error

func GetViper

func GetViper() *viper.Viper

func GzipData

func GzipData(input []byte) ([]byte, error)

func HashStringToLong

func HashStringToLong(dir string) (v int64)

returns a 64 bit big int

func HashToInt32

func HashToInt32(data []byte) (v int32)

func Head

func Head(url string) (http.Header, error)

func HumanReadableInts

func HumanReadableInts(ids ...int) string

HumanReadableInts joins a serials of inits into a smart one like 1-3 5 7-10 for human readable.

func HumanReadableIntsMax

func HumanReadableIntsMax(max int, ids ...int) string

HumanReadableIntsMax joins a serials of inits into a smart one like 1-3 5 ... for human readable.

func IsCompressableFileType

func IsCompressableFileType(ext, mtype string) (shouldBeCompressed, iAmSure bool)

func IsGzippedContent

func IsGzippedContent(data []byte) bool

func IsZstdContent

func IsZstdContent(data []byte) bool

func Join

func Join(names ...string) string

func LoadConfiguration

func LoadConfiguration(configFileName string, required bool) (loaded bool)

func MaybeDecompressData

func MaybeDecompressData(input []byte) []byte

func MaybeGzipData

func MaybeGzipData(input []byte) []byte

func Md5

func Md5(data []byte) []byte

func Md5String

func Md5String(data []byte) string

func NewListener

func NewListener(addr string, timeout time.Duration) (net.Listener, error)

func NormalizeUrl

func NormalizeUrl(url string) string

func ParseFilerUrl

func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error)

func ParseHostPort

func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error)

func ParseInt

func ParseInt(text string, defaultValue int) int

func ParseUint64

func ParseUint64(text string, defaultValue uint64) uint64

func Post

func Post(url string, values url.Values) ([]byte, error)

func PostBytes

func PostBytes(url string, body []byte) ([]byte, error)

func RandomInt32

func RandomInt32() int32

func ReadUrl

func ReadUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error)

func ReadUrlAsReaderCloser

func ReadUrlAsReaderCloser(fileUrl string, rangeHeader string) (io.ReadCloser, error)

func ReadUrlAsStream

func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) error

func ResolvePath

func ResolvePath(path string) string

func TestFolderWritable

func TestFolderWritable(folder string) (err error)

func Uint16toBytes

func Uint16toBytes(b []byte, v uint16)

func Uint32toBytes

func Uint32toBytes(b []byte, v uint32)

func Uint64toBytes

func Uint64toBytes(b []byte, v uint64)

func Uint8toBytes

func Uint8toBytes(b []byte, v uint8)

func Version

func Version() string

func ZstdData

func ZstdData(input []byte) ([]byte, error)

Types

type CipherKey

type CipherKey []byte

func GenCipherKey

func GenCipherKey() CipherKey

type ConcurrentReadMap

type ConcurrentReadMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

A mostly for read map, which can thread-safely initialize the map entries.

func NewConcurrentReadMap

func NewConcurrentReadMap() *ConcurrentReadMap

func (*ConcurrentReadMap) Delete

func (m *ConcurrentReadMap) Delete(key string)

func (*ConcurrentReadMap) Find

func (m *ConcurrentReadMap) Find(key string) (interface{}, bool)

func (*ConcurrentReadMap) Get

func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{}

func (*ConcurrentReadMap) Items

func (m *ConcurrentReadMap) Items() (itemsCopy []interface{})

type Configuration

type Configuration interface {
	GetString(key string) string
	GetBool(key string) bool
	GetInt(key string) int
	GetStringSlice(key string) []string
	SetDefault(key string, value interface{})
}

type Conn

type Conn struct {
	net.Conn
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
	// contains filtered or unexported fields
}

Conn wraps a net.Conn, and sets a deadline for every read and write operation.

func (*Conn) Close

func (c *Conn) Close() error

func (*Conn) Read

func (c *Conn) Read(b []byte) (count int, e error)

func (*Conn) Write

func (c *Conn) Write(b []byte) (count int, e error)

type FullPath

type FullPath string

func JoinPath

func JoinPath(names ...string) FullPath

func NewFullPath

func NewFullPath(dir, name string) FullPath

func (FullPath) AsInode

func (fp FullPath) AsInode() uint64

func (FullPath) Child

func (fp FullPath) Child(name string) FullPath

func (FullPath) DirAndName

func (fp FullPath) DirAndName() (string, string)

func (FullPath) Name

func (fp FullPath) Name() string

func (FullPath) Split

func (fp FullPath) Split() []string

split, but skipping the root

type Listener

type Listener struct {
	net.Listener
	ReadTimeout  time.Duration
	WriteTimeout time.Duration
}

Listener wraps a net.Listener, and gives a place to store the timeout parameters. On Accept, it will wrap the net.Conn with our own Conn for us.

func (*Listener) Accept

func (l *Listener) Accept() (net.Conn, error)

type Queue

type Queue struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewQueue

func NewQueue() *Queue

func (*Queue) Dequeue

func (q *Queue) Dequeue() interface{}

func (*Queue) Enqueue

func (q *Queue) Enqueue(item interface{})

func (*Queue) Len

func (q *Queue) Len() int

type UnboundedQueue

type UnboundedQueue struct {
	// contains filtered or unexported fields
}

func NewUnboundedQueue

func NewUnboundedQueue() *UnboundedQueue

func (*UnboundedQueue) Consume

func (q *UnboundedQueue) Consume(fn func([]string))

func (*UnboundedQueue) EnQueue

func (q *UnboundedQueue) EnQueue(items ...string)

type WriteThrottler

type WriteThrottler struct {
	// contains filtered or unexported fields
}

func NewWriteThrottler

func NewWriteThrottler(bytesPerSecond int64) *WriteThrottler

func (*WriteThrottler) MaybeSlowdown

func (wt *WriteThrottler) MaybeSlowdown(delta int64)