filesystem

package
v1.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 15, 2022 License: Apache-2.0 Imports: 25 Imported by: 3

Documentation

Overview

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

  • Copyright (C) 2020-2022 Arm Limited or its affiliates and Contributors. All rights reserved.

  • SPDX-License-Identifier: Apache-2.0

Distributed lock using lock files https://fileinfo.com/extension/lock

Index

Constants

View Source 
const (
	StandardFS int = iota
	InMemoryFS
)
View Source 
const LockFilePrefix = "lockfile"
View Source 
const (
	UnsetFileHandle = ^uint64(0)
)

Variables

View Source 
var (
	ErrLinkNotImplemented  = fmt.Errorf("link not implemented: %w", commonerrors.ErrNotImplemented)
	ErrChownNotImplemented = fmt.Errorf("chown not implemented: %w", commonerrors.ErrNotImplemented)
	ErrPathNotExist        = errors.New("readdirent: no such file or directory")
)
View Source 
var (
	FileSystemTypes = []int{StandardFS, InMemoryFS}
)

Functions

func CleanDir 

func CleanDir(dir string) (err error)

Removes all the files in a directory (equivalent rm -rf .../*)

func ConvertFileSytemError 

func ConvertFileSytemError(err error) error

Converts file system error into common errors

func Copy 

func Copy(src string, dest string) (err error)

func CopyBetweenFS 

func CopyBetweenFS(srcFs FS, src string, destFs FS, dest string) (err error)

func CurrentDirectory 

func CurrentDirectory() (string, error)

func ExcludeAll 

func ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error)

Excludes files

func Exists 

func Exists(path string) bool

Checks if a file or folder exists

func FindAll 

func FindAll(dir string, extensions ...string) (files []string, err error)

Finds all the files with extensions

func GetFileSize 

func GetFileSize(name string) (size int64, err error)

func GetType 

func GetType() int

func IdentityPathConverterFunc 

func IdentityPathConverterFunc(path string) string

func IsDir 

func IsDir(path string) (result bool, err error)

States whether it is a directory or not

func IsDirectory 

func IsDirectory(fi os.FileInfo) bool

func IsEmpty 

func IsEmpty(name string) (empty bool, err error)

Checks whether a path is empty or not

func IsFile 

func IsFile(path string) (result bool, err error)

States whether it is a file or not

func IsFileHandleUnset 

func IsFileHandleUnset(fh uintptr) bool

func IsPathNotExist 

func IsPathNotExist(err error) bool

func IsRegularFile 

func IsRegularFile(fi os.FileInfo) bool
func IsSymLink(fi os.FileInfo) bool

func ListDirTree 

func ListDirTree(dirPath string, list *[]string) error

func Ls 

func Ls(dir string) (files []string, err error)

func Lstat 

func Lstat(name string) (fileInfo os.FileInfo, err error)

func MkDir 

func MkDir(dir string) (err error)

Makes directory (equivalent to mkdir -p)

func Move 

func Move(src string, dest string) (err error)

Moves a file (equivalent to mv)

func MoveBetweenFS 

func MoveBetweenFS(srcFs FS, src string, destFs FS, dest string) (err error)

func NewExtendedOsFs 

func NewExtendedOsFs() afero.Fs

func PathSeparator 

func PathSeparator() rune

func ReadFile 

func ReadFile(name string) ([]byte, error)

func Rm 

func Rm(dir string) (err error)

Removes directory (equivalent to rm -r)

func Stat 

func Stat(name string) (os.FileInfo, error)

func SubDirectories 

func SubDirectories(directory string) ([]string, error)

func TempDir 

func TempDir(dir string, prefix string) (name string, err error)

func TempDirInTempDir 

func TempDirInTempDir(prefix string) (name string, err error)

func TempDirectory 

func TempDirectory() string

func Unzip 

func Unzip(source string, destination string) ([]string, error)

func Zip 

func Zip(source string, destination string) error

Types

type Chowner 

type Chowner interface {
	ChownIfPossible(string, int, int) error
}

Optional interface. It is only implemented by the filesystems saying so.

type DiskUsage 

type DiskUsage interface {
	GetTotal() uint64
	GetFree() uint64
	GetUsed() uint64
	GetUsedPercent() float64
	GetInodesTotal() uint64
	GetInodesUsed() uint64
	GetInodesFree() uint64
	GetInodesUsedPercent() float64
}

type ExtendedOsFs 

type ExtendedOsFs struct {
	afero.OsFs
}

func (*ExtendedOsFs) ChownIfPossible 

func (c *ExtendedOsFs) ChownIfPossible(name string, uid int, gid int) error

func (*ExtendedOsFs) LinkIfPossible 

func (c *ExtendedOsFs) LinkIfPossible(oldname, newname string) (err error)

type FS 

type FS interface {
	//The following is for being able to use doublestar
	Open(name string) (doublestar.File, error)
	// Open a file for reading. It opens the named file with specified flag (O_RDONLY etc.).
	// See os.Open()
	GenericOpen(name string) (File, error)
	// OpenFile opens a file using the given flags and the given mode.
	// OpenFile is the generalized open call
	// most users will use GenericOpen or Create instead.
	// See os.OpenFile
	OpenFile(name string, flag int, perm os.FileMode) (File, error)
	// Creates a file.
	CreateFile(name string) (File, error)
	// Gets the path separator character.
	PathSeparator() rune
	Stat(name string) (os.FileInfo, error)
	Lstat(name string) (os.FileInfo, error)
	// Gets file time information.
	StatTimes(name string) (FileTimeInfo, error)
	// Gets the type of the file system.
	GetType() int
	// Removes all the files in a directory (equivalent rm -rf .../*)
	CleanDir(dir string) (err error)
	// Checks if a file or folder exists
	Exists(path string) bool
	// Removes directory (equivalent to rm -r)
	Rm(dir string) (err error)
	// States whether it is a file or not
	IsFile(path string) (result bool, err error)
	// States whether it is a directory or not
	IsDir(path string) (result bool, err error)
	// States whether it is a link or not
	IsLink(path string) (result bool, err error)
	// Checks whether a path is empty or not
	IsEmpty(name string) (empty bool, err error)
	// Makes directory (equivalent to mkdir -p)
	MkDir(dir string) (err error)
	// Makes directory (equivalent to mkdir -p)
	MkDirAll(dir string, perm os.FileMode) (err error)
	// Excludes files from a list. Returns the list without the path matching the exclusion patterns.
	ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error)
	// Finds all the files with extensions
	FindAll(dir string, extensions ...string) (files []string, err error)
	// Walks  the file tree rooted at root, calling fn for each file or
	// directory in the tree, including root. See https://golang.org/pkg/path/filepath/#WalkDir
	Walk(root string, fn filepath.WalkFunc) error
	// Lists all files and directory (equivalent to ls)
	Ls(dir string) (files []string, err error)
	LsFromOpenedDirectory(dir File) (files []string, err error)
	// Lists all files and directory (equivalent to ls -l)
	Lls(dir string) (files []os.FileInfo, err error)
	LlsFromOpenedDirectory(dir File) (files []os.FileInfo, err error)
	// Copy files and directory (equivalent to cp -r)
	Copy(src string, dest string) (err error)
	// Moves a file (equivalent to mv)
	Move(src string, dest string) (err error)
	// Creates a temp directory
	TempDir(dir string, prefix string) (name string, err error)
	// Creates a temp directory in temp directory.
	TempDirInTempDir(prefix string) (name string, err error)
	// Creates a temp file
	TempFile(dir string, pattern string) (f File, err error)
	// Creates a temp file in temp directory.
	TempFileInTempDir(pattern string) (f File, err error)
	// Gets temp directory.
	TempDirectory() string
	// Gets current directory.
	CurrentDirectory() (string, error)
	// Reads a file and return its content.
	ReadFile(filename string) ([]byte, error)
	// Writes data to a file named by filename.
	// If the file does not exist, WriteFile creates it with permissions perm;
	// otherwise WriteFile truncates it before writing.
	WriteFile(filename string, data []byte, perm os.FileMode) error
	// Runs the Garbage collector on the filesystem (removes any file which has not been accessed for a certain duration)
	GarbageCollect(root string, durationSinceLastAccess time.Duration) error
	// Changes the mode of the named file to mode.
	Chmod(name string, mode os.FileMode) error
	// Changes the access and modification times of the named file
	Chtimes(name string, atime time.Time, mtime time.Time) error
	// Changes the numeric uid and gid of the named file.
	Chown(name string, uid, gid int) error
	// Creates newname as a hard link to the oldname file
	Link(oldname, newname string) error
	// Returns the destination of the named symbolic link.
	Readlink(name string) (string, error)
	// Creates newname as a symbolic link to oldname.
	Symlink(oldname string, newname string) error
	// Determines Disk usage
	DiskUsage(name string) (DiskUsage, error)
	// Gets file size
	GetFileSize(filename string) (int64, error)
	// Returns a list of all subdirectories (which are not hidden)
	SubDirectories(directory string) ([]string, error)
	// Lists the content of directory recursively
	ListDirTree(dirPath string, list *[]string) error
	// Gets FS file path instead of real file path. In most cases, returned file path
	// should be identical however this may not be true for some particular file systems e.g. for base FS, file path
	// returned will have any base prefix removed.
	ConvertFilePath(name string) string
	// Converts a list of paths to relative paths
	ConvertToRelativePath(rootPath string, paths []string) ([]string, error)
	// Converts a list of paths to relative paths
	ConvertToAbsolutePath(rootPath string, paths []string) ([]string, error)
	// Creates a lock file on a remote location (NFS)
	NewRemoteLockFile(id string, dirToLock string) ILock
	// Compresses a file tree (source) into a zip file (destination)
	Zip(source string, destination string) error
	// Decompresses a source zip archive into the destination
	Unzip(source string, destination string) ([]string, error)
	// Calculates file hash
	FileHash(hashAlgo string, path string) (string, error)
}

func GetGlobalFileSystem 

func GetGlobalFileSystem() FS

func NewFs 

func NewFs(fsType int) FS

func NewInMemoryFileSystem 

func NewInMemoryFileSystem() FS

func NewStandardFileSystem 

func NewStandardFileSystem() FS

func NewVirtualFileSystem 

func NewVirtualFileSystem(vfs afero.Fs, fsType int, pathConverter func(path string) string) FS

type File 

type File interface {
	afero.File
	Fd() uintptr
}

func CreateFile 

func CreateFile(name string) (File, error)

func GenericOpen 

func GenericOpen(name string) (File, error)

func OpenFile added in v1.5.0 

func OpenFile(name string, flag int, perm os.FileMode) (File, error)

func TempFile 

func TempFile(dir string, pattern string) (f File, err error)

func TempFileInTempDir 

func TempFileInTempDir(pattern string) (f File, err error)

type FileTimeInfo 

type FileTimeInfo interface {
	ModTime() time.Time
	AccessTime() time.Time
	ChangeTime() time.Time
	BirthTime() time.Time
	HasChangeTime() bool
	HasBirthTime() bool
	HasAccessTime() bool
}

func DetermineFileTimes 

func DetermineFileTimes(info os.FileInfo) (times FileTimeInfo, err error)

type IFileHash 

type IFileHash interface {
	Calculate(f File) (string, error)
	CalculateFile(fs FS, path string) (string, error)
	GetType() string
}

For reference. https://stackoverflow.com/questions/1761607/what-is-the-fastest-hash-algorithm-to-check-if-two-files-are-equal

func NewFileHash 

func NewFileHash(hashType string) (IFileHash, error)

type ILock 

type ILock interface {
	Lock(ctx context.Context) error                                   // Locks the lock. This call will wait (i.e. block) until the lock is available.
	LockWithTimeout(ctx context.Context, timeout time.Duration) error // Tries to lock the lock until the timeout expires. If the timeout expires, this method will return commonerror.ErrTimeout.
	TryLock(ctx context.Context) error                                // Attempts to lock the lock instantly. This method will return commonerrors.ErrLocked immediately if the lock cannot be acquired straight away.
	IsStale() bool                                                    // Determines whether a lock is stale (the owner forgot to release it or is dead) or not.
	Unlock(ctx context.Context) error                                 // Releases the lock. This takes precedence over any current lock.
	ReleaseIfStale(ctx context.Context) error                         // Forces the lock to be released if it is considered as stale.
	MakeStale(ctx context.Context) error                              // Makes the lock stale. This is mostly for testing purposes.
}

FIXME it should be noted that despite being possible to use the lock with an in-memory filesystem, it should be avoided at all cost. The implementation of the in-memory FS used (afero) has shown several thread safety issues (e.g. https://github.com/spf13/afero/issues/298) and therefore, should not be used for scenarios involving concurrency until it is fixed.

func NewGenericRemoteLockFile 

func NewGenericRemoteLockFile(fs *VFS, lockID string, dirPath string, overrideStaleLock bool) ILock

NewGenericRemoteLockFile creates a new remote lock using the file system.

func NewRemoteLockFile 

func NewRemoteLockFile(fs *VFS, lockID string, dirPath string) ILock

NewRemoteLockFile creates a new remote lock using the file system. lockID Id for the lock. dirPath path where the lock should be applied to.

type Linker 

type Linker interface {
	LinkIfPossible(string, string) error
}

Optional interface. It is only implemented by the filesystems saying so.

type RemoteLockFile 

type RemoteLockFile struct {
	// contains filtered or unexported fields
}

RemoteLockFile describes a distributed lock using only the file system. The locking mechanism is performed using directories and the atomic function `mkdir`. A major issue of distributed locks is the presence of stale locks due to many factors such as the loss of the holder of a lock for various reasons. To mitigate this problem, a "heart bit" file is modified regularly by the lock holder in order to specify the holder is still alive and the lock still valid.

func (*RemoteLockFile) IsStale 

func (l *RemoteLockFile) IsStale() bool

IsStale checks whether the lock is stale (i.e. no heart beat detected) or not.

func (*RemoteLockFile) Lock 

func (l *RemoteLockFile) Lock(ctx context.Context) error

Lock locks the lock. This call will block until the lock is available.

func (*RemoteLockFile) LockWithTimeout 

func (l *RemoteLockFile) LockWithTimeout(ctx context.Context, timeout time.Duration) error

LockWithTimeout tries to lock the lock until the timeout expires

func (*RemoteLockFile) MakeStale 

func (l *RemoteLockFile) MakeStale(ctx context.Context) error

MakeStale is mostly useful for testing purposes and tries to mock locks going stale.

func (*RemoteLockFile) ReleaseIfStale 

func (l *RemoteLockFile) ReleaseIfStale(ctx context.Context) error

func (*RemoteLockFile) TryLock 

func (l *RemoteLockFile) TryLock(ctx context.Context) (err error)

TryLock attempts to lock the lock straight away.

func (*RemoteLockFile) Unlock 

func (l *RemoteLockFile) Unlock(context.Context) error

Unlock unlocks the lock

type UsageStat 

type UsageStat struct {
	Total             uint64
	Free              uint64
	Used              uint64
	UsedPercent       float64
	InodesTotal       uint64
	InodesUsed        uint64
	InodesFree        uint64
	InodesUsedPercent float64
}

func (*UsageStat) GetFree 

func (d *UsageStat) GetFree() uint64

func (*UsageStat) GetInodesFree 

func (d *UsageStat) GetInodesFree() uint64

func (*UsageStat) GetInodesTotal 

func (d *UsageStat) GetInodesTotal() uint64

func (*UsageStat) GetInodesUsed 

func (d *UsageStat) GetInodesUsed() uint64

func (*UsageStat) GetInodesUsedPercent 

func (d *UsageStat) GetInodesUsedPercent() float64

func (*UsageStat) GetTotal 

func (d *UsageStat) GetTotal() uint64

func (*UsageStat) GetUsed 

func (d *UsageStat) GetUsed() uint64

func (*UsageStat) GetUsedPercent 

func (d *UsageStat) GetUsedPercent() float64

type VFS 

type VFS struct {
	// contains filtered or unexported fields
}

func (*VFS) Chmod 

func (fs *VFS) Chmod(name string, mode os.FileMode) error

func (*VFS) Chown 

func (fs *VFS) Chown(name string, uid, gid int) (err error)

func (*VFS) Chtimes 

func (fs *VFS) Chtimes(name string, atime time.Time, mtime time.Time) error

func (*VFS) CleanDir 

func (fs *VFS) CleanDir(dir string) (err error)

func (*VFS) ConvertFilePath 

func (fs *VFS) ConvertFilePath(name string) string

func (*VFS) ConvertToAbsolutePath 

func (fs *VFS) ConvertToAbsolutePath(rootPath string, paths []string) ([]string, error)

func (*VFS) ConvertToRelativePath 

func (fs *VFS) ConvertToRelativePath(rootPath string, paths []string) ([]string, error)

func (*VFS) Copy 

func (fs *VFS) Copy(src string, dest string) (err error)

func (*VFS) CreateFile 

func (fs *VFS) CreateFile(name string) (File, error)

func (*VFS) CurrentDirectory 

func (fs *VFS) CurrentDirectory() (string, error)

func (*VFS) DiskUsage 

func (fs *VFS) DiskUsage(name string) (usage DiskUsage, err error)

func (*VFS) ExcludeAll 

func (fs *VFS) ExcludeAll(files []string, exclusionPatterns ...string) ([]string, error)

func (*VFS) Exists 

func (fs *VFS) Exists(path string) bool

func (*VFS) FileHash 

func (fs *VFS) FileHash(hashAlgo string, path string) (hash string, err error)

func (*VFS) FindAll 

func (fs *VFS) FindAll(dir string, extensions ...string) (files []string, err error)

func (*VFS) GarbageCollect 

func (fs *VFS) GarbageCollect(root string, durationSinceLastAccess time.Duration) error

func (*VFS) GenericOpen 

func (fs *VFS) GenericOpen(name string) (File, error)

func (*VFS) GetFileSize 

func (fs *VFS) GetFileSize(name string) (size int64, err error)

func (*VFS) GetType 

func (fs *VFS) GetType() int

func (*VFS) IsDir 

func (fs *VFS) IsDir(path string) (result bool, err error)

func (*VFS) IsEmpty 

func (fs *VFS) IsEmpty(name string) (empty bool, err error)

func (*VFS) IsFile 

func (fs *VFS) IsFile(path string) (result bool, err error)
func (fs *VFS) IsLink(path string) (result bool, err error)
func (fs *VFS) Link(oldname, newname string) (err error)

func (*VFS) ListDirTree 

func (fs *VFS) ListDirTree(dirPath string, list *[]string) error

Return a list of files and directories recursively available under specified path

func (*VFS) Lls 

func (fs *VFS) Lls(dir string) (files []os.FileInfo, err error)

func (*VFS) LlsFromOpenedDirectory 

func (fs *VFS) LlsFromOpenedDirectory(dir File) ([]os.FileInfo, error)

func (*VFS) Ls 

func (fs *VFS) Ls(dir string) (names []string, err error)

func (*VFS) LsFromOpenedDirectory 

func (fs *VFS) LsFromOpenedDirectory(dir File) ([]string, error)

func (*VFS) Lstat 

func (fs *VFS) Lstat(name string) (fileInfo os.FileInfo, err error)

func (*VFS) MkDir 

func (fs *VFS) MkDir(dir string) (err error)

func (*VFS) MkDirAll 

func (fs *VFS) MkDirAll(dir string, perm os.FileMode) (err error)

func (*VFS) Move 

func (fs *VFS) Move(src string, dest string) (err error)

func (*VFS) NewRemoteLockFile 

func (fs *VFS) NewRemoteLockFile(id string, dirToLock string) ILock

func (*VFS) Open 

func (fs *VFS) Open(name string) (doublestar.File, error)

func (*VFS) OpenFile 

func (fs *VFS) OpenFile(name string, flag int, perm os.FileMode) (File, error)

func (*VFS) PathSeparator 

func (fs *VFS) PathSeparator() rune

func (*VFS) ReadFile 

func (fs *VFS) ReadFile(filename string) (content []byte, err error)
func (fs *VFS) Readlink(name string) (value string, err error)

func (*VFS) Rm 

func (fs *VFS) Rm(dir string) (err error)

func (*VFS) Stat 

func (fs *VFS) Stat(name string) (os.FileInfo, error)

func (*VFS) StatTimes 

func (fs *VFS) StatTimes(name string) (info FileTimeInfo, err error)

func (*VFS) SubDirectories 

func (fs *VFS) SubDirectories(directory string) ([]string, error)

Return a list of all subdirectories (which are not hidden) 

func (fs *VFS) Symlink(oldname string, newname string) (err error)

func (*VFS) TempDir 

func (fs *VFS) TempDir(dir string, prefix string) (name string, err error)

func (*VFS) TempDirInTempDir 

func (fs *VFS) TempDirInTempDir(prefix string) (name string, err error)

func (*VFS) TempDirectory 

func (fs *VFS) TempDirectory() string

func (*VFS) TempFile 

func (fs *VFS) TempFile(dir string, prefix string) (f File, err error)

func (*VFS) TempFileInTempDir 

func (fs *VFS) TempFileInTempDir(prefix string) (f File, err error)

func (*VFS) Unzip 

func (fs *VFS) Unzip(source string, destination string) (fileList []string, err error)

func (*VFS) Walk 

func (fs *VFS) Walk(root string, fn filepath.WalkFunc) error

Walks https://golang.org/pkg/path/filepath/#WalkDir

func (*VFS) WriteFile 

func (fs *VFS) WriteFile(filename string, data []byte, perm os.FileMode) (err error)

func (*VFS) Zip 

func (fs *VFS) Zip(source, destination string) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.