filesystem

package module
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 3, 2024 License: MIT Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	AlgoDFS = "DFS"
	AlgoBFS = "BFS"
)
View Source 
const (
	DirStubFileName    = ".dir" // name of stub file to imitate empty folder
	DirStubFileContent = "!"    // content of stub file to imitate empty folder
	TempDir            = "tmp"
)

constants regarding stubs and directories

Variables

View Source 
var (
	ErrCantOpenS3Directory           = errors.New("can't open S3 directory")
	ErrDirectoryNotEmpty             = errors.New("directory not empty")
	ErrDestinationPathIsNotDirectory = errors.New("destination path is not directory while source is")
	ErrCantUseRenameWithStubObject   = errors.New("can't use rename with stub object")
	ErrRenamingNonExistentDirectory  = errors.New("can't rename non-existent directory")
	ErrNotADirectory                 = errors.New("given path is not a directory")
	ErrDirectoryNotExists            = errors.New("directory not exists")
	ErrUnknownFileMode               = errors.New("unknown file mode")
	ErrFileAlreadyOpened             = errors.New("file already opened")
)

errors

View Source 
var ErrSkipDir = fs.SkipDir

ErrSkipDir should be returned from WalkDirFunc to skip walking inside directory

Functions

func AfterOperationCB added in v1.1.0 

func AfterOperationCB() cbFunc

AfterOperationCB returns callback that will be invoked after each operation

func BeforeOperationCB added in v1.1.0 

func BeforeOperationCB() cbCtxFunc

BeforeOperationCB returns callback that will be invoked before each operation

func IsDir added in v1.1.14 

func IsDir(ctx context.Context, fs FileSystem, name string) (bool, error)

IsDir returns true if given name is a directory

func RemoveEmptyDirs added in v1.1.14 

func RemoveEmptyDirs(ctx context.Context, fs FileSystem, logger logrus.FieldLogger, basePath, algo string) (int, error)

RemoveEmptyDirs removed all subtrees of directories inside basePath which contains only empty directories recursively

func SetAfterOperationCB added in v1.1.0 

func SetAfterOperationCB(f cbFunc)

SetAfterOperationCB sets callback that will be invoked after each operation

func SetBeforeOperationCB added in v1.1.0 

func SetBeforeOperationCB(f cbCtxFunc)

SetBeforeOperationCB sets callback that will be invoked before each operation

Types

type DirEntry 

type DirEntry interface {
	fs.DirEntry
	FullName() string
}

DirEntry abstracts directory walkDirEntry

type EmptySubtreeCleaner added in v1.1.14 

type EmptySubtreeCleaner struct {
	FS    FileSystem
	Count int

	Logger logrus.FieldLogger
	// contains filtered or unexported fields
}

EmptySubtreeCleaner removes empty directories subtrees

type File 

type File interface {
	fs.File
	io.ReaderAt
	io.Writer
	io.Seeker
	Truncate(int64) error
	Sync() error
	Name() string
}

File abstracts a file

type FileInfo 

type FileInfo interface {
	fs.FileInfo
	FS() FileSystem
	FullName() string
}

FileInfo abstracts file information

type FileNameData added in v1.0.10 

type FileNameData struct {
	Name string
	Data []byte
}

FileNameData represents file name and data

type FileSystem 

type FileSystem interface {
	Create(context.Context, string) (File, error)
	Open(context.Context, string) (File, error)
	OpenW(context.Context, string) (File, error)
	ReadFile(context.Context, string) ([]byte, error)
	WriteFile(context.Context, string, []byte) error
	WriteFiles(context.Context, []FileNameData) error
	Reader(context.Context, string) (io.ReadCloser, error)
	Exists(context.Context, string) (bool, error)
	MakePathAll(context.Context, string) error
	Remove(context.Context, string) error
	RemoveFiles(context.Context, []string) ([]string, error)
	RemoveAll(context.Context, string) error
	IsNotExist(error) bool
	IsEmptyPath(context.Context, string) (bool, error)
	PreparePath(context.Context, string) (string, error)
	Rename(context.Context, string, string) error
	Stat(context.Context, string) (FileInfo, error)
	ReadDir(context.Context, string) (FilesInfo, error)
	WalkDir(context.Context, string, WalkDirFunc) error
	Join(...string) string
	Dir(string) string
	Ext(string) string
	Base(string) string
}

FileSystem abstracts a file system

func NewLocal 

func NewLocal() FileSystem

NewLocal returns a pointer to a new Local object

type FilesInfo 

type FilesInfo []FileInfo

FilesInfo is a slice of FileInfo

func (FilesInfo) FullNames added in v1.0.7 

func (fsi FilesInfo) FullNames() []string

FullNames returns a slice of full names derived from the receiver slice

type Local 

type Local struct{}

Local implements FileSystem. The implementation is not concurrent-safe

func (*Local) Base added in v1.1.12 

func (l *Local) Base(name string) string

Base returns the last element of path

func (*Local) Create 

func (l *Local) Create(ctx context.Context, name string) (f File, err error)

Create file in the FileSystem

func (*Local) Dir added in v1.1.8 

func (l *Local) Dir(name string) string

Dir returns parent directory path

func (*Local) Exists 

func (l *Local) Exists(ctx context.Context, name string) (e bool, err error)

Exists returns whether file exists or not

func (*Local) Ext added in v1.1.9 

func (l *Local) Ext(name string) string

Ext returns file name extension

func (*Local) IsEmptyPath 

func (l *Local) IsEmptyPath(ctx context.Context, name string) (e bool, err error)

IsEmptyPath returns whether given name is empty (does not contain any subpaths)

func (*Local) IsNotExist 

func (l *Local) IsNotExist(err error) bool

IsNotExist returns whether err is a "file not exists" error

func (*Local) Join added in v1.1.8 

func (l *Local) Join(names ...string) string

Join the path segments

func (*Local) MakePathAll 

func (l *Local) MakePathAll(ctx context.Context, name string) (err error)

MakePathAll makes name recursively

func (*Local) Open 

func (l *Local) Open(ctx context.Context, name string) (f File, err error)

Open file in the FileSystem

func (*Local) OpenW added in v1.0.4 

func (l *Local) OpenW(ctx context.Context, name string) (f File, err error)

OpenW opens file in the FileSystem for writing

func (*Local) PreparePath 

func (l *Local) PreparePath(ctx context.Context, name string) (absolutePath string, err error)

PreparePath constructs an absolute name from. If it does not exists, creates it.

func (*Local) ReadDir 

func (l *Local) ReadDir(ctx context.Context, name string) (fi FilesInfo, err error)

ReadDir with the name given

func (*Local) ReadFile 

func (l *Local) ReadFile(ctx context.Context, name string) (b []byte, err error)

ReadFile by name

func (*Local) Reader 

func (l *Local) Reader(ctx context.Context, name string) (r io.ReadCloser, err error)

Reader returns io.Reader file abstraction

func (*Local) Remove 

func (l *Local) Remove(ctx context.Context, name string) (err error)

Remove file

func (*Local) RemoveAll 

func (l *Local) RemoveAll(ctx context.Context, name string) (err error)

RemoveAll removes the entire name

func (*Local) RemoveFiles added in v1.0.13 

func (l *Local) RemoveFiles(ctx context.Context, names []string) (failed []string, err error)

RemoveFiles files given

func (*Local) Rename 

func (l *Local) Rename(ctx context.Context, from, to string) (err error)

Rename file

func (*Local) Stat 

func (l *Local) Stat(ctx context.Context, name string) (fi FileInfo, err error)

Stat returns a FileInfo describing the named file

func (*Local) WalkDir 

func (l *Local) WalkDir(ctx context.Context, root string, walkDirFunc WalkDirFunc) (err error)

WalkDir traverses the filesystem from the given directory

func (*Local) WriteFile 

func (l *Local) WriteFile(ctx context.Context, name string, data []byte) (err error)

WriteFile by name

func (*Local) WriteFiles added in v1.0.10 

func (l *Local) WriteFiles(ctx context.Context, f []FileNameData) (err error)

WriteFiles by the data given

type LocalDirEntry added in v1.0.7 

type LocalDirEntry struct {
	// contains filtered or unexported fields
}

LocalDirEntry implements DirEntry

func (LocalDirEntry) FullName added in v1.0.7 

func (s LocalDirEntry) FullName() string

FullName makes LocalDirEntry to implement DirEntry. Returns full name of the underlying FileInfo object

func (LocalDirEntry) Info added in v1.0.7 

func (s LocalDirEntry) Info() (fs.FileInfo, error)

Info makes LocalDirEntry to implement DirEntry. It returns the underlying FileInfo object

func (LocalDirEntry) IsDir added in v1.0.7 

func (s LocalDirEntry) IsDir() bool

IsDir makes LocalDirEntry to implement DirEntry. It calls IsDir() of the underlying FileInfo object

func (LocalDirEntry) Name added in v1.0.7 

func (s LocalDirEntry) Name() string

Name makes LocalDirEntry to implement DirEntry. Returns name of the underlying FileInfo object

func (LocalDirEntry) Type added in v1.0.7 

func (s LocalDirEntry) Type() fs.FileMode

Type makes LocalDirEntry to implement DirEntry. It calls Mode() of the underlying FileInfo object

type LocalFileInfo added in v1.0.7 

type LocalFileInfo struct {
	// contains filtered or unexported fields
}

LocalFileInfo implements FileInfo

func NewLocalFileInfo added in v1.0.7 

func NewLocalFileInfo(fs *Local, fi os.FileInfo, fullName string) LocalFileInfo

NewLocalFileInfo returns new LocalFileInfo object

func (LocalFileInfo) FS added in v1.2.1 

func (s LocalFileInfo) FS() FileSystem

FS makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) FullName added in v1.0.7 

func (s LocalFileInfo) FullName() string

FullName makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) IsDir added in v1.0.7 

func (s LocalFileInfo) IsDir() bool

IsDir makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) ModTime added in v1.0.7 

func (s LocalFileInfo) ModTime() time.Time

ModTime makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) Mode added in v1.0.7 

func (s LocalFileInfo) Mode() fs.FileMode

Mode makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) Name added in v1.0.7 

func (s LocalFileInfo) Name() string

Name makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) Size added in v1.0.7 

func (s LocalFileInfo) Size() int64

Size makes LocalFileInfo to implement FileInfo

func (LocalFileInfo) Sys added in v1.0.7 

func (s LocalFileInfo) Sys() interface{}

Sys makes LocalFileInfo to implement FileInfo

type Node added in v1.1.14 

type Node struct {
	Path     string
	Children []*Node
}

Node is a tree node

type S3 

type S3 struct {
	// contains filtered or unexported fields
}

S3 implements FileSystem. The implementation is not concurrent-safe

func NewS3 

func NewS3(ctx context.Context, p S3Params) (s3 *S3, err error)

NewS3 returns a pointer to a new Local object

func (*S3) Base added in v1.1.12 

func (s *S3) Base(name string) string

Base returns the last element of path

func (*S3) Count 

func (s *S3) Count(ctx context.Context, name string, recursive bool,
	countFunc func(oi minio.ObjectInfo, num int64) (proceed bool, e error)) (c int64, err error)

Count returns count of items in a folder. May count in childs also if recursive param set to true.

func (*S3) Create added in v1.0.2 

func (s *S3) Create(ctx context.Context, name string) (f File, err error)

Create file with given name in the client's bucket. A file will be created locally for reading, writing, truncating. To remove the actual local file and write out into S3 object it should be properly closed by calling Close() on the caller's side. Calls to Open, Create, OpenW and S3OpenedFile.Close are concurrent-safe and mutually locking.

func (*S3) Dir added in v1.1.8 

func (s *S3) Dir(name string) string

Dir returns parent directory path

func (*S3) Exists 

func (s *S3) Exists(ctx context.Context, name string) (e bool, err error)

Exists checks whether an object exists

func (*S3) Ext added in v1.1.9 

func (s *S3) Ext(name string) string

Ext returns object name extension

func (*S3) IsEmptyPath 

func (s *S3) IsEmptyPath(ctx context.Context, name string) (e bool, err error)

IsEmptyPath works according to the MakePathAll implementation. Returns true only if specified path contains only a dir stub file.

func (*S3) IsNotExist 

func (s *S3) IsNotExist(err error) bool

IsNotExist returns whether err is an 'bucket not exists' error or 'object not exists' error

func (*S3) Join added in v1.1.8 

func (s *S3) Join(paths ...string) string

Join the path segments

func (*S3) Logger 

func (s *S3) Logger() logrus.FieldLogger

Logger provides access to a logger

func (*S3) MakePathAll 

func (s *S3) MakePathAll(ctx context.Context, name string) (err error)

MakePathAll does nothing in S3 FileSystem. As a workaround to make sure that we have a precreated folder, we create a small file in it. refer to: https://github.com/minio/minio/issues/3555, https://github.com/minio/minio/issues/2423

func (*S3) MinioClient 

func (s *S3) MinioClient() *minio.Client

MinioClient provides access to Minio Client, use mainly for tests

func (*S3) Open 

func (s *S3) Open(ctx context.Context, name string) (f File, err error)

Open file with given name in the client's bucket. An object will be downloaded from S3 storage and opened as a local file for reading. To remove the actual local file and write out into S3 object it should be properly closed by calling Close() on the caller's side. Calls to Open, Create, OpenW and S3OpenedFile.Close are concurrent-safe and mutually locking.

func (*S3) OpenW added in v1.0.4 

func (s *S3) OpenW(ctx context.Context, name string) (f File, err error)

OpenW opens file in the FileSystem for writing. An object will be downloaded from S3 storage and opened as a local file for writing. To remove the actual local file and write out into S3 object it should be properly closed by calling Close() on the caller's side. Calls to Open, Create, OpenW and S3OpenedFile.Close are concurrent-safe and mutually locking.

func (*S3) OpenedFilesList 

func (s *S3) OpenedFilesList() *S3OpenedFilesList

OpenedFilesList provides access to opened files list, use mainly for tests

func (*S3) OpenedFilesListLock 

func (s *S3) OpenedFilesListLock()

OpenedFilesListLock locks opened files list associated mutex, use mainly for tests

func (*S3) OpenedFilesListUnlock 

func (s *S3) OpenedFilesListUnlock()

OpenedFilesListUnlock unlocks opened files list associated mutex, use mainly for tests

func (*S3) PreparePath 

func (s *S3) PreparePath(ctx context.Context, name string) (_ string, err error)

PreparePath works according to the MakePathAll implementation.

func (*S3) ReadDir 

func (s *S3) ReadDir(ctx context.Context, name string) (fi FilesInfo, err error)

ReadDir simulates directory reading by the given name

func (*S3) ReadFile 

func (s *S3) ReadFile(ctx context.Context, name string) (b []byte, err error)

ReadFile by it's name from the client's bucket

func (*S3) Reader 

func (s *S3) Reader(ctx context.Context, name string) (r io.ReadCloser, err error)

Reader returns reader by it's name

func (*S3) Remove 

func (s *S3) Remove(ctx context.Context, name string) (err error)

Remove object by the given name. Returns no error even if object does not exists

func (*S3) RemoveAll 

func (s *S3) RemoveAll(ctx context.Context, name string) (err error)

RemoveAll objects by the given filepath

func (*S3) RemoveFiles added in v1.0.13 

func (s *S3) RemoveFiles(ctx context.Context, names []string) (failed []string, err error)

RemoveFiles removes multiple objects in batch by the given names. Returns no error even if any object does not exists

func (*S3) Rename 

func (s *S3) Rename(ctx context.Context, from string, to string) (err error)

Rename object

func (*S3) SetListDirectoryEntries 

func (s *S3) SetListDirectoryEntries(v bool)

SetListDirectoryEntries or unset it, use mainly for tests

func (*S3) Stat 

func (s *S3) Stat(ctx context.Context, name string) (fi FileInfo, err error)

Stat returns S3 object information as FileInfo interface

func (*S3) TempFileName added in v1.0.5 

func (s *S3) TempFileName(name string) string

TempFileName converts file name to a temporary file name

func (*S3) WalkDir 

func (s *S3) WalkDir(ctx context.Context, name string, walkDirFunc WalkDirFunc) (err error)

WalkDir simulates traversing the filesystem from the given directory

func (*S3) WriteFile 

func (s *S3) WriteFile(ctx context.Context, name string, b []byte) (err error)

WriteFile by it's name to the client's bucket

func (*S3) WriteFiles added in v1.0.10 

func (s *S3) WriteFiles(ctx context.Context, f []FileNameData) (err error)

WriteFiles by the data given. An archive will be created by the underlying minio client

type S3DirEntry 

type S3DirEntry struct {
	// contains filtered or unexported fields
}

S3DirEntry implements DirEntry

func (S3DirEntry) FullName added in v1.0.7 

func (s S3DirEntry) FullName() string

FullName makes S3DirEntry to implement DirEntry. Returns full name of the underlying FileInfo object

func (S3DirEntry) Info 

func (s S3DirEntry) Info() (fs.FileInfo, error)

Info makes S3DirEntry to implement DirEntry. It returns the underlying FileInfo object

func (S3DirEntry) IsDir 

func (s S3DirEntry) IsDir() bool

IsDir makes S3DirEntry to implement DirEntry. It calls IsDir() of the underlying FileInfo object

func (S3DirEntry) Name 

func (s S3DirEntry) Name() string

Name makes S3DirEntry to implement DirEntry. Returns name of the underlying FileInfo object

func (S3DirEntry) Type 

func (s S3DirEntry) Type() fs.FileMode

Type makes S3DirEntry to implement DirEntry. It calls Mode() of the underlying FileInfo object

type S3FileInfo 

type S3FileInfo struct {
	// contains filtered or unexported fields
}

S3FileInfo implements FileInfo

func NewS3FileInfo added in v1.0.1 

func NewS3FileInfo(fs *S3, oi minio.ObjectInfo) S3FileInfo

NewS3FileInfo returns new S3FileInfo object

func NewS3FileInfoStub added in v1.0.1 

func NewS3FileInfoStub(fs *S3, key string, modTime time.Time) S3FileInfo

NewS3FileInfoStub returns new stub S3FileInfo object

func (S3FileInfo) FS added in v1.2.1 

func (s S3FileInfo) FS() FileSystem

FS makes S3FileInfo to implement FileInfo. Returns same as Sys() method, but an interface.

func (S3FileInfo) FullName added in v1.0.7 

func (s S3FileInfo) FullName() string

FullName makes S3FileInfo to implement FileInfo. Returns last part (file name) of the key of S3 object

func (S3FileInfo) IsDir 

func (s S3FileInfo) IsDir() bool

IsDir makes S3FileInfo to implement FileInfo. It returns whether an object key ends in '/' (and it's size is 0)

func (S3FileInfo) ModTime 

func (s S3FileInfo) ModTime() time.Time

ModTime makes S3FileInfo to implement FileInfo. Returns last modified time

func (S3FileInfo) Mode 

func (s S3FileInfo) Mode() fs.FileMode

Mode makes S3FileInfo to implement FileInfo. It always returns 0

func (S3FileInfo) Name 

func (s S3FileInfo) Name() string

Name makes S3FileInfo to implement FileInfo. Returns last part (file name) of the key of S3 object

func (S3FileInfo) Size 

func (s S3FileInfo) Size() int64

Size makes S3FileInfo to implement FileInfo. Returns size of S3 object

func (S3FileInfo) Sys 

func (s S3FileInfo) Sys() interface{}

Sys makes S3FileInfo to implement FileInfo. It returns a value of type *S3: a pointer to the underlying FileSystem-implementing object

type S3OpenedFile 

type S3OpenedFile struct {
	// contains filtered or unexported fields
}

S3OpenedFile implements a wrapper around File

func (*S3OpenedFile) Close 

func (of *S3OpenedFile) Close() error

Close makes S3OpenedFile to implement File. It closed the underlying File and removes it from local file system.

func (*S3OpenedFile) LocalName added in v1.0.5 

func (of *S3OpenedFile) LocalName() string

LocalName returns local file name

func (*S3OpenedFile) Name 

func (of *S3OpenedFile) Name() string

Name returns S3 object name

func (*S3OpenedFile) Read 

func (of *S3OpenedFile) Read(bytes []byte) (int, error)

Read makes S3OpenedFile to implement File

func (*S3OpenedFile) ReadAt 

func (of *S3OpenedFile) ReadAt(p []byte, off int64) (n int, err error)

ReadAt makes S3OpenedFile to implement File

func (*S3OpenedFile) Seek 

func (of *S3OpenedFile) Seek(offset int64, whence int) (int64, error)

Seek makes S3OpenedFile to implement File

func (*S3OpenedFile) SetUnderlying added in v1.0.5 

func (of *S3OpenedFile) SetUnderlying(f File)

SetUnderlying file

func (*S3OpenedFile) Stat 

func (of *S3OpenedFile) Stat() (fs.FileInfo, error)

Stat makes S3OpenedFile to implement File

func (*S3OpenedFile) Sync 

func (of *S3OpenedFile) Sync() error

Sync makes S3OpenedFile to implement File

func (*S3OpenedFile) Truncate 

func (of *S3OpenedFile) Truncate(size int64) error

Truncate makes S3OpenedFile to implement File

func (*S3OpenedFile) Underlying added in v1.0.5 

func (of *S3OpenedFile) Underlying() File

Underlying returns an underlying file

func (*S3OpenedFile) Write 

func (of *S3OpenedFile) Write(p []byte) (n int, err error)

Write makes S3OpenedFile to implement File

type S3OpenedFilesList 

type S3OpenedFilesList struct {
	sync.Mutex
	// contains filtered or unexported fields
}

S3OpenedFilesList represents S3 opened files list

func NewS3OpenedFilesList 

func NewS3OpenedFilesList() *S3OpenedFilesList

NewS3OpenedFilesList returns a pointer to new S3 opened files list instance

func (*S3OpenedFilesList) AddAndLockEntry added in v1.0.5 

func (ofl *S3OpenedFilesList) AddAndLockEntry(localFileName string, entry *S3OpenedFilesListEntry)

AddAndLockEntry adds an entry to the list and locks it

func (*S3OpenedFilesList) DeleteAndUnlockEntry added in v1.0.5 

func (ofl *S3OpenedFilesList) DeleteAndUnlockEntry(localFileName string)

DeleteAndUnlockEntry deletes and unlocks an entry from the list if it exists

func (*S3OpenedFilesList) Len added in v1.0.5 

func (ofl *S3OpenedFilesList) Len() int

Len returns length of list

func (*S3OpenedFilesList) Map 

func (ofl *S3OpenedFilesList) Map() map[string]*S3OpenedFilesListEntry

Map returns opened file list underlying map

type S3OpenedFilesListEntry 

type S3OpenedFilesListEntry struct {
	Added  time.Time
	S3File *S3OpenedFile
	// contains filtered or unexported fields
}

S3OpenedFilesListEntry is an walkDirEntry of opened files list

func (*S3OpenedFilesListEntry) Lock added in v1.0.5 

func (ofle *S3OpenedFilesListEntry) Lock()

Lock or RLock the entry according to S3File Open mode

func (*S3OpenedFilesListEntry) Unlock added in v1.0.5 

func (ofle *S3OpenedFilesListEntry) Unlock()

Unlock or RUnlock the entry according to S3File Open mode

type S3Params 

type S3Params struct {
	Endpoint   string
	Region     string
	AccessKey  string
	SecretKey  string
	UseSSL     bool
	BucketName string

	OpenedFilesTTL     time.Duration
	OpenedFilesTempDir string

	Logger logrus.FieldLogger

	EmulateEmptyDirs     bool // without this directory modification time will not be available
	ListDirectoryEntries bool // in the ReadDir output
}

S3Params are parameters for S3 filesystem client

type WalkDirFunc 

type WalkDirFunc func(string, DirEntry, error) error

WalkDirFunc is a wrapper around fs.WalkDirFunc

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.