file

package
v0.20.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 30, 2023 License: AGPL-3.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	FingerprintTypeOshash = "oshash"
	FingerprintTypeMD5    = "md5"
	FingerprintTypePhash  = "phash"
)

Functions

func Destroy added in v0.17.0 

func Destroy(ctx context.Context, destroyer Destroyer, f File, fileDeleter *Deleter, deleteFile bool) error

Types

type BaseFile added in v0.17.0 

type BaseFile struct {
	ID ID `json:"id"`

	DirEntry

	// resolved from parent folder and basename only - not stored in DB
	Path string `json:"path"`

	Basename       string   `json:"basename"`
	ParentFolderID FolderID `json:"parent_folder_id"`

	Fingerprints Fingerprints `json:"fingerprints"`

	Size int64 `json:"size"`

	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
}

BaseFile represents a file in the file system.

func (*BaseFile) Base added in v0.17.0 

func (f *BaseFile) Base() *BaseFile

Base is used to fulfil the File interface.

func (*BaseFile) Info added in v0.17.0 

func (f *BaseFile) Info(fs FS) (fs.FileInfo, error)

func (*BaseFile) Open added in v0.17.0 

func (f *BaseFile) Open(fs FS) (io.ReadCloser, error)

func (*BaseFile) Serve added in v0.17.0 

func (f *BaseFile) Serve(fs FS, w http.ResponseWriter, r *http.Request) error

func (*BaseFile) SetFingerprint added in v0.17.0 

func (f *BaseFile) SetFingerprint(fp Fingerprint)

SetFingerprint sets the fingerprint of the file. If a fingerprint of the same type already exists, it is overwritten.

func (*BaseFile) SetFingerprints added in v0.17.0 

func (f *BaseFile) SetFingerprints(fp Fingerprints)

SetFingerprints sets the fingerprints of the file. If a fingerprint of the same type already exists, it is overwritten.

type CleanHandler added in v0.17.0 

type CleanHandler interface {
	HandleFile(ctx context.Context, fileDeleter *Deleter, fileID ID) error
	HandleFolder(ctx context.Context, fileDeleter *Deleter, folderID FolderID) error
}

CleanHandler provides a handler for cleaning Files and Folders.

type CleanOptions added in v0.17.0 

type CleanOptions struct {
	Paths []string

	// Do a dry run. Don't delete any files
	DryRun bool

	// PathFilter are used to determine if a file should be included.
	// Excluded files are marked for cleaning.
	PathFilter PathFilter
}

ScanOptions provides options for scanning files.

type Cleaner added in v0.17.0 

type Cleaner struct {
	FS         FS
	Repository Repository

	Handlers []CleanHandler
}

Cleaner scans through stored file and folder instances and removes those that are no longer present on disk.

func (*Cleaner) Clean added in v0.17.0 

func (s *Cleaner) Clean(ctx context.Context, options CleanOptions, progress *job.Progress)

Clean starts the clean process.

type Counter added in v0.17.0 

type Counter interface {
	CountAllInPaths(ctx context.Context, p []string) (int, error)
}

type Creator added in v0.17.0 

type Creator interface {
	Create(ctx context.Context, f File) error
}

Creator provides methods to create Files.

type Decorator added in v0.17.0 

type Decorator interface {
	Decorate(ctx context.Context, fs FS, f File) (File, error)
	IsMissingMetadata(ctx context.Context, fs FS, f File) bool
}

Decorator wraps the Decorate method to add additional functionality while scanning files.

type Deleter added in v0.12.0 

type Deleter struct {
	RenamerRemover RenamerRemover
	// contains filtered or unexported fields
}

Deleter is used to safely delete files and directories from the filesystem. During a transaction, files and directories are marked for deletion using the Files and Dirs methods. This will rename the files/directories to be deleted. If the transaction is rolled back, then the files/directories can be restored to their original state with the Abort method. If the transaction is committed, the marked files are then deleted from the filesystem using the Complete method.

func NewDeleter added in v0.12.0 

func NewDeleter() *Deleter

func (*Deleter) Commit added in v0.12.0 

func (d *Deleter) Commit()

Commit deletes all files marked for deletion and clears the marked list. Any errors encountered are logged. All files will be attempted, regardless of the errors encountered.

func (*Deleter) Dirs added in v0.12.0 

func (d *Deleter) Dirs(paths []string) error

Dirs designates directories to be deleted. Each directory marked will be renamed to add a `.delete` suffix. An error is returned if a directory could not be renamed. Note that if an error is returned, then some directories may be left renamed. Abort should be called to restore marked files/directories if this function returns an error.

func (*Deleter) Files added in v0.12.0 

func (d *Deleter) Files(paths []string) error

Files designates files to be deleted. Each file marked will be renamed to add a `.delete` suffix. An error is returned if a file could not be renamed. Note that if an error is returned, then some files may be left renamed. Abort should be called to restore marked files if this function returns an error.

func (*Deleter) RegisterHooks added in v0.17.0 

func (d *Deleter) RegisterHooks(ctx context.Context)

RegisterHooks registers post-commit and post-rollback hooks.

func (*Deleter) Rollback added in v0.12.0 

func (d *Deleter) Rollback()

Rollback tries to rename all marked files and directories back to their original names and clears the marked list. Any errors encountered are logged. All files will be attempted regardless of any errors occurred.

type Destroyer added in v0.17.0 

type Destroyer interface {
	Destroy(ctx context.Context, id ID) error
}

type DirEntry added in v0.17.0 

type DirEntry struct {
	ZipFileID *ID `json:"zip_file_id"`

	// transient - not persisted
	// only guaranteed to have id, path and basename set
	ZipFile File

	ModTime time.Time `json:"mod_time"`
}

DirEntry represents a file or directory in the file system.

type DirMakerStatRenamer added in v0.20.0 

type DirMakerStatRenamer interface {
	Statter
	Renamer
	Mkdir(name string, perm os.FileMode) error
	Remove(name string) error
}

type FS added in v0.17.0 

type FS interface {
	Stat(name string) (fs.FileInfo, error)
	Lstat(name string) (fs.FileInfo, error)
	Open(name string) (fs.ReadDirFile, error)
	OpenZip(name string) (*ZipFS, error)
	IsPathCaseSensitive(path string) (bool, error)
}

FS represents a file system.

type File added in v0.17.0 

type File interface {
	Base() *BaseFile
	SetFingerprints(fp Fingerprints)
	Open(fs FS) (io.ReadCloser, error)
}

File represents a file in the file system.

type Filter added in v0.17.0 

type Filter interface {
	Accept(ctx context.Context, f File) bool
}

Filter provides a filter function for Files.

type FilterFunc added in v0.17.0 

type FilterFunc func(ctx context.Context, f File) bool

func (FilterFunc) Accept added in v0.17.0 

func (ff FilterFunc) Accept(ctx context.Context, f File) bool

type FilteredDecorator added in v0.17.0 

type FilteredDecorator struct {
	Decorator
	Filter
}

func (*FilteredDecorator) Decorate added in v0.17.0 

func (d *FilteredDecorator) Decorate(ctx context.Context, fs FS, f File) (File, error)

Decorate runs the decorator if the filter accepts the file.

func (*FilteredDecorator) IsMissingMetadata added in v0.17.0 

func (d *FilteredDecorator) IsMissingMetadata(ctx context.Context, fs FS, f File) bool

type FilteredHandler added in v0.17.0 

type FilteredHandler struct {
	Handler
	Filter
}

FilteredHandler is a Handler runs only if the filter accepts the file.

func (*FilteredHandler) Handle added in v0.17.0 

func (h *FilteredHandler) Handle(ctx context.Context, f File, oldFile File) error

Handle runs the handler if the filter accepts the file.

type Finder added in v0.17.0 

type Finder interface {
	Find(ctx context.Context, id ...ID) ([]File, error)
}

type Fingerprint added in v0.17.0 

type Fingerprint struct {
	Type        string
	Fingerprint interface{}
}

Fingerprint represents a fingerprint of a file.

type FingerprintCalculator added in v0.17.0 

type FingerprintCalculator interface {
	CalculateFingerprints(f *BaseFile, o Opener, useExisting bool) ([]Fingerprint, error)
}

FingerprintCalculator calculates a fingerprint for the provided file.

type Fingerprints added in v0.17.0 

type Fingerprints []Fingerprint

func (Fingerprints) AppendUnique added in v0.17.0 

func (f Fingerprints) AppendUnique(o Fingerprint) Fingerprints

AppendUnique appends a fingerprint to the list if a Fingerprint of the same type does not already exist in the list. If one does, then it is updated with o's Fingerprint value.

func (Fingerprints) ContentsChanged added in v0.17.0 

func (f Fingerprints) ContentsChanged(other Fingerprints) bool

ContentsChanged returns true if this Fingerprints slice contains any Fingerprints that different Fingerprint values for the matching type in other, or if this slice contains any Fingerprint types that are not in other.

func (Fingerprints) Equals added in v0.17.0 

func (f Fingerprints) Equals(other Fingerprints) bool

Equals returns true if the contents of this slice are equal to those in the other slice.

func (Fingerprints) For added in v0.17.0 

func (f Fingerprints) For(type_ string) *Fingerprint

For returns a pointer to the first Fingerprint element matching the provided type.

func (Fingerprints) Get added in v0.17.0 

func (f Fingerprints) Get(type_ string) interface{}

func (Fingerprints) GetInt64 added in v0.17.0 

func (f Fingerprints) GetInt64(type_ string) int64

func (Fingerprints) GetString added in v0.17.0 

func (f Fingerprints) GetString(type_ string) string

func (*Fingerprints) Remove added in v0.17.0 

func (f *Fingerprints) Remove(type_ string)

type Folder added in v0.17.0 

type Folder struct {
	ID FolderID `json:"id"`
	DirEntry
	Path           string    `json:"path"`
	ParentFolderID *FolderID `json:"parent_folder_id"`

	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
}

Folder represents a folder in the file system.

func GetOrCreateFolderHierarchy added in v0.20.0 

func GetOrCreateFolderHierarchy(ctx context.Context, fc FolderFinderCreator, path string) (*Folder, error)

GetOrCreateFolderHierarchy gets the folder for the given path, or creates a folder hierarchy for the given path if one if no existing folder is found. Does not create any folders in the file system

func (*Folder) Info added in v0.17.0 

func (f *Folder) Info(fs FS) (fs.FileInfo, error)

type FolderCounter added in v0.17.0 

type FolderCounter interface {
	CountAllInPaths(ctx context.Context, p []string) (int, error)
}

type FolderCreator added in v0.17.0 

type FolderCreator interface {
	Create(ctx context.Context, f *Folder) error
}

FolderCreator provides methods to create Folders.

type FolderDestroyer added in v0.17.0 

type FolderDestroyer interface {
	Destroy(ctx context.Context, id FolderID) error
}

type FolderFinder added in v0.20.0 

type FolderFinder interface {
	Find(ctx context.Context, id FolderID) (*Folder, error)
}

type FolderFinderCreator added in v0.20.0 

type FolderFinderCreator interface {
	FolderPathFinder
	FolderCreator
}

type FolderGetter added in v0.17.0 

type FolderGetter interface {
	FolderFinder
	FolderPathFinder
	FindByZipFileID(ctx context.Context, zipFileID ID) ([]*Folder, error)
	FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]*Folder, error)
	FindByParentFolderID(ctx context.Context, parentFolderID FolderID) ([]*Folder, error)
}

FolderGetter provides methods to find Folders.

type FolderGetterDestroyer added in v0.17.0 

type FolderGetterDestroyer interface {
	FolderGetter
	FolderDestroyer
}

type FolderID added in v0.17.0 

type FolderID int32

FolderID represents an ID of a folder.

func (FolderID) String added in v0.17.0 

func (i FolderID) String() string

String converts the ID to a string.

type FolderPathFinder added in v0.20.0 

type FolderPathFinder interface {
	FindByPath(ctx context.Context, path string) (*Folder, error)
}

FolderPathFinder finds Folders by their path.

type FolderStore added in v0.17.0 

type FolderStore interface {
	FolderGetter
	FolderCounter
	FolderCreator
	FolderUpdater
	FolderDestroyer
}

FolderStore provides methods to find, create and update Folders.

type FolderUpdater added in v0.17.0 

type FolderUpdater interface {
	Update(ctx context.Context, f *Folder) error
}

FolderUpdater provides methods to update Folders.

type Getter added in v0.17.0 

type Getter interface {
	Finder
	FindByPath(ctx context.Context, path string) (File, error)
	FindAllByPath(ctx context.Context, path string) ([]File, error)
	FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error)
	FindByZipFileID(ctx context.Context, zipFileID ID) ([]File, error)
	FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error)
}

Getter provides methods to find Files.

type GetterDestroyer added in v0.17.0 

type GetterDestroyer interface {
	Getter
	Destroyer
}

type GetterUpdater added in v0.20.0 

type GetterUpdater interface {
	Getter
	Updater
}

type Handler added in v0.17.0 

type Handler interface {
	Handle(ctx context.Context, f File, oldFile File) error
}

Handler provides a handler for Files.

type ID added in v0.17.0 

type ID int32

ID represents an ID of a file.

func (ID) String added in v0.17.0 

func (i ID) String() string

type ImageFile added in v0.17.0 

type ImageFile struct {
	*BaseFile
	Format string `json:"format"`
	Width  int    `json:"width"`
	Height int    `json:"height"`
}

ImageFile is an extension of BaseFile to represent image files.

type Mover added in v0.20.0 

type Mover struct {
	Renamer DirMakerStatRenamer
	Files   GetterUpdater
	Folders FolderStore
	// contains filtered or unexported fields
}

func NewMover added in v0.20.0 

func NewMover(fileStore GetterUpdater, folderStore FolderStore) *Mover

func (*Mover) CreateFolderHierarchy added in v0.20.0 

func (m *Mover) CreateFolderHierarchy(path string) error

func (*Mover) Move added in v0.20.0 

func (m *Mover) Move(ctx context.Context, f File, folder *Folder, basename string) error

Move moves the file to the given folder and basename. If basename is empty, then the existing basename is used. Assumes that the parent folder exists in the filesystem.

func (*Mover) RegisterHooks added in v0.20.0 

func (m *Mover) RegisterHooks(ctx context.Context, mgr txn.Manager)

type Opener added in v0.17.0 

type Opener interface {
	Open() (io.ReadCloser, error)
}

Opener provides an interface to open a file.

type OsFS added in v0.17.0 

type OsFS struct{}

OsFS is a file system backed by the OS.

func (*OsFS) Create added in v0.20.0 

func (f *OsFS) Create(name string) (*os.File, error)

func (*OsFS) IsPathCaseSensitive added in v0.17.2 

func (f *OsFS) IsPathCaseSensitive(path string) (bool, error)

func (*OsFS) Lstat added in v0.17.0 

func (f *OsFS) Lstat(name string) (fs.FileInfo, error)

func (*OsFS) MkdirAll added in v0.20.0 

func (f *OsFS) MkdirAll(path string, perm fs.FileMode) error

func (*OsFS) Open added in v0.17.0 

func (f *OsFS) Open(name string) (fs.ReadDirFile, error)

func (*OsFS) OpenZip added in v0.17.0 

func (f *OsFS) OpenZip(name string) (*ZipFS, error)

func (*OsFS) Remove added in v0.20.0 

func (f *OsFS) Remove(name string) error

func (*OsFS) RemoveAll added in v0.20.0 

func (f *OsFS) RemoveAll(path string) error

func (*OsFS) Rename added in v0.20.0 

func (f *OsFS) Rename(oldpath, newpath string) error

func (*OsFS) Stat added in v0.17.2 

func (f *OsFS) Stat(name string) (fs.FileInfo, error)

type PathFilter added in v0.17.0 

type PathFilter interface {
	Accept(ctx context.Context, path string, info fs.FileInfo) bool
}

PathFilter provides a filter function for paths.

type PathFilterFunc added in v0.17.0 

type PathFilterFunc func(path string) bool

func (PathFilterFunc) Accept added in v0.17.0 

func (pff PathFilterFunc) Accept(path string) bool

type ProgressReporter added in v0.17.0 

type ProgressReporter interface {
	AddTotal(total int)
	Increment()
	Definite()
	ExecuteTask(description string, fn func())
}

ProgressReporter is used to report progress of the scan.

type Renamer added in v0.20.0 

type Renamer interface {
	Rename(oldpath, newpath string) error
}

type RenamerRemover added in v0.12.0 

type RenamerRemover interface {
	Renamer
	Remove(name string) error
	RemoveAll(path string) error
	Statter
}

RenamerRemover provides access to the Rename and Remove functions.

type Repository added in v0.17.0 

type Repository struct {
	txn.Manager
	txn.DatabaseProvider
	Store

	FolderStore FolderStore
}

Repository provides access to storage methods for files and folders.

type ScanOptions added in v0.17.0 

type ScanOptions struct {
	Paths []string

	// ZipFileExtensions is a list of file extensions that are considered zip files.
	// Extension does not include the . character.
	ZipFileExtensions []string

	// ScanFilters are used to determine if a file should be scanned.
	ScanFilters []PathFilter

	// HandlerRequiredFilters are used to determine if an unchanged file needs to be handled
	HandlerRequiredFilters []Filter

	ParallelTasks int
}

ScanOptions provides options for scanning files.

type Scanner 

type Scanner struct {
	FS                    FS
	Repository            Repository
	FingerprintCalculator FingerprintCalculator

	// FileDecorators are applied to files as they are scanned.
	FileDecorators []Decorator
}

Scanner scans files into the database.

The scan process works using two goroutines. The first walks through the provided paths in the filesystem. It runs each directory entry through the provided ScanFilters. If none of the filter Accept methods return true, then the file/directory is ignored. Any folders found are handled immediately. Files inside zip files are also handled immediately. All other files encountered are sent to the second goroutine queue.

Folders are handled by checking if the folder exists in the database, by its full path. If a folder entry already exists, then its mod time is updated (if applicable). If the folder does not exist in the database, then a new folder entry its created.

Files are handled by first querying for the file by its path. If the file entry exists in the database, then the mod time is compared to the value in the database. If the mod time is different then file is marked as updated - it recalculates any fingerprints and fires decorators, then the file entry is updated and any applicable handlers are fired.

If the file entry does not exist in the database, then fingerprints are calculated for the file. It then determines if the file is a rename of an existing file by querying for file entries with the same fingerprint. If any are found, it checks each to see if any are missing in the file system. If one is, then the file is treated as renamed and its path is updated. If none are missing, or many are, then the file is treated as a new file.

If the file is not a renamed file, then the decorators are fired and the file is created, then the applicable handlers are fired.

func (*Scanner) Scan added in v0.17.0 

func (s *Scanner) Scan(ctx context.Context, handlers []Handler, options ScanOptions, progressReporter ProgressReporter)

Scan starts the scanning process.

type Statter added in v0.20.0 

type Statter interface {
	Stat(name string) (fs.FileInfo, error)
}

type Store added in v0.17.0 

type Store interface {
	Getter
	Counter
	Creator
	Updater
	Destroyer

	IsPrimary(ctx context.Context, fileID ID) (bool, error)
}

Store provides methods to find, create and update Files.

type Updater added in v0.17.0 

type Updater interface {
	Update(ctx context.Context, f File) error
}

Updater provides methods to update Files.

type VideoFile added in v0.17.0 

type VideoFile struct {
	*BaseFile
	Format     string  `json:"format"`
	Width      int     `json:"width"`
	Height     int     `json:"height"`
	Duration   float64 `json:"duration"`
	VideoCodec string  `json:"video_codec"`
	AudioCodec string  `json:"audio_codec"`
	FrameRate  float64 `json:"frame_rate"`
	BitRate    int64   `json:"bitrate"`

	Interactive      bool `json:"interactive"`
	InteractiveSpeed *int `json:"interactive_speed"`
}

VideoFile is an extension of BaseFile to represent video files.

func (VideoFile) GetMinResolution added in v0.17.0 

func (f VideoFile) GetMinResolution() int

type ZipDestroyer added in v0.17.0 

type ZipDestroyer struct {
	FileDestroyer   GetterDestroyer
	FolderDestroyer FolderGetterDestroyer
}

func (*ZipDestroyer) DestroyZip added in v0.17.0 

func (d *ZipDestroyer) DestroyZip(ctx context.Context, f File, fileDeleter *Deleter, deleteFile bool) error

type ZipFS added in v0.17.0 

type ZipFS struct {
	*zip.Reader
	// contains filtered or unexported fields
}

ZipFS is a file system backed by a zip file.

func (*ZipFS) Close added in v0.17.0 

func (f *ZipFS) Close() error

func (*ZipFS) IsPathCaseSensitive added in v0.17.2 

func (f *ZipFS) IsPathCaseSensitive(path string) (bool, error)

func (*ZipFS) Lstat added in v0.17.0 

func (f *ZipFS) Lstat(name string) (fs.FileInfo, error)

func (*ZipFS) Open added in v0.17.0 

func (f *ZipFS) Open(name string) (fs.ReadDirFile, error)

func (*ZipFS) OpenOnly added in v0.17.0 

func (f *ZipFS) OpenOnly(name string) (io.ReadCloser, error)

openOnly returns a ReadCloser where calling Close will close the zip fs as well.

func (*ZipFS) OpenZip added in v0.17.0 

func (f *ZipFS) OpenZip(name string) (*ZipFS, error)

func (*ZipFS) Stat added in v0.17.2 

func (f *ZipFS) Stat(name string) (fs.FileInfo, error)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.