backup

package
v1.6.14 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 4, 2024 License: AGPL-3.0 Imports: 18 Imported by: 0

Documentation

Overview

Package backup is providing commands to inspect a file system (hard-drive, USB, Android, S3) and backup medias to a remote DPhoto storage.

Index

Constants

View Source 
const (
	MediaTypeImage MediaType = "IMAGE"
	MediaTypeVideo MediaType = "VIDEO"
	MediaTypeOther MediaType = "OTHER"

	OrientationUpperLeft  ImageOrientation = "UPPER_LEFT"
	OrientationLowerRight ImageOrientation = "LOWER_RIGHT"
	OrientationUpperRight ImageOrientation = "UPPER_RIGHT"
	OrientationLowerLeft  ImageOrientation = "LOWER_LEFT"

	ProgressEventScanComplete      ProgressEventType = "scan-complete"       // ProgressEventScanComplete set the total of files
	ProgressEventAnalysed          ProgressEventType = "analysed"            // ProgressEventAnalysed is not useful for progress, it will be fined grained before upload
	ProgressEventAnalysedFromCache ProgressEventType = "analysed-from-cache" // ProgressEventAnalysedFromCache is sent instead of ProgressEventAnalysed when the analysis has been cached
	ProgressEventCatalogued        ProgressEventType = "catalogued"          // ProgressEventCatalogued is not useful for progress, it will be fined grained before upload
	ProgressEventWrongAlbum        ProgressEventType = "wrong-album"         // ProgressEventWrongAlbum count files in filtered out albums (if filter used), subtracted from ProgressEventScanComplete
	ProgressEventAlreadyExists     ProgressEventType = "duplicate-catalog"   // ProgressEventAlreadyExists count files already known in catalog, subtracted from ProgressEventScanComplete
	ProgressEventDuplicate         ProgressEventType = "duplicate-unique"    // ProgressEventDuplicate count files present twice in this backup/scan process, subtracted from ProgressEventScanComplete
	ProgressEventReadyForUpload    ProgressEventType = "upload-ready"        // ProgressEventReadyForUpload files remaining after analysis and filters: ProgressEventReadyForUpload = ProgressEventScanComplete - ProgressEventDuplicate - ProgressEventAlreadyExists - ProgressEventWrongAlbum
	ProgressEventUploaded          ProgressEventType = "uploaded"            // ProgressEventUploaded files uploaded, is equals to ProgressEventReadyForUpload when complete
	ProgressEventAlbumCreated      ProgressEventType = "album-created"       // ProgressEventAlbumCreated notify when a new album is created
)

Variables

View Source 
var (
	MediaCounterZero = MediaCounter{}
)
View Source 
var SupportedExtensions = map[string]MediaType{
	"jpg":  MediaTypeImage,
	"jpeg": MediaTypeImage,
	"png":  MediaTypeImage,
	"gif":  MediaTypeImage,
	"webp": MediaTypeImage,
	"raw":  MediaTypeImage,

	"svg": MediaTypeImage,
	"eps": MediaTypeImage,

	"mkv":  MediaTypeVideo,
	"mts":  MediaTypeVideo,
	"avi":  MediaTypeVideo,
	"mp4":  MediaTypeVideo,
	"mpeg": MediaTypeVideo,
	"mov":  MediaTypeVideo,
	"wmv":  MediaTypeVideo,
	"webm": MediaTypeVideo,
}

SupportedExtensions is used by SourceVolume adapters to filter files they find

Functions

func ClearDetailsReader added in v1.5.57 

func ClearDetailsReader()

func Init 

func Init(archive BArchiveAdapter, refFactory ReferencerFactory, insertMedia InsertMediaPort)

Init for scan or backup (but only refFactory is required for scan)

func RegisterDetailsReader 

func RegisterDetailsReader(reader DetailsReaderAdapter)

func Scan 

func Scan(owner string, volume SourceVolume, optionSlice ...Options) ([]*ScannedFolder, []FoundMedia, error)

Scan a source to discover albums based on original folder structure. Use listeners will be notified on the progress of the scan.

Types

type AlbumLookupPort added in v1.5.54 

type AlbumLookupPort interface {
	FindOrCreateAlbum(owner ownermodel.Owner, mediaTime time.Time) (folderName string, created bool, err error)
}

type AnalysedMedia 

type AnalysedMedia struct {
	FoundMedia FoundMedia    // FoundMedia is the reference of the file, implementation depends on the VolumeType
	Type       MediaType     // Type is 'photo' or 'video'
	Sha256Hash string        // Sha256Hash sha256 of the file
	Details    *MediaDetails // Details are data found within the file (location, date, ...)
}

AnalysedMedia is a FoundMedia to which has been attached its type (photo / video) and other details usually found within the file.

type AnalyserDecorator added in v1.5.0 

type AnalyserDecorator interface {
	Decorate(analyseFunc RunnerAnalyser) RunnerAnalyser
}

type BArchiveAdapter 

type BArchiveAdapter interface {
	// ArchiveMedia uploads the file in the right folder but might change the name to avoid clash with other existing files. Use files name is always returned.
	ArchiveMedia(owner string, media *BackingUpMediaRequest) (string, error)
}

type BackingUpMediaRequest 

type BackingUpMediaRequest struct {
	AnalysedMedia    *AnalysedMedia
	CatalogReference CatalogReference
}

BackingUpMediaRequest is the requests that must be executed to back up the media

type CatalogAdapter 

type CatalogAdapter interface {
	// IndexMedias add to the catalog following medias
	IndexMedias(owner string, requests []*CatalogMediaRequest) error
}

type CatalogMediaRequest 

type CatalogMediaRequest struct {
	BackingUpMediaRequest *BackingUpMediaRequest
	ArchiveFilename       string // ArchiveFilename is a normalised named generated and used in archive.
}

CatalogMediaRequest is the request passed to Archive domain

type CatalogReference added in v1.5.54 

type CatalogReference interface {
	// Exists returns true if the media exists in the catalog
	Exists() bool

	// AlbumCreated returns true if the album was created during the cataloger process
	AlbumCreated() bool

	// AlbumFolderName returns the name of the album where the media would be stored
	AlbumFolderName() string

	// UniqueIdentifier is identifying the media no matter its filename, its id in the catalog (if it's in it or not), its album, ... It's its signature.
	UniqueIdentifier() string

	// MediaId is the id of the media in the catalog and in the archive
	MediaId() string
}

CatalogReference is used to project where a media will fit in the catalog: its ID and its album.

type CatalogReferencer added in v1.5.54 

type CatalogReferencer interface {
	Reference(ctx context.Context, medias []*AnalysedMedia) (map[*AnalysedMedia]CatalogReference, error)
}

type Cataloger added in v1.5.54 

type Cataloger struct {
	CatalogReferencer CatalogReferencer
	CatalogerFilters  []CatalogerFilter
}

Cataloger returns a BackingUpMediaRequest with an album all the time: it will have been created if necessary.

func (*Cataloger) Catalog added in v1.5.54 

func (c *Cataloger) Catalog(ctx context.Context, medias []*AnalysedMedia, progressChannel chan *ProgressEvent) ([]*BackingUpMediaRequest, error)

type CatalogerFilter added in v1.5.54 

type CatalogerFilter interface {
	// FilterOut returns true if the media should be filtered out, with the cause of behind excluded
	FilterOut(media AnalysedMedia, reference CatalogReference) (ProgressEventType, bool)
}

type CatalogerFilterFunc added in v1.5.54 

type CatalogerFilterFunc func(media AnalysedMedia, reference CatalogReference) (ProgressEventType, bool)

func (CatalogerFilterFunc) FilterOut added in v1.5.54 

func (f CatalogerFilterFunc) FilterOut(media AnalysedMedia, reference CatalogReference) (ProgressEventType, bool)

type ClosableFoundMedia 

type ClosableFoundMedia interface {
	Close() error
}

ClosableFoundMedia can be implemented alongside FoundMedia if the implementation requires to release resources once the media has been handled.

type CompletionReport 

type CompletionReport interface {
	NewAlbums() []string
	Skipped() MediaCounter
	CountPerAlbum() map[string]*TypeCounter
}

func Backup 

func Backup(owner ownermodel.Owner, volume SourceVolume, optionsSlice ...Options) (CompletionReport, error)

Backup is analysing each media and is backing it up if not already in the catalog.

type DetailsReaderAdapter 

type DetailsReaderAdapter interface {
	// Supports returns true if the file can be parsed with this reader. False otherwise.
	Supports(media FoundMedia, mediaType MediaType) bool
	// ReadDetails extracts metadata from the content of the file.
	ReadDetails(reader io.Reader, options DetailsReaderOptions) (*MediaDetails, error)
}

type DetailsReaderOptions 

type DetailsReaderOptions struct {
	Fast bool // Fast true indicate the parser should focus at extracting the date, nothing else
}

type FoundMedia 

type FoundMedia interface {
	// MediaPath return breakdown of the absolute path of the media.
	MediaPath() MediaPath
	// ReadMedia reads content of the file ; it might not be optimised to call it several times (see VolumeToBackup)
	ReadMedia() (io.ReadCloser, error)
	// Size returns the size of the file
	Size() int

	LastModification() time.Time

	String() string
}

FoundMedia represents files found on the scanned volume

func NewInMemoryMedia added in v1.5.0 

func NewInMemoryMedia(name string, date time.Time, content []byte) FoundMedia

NewInMemoryMedia creates a new FoundMedia for TESTING PURPOSE ONLY

type FullMediaSignature 

type FullMediaSignature struct {
	Sha256 string
	Size   uint
}

FullMediaSignature is the business key of the media, unique per user

func (*FullMediaSignature) String 

func (s *FullMediaSignature) String() string

type ImageOrientation 

type ImageOrientation string

ImageOrientation is teh start point of stored data

type InMemoryMedia added in v1.5.0 

type InMemoryMedia struct {
	// contains filtered or unexported fields
}

func (*InMemoryMedia) LastModification added in v1.5.0 

func (i *InMemoryMedia) LastModification() time.Time

func (*InMemoryMedia) MediaPath added in v1.5.0 

func (i *InMemoryMedia) MediaPath() MediaPath

func (*InMemoryMedia) ReadMedia added in v1.5.0 

func (i *InMemoryMedia) ReadMedia() (io.ReadCloser, error)

func (*InMemoryMedia) Size added in v1.5.0 

func (i *InMemoryMedia) Size() int

func (*InMemoryMedia) String added in v1.5.0 

func (i *InMemoryMedia) String() string

type InsertMediaPort added in v1.5.57 

type InsertMediaPort interface {
	IndexMedias(ctx context.Context, owner ownermodel.Owner, requests []*CatalogMediaRequest) error
}

type MediaCounter 

type MediaCounter struct {
	Count int // Count is the number of medias
	Size  int // Size is the sum of the size of the medias
}

func NewMediaCounter 

func NewMediaCounter(count int, size int) MediaCounter

func (MediaCounter) Add 

func (c MediaCounter) Add(count int, size int) MediaCounter

Add creates a new MediaCounter with the delta applied ; initial MediaCounter is not updated.

func (MediaCounter) AddCounter 

func (c MediaCounter) AddCounter(counter MediaCounter) MediaCounter

AddCounter creates a new MediaCounter which is the sum of the 2 counters provided.

func (MediaCounter) IsZero 

func (c MediaCounter) IsZero() bool

IsZero returns true if it's the default value

type MediaDetails 

type MediaDetails struct {
	Width, Height             int
	DateTime                  time.Time
	Orientation               ImageOrientation
	Make                      string
	Model                     string
	GPSLatitude, GPSLongitude float64
	Duration                  int64  // Duration is the length, in milliseconds, of a video
	VideoEncoding             string // VideoEncoding is the codec used to encode the video (ex: 'H264')
}

func (*MediaDetails) String 

func (s *MediaDetails) String() string

type MediaPath 

type MediaPath struct {
	ParentFullPath string // ParentFullPath is the absolute path of the media folder (URL = ParentFullPath + Filename)
	Root           string // Root is the path or URL representing the volume in which the media has been found. (URL = Root + Path + Filename)
	Path           string // Path is the directory path relative to Root: URL = Root + Path + Filename
	Filename       string // Filename does not contain any slash, and contains the extension
	ParentDir      string // ParentDir is the name of the directory: dirname(Root + Path)
}

MediaPath is a breakdown of an absolute path, or URL, agnostic of its origin.

func (MediaPath) Absolute added in v1.5.0 

func (m MediaPath) Absolute() string

func (MediaPath) String added in v1.5.0 

func (m MediaPath) String() string

type MediaType 

type MediaType string

MediaType is photo or video

type NopeAnalyserDecorator added in v1.5.0 

type NopeAnalyserDecorator struct {
}

NopeAnalyserDecorator is a default implementation for AnalyserDecorator which doesn't decorate the AnalyseMediaFunc.

func (*NopeAnalyserDecorator) Decorate added in v1.5.0 

func (n *NopeAnalyserDecorator) Decorate(analyseFunc RunnerAnalyser) RunnerAnalyser

type Options 

type Options struct {
	RestrictedAlbumFolderName map[string]interface{} // RestrictedAlbumFolderName will restrict the media to only back up medias that are in one of these albums
	Listener                  interface{}            // Listener will receive progress events.
	SkipRejects               bool                   // SkipRejects mode will report any analysis error, or missing timestamp, and continue.
	AnalyserDecorator         AnalyserDecorator      // AnalyserDecorator is an optional decorator to add concept like caching (might be nil)
	DryRun                    bool                   // DryRun mode will not upload anything and do not create albums, still analyse
	ConcurrentAnalyser        int                    // ConcurrentAnalyser is the number of concurrent analyser (read files, compute hash, filter out duplicates, ...)
	ConcurrentCataloguer      int                    // ConcurrentCataloguer is the number of concurrent cataloguer (find album, create new albums)
	ConcurrentUploader        int                    // ConcurrentUploader is the number of concurrent uploader (upload files)
	BatchSize                 int                    // BatchSize is the number of items to read from the database at once (used by analyser) ; default to the maximum DynamoDB can handle
}

func OptionOnlyAlbums 

func OptionOnlyAlbums(albums ...string) Options

OptionOnlyAlbums restricts backed up medias to those in these albums

func OptionSkipRejects 

func OptionSkipRejects(skip bool) Options

OptionSkipRejects disables the strict mode and ignores invalid files (wrong / no date, ...)

func OptionWithListener 

func OptionWithListener(listener interface{}) Options

OptionWithListener creates an option with a listener

func WithBatchSize added in v1.6.13 

func WithBatchSize(batchSize int) Options

func WithConcurrentAnalyser added in v1.6.13 

func WithConcurrentAnalyser(concurrent int) Options

func WithConcurrentCataloguer added in v1.6.13 

func WithConcurrentCataloguer(concurrent int) Options

func WithConcurrentUploader added in v1.6.13 

func WithConcurrentUploader(concurrent int) Options

func (Options) GetAnalyserDecorator added in v1.5.0 

func (o Options) GetAnalyserDecorator() AnalyserDecorator

GetAnalyserDecorator is returning the AnalyserDecorator or NopeAnalyserDecorator, never nil.

func (Options) WithCachedAnalysis added in v1.5.0 

func (o Options) WithCachedAnalysis(analyserDecorator AnalyserDecorator) Options

WithCachedAnalysis adds a decorator on analysis function ; argument can be nil.

type PostAnalyseFilter 

type PostAnalyseFilter interface {
	// AcceptAnalysedMedia returns TRUE if the media should be backed-up.
	AcceptAnalysedMedia(media *AnalysedMedia, folderName string) bool
}

type ProgressEvent 

type ProgressEvent struct {
	Type      ProgressEventType // Type defines what's count, and size are about ; some might not be used.
	Count     int               // Count is the number of media
	Size      int               // Size is the sum of the size of the media concerned by this event
	Album     string            // Album is the folder name of the medias concerned by this event
	MediaType MediaType         // MediaType is the type of media ; only mandatory with 'uploaded' event
}

type ProgressEventType 

type ProgressEventType string

type ReferencerFactory added in v1.5.54 

type ReferencerFactory interface {
	// NewCreatorReferencer returns a Referencer that will create the album if the date is not yet covered.
	NewCreatorReferencer(ctx context.Context, owner ownermodel.Owner) (CatalogReferencer, error)
	// NewDryRunReferencer returns a Referencer that will not create any album.
	NewDryRunReferencer(ctx context.Context, owner ownermodel.Owner) (CatalogReferencer, error)
}

type RunnerAnalyser added in v1.5.20 

type RunnerAnalyser interface {
	Analyse(found FoundMedia, progressChannel chan *ProgressEvent) (*AnalysedMedia, error)
}

type RunnerAnalyserFunc added in v1.5.0 

type RunnerAnalyserFunc func(found FoundMedia, progressChannel chan *ProgressEvent) (*AnalysedMedia, error)

func (RunnerAnalyserFunc) Analyse added in v1.5.20 

func (r RunnerAnalyserFunc) Analyse(found FoundMedia, progressChannel chan *ProgressEvent) (*AnalysedMedia, error)

type RunnerCataloger added in v1.5.54 

type RunnerCataloger interface {
	Catalog(ctx context.Context, medias []*AnalysedMedia, progressChannel chan *ProgressEvent) ([]*BackingUpMediaRequest, error)
}

func NewCataloger added in v1.5.54 

func NewCataloger(owner ownermodel.Owner, options Options) (RunnerCataloger, error)

type RunnerCatalogerFunc added in v1.5.54 

type RunnerCatalogerFunc func(ctx context.Context, medias []*AnalysedMedia, progressChannel chan *ProgressEvent) ([]*BackingUpMediaRequest, error)

func (RunnerCatalogerFunc) Catalog added in v1.5.54 

func (r RunnerCatalogerFunc) Catalog(ctx context.Context, medias []*AnalysedMedia, progressChannel chan *ProgressEvent) ([]*BackingUpMediaRequest, error)

type RunnerUploader added in v1.5.57 

type RunnerUploader interface {
	Upload(buffer []*BackingUpMediaRequest, progressChannel chan *ProgressEvent) error
}

type RunnerUploaderFunc added in v1.5.57 

type RunnerUploaderFunc func(buffer []*BackingUpMediaRequest, progressChannel chan *ProgressEvent) error

func (RunnerUploaderFunc) Upload added in v1.5.57 

func (r RunnerUploaderFunc) Upload(buffer []*BackingUpMediaRequest, progressChannel chan *ProgressEvent) error

type ScannedFolder 

type ScannedFolder struct {
	Name         string
	RelativePath string                  // RelativePath can be used for display purpose
	FolderName   string                  // FolderName is the original folder name (Name with date prefix that have been removed)
	AbsolutePath string                  // AbsolutePath is used to create a new SourceVolume
	Start, End   time.Time               // Start and End are the beginning of the day of the first media, and the beginning of the day following the last media.
	Distribution map[string]MediaCounter // Distribution is the number of media found for each day (format YYYY-MM-DD)
}

ScannedFolder represents a (sub)folder in the scanned target

func (*ScannedFolder) PushBoundaries 

func (f *ScannedFolder) PushBoundaries(date time.Time, size int)

PushBoundaries is updating the ScannedFolder dates, and update the counter.

type SourceVolume 

type SourceVolume interface {
	String() string
	FindMedias() ([]FoundMedia, error)
}

type TimelineAdapter 

type TimelineAdapter interface {
	FindOrCreateAlbum(mediaTime time.Time) (folderName string, created bool, err error)
	FindAlbum(dateTime time.Time) (folderName string, exists bool, err error)
}

type TrackAnalysed 

type TrackAnalysed interface {
	OnAnalysed(done, total, cached MediaCounter)
}

type TrackEvents 

type TrackEvents interface {
	OnEvent(event ProgressEvent)
}

type TrackScanComplete 

type TrackScanComplete interface {
	OnScanComplete(total MediaCounter)
}

type TrackUploaded 

type TrackUploaded interface {
	OnUploaded(done, total MediaCounter)
}

TrackUploaded includes both uploaded and skipped

type Tracker 

type Tracker struct {
	Done chan struct{} // Done is closed when all events have been processed.
	// contains filtered or unexported fields
}

Tracker is consuming progress channel, keep a record of counts, and call listeners

func NewTracker 

func NewTracker(progressChannel chan *ProgressEvent, listeners ...interface{}) *Tracker

NewTracker creates the Tracker and start consuming (async)

func (*Tracker) CountPerAlbum 

func (t *Tracker) CountPerAlbum() map[string]*TypeCounter

func (*Tracker) MediaCount added in v1.6.13 

func (t *Tracker) MediaCount() int

func (*Tracker) NewAlbums 

func (t *Tracker) NewAlbums() []string

func (*Tracker) Skipped 

func (t *Tracker) Skipped() MediaCounter

func (*Tracker) WaitToComplete 

func (t *Tracker) WaitToComplete()

type TypeCounter 

type TypeCounter struct {
	// contains filtered or unexported fields
}

func NewTypeCounter 

func NewTypeCounter(mediaType MediaType, count int, size int) *TypeCounter

NewTypeCounter is a convenience method for testing or mocking 'backup' domain

func (*TypeCounter) GetMediaIndex 

func (c *TypeCounter) GetMediaIndex(mediaType MediaType) int

func (*TypeCounter) IncrementCounter 

func (c *TypeCounter) IncrementCounter(counter *[numberOfMediaType]int, mediaType MediaType, delta int)

func (*TypeCounter) IncrementFoundCounter 

func (c *TypeCounter) IncrementFoundCounter(mediaType MediaType, count int, size int) *TypeCounter

func (*TypeCounter) OfType 

func (c *TypeCounter) OfType(mediaType MediaType) MediaCounter

func (*TypeCounter) Total 

func (c *TypeCounter) Total() MediaCounter

type Uploader added in v1.5.57 

type Uploader struct {
	Owner           ownermodel.Owner
	InsertMediaPort InsertMediaPort
}

func (*Uploader) Upload added in v1.5.57 

func (u *Uploader) Upload(buffer []*BackingUpMediaRequest, progressChannel chan *ProgressEvent) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.