diskmgr

package module
v0.0.0-...-d1719c8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 10, 2022 License: Apache-2.0 Imports: 18 Imported by: 0

README

diskmgr

DiskMgr manages the files and space on the disk.

Modules

It consists of the following modules:

  • sqlite db: record cache file info and download file info
  • local cache: cache file info and header info
  • folderMgr: manages folders and disk space
  • downloadMgr: download files
Provide folder

You can set several folders(at least one folder) for file storage. Files will be saved in these folders. If you have multiple folders, files will be stored in these folders proportionally according to the size of the folders.

Background jobs

There are some background jobs: 1 scan leak files: scan disk and delete file which on disk but not in db. It will run every 12 hours, and sleep 5 seconds after scan 100 files.

2 delete empty folder: check path(which from a channel) is empty folder or not, and delete empty folder. A file path will be push into a channel when the file is deleted, this bgjob will check these path and delete empty folder

3 retry download notify: onDownloadSuccess or onDownloadFailed will be called when download finish(success or failed) to notify server. If notify failed, this job will retry the notification till server receive the message.

Interface

The main interface of diskMgr is SaveFile, DeleteFile, and GetFile

GetSpaceInfo will used to monitor space usage.

GetFilesBeforeAccessTime and GetFilesByAccessTime is used to get files which have not been accessed for a long time.

static function tool

diskmgr.CheckFolder is a static tool to check folder size. This function will generate the space holder files to make sure you have enough permissions and space. Those files will be deleted after check.

Example


package main

import (
	"fmt"
	"log"
	"time"

	"github.com/meson-network/diskMgr"
	"github.com/meson-network/diskMgr/folderMgr"
	"github.com/meson-network/downloadMgr"
	"github.com/coreservice-io/LogrusULog"
	"github.com/coreservice-io/ULog"
)

func main() {
	//logger is needed
	logger, _ := LogrusULog.New("/Users/bruce/workspace/go/project/diskmgr/testfolder/logs", 2, 20, 30)
	logger.SetLevel(ULog.DebugLevel)

	//some callback
	onSuccess := func(nameHash string, size int64) error {
		logger.Debugln("success", "name:", nameHash, "size:", size)
		return nil
	}
	onFail := func(nameHash string, failReason downloadmgr.FailReasonType) error {
		logger.Debugln("fail", "name:", nameHash)
		switch failReason {
		case downloadmgr.Fail_NoSpace:
			logger.Debugln("fail reason", "no space")
		case downloadmgr.Fail_Other, downloadmgr.Fail_RequestError:
			logger.Debugln("fail reason", "request error")
		case downloadmgr.Fail_SizeLimit:
			logger.Debugln("fail reason", "file too big")
		default:
			logger.Debugln("fail reason", "other")

		}
		return nil
	}
	onCancel := func(task *downloadmgr.Task) error {
		logger.Debugln("cancel", task.Id, task.NameHash)
		return nil
	}
	onDownloading := func(task *downloadmgr.Task) error {
		logger.Debugln("downloading", task.Id, task.NameHash)
		return nil
	}
	onLowSpeed := func(task *downloadmgr.Task) error {
		logger.Debugln("lowSpeed", task.Id, task.NameHash)
		return nil
	}

	onCachedFileDeleted := func(deletedFile []string) error {
		if len(deletedFile) <= 0 {
			return nil
		}
		logger.Debugln("deletedCachedFile:")
		for _, v := range deletedFile {
			logger.Debugln(v)
		}
		return nil
	}
	onDownloadingFileDeleted := func(deletedFile []string) error {
		if len(deletedFile) <= 0 {
			return nil
		}
		logger.Debugln("deletedDownloadingFile:")
		for _, v := range deletedFile {
			logger.Debugln(v)
		}
		return nil
	}
	onFileMissing := func(missingFile []string) error {
		if len(missingFile) <= 0 {
			return nil
		}
		logger.Debugln("missingFile:")
		for _, v := range missingFile {
			logger.Debugln(v)
		}
		return nil
	}

	//provide folder
	//form config file in real project
	folderInfo := []folderMgr.FolderConfig{
		{"/Users/bruce/workspace/go/project/diskmgr/testfolder/1", 4},
		{"/Users/bruce/workspace/go/project/diskmgr/testfolder/2", 4},
	}

	for _, v := range folderInfo {
		err := diskmgr.CheckFolder(v.AbsPath, v.SizeGB, 40, 4)
		if err != nil {
			logger.Fatalln(err)
		}
	}

        //new diskMgr
	diskMgr := diskmgr.New(
		logger,
		func(panic interface{}) {
			logger.Errorln(panic)
		},
	)
	err := diskMgr.StartUp(
		folderInfo,
		"/Users/bruce/workspace/go/project/diskmgr/testfolder",
		onSuccess,
		onFail,
		onCancel,
		onDownloading,
		onLowSpeed,
		onCachedFileDeleted,
		onDownloadingFileDeleted,
		onFileMissing,
	)
	if err != nil {
		logger.Fatalln(err)
	}
  
	//get space info
	log.Println(diskMgr.GetSpaceInfo())

	//cache some files
	for i := 0; i < 10; i++ {
		nameHash := fmt.Sprintf("123456789012%04d", i)
		err := diskMgr.CacheFile(
			nameHash,
			[]string{
	            "https://images-na.ssl-images-amazon.com/images/I/41eXG7xY09L._SY291_BO1,204,203,200_QL40_FMwebp_.jpg",
				},
			false,
			500*1024*1024)
		if err != nil {
			log.Println(err)
		}
	}

	//try to get some files
	go func() {
		files, _ := diskMgr.GetFilesByAccessTime(200)
		for _, v := range files {
			err := diskMgr.DeleteFile([]string{v.NameHash})
			if err != nil {
				log.Println("delete file err:", err)
			}
		}
	}()

	time.Sleep(1 * time.Hour)
}

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func CheckFolder 

func CheckFolder(folderAbsPath string, sizeGB int, checkLimitGB int, bottomSizeGB int) error

CheckFolder check folder operation auth and folder space

Types

type CachedFileWithHeader 

type CachedFileWithHeader struct {
	*model.CachedFile
	Header map[string][]string
}

type DiskFile 

type DiskFile struct {
	// contains filtered or unexported fields
}

type DiskMgr 

type DiskMgr struct {
	// contains filtered or unexported fields
}

func New 

func New(logger ULog.Logger, onPanic func(panic interface{})) *DiskMgr

func (*DiskMgr) CacheFile 

func (dm *DiskMgr) CacheFile(nameHash string, originUrl []string, encrypted bool, sizeLimit int64) error

SaveFile add normal download task,if return err, err will be diskMgrErr.ErrNameHashError,diskMgrErr.ErrTaskExist, diskMgrErr.ErrDiskNoSpace,diskMgrErr.ErrOriginUrlProtocol

func (*DiskMgr) CacheTempFile 

func (dm *DiskMgr) CacheTempFile(nameHash string, originUrl []string, encrypted bool, sizeLimit int64, downloadTimeLimit int64) error

SaveTempFile add quick download task,if return err, err will be diskMgrErr.ErrNameHashError,diskMgrErr.ErrTaskExist, diskMgrErr.ErrDiskNoSpace,diskMgrErr.ErrOriginUrlProtocol

func (*DiskMgr) DeleteFile 

func (dm *DiskMgr) DeleteFile(nameHash []string) error

func (*DiskMgr) GetFile 

func (dm *DiskMgr) GetFile(nameHash string) (fileAbsPath string, fileInfo *CachedFileWithHeader, err error)

func (*DiskMgr) GetFilesBeforeAccessTime 

func (dm *DiskMgr) GetFilesBeforeAccessTime(timeStamp int64, limit int, offset int) ([]*model.CachedFile, error)

GetFilesBeforeAccessTime get no access files since limit line

func (*DiskMgr) GetFilesByAccessTime 

func (dm *DiskMgr) GetFilesByAccessTime(limit int) ([]*model.CachedFile, error)

GetFilesByAccessTime get some no access files order by last access time.

func (*DiskMgr) GetSpaceInfo 

func (dm *DiskMgr) GetSpaceInfo() (total int64, used int64, free int64)

func (*DiskMgr) StartUp 

func (dm *DiskMgr) StartUp(
	folderConfig []folderMgr.FolderConfig,
	absDbFolder string,

	onDownloadSuccess TerminalOnDownloadSuccess,
	onDownloadFailed TerminalOnDownloadFailed,
	onDownloadCanceled TerminalOnDownloadCanceled,
	onDownloading TerminalOnDownloading,
	onDownloadSlowSpeed TerminalOnDownloadSlowSpeed,

	onCachedFileDeleted TerminalOnCachedFileDeleted,
	onDownloadingFileDeleted TerminalOnDownloadingFileDeleted,
	onFileMissing TerminalOnFileMissing,
) error

type IDiskMgr 

type IDiskMgr interface {
	StartUp(
		folderConfig []folderMgr.FolderConfig,
		absDbFolder string,

		onDownloadSuccess TerminalOnDownloadSuccess,
		onDownloadFailed TerminalOnDownloadFailed,
		onDownloadCanceled TerminalOnDownloadCanceled,
		onDownloading TerminalOnDownloading,
		onDownloadSlowSpeed TerminalOnDownloadSlowSpeed,

		onCachedFileDeleted TerminalOnCachedFileDeleted,
		onDownloadingFileDeleted TerminalOnDownloadingFileDeleted,
		onFileMissing TerminalOnFileMissing,
	) error

	CacheTempFile(nameHash string, originUrl []string, needEncrypt bool, sizeLimit int64, downloadTimeLimit int64) error
	CacheFile(nameHash string, originUrl []string, needEncrypt bool, sizeLimit int64) error
	DeleteFile(nameHash []string) error

	GetFile(nameHash string) (fileAbsPath string, fileInfo *CachedFileWithHeader, err error)

	GetSpaceInfo() (total int64, used int64, free int64)
	GetFilesBeforeAccessTime(timeStamp int64, limit int, offset int) ([]*model.CachedFile, error)
	GetFilesByAccessTime(limit int) ([]*model.CachedFile, error)
}

type TerminalOnCachedFileDeleted 

type TerminalOnCachedFileDeleted = func(deletedFile []string) error

type TerminalOnDownloadCanceled 

type TerminalOnDownloadCanceled = func(task *downloadmgr.Task) error

type TerminalOnDownloadFailed 

type TerminalOnDownloadFailed = func(nameHash string, failReason downloadmgr.FailReasonType) error

type TerminalOnDownloadSlowSpeed 

type TerminalOnDownloadSlowSpeed = func(task *downloadmgr.Task) error

type TerminalOnDownloadSuccess 

type TerminalOnDownloadSuccess = func(nameHash string, fileSize int64) error

type TerminalOnDownloading 

type TerminalOnDownloading = func(task *downloadmgr.Task) error

type TerminalOnDownloadingFileDeleted 

type TerminalOnDownloadingFileDeleted = func(deletedFile []string) error

type TerminalOnFileMissing 

type TerminalOnFileMissing = func(missingFile []string) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.