dejavu

package module
v0.0.0-...-e418aab Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 21, 2024 License: AGPL-3.0 Imports: 29 Imported by: 2

README

DejaVu

中文

💡 Introduction

DejaVu is the component of data snapshot and sync for SiYuan.

✨ Features

  • Git-like version control
  • File deduplication in chunks
  • Data compression
  • AES Encrypted
  • Cloud sync and backup

⚠️ Attention

  • Folders are not supported
  • Permission attributes are not supported
  • Symbolic links are not supported

🎨 Design

Design reference from ArtiVC.

Entity
  • ID Each entity is identified by SHA-1
  • Index file list, each index operation generates a new index
    • memo index memo
    • created index time
    • files file list
    • count count of total files
    • size size of total files
  • File file, a new file is generated when the actual data file path or content changes
    • path file path
    • size file size
    • updated last update time
    • chunks file chunk list
  • Chunk file chunk
    • data actual data
  • Ref refers to the index
    • latest built-in reference, automatically points to the latest index
    • tag tag reference, manually point to the specified index
  • Repo repository
Repo
  • DataPath data folder path, the folder where the actual data file is located
  • Path repo folder path, the repo is not stored in the data folder, we need to specify the repo folder path separately

The repo folder layout is as follows:

├─indexes
│      0531732dca85404e716abd6bb896319a41fa372b
│      19fc2c2e5317b86f9e048f8d8da2e4ed8300d8af
│      5f32d78d69e314beee36ad7de302b984da47ddd2
│      cbd254ca246498978d4f47e535bac87ad7640fe6
│
├─objects
│  ├─1e
│  │      0ac5f319f5f24b3fe5bf63639e8dbc31a52e3b
│  │
│  ├─56
│  │      322ccdb61feab7f2f76f5eb82006bd51da7348
│  │
│  ├─7e
│  │      dccca8340ebe149b10660a079f34a20f35c4d4
│  │
│  ├─83
│  │      a7d72fe9a071b696fc81a3dc041cf36cbde802
│  │
│  ├─85
│  │      26b9a7efde615b67b4666ae509f9fbc91d370b
│  │
│  ├─87
│  │      1355acd062116d1713e8f7f55969dbb507a040
│  │
│  ├─96
│  │      46ba13a4e8eabeca4f5259bfd7da41d368a1a6
│  │
│  ├─a5
│  │      5b8e6b9ccad3fc9b792d3d453a0793f8635b9f
│  │      b28787922f4e2a477b4f027e132aa7e35253d4
│  │
│  ├─be
│  │      c7a729d1b5f021f8eca0dd8b6ef689ad753567
│  │
│  ├─d1
│  │      324c714bde18442b5629a84a361b5e7528b14a
│  │
│  ├─f1
│  │      d7229171f4fa1c5eacb411995b16938a04f7f6
│  │
│  └─f7
│          ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0
│
└─refs
    │  latest
    │
    └─tags
            v1.0.0
            v1.0.1

📄 License

DejaVu uses the GNU AFFERO GENERAL PUBLIC LICENSE, Version 3 open source license.

🙏 Acknowledgement

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrRepoFatal  = errors.New("repo fatal error")
	ErrEmptyIndex = errors.New("empty index")
	// ErrIndexFileChanged indicates that the file has changed during the index process.
	// Improve data snapshot and sync robustness https://github.com/siyuan-note/siyuan/issues/9941
	ErrIndexFileChanged = errors.New("file changed")
)
View Source 
var (
	ErrCloudStorageSizeExceeded = errors.New("cloud storage limit size exceeded")
	ErrCloudBackupCountExceeded = errors.New("cloud backup count exceeded")

	ErrCloudGenerateConflictHistory = errors.New("generate conflict history failed")
)
View Source 
var (
	ErrLockCloudFailed = errors.New("lock cloud repo failed")
	ErrCloudLocked     = errors.New("cloud repo is locked")
)
View Source 
var ErrNotFoundIndex = errors.New("not found index")
View Source 
var ErrNotFoundObject = errors.New("not found object")

Functions

This section is empty.

Types

type APITrafficStat 

type APITrafficStat struct {
	APIGet int
	APIPut int
}

type DownloadTrafficStat 

type DownloadTrafficStat struct {
	DownloadFileCount  int
	DownloadChunkCount int
	DownloadBytes      int64
}

type LeftRightDiff 

type LeftRightDiff struct {
	LeftIndex    *entity.Index
	RightIndex   *entity.Index
	AddsLeft     []*entity.File
	UpdatesLeft  []*entity.File
	UpdatesRight []*entity.File
	RemovesRight []*entity.File
}

type Log 

type Log struct {
	ID          string         `json:"id"`          // 索引 ID
	Memo        string         `json:"memo"`        // 索引备注
	Created     int64          `json:"created"`     // 索引时间
	HCreated    string         `json:"hCreated"`    // 索引时间 "2006-01-02 15:04:05"
	Files       []*entity.File `json:"files"`       // 文件列表
	Count       int            `json:"count"`       // 文件总数
	Size        int64          `json:"size"`        // 文件总大小
	HSize       string         `json:"hSize"`       // 格式化好的文件总大小 "10.00 MB"
	SystemID    string         `json:"systemID"`    // 设备 ID
	SystemName  string         `json:"systemName"`  // 设备名称
	SystemOS    string         `json:"systemOS"`    // 设备操作系统
	Tag         string         `json:"tag"`         // 索引标记名称
	HTagUpdated string         `json:"hTagUpdated"` // 标记时间 "2006-01-02 15:04:05"
}

func (*Log) String 

func (log *Log) String() string

type MergeResult 

type MergeResult struct {
	Time                        time.Time
	Upserts, Removes, Conflicts []*entity.File
}

func (*MergeResult) DataChanged 

func (mr *MergeResult) DataChanged() bool

type Repo 

type Repo struct {
	DataPath    string   // 数据文件夹的绝对路径，如：F:\\SiYuan\\data\\
	Path        string   // 仓库的绝对路径，如：F:\\SiYuan\\repo\\
	HistoryPath string   // 数据历史文件夹的绝对路径，如：F:\\SiYuan\\history\\
	TempPath    string   // 临时文件夹的绝对路径，如：F:\\SiYuan\\temp\\
	DeviceID    string   // 设备 ID
	DeviceName  string   // 设备名称
	DeviceOS    string   // 操作系统
	IgnoreLines []string // 忽略配置文件内容行，是用 .gitignore 语法
	// contains filtered or unexported fields
}

Repo 描述了逮虾户数据仓库。

func NewRepo 

func NewRepo(dataPath, repoPath, historyPath, tempPath, deviceID, deviceName, deviceOS string, aesKey []byte, ignoreLines []string, cloud cloud.Cloud) (ret *Repo, err error)

NewRepo 创建一个新的仓库。

func (*Repo) AddTag 

func (repo *Repo) AddTag(id, tag string) (err error)

func (*Repo) Checkout 

func (repo *Repo) Checkout(id string, context map[string]interface{}) (upserts, removes []*entity.File, err error)

Checkout 将仓库中的数据迁出到 repo 数据文件夹下。context 参数用于发布事件时传递调用上下文。

func (*Repo) CheckoutFilesFromCloud 

func (repo *Repo) CheckoutFilesFromCloud(files []*entity.File, context map[string]interface{}) (stat *DownloadTrafficStat, err error)

func (*Repo) CountIndexes 

func (repo *Repo) CountIndexes() (ret int, err error)

func (*Repo) CreateCloudRepo 

func (repo *Repo) CreateCloudRepo(name string) (err error)

func (*Repo) DiffIndex 

func (repo *Repo) DiffIndex(leftIndexID, rightIndexID string) (ret *LeftRightDiff, err error)

DiffIndex 返回索引 left 比索引 right 新增、更新和删除的文件列表。

func (*Repo) DownloadIndex 

func (repo *Repo) DownloadIndex(id string, context map[string]interface{}) (downloadFileCount, downloadChunkCount int, downloadBytes int64, err error)

func (*Repo) DownloadTagIndex 

func (repo *Repo) DownloadTagIndex(tag, id string, context map[string]interface{}) (downloadFileCount, downloadChunkCount int, downloadBytes int64, err error)

func (*Repo) GetCloudAvailableSize 

func (repo *Repo) GetCloudAvailableSize() (ret int64)

func (*Repo) GetCloudLatest 

func (repo *Repo) GetCloudLatest(context map[string]interface{}) (cloudLatest *entity.Index, err error)

func (*Repo) GetCloudRepoLogs 

func (repo *Repo) GetCloudRepoLogs(page int) (ret []*Log, pageCount, totalCount int, err error)

func (*Repo) GetCloudRepoStat 

func (repo *Repo) GetCloudRepoStat() (stat *cloud.Stat, err error)

func (*Repo) GetCloudRepoTagLogs 

func (repo *Repo) GetCloudRepoTagLogs(context map[string]interface{}) (ret []*Log, err error)

func (*Repo) GetCloudRepos 

func (repo *Repo) GetCloudRepos() (repos []*cloud.Repo, size int64, err error)

func (*Repo) GetFile 

func (repo *Repo) GetFile(fileID string) (ret *entity.File, err error)

func (*Repo) GetFiles 

func (repo *Repo) GetFiles(index *entity.Index) (ret []*entity.File, err error)

GetFiles 返回快照索引 index 中的文件列表。

func (*Repo) GetIndex 

func (repo *Repo) GetIndex(id string) (index *entity.Index, err error)

GetIndex 从仓库根据 id 获取索引。

func (*Repo) GetIndexLogs 

func (repo *Repo) GetIndexLogs(page, pageSize int) (ret []*Log, pageCount, totalCount int, err error)

func (*Repo) GetIndexes 

func (repo *Repo) GetIndexes(page, pageSize int) (ret []*entity.Index, totalCount, pageCount int, err error)

func (*Repo) GetSyncCloudFiles 

func (repo *Repo) GetSyncCloudFiles(cloudLatest *entity.Index, context map[string]interface{}) (fetchedFiles []*entity.File, err error)

func (*Repo) GetTag 

func (repo *Repo) GetTag(tag string) (id string, err error)

func (*Repo) GetTagLogs 

func (repo *Repo) GetTagLogs() (ret []*Log, err error)

func (*Repo) Index 

func (repo *Repo) Index(memo string, context map[string]interface{}) (ret *entity.Index, err error)

Index 将 repo 数据文件夹中的文件索引到仓库中。context 参数用于发布事件时传递调用上下文。

func (*Repo) Latest 

func (repo *Repo) Latest() (ret *entity.Index, err error)

func (*Repo) OpenFile 

func (repo *Repo) OpenFile(file *entity.File) (ret []byte, err error)

func (*Repo) Purge 

func (repo *Repo) Purge(retentionIndexIDs ...string) (ret *entity.PurgeStat, err error)

Purge 清理所有未引用数据，retentionIndexIDs 为保留的索引 ID 列表，如果不传入的话则清理所有未引用数据。

func (*Repo) PurgeCloud 

func (repo *Repo) PurgeCloud() (ret *entity.PurgeStat, err error)

PurgeCloud 清理云端所有未引用数据。 Support manual purge of unreferenced data snapshots in the S3/WebDAV cloud storage https://github.com/siyuan-note/siyuan/issues/10081

func (*Repo) PutIndex 

func (repo *Repo) PutIndex(index *entity.Index) (err error)

PutIndex 将索引 index 写入仓库。

func (*Repo) RemoveCloudRepo 

func (repo *Repo) RemoveCloudRepo(name string) (err error)

func (*Repo) RemoveCloudRepoTag 

func (repo *Repo) RemoveCloudRepoTag(tag string) (err error)

func (*Repo) RemoveTag 

func (repo *Repo) RemoveTag(tag string) (err error)

func (*Repo) Reset 

func (repo *Repo) Reset() (err error)

Reset 重置仓库，清空所有数据。

func (*Repo) Sync 

func (repo *Repo) Sync(context map[string]interface{}) (mergeResult *MergeResult, trafficStat *TrafficStat, err error)

func (*Repo) SyncDownload 

func (repo *Repo) SyncDownload(context map[string]interface{}) (mergeResult *MergeResult, trafficStat *TrafficStat, err error)

func (*Repo) SyncUpload 

func (repo *Repo) SyncUpload(context map[string]interface{}) (trafficStat *TrafficStat, err error)

func (*Repo) UpdateLatest 

func (repo *Repo) UpdateLatest(index *entity.Index) (err error)

func (*Repo) UpdateLatestSync 

func (repo *Repo) UpdateLatestSync(index *entity.Index) (err error)

func (*Repo) UploadTagIndex 

func (repo *Repo) UploadTagIndex(tag, id string, context map[string]interface{}) (uploadFileCount, uploadChunkCount int, uploadBytes int64, err error)

type Store 

type Store struct {
	Path   string // 存储库文件夹的绝对路径，如：F:\\SiYuan\\repo\\
	AesKey []byte
	// contains filtered or unexported fields
}

Store 描述了存储库。

func NewStore 

func NewStore(path string, aesKey []byte) (ret *Store, err error)

func (*Store) AbsPath 

func (store *Store) AbsPath(id string) (dir, file string)

func (*Store) GetChunk 

func (store *Store) GetChunk(id string) (ret *entity.Chunk, err error)

func (*Store) GetFile 

func (store *Store) GetFile(id string) (ret *entity.File, err error)

func (*Store) GetIndex 

func (store *Store) GetIndex(id string) (ret *entity.Index, err error)

func (*Store) IndexAbsPath 

func (store *Store) IndexAbsPath(id string) (dir, file string)

func (*Store) Purge 

func (store *Store) Purge(retentionIndexIDs ...string) (ret *entity.PurgeStat, err error)

func (*Store) PutChunk 

func (store *Store) PutChunk(chunk *entity.Chunk) (err error)

func (*Store) PutFile 

func (store *Store) PutFile(file *entity.File) (err error)

func (*Store) PutIndex 

func (store *Store) PutIndex(index *entity.Index) (err error)

func (*Store) Remove 

func (store *Store) Remove(id string) (err error)

func (*Store) Stat 

func (store *Store) Stat(id string) (stat os.FileInfo, err error)

type TrafficStat 

type TrafficStat struct {
	DownloadTrafficStat
	UploadTrafficStat
	APITrafficStat
	// contains filtered or unexported fields
}

type UploadTrafficStat 

type UploadTrafficStat struct {
	UploadFileCount  int
	UploadChunkCount int
	UploadBytes      int64
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.