fsutil

README

Incremental file directory sync tools in golang.

BENCH_FILE_SIZE=10000 ./bench.test --test.bench .
BenchmarkCopyWithTar10-4                	    2000	    995242 ns/op
BenchmarkCopyWithTar50-4                	     300	   4710021 ns/op
BenchmarkCopyWithTar200-4               	     100	  16627260 ns/op
BenchmarkCopyWithTar1000-4              	      20	  60031459 ns/op
BenchmarkCPA10-4                        	    1000	   1678367 ns/op
BenchmarkCPA50-4                        	     500	   3690306 ns/op
BenchmarkCPA200-4                       	     200	   9495066 ns/op
BenchmarkCPA1000-4                      	      50	  29769289 ns/op
BenchmarkDiffCopy10-4                   	    2000	    943889 ns/op
BenchmarkDiffCopy50-4                   	     500	   3285950 ns/op
BenchmarkDiffCopy200-4                  	     200	   8563792 ns/op
BenchmarkDiffCopy1000-4                 	      50	  29511340 ns/op
BenchmarkDiffCopyProto10-4              	    2000	    944615 ns/op
BenchmarkDiffCopyProto50-4              	     500	   3334940 ns/op
BenchmarkDiffCopyProto200-4             	     200	   9420038 ns/op
BenchmarkDiffCopyProto1000-4            	      50	  30632429 ns/op
BenchmarkIncrementalDiffCopy10-4        	    2000	    691993 ns/op
BenchmarkIncrementalDiffCopy50-4        	    1000	   1304253 ns/op
BenchmarkIncrementalDiffCopy200-4       	     500	   3306519 ns/op
BenchmarkIncrementalDiffCopy1000-4      	     200	  10211343 ns/op
BenchmarkIncrementalDiffCopy5000-4      	      20	  55194427 ns/op
BenchmarkIncrementalDiffCopy10000-4     	      20	  91759289 ns/op
BenchmarkIncrementalCopyWithTar10-4     	    2000	   1020258 ns/op
BenchmarkIncrementalCopyWithTar50-4     	     300	   5348786 ns/op
BenchmarkIncrementalCopyWithTar200-4    	     100	  19495000 ns/op
BenchmarkIncrementalCopyWithTar1000-4   	      20	  70338507 ns/op
BenchmarkIncrementalRsync10-4           	      30	  45215754 ns/op
BenchmarkIncrementalRsync50-4           	      30	  45837260 ns/op
BenchmarkIncrementalRsync200-4          	      30	  48780614 ns/op
BenchmarkIncrementalRsync1000-4         	      20	  54801892 ns/op
BenchmarkIncrementalRsync5000-4         	      20	  84782542 ns/op
BenchmarkIncrementalRsync10000-4        	      10	 103355108 ns/op
BenchmarkRsync10-4                      	      30	  46776470 ns/op
BenchmarkRsync50-4                      	      30	  48601555 ns/op
BenchmarkRsync200-4                     	      20	  59642691 ns/op
BenchmarkRsync1000-4                    	      20	 101343010 ns/op
BenchmarkGnuTar10-4                     	     500	   3171448 ns/op
BenchmarkGnuTar50-4                     	     300	   5030296 ns/op
BenchmarkGnuTar200-4                    	     100	  10464313 ns/op
BenchmarkGnuTar1000-4                   	      50	  30375257 ns/op

Documentation

Index

• func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error
• func ComparePath(p1, p2 string) int
• func FollowLinks(root string, paths []string) ([]string, error)
• func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error
• func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error
• func Stat(path string) (*types.Stat, error)
• func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error
• func WriteTar(ctx context.Context, fs FS, w io.Writer) error
• type ChangeFunc
• type ChangeKind
  • func (k ChangeKind) String() string
• type ContentHasher
• type DiffType
• type Dir
• type DiskWriter
  • func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)
  • func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)
  • func (dw *DiskWriter) Wait(ctx context.Context) error
• type DiskWriterOpt
• type FS
  • func NewFS(root string, opt *WalkOpt) FS
  • func SubDirFS(dirs []Dir) (FS, error)
• type FilterFunc
• type HandleChangeFn
• type Hardlinks
  • func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error
• type MapFunc
• type MapResult
• type ReceiveOpt
• type StatInfo
  • func (s *StatInfo) IsDir() bool
  • func (s *StatInfo) ModTime() time.Time
  • func (s *StatInfo) Mode() os.FileMode
  • func (s *StatInfo) Name() string
  • func (s *StatInfo) Size() int64
  • func (s *StatInfo) Sys() interface{}
• type Stream
• type Validator
  • func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)
• type WalkOpt
• type WriteToFunc

Functions

func Changes

func Changes(ctx context.Context, a, b walkerFn, changeFn ChangeFunc) error

func ComparePath

func ComparePath(p1, p2 string) int

func FollowLinks

func FollowLinks(root string, paths []string) ([]string, error)

func Receive

func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error

func Send

func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error

func Stat

func Stat(path string) (*types.Stat, error)

func Walk

func Walk(ctx context.Context, p string, opt *WalkOpt, fn filepath.WalkFunc) error

func WriteTar

func WriteTar(ctx context.Context, fs FS, w io.Writer) error

Types

type ChangeFunc

type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error

ChangeFunc is the type of function called for each change computed during a directory changes calculation.

type ChangeKind

type ChangeKind int

ChangeKind is the type of modification that a change is making.

const (
	// ChangeKindAdd represents an addition of
	// a file
	ChangeKindAdd ChangeKind = iota

	// ChangeKindModify represents a change to
	// an existing file
	ChangeKindModify

	// ChangeKindDelete represents a delete of
	// a file
	ChangeKindDelete
)

func (ChangeKind) String

func (k ChangeKind) String() string

type ContentHasher

type ContentHasher func(*types.Stat) (hash.Hash, error)

type DiffType

type DiffType int

const (
	DiffMetadata DiffType = iota
	DiffNone
	DiffContent
)

type Dir

type Dir struct {
	Stat types.Stat
	FS   FS
}

type DiskWriter

type DiskWriter struct {
	// contains filtered or unexported fields
}

func NewDiskWriter

func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)

func (*DiskWriter) HandleChange

func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*DiskWriter) Wait

func (dw *DiskWriter) Wait(ctx context.Context) error

type DiskWriterOpt

type DiskWriterOpt struct {
	AsyncDataCb   WriteToFunc
	SyncDataCb    WriteToFunc
	NotifyCb      func(ChangeKind, string, os.FileInfo, error) error
	ContentHasher ContentHasher
	Filter        FilterFunc
}

type FS

type FS interface {
	Walk(context.Context, filepath.WalkFunc) error
	Open(string) (io.ReadCloser, error)
}

func NewFS

func NewFS(root string, opt *WalkOpt) FS

func SubDirFS

func SubDirFS(dirs []Dir) (FS, error)

type FilterFunc

type FilterFunc func(string, *types.Stat) bool

type HandleChangeFn

type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error

type Hardlinks

type Hardlinks struct {
	// contains filtered or unexported fields
}

func (*Hardlinks) HandleChange

func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error

type MapFunc

type MapFunc func(string, *types.Stat) MapResult

type MapResult

type MapResult int

The result of the walk function controls both how WalkDir continues and whether the path is kept.

const (
	// Keep the current path and continue.
	MapResultKeep MapResult = iota

	// Exclude the current path and continue.
	MapResultExclude

	// Exclude the current path, and skip the rest of the dir.
	// If path is a dir, skip the current directory.
	// If path is a file, skip the rest of the parent directory.
	// (This matches the semantics of fs.SkipDir.)
	MapResultSkipDir
)

type ReceiveOpt

type ReceiveOpt struct {
	NotifyHashed  ChangeFunc
	ContentHasher ContentHasher
	ProgressCb    func(int, bool)
	Merge         bool
	Filter        FilterFunc
	Differ        DiffType
}

type StatInfo

type StatInfo struct {
	*types.Stat
}

func (*StatInfo) IsDir

func (s *StatInfo) IsDir() bool

func (*StatInfo) ModTime

func (s *StatInfo) ModTime() time.Time

func (*StatInfo) Mode

func (s *StatInfo) Mode() os.FileMode

func (*StatInfo) Name

func (s *StatInfo) Name() string

func (*StatInfo) Size

func (s *StatInfo) Size() int64

func (*StatInfo) Sys

func (s *StatInfo) Sys() interface{}

type Stream

type Stream interface {
	RecvMsg(interface{}) error
	SendMsg(m interface{}) error
	Context() context.Context
}

type Validator

type Validator struct {
	// contains filtered or unexported fields
}

func (*Validator) HandleChange

func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

type WalkOpt

type WalkOpt struct {
	IncludePatterns []string
	ExcludePatterns []string
	// FollowPaths contains symlinks that are resolved into include patterns
	// before performing the fs walk
	FollowPaths []string
	Map         MapFunc
}

type WriteToFunc

type WriteToFunc func(context.Context, string, io.WriteCloser) error