sparse

Documentation

Index

• Constants
• Variables
• func FileReader(fileStream <-chan FileInterval, file *os.File, ...)
• func FoldFile(childFileName, parentFileName string) error
• func IntervalSplitter(spltterStream <-chan FileInterval, fileStream chan<- FileInterval)
• func OrderIntervals(prefix string, unorderedStream <-chan HashedDataInterval, ...)
• func PunchHole(file *os.File, hole Interval) error
• func RetrieveLayoutStream(abortStream <-chan error, file *os.File, r Interval, ...)
• func RetrieveLayoutStream0(abortStream <-chan error, file *os.File, r Interval, ...)
• func Server(addr TCPEndPoint, timeout int)
• func SyncFile(localPath string, addr TCPEndPoint, remotePath string, timeout int) (hashLocal []byte, err error)
• func Tee(orderedStream <-chan HashedDataInterval, netOutStream chan<- HashedInterval, ...)
• func TestServer(addr TCPEndPoint, timeout int)
• func Validator(checksumStream, netInStream <-chan DataInterval, resultStream chan<- []byte)
• type DataInterval
• type FileInterval
  • func RetrieveLayout(file *os.File, r Interval) ([]FileInterval, error)
  • func (i FileInterval) String() string
• type FileIntervalKind
• type HashedDataInterval
• type HashedInterval
  • func (i HashedInterval) String() string
• type Interval
  • func (interval Interval) Len() int64
  • func (interval Interval) String() string
• type TCPEndPoint

Constants

const (
	Blocks int64 = 4 << 10 // 4k
)

Storage block size in bytes

Variables

var HashSalt = []byte("TODO: randomize and exchange between client/server")

HashSalt is common client/server hash salt

Functions

func FileReader

func FileReader(fileStream <-chan FileInterval, file *os.File, unorderedStream chan<- HashedDataInterval)

FileReader supports concurrent file reading

func FoldFile

func FoldFile(childFileName, parentFileName string) error

FoldFile folds child snapshot data into its parent

func IntervalSplitter

func IntervalSplitter(spltterStream <-chan FileInterval, fileStream chan<- FileInterval)

IntervalSplitter limits file intervals to predefined batch size

func OrderIntervals

func OrderIntervals(prefix string, unorderedStream <-chan HashedDataInterval, orderedStream chan<- HashedDataInterval)

OrderIntervals puts back "out of order" read results

func PunchHole

func PunchHole(file *os.File, hole Interval) error

PunchHole in a sparse file, preserve file size

func RetrieveLayoutStream

func RetrieveLayoutStream(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error)

RetrieveLayoutStream streams sparse file data/hole layout Based on fiemap To abort: abortStream <- error Check status: err := <- errStream Usage: go RetrieveLayoutStream(...)

func RetrieveLayoutStream0

func RetrieveLayoutStream0(abortStream <-chan error, file *os.File, r Interval, layoutStream chan<- FileInterval, errStream chan<- error)

RetrieveLayoutStream0 streams sparse file data/hole layout Deprecated; Based on file.seek; use RetrieveLayoutStream instead To abort: abortStream <- error Check status: err := <- errStream Usage: go RetrieveLayoutStream(...)

func Server

func Server(addr TCPEndPoint, timeout int)

Server daemon

func SyncFile

func SyncFile(localPath string, addr TCPEndPoint, remotePath string, timeout int) (hashLocal []byte, err error)

SyncFile synchronizes local file to remote host

func Tee

func Tee(orderedStream <-chan HashedDataInterval, netOutStream chan<- HashedInterval, checksumStream chan<- DataInterval)

Tee ordered intervals into the network and checksum checker

func TestServer

func TestServer(addr TCPEndPoint, timeout int)

TestServer daemon serves only one connection for each test then exits

func Validator

func Validator(checksumStream, netInStream <-chan DataInterval, resultStream chan<- []byte)

Validator merges source and diff data; produces hash of the destination file

Types

type DataInterval

type DataInterval struct {
	FileInterval
	Data []byte
}

DataInterval FileInterval plus its data

type FileInterval

type FileInterval struct {
	Kind FileIntervalKind
	Interval
}

FileInterval describes either sparse data Interval or a hole

func RetrieveLayout

func RetrieveLayout(file *os.File, r Interval) ([]FileInterval, error)

RetrieveLayout retrieves sparse file hole and data layout

func (FileInterval) String

func (i FileInterval) String() string

type FileIntervalKind

type FileIntervalKind int

FileIntervalKind distinguishes between data and hole

const (
	SparseData FileIntervalKind = 1 + iota
	SparseHole
	SparseIgnore // ignore file interval (equal src vs dst part)
)

Sparse file Interval types

type HashedDataInterval

type HashedDataInterval struct {
	HashedInterval
	Data []byte
}

HashedDataInterval FileInterval plus its hash and data

type HashedInterval

type HashedInterval struct {
	FileInterval
	Hash []byte
}

HashedInterval FileInterval plus its data hash (to be sent to the client)

func (HashedInterval) String

func (i HashedInterval) String() string

type Interval

type Interval struct {
	Begin, End int64
}

Interval [Begin, End) is non-inclusive at the End

func (Interval) Len

func (interval Interval) Len() int64

Len returns length of Interval

func (Interval) String

func (interval Interval) String() string

String conversion

type TCPEndPoint

type TCPEndPoint struct {
	Host string
	Port int16
}

TCPEndPoint tcp connection address