pack

package
v0.0.0-...-a0bb070 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 19, 2019 License: Apache-2.0 Imports: 36 Imported by: 0

Documentation

Overview

Provide necessary API for object replicator

Package pack is a generated protocol buffer package.

It is generated from these files: 

object.proto
rpc.proto

It has these top-level messages: 

ObjectMeta
NeedleIndex
DBIndex
ObjectTimestamps
CheckedObjects
WantedParts
WantedObjects
Partition
PartitionSuffixesReply
SuffixHashesMsg
SuffixHashesReply
SyncMsg
SyncReply
PartitionDeletionReply
PartitionAuditionReply

RPC API for object replication

Index

Constants

View Source 
const (
	BundleVersion1       = BundleVersion(1)
	CurrentBundleVersion = BundleVersion1
)
View Source 
const (
	SuperBlockSize     = 64
	SuperBlockDiskSize = NeedleAlignment
	BundleFileMode     = 0644
)
View Source 
const (
	FALLOC_FL_KEEP_SIZE  = 0x1
	FALLOC_FL_PUNCH_HOLE = 0x2
)
View Source 
const (
	NANO            = 1e9
	FILES_INCREMENT = 1
	LIMITER_BUFFER  = 5
)
View Source 
const (
	NEEDLE_THRESHOLD = 4 * 1024 * 1024
	RECLAIM_AGE      = 60 * 60 * 24 * 7
)

File above 4M will be save as standalone file

View Source 
const (
	NeedleMagicNumber     = 0xdeadbeef
	NeedleAlignment       = 4096
	NeedleHeaderSize      = 40
	DefaultDataBufferSize = 1024 * 256
	DefaultMetaBufferSize = 512
)
View Source 
const (
	NAME                    = "pack"
	ONE_WEEK                = 604800
	METADATA_CHUNK_SIZE     = 65536
	HASH_FILE               = "hashes.pkl"
	HASH_INVALIDATIONS_FILE = "hashes.invalid"
)
View Source 
const (
	// Because suffix is a hexadecimal string of size 3,
	// thus the maximum suffixes per partition is 4096
	MaxPartitionSuffixes = 4096
)

Variables

View Source 
var (
	ErrLockPath                  = errors.New("Error locking path")
	ErrPathNotDir                = errors.New("Path is not a directory")
	ErrNeedleNotAligned          = errors.New("needle offset not aligned")
	ErrNotImplementedErr         = errors.New("not implemented yet")
	ErrDataNotEmpty              = errors.New("data must be empty in meta needle")
	ErrPackDeviceManagerNotFound = errors.New("pack device manager not found")
	ErrPackDeviceNotFound        = errors.New("pack device not found")
	ErrPartitionNotFound         = errors.New("partition not found")
	ErrEmptyPartition            = errors.New("partition is empty")
	ErrDBIndexCorrupted          = errors.New("data in RocksDB can not be deserialized")
	ErrMalformedPickleFile       = errors.New("pickle file is malformed")
	ErrMalformedData             = errors.New("pickle data is malformed")
	ErrObjectNotDelete           = errors.New("unable to delete remote object")
	ErrObjectDataNotSynced       = errors.New("unable to sync object data")
	ErrObjectMetaNotSynced       = errors.New("unable to sync object meta")
	ErrBundleModifiedErr         = errors.New("bundle has been modified")
	ErrObjectsDiff               = errors.New("unable to diff objects with remotes")
	ErrWrongDeallocaion          = errors.New("deallocation on wrong type")
	ErrMetaNotLoaded             = errors.New("unable to load meta data")
	ErrNotPackEngine             = errors.New("engine is not pack type")
	ErrWrongDataWriter           = errors.New("data writer is not correct type")
	ErrRemoteDiskUnmounted       = errors.New("remote disk is unmounted")
	ErrRemoteHash                = errors.New("unable to get remote hash")
	ErrHashConfNotFound          = errors.New("unable to read hash prefix and suffxi")
)

Pack engine errors

Functions

func CalculateBufferSize 

func CalculateBufferSize(headerSize int32, dataSize int64) int64

Calculate memory buffer size for SO

func CalculateDiskSize 

func CalculateDiskSize(headerSize int32, dataSize int64, metaSize int32) int64

func ConsolidateHashes 

func ConsolidateHashes(pklPath, invalidPath string) (map[string]string, error)

func GetFsBlockSize 

func GetFsBlockSize(file *os.File) (int64, error)

func GetRealFsBlocks 

func GetRealFsBlocks(file *os.File) (int64, error)

func HashCleanupListDir 

func HashCleanupListDir(hashDir string, reclaimAge int64) ([]string, error)

func InitAuditor 

func InitAuditor(cnf conf.Config, flags *flag.FlagSet) (srv.Daemon, error)

func InitReplicator 

func InitReplicator(cnf conf.Config, flags *flag.FlagSet) (srv.Daemon, error)

func InvalidateHash 

func InvalidateHash(hashDir string) error

func LoadInvalidSuffixes 

func LoadInvalidSuffixes(invalidPath string) ([]string, error)

Load hash list from hashes.invalid

func LoadPklHashes 

func LoadPklHashes(pklPath string) (map[string]string, error)

Load hash list from hashes.pkl TODO: need to remove corrupted hashes.pkl file

func PackDevicePaths 

func PackDevicePaths(device, driveRoot string, policy int) (string, string)

func PackEngineConstructor 

func PackEngineConstructor(config conf.Config, policy *conf.Policy,
	flags *flag.FlagSet, wg *sync.WaitGroup) (engine.ObjectEngine, error)

func QuarantineDir 

func QuarantineDir(driveRoot string, device string, policy int) string

func RawReadMetadata 

func RawReadMetadata(fileNameOrFd interface{}) ([]byte, error)

func RawWriteMetadata 

func RawWriteMetadata(fd uintptr, buf []byte) error

func ReadMetadata 

func ReadMetadata(fileNameOrFd interface{}) (map[string]string, error)

func RegisterPackRpcServiceServer 

func RegisterPackRpcServiceServer(s *grpc.Server, srv PackRpcServiceServer)

func SaveHashesPkl 

func SaveHashesPkl(
	hashes map[string]string, pklPath, tempDir, partitionDir string) error

func WriteMetadata 

func WriteMetadata(fd uintptr, v map[string]string) error

Types

type AuditStat 

type AuditStat struct {
	ProcessedBytes int64
	ProcessedFiles int64
	Quarantines    int64
	Errors         int64
}

type Auditor 

type Auditor struct {
	// contains filtered or unexported fields
}

func (*Auditor) Run 

func (a *Auditor) Run()

func (*Auditor) RunForever 

func (a *Auditor) RunForever()

type Bundle 

type Bundle struct {
	*SuperBlock
	*os.File
	sync.Mutex
	// contains filtered or unexported fields
}

func OpenBundle 

func OpenBundle(devPath, partition string) (*Bundle, error)

func (*Bundle) BundleSize 

func (b *Bundle) BundleSize() int64

func (*Bundle) Cleanup 

func (b *Bundle) Cleanup() error

func (*Bundle) FlushSuperBlock 

func (b *Bundle) FlushSuperBlock() error

func (*Bundle) PunchHole 

func (b *Bundle) PunchHole(offset, len int64) error

type BundleVersion 

type BundleVersion uint8

type CheckedObjects 

type CheckedObjects struct {
	Objects map[string]*ObjectTimestamps `` /* 134-byte string literal not displayed */
}

func (*CheckedObjects) Descriptor 

func (*CheckedObjects) Descriptor() ([]byte, []int)

func (*CheckedObjects) GetObjects 

func (m *CheckedObjects) GetObjects() map[string]*ObjectTimestamps

func (*CheckedObjects) ProtoMessage 

func (*CheckedObjects) ProtoMessage()

func (*CheckedObjects) Reset 

func (m *CheckedObjects) Reset()

func (*CheckedObjects) String 

func (m *CheckedObjects) String() string

type DBIndex 

type DBIndex struct {
	Index *NeedleIndex `protobuf:"bytes,1,opt,name=index" json:"index,omitempty"`
	Meta  *ObjectMeta  `protobuf:"bytes,2,opt,name=meta" json:"meta,omitempty"`
}

func (*DBIndex) Descriptor 

func (*DBIndex) Descriptor() ([]byte, []int)

func (*DBIndex) GetIndex 

func (m *DBIndex) GetIndex() *NeedleIndex

func (*DBIndex) GetMeta 

func (m *DBIndex) GetMeta() *ObjectMeta

func (*DBIndex) ProtoMessage 

func (*DBIndex) ProtoMessage()

func (*DBIndex) Reset 

func (m *DBIndex) Reset()

func (*DBIndex) String 

func (m *DBIndex) String() string

type NeedleHeader 

type NeedleHeader struct {
	MagicNumber uint32
	NeedleSize  int64
	MetaOffset  int64
	MetaSize    int32
	DataOffset  int64
	DataSize    int64
}

func (*NeedleHeader) DeserializeFrom 

func (n *NeedleHeader) DeserializeFrom(b []byte)

func (*NeedleHeader) WriteToBuffer 

func (n *NeedleHeader) WriteToBuffer(b []byte)

type NeedleIndex 

type NeedleIndex struct {
	Offset     int64 `protobuf:"varint,1,opt,name=offset" json:"offset,omitempty"`
	Size       int64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
	DataOffset int64 `protobuf:"varint,3,opt,name=dataOffset" json:"dataOffset,omitempty"`
	DataSize   int64 `protobuf:"varint,4,opt,name=dataSize" json:"dataSize,omitempty"`
	MetaOffset int64 `protobuf:"varint,5,opt,name=metaOffset" json:"metaOffset,omitempty"`
	MetaSize   int32 `protobuf:"varint,6,opt,name=metaSize" json:"metaSize,omitempty"`
}

func (*NeedleIndex) Descriptor 

func (*NeedleIndex) Descriptor() ([]byte, []int)

func (*NeedleIndex) GetDataOffset 

func (m *NeedleIndex) GetDataOffset() int64

func (*NeedleIndex) GetDataSize 

func (m *NeedleIndex) GetDataSize() int64

func (*NeedleIndex) GetMetaOffset 

func (m *NeedleIndex) GetMetaOffset() int64

func (*NeedleIndex) GetMetaSize 

func (m *NeedleIndex) GetMetaSize() int32

func (*NeedleIndex) GetOffset 

func (m *NeedleIndex) GetOffset() int64

func (*NeedleIndex) GetSize 

func (m *NeedleIndex) GetSize() int64

func (*NeedleIndex) ProtoMessage 

func (*NeedleIndex) ProtoMessage()

func (*NeedleIndex) Reset 

func (m *NeedleIndex) Reset()

func (*NeedleIndex) String 

func (m *NeedleIndex) String() string

type NodeChain 

type NodeChain struct {
	// contains filtered or unexported fields
}

func (*NodeChain) Next 

func (c *NodeChain) Next() *ring.Device

type ObjectMeta 

type ObjectMeta struct {
	Name       string            `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
	Timestamp  string            `protobuf:"bytes,2,opt,name=timestamp" json:"timestamp,omitempty"`
	DataSize   int64             `protobuf:"varint,3,opt,name=dataSize" json:"dataSize,omitempty"`
	SystemMeta map[string]string `` /* 140-byte string literal not displayed */
	UserMeta   map[string]string `` /* 136-byte string literal not displayed */
}

func (*ObjectMeta) DeepCopy 

func (m *ObjectMeta) DeepCopy() *ObjectMeta

func (*ObjectMeta) Descriptor 

func (*ObjectMeta) Descriptor() ([]byte, []int)

func (*ObjectMeta) GetDataSize 

func (m *ObjectMeta) GetDataSize() int64

func (*ObjectMeta) GetName 

func (m *ObjectMeta) GetName() string

func (*ObjectMeta) GetSystemMeta 

func (m *ObjectMeta) GetSystemMeta() map[string]string

func (*ObjectMeta) GetTimestamp 

func (m *ObjectMeta) GetTimestamp() string

func (*ObjectMeta) GetUserMeta 

func (m *ObjectMeta) GetUserMeta() map[string]string

func (*ObjectMeta) ProtoMessage 

func (*ObjectMeta) ProtoMessage()

func (*ObjectMeta) Reset 

func (m *ObjectMeta) Reset()

func (*ObjectMeta) String 

func (m *ObjectMeta) String() string

type ObjectTimestamps 

type ObjectTimestamps struct {
	DataTimestamp string `protobuf:"bytes,1,opt,name=dataTimestamp" json:"dataTimestamp,omitempty"`
	MetaTimestamp string `protobuf:"bytes,2,opt,name=metaTimestamp" json:"metaTimestamp,omitempty"`
}

func (*ObjectTimestamps) Descriptor 

func (*ObjectTimestamps) Descriptor() ([]byte, []int)

func (*ObjectTimestamps) GetDataTimestamp 

func (m *ObjectTimestamps) GetDataTimestamp() string

func (*ObjectTimestamps) GetMetaTimestamp 

func (m *ObjectTimestamps) GetMetaTimestamp() string

func (*ObjectTimestamps) ProtoMessage 

func (*ObjectTimestamps) ProtoMessage()

func (*ObjectTimestamps) Reset 

func (m *ObjectTimestamps) Reset()

func (*ObjectTimestamps) String 

func (m *ObjectTimestamps) String() string

type PackConfig 

type PackConfig struct {
	// Object Server
	AllowedHeaders map[string]bool

	// Auditor configuration
	AuditorFPS int64 // rate of auditor: files per seconds
	AuditorBPS int64 // rate of auditor: bytes per seconds

	// QUSE
	LazyMigration     bool
	PackChunkedObject bool
}

type PackDevice 

type PackDevice struct {
	// contains filtered or unexported fields
}

func NewPackDevice 

func NewPackDevice(device, driveRoot string, policy int) *PackDevice

func (*PackDevice) AuditPartition 

func (d *PackDevice) AuditPartition(partition string) (*AuditStat, error)

func (*PackDevice) CalculateSuffixHash 

func (d *PackDevice) CalculateSuffixHash(partition, suffix string,
	reclaimAge int64) (string, error)

FIXME: identify the empty string because empty string has valid MD5 checksum.

func (*PackDevice) Close 

func (d *PackDevice) Close()

func (*PackDevice) CloseVolume 

func (d *PackDevice) CloseVolume(partition string) error

func (*PackDevice) CommitDeletion 

func (d *PackDevice) CommitDeletion(obj *PackObject) error

func (*PackDevice) CommitUpdate 

func (d *PackDevice) CommitUpdate(obj *PackObject) error

func (*PackDevice) CommitWrite 

func (d *PackDevice) CommitWrite(obj *PackObject) error

func (*PackDevice) DeleteHandoff 

func (d *PackDevice) DeleteHandoff(partition string) error

func (*PackDevice) DiffReplica 

func (d *PackDevice) DiffReplica(partition, objHash string,
	timestamps *ObjectTimestamps) (*WantedParts, error)

func (*PackDevice) GetHashes 

func (d *PackDevice) GetHashes(partition string,
	recalculate []string,
	lsSuffixes bool,
	reclaimAge int64) (hashed int64, hashes map[string]string, err error)

A port of from its Python counterpart. It is kind of complex so from Newton, they improve the hashes.pkl mechanism which requires change to the format of hashes.pkl file. NO plan to implement the new design in version 1 unless we have to.

func (*PackDevice) ListSuffixTimestamps 

func (d *PackDevice) ListSuffixTimestamps(partition, suffix string) (
	map[string]*ObjectTimestamps, error)

func (*PackDevice) ListSuffixes 

func (d *PackDevice) ListSuffixes(partition string) []string

The algorithm should work in product environment. Rough benchmark shows that it takes about 100ms to get the suffix list in a partition. To be honest, it is not efficient but it is simple. If it is inadequent, extra complex data structures should be maintained.

func (*PackDevice) LoadObjectMeta 

func (d *PackDevice) LoadObjectMeta(obj *PackObject) error

************************** Re-implemented API ***************************

func (*PackDevice) NewMetaWriter 

func (d *PackDevice) NewMetaWriter(obj *PackObject) (*dataWriter, error)

func (*PackDevice) NewRangeReader 

func (d *PackDevice) NewRangeReader(
	obj *PackObject, offset, size int64) (*dataReader, error)

func (*PackDevice) NewReader 

func (d *PackDevice) NewReader(obj *PackObject) (*dataReader, error)

func (*PackDevice) NewWriter 

func (d *PackDevice) NewWriter(obj *PackObject) (*dataWriter, error)

func (*PackDevice) QuarantineObject 

func (d *PackDevice) QuarantineObject(obj *PackObject) error

type PackDeviceMgr 

type PackDeviceMgr struct {
	DriveRoot string
	Policy    int
	Port      int
	// contains filtered or unexported fields
}

func NewPackDeviceMgr 

func NewPackDeviceMgr(port int, driveRoot string, policy int) *PackDeviceMgr

func (*PackDeviceMgr) Close 

func (dm *PackDeviceMgr) Close()

func (*PackDeviceMgr) GetPackDevice 

func (dm *PackDeviceMgr) GetPackDevice(device string) *PackDevice

type PackEngine 

type PackEngine struct {
	// contains filtered or unexported fields
}

func (*PackEngine) Close 

func (f *PackEngine) Close() error

func (*PackEngine) DiffReplicas 

func (f *PackEngine) DiffReplicas(device, partition string,
	objects map[string]*ObjectTimestamps) (map[string]*WantedParts, error)

func (*PackEngine) GetHashes 

func (f *PackEngine) GetHashes(
	device, partition string, recalculate []string) (map[string]string, error)

func (*PackEngine) New 

func (f *PackEngine) New(vars map[string]string,

	needData bool) (engine.Object, error)

type PackObject 

type PackObject struct {
	// contains filtered or unexported fields
}

func (*PackObject) Close 

func (o *PackObject) Close() error

Close releases any resources used by the instance of PackObject This method is very important. If we don't close the reader/writer explicitly, file descriptors may be leaked.

func (*PackObject) Commit 

func (o *PackObject) Commit(metadata map[string]string) error

func (*PackObject) CommitMeta 

func (o *PackObject) CommitMeta(metadata map[string]string) error

Handing swift meta is not easy. See https://docs.openstack.org/swift/latest/development_middleware.html#swift-metadata

func (*PackObject) ContentLength 

func (o *PackObject) ContentLength() int64

func (*PackObject) Copy 

func (o *PackObject) Copy(dsts ...io.Writer) (int64, error)

func (*PackObject) CopyRange 

func (o *PackObject) CopyRange(w io.Writer, start int64, end int64) (int64, error)

func (*PackObject) Delete 

func (o *PackObject) Delete(metadata map[string]string) error

func (*PackObject) Exists 

func (o *PackObject) Exists() bool

func (*PackObject) Metadata 

func (o *PackObject) Metadata() map[string]string

func (*PackObject) Migrate 

func (o *PackObject) Migrate() bool

func (*PackObject) Quarantine 

func (o *PackObject) Quarantine() error

func (*PackObject) Repr 

func (o *PackObject) Repr() string

func (*PackObject) SetData 

func (o *PackObject) SetData(size int64) (io.Writer, error)

N.B. CommitMeta shall NOT call this method as NewWriter will choose the Writer between LO and SO based on the size.

type PackRpcServer 

type PackRpcServer struct {
	// contains filtered or unexported fields
}

func NewRpcServer 

func NewRpcServer(port int) *PackRpcServer

func (*PackRpcServer) AuditPartition 

func (s *PackRpcServer) AuditPartition(
	ctx context.Context, msg *Partition) (*PartitionAuditionReply, error)

func (*PackRpcServer) DeleteHandoff 

func (s *PackRpcServer) DeleteHandoff(ctx context.Context, msg *Partition) (
	*PartitionDeletionReply, error)

func (*PackRpcServer) GetHashes 

func (s *PackRpcServer) GetHashes(
	ctx context.Context, msg *SuffixHashesMsg) (*SuffixHashesReply, error)

func (*PackRpcServer) ListPartitionSuffixes 

func (s *PackRpcServer) ListPartitionSuffixes(
	ctx context.Context, msg *Partition) (*PartitionSuffixesReply, error)

func (*PackRpcServer) RegisterPackDeviceMgr 

func (s *PackRpcServer) RegisterPackDeviceMgr(bdm *PackDeviceMgr)

func (*PackRpcServer) Sync 

func (s *PackRpcServer) Sync(ctx context.Context, msg *SyncMsg) (*SyncReply, error)

A successful flag would cause handoff partition to be deleted, so this method should fail fast at any error. The sucessful flag will be return only when ALL the suffixes are replicated sucessfully.

type PackRpcServiceClient 

type PackRpcServiceClient interface {
	ListPartitionSuffixes(ctx context.Context, in *Partition, opts ...grpc.CallOption) (*PartitionSuffixesReply, error)
	GetHashes(ctx context.Context, in *SuffixHashesMsg, opts ...grpc.CallOption) (*SuffixHashesReply, error)
	Sync(ctx context.Context, in *SyncMsg, opts ...grpc.CallOption) (*SyncReply, error)
	DeleteHandoff(ctx context.Context, in *Partition, opts ...grpc.CallOption) (*PartitionDeletionReply, error)
	AuditPartition(ctx context.Context, in *Partition, opts ...grpc.CallOption) (*PartitionAuditionReply, error)
}

func NewPackRpcServiceClient 

func NewPackRpcServiceClient(cc *grpc.ClientConn) PackRpcServiceClient

type PackRpcServiceServer 

type PackRpcServiceServer interface {
	ListPartitionSuffixes(context.Context, *Partition) (*PartitionSuffixesReply, error)
	GetHashes(context.Context, *SuffixHashesMsg) (*SuffixHashesReply, error)
	Sync(context.Context, *SyncMsg) (*SyncReply, error)
	DeleteHandoff(context.Context, *Partition) (*PartitionDeletionReply, error)
	AuditPartition(context.Context, *Partition) (*PartitionAuditionReply, error)
}

type PartType 

type PartType string
const (
	DATA      PartType = "data"
	META      PartType = "meta"
	TOMBSTONE PartType = "ts"
)

type Partition 

type Partition struct {
	Device    string `protobuf:"bytes,1,opt,name=device" json:"device,omitempty"`
	Policy    uint32 `protobuf:"varint,2,opt,name=policy" json:"policy,omitempty"`
	Partition string `protobuf:"bytes,3,opt,name=partition" json:"partition,omitempty"`
}

func (*Partition) Descriptor 

func (*Partition) Descriptor() ([]byte, []int)

func (*Partition) GetDevice 

func (m *Partition) GetDevice() string

func (*Partition) GetPartition 

func (m *Partition) GetPartition() string

func (*Partition) GetPolicy 

func (m *Partition) GetPolicy() uint32

func (*Partition) ProtoMessage 

func (*Partition) ProtoMessage()

func (*Partition) Reset 

func (m *Partition) Reset()

func (*Partition) String 

func (m *Partition) String() string

type PartitionAuditionReply 

type PartitionAuditionReply struct {
	ProcessedBytes int64 `protobuf:"varint,1,opt,name=processedBytes" json:"processedBytes,omitempty"`
	ProcessedFiles int64 `protobuf:"varint,2,opt,name=processedFiles" json:"processedFiles,omitempty"`
	Quarantines    int64 `protobuf:"varint,3,opt,name=quarantines" json:"quarantines,omitempty"`
	Errors         int64 `protobuf:"varint,4,opt,name=errors" json:"errors,omitempty"`
}

func (*PartitionAuditionReply) Descriptor 

func (*PartitionAuditionReply) Descriptor() ([]byte, []int)

func (*PartitionAuditionReply) GetErrors 

func (m *PartitionAuditionReply) GetErrors() int64

func (*PartitionAuditionReply) GetProcessedBytes 

func (m *PartitionAuditionReply) GetProcessedBytes() int64

func (*PartitionAuditionReply) GetProcessedFiles 

func (m *PartitionAuditionReply) GetProcessedFiles() int64

func (*PartitionAuditionReply) GetQuarantines 

func (m *PartitionAuditionReply) GetQuarantines() int64

func (*PartitionAuditionReply) ProtoMessage 

func (*PartitionAuditionReply) ProtoMessage()

func (*PartitionAuditionReply) Reset 

func (m *PartitionAuditionReply) Reset()

func (*PartitionAuditionReply) String 

func (m *PartitionAuditionReply) String() string

type PartitionDeletionReply 

type PartitionDeletionReply struct {
	Success bool `protobuf:"varint,1,opt,name=success" json:"success,omitempty"`
}

func (*PartitionDeletionReply) Descriptor 

func (*PartitionDeletionReply) Descriptor() ([]byte, []int)

func (*PartitionDeletionReply) GetSuccess 

func (m *PartitionDeletionReply) GetSuccess() bool

func (*PartitionDeletionReply) ProtoMessage 

func (*PartitionDeletionReply) ProtoMessage()

func (*PartitionDeletionReply) Reset 

func (m *PartitionDeletionReply) Reset()

func (*PartitionDeletionReply) String 

func (m *PartitionDeletionReply) String() string

type PartitionSuffixesReply 

type PartitionSuffixesReply struct {
	Suffixes []string `protobuf:"bytes,1,rep,name=suffixes" json:"suffixes,omitempty"`
}

func (*PartitionSuffixesReply) Descriptor 

func (*PartitionSuffixesReply) Descriptor() ([]byte, []int)

func (*PartitionSuffixesReply) GetSuffixes 

func (m *PartitionSuffixesReply) GetSuffixes() []string

func (*PartitionSuffixesReply) ProtoMessage 

func (*PartitionSuffixesReply) ProtoMessage()

func (*PartitionSuffixesReply) Reset 

func (m *PartitionSuffixesReply) Reset()

func (*PartitionSuffixesReply) String 

func (m *PartitionSuffixesReply) String() string

type ReplicationStat 

type ReplicationStat struct {
	// contains filtered or unexported fields
}

type Replicator 

type Replicator struct {
	// contains filtered or unexported fields
}

func (*Replicator) Run 

func (r *Replicator) Run()

func (*Replicator) RunForever 

func (r *Replicator) RunForever()

type SuffixHashesMsg 

type SuffixHashesMsg struct {
	Device      string   `protobuf:"bytes,1,opt,name=device" json:"device,omitempty"`
	Policy      uint32   `protobuf:"varint,2,opt,name=policy" json:"policy,omitempty"`
	Partition   string   `protobuf:"bytes,3,opt,name=partition" json:"partition,omitempty"`
	Recalculate []string `protobuf:"bytes,4,rep,name=recalculate" json:"recalculate,omitempty"`
	ListDir     bool     `protobuf:"varint,5,opt,name=listDir" json:"listDir,omitempty"`
	ReclaimAge  uint64   `protobuf:"varint,6,opt,name=reclaimAge" json:"reclaimAge,omitempty"`
}

func (*SuffixHashesMsg) Descriptor 

func (*SuffixHashesMsg) Descriptor() ([]byte, []int)

func (*SuffixHashesMsg) GetDevice 

func (m *SuffixHashesMsg) GetDevice() string

func (*SuffixHashesMsg) GetListDir 

func (m *SuffixHashesMsg) GetListDir() bool

func (*SuffixHashesMsg) GetPartition 

func (m *SuffixHashesMsg) GetPartition() string

func (*SuffixHashesMsg) GetPolicy 

func (m *SuffixHashesMsg) GetPolicy() uint32

func (*SuffixHashesMsg) GetRecalculate 

func (m *SuffixHashesMsg) GetRecalculate() []string

func (*SuffixHashesMsg) GetReclaimAge 

func (m *SuffixHashesMsg) GetReclaimAge() uint64

func (*SuffixHashesMsg) ProtoMessage 

func (*SuffixHashesMsg) ProtoMessage()

func (*SuffixHashesMsg) Reset 

func (m *SuffixHashesMsg) Reset()

func (*SuffixHashesMsg) String 

func (m *SuffixHashesMsg) String() string

type SuffixHashesReply 

type SuffixHashesReply struct {
	Hashed int64             `protobuf:"varint,1,opt,name=hashed" json:"hashed,omitempty"`
	Hashes map[string]string `` /* 132-byte string literal not displayed */
}

func (*SuffixHashesReply) Descriptor 

func (*SuffixHashesReply) Descriptor() ([]byte, []int)

func (*SuffixHashesReply) GetHashed 

func (m *SuffixHashesReply) GetHashed() int64

func (*SuffixHashesReply) GetHashes 

func (m *SuffixHashesReply) GetHashes() map[string]string

func (*SuffixHashesReply) ProtoMessage 

func (*SuffixHashesReply) ProtoMessage()

func (*SuffixHashesReply) Reset 

func (m *SuffixHashesReply) Reset()

func (*SuffixHashesReply) String 

func (m *SuffixHashesReply) String() string

type SuperBlock 

type SuperBlock struct {
	Version BundleVersion
}

* Currently, A superblock only require 1 byte to track the bundle format * version. However, since every needle is 4K aligned in the bundle file, * the first 4K of every bundle will be dedicated to superblock. * This means there a lot of space "reserved" for superblock and should be * sufficient for later extending.

func NewSuperBlock 

func NewSuperBlock(header []byte) *SuperBlock

func (*SuperBlock) Bytes 

func (s *SuperBlock) Bytes() []byte

type SyncMsg 

type SyncMsg struct {
	LocalDevice string   `protobuf:"bytes,1,opt,name=localDevice" json:"localDevice,omitempty"`
	Host        string   `protobuf:"bytes,2,opt,name=host" json:"host,omitempty"`
	Port        int32    `protobuf:"varint,3,opt,name=port" json:"port,omitempty"`
	Device      string   `protobuf:"bytes,4,opt,name=device" json:"device,omitempty"`
	Policy      uint32   `protobuf:"varint,5,opt,name=policy" json:"policy,omitempty"`
	Partition   string   `protobuf:"bytes,6,opt,name=partition" json:"partition,omitempty"`
	Suffixes    []string `protobuf:"bytes,7,rep,name=suffixes" json:"suffixes,omitempty"`
}

func (*SyncMsg) Descriptor 

func (*SyncMsg) Descriptor() ([]byte, []int)

func (*SyncMsg) GetDevice 

func (m *SyncMsg) GetDevice() string

func (*SyncMsg) GetHost 

func (m *SyncMsg) GetHost() string

func (*SyncMsg) GetLocalDevice 

func (m *SyncMsg) GetLocalDevice() string

func (*SyncMsg) GetPartition 

func (m *SyncMsg) GetPartition() string

func (*SyncMsg) GetPolicy 

func (m *SyncMsg) GetPolicy() uint32

func (*SyncMsg) GetPort 

func (m *SyncMsg) GetPort() int32

func (*SyncMsg) GetSuffixes 

func (m *SyncMsg) GetSuffixes() []string

func (*SyncMsg) ProtoMessage 

func (*SyncMsg) ProtoMessage()

func (*SyncMsg) Reset 

func (m *SyncMsg) Reset()

func (*SyncMsg) String 

func (m *SyncMsg) String() string

type SyncReply 

type SyncReply struct {
	Success    bool              `protobuf:"varint,1,opt,name=success" json:"success,omitempty"`
	Candidates map[string]string `` /* 140-byte string literal not displayed */
}

func (*SyncReply) Descriptor 

func (*SyncReply) Descriptor() ([]byte, []int)

func (*SyncReply) GetCandidates 

func (m *SyncReply) GetCandidates() map[string]string

func (*SyncReply) GetSuccess 

func (m *SyncReply) GetSuccess() bool

func (*SyncReply) ProtoMessage 

func (*SyncReply) ProtoMessage()

func (*SyncReply) Reset 

func (m *SyncReply) Reset()

func (*SyncReply) String 

func (m *SyncReply) String() string

type WantedObjects 

type WantedObjects struct {
	Objects map[string]*WantedParts `` /* 134-byte string literal not displayed */
}

func (*WantedObjects) Descriptor 

func (*WantedObjects) Descriptor() ([]byte, []int)

func (*WantedObjects) GetObjects 

func (m *WantedObjects) GetObjects() map[string]*WantedParts

func (*WantedObjects) ProtoMessage 

func (*WantedObjects) ProtoMessage()

func (*WantedObjects) Reset 

func (m *WantedObjects) Reset()

func (*WantedObjects) String 

func (m *WantedObjects) String() string

type WantedParts 

type WantedParts struct {
	Data bool `protobuf:"varint,1,opt,name=data" json:"data,omitempty"`
	Meta bool `protobuf:"varint,2,opt,name=meta" json:"meta,omitempty"`
}

func (*WantedParts) Descriptor 

func (*WantedParts) Descriptor() ([]byte, []int)

func (*WantedParts) GetData 

func (m *WantedParts) GetData() bool

func (*WantedParts) GetMeta 

func (m *WantedParts) GetMeta() bool

func (*WantedParts) ProtoMessage 

func (*WantedParts) ProtoMessage()

func (*WantedParts) Reset 

func (m *WantedParts) Reset()

func (*WantedParts) String 

func (m *WantedParts) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.