shards

package
v1.0.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 16, 2020 License: GPL-3.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const BOMxorNumRequired = 2

Only supporting mxor_2_6

View Source 
const BOMxorNumTotal = 6
View Source 
const CRC32Len = 4

in bytes

View Source 
const DataLenBytes = 4

At most 4G in our data buffers

View Source 
const FileContainerCurrentVersion = 0
View Source 
const MxorCurVersion = 0
View Source 
const RedundacyLen = 4
View Source 
const SPHeaderMarker = 0x80

Variables

This section is empty.

Functions

func CorrectRSI 

func CorrectRSI(input []byte, parity [][]byte, dataShards, parityShards int) (output []byte, err error)

CorrectRSI takes input as data and some parity shards previously generated and returns the corrected data as output. Using infectious

func DecodeMxor 

func DecodeMxor(shard1 []byte, shard1Ix int, shard2 []byte, shard2Ix int, code ErasureCodeType) ([]byte, error)

DecodeMxor returns the decoded data block as generated by the codec (may end with padded zeros)

func EncodeMxor 

func EncodeMxor(data []byte, code ErasureCodeType) ([][]byte, error)

EncodeMxor returns a slice of shards encoded by the codec

func GenerateRSIParity 

func GenerateRSIParity(input []byte, dataShards, parityShards int) ([][]byte, error)

func GenerateRSIParity_25_26 

func GenerateRSIParity_25_26(input []byte) ([]byte, error)

func GenerateRSIParity_4_5 

func GenerateRSIParity_4_5(input []byte) ([]byte, error)

func GetReedSolomonShareSize 

func GetReedSolomonShareSize(inputSize int64, required int) int64

func GetShardDataLen 

func GetShardDataLen(fileLen int64, shardContainerType ShardContainerType, numRequired int) (ln int64, err error)

func GetShardNumBytes 

func GetShardNumBytes(h ShardHeader) int64

func GetShardTotalLen 

func GetShardTotalLen(fileLen int64, shardContainerType ShardContainerType, numRequired int) (ln int64, err error)

Types

type AorsShardHeader 

type AorsShardHeader struct {
	Version         byte               // aorsCurVersion
	Type            ShardContainerType // 2=ArchiveOptimizedReedSolomon
	ErasureCode     ErasureCodeType    // RSinfectious
	TotalShardsNum  int                // Total number of shards
	NeededShardsNum int                // Number needed for reconstruction
	ShardIndex      int
	// The following two refer the FileContainer for the data
	FileContainerLen  int64  // whole file container
	FileContainerHash []byte // whole file container
}

This is encoded in the shard header bytes as big-endian

func NewAorHeaderFromReader 

func NewAorHeaderFromReader(dataR io.Reader) (h *AorsShardHeader, err error)

func NewAorsShardHeader 

func NewAorsShardHeader(index, total, needed int, fcLen int64, fcHash []byte) *AorsShardHeader

func (*AorsShardHeader) GetContainerType 

func (h *AorsShardHeader) GetContainerType() ShardContainerType

ShardHeader interface

func (*AorsShardHeader) GetErasureCode 

func (h *AorsShardHeader) GetErasureCode() ErasureCodeType

ShardHeader interface

func (*AorsShardHeader) GetFileContainerHash 

func (h *AorsShardHeader) GetFileContainerHash() []byte

ShardHeader interface

func (*AorsShardHeader) GetFileContainerLen 

func (h *AorsShardHeader) GetFileContainerLen() int64

ShardHeader interface

func (*AorsShardHeader) GetNumRequired 

func (h *AorsShardHeader) GetNumRequired() int

ShardHeader interface

func (*AorsShardHeader) GetNumTotal 

func (h *AorsShardHeader) GetNumTotal() int

ShardHeader interface

func (*AorsShardHeader) GetShardIndex 

func (h *AorsShardHeader) GetShardIndex() int

ShardHeader interface

func (*AorsShardHeader) GetShardPayloadLen 

func (h *AorsShardHeader) GetShardPayloadLen() int64

ShardHeader interface

func (*AorsShardHeader) StoreSPShard 

func (h *AorsShardHeader) StoreSPShard(w io.Writer, r io.Reader) (hash, uploaderSignature []byte, err error)

ShardHeader interface r provides the shard data

func (*AorsShardHeader) String 

func (h *AorsShardHeader) String() string

ShardHeader interface

func (*AorsShardHeader) ToBytes 

func (h *AorsShardHeader) ToBytes() []byte

ShardHeader interface. Generates the byte needed for staorage

type BorsShardHeader 

type BorsShardHeader struct {
	Version         byte               // borsCurVersion
	Type            ShardContainerType // 1=BrowserOptimizedReedSolomon
	ErasureCode     ErasureCodeType    // RSinfectious
	TotalShardsNum  byte               // Total number of shards
	NeededShardsNum byte               // Number needed for reconstruction
	ShardIndex      byte
	// The following two refer the FileContainer for the data
	FileContainerDataLen int64  // whole file container
	FileContainerHash    []byte // whole file container
}

This is encoded in the shard header bytes as big-endian

func NewBorsHeaderFromReader 

func NewBorsHeaderFromReader(r io.Reader) (hdr *BorsShardHeader, err error)

func NewBorsShardHeader 

func NewBorsShardHeader(index, total, needed uint, fcLen int64, fcHash []byte) *BorsShardHeader

func NewBorsShardHeaderFromBytes 

func NewBorsShardHeaderFromBytes(data []byte) (*BorsShardHeader, error)

func (*BorsShardHeader) GetContainerType 

func (h *BorsShardHeader) GetContainerType() ShardContainerType

ShardHeader interface

func (*BorsShardHeader) GetErasureCode 

func (h *BorsShardHeader) GetErasureCode() ErasureCodeType

ShardHeader interface

func (*BorsShardHeader) GetFileContainerHash 

func (h *BorsShardHeader) GetFileContainerHash() []byte

ShardHeader interface

func (*BorsShardHeader) GetFileContainerLen 

func (h *BorsShardHeader) GetFileContainerLen() int64

ShardHeader interface

func (*BorsShardHeader) GetNumRequired 

func (h *BorsShardHeader) GetNumRequired() int

ShardHeader interface

func (*BorsShardHeader) GetNumTotal 

func (h *BorsShardHeader) GetNumTotal() int

ShardHeader interface

func (*BorsShardHeader) GetShardIndex 

func (h *BorsShardHeader) GetShardIndex() int

ShardHeader interface

func (*BorsShardHeader) GetShardPayloadLen 

func (h *BorsShardHeader) GetShardPayloadLen() int64

ShardHeader interface

func (*BorsShardHeader) StoreSPShard 

func (h *BorsShardHeader) StoreSPShard(w io.Writer, r io.Reader) (hash, uploaderSignature []byte, err error)

ShardHeader interface r provides the payload

func (*BorsShardHeader) String 

func (h *BorsShardHeader) String() string

ShardHeader interface

func (*BorsShardHeader) ToBytes 

func (h *BorsShardHeader) ToBytes() []byte

ShardHeader interface

type EncryptionType 

type EncryptionType uint8
const (
	NoEncryption EncryptionType = iota
)

type ErasureCodeType 

type ErasureCodeType int8
const (
	ErasureCodeNone ErasureCodeType = -1
	Mxor_2_3        ErasureCodeType = 0
	Mxor_2_4        ErasureCodeType = 1
	Mxor_2_6        ErasureCodeType = 2 // Only this mxor is currently implemented (no plan for the others)
	Mxor_2_7        ErasureCodeType = 3
	RSinfectious    ErasureCodeType = 4 // Reed-Solomon using infectious package
)

func (ErasureCodeType) String 

func (e ErasureCodeType) String() string

type FileContainer 

type FileContainer struct {
	Version         uint8 // currently 0
	Type            FileContainerType
	EncryptionType  EncryptionType
	CompressionType uint8 // 0=no compression
	Size            int64 // bytes
	Signature       []byte
	Shards          *ShardsContainer
	Hash            []byte
}

func NewFileContainer 

func NewFileContainer(s *ShardsContainer, r io.Reader, account interfaces.IAccount) (*FileContainer, error)

NewFileContainer reads the payload data from r and encodes into s

func (*FileContainer) CreateContainerBytes 

func (f *FileContainer) CreateContainerBytes(r io.Reader, uploaderAccount interfaces.IAccount) (fileContainerBytes bytes.Buffer, err error)

CreateContainerBytes writes the file container data to w. Reads the payload data from r

type FileContainerType 

type FileContainerType uint8
const (
	NoContainer FileContainerType = iota // for non-container upload/download (raw file)
	BrowserOptimized
	ArchiveOptimized // using Infectious RS
)

func (FileContainerType) HdrLen 

func (fct FileContainerType) HdrLen() int

func (FileContainerType) PayLoadEnd 

func (fct FileContainerType) PayLoadEnd(fcLen int64) int64

func (FileContainerType) TaiLen 

func (fct FileContainerType) TaiLen() int

type MxorShardHeader 

type MxorShardHeader struct {
	Version     uint8              // MxorCurVersion
	Type        ShardContainerType // 0=BrowserOptimizedMxor
	Flags       uint8
	ErasureCode ErasureCodeType
	ShardIndex  int
	// The following two refer the FileContainer for the data
	FileContainerDataLen uint32 // Max 4G, including file container
	FileContainerHash    []byte // including file container
}

This is encoded in the shard header bytes as big-endian

func NewMxorHeaderFromBytes 

func NewMxorHeaderFromBytes(data []byte) (*MxorShardHeader, error)

func NewMxorHeaderFromReader 

func NewMxorHeaderFromReader(r io.Reader) (hdr *MxorShardHeader, err error)

func NewMxorShardHeader 

func NewMxorShardHeader(index int, fcLen uint32, fcHash []byte) *MxorShardHeader

func (*MxorShardHeader) GetContainerType 

func (h *MxorShardHeader) GetContainerType() ShardContainerType

ShardHeader interface

func (*MxorShardHeader) GetErasureCode 

func (h *MxorShardHeader) GetErasureCode() ErasureCodeType

ShardHeader interface

func (*MxorShardHeader) GetFileContainerHash 

func (h *MxorShardHeader) GetFileContainerHash() []byte

ShardHeader interface

func (*MxorShardHeader) GetFileContainerLen 

func (h *MxorShardHeader) GetFileContainerLen() int64

ShardHeader interface

func (*MxorShardHeader) GetNumRequired 

func (h *MxorShardHeader) GetNumRequired() int

ShardHeader interface

func (*MxorShardHeader) GetNumTotal 

func (h *MxorShardHeader) GetNumTotal() int

ShardHeader interface

func (*MxorShardHeader) GetShardIndex 

func (h *MxorShardHeader) GetShardIndex() int

ShardHeader interface

func (*MxorShardHeader) GetShardPayloadLen 

func (h *MxorShardHeader) GetShardPayloadLen() int64

ShardHeader interface

func (*MxorShardHeader) MarkAsSP 

func (h *MxorShardHeader) MarkAsSP()

func (*MxorShardHeader) StoreSPShard 

func (h *MxorShardHeader) StoreSPShard(w io.Writer, r io.Reader) (hash, uploaderSignature []byte, err error)

ShardHeader interface

func (*MxorShardHeader) String 

func (h *MxorShardHeader) String() string

ShardHeader interface

func (*MxorShardHeader) ToBytes 

func (h *MxorShardHeader) ToBytes() []byte

ShardHeader interface

type Shard 

type Shard struct {
	Hdr  []byte // shard header
	Data []byte // shard data
}

func MxorShard 

func MxorShard(r io.Reader) (*Shard, error)

func NewShardFromSP 

func NewShardFromSP(shardBytes []byte) (sh *Shard, err error)

func (*Shard) GetContainerType 

func (shard *Shard) GetContainerType() ShardContainerType

func (*Shard) GetShardHeader 

func (shard *Shard) GetShardHeader() (hdr ShardHeader, err error)

func (*Shard) Len 

func (shard *Shard) Len() int

func (*Shard) VerifyShardDataSize 

func (shard *Shard) VerifyShardDataSize() error

sanity check

func (*Shard) WriteShardContainer 

func (shard *Shard) WriteShardContainer(w io.Writer, acc interfaces.IAccount) (err error)

WriteShardContainer writes the whole container to w

type ShardContainerType 

type ShardContainerType int8
const (
	ShardContainerNone          ShardContainerType = -1
	BrowserOptimizedMxor        ShardContainerType = 0
	BrowserOptimizedReedSolomon ShardContainerType = 1
	ArchiveOptimizedReedSolomon ShardContainerType = 2
)

func (ShardContainerType) String 

func (s ShardContainerType) String() string

type ShardHeader 

type ShardHeader interface {
	GetContainerType() ShardContainerType
	GetErasureCode() ErasureCodeType
	GetNumTotal() int
	GetNumRequired() int
	GetShardIndex() int
	GetShardPayloadLen() int64 // just shard payload, no header, no parity, no CRC

	// file container, same for all shards
	GetFileContainerLen() int64 // the whole file container
	GetFileContainerHash() []byte

	String() string
	ToBytes() []byte
	// r is the payload (shard data). To be used on the SP
	StoreSPShard(w io.Writer, r io.Reader) (hash, uploaderSignature []byte, err error)
}

func NewShardHeader 

func NewShardHeader(r io.ReadSeeker) (shdr ShardHeader, err error)

type ShardsContainer 

type ShardsContainer struct {
	// Not thread safe. Should not be shared between go routines
	ContainerType ShardContainerType

	NumTotal          int    // Total number of shards
	NumRequired       int    // Min number required for reconstruction
	FileContainerHash []byte // including file container

	// Encode encodes a file container (to shards)
	Encode func(sc *ShardsContainer, fileContainer []byte) error
	// Decode decodes from shards to original data, discarding file container
	Decode func(sc *ShardsContainer, writer io.Writer) error
	// contains filtered or unexported fields
}

ShardsContainer contains all the shards

func NewAors 

func NewAors(total, needed int) *ShardsContainer

func NewBOMxor 

func NewBOMxor() *ShardsContainer

func NewBors 

func NewBors(total, needed int) *ShardsContainer

func NewShardsContainer 

func NewShardsContainer(shardBytes []byte) (s *ShardsContainer, err error)

func (*ShardsContainer) GetContainerType 

func (sc *ShardsContainer) GetContainerType() ShardContainerType

func (*ShardsContainer) GetErasureCode 

func (sc *ShardsContainer) GetErasureCode() ErasureCodeType

func (*ShardsContainer) GetFirstShard 

func (sc *ShardsContainer) GetFirstShard() *Shard

GetFirstShard returns first existing shards, nil if there are none

func (*ShardsContainer) GetNumShards 

func (sc *ShardsContainer) GetNumShards() int

func (*ShardsContainer) GetOriginDataHash 

func (sc *ShardsContainer) GetOriginDataHash() *ArchonHashBytes

func (*ShardsContainer) GetOriginDataHashString 

func (sc *ShardsContainer) GetOriginDataHashString() string

func (*ShardsContainer) GetOriginalDataFromContainer 

func (s *ShardsContainer) GetOriginalDataFromContainer(container []byte) (io.Reader, error)

buf is the file container data. Returns reader of original data

func (*ShardsContainer) GetSampleShardHdr 

func (sc *ShardsContainer) GetSampleShardHdr() (hdr ShardHeader, err error)

GetSampleShardHdr returns the header of the first shard. Useful when needing info that is common to all shards

func (*ShardsContainer) GetShard 

func (sc *ShardsContainer) GetShard(index int) *Shard

func (*ShardsContainer) GetShardDataLen 

func (sc *ShardsContainer) GetShardDataLen() int64

GetShardPayloadLen returns shard data size (without header). All shards have same size

func (*ShardsContainer) GetShardNumBytes 

func (sc *ShardsContainer) GetShardNumBytes() int64

GetShardNumBytes returns the numbers of bytes in the shard

func (*ShardsContainer) HaveAllNeededShards 

func (sc *ShardsContainer) HaveAllNeededShards() bool

func (*ShardsContainer) SetShard 

func (sc *ShardsContainer) SetShard(index int, shard *Shard) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.