blocks

Documentation

Index

• Constants
• Variables
• type Block
  • func CreateBlock(data []byte) *Block
  • func (block *Block) AsSlice() []byte
  • func (block *Block) Hash() *BlockHash
  • func (block *Block) Size() uint64
  • func (block *Block) Update(data []byte)
• type BlockHash
  • func HashBlock(data []byte) *BlockHash
  • func HashFromSlice(hash []byte) (*BlockHash, error)
  • func (block *BlockHash) AsSlice() []byte
  • func (block *BlockHash) Equal(other *BlockHash) bool
  • func (b *BlockHash) String() string
• type BlockPipeline
  • func NewBlockPipeline(ctx context.Context, chunkSize uint64, bufferSize int) *BlockPipeline
  • func (b *BlockPipeline) FromDirectory(root string) *BlockPipeline
  • func (b *BlockPipeline) FromSlice(blocks []FileBlockData) *BlockPipeline
  • func (b *BlockPipeline) FromZip(r *zip.Reader) *BlockPipeline
  • func (b *BlockPipeline) Run(p BlockProcessor) error
  • func (b *BlockPipeline) RunBatch(p []BlockProcessor) error
• type BlockProcessor
  • func DedupBlockProcessor() BlockProcessor
  • func HTTPFetchBlockProcessor(prefix string) BlockProcessor
  • func HashBlockProcessor() BlockProcessor
  • func PrintBlockProcessor() BlockProcessor
  • func ValidateBlockProcessor() BlockProcessor
  • func ZlibCompressBlockProcessor(level int) BlockProcessor
  • func ZlibDecompressBlockProcessor() BlockProcessor
• type FileBlockData
  • func (block *FileBlockData) ComputeHash() (*BlockHash, error)
  • func (block *FileBlockData) SetBlock(data []byte)
  • func (b *FileBlockData) String() string
  • func (block *FileBlockData) UpdateHash() (*BlockHash, error)

Constants

const DefaultBlockSize = 1024 * 1024

const HashSize = sha512.Size

Variables

var Encoding = base64.RawURLEncoding

Types

type Block

type Block struct {
	// contains filtered or unexported fields
}

func CreateBlock

func CreateBlock(data []byte) *Block

func (*Block) AsSlice

func (block *Block) AsSlice() []byte

func (*Block) Hash

func (block *Block) Hash() *BlockHash

func (*Block) Size

func (block *Block) Size() uint64

func (*Block) Update

func (block *Block) Update(data []byte)

type BlockHash

type BlockHash struct {
	// contains filtered or unexported fields
}

func HashBlock

func HashBlock(data []byte) *BlockHash

func HashFromSlice

func HashFromSlice(hash []byte) (*BlockHash, error)

func (*BlockHash) AsSlice

func (block *BlockHash) AsSlice() []byte

func (*BlockHash) Equal

func (block *BlockHash) Equal(other *BlockHash) bool

func (*BlockHash) String

func (b *BlockHash) String() string

type BlockPipeline

type BlockPipeline struct {
	context.Context
	sync.WaitGroup
	ChunkSize uint64
	Blocks    chan *FileBlockData
}

func NewBlockPipeline

func NewBlockPipeline(ctx context.Context, chunkSize uint64, bufferSize int) *BlockPipeline

func (*BlockPipeline) FromDirectory

func (b *BlockPipeline) FromDirectory(root string) *BlockPipeline

func (*BlockPipeline) FromSlice

func (b *BlockPipeline) FromSlice(blocks []FileBlockData) *BlockPipeline

func (*BlockPipeline) FromZip

func (b *BlockPipeline) FromZip(r *zip.Reader) *BlockPipeline

func (*BlockPipeline) Run

func (b *BlockPipeline) Run(p BlockProcessor) error

func (*BlockPipeline) RunBatch

func (b *BlockPipeline) RunBatch(p []BlockProcessor) error

type BlockProcessor

type BlockProcessor func(b *FileBlockData) (*FileBlockData, error)

func DedupBlockProcessor

func DedupBlockProcessor() BlockProcessor

func HTTPFetchBlockProcessor

func HTTPFetchBlockProcessor(prefix string) BlockProcessor

func HashBlockProcessor

func HashBlockProcessor() BlockProcessor

func PrintBlockProcessor

func PrintBlockProcessor() BlockProcessor

func ValidateBlockProcessor

func ValidateBlockProcessor() BlockProcessor

func ZlibCompressBlockProcessor

func ZlibCompressBlockProcessor(level int) BlockProcessor

func ZlibDecompressBlockProcessor

func ZlibDecompressBlockProcessor() BlockProcessor

type FileBlockData

type FileBlockData struct {
	File    string
	Offset  uint64
	BlockId uint64
	Size    uint64
	Hash    *BlockHash
	Data    *Block
}

func (*FileBlockData) ComputeHash

func (block *FileBlockData) ComputeHash() (*BlockHash, error)

func (*FileBlockData) SetBlock

func (block *FileBlockData) SetBlock(data []byte)

func (*FileBlockData) String

func (b *FileBlockData) String() string

func (*FileBlockData) UpdateHash

func (block *FileBlockData) UpdateHash() (*BlockHash, error)