block

package
v0.0.0-...-0ae6773 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 29, 2017 License: MPL-2.0 Imports: 17 Imported by: 3

Documentation

Index

Constants

View Source 
const (
	SchemeMemory = "memory"
	SchemeFile   = "file"
	SchemeTCP    = "tcp"
)
View Source 
const (
	// DefaultBlockSize is 1MB
	DefaultBlockSize uint64 = 1024 * 1024
)

Variables

View Source 
var (
	// ErrInvalidBlock is used when the block is of invalid format
	ErrInvalidBlock = errors.New("invalid block")
	// ErrBlockNotFound is used when the block doesn't exist
	ErrBlockNotFound = errors.New("block not found")
	// ErrBlockExists is used when a block already exists
	ErrBlockExists = errors.New("block exists")
	// ErrInvalidBlockType is used if an unsupported block type is encountered
	ErrInvalidBlockType = errors.New("invalid block type")
	// ErrReadBlockType is an error when the type cannot be read
	ErrReadBlockType = errors.New("failed to read BlockType")
	// ErrWriteBlockType is an error when the type cannot be written
	ErrWriteBlockType    = errors.New("failed to write BlockType")
	ErrUnsupportedScheme = errors.New("unsupported scheme")
)

Functions

func ParseError 

func ParseError(e string) error

ParseError parses a error string to an actual error

func WriteBlockType 

func WriteBlockType(wr io.Writer, typ BlockType) error

WriteBlockType writes the block type to the writer ensuring a successful write

Types

type Block 

type Block interface {
	// Returns the hash id of the block.  This is the hash of the overall block data.
	ID() []byte
	// Type of block
	Type() BlockType
	// Size of the block data
	Size() uint64
	// Set the size of the block
	SetSize(size uint64)
	// Reader to read data from block
	Reader() (io.ReadCloser, error)
	// Writer to write data to block
	Writer() (io.WriteCloser, error)
	// Compute and return hash id
	Hash() []byte
	// URI returns the location uri of the block
	URI() *URI
}

Block represents a block interface. Blocks may live in-memory, on-disk or remote.

func New 

func New(typ BlockType, uri *URI, hasher func() hash.Hash) (blk Block, err error)

New returns a new Block of the given type. It takes a uri used to determine the source of the block and a hasher. The hasher is only required for TreeBlocks

func NewDataBlock 

func NewDataBlock(uri *URI, hasher func() hash.Hash) Block

NewDataBlock inits a new DataBlock based on the given scheme

type BlockType 

type BlockType byte

BlockType holds the type of block 

const (
	// BlockTypeData is data block. This type of block contains a chunk of arbitrary data.
	// This could be a whole or piece of a file or data
	BlockTypeData BlockType = iota + 1
	// BlockTypeIndex is an index block
	BlockTypeIndex
	// BlockTypeTree defines an tree block containing other data, index, or tree entries
	BlockTypeTree
	// BlockTypeMeta defines a metadata block containing the id of the a tree, index or
	// data block and key-value metadata
	BlockTypeMeta
)

func ParseBlockType 

func ParseBlockType(typ string) (btyp BlockType, err error)

func ReadBlockType 

func ReadBlockType(r io.Reader) (BlockType, error)

ReadBlockType reads the BlockType from the reader ensuring a successful read

func (BlockType) String 

func (blockType BlockType) String() (str string)

type FileDataBlock 

type FileDataBlock struct {
	// contains filtered or unexported fields
}

FileDataBlock is a block with a file as its store.

func LoadFileDataBlock 

func LoadFileDataBlock(uri *URI, hasher func() hash.Hash) (*FileDataBlock, error)

LoadFileDataBlock loads a FileDataBlock from a file on disk. It does not actually open the file

func NewFileDataBlock 

func NewFileDataBlock(uri *URI, hasher func() hash.Hash) *FileDataBlock

NewFileDataBlock instantiates a new Block for the given type

func (*FileDataBlock) Close 

func (block *FileDataBlock) Close() error

Close closes the Writer, writes the hash id to the block and resets the writer.

func (*FileDataBlock) Hash 

func (block *FileDataBlock) Hash() []byte

Hash computes the hash of the underlying file, updates the internal hash id and returns the hash

func (FileDataBlock) ID 

func (block FileDataBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*FileDataBlock) Reader 

func (block *FileDataBlock) Reader() (io.ReadCloser, error)

Reader reads data from block. It first burns the 1-byte type then returns a ReadCloser to the actual data

func (FileDataBlock) SetSize 

func (block FileDataBlock) SetSize(size uint64)

func (FileDataBlock) Size 

func (block FileDataBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (FileDataBlock) Type 

func (block FileDataBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (FileDataBlock) URI 

func (block FileDataBlock) URI() *URI

func (*FileDataBlock) Write 

func (block *FileDataBlock) Write(b []byte) (int, error)

Write writes and hashes the data by writing it to the underlying writer. It also updates the block size

func (*FileDataBlock) Writer 

func (block *FileDataBlock) Writer() (io.WriteCloser, error)

Writer returns a new writer closer to write data to the block. It initializes a hashing writer, writing the type first before returning the writer. It writes to a temp file first and gets moved into the path directory specified in the uri.

type HasherReader 

type HasherReader struct {
	// contains filtered or unexported fields
}

HasherReader hashes all data that is read from the underlying reader

func NewHasherReader 

func NewHasherReader(hasher hash.Hash, r io.Reader) *HasherReader

NewHasherReader implments a reader that hashes the data

func (*HasherReader) DataSize 

func (w *HasherReader) DataSize() uint64

DataSize returns the total bytes read

func (*HasherReader) Hash 

func (w *HasherReader) Hash() []byte

Hash returns the hash of the data written so far

func (*HasherReader) Read 

func (w *HasherReader) Read(b []byte) (int, error)

Read reads from the underlying reader, writes the data to the hasher and updates the bytes read count.

type HasherWriter 

type HasherWriter struct {
	// contains filtered or unexported fields
}

HasherWriter writes and hashes the data

func NewHasherWriter 

func NewHasherWriter(hasher hash.Hash, w io.Writer) *HasherWriter

NewHasherWriter instantiates a new HasherWriter with given underlying writer.

func (*HasherWriter) DataSize 

func (w *HasherWriter) DataSize() uint64

DataSize returns total bytes written

func (*HasherWriter) Hash 

func (w *HasherWriter) Hash() []byte

Hash returns the hash of the data written so far.

func (*HasherWriter) Write 

func (w *HasherWriter) Write(p []byte) (int, error)

type IndexBlock 

type IndexBlock struct {
	// contains filtered or unexported fields
}

IndexBlock is an index of data blocks. It contains an ordered list of block ids making up the data set. This is essentially an index of shards making up the whole file. It is thread safe

func NewIndexBlock 

func NewIndexBlock(uri *URI, hasher func() hash.Hash) *IndexBlock

NewIndexBlock instantiates a new Data index.

func (*IndexBlock) AddBlock 

func (block *IndexBlock) AddBlock(index uint64, blk Block)

AddBlock adds a block to the IndexBlock at the given index.

func (*IndexBlock) BlockCount 

func (block *IndexBlock) BlockCount() int

BlockCount returns the number of blocks in the index

func (*IndexBlock) BlockSize 

func (block *IndexBlock) BlockSize() uint64

BlockSize returns the block size for the overall dataset.

func (*IndexBlock) Blocks 

func (block *IndexBlock) Blocks() [][]byte

Blocks returns sorted block ids by file order. It assumes there are no wholes in the file

func (*IndexBlock) Close 

func (block *IndexBlock) Close() error

Close closes the reader buffer by setting it to nil

func (*IndexBlock) FileSize 

func (block *IndexBlock) FileSize() uint64

FileSize returns the file size represented by this block

func (*IndexBlock) Hash 

func (block *IndexBlock) Hash() []byte

Hash computes the hash of the block given the hash function updating the indertal id and returns the hash id.

func (IndexBlock) ID 

func (block IndexBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*IndexBlock) IndexBlock 

func (block *IndexBlock) IndexBlock(index uint64, id []byte, size uint64)

IndexBlock adds a block to the index by the index, id and size

func (*IndexBlock) Iter 

func (block *IndexBlock) Iter(f func(index uint64, id []byte) error) error

Iter iterates over each block id in order. It sorts based on block posittion and issues the callback with the index and id.

func (*IndexBlock) MarshalBinary 

func (block *IndexBlock) MarshalBinary() []byte

MarshalBinary marshals the IndexBlock into bytes. It writes the type, size, blocksize, and finally the block ids in that order.

func (*IndexBlock) MarshalJSON 

func (block *IndexBlock) MarshalJSON() ([]byte, error)

MarshalJSON is custom json marshaller for IndexBlock

func (*IndexBlock) Read 

func (block *IndexBlock) Read(p []byte) (int, error)

func (*IndexBlock) Reader 

func (block *IndexBlock) Reader() (io.ReadCloser, error)

Reader returns a ReadCloser to this block. It contains a byte stream with the 1-byte replica count, 8-byte size, 8-byte block size followed by an ordered list of block id's.

func (*IndexBlock) Replicas 

func (block *IndexBlock) Replicas() uint8

Replicas returns the replica count set on the index and its blocks

func (*IndexBlock) SetBlockSize 

func (block *IndexBlock) SetBlockSize(size uint64)

SetBlockSize sets the block size for the data blocks in this index. A re-index is required after setting the blocksize

func (*IndexBlock) SetFileSize 

func (block *IndexBlock) SetFileSize(size uint64)

SetFileSize sets the file size for the index the file is representing

func (*IndexBlock) SetReplicas 

func (block *IndexBlock) SetReplicas(replicas uint8)

SetReplicas sets the replica count for the index as well as data blocks it contains

func (IndexBlock) SetSize 

func (block IndexBlock) SetSize(size uint64)

func (IndexBlock) Size 

func (block IndexBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (IndexBlock) Type 

func (block IndexBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (IndexBlock) URI 

func (block IndexBlock) URI() *URI

func (*IndexBlock) UnmarshalBinary 

func (block *IndexBlock) UnmarshalBinary(b []byte) error

UnmarshalBinary takes the byte slice and unmarshals it into an IndexBlock.

func (*IndexBlock) Write 

func (block *IndexBlock) Write(p []byte) (int, error)

func (*IndexBlock) Writer 

func (block *IndexBlock) Writer() (io.WriteCloser, error)

Writer inits a new WriterCloser backed by hasher. It writes the type and returns the WriteCloser

type MemDataBlock 

type MemDataBlock struct {
	// contains filtered or unexported fields
}

MemDataBlock is an in-memory raw data block

func NewMemDataBlock 

func NewMemDataBlock(uri *URI, hasher func() hash.Hash) *MemDataBlock

NewMemDataBlock inits a new DataBlock

func (*MemDataBlock) Close 

func (block *MemDataBlock) Close() error

Close closes the writer if it is not nil. For a read close this is simply a no-op

func (*MemDataBlock) Hash 

func (block *MemDataBlock) Hash() []byte

Hash returns the hash id of the block given the hash function

func (MemDataBlock) ID 

func (block MemDataBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*MemDataBlock) Read 

func (block *MemDataBlock) Read(p []byte) (int, error)

func (*MemDataBlock) Reader 

func (block *MemDataBlock) Reader() (io.ReadCloser, error)

Reader initializes the read buffer with the data returning a ReadCloser

func (MemDataBlock) SetSize 

func (block MemDataBlock) SetSize(size uint64)

func (MemDataBlock) Size 

func (block MemDataBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (MemDataBlock) Type 

func (block MemDataBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (MemDataBlock) URI 

func (block MemDataBlock) URI() *URI

func (*MemDataBlock) Write 

func (block *MemDataBlock) Write(p []byte) (int, error)

func (*MemDataBlock) Writer 

func (block *MemDataBlock) Writer() (io.WriteCloser, error)

Writer initializes a write buffer returning a WriteCloser

type MetaBlock 

type MetaBlock struct {
	// contains filtered or unexported fields
}

MetaBlock is a metadata block. It contains an id that points to an actual data block i.e. tree, index, data and key-value metadata

func NewMetaBlock 

func NewMetaBlock(uri *URI, hasher func() hash.Hash) *MetaBlock

func (*MetaBlock) Hash 

func (blk *MetaBlock) Hash() []byte

func (MetaBlock) ID 

func (block MetaBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*MetaBlock) MarshalBinary 

func (blk *MetaBlock) MarshalBinary() []byte

func (*MetaBlock) SetMetadata 

func (blk *MetaBlock) SetMetadata(m map[string]string)

func (MetaBlock) SetSize 

func (block MetaBlock) SetSize(size uint64)

func (MetaBlock) Size 

func (block MetaBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (MetaBlock) Type 

func (block MetaBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (MetaBlock) URI 

func (block MetaBlock) URI() *URI

func (*MetaBlock) UnmarshalBinary 

func (blk *MetaBlock) UnmarshalBinary(b []byte) error

type StreamedBlock 

type StreamedBlock struct {
	// contains filtered or unexported fields
}

StreamedBlock is a block backed by an underly reader/writer

func NewStreamedBlock 

func NewStreamedBlock(typ BlockType, uri *URI, hasher func() hash.Hash, fh io.ReadWriteCloser, size uint64) *StreamedBlock

NewStreamedBlock initializes a block with a read/writer. It hashes both on reads as well as writes. It takes a BlockType, hash function for read operations, network connection as the reader/writer and the size of the block as parameters.

func (*StreamedBlock) Close 

func (block *StreamedBlock) Close() error

Close closes the Reader or Writer depending on which one is open. It does not close the underlying reader

func (*StreamedBlock) Hash 

func (block *StreamedBlock) Hash() []byte

Hash is a no-op to satisfy the block interface

func (StreamedBlock) ID 

func (block StreamedBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*StreamedBlock) Read 

func (block *StreamedBlock) Read(b []byte) (int, error)

Read reads until the block size then returns an EOF

func (*StreamedBlock) Reader 

func (block *StreamedBlock) Reader() (io.ReadCloser, error)

Reader gets a reader that wraps the underlying network connection in to a block size limited reader.

func (StreamedBlock) SetSize 

func (block StreamedBlock) SetSize(size uint64)

func (StreamedBlock) Size 

func (block StreamedBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (StreamedBlock) Type 

func (block StreamedBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (StreamedBlock) URI 

func (block StreamedBlock) URI() *URI

func (*StreamedBlock) Write 

func (block *StreamedBlock) Write(p []byte) (int, error)

func (*StreamedBlock) Writer 

func (block *StreamedBlock) Writer() (io.WriteCloser, error)

Writer returns a write to a remote block.

type TreeBlock 

type TreeBlock struct {
	// contains filtered or unexported fields
}

TreeBlock is a block containing other types of blocks as it's children

func NewTreeBlock 

func NewTreeBlock(uri *URI, hasher func() hash.Hash) *TreeBlock

NewTreeBlock inits a new TreeBlock with the uri and hasher. The uri may be nil.

func (*TreeBlock) AddNodes 

func (block *TreeBlock) AddNodes(nodes ...*TreeNode)

AddNodes adds TreeNodes to the TreeBlock. It also updates the size of the block by computing the byte slice size

func (*TreeBlock) Close 

func (block *TreeBlock) Close() error

Close closed the reader and writer

func (*TreeBlock) GetNodeByName 

func (block *TreeBlock) GetNodeByName(name string) (*TreeNode, bool)

func (*TreeBlock) Hash 

func (block *TreeBlock) Hash() []byte

Hash returns the hash id of the block given the hash function

func (TreeBlock) ID 

func (block TreeBlock) ID() []byte

ID returns the cached hash id of the block. The hex value of the id is used as the file basename

func (*TreeBlock) Iter 

func (block *TreeBlock) Iter(cb func(*TreeNode) error) error

Iter iterates over each child TreeNode sorted by the name

func (*TreeBlock) MarshalBinary 

func (block *TreeBlock) MarshalBinary() []byte

MarshalBinary marshals the TreeNodes sorted by name. It writes a 1-byte type followed by each node 1 per line.

func (*TreeBlock) MarshalJSON 

func (block *TreeBlock) MarshalJSON() ([]byte, error)

func (*TreeBlock) NodeCount 

func (block *TreeBlock) NodeCount() int

NodeCount returns the total number of child nodes to the TreeBlock

func (*TreeBlock) Read 

func (block *TreeBlock) Read(b []byte) (int, error)

func (*TreeBlock) Reader 

func (block *TreeBlock) Reader() (io.ReadCloser, error)

Reader inits the internal buffer for reading and writes the bytes to it. It returns a io.ReadCloser

func (TreeBlock) SetSize 

func (block TreeBlock) SetSize(size uint64)

func (TreeBlock) Size 

func (block TreeBlock) Size() uint64

Size returns the size of the block data. The value is internally set when the block is loaded. This is strictly the size of the data.

func (TreeBlock) Type 

func (block TreeBlock) Type() BlockType

Type returns the type of block. The value is internally set when the block is loaded.

func (TreeBlock) URI 

func (block TreeBlock) URI() *URI

func (*TreeBlock) UnmarshalBinary 

func (block *TreeBlock) UnmarshalBinary(b []byte) error

UnmarshalBinary unmarshals the byte slice to a tree block

func (*TreeBlock) Write 

func (block *TreeBlock) Write(p []byte) (int, error)

func (*TreeBlock) Writer 

func (block *TreeBlock) Writer() (io.WriteCloser, error)

Writer returns a new writer to allow writing raw bytes to the the TreeBlock. Data is actually written to the structure once the writer is closed.

type TreeNode 

type TreeNode struct {
	Name    string // name of the file or directory
	Address []byte // hash address to its index block
	Type    BlockType
	Mode    os.FileMode
}

TreeNode contains child entries pointing to other blocks

func NewDirTreeNode 

func NewDirTreeNode(name string, addr []byte) *TreeNode

NewDirTreeNode inits a new TreeNode for a directory

func NewFileTreeNode 

func NewFileTreeNode(name string, addr []byte) *TreeNode

NewFileTreeNode inits a new TreeNode for a file

func (TreeNode) MarshalBinary 

func (node TreeNode) MarshalBinary() []byte

MarshalBinary marshals the TreeNode into bytes. It writes mode, space, block type, space, hash address, space, and finally the name as a string and returns the byte representation of the string

func (*TreeNode) MarshalJSON 

func (node *TreeNode) MarshalJSON() ([]byte, error)

func (*TreeNode) UnmarshalBinary 

func (node *TreeNode) UnmarshalBinary(b []byte) error

UnmarshalBinary unmarshals the given bytes into a TreeNode. it returns an error if the format is not as expected

type URI 

type URI struct {
	*url.URL
}

func NewURI 

func NewURI(str string) *URI

func (*URI) ReadOnly 

func (uri *URI) ReadOnly() bool

ReadOnly returns whether or not the uri is meant for read-only operations. By default all uri's are readonly. It looks for the 'wr' param in the url and if present sets the uri to be writable

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.