device

Documentation

Index

• Constants
• type BlockDevice
  • func NewBlockDevice(idx BlockIndex, dev RawDevice) *BlockDevice
  • func (dev *BlockDevice) BlockExists(id []byte) (bool, error)
  • func (dev *BlockDevice) Close() error
  • func (dev *BlockDevice) GetBlock(id []byte) (blk block.Block, err error)
  • func (dev *BlockDevice) Hasher() func() hash.Hash
  • func (dev *BlockDevice) Reindex()
  • func (dev *BlockDevice) RemoveBlock(id []byte) error
  • func (dev *BlockDevice) SetBlock(blk block.Block) ([]byte, error)
  • func (dev *BlockDevice) SetDelegate(delegate Delegate)
  • func (dev *BlockDevice) Stats() *Stats
• type BlockIndex
• type Delegate
• type FileRawDevice
  • func NewFileRawDevice(datadir string, hasher func() hash.Hash) (*FileRawDevice, error)
  • func (st *FileRawDevice) Close() error
  • func (st *FileRawDevice) Count() int
  • func (st *FileRawDevice) Exists(id []byte) bool
  • func (st *FileRawDevice) GetBlock(id []byte) (block.Block, error)
  • func (st *FileRawDevice) Hasher() func() hash.Hash
  • func (st *FileRawDevice) IterIDs(f func(id []byte) error) error
  • func (st *FileRawDevice) NewBlock() block.Block
  • func (st *FileRawDevice) RemoveBlock(id []byte) error
  • func (st *FileRawDevice) SetBlock(blk block.Block) ([]byte, error)
• type IndexEntry
  • func (je *IndexEntry) Data() []byte
  • func (je *IndexEntry) ID() []byte
  • func (je *IndexEntry) MarshalBinary() ([]byte, error)
  • func (je *IndexEntry) Size() uint64
  • func (je *IndexEntry) Type() block.BlockType
  • func (je *IndexEntry) UnmarshalBinary(b []byte) error
• type InmemIndex
  • func NewInmemIndex() *InmemIndex
  • func (j *InmemIndex) Close() error
  • func (j *InmemIndex) Exists(id []byte) bool
  • func (j *InmemIndex) Get(id []byte) (*IndexEntry, error)
  • func (j *InmemIndex) Iter(cb func(*IndexEntry) error) error
  • func (j *InmemIndex) Remove(id []byte) (*IndexEntry, error)
  • func (j *InmemIndex) Set(entry *IndexEntry) error
  • func (j *InmemIndex) Stats() *Stats
• type RawDevice
• type Stats

Constants

const DefaultFilePerms = 0444

DefaultFilePerms are the default file permissions used when creating files on disk.

Types

type BlockDevice

type BlockDevice struct {
	// contains filtered or unexported fields
}

BlockDevice holds and stores the actual blocks. It contians an underlying block device used primarily to store data blocks. It maintains an index of all blocks, that includes the type and size of the block indexed by its hash id. Index and Tree blocks are stored in the index/journal.

func NewBlockDevice

func NewBlockDevice(idx BlockIndex, dev RawDevice) *BlockDevice

NewBlockDevice inits a new BlockDevice with the underlying raw BlockDevice.

func (*BlockDevice) BlockExists

func (dev *BlockDevice) BlockExists(id []byte) (bool, error)

BlockExists returns true if the id exists in the journal

func (*BlockDevice) Close

func (dev *BlockDevice) Close() error

Close stops all operations on the device and closes it

func (*BlockDevice) GetBlock

func (dev *BlockDevice) GetBlock(id []byte) (blk block.Block, err error)

GetBlock returns a block from the volume. Index and tree blocks will be returned in their entirity while a DataBlock will only contain the type and size. The Reader must be used to access the block contents.

func (*BlockDevice) Hasher

func (dev *BlockDevice) Hasher() func() hash.Hash

Hasher returns the underlying hasher used for hash id generation

func (*BlockDevice) Reindex

func (dev *BlockDevice) Reindex()

Reindex scans the raw device and adds indexes for earch block not found in the index

func (*BlockDevice) RemoveBlock

func (dev *BlockDevice) RemoveBlock(id []byte) error

RemoveBlock removes a block from the volume as well as journal by the given hash id

func (*BlockDevice) SetBlock

func (dev *BlockDevice) SetBlock(blk block.Block) ([]byte, error)

SetBlock stores the block in the volume. For DataBlocks the ID is expected to be present.

func (*BlockDevice) SetDelegate

func (dev *BlockDevice) SetDelegate(delegate Delegate)

SetDelegate set the block device delegate. It should be set before the device is used as it is not thread-safe

func (*BlockDevice) Stats

func (dev *BlockDevice) Stats() *Stats

Stats returns stats about the device

type BlockIndex

type BlockIndex interface {
	Get(id []byte) (*IndexEntry, error)
	Exists(id []byte) bool

	// Iterate over all index entries in the store
	Iter(cb func(*IndexEntry) error) error

	// Set an block index entry to the index store
	Set(idx *IndexEntry) error
	Remove(id []byte) (*IndexEntry, error)
	Close() error

	// Stats returns statistics.  Also contains raw device stats
	Stats() *Stats
}

BlockIndex implements a BlockDevice index containing type and size. IndexBlock and TreeBlock are stored in their entirity only in the index. DataBlock is stored in the index if the size is smaller than maxIndexDataValSize.

type Delegate

type Delegate interface {
	// Called when a block is successfully set ie. created
	BlockSet(idx IndexEntry)

	// Called when a block is successfully removed
	BlockRemove(id []byte)
}

Delegate implements a block device Delegate for write operations

type FileRawDevice

type FileRawDevice struct {
	// contains filtered or unexported fields
}

FileRawDevice implements a file based block device. Blocks are stored in files 1 file per block in the data dir.

func NewFileRawDevice

func NewFileRawDevice(datadir string, hasher func() hash.Hash) (*FileRawDevice, error)

NewFileRawDevice instantiates a new FileRawDevice setting the defaults permissions, flush interval provided data directory.

func (*FileRawDevice) Close

func (st *FileRawDevice) Close() error

Close closes the device performing necessary cleanup and shutdown

func (*FileRawDevice) Count

func (st *FileRawDevice) Count() int

Count returns the total number of blocks about the device

func (*FileRawDevice) Exists

func (st *FileRawDevice) Exists(id []byte) bool

Exists stats the block file and returns whether it exists

func (*FileRawDevice) GetBlock

func (st *FileRawDevice) GetBlock(id []byte) (block.Block, error)

GetBlock returns a block with the given id if it exists. It loads the type and size from the file then closes the file.

func (*FileRawDevice) Hasher

func (st *FileRawDevice) Hasher() func() hash.Hash

Hasher returns the underlying hash function generator used to generate hash id

func (*FileRawDevice) IterIDs

func (st *FileRawDevice) IterIDs(f func(id []byte) error) error

IterIDs iterates over all block ids

func (*FileRawDevice) NewBlock

func (st *FileRawDevice) NewBlock() block.Block

NewBlock returns a new Block backed by the store. It initially sets the block uri to the data directory for the store. On write closing it updates with the path including the hash id

func (*FileRawDevice) RemoveBlock

func (st *FileRawDevice) RemoveBlock(id []byte) error

RemoveBlock removes a Block from the in-mem buffer as well as stable store.

func (*FileRawDevice) SetBlock

func (st *FileRawDevice) SetBlock(blk block.Block) ([]byte, error)

SetBlock writes the block to the store. It gets a reader from the provided Block, instantiates a new Block in the store and copies the data to the new Block. It returns the id i.e. hash of the newly written block. It returns a ErrBlockExists if the block exists along with the id.

type IndexEntry

type IndexEntry struct {
	// contains filtered or unexported fields
}

IndexEntry represents a single entry for a block

func (*IndexEntry) Data

func (je *IndexEntry) Data() []byte

Data returns the raw block data as returned from the block Writer

func (*IndexEntry) ID

func (je *IndexEntry) ID() []byte

ID returns the id for the entry

func (*IndexEntry) MarshalBinary

func (je *IndexEntry) MarshalBinary() ([]byte, error)

MarshalBinary marshals the entry into a 1-8-hash-null-data - type, size, id, followed by a null byte then the data

func (*IndexEntry) Size

func (je *IndexEntry) Size() uint64

Size returns the actual consumed size of the block

func (*IndexEntry) Type

func (je *IndexEntry) Type() block.BlockType

Type returns the type of block

func (*IndexEntry) UnmarshalBinary

func (je *IndexEntry) UnmarshalBinary(b []byte) error

UnmarshalBinary unmarshals a byte slice into an IndexEntry. It returns an error if there is not enough data

type InmemIndex

type InmemIndex struct {
	// contains filtered or unexported fields
}

InmemIndex implements an in-memory Index interface

func NewInmemIndex

func NewInmemIndex() *InmemIndex

NewInmemIndex inits a new in-memory journal.

func (*InmemIndex) Close

func (j *InmemIndex) Close() error

Close is a no-op to satifsy the journal interface

func (*InmemIndex) Exists

func (j *InmemIndex) Exists(id []byte) bool

Exists returns true if the journal contains the id

func (*InmemIndex) Get

func (j *InmemIndex) Get(id []byte) (*IndexEntry, error)

Get retreives the value for the given id. It returns a ErrNotFoundError if the id is not found

func (*InmemIndex) Iter

func (j *InmemIndex) Iter(cb func(*IndexEntry) error) error

Iter obtains a read-lock and interates over each key-value pair issuing the callback for each

func (*InmemIndex) Remove

func (j *InmemIndex) Remove(id []byte) (*IndexEntry, error)

Remove removes the block from the journal and return true if the block was inline and an error if it doesn't exist

func (*InmemIndex) Set

func (j *InmemIndex) Set(entry *IndexEntry) error

Set sets the id to the value in the journal. It returns an error if the block exists.

func (*InmemIndex) Stats

func (j *InmemIndex) Stats() *Stats

Stats returns index stats

type RawDevice

type RawDevice interface {
	// Hasher i.e. hash function for id generation
	Hasher() func() hash.Hash

	// New block.  Upon close write the id
	NewBlock() block.Block

	// Write the block by the id
	SetBlock(block.Block) ([]byte, error)

	// Get a block by id
	GetBlock(id []byte) (block.Block, error)

	// Remove a block by id
	RemoveBlock(id []byte) error

	// Check if a block by the id exists
	Exists(id []byte) bool

	// IterIDs
	IterIDs(f func(id []byte) error) error

	// Count returns the total number of block on the device
	Count() int

	// Closes the device
	Close() error
}

RawDevice represents a block storage interface specifically for data blocks. It contains no smarts

type Stats

type Stats struct {
	DataBlocks   int
	IndexBlocks  int
	TreeBlocks   int
	MetaBlocks   int
	TotalBlocks  int
	BlocksOnDisk int
	UsedBytes    uint64
}

Stats contains information regarding blocks for a given device