blockstore

Documentation

Index

• Constants
• Variables
• type BlockAllocator
• type BlockReader
• type BlockStorage
  • func NewBlockFile(path string, storageOptions ...BufferedBlockStorageOption) (BlockStorage, error)
  • func NewInMemBlockStorage(blockSize int) BlockStorage
  • func OpenBlockFile(path string, storageOptions ...BufferedBlockStorageOption) (BlockStorage, error)
• type BlockWriter
• type BufferedBlockStorage
  • func (s *BufferedBlockStorage) Allocate(nblocks int) (nAllocated int, err error)
  • func (s *BufferedBlockStorage) BlockSize() int
  • func (s *BufferedBlockStorage) Close() error
  • func (s *BufferedBlockStorage) Flush() error
  • func (s *BufferedBlockStorage) NumBlocks() int
  • func (s *BufferedBlockStorage) Read(data []byte) (n int, err error)
  • func (s *BufferedBlockStorage) ReadBlock(index uint, buffer *bytes.Buffer) (n int, err error)
  • func (s *BufferedBlockStorage) ReadPosition() Position
  • func (s *BufferedBlockStorage) Sync() error
  • func (s *BufferedBlockStorage) Write(data []byte) (n int, err error)
  • func (s *BufferedBlockStorage) WriteBlock(index uint, buffer *bytes.Buffer) (n int, err error)
  • func (s *BufferedBlockStorage) WritePosition() Position
  • func (b *BufferedBlockStorage) WriteWithCallback(data []byte, callback WriteEventCallback) (n int, err error)
• type BufferedBlockStorageOption
  • func WithAutoSync() BufferedBlockStorageOption
  • func WithBlockSize(value int) BufferedBlockStorageOption
  • func WithSyncFileIO() BufferedBlockStorageOption
• type BufferedBlockStorageOptions
• type CallbackWriter
• type InMemBlockStorage
  • func (b *InMemBlockStorage) Allocate(nblocks int) (n int, err error)
  • func (b *InMemBlockStorage) BlockSize() int
  • func (b *InMemBlockStorage) Close() error
  • func (b *InMemBlockStorage) Flush() error
  • func (b *InMemBlockStorage) NumBlocks() int
  • func (b *InMemBlockStorage) Read(p []byte) (n int, err error)
  • func (b *InMemBlockStorage) ReadBlock(index uint, buffer *bytes.Buffer) (n int, err error)
  • func (b *InMemBlockStorage) ReadPosition() Position
  • func (b *InMemBlockStorage) Sync() error
  • func (b *InMemBlockStorage) Write(p []byte) (n int, err error)
  • func (b *InMemBlockStorage) WriteBlock(index uint, buffer *bytes.Buffer) (n int, err error)
  • func (b *InMemBlockStorage) WritePosition() Position
  • func (b *InMemBlockStorage) WriteWithCallback(data []byte, callback WriteEventCallback) (n int, err error)
• type Position
  • func (p *Position) DecodeUint64(x uint64)
  • func (p *Position) EncodeUint64() uint64
• type WriteEventCallback

Constants

const BaseBlockSize = 4 * 1024

const (
	FsBaseBlockSize uint = 4 * 1024
)

Variables

var SizeExceedBlockSize = errors.New("size exceeded")

var UninitializedPosition = Position{
	-1, -1,
}

Types

type BlockAllocator

type BlockAllocator interface {
	Allocate(nblocks int) (nAllocated int, err error)
}

type BlockReader

type BlockReader interface {
	ReadBlock(index uint, buffer *bytes.Buffer) (n int, err error)
}

Random access to blocks in a file. IO is aligned. Returns number of bytes read or error

type BlockStorage

type BlockStorage interface {
	BlockReader
	BlockWriter
	BlockAllocator

	io.Reader
	io.Writer
	io.Closer

	CallbackWriter

	NumBlocks() int
	BlockSize() int

	WritePosition() Position
	ReadPosition() Position

	Sync() error
	Flush() error
	// contains filtered or unexported methods
}

func NewBlockFile

func NewBlockFile(path string, storageOptions ...BufferedBlockStorageOption) (BlockStorage, error)

func NewInMemBlockStorage

func NewInMemBlockStorage(blockSize int) BlockStorage

func OpenBlockFile

func OpenBlockFile(path string, storageOptions ...BufferedBlockStorageOption) (BlockStorage, error)

type BlockWriter

type BlockWriter interface {
	WriteBlock(index uint, buffer *bytes.Buffer) (n int, err error)
}

Random write to block in a file. Only write data up to block size, or up to size of buffer, which ever is smaller IO is aligned. Returns number of bytes written or error

type BufferedBlockStorage

type BufferedBlockStorage struct {
	FilePath string
	// contains filtered or unexported fields
}

A class to provide block buffering for io. Data is aligned in blocks

func (*BufferedBlockStorage) Allocate

func (s *BufferedBlockStorage) Allocate(nblocks int) (nAllocated int, err error)

func (*BufferedBlockStorage) BlockSize

func (s *BufferedBlockStorage) BlockSize() int

func (*BufferedBlockStorage) Close

func (s *BufferedBlockStorage) Close() error

func (*BufferedBlockStorage) Flush

func (s *BufferedBlockStorage) Flush() error

func (*BufferedBlockStorage) NumBlocks

func (s *BufferedBlockStorage) NumBlocks() int

func (*BufferedBlockStorage) Read

func (s *BufferedBlockStorage) Read(data []byte) (n int, err error)

func (*BufferedBlockStorage) ReadBlock

func (s *BufferedBlockStorage) ReadBlock(index uint, buffer *bytes.Buffer) (n int, err error)

func (*BufferedBlockStorage) ReadPosition

func (s *BufferedBlockStorage) ReadPosition() Position

func (*BufferedBlockStorage) Sync

func (s *BufferedBlockStorage) Sync() error

func (*BufferedBlockStorage) Write

func (s *BufferedBlockStorage) Write(data []byte) (n int, err error)

func (*BufferedBlockStorage) WriteBlock

func (s *BufferedBlockStorage) WriteBlock(index uint, buffer *bytes.Buffer) (n int, err error)

func (*BufferedBlockStorage) WritePosition

func (s *BufferedBlockStorage) WritePosition() Position

func (*BufferedBlockStorage) WriteWithCallback

func (b *BufferedBlockStorage) WriteWithCallback(data []byte, callback WriteEventCallback) (n int, err error)

type BufferedBlockStorageOption

type BufferedBlockStorageOption func(options *BufferedBlockStorageOptions)

func WithAutoSync

func WithAutoSync() BufferedBlockStorageOption

func WithBlockSize

func WithBlockSize(value int) BufferedBlockStorageOption

func WithSyncFileIO

func WithSyncFileIO() BufferedBlockStorageOption

type BufferedBlockStorageOptions

type BufferedBlockStorageOptions struct {
	AutoSync   bool
	SyncFileIO bool
	BlockSize  int
}

type CallbackWriter

type CallbackWriter interface {
	WriteWithCallback(data []byte, callback WriteEventCallback) (n int, err error)
}

type InMemBlockStorage

type InMemBlockStorage struct {
	// contains filtered or unexported fields
}

Im Memory block storage is uesd for testing only

func (*InMemBlockStorage) Allocate

func (b *InMemBlockStorage) Allocate(nblocks int) (n int, err error)

func (*InMemBlockStorage) BlockSize

func (b *InMemBlockStorage) BlockSize() int

func (*InMemBlockStorage) Close

func (b *InMemBlockStorage) Close() error

func (*InMemBlockStorage) Flush

func (b *InMemBlockStorage) Flush() error

func (*InMemBlockStorage) NumBlocks

func (b *InMemBlockStorage) NumBlocks() int

func (*InMemBlockStorage) Read

func (b *InMemBlockStorage) Read(p []byte) (n int, err error)

Read bytes from block starts a new block if remaining size is too small

func (*InMemBlockStorage) ReadBlock

func (b *InMemBlockStorage) ReadBlock(index uint, buffer *bytes.Buffer) (n int, err error)

Read block into a buffer

func (*InMemBlockStorage) ReadPosition

func (b *InMemBlockStorage) ReadPosition() Position

func (*InMemBlockStorage) Sync

func (b *InMemBlockStorage) Sync() error

func (*InMemBlockStorage) Write

func (b *InMemBlockStorage) Write(p []byte) (n int, err error)

Pack bytes into one block at a time, start a new block if space left in current block is too small

func (*InMemBlockStorage) WriteBlock

func (b *InMemBlockStorage) WriteBlock(index uint, buffer *bytes.Buffer) (n int, err error)

write data in buffer into a block.

func (*InMemBlockStorage) WritePosition

func (b *InMemBlockStorage) WritePosition() Position

func (*InMemBlockStorage) WriteWithCallback

func (b *InMemBlockStorage) WriteWithCallback(data []byte, callback WriteEventCallback) (n int, err error)

type Position

type Position struct {
	Block  int
	Offset int
}

func (*Position) DecodeUint64

func (p *Position) DecodeUint64(x uint64)

func (*Position) EncodeUint64

func (p *Position) EncodeUint64() uint64

type WriteEventCallback

type WriteEventCallback func(position Position, err error)