wal

Documentation

Index

• Constants
• func CreateCheckpointWriter(dir string, fileNo int) (io.WriteCloser, error)
• func CreateCheckpointWriterForFile(filename string) (io.WriteCloser, error)
• func EncodeDelete(stateCommitment flow.StateCommitment) []byte
• func EncodeStorableNode(storableNode *flattener.StorableNode) []byte
• func EncodeStorableTrie(storableTrie *flattener.StorableTrie) []byte
• func EncodeUpdate(stateCommitment flow.StateCommitment, keys [][]byte, values [][]byte) []byte
• func LoadCheckpoint(filepath string) (*flattener.FlattenedForest, error)
• func ReadStorableNode(reader io.Reader) (*flattener.StorableNode, error)
• func ReadStorableTrie(reader io.Reader, version uint16) (*flattener.StorableTrie, error)
• func StoreCheckpoint(forestSequencing *flattener.FlattenedForest, writer io.WriteCloser) error
• type Checkpointer
  • func NewCheckpointer(wal *LedgerWAL, keyByteSize int, forestCapacity int) *Checkpointer
  • func (c *Checkpointer) Checkpoint(to int, targetWriter func() (io.WriteCloser, error)) error
  • func (c *Checkpointer) CheckpointWriter(to int) (io.WriteCloser, error)
  • func (c *Checkpointer) HasRootCheckpoint() (bool, error)
  • func (c *Checkpointer) LatestCheckpoint() (int, error)
  • func (c *Checkpointer) LoadCheckpoint(checkpoint int) (*flattener.FlattenedForest, error)
  • func (c *Checkpointer) LoadRootCheckpoint() (*flattener.FlattenedForest, error)
  • func (c *Checkpointer) NotCheckpointedSegments() (from, to int, err error)
• type Compactor
  • func NewCompactor(checkpointer *Checkpointer, interval time.Duration, checkpointDistance uint) *Compactor
  • func (c *Compactor) Done() <-chan struct{}
  • func (c *Compactor) Ready() <-chan struct{}
  • func (c *Compactor) Run() error
• type LedgerWAL
  • func NewWAL(logger log.Logger, reg prometheus.Registerer, dir string, forestCapacity int, ...) (*LedgerWAL, error)
  • func (w *LedgerWAL) Close() error
  • func (w *LedgerWAL) NewCheckpointer() (*Checkpointer, error)
  • func (w *LedgerWAL) RecordDelete(stateCommitment flow.StateCommitment) error
  • func (w *LedgerWAL) RecordUpdate(stateCommitment flow.StateCommitment, keys [][]byte, values [][]byte) error
  • func (w *LedgerWAL) Replay(checkpointFn func(forestSequencing *flattener.FlattenedForest) error, ...) error
  • func (w *LedgerWAL) ReplayLogsOnly(checkpointFn func(forestSequencing *flattener.FlattenedForest) error, ...) error
  • func (w *LedgerWAL) ReplayOnMForest(mForest *mtrie.MForest) error
• type SyncOnCloseFile
  • func (s *SyncOnCloseFile) Close() error
• type WALOperation
  • func Decode(data []byte) (operation WALOperation, stateCommitment flow.StateCommitment, keys [][]byte, ...)

Constants

const MagicBytes uint16 = 0x2137

const RootCheckpointFilename = "root.checkpoint"

const SegmentSize = 32 * 1024 * 1024

const VersionV1 uint16 = 0x01

const VersionV2 uint16 = 0x02

Functions

func CreateCheckpointWriter

func CreateCheckpointWriter(dir string, fileNo int) (io.WriteCloser, error)

func CreateCheckpointWriterForFile

func CreateCheckpointWriterForFile(filename string) (io.WriteCloser, error)

func EncodeDelete

func EncodeDelete(stateCommitment flow.StateCommitment) []byte

func EncodeStorableNode

func EncodeStorableNode(storableNode *flattener.StorableNode) []byte

EncodeStorableNode encodes StorableNode 2-bytes Big Endian uint16 height 8-bytes Big Endian uint64 LIndex 8-bytes Big Endian uint64 RIndex 2-bytes Big Endian uint16 key length key bytes 4-bytes Big Endian uint32 value length value bytes 2-bytes Big Endian uint16 hashValue length hash value bytes

func EncodeStorableTrie

func EncodeStorableTrie(storableTrie *flattener.StorableTrie) []byte

EncodeStorableTrie encodes StorableTrie 8-bytes Big Endian uint64 RootIndex 2-bytes Big Endian uint16 RootHash length RootHash bytes

func EncodeUpdate

func EncodeUpdate(stateCommitment flow.StateCommitment, keys [][]byte, values [][]byte) []byte

func LoadCheckpoint

func LoadCheckpoint(filepath string) (*flattener.FlattenedForest, error)

func ReadStorableNode

func ReadStorableNode(reader io.Reader) (*flattener.StorableNode, error)

func ReadStorableTrie

func ReadStorableTrie(reader io.Reader, version uint16) (*flattener.StorableTrie, error)

func StoreCheckpoint

func StoreCheckpoint(forestSequencing *flattener.FlattenedForest, writer io.WriteCloser) error

Types

type Checkpointer

type Checkpointer struct {
	// contains filtered or unexported fields
}

func NewCheckpointer

func NewCheckpointer(wal *LedgerWAL, keyByteSize int, forestCapacity int) *Checkpointer

func (*Checkpointer) Checkpoint

func (c *Checkpointer) Checkpoint(to int, targetWriter func() (io.WriteCloser, error)) error

Checkpoint creates new checkpoint stopping at given segment

func (*Checkpointer) CheckpointWriter

func (c *Checkpointer) CheckpointWriter(to int) (io.WriteCloser, error)

func (*Checkpointer) HasRootCheckpoint

func (c *Checkpointer) HasRootCheckpoint() (bool, error)

func (*Checkpointer) LatestCheckpoint

func (c *Checkpointer) LatestCheckpoint() (int, error)

LatestCheckpoint returns number of latest checkpoint or -1 if there are no checkpoints

func (*Checkpointer) LoadCheckpoint

func (c *Checkpointer) LoadCheckpoint(checkpoint int) (*flattener.FlattenedForest, error)

func (*Checkpointer) LoadRootCheckpoint

func (c *Checkpointer) LoadRootCheckpoint() (*flattener.FlattenedForest, error)

func (*Checkpointer) NotCheckpointedSegments

func (c *Checkpointer) NotCheckpointedSegments() (from, to int, err error)

NotCheckpointedSegments - returns numbers of segments which are not checkpointed yet, or -1, -1 if there are no segments

type Compactor

type Compactor struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewCompactor

func NewCompactor(checkpointer *Checkpointer, interval time.Duration, checkpointDistance uint) *Compactor

func (*Compactor) Done

func (c *Compactor) Done() <-chan struct{}

func (*Compactor) Ready

func (c *Compactor) Ready() <-chan struct{}

Ready periodically fires Run function, every `interval` If called more then once, behaviour is undefined.

func (*Compactor) Run

func (c *Compactor) Run() error

type LedgerWAL

type LedgerWAL struct {
	// contains filtered or unexported fields
}

func NewWAL

func NewWAL(logger log.Logger, reg prometheus.Registerer, dir string, forestCapacity int, keyByteSize int, segmentSize int) (*LedgerWAL, error)

TODO use real logger and metrics, but that would require passing them to Trie storage

func (*LedgerWAL) Close

func (w *LedgerWAL) Close() error

func (*LedgerWAL) NewCheckpointer

func (w *LedgerWAL) NewCheckpointer() (*Checkpointer, error)

NewCheckpointer returns a Checkpointer for this WAL

func (*LedgerWAL) RecordDelete

func (w *LedgerWAL) RecordDelete(stateCommitment flow.StateCommitment) error

func (*LedgerWAL) RecordUpdate

func (w *LedgerWAL) RecordUpdate(stateCommitment flow.StateCommitment, keys [][]byte, values [][]byte) error

func (*LedgerWAL) Replay

func (w *LedgerWAL) Replay(
	checkpointFn func(forestSequencing *flattener.FlattenedForest) error,
	updateFn func(flow.StateCommitment, [][]byte, [][]byte) error,
	deleteFn func(flow.StateCommitment) error,
) error

func (*LedgerWAL) ReplayLogsOnly

func (w *LedgerWAL) ReplayLogsOnly(
	checkpointFn func(forestSequencing *flattener.FlattenedForest) error,
	updateFn func(flow.StateCommitment, [][]byte, [][]byte) error,
	deleteFn func(flow.StateCommitment) error,
) error

func (*LedgerWAL) ReplayOnMForest

func (w *LedgerWAL) ReplayOnMForest(mForest *mtrie.MForest) error

type SyncOnCloseFile

type SyncOnCloseFile struct {
	*bufio.Writer
	// contains filtered or unexported fields
}

func (*SyncOnCloseFile) Close

func (s *SyncOnCloseFile) Close() error

type WALOperation

type WALOperation uint8

const WALDelete WALOperation = 2

const WALUpdate WALOperation = 1

func Decode

func Decode(data []byte) (operation WALOperation, stateCommitment flow.StateCommitment, keys [][]byte, values [][]byte, err error)