datatree

package
v0.0.0-...-19f5321 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 14, 2020 License: BSD-2-Clause Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MinimumJobsInGoroutine = 20
	MaximumGoroutines      = 16
)
View Source 
const (
	BufferSize      = 16 * 1024 * 1024
	SmallBufferSize = 32 * 1024 //For UnitTest
	PreReadBufSize  = 256 * 1024
)
View Source 
const (
	FirstLevelAboveTwig int   = 13
	MinPruneCount       int64 = 2

	DeactivedSNListMaxLen = 64
)
View Source 
const (
	TwigShift       = 11
	LeafCountInTwig = 1 << TwigShift // 2048
	TwigMask        = LeafCountInTwig - 1
)
View Source 
const Debug = false

var Debug bool

View Source 
const (
	MSB32 = uint32(1 << 31)
)
View Source 
const MaxEntryBytes int = (1 << 24) - 1
View Source 
const OtherNodeCount = 1 + 11 + 1 + 3 + 1
View Source 
const TwigMtEntryCount = 4095
View Source 
const TwigMtSize = 12 + TwigMtEntryCount*32

Variables

View Source 
var MagicBytes = [8]byte{255, 254, 253, 252, 252, 253, 254, 255}
View Source 
var NullMT4Twig [4096][32]byte
View Source 
var NullNodeInHigherTree [64][32]byte
View Source 
var Phase1Time, Phase2Time, Phase3Time uint64
View Source 
var TotalWriteTime, TotalReadTime, TotalSyncTime uint64

Functions

func CheckHashConsistency 

func CheckHashConsistency(tree *Tree)

func CompareTreeNodes 

func CompareTreeNodes(treeA, treeB *Tree)

func CompareTreeTwigs 

func CompareTreeTwigs(treeA, treeB *Tree)

func CompareTwig 

func CompareTwig(twigID int64, a, b *Twig)

func EdgeNodesToBytes 

func EdgeNodesToBytes(edgeNodes []*EdgeNode) []byte

func EntryToBytes 

func EntryToBytes(entry Entry, deactivedSerialNumList []int64) []byte

func ExtractKeyFromRawBytes 

func ExtractKeyFromRawBytes(b []byte) []byte

func ExtractSerialNum 

func ExtractSerialNum(entryBz []byte) int64

func GetUint24 

func GetUint24(b []byte) (n uint32)

func ParrallelRun 

func ParrallelRun(workerCount int, fn func(workerID int))

func PutUint24 

func PutUint24(b []byte, n uint32)

func SNListToBytes 

func SNListToBytes(deactivedSerialNumList []int64) []byte

func UpdateSerialNum 

func UpdateSerialNum(entryBz []byte, sn int64)

Types

type EdgeNode 

type EdgeNode struct {
	Pos   NodePos
	Value []byte
}

func BytesToEdgeNodes 

func BytesToEdgeNodes(bz []byte) []*EdgeNode

type Entry 

type Entry = types.Entry

func DummyEntry 

func DummyEntry(sn int64) *Entry

func EntryFromBytes 

func EntryFromBytes(b []byte, numberOfSN int) (*Entry, []int64)

func EntryFromRawBytes 

func EntryFromRawBytes(b []byte) *Entry

func NullEntry 

func NullEntry() Entry

type EntryFile 

type EntryFile struct {
	HPFile
}

func NewEntryFile 

func NewEntryFile(bufferSize, blockSize int, dirName string) (res EntryFile, err error)

func (*EntryFile) Append 

func (ef *EntryFile) Append(b [2][]byte) (pos int64)

func (*EntryFile) Close 

func (ef *EntryFile) Close()

func (*EntryFile) Flush 

func (ef *EntryFile) Flush()

func (*EntryFile) FlushAsync 

func (ef *EntryFile) FlushAsync()

func (*EntryFile) GetActiveEntriesInTwig 

func (ef *EntryFile) GetActiveEntriesInTwig(twig *Twig) chan []byte

func (*EntryFile) PruneHead 

func (ef *EntryFile) PruneHead(off int64)

func (*EntryFile) ReadEntry 

func (ef *EntryFile) ReadEntry(off int64) (entry *Entry, nextPos int64)

func (*EntryFile) ReadEntryAndSNList 

func (ef *EntryFile) ReadEntryAndSNList(off int64) (entry *Entry, deactivedSerialNumList []int64, nextPos int64)

func (*EntryFile) ReadEntryRawBytes 

func (ef *EntryFile) ReadEntryRawBytes(off int64) (entryBz []byte, nextPos int64)

func (*EntryFile) Size 

func (ef *EntryFile) Size() int64

func (*EntryFile) Truncate 

func (ef *EntryFile) Truncate(size int64)

type HPFile 

type HPFile struct {
	// contains filtered or unexported fields
}

Head prune-able file

func NewHPFile 

func NewHPFile(bufferSize, blockSize int, dirName string) (HPFile, error)

func (*HPFile) Append 

func (hpf *HPFile) Append(bufList [][]byte) (int64, error)

func (*HPFile) Close 

func (hpf *HPFile) Close() error

func (*HPFile) Flush 

func (hpf *HPFile) Flush()

func (*HPFile) FlushAsync 

func (hpf *HPFile) FlushAsync()

func (*HPFile) InitPreReader 

func (hpf *HPFile) InitPreReader()

func (*HPFile) PruneHead 

func (hpf *HPFile) PruneHead(off int64) error

func (*HPFile) ReadAt 

func (hpf *HPFile) ReadAt(buf []byte, off int64, withBuf bool) (err error)

func (*HPFile) Size 

func (hpf *HPFile) Size() int64

func (*HPFile) Truncate 

func (hpf *HPFile) Truncate(size int64) error

type Hasher 

type Hasher struct {
	// contains filtered or unexported fields
}

func (*Hasher) Add 

func (h *Hasher) Add(level byte, target, srcA, srcB []byte)

func (*Hasher) Run 

func (h *Hasher) Run()

type MockDataTree 

type MockDataTree struct {
	// contains filtered or unexported fields
}

func NewMockDataTree 

func NewMockDataTree() *MockDataTree

func (*MockDataTree) AppendEntry 

func (dt *MockDataTree) AppendEntry(entry *Entry) int64

func (*MockDataTree) AppendEntryRawBytes 

func (dt *MockDataTree) AppendEntryRawBytes(entryBz []byte, sn int64) int64

func (*MockDataTree) Close 

func (dt *MockDataTree) Close()

func (*MockDataTree) DeactiviateEntry 

func (dt *MockDataTree) DeactiviateEntry(sn int64) int

func (*MockDataTree) EndBlock 

func (dt *MockDataTree) EndBlock() []byte

func (*MockDataTree) EvictTwig 

func (dt *MockDataTree) EvictTwig(twigID int64)

func (*MockDataTree) Flush 

func (dt *MockDataTree) Flush()

func (*MockDataTree) GetActiveBit 

func (dt *MockDataTree) GetActiveBit(sn int64) bool

func (*MockDataTree) GetActiveEntriesInTwig 

func (dt *MockDataTree) GetActiveEntriesInTwig(twigID int64) chan []byte

func (*MockDataTree) GetFileSizes 

func (dt *MockDataTree) GetFileSizes() (int64, int64)

func (*MockDataTree) PruneTwigs 

func (dt *MockDataTree) PruneTwigs(startID, endID int64) []byte

func (*MockDataTree) ReadEntry 

func (dt *MockDataTree) ReadEntry(pos int64) *Entry

func (*MockDataTree) ScanEntries 

func (dt *MockDataTree) ScanEntries(oldestActiveTwigID int64, outChan chan types.EntryX)

func (*MockDataTree) ScanEntriesLite 

func (dt *MockDataTree) ScanEntriesLite(oldestActiveTwigID int64, outChan chan types.KeyAndPos)

func (*MockDataTree) TwigCanBePruned 

func (dt *MockDataTree) TwigCanBePruned(twigID int64) bool

type MockTwig 

type MockTwig struct {
	// contains filtered or unexported fields
}

type NodePos 

type NodePos int64

func Pos 

func Pos(level int, n int64) NodePos

type PreReader 

type PreReader struct {
	// contains filtered or unexported fields
}

func (*PreReader) GetToFill 

func (pr *PreReader) GetToFill(fileID, start, end int64) []byte

func (*PreReader) Init 

func (pr *PreReader) Init()

func (*PreReader) TryRead 

func (pr *PreReader) TryRead(fileID, start int64, buf []byte) bool

type ProofNode 

type ProofNode struct {
	SelfHash   [32]byte
	PeerHash   [32]byte
	PeerAtLeft bool
}

type ProofPath 

type ProofPath struct {
	LeftOfTwig  [11]ProofNode
	RightOfTwig [3]ProofNode
	UpperPath   []ProofNode
	SerialNum   int64
	Root        [32]byte
}

func BytesToProofPath 

func BytesToProofPath(bz []byte) (*ProofPath, error)

func (*ProofPath) Check 

func (pp *ProofPath) Check(complete bool) error

func (*ProofPath) ToBytes 

func (pp *ProofPath) ToBytes() []byte

type Tree 

type Tree struct {
	// contains filtered or unexported fields
}

func LoadTree 

func LoadTree(bufferSize, blockSize int, dirName string) *Tree

func NewEmptyTree 

func NewEmptyTree(bufferSize, blockSize int, dirName string) *Tree

func RecoverTree 

func RecoverTree(bufferSize, blockSize int, dirName string, edgeNodes []*EdgeNode, lastPrunedTwigID, oldestActiveTwigID, youngestTwigID int64) *Tree

func (*Tree) ActiviateEntry 

func (tree *Tree) ActiviateEntry(sn int64)

func (*Tree) AppendEntry 

func (tree *Tree) AppendEntry(entry *Entry) int64

func (*Tree) AppendEntryRawBytes 

func (tree *Tree) AppendEntryRawBytes(entryBz []byte, sn int64) int64

func (*Tree) Close 

func (tree *Tree) Close()

func (*Tree) DeactiviateEntry 

func (tree *Tree) DeactiviateEntry(sn int64) int

func (*Tree) DumpMtree4YT 

func (tree *Tree) DumpMtree4YT(outfile io.Writer) error

func (*Tree) DumpNodes 

func (tree *Tree) DumpNodes(outfile io.Writer) error

func (*Tree) EndBlock 

func (tree *Tree) EndBlock() (rootHash []byte)

func (*Tree) EvictTwig 

func (tree *Tree) EvictTwig(twigID int64)

func (*Tree) Flush 

func (tree *Tree) Flush()

func (*Tree) GetActiveBit 

func (tree *Tree) GetActiveBit(sn int64) bool

func (*Tree) GetActiveEntriesInTwig 

func (tree *Tree) GetActiveEntriesInTwig(twigID int64) chan []byte

func (*Tree) GetFileSizes 

func (tree *Tree) GetFileSizes() (int64, int64)

func (*Tree) GetProof 

func (tree *Tree) GetProof(sn int64) *ProofPath

func (*Tree) LoadMtree4YT 

func (tree *Tree) LoadMtree4YT(infile io.Reader) error

func (*Tree) LoadNodes 

func (tree *Tree) LoadNodes(infile io.Reader) error

func (*Tree) PruneTwigs 

func (tree *Tree) PruneTwigs(startID, endID int64) []byte

Prune the twigs between startID and endID

func (*Tree) ReadEntry 

func (tree *Tree) ReadEntry(pos int64) (entry *Entry)

func (*Tree) ReapNodes 

func (tree *Tree) ReapNodes(start, end int64) []byte

func (*Tree) RecoverActiveTwigs 

func (tree *Tree) RecoverActiveTwigs(oldestActiveTwigID int64) []int64

func (*Tree) RecoverEntry 

func (tree *Tree) RecoverEntry(pos int64, entry *Entry, deactivedSNList []int64, oldestActiveTwigID int64)

func (*Tree) RecoverInactiveTwigRoots 

func (tree *Tree) RecoverInactiveTwigRoots(lastPrunedTwigID, oldestActiveTwigID int64) (newList []int64)

func (*Tree) RecoverUpperNodes 

func (tree *Tree) RecoverUpperNodes(edgeNodes []*EdgeNode, nList []int64)

func (*Tree) ScanEntries 

func (tree *Tree) ScanEntries(oldestActiveTwigID int64, outChan chan types.EntryX)

func (*Tree) ScanEntriesLite 

func (tree *Tree) ScanEntriesLite(oldestActiveTwigID int64, outChan chan types.KeyAndPos)

func (*Tree) TruncateFiles 

func (tree *Tree) TruncateFiles(entryFileSize, twigMtFileSize int64)

func (*Tree) TwigCanBePruned 

func (tree *Tree) TwigCanBePruned(twigID int64) bool

type Twig 

type Twig struct {
	FirstEntryPos int64
	// contains filtered or unexported fields
}
var NullTwig Twig

func CopyNullTwig 

func CopyNullTwig() *Twig

func LoadTwigFromFile 

func LoadTwigFromFile(infile io.Reader) (twigID int64, twig Twig, err error)

func (*Twig) Dump 

func (twig *Twig) Dump(twigID int64, outfile io.Writer) error

type TwigMtFile 

type TwigMtFile struct {
	HPFile
}

func NewTwigMtFile 

func NewTwigMtFile(bufferSize, blockSize int, dirName string) (res TwigMtFile, err error)

func (*TwigMtFile) AppendTwig 

func (tf *TwigMtFile) AppendTwig(mtree [][32]byte, firstEntryPos int64)

func (*TwigMtFile) Close 

func (tf *TwigMtFile) Close()

func (*TwigMtFile) Flush 

func (tf *TwigMtFile) Flush()

func (*TwigMtFile) FlushAsync 

func (tf *TwigMtFile) FlushAsync()

func (*TwigMtFile) GetFirstEntryPos 

func (tf *TwigMtFile) GetFirstEntryPos(twigID int64) int64

func (*TwigMtFile) GetHashNode 

func (tf *TwigMtFile) GetHashNode(twigID int64, hashID int) []byte

func (*TwigMtFile) PruneHead 

func (tf *TwigMtFile) PruneHead(off int64)

func (*TwigMtFile) Size 

func (tf *TwigMtFile) Size() int64

func (*TwigMtFile) Truncate 

func (tf *TwigMtFile) Truncate(size int64)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.