ethdb

package
v2.2.1+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 26, 2018 License: GPL-3.0 Imports: 25 Imported by: 104

Documentation

Index

Constants

View Source 
const (
	HeaderPrefix        = "h"
	BlockHashPrefix     = "H"
	BodyPrefix          = "b"
	BlockReceiptsPrefix = "r"
	LookupPrefix        = "l"
	BloomBitsPrefix     = "B"
)

Prefix used for each table type.

View Source 
const (
	// DefaultPartitionSize is the default number of blocks per partition.
	DefaultPartitionSize = 1024

	// DefaultMinMutableSegmentCount is the minimum number of mutable segments on a given table.
	DefaultMinMutableSegmentCount = 40

	// DefaultMinCompactionAge is the minimum age after creation before an LDB
	// segment can be compacted into a file segment.
	DefaultMinCompactionAge = 1 * time.Minute
)
View Source 
const (
	SegmentETH1 = "eth1"
	SegmentLDB1 = "ldb1"
)

Segment file types.

View Source 
const (
	// FileSegmentMagic is the magic number at the beginning of the file segment.
	FileSegmentMagic = "ETH1"

	// FileSegmentChecksumSize is the size of the checksum, in bytes.
	FileSegmentChecksumSize = 8

	// FileSegmentIndexOffsetSize is the size of the index offset, in bytes.
	FileSegmentIndexOffsetSize = 8

	// FileSegmentIndexCountSize is the size of the index element count, in bytes.
	FileSegmentIndexCountSize = 8

	// FileSegmentIndexCapacitySize is the size of the index capacity, in bytes.
	FileSegmentIndexCapacitySize = 8

	// FileSegmentHeaderSize is the total size of the fixed length FileSegment header.
	FileSegmentHeaderSize = 0 +
		len(FileSegmentMagic) +
		FileSegmentChecksumSize +
		FileSegmentIndexOffsetSize +
		FileSegmentIndexCountSize +
		FileSegmentIndexCapacitySize
)
View Source 
const (
	DefaultMaxOpenSegmentCount = 10
)

Configuration defaults.

View Source 
const IdealBatchSize = 100 * 1024

Code using batches should try to add this much data to the batch. The value was determined empirically.

Variables

View Source 
var (
	ErrImmutableSegment            = errors.New("ethdb: immutable segment")
	ErrSegmentTypeUnknown          = errors.New("ethdb: segment type unknown")
	ErrFileSegmentChecksumMismatch = errors.New("ethdb: file segment checksum mismatch")
)
View Source 
var (
	ErrInvalidSegmentType = errors.New("ethdb: Invalid segment type")
)

Database errors.

Functions

func ChecksumFileSegment 

func ChecksumFileSegment(path string) ([]byte, error)

ChecksumFileSegment calculates the checksum for the file segment at path.

func IsDBDir 

func IsDBDir(path string) (bool, error)

IsDBDir returns true if path contains an ethdb.DB. Checks if the "global" table exists.

func IsLevelDBDir 

func IsLevelDBDir(path string) (bool, error)

IsLevelDBDir returns true if path contains a goleveldb database. Verifies that path contains a CURRENT file that starts with MANIFEST.

func KeyBlockNumber 

func KeyBlockNumber(key []byte) (num uint64, ok bool)

KeyBlockNumber returns the block number for a given key and returns ok true. If the key does not encode the block number then ok is false.

func SegmentFileType 

func SegmentFileType(path string) (string, error)

SegmentFileType returns the file type at path.

func SortSegments 

func SortSegments(a []Segment)

SortSegments sorts a by name.

func UncompactSegmentTo 

func UncompactSegmentTo(s Segment, path string) error

UncompactSegmentTo writes an LDB segment from a file segment.

func VerifyFileSegment 

func VerifyFileSegment(path string) error

VerifyFileSegment compares the calculated and stored checksum of the segment at path.

Types

type BlockNumberPartitioner 

type BlockNumberPartitioner struct {
	Size uint64
}

BlockNumberPartitioner represents a partitioner that returns a partition based on block number.

func NewBlockNumberPartitioner 

func NewBlockNumberPartitioner(size uint64) *BlockNumberPartitioner

NewBlockNumberPartitioner returns a new instance of BlockNumberPartitioner.

func (*BlockNumberPartitioner) Partition 

func (p *BlockNumberPartitioner) Partition(key []byte) string

Partition always returns a partition name based on a grouping of block numbers.

type Config 

type Config struct {
	// S3 archive options
	Endpoint        string `toml:",omitempty"`
	Bucket          string `toml:",omitempty"`
	AccessKeyID     string `toml:",omitempty"`
	SecretAccessKey string `toml:",omitempty"`

	// Per-table LRU cache settings.
	MaxOpenSegmentCount int `toml:",omitempty"`
}

func NewConfig 

func NewConfig() Config

NewConfig returns a new instance of Config with defaults set.

type DB 

type DB struct {

	// Filename of the root of the database.
	Path string

	// Number of blocks grouped together.
	PartitionSize uint64

	// Maximum number of segments that can be opened at once.
	MaxOpenSegmentCount int

	// Age before LDB segment can be compacted to a file segment.
	MinCompactionAge time.Duration

	SegmentOpener    SegmentOpener
	SegmentCompactor SegmentCompactor
	// contains filtered or unexported fields
}

DB is the top-level database and contains a mixture of LevelDB & File storage layers.

func NewDB 

func NewDB(path string) *DB

NewDB returns a new instance of DB.

func (*DB) BodyTable 

func (db *DB) BodyTable() common.Table

BodyTable returns the table which holds body data.

func (*DB) Close 

func (db *DB) Close() error

Close closes all underlying tables.

func (*DB) GlobalTable 

func (db *DB) GlobalTable() common.Table

GlobalTable returns the global, statically partitioned table.

func (*DB) HeaderTable 

func (db *DB) HeaderTable() common.Table

HeaderTable returns the table which holds header data.

func (*DB) Open 

func (db *DB) Open() error

Open initializes and opens the database.

func (*DB) ReceiptTable 

func (db *DB) ReceiptTable() common.Table

ReceiptTable returns the table which holds receipt data.

func (*DB) TablePath 

func (db *DB) TablePath(name string) string

TablePath returns the filename for the given table.

func (*DB) Tables 

func (db *DB) Tables() []*Table

Tables returns a sorted list of all tables.

type FileSegment 

type FileSegment struct {
	// contains filtered or unexported fields
}

FileSegment represents an immutable key/value file segment for a table.

func NewFileSegment 

func NewFileSegment(name, path string) *FileSegment

NewFileSegment returns a new instance of FileSegment.

func (*FileSegment) Cap 

func (s *FileSegment) Cap() int

capacity returns the capacity of the index.

func (*FileSegment) Checksum 

func (s *FileSegment) Checksum() []byte

Checksum returns the checksum written to the segment file.

func (*FileSegment) Close 

func (s *FileSegment) Close() error

Close closes the file and its mmap.

func (*FileSegment) Data 

func (s *FileSegment) Data() []byte

Data returns the underlying mmap data.

func (*FileSegment) Get 

func (s *FileSegment) Get(key []byte) ([]byte, error)

Get returns the value of the given key.

func (*FileSegment) Has 

func (s *FileSegment) Has(key []byte) (bool, error)

Has returns true if the key exists.

func (*FileSegment) Index 

func (s *FileSegment) Index() []byte

index returns the byte slice containing the index.

func (*FileSegment) IndexOffset 

func (s *FileSegment) IndexOffset() int64

indexOffset returns the file offset where the index starts.

func (*FileSegment) Iterator 

func (s *FileSegment) Iterator() SegmentIterator

Iterator returns an iterator for iterating over all key/value pairs.

func (*FileSegment) Len 

func (s *FileSegment) Len() int

Len returns the number of keys in the file.

func (*FileSegment) Name 

func (s *FileSegment) Name() string

Name returns the name of the segment.

func (*FileSegment) Open 

func (s *FileSegment) Open() error

Open opens and initializes the file segment.

func (*FileSegment) Path 

func (s *FileSegment) Path() string

Path returns the path of the segment.

func (*FileSegment) Size 

func (s *FileSegment) Size() int

Size returns the size of the underlying data file.

type FileSegmentCompactor 

type FileSegmentCompactor struct{}

FileSegmentCompactor locally compacts LDB segments into file segments.

func NewFileSegmentCompactor 

func NewFileSegmentCompactor() *FileSegmentCompactor

NewFileSegmentCompactor returns a new instance of FileSegmentCompactor.

func (*FileSegmentCompactor) CompactSegment 

func (c *FileSegmentCompactor) CompactSegment(ctx context.Context, table string, s *LDBSegment) (Segment, error)

CompactSegment compacts an LDB segment into a file segment.

func (*FileSegmentCompactor) CompactSegmentTo 

func (c *FileSegmentCompactor) CompactSegmentTo(ctx context.Context, s *LDBSegment, path string) error

CompactSegmentTo compacts an LDB segment to a specified path.

func (*FileSegmentCompactor) RenameSegment 

func (c *FileSegmentCompactor) RenameSegment(ctx context.Context, dst, src string) error

RenameSegment removes dst and renames the new segment at path.

func (*FileSegmentCompactor) UncompactSegment 

func (c *FileSegmentCompactor) UncompactSegment(ctx context.Context, table string, s Segment) (*LDBSegment, error)

UncompactSegment converts an LDB segment back into a file segment.

func (*FileSegmentCompactor) UncompactSegmentTo 

func (c *FileSegmentCompactor) UncompactSegmentTo(ctx context.Context, s Segment, path string) error

UncompactSegmentTo converts a segment back to an LDB segment at path.

type FileSegmentEncoder 

type FileSegmentEncoder struct {

	// Filename of file segment to encode.
	Path string
	// contains filtered or unexported fields
}

FileSegmentEncoder represents a encoder for building a ethdb.FileSegment.

func NewFileSegmentEncoder 

func NewFileSegmentEncoder(path string) *FileSegmentEncoder

func (*FileSegmentEncoder) Close 

func (enc *FileSegmentEncoder) Close() error

Close closes the file handle. File must be flushed before calling close.

func (*FileSegmentEncoder) EncodeKeyValue 

func (enc *FileSegmentEncoder) EncodeKeyValue(key, value []byte) error

EncodeKeyValue writes framed key & value byte slices to the file and records their offset.

func (*FileSegmentEncoder) Flush 

func (enc *FileSegmentEncoder) Flush() error

Flush finalizes the file segment and appends a hashmap & trailer.

func (*FileSegmentEncoder) Open 

func (enc *FileSegmentEncoder) Open() (err error)

Open opens and initializes the output file segment.

type FileSegmentIterator 

type FileSegmentIterator struct {
	// contains filtered or unexported fields
}

FileSegmentIterator returns an error for sequentially iterating over a FileSegment's key/value pairs.

func (*FileSegmentIterator) Close 

func (itr *FileSegmentIterator) Close() error

Close releases the iterator.

func (*FileSegmentIterator) Key 

func (itr *FileSegmentIterator) Key() []byte

Key returns the current key. Must be called after Next().

func (*FileSegmentIterator) Next 

func (itr *FileSegmentIterator) Next() bool

Next reads the next key/value pair into the buffer.

func (*FileSegmentIterator) Value 

func (itr *FileSegmentIterator) Value() []byte

Value returns the current key. Must be called after Next().

type FileSegmentOpener 

type FileSegmentOpener struct{}

FileSegmentOpener initializes and opens segments.

func NewFileSegmentOpener 

func NewFileSegmentOpener() *FileSegmentOpener

NewFileSegmentOpener returns a new instance of FileSegmentOpener.

func (*FileSegmentOpener) ListSegmentNames 

func (o *FileSegmentOpener) ListSegmentNames(path, table string) ([]string, error)

ListSegmentNames returns a list of all segment names for a table.

func (*FileSegmentOpener) OpenSegment 

func (o *FileSegmentOpener) OpenSegment(table, name, path string) (Segment, error)

OpenSegment returns an initialized and opened segment.

type LDBSegment 

type LDBSegment struct {
	// contains filtered or unexported fields
}

LDBSegement represents a mutable segment in a Table. These segments can eventually be rebuilt into immutable FileSegments.

func NewLDBSegment 

func NewLDBSegment(name, path string) *LDBSegment

NewLDBSegment returns a LevelDB-based database segment.

func (*LDBSegment) Close 

func (s *LDBSegment) Close() error

Close closes the underlying database.

func (*LDBSegment) CompactTo 

func (s *LDBSegment) CompactTo(path string) error

CompactTo writes the segment to disk as a file segment.

func (*LDBSegment) Delete 

func (s *LDBSegment) Delete(key []byte) error

Delete deletes the key from the queue and database

func (*LDBSegment) Get 

func (s *LDBSegment) Get(key []byte) ([]byte, error)

Get returns the given key if it's present.

func (*LDBSegment) Has 

func (s *LDBSegment) Has(key []byte) (bool, error)

Has returns true if the segment contains key.

func (*LDBSegment) Iterator 

func (s *LDBSegment) Iterator() SegmentIterator

Iterator returns a sequential iterator for the segment.

func (*LDBSegment) LDB 

func (s *LDBSegment) LDB() *leveldb.DB

LDB returns the underlying LevelDB database.

func (*LDBSegment) Name 

func (s *LDBSegment) Name() string

Name returns the name of the segment.

func (*LDBSegment) Open 

func (s *LDBSegment) Open() (err error)

Open initializes the underlying segment database.

func (*LDBSegment) Path 

func (s *LDBSegment) Path() string

Path returns the path to the segment.

func (*LDBSegment) Put 

func (s *LDBSegment) Put(key []byte, value []byte) error

Put inserts a value into a given key.

type MemDatabase 

type MemDatabase struct {
	// contains filtered or unexported fields
}

* This is a test memory database. Do not use for any production it does not get persisted

func NewMemDatabase 

func NewMemDatabase() *MemDatabase

func NewMemDatabaseWithCap 

func NewMemDatabaseWithCap(size int) (*MemDatabase, error)

func (*MemDatabase) BodyTable 

func (db *MemDatabase) BodyTable() common.Table

func (*MemDatabase) Close 

func (db *MemDatabase) Close() error

func (*MemDatabase) Delete 

func (db *MemDatabase) Delete(key []byte) error

func (*MemDatabase) Get 

func (db *MemDatabase) Get(key []byte) ([]byte, error)

func (*MemDatabase) GlobalTable 

func (db *MemDatabase) GlobalTable() common.Table

func (*MemDatabase) Has 

func (db *MemDatabase) Has(key []byte) (bool, error)

func (*MemDatabase) HeaderTable 

func (db *MemDatabase) HeaderTable() common.Table

func (*MemDatabase) Keys 

func (db *MemDatabase) Keys() [][]byte

func (*MemDatabase) Len 

func (db *MemDatabase) Len() int

func (*MemDatabase) NewBatch 

func (db *MemDatabase) NewBatch() common.Batch

func (*MemDatabase) Put 

func (db *MemDatabase) Put(key []byte, value []byte) error

func (*MemDatabase) ReceiptTable 

func (db *MemDatabase) ReceiptTable() common.Table

type MutableSegment 

type MutableSegment interface {
	Segment
	Put(key, value []byte) error
	Delete(key []byte) error
}

MutableSegment represents a segment that can be altered. These segments are eventually compacted into immutable segments.

type PartitionFunc 

type PartitionFunc func(key []byte) string

PartitionFunc implements Partitioner.

func (PartitionFunc) Partition 

func (fn PartitionFunc) Partition(key []byte) string

type Partitioner 

type Partitioner interface {
	Partition(key []byte) string
}

Partitioner represents an object that returns a partition name for a given key.

type Segment 

type Segment interface {
	io.Closer

	Name() string
	Path() string

	Has(key []byte) (bool, error)
	Get(key []byte) ([]byte, error)
	Iterator() SegmentIterator
}

Segment represents a subset of Table data.

type SegmentCompactor 

type SegmentCompactor interface {
	CompactSegment(ctx context.Context, table string, s *LDBSegment) (Segment, error)
	UncompactSegment(ctx context.Context, table string, s Segment) (*LDBSegment, error)
}

SegmentCompactor represents an object that can compact from an LDB segment to an immutable segment and back.

type SegmentIterator 

type SegmentIterator interface {
	io.Closer
	Next() bool
	Key() []byte
	Value() []byte
}

SegmentIterator represents a sequentially iterator over all the key/value pairs inside a segment.

type SegmentOpener 

type SegmentOpener interface {
	OpenSegment(table, name, path string) (Segment, error)
	ListSegmentNames(path, table string) ([]string, error)
}

SegmentOpener represents an object that can instantiate and load an immutable segment.

type SegmentSet 

type SegmentSet struct {
	// contains filtered or unexported fields
}

SegmentSet represents a set of segments.

func NewSegmentSet 

func NewSegmentSet(maxOpenCount int) *SegmentSet

NewSegmentSet returns a new instance of SegmentSet.

func (*SegmentSet) Acquire 

func (ss *SegmentSet) Acquire(name string) (Segment, error)

Acquire returns a segment by name from the set and adds increments the semaphore. If the segment is unopened then it is opened before returning. If a segment is successfully retruns then Release() must always be called by the caller.

func (*SegmentSet) Add 

func (ss *SegmentSet) Add(s Segment)

Add adds s to the set.

func (*SegmentSet) Contains 

func (ss *SegmentSet) Contains(name string) bool

Contains returns true if name is in the set.

func (*SegmentSet) Len 

func (ss *SegmentSet) Len() int

Len returns the number of segments in the set.

func (*SegmentSet) Release 

func (ss *SegmentSet) Release()

Release decrements the semaphore on the set.

func (*SegmentSet) Remove 

func (ss *SegmentSet) Remove(name string)

Remove removes the segment with the given name from the set.

func (*SegmentSet) Slice 

func (ss *SegmentSet) Slice() []Segment

Slice returns a slice of all segments.

type StaticPartitioner 

type StaticPartitioner struct {
	Name string
}

StaticPartitioner represents a partitioner that always returns the same partition name.

func (*StaticPartitioner) Partition 

func (p *StaticPartitioner) Partition(key []byte) string

Partition always returns the same partition name.

type Table 

type Table struct {
	Name        string
	Path        string
	Partitioner Partitioner

	MinMutableSegmentCount int
	MinCompactionAge       time.Duration

	// Maximum number of segments that can be opened at once.
	MaxOpenSegmentCount int

	SegmentOpener    SegmentOpener
	SegmentCompactor SegmentCompactor
	// contains filtered or unexported fields
}

Table represents key/value storage for a particular data type. Contains zero or more segments that are separated by partitioner.

func NewTable 

func NewTable(name, path string, partitioner Partitioner) *Table

NewTable returns a new instance of Table.

func (*Table) AcquireSegment 

func (t *Table) AcquireSegment(name string) (Segment, error)

AcquireSegment returns a segment by name. Returns nil if segment does not exist. Must call ReleaseSegment when finished with the segment.

func (*Table) ActiveSegment 

func (t *Table) ActiveSegment() MutableSegment

ActiveSegment returns the active segment.

func (*Table) ActiveSegmentName 

func (t *Table) ActiveSegmentName() string

ActiveSegmentName the name of the current active segment.

func (*Table) Close 

func (t *Table) Close() error

Close closes all segments within the table.

func (*Table) Compact 

func (t *Table) Compact(ctx context.Context) error

Compact converts LDB segments into immutable file segments.

func (*Table) CreateSegmentIfNotExists 

func (t *Table) CreateSegmentIfNotExists(name string) (*LDBSegment, error)

CreateSegmentIfNotExists returns a mutable segment by name. Creates a new segment if it does not exist.

func (*Table) Delete 

func (t *Table) Delete(key []byte) error

Delete removes key from the database.

func (*Table) Get 

func (t *Table) Get(key []byte) ([]byte, error)

Get returns the value associated with key.

func (*Table) Has 

func (t *Table) Has(key []byte) (bool, error)

Has returns true if key exists in the table.

func (*Table) NewBatch 

func (t *Table) NewBatch() common.Batch

func (*Table) Open 

func (t *Table) Open() error

Open initializes the table and all existing segments.

func (*Table) Put 

func (t *Table) Put(key, value []byte) error

Put associates a value with key.

func (*Table) ReleaseSegment 

func (t *Table) ReleaseSegment(s Segment)

ReleaseSegment releases a given segment.

func (*Table) SegmentNames 

func (t *Table) SegmentNames() []string

SegmentNames a sorted list of all segments names.

func (*Table) SegmentPath 

func (t *Table) SegmentPath(name string) string

SegmentPath returns the path of the named segment.

func (*Table) SegmentSlice 

func (t *Table) SegmentSlice() []Segment

SegmentsSlice returns a sorted slice of all segments.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.