ethdb

package
v2.1.109+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 24, 2018 License: GPL-3.0 Imports: 25 Imported by: 104

Documentation

Index

Constants

View Source
const (
	HeaderPrefix        = "h"
	BlockHashPrefix     = "H"
	BodyPrefix          = "b"
	BlockReceiptsPrefix = "r"
	LookupPrefix        = "l"
	BloomBitsPrefix     = "B"
)

Prefix used for each table type.

View Source
const (
	// DefaultPartitionSize is the default number of blocks per partition.
	DefaultPartitionSize = 1024

	// DefaultMinMutableSegmentCount is the minimum number of mutable segments on a given table.
	DefaultMinMutableSegmentCount = 40

	// DefaultMinCompactionAge is the minimum age after creation before an LDB
	// segment can be compacted into a file segment.
	DefaultMinCompactionAge = 1 * time.Minute
)
View Source
const (
	SegmentETH1 = "eth1"
	SegmentLDB1 = "ldb1"
)

Segment file types.

View Source
const (
	// FileSegmentMagic is the magic number at the beginning of the file segment.
	FileSegmentMagic = "ETH1"

	// FileSegmentChecksumSize is the size of the checksum, in bytes.
	FileSegmentChecksumSize = 8

	// FileSegmentIndexOffsetSize is the size of the index offset, in bytes.
	FileSegmentIndexOffsetSize = 8

	// FileSegmentIndexCountSize is the size of the index element count, in bytes.
	FileSegmentIndexCountSize = 8

	// FileSegmentIndexCapacitySize is the size of the index capacity, in bytes.
	FileSegmentIndexCapacitySize = 8

	// FileSegmentHeaderSize is the total size of the fixed length FileSegment header.
	FileSegmentHeaderSize = 0 +
		len(FileSegmentMagic) +
		FileSegmentChecksumSize +
		FileSegmentIndexOffsetSize +
		FileSegmentIndexCountSize +
		FileSegmentIndexCapacitySize
)
View Source
const (
	DefaultMaxOpenSegmentCount = 10
)

Configuration defaults.

View Source
const IdealBatchSize = 100 * 1024

Code using batches should try to add this much data to the batch. The value was determined empirically.

Variables

View Source
var (
	ErrKeyNotFound                 = errors.New("ethdb: key not found")
	ErrImmutableSegment            = errors.New("ethdb: immutable segment")
	ErrSegmentTypeUnknown          = errors.New("ethdb: segment type unknown")
	ErrFileSegmentChecksumMismatch = errors.New("ethdb: file segment checksum mismatch")
)
View Source
var (
	ErrInvalidSegmentType = errors.New("ethdb: Invalid segment type")
)

Database errors.

Functions

func ChecksumFileSegment

func ChecksumFileSegment(path string) ([]byte, error)

ChecksumFileSegment calculates the checksum for the file segment at path.

func IsDBDir

func IsDBDir(path string) (bool, error)

IsDBDir returns true if path contains an ethdb.DB. Checks if the "global" table exists.

func IsLevelDBDir

func IsLevelDBDir(path string) (bool, error)

IsLevelDBDir returns true if path contains a goleveldb database. Verifies that path contains a CURRENT file that starts with MANIFEST.

func KeyBlockNumber

func KeyBlockNumber(key []byte) (num uint64, ok bool)

KeyBlockNumber returns the block number for a given key and returns ok true. If the key does not encode the block number then ok is false.

func SegmentFileType

func SegmentFileType(path string) (string, error)

SegmentFileType returns the file type at path.

func SortSegments

func SortSegments(a []Segment)

SortSegments sorts a by name.

func UncompactSegmentTo

func UncompactSegmentTo(s Segment, path string) error

UncompactSegmentTo writes an LDB segment from a file segment.

func VerifyFileSegment

func VerifyFileSegment(path string) error

VerifyFileSegment compares the calculated and stored checksum of the segment at path.

Types

type BlockNumberPartitioner

type BlockNumberPartitioner struct {
	Size uint64
}

BlockNumberPartitioner represents a partitioner that returns a partition based on block number.

func NewBlockNumberPartitioner

func NewBlockNumberPartitioner(size uint64) *BlockNumberPartitioner

NewBlockNumberPartitioner returns a new instance of BlockNumberPartitioner.

func (*BlockNumberPartitioner) Partition

func (p *BlockNumberPartitioner) Partition(key []byte) string

Partition always returns a partition name based on a grouping of block numbers.

type Config

type Config struct {
	// S3 archive options
	Endpoint        string `toml:",omitempty"`
	Bucket          string `toml:",omitempty"`
	AccessKeyID     string `toml:",omitempty"`
	SecretAccessKey string `toml:",omitempty"`

	// Per-table LRU cache settings.
	MaxOpenSegmentCount int `toml:",omitempty"`
}

func NewConfig

func NewConfig() Config

NewConfig returns a new instance of Config with defaults set.

type DB

type DB struct {

	// Filename of the root of the database.
	Path string

	// Number of blocks grouped together.
	PartitionSize uint64

	// Maximum number of segments that can be opened at once.
	MaxOpenSegmentCount int

	// Age before LDB segment can be compacted to a file segment.
	MinCompactionAge time.Duration

	SegmentOpener    SegmentOpener
	SegmentCompactor SegmentCompactor
	// contains filtered or unexported fields
}

DB is the top-level database and contains a mixture of LevelDB & File storage layers.

func NewDB

func NewDB(path string) *DB

NewDB returns a new instance of DB.

func (*DB) BodyTable

func (db *DB) BodyTable() common.Table

BodyTable returns the table which holds body data.

func (*DB) Close

func (db *DB) Close() error

Close closes all underlying tables.

func (*DB) GlobalTable

func (db *DB) GlobalTable() common.Table

GlobalTable returns the global, statically partitioned table.

func (*DB) HeaderTable

func (db *DB) HeaderTable() common.Table

HeaderTable returns the table which holds header data.

func (*DB) Open

func (db *DB) Open() error

Open initializes and opens the database.

func (*DB) ReceiptTable

func (db *DB) ReceiptTable() common.Table

ReceiptTable returns the table which holds receipt data.

func (*DB) TablePath

func (db *DB) TablePath(name string) string

TablePath returns the filename for the given table.

func (*DB) Tables

func (db *DB) Tables() []*Table

Tables returns a sorted list of all tables.

type FileSegment

type FileSegment struct {
	// contains filtered or unexported fields
}

FileSegment represents an immutable key/value file segment for a table.

func NewFileSegment

func NewFileSegment(name, path string) *FileSegment

NewFileSegment returns a new instance of FileSegment.

func (*FileSegment) Cap

func (s *FileSegment) Cap() int

capacity returns the capacity of the index.

func (*FileSegment) Checksum

func (s *FileSegment) Checksum() []byte

Checksum returns the checksum written to the segment file.

func (*FileSegment) Close

func (s *FileSegment) Close() error

Close closes the file and its mmap.

func (*FileSegment) Data

func (s *FileSegment) Data() []byte

Data returns the underlying mmap data.

func (*FileSegment) Get

func (s *FileSegment) Get(key []byte) ([]byte, error)

Get returns the value of the given key.

func (*FileSegment) Has

func (s *FileSegment) Has(key []byte) (bool, error)

Has returns true if the key exists.

func (*FileSegment) Index

func (s *FileSegment) Index() []byte

index returns the byte slice containing the index.

func (*FileSegment) IndexOffset

func (s *FileSegment) IndexOffset() int64

indexOffset returns the file offset where the index starts.

func (*FileSegment) Iterator

func (s *FileSegment) Iterator() SegmentIterator

Iterator returns an iterator for iterating over all key/value pairs.

func (*FileSegment) Len

func (s *FileSegment) Len() int

Len returns the number of keys in the file.

func (*FileSegment) Name

func (s *FileSegment) Name() string

Name returns the name of the segment.

func (*FileSegment) Open

func (s *FileSegment) Open() error

Open opens and initializes the file segment.

func (*FileSegment) Path

func (s *FileSegment) Path() string

Path returns the path of the segment.

func (*FileSegment) Size

func (s *FileSegment) Size() int

Size returns the size of the underlying data file.

type FileSegmentCompactor

type FileSegmentCompactor struct{}

FileSegmentCompactor locally compacts LDB segments into file segments.

func NewFileSegmentCompactor

func NewFileSegmentCompactor() *FileSegmentCompactor

NewFileSegmentCompactor returns a new instance of FileSegmentCompactor.

func (*FileSegmentCompactor) CompactSegment

func (c *FileSegmentCompactor) CompactSegment(ctx context.Context, table string, s *LDBSegment) (Segment, error)

CompactSegment compacts an LDB segment into a file segment.

func (*FileSegmentCompactor) CompactSegmentTo

func (c *FileSegmentCompactor) CompactSegmentTo(ctx context.Context, s *LDBSegment, path string) error

CompactSegmentTo compacts an LDB segment to a specified path.

func (*FileSegmentCompactor) RenameSegment

func (c *FileSegmentCompactor) RenameSegment(ctx context.Context, dst, src string) error

RenameSegment removes dst and renames the new segment at path.

func (*FileSegmentCompactor) UncompactSegment

func (c *FileSegmentCompactor) UncompactSegment(ctx context.Context, table string, s Segment) (*LDBSegment, error)

UncompactSegment converts an LDB segment back into a file segment.

func (*FileSegmentCompactor) UncompactSegmentTo

func (c *FileSegmentCompactor) UncompactSegmentTo(ctx context.Context, s Segment, path string) error

UncompactSegmentTo converts a segment back to an LDB segment at path.

type FileSegmentEncoder

type FileSegmentEncoder struct {

	// Filename of file segment to encode.
	Path string
	// contains filtered or unexported fields
}

FileSegmentEncoder represents a encoder for building a ethdb.FileSegment.

func NewFileSegmentEncoder

func NewFileSegmentEncoder(path string) *FileSegmentEncoder

func (*FileSegmentEncoder) Close

func (enc *FileSegmentEncoder) Close() error

Close closes the file handle. File must be flushed before calling close.

func (*FileSegmentEncoder) EncodeKeyValue

func (enc *FileSegmentEncoder) EncodeKeyValue(key, value []byte) error

EncodeKeyValue writes framed key & value byte slices to the file and records their offset.

func (*FileSegmentEncoder) Flush

func (enc *FileSegmentEncoder) Flush() error

Flush finalizes the file segment and appends a hashmap & trailer.

func (*FileSegmentEncoder) Open

func (enc *FileSegmentEncoder) Open() (err error)

Open opens and initializes the output file segment.

type FileSegmentIterator

type FileSegmentIterator struct {
	// contains filtered or unexported fields
}

FileSegmentIterator returns an error for sequentially iterating over a FileSegment's key/value pairs.

func (*FileSegmentIterator) Close

func (itr *FileSegmentIterator) Close() error

Close releases the iterator.

func (*FileSegmentIterator) Key

func (itr *FileSegmentIterator) Key() []byte

Key returns the current key. Must be called after Next().

func (*FileSegmentIterator) Next

func (itr *FileSegmentIterator) Next() bool

Next reads the next key/value pair into the buffer.

func (*FileSegmentIterator) Value

func (itr *FileSegmentIterator) Value() []byte

Value returns the current key. Must be called after Next().

type FileSegmentOpener

type FileSegmentOpener struct{}

FileSegmentOpener initializes and opens segments.

func NewFileSegmentOpener

func NewFileSegmentOpener() *FileSegmentOpener

NewFileSegmentOpener returns a new instance of FileSegmentOpener.

func (*FileSegmentOpener) ListSegmentNames

func (o *FileSegmentOpener) ListSegmentNames(path, table string) ([]string, error)

ListSegmentNames returns a list of all segment names for a table.

func (*FileSegmentOpener) OpenSegment

func (o *FileSegmentOpener) OpenSegment(table, name, path string) (Segment, error)

OpenSegment returns an initialized and opened segment.

type LDBSegment

type LDBSegment struct {
	// contains filtered or unexported fields
}

LDBSegement represents a mutable segment in a Table. These segments can eventually be rebuilt into immutable FileSegments.

func NewLDBSegment

func NewLDBSegment(name, path string) *LDBSegment

NewLDBSegment returns a LevelDB-based database segment.

func (*LDBSegment) Close

func (s *LDBSegment) Close() error

Close closes the underlying database.

func (*LDBSegment) CompactTo

func (s *LDBSegment) CompactTo(path string) error

CompactTo writes the segment to disk as a file segment.

func (*LDBSegment) Delete

func (s *LDBSegment) Delete(key []byte) error

Delete deletes the key from the queue and database

func (*LDBSegment) Get

func (s *LDBSegment) Get(key []byte) ([]byte, error)

Get returns the given key if it's present.

func (*LDBSegment) Has

func (s *LDBSegment) Has(key []byte) (bool, error)

Has returns true if the segment contains key.

func (*LDBSegment) Iterator

func (s *LDBSegment) Iterator() SegmentIterator

Iterator returns a sequential iterator for the segment.

func (*LDBSegment) LDB

func (s *LDBSegment) LDB() *leveldb.DB

LDB returns the underlying LevelDB database.

func (*LDBSegment) Name

func (s *LDBSegment) Name() string

Name returns the name of the segment.

func (*LDBSegment) Open

func (s *LDBSegment) Open() (err error)

Open initializes the underlying segment database.

func (*LDBSegment) Path

func (s *LDBSegment) Path() string

Path returns the path to the segment.

func (*LDBSegment) Put

func (s *LDBSegment) Put(key []byte, value []byte) error

Put inserts a value into a given key.

type MemDatabase

type MemDatabase struct {
	// contains filtered or unexported fields
}

* This is a test memory database. Do not use for any production it does not get persisted

func NewMemDatabase

func NewMemDatabase() *MemDatabase

func NewMemDatabaseWithCap

func NewMemDatabaseWithCap(size int) (*MemDatabase, error)

func (*MemDatabase) BodyTable

func (db *MemDatabase) BodyTable() common.Table

func (*MemDatabase) Close

func (db *MemDatabase) Close() error

func (*MemDatabase) Delete

func (db *MemDatabase) Delete(key []byte) error

func (*MemDatabase) Get

func (db *MemDatabase) Get(key []byte) ([]byte, error)

func (*MemDatabase) GlobalTable

func (db *MemDatabase) GlobalTable() common.Table

func (*MemDatabase) Has

func (db *MemDatabase) Has(key []byte) (bool, error)

func (*MemDatabase) HeaderTable

func (db *MemDatabase) HeaderTable() common.Table

func (*MemDatabase) Keys

func (db *MemDatabase) Keys() [][]byte

func (*MemDatabase) Len

func (db *MemDatabase) Len() int

func (*MemDatabase) NewBatch

func (db *MemDatabase) NewBatch() common.Batch

func (*MemDatabase) Put

func (db *MemDatabase) Put(key []byte, value []byte) error

func (*MemDatabase) ReceiptTable

func (db *MemDatabase) ReceiptTable() common.Table

type MutableSegment

type MutableSegment interface {
	Segment
	Put(key, value []byte) error
	Delete(key []byte) error
}

MutableSegment represents a segment that can be altered. These segments are eventually compacted into immutable segments.

type PartitionFunc

type PartitionFunc func(key []byte) string

PartitionFunc implements Partitioner.

func (PartitionFunc) Partition

func (fn PartitionFunc) Partition(key []byte) string

type Partitioner

type Partitioner interface {
	Partition(key []byte) string
}

Partitioner represents an object that returns a partition name for a given key.

type Segment

type Segment interface {
	io.Closer

	Name() string
	Path() string

	Has(key []byte) (bool, error)
	Get(key []byte) ([]byte, error)
	Iterator() SegmentIterator
}

Segment represents a subset of Table data.

type SegmentCompactor

type SegmentCompactor interface {
	CompactSegment(ctx context.Context, table string, s *LDBSegment) (Segment, error)
	UncompactSegment(ctx context.Context, table string, s Segment) (*LDBSegment, error)
}

SegmentCompactor represents an object that can compact from an LDB segment to an immutable segment and back.

type SegmentIterator

type SegmentIterator interface {
	io.Closer
	Next() bool
	Key() []byte
	Value() []byte
}

SegmentIterator represents a sequentially iterator over all the key/value pairs inside a segment.

type SegmentOpener

type SegmentOpener interface {
	OpenSegment(table, name, path string) (Segment, error)
	ListSegmentNames(path, table string) ([]string, error)
}

SegmentOpener represents an object that can instantiate and load an immutable segment.

type SegmentSet

type SegmentSet struct {
	// contains filtered or unexported fields
}

SegmentSet represents a set of segments.

func NewSegmentSet

func NewSegmentSet(maxOpenCount int) *SegmentSet

NewSegmentSet returns a new instance of SegmentSet.

func (*SegmentSet) Acquire

func (ss *SegmentSet) Acquire(name string) (Segment, error)

Acquire returns a segment by name from the set and adds increments the semaphore. If the segment is unopened then it is opened before returning. If a segment is successfully retruns then Release() must always be called by the caller.

func (*SegmentSet) Add

func (ss *SegmentSet) Add(s Segment)

Add adds s to the set.

func (*SegmentSet) Contains

func (ss *SegmentSet) Contains(name string) bool

Contains returns true if name is in the set.

func (*SegmentSet) Len

func (ss *SegmentSet) Len() int

Len returns the number of segments in the set.

func (*SegmentSet) Release

func (ss *SegmentSet) Release()

Release decrements the semaphore on the set.

func (*SegmentSet) Remove

func (ss *SegmentSet) Remove(name string)

Remove removes the segment with the given name from the set.

func (*SegmentSet) Slice

func (ss *SegmentSet) Slice() []Segment

Slice returns a slice of all segments.

type StaticPartitioner

type StaticPartitioner struct {
	Name string
}

StaticPartitioner represents a partitioner that always returns the same partition name.

func (*StaticPartitioner) Partition

func (p *StaticPartitioner) Partition(key []byte) string

Partition always returns the same partition name.

type Table

type Table struct {
	Name        string
	Path        string
	Partitioner Partitioner

	MinMutableSegmentCount int
	MinCompactionAge       time.Duration

	// Maximum number of segments that can be opened at once.
	MaxOpenSegmentCount int

	SegmentOpener    SegmentOpener
	SegmentCompactor SegmentCompactor
	// contains filtered or unexported fields
}

Table represents key/value storage for a particular data type. Contains zero or more segments that are separated by partitioner.

func NewTable

func NewTable(name, path string, partitioner Partitioner) *Table

NewTable returns a new instance of Table.

func (*Table) AcquireSegment

func (t *Table) AcquireSegment(name string) (Segment, error)

AcquireSegment returns a segment by name. Returns nil if segment does not exist. Must call ReleaseSegment when finished with the segment.

func (*Table) ActiveSegment

func (t *Table) ActiveSegment() MutableSegment

ActiveSegment returns the active segment.

func (*Table) ActiveSegmentName

func (t *Table) ActiveSegmentName() string

ActiveSegmentName the name of the current active segment.

func (*Table) Close

func (t *Table) Close() error

Close closes all segments within the table.

func (*Table) Compact

func (t *Table) Compact(ctx context.Context) error

Compact converts LDB segments into immutable file segments.

func (*Table) CreateSegmentIfNotExists

func (t *Table) CreateSegmentIfNotExists(name string) (*LDBSegment, error)

CreateSegmentIfNotExists returns a mutable segment by name. Creates a new segment if it does not exist.

func (*Table) Delete

func (t *Table) Delete(key []byte) error

Delete removes key from the database.

func (*Table) Get

func (t *Table) Get(key []byte) ([]byte, error)

Get returns the value associated with key.

func (*Table) Has

func (t *Table) Has(key []byte) (bool, error)

Has returns true if key exists in the table.

func (*Table) NewBatch

func (t *Table) NewBatch() common.Batch

func (*Table) Open

func (t *Table) Open() error

Open initializes the table and all existing segments.

func (*Table) Put

func (t *Table) Put(key, value []byte) error

Put associates a value with key.

func (*Table) ReleaseSegment

func (t *Table) ReleaseSegment(s Segment)

ReleaseSegment releases a given segment.

func (*Table) SegmentNames

func (t *Table) SegmentNames() []string

SegmentNames a sorted list of all segments names.

func (*Table) SegmentPath

func (t *Table) SegmentPath(name string) string

SegmentPath returns the path of the named segment.

func (*Table) SegmentSlice

func (t *Table) SegmentSlice() []Segment

SegmentsSlice returns a sorted slice of all segments.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL