birdsnest

package
v2.3.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 4, 2024 License: Apache-2.0 Imports: 22 Imported by: 5

Documentation

Overview

Package birdsnest bird's nest implement

Package birdsnest configuration implement

Package birdsnest contains

Package birdsnest cuckoo

Package birdsnest error

Package birdsnest filter extension

Package birdsnest filter extension factory

Package birdsnest interface

Package birdsnest key

Package birdsnest rules

Package birdsnest serialize

Package birdsnest snapshot

Package birdsnest strategy

Package birdsnest test util

Package birdsnest utils

Index

Constants

View Source 
const (
	// DefaultLoadFactor Default load factor
	DefaultLoadFactor = 0.98
)
View Source 
const (
	// Filepath birdsnest
	Filepath = "birdsnest"
)
View Source 
const TestDir = "./data/timestamp_birds_nest"

TestDir test path

Variables

View Source 
var (
	// ErrKeyCannotBeEmpty key can not be empty error
	ErrKeyCannotBeEmpty = errors.New("key cannot be empty")
	// ErrCannotModifyTheNestConfiguration cannot modify the nest configuration error
	ErrCannotModifyTheNestConfiguration = errors.New("when historical data exists, you cannot modify the nest " +
		"configuration")
	// ErrBirdsNestSizeCannotBeZero bird's nest size cannot be zero error
	ErrBirdsNestSizeCannotBeZero = errors.New("the size cannot be 0")
)
View Source 
var (
	// SeparatorString -
	SeparatorString = "-"
	// Separator chainmaker ca
	Separator = byte(202)

	// ErrKeyLengthCannotBeZero key length cannot be zero error
	ErrKeyLengthCannotBeZero = errors.New("the key length cannot be 0")
	// ErrTimestampKeyLengthCannotLessNine timestamp key length cannot less 9 error
	ErrTimestampKeyLengthCannotLessNine = errors.New("the timestamp key length cannot less 9")
	// ErrNotTimestampKey not timestamp key error
	ErrNotTimestampKey = errors.New("not timestamp txid")
	// ErrTimestampKeyIsInvalid timestamp key is invalid
	ErrTimestampKeyIsInvalid = errors.New("TxId nanosecond is invalid")
)
View Source 
var (
	// ErrFilterExtensionNotSupportMessage error filter extension not support message
	ErrFilterExtensionNotSupportMessage = "filter extension not support type: %v"
)
View Source 
var (
	// ErrKeyItsSoLongAgoError key it's so long ago error
	ErrKeyItsSoLongAgoError = errors.New("key is out of the range")
)
View Source 
var (
	// ErrKeyTimeIsNotInTheFilterRange Not error; Key time is not in the filter range
	ErrKeyTimeIsNotInTheFilterRange = errors.New("key time is not in the filter range")
)
View Source 
var KeyType_name = map[KeyType]string{
	KeyType_KTDefault:      "Default",
	KeyType_KTTimestampKey: "TimestampKey",
}

KeyType_name key type name map

View Source 
var SerializeIntervalType_name = map[SerializeIntervalType]string{
	SerializeIntervalType_Height: "Height",
	SerializeIntervalType_Timed:  "Timed",
	SerializeIntervalType_Exit:   "Exit",
}

SerializeIntervalType_name SerializeIntervalType name map

Functions

func CurrentTimestampNano 

func CurrentTimestampNano() int64

CurrentTimestampNano get current timestamp nanosecond

func Factory 

func Factory() *factory

Factory return the global tx filter factory. nolint: revive

func GenTimestampKey 

func GenTimestampKey() string

GenTimestampKey generate timestamp key

func GenTimestampKeyByNano 

func GenTimestampKeyByNano(nano int64) string

GenTimestampKeyByNano generate timestamp key by nanosecond

func GenTxId 

func GenTxId() string

GenTxId generate tx id

func LruStrategy 

func LruStrategy(bn *BirdsNestImpl) error

LruStrategy Nest filter cycle elimination strategy

func ToStrings 

func ToStrings(keys []Key) []string

ToStrings TimestampKey to string

func ToTimestampKeysAndNormalKeys 

func ToTimestampKeysAndNormalKeys(key []string) (timestampKeys []Key, normalKeys []Key)

ToTimestampKeysAndNormalKeys string to TimestampKey return timestampKeys and normalKeys

Types

type AbsoluteExpireTimeRule 

type AbsoluteExpireTimeRule struct {
	// contains filtered or unexported fields
}

AbsoluteExpireTimeRule absolute expire time rule

func NewAETRule 

func NewAETRule(absoluteExpireTime int64, logger Logger) AbsoluteExpireTimeRule

NewAETRule new absolute expire time rule

func (AbsoluteExpireTimeRule) Validate 

func (r AbsoluteExpireTimeRule) Validate(key Key) error

Validate timestamp key

type BirdsNest 

type BirdsNest interface {
	GetHeight() uint64
	SetHeight(height uint64)
	// Add the key
	Add(key Key) error
	// Adds adding Multiple Keys
	Adds(keys []Key) (result error)
	// AddsAndSetHeight Adds and SetHeight
	AddsAndSetHeight(keys []Key, height uint64) (result error)
	// Contains the key
	Contains(key Key, rules ...RuleType) (bool, error)
	ValidateRule(key Key, rules ...RuleType) error
	// Info Current cuckoos nest information and status
	Info() []uint64

	Start()
}

BirdsNest Bird's Nest

type BirdsNestConfig added in v2.3.0 

type BirdsNestConfig struct {
	// ChainId
	ChainId string `json:"chain_id,omitempty"`
	// Length cuckoo numbers
	Length uint32 `json:"length,omitempty"`
	// rules configuration
	Rules *RulesConfig `json:"rules,omitempty"`
	// Cuckoo configuration
	Cuckoo *CuckooConfig `json:"cuckoo,omitempty"`
	// Snapshot configuration
	Snapshot *SnapshotSerializerConfig `json:"snapshot,omitempty"`
}

BirdsNestConfig bird's nest configuration

type BirdsNestImpl 

type BirdsNestImpl struct {
	// contains filtered or unexported fields
}

BirdsNestImpl impl

func NewBirdsNest 

func NewBirdsNest(config *BirdsNestConfig, exitC chan struct{}, strategy Strategy, logger Logger) (
	*BirdsNestImpl, error)

NewBirdsNest Create a BirdsNest

func NewBirdsNestByNumber 

func NewBirdsNestByNumber(config *BirdsNestConfig, exitC chan struct{}, strategy Strategy, logger Logger,
	number int) (*BirdsNestImpl, error)

NewBirdsNestByNumber Create a numbered BirdsNest

func (*BirdsNestImpl) Add 

func (b *BirdsNestImpl) Add(key Key) error

Add a Key

func (*BirdsNestImpl) Adds 

func (b *BirdsNestImpl) Adds(keys []Key) error

Adds Add multiple Key

func (*BirdsNestImpl) AddsAndSetHeight 

func (b *BirdsNestImpl) AddsAndSetHeight(keys []Key, height uint64) (result error)

AddsAndSetHeight Add multiple Key and set height

func (*BirdsNestImpl) Contains 

func (b *BirdsNestImpl) Contains(key Key, rules ...RuleType) (bool, error)

Contains returns whether the filter exists sync

func (*BirdsNestImpl) Deserialize 

func (b *BirdsNestImpl) Deserialize() error

Deserialize deserialize Bird's nest

func (*BirdsNestImpl) GetHeight 

func (b *BirdsNestImpl) GetHeight() uint64

GetHeight get current height

func (*BirdsNestImpl) Info 

func (b *BirdsNestImpl) Info() []uint64

Info info index 0 height index 1 cuckoo size index 2 current index index 3 total cuckoo size index 4 total space occupied by cuckoo

func (*BirdsNestImpl) Serialize 

func (b *BirdsNestImpl) Serialize(index int) error

Serialize cuckoo of index in current BirdsNest

func (*BirdsNestImpl) SetHeight 

func (b *BirdsNestImpl) SetHeight(height uint64)

SetHeight set height

func (*BirdsNestImpl) Start 

func (b *BirdsNestImpl) Start()

Start serialize monitor TODO Goroutinue should be turned off using context.Context here

func (*BirdsNestImpl) ValidateRule 

func (b *BirdsNestImpl) ValidateRule(key Key, rules ...RuleType) error

ValidateRule validate rule

type BirdsNestSerialize added in v2.3.0 

type BirdsNestSerialize struct {
	// Bird's Nest config
	Config *BirdsNestConfig `protobuf:"bytes,1,opt,name=config,proto3" json:"config,omitempty"`
	// The final height
	Height uint64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"`
	// current index
	CurrentIndex uint32 `protobuf:"varint,3,opt,name=currentIndex,proto3" json:"currentIndex,omitempty"`
	// A group of cuckoos filter
	Filters []*CuckooFilterSerialize `protobuf:"bytes,4,rep,name=filters,proto3" json:"filters,omitempty"`
}

BirdsNestSerialize Bird's nest serialize

type BlockHeightSerializeIntervalConfig added in v2.3.0 

type BlockHeightSerializeIntervalConfig struct {
	// Block height Interval
	Interval uint64 `json:"interval,omitempty"`
}

BlockHeightSerializeIntervalConfig Block height serialization interval configuration

type CuckooConfig added in v2.3.0 

type CuckooConfig struct {
	// KeyType key type
	KeyType KeyType `json:"key_type,omitempty"`
	// TagsPerBucket num of tags for each bucket, which is b in paper. tag is fingerprint, which is f in paper.
	TagsPerBucket uint32 `json:"tags_per_bucket,omitempty"`
	// BitsPerItem num of bits for each item, which is length of tag(fingerprint)
	BitsPerItem uint32 `json:"bits_per_item,omitempty"`
	// MaxNumKeys num of keys that filter will store. this value should close to and lower
	//					 nextPow2(maxNumKeys/tagsPerBucket) * maxLoadFactor. cause table.NumBuckets is always a
	//					 power of two
	MaxNumKeys uint32 `json:"max_num_keys,omitempty"`
	// TableType has two constant parameters to choose from:
	// TableTypeSingle normal single table
	// TableTypePacked packed table, use semi-sort to save 1 bit per item
	TableType uint32 `json:"table_type,omitempty"`
}

CuckooConfig Cuckoo configuration

type CuckooFilter 

type CuckooFilter interface {
	// IsFull cuckoo filter is full
	IsFull() bool
	// Add cuckoo filter
	Add(key Key) (bool, error)
	// Contains cuckoo filter
	Contains(key Key) (bool, error)
	// Encode cuckoo filter
	Encode() (FilterEncoder, error)
	// Extension cuckoo filter
	Extension() FilterExtension
	// Info print cuckoo filter
	Info() []uint64
}

CuckooFilter cuckoo filter

func NewCuckooFilter 

func NewCuckooFilter(config *CuckooConfig) CuckooFilter

NewCuckooFilter new cuckoo filter Params: common.CuckooConfig.TableType : has two constant parameters to choose from: 1. TableTypeSingle normal single table 2. TableTypePacked packed table, use semi-sort to save 1 bit per item common.CuckooConfig.TagsPerBucket: num of tags for each bucket, which is b in paper. tag is fingerprint, which is f in paper. common.CuckooConfig.MaxNumKeys : num of keys that filter will store. this value should close to and lower nextPow2(maxNumKeys/tagsPerBucket) * maxLoadFactor. cause table.NumBuckets is always a power of two common.CuckooConfig.BitsPerItem : num of bits for each item, which is length of tag(fingerprint) common.CuckooConfig.TableType : common.CuckooConfig.KeyType : 0 TableTypeSingle normal single table 1 TableTypePacked packed table, use semi-sort to save 1 bit per item 1 is recommended Result: CuckooFilter

type CuckooFilterImpl 

type CuckooFilterImpl struct {
	// contains filtered or unexported fields
}

CuckooFilterImpl Cuckoo Filter

func NewCuckooFilterByDecode 

func NewCuckooFilterByDecode(filter *birdsnestpb.CuckooFilter) (*CuckooFilterImpl, error)

NewCuckooFilterByDecode new cuckoo filter by decode

func (*CuckooFilterImpl) Add 

func (c *CuckooFilterImpl) Add(key Key) (bool, error)

Add key to cuckoo filter

func (*CuckooFilterImpl) Config 

func (c *CuckooFilterImpl) Config() ([]byte, error)

Config config

func (*CuckooFilterImpl) Contains 

func (c *CuckooFilterImpl) Contains(key Key) (bool, error)

Contains Whether the cuckoo filter contains keys

func (*CuckooFilterImpl) Encode 

func (c *CuckooFilterImpl) Encode() (FilterEncoder, error)

Encode cuckoo filter

func (*CuckooFilterImpl) Extension 

func (c *CuckooFilterImpl) Extension() FilterExtension

Extension cuckoo filter

func (*CuckooFilterImpl) Info 

func (c *CuckooFilterImpl) Info() []uint64

Info info index 0 cuckoo size index 1 Space occupied by cuckoo

func (*CuckooFilterImpl) IsFull 

func (c *CuckooFilterImpl) IsFull() bool

IsFull is full

type CuckooFilterSerialize added in v2.3.0 

type CuckooFilterSerialize struct {
	// The field "cuckoo" is used to hold the serialized data of the cuckoo
	// Pb limit: The size of bytes cannot be larger than 4 GB
	Cuckoo []byte `protobuf:"bytes,1,opt,name=cuckoo,proto3" json:"cuckoo,omitempty"`
	// Carries the ID of the time
	Extension []byte `protobuf:"bytes,2,opt,name=extension,proto3" json:"extension,omitempty"`
	// cuckoo configuration
	Config []byte `protobuf:"bytes,3,opt,name=config,proto3" json:"config,omitempty"`
}

CuckooFilterSerialize cuckoo filter serialize struct

type DefaultFilterExtension 

type DefaultFilterExtension struct {
}

DefaultFilterExtension default filter extension

func NewDefaultFilterExtension 

func NewDefaultFilterExtension() *DefaultFilterExtension

NewDefaultFilterExtension new default filter extension

func (DefaultFilterExtension) Serialize 

func (d DefaultFilterExtension) Serialize() []byte

Serialize serialize

func (DefaultFilterExtension) Store 

func (d DefaultFilterExtension) Store(Key) error

Store store

func (DefaultFilterExtension) Validate 

func (d DefaultFilterExtension) Validate(Key, bool) error

Validate validate

type FilterEncoder 

type FilterEncoder struct {
	// contains filtered or unexported fields
}

FilterEncoder filter encoder

type FilterExtension 

type FilterExtension interface {
	// Validate validate key
	Validate(key Key, full bool) error
	// Store store filter extension
	Store(key Key) error
	// Serialize filter extension
	Serialize() []byte
}

FilterExtension filter extension

func DeserializeDefault 

func DeserializeDefault() FilterExtension

DeserializeDefault new default deserialize extension

func ExtensionDeserialize 

func ExtensionDeserialize(bytes []byte) (FilterExtension, error)

ExtensionDeserialize Extension deserialize

func NewTimestampFilterExtension 

func NewTimestampFilterExtension() FilterExtension

NewTimestampFilterExtension new timestamp filter extension

type FilterExtensionType added in v2.3.0 

type FilterExtensionType int32

FilterExtensionType filter extension type 

const (
	// FilterExtensionType_FETDefault default filter extension type
	FilterExtensionType_FETDefault FilterExtensionType = 0
	// FilterExtensionType_FETTimestamp timestamp filter extension type
	FilterExtensionType_FETTimestamp FilterExtensionType = 1
)

type Key 

type Key interface {
	// Parse the key
	Parse() ([][]byte, error)
	// Key bytes
	Key() []byte
	// Len The length of the key
	Len() int
	// String key to string
	String() string
	// GetNano get nanosecond
	GetNano() int64
}

Key filter key

func GetTimestampKey 

func GetTimestampKey() Key

GetTimestampKey get timestamp key

func GetTimestampKeyByNano 

func GetTimestampKeyByNano(i int64) Key

GetTimestampKeyByNano get timestamp key by nanosecond

func GetTimestampKeys 

func GetTimestampKeys(n int) []Key

GetTimestampKeys get timestamp key collections

type KeyType added in v2.3.0 

type KeyType int32

KeyType key type 

const (
	// KeyType_KTDefault default type
	KeyType_KTDefault KeyType = 0
	// KeyType_KTTimestampKey timestamp type
	KeyType_KTTimestampKey KeyType = 1
)

type Logger 

type Logger interface {
	// Debugf debug format
	Debugf(format string, args ...interface{})
	// Errorf error format
	Errorf(format string, args ...interface{})
	// Infof info format
	Infof(format string, args ...interface{})
	// Warnf warn format
	Warnf(format string, args ...interface{})
}

Logger logger

type Rule 

type Rule interface {
	Validate(Key) error
}

Rule rule

type RuleType added in v2.3.0 

type RuleType int32

RuleType rule type 

const (
	// RuleType_AbsoluteExpireTime absolute expire time
	RuleType_AbsoluteExpireTime RuleType = 0
)

type RulesConfig added in v2.3.0 

type RulesConfig struct {
	// absolute expire time second
	AbsoluteExpireTime int64 `json:"absolute_expire_time,omitempty"`
}

RulesConfig rules configuration

type SerializeIntervalType added in v2.3.0 

type SerializeIntervalType int32

SerializeIntervalType Serialize interval type 

const (
	// SerializeIntervalType_Height Timed serialize type
	SerializeIntervalType_Height SerializeIntervalType = 0
	// SerializeIntervalType_Timed Timed serialize type
	SerializeIntervalType_Timed SerializeIntervalType = 1
	// SerializeIntervalType_Exit  Exit serialize type
	SerializeIntervalType_Exit SerializeIntervalType = 2
)

type Serializer 

type Serializer interface {
	// Serialize serialize
	Serialize() error
	// Deserialize deserialize
	Deserialize() error
}

Serializer serializer

type Snapshot added in v2.3.0 

type Snapshot interface {
	// Write write
	Write(data []byte) error
	// Read read
	Read() ([]byte, error)
}

Snapshot snapshot

type SnapshotSerializerConfig added in v2.3.0 

type SnapshotSerializerConfig struct {
	// Type serialize interval type
	Type SerializeIntervalType `json:"type,omitempty"`
	// Timed serialize interval configuration
	Timed *TimedSerializeIntervalConfig `json:"timed,omitempty"`
	// BlockHeight serialize interval configuration
	BlockHeight *BlockHeightSerializeIntervalConfig `json:"block_height,omitempty"`
	// Path filepath
	Path string `json:"path,omitempty"`
}

SnapshotSerializerConfig Snapshot serializer config

type SprintfError 

type SprintfError string

SprintfError error

func NewError 

func NewError(s string, args ...interface{}) SprintfError

NewError new error

func (SprintfError) Error 

func (s SprintfError) Error() string

Error error information

type Strategy 

type Strategy func(bn *BirdsNestImpl) error

Strategy function

type TestLogger 

type TestLogger struct {
	T *testing.T
}

TestLogger test logger

func (TestLogger) Debugf 

func (t TestLogger) Debugf(format string, args ...interface{})

Debugf debug format

func (TestLogger) Errorf 

func (t TestLogger) Errorf(format string, args ...interface{})

Errorf error format

func (TestLogger) Infof 

func (t TestLogger) Infof(format string, args ...interface{})

Infof info format

func (TestLogger) Warnf added in v2.3.0 

func (t TestLogger) Warnf(format string, args ...interface{})

Warnf warn format

type TimedSerializeIntervalConfig added in v2.3.0 

type TimedSerializeIntervalConfig struct {
	// Timed Interval
	Interval int64 `json:"interval,omitempty"`
}

TimedSerializeIntervalConfig Timed serialization interval configuration

type TimestampFilterExtension 

type TimestampFilterExtension struct {
	// contains filtered or unexported fields
}

TimestampFilterExtension timestamp filter extension

func DeserializeTimestamp 

func DeserializeTimestamp(bytes []byte) (*TimestampFilterExtension, error)

DeserializeTimestamp deserialize timestamp by bytes

func (*TimestampFilterExtension) Serialize 

func (t *TimestampFilterExtension) Serialize() []byte

Serialize serialize

func (*TimestampFilterExtension) Store 

func (t *TimestampFilterExtension) Store(key Key) error

Store store start time and end time

func (*TimestampFilterExtension) Validate 

func (t *TimestampFilterExtension) Validate(key Key, full bool) error

Validate validate

type TimestampKey 

type TimestampKey []byte

TimestampKey Converting TxId directly using TimestampKey is not allowed, see ToTimestampKey

func ToTimestampKey 

func ToTimestampKey(txId string) (TimestampKey, error)

ToTimestampKey strings txid to timestamp key It believes that you should be a string. For the format like "0 000", it is not in hexadecimal format. Hexadecimal requires one character to occupy half a byte, and at least two characters are needed for a byte. For "00", it will be considered as hexadecimal, which is just "0". Both "00" and "000" will be considered as decimal, and it will only return an error. Later on, it will directly query the database, so there's no issue. "00" will be considered as hexadecimal, and if the length of the split is insufficient later on, it will panic.

func (TimestampKey) GetNano 

func (k TimestampKey) GetNano() int64

GetNano get nanosecond

func (TimestampKey) Key 

func (k TimestampKey) Key() []byte

Key to bytes

func (TimestampKey) Len 

func (k TimestampKey) Len() int

Len length

func (TimestampKey) Parse 

func (k TimestampKey) Parse() ([][]byte, error)

Parse parse

func (TimestampKey) String 

func (k TimestampKey) String() string

type WalSnapshot 

type WalSnapshot struct {
	// contains filtered or unexported fields
}

WalSnapshot wal snapshot

func NewWalSnapshot 

func NewWalSnapshot(path, name string, number int) (*WalSnapshot, error)

NewWalSnapshot new wal snapshot eg: data/tx_filter/chainN/birdsnestN

func (*WalSnapshot) Read 

func (s *WalSnapshot) Read() ([]byte, error)

Read safe read wal

func (*WalSnapshot) Write 

func (s *WalSnapshot) Write(data []byte) error

Write safe write wal

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.