Documentation ¶
Overview ¶
Package birdsnest bird's nest implement
Package birdsnest configuration implement ¶
Copyright (C) BABEC. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Copyright (C) BABEC. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Copyright (C) BABEC. All rights reserved.
SPDX-License-Identifier: Apache-2.0
Package birdsnest filter extension ¶
Package birdsnest interface ¶
Package birdsnest key ¶
Package birdsnest rules ¶
Package birdsnest serialize ¶
Package birdsnest snapshot ¶
Package birdsnest strategy ¶
Package birdsnest test util ¶
Package birdsnest utils
Index ¶
- Constants
- Variables
- func CurrentTimestampNano() int64
- func Factory() *factory
- func GenTimestampKey() string
- func GenTimestampKeyByNano(nano int64) string
- func GenTxId() string
- func LruStrategy(bn *BirdsNestImpl) error
- func ToStrings(keys []Key) []string
- func ToTimestampKeysAndNormalKeys(key []string) (timestampKeys []Key, normalKeys []Key)
- type AbsoluteExpireTimeRule
- type BirdsNest
- type BirdsNestConfig
- type BirdsNestImpl
- func (b *BirdsNestImpl) Add(key Key) error
- func (b *BirdsNestImpl) Adds(keys []Key) error
- func (b *BirdsNestImpl) AddsAndSetHeight(keys []Key, height uint64) (result error)
- func (b *BirdsNestImpl) Contains(key Key, rules ...RuleType) (bool, error)
- func (b *BirdsNestImpl) Deserialize() error
- func (b *BirdsNestImpl) GetHeight() uint64
- func (b *BirdsNestImpl) Info() []uint64
- func (b *BirdsNestImpl) Serialize(index int) error
- func (b *BirdsNestImpl) SetHeight(height uint64)
- func (b *BirdsNestImpl) Start()
- func (b *BirdsNestImpl) ValidateRule(key Key, rules ...RuleType) error
- type BirdsNestSerialize
- type BlockHeightSerializeIntervalConfig
- type CuckooConfig
- type CuckooFilter
- type CuckooFilterImpl
- func (c *CuckooFilterImpl) Add(key Key) (bool, error)
- func (c *CuckooFilterImpl) Config() ([]byte, error)
- func (c *CuckooFilterImpl) Contains(key Key) (bool, error)
- func (c *CuckooFilterImpl) Encode() (FilterEncoder, error)
- func (c *CuckooFilterImpl) Extension() FilterExtension
- func (c *CuckooFilterImpl) Info() []uint64
- func (c *CuckooFilterImpl) IsFull() bool
- type CuckooFilterSerialize
- type DefaultFilterExtension
- type FilterEncoder
- type FilterExtension
- type FilterExtensionType
- type Key
- type KeyType
- type Logger
- type Rule
- type RuleType
- type RulesConfig
- type SerializeIntervalType
- type Serializer
- type Snapshot
- type SnapshotSerializerConfig
- type SprintfError
- type Strategy
- type TestLogger
- type TimedSerializeIntervalConfig
- type TimestampFilterExtension
- type TimestampKey
- type WalSnapshot
Constants ¶
const (
// DefaultLoadFactor Default load factor
DefaultLoadFactor = 0.98
)
const (
Filepath = "birdsnest"
)
const TestDir = "./data/timestamp_birds_nest"
TestDir test path
Variables ¶
var ( // ErrKeyCannotBeEmpty key can not be empty error ErrKeyCannotBeEmpty = errors.New("key cannot be empty") // ErrCannotModifyTheNestConfiguration cannot modify the nest configuration error ErrCannotModifyTheNestConfiguration = errors.New("when historical data exists, you cannot modify the nest " + "configuration") // ErrBirdsNestSizeCannotBeZero bird's nest size cannot be zero error ErrBirdsNestSizeCannotBeZero = errors.New("the size cannot be 0") )
var ( // SeparatorString - SeparatorString = "-" // Separator chainmaker ca Separator = byte(202) // ErrKeyLengthCannotBeZero key length cannot be zero error ErrKeyLengthCannotBeZero = errors.New("the key length cannot be 0") // ErrNotTimestampKey not timestamp key error ErrNotTimestampKey = errors.New("not timestamp txid") // ErrTimestampKeyIsInvalid timestamp key is invalid ErrTimestampKeyIsInvalid = errors.New("TxId nanosecond is invalid") )
var (
ErrFilterExtensionNotSupportMessage = "filter extension not support type: %v"
)
var ( // ErrKeyItsSoLongAgoError key it's so long ago error ErrKeyItsSoLongAgoError = errors.New("key is out of the range") )
var ( // ErrKeyTimeIsNotInTheFilterRange Not error; Key time is not in the filter range ErrKeyTimeIsNotInTheFilterRange = errors.New("key time is not in the filter range") )
var KeyType_name = map[KeyType]string{ KeyType_KTDefault: "Default", KeyType_KTTimestampKey: "TimestampKey", }
KeyType_name key type name map
var SerializeIntervalType_name = map[SerializeIntervalType]string{ SerializeIntervalType_Height: "Height", SerializeIntervalType_Timed: "Timed", SerializeIntervalType_Exit: "Exit", }
SerializeIntervalType_name SerializeIntervalType name map
Functions ¶
func CurrentTimestampNano ¶
func CurrentTimestampNano() int64
CurrentTimestampNano get current timestamp nanosecond
func Factory ¶
func Factory() *factory
func GenTimestampKeyByNano ¶
GenTimestampKeyByNano generate timestamp key by nanosecond
func LruStrategy ¶
func LruStrategy(bn *BirdsNestImpl) error
LruStrategy Nest filter cycle elimination strategy
Types ¶
type AbsoluteExpireTimeRule ¶
type AbsoluteExpireTimeRule struct {
// contains filtered or unexported fields
}
AbsoluteExpireTimeRule absolute expire time rule
func NewAETRule ¶
func NewAETRule(absoluteExpireTime int64, logger Logger) AbsoluteExpireTimeRule
NewAETRule new absolute expire time rule
func (AbsoluteExpireTimeRule) Validate ¶
func (r AbsoluteExpireTimeRule) Validate(key Key) error
Validate timestamp key
type BirdsNest ¶
type BirdsNest interface { GetHeight() uint64 SetHeight(height uint64) // Add the key Add(key Key) error // Adds adding Multiple Keys Adds(keys []Key) (result error) // AddsAndSetHeight Adds and SetHeight AddsAndSetHeight(keys []Key, height uint64) (result error) // Contains the key Contains(key Key, rules ...RuleType) (bool, error) ValidateRule(key Key, rules ...RuleType) error // Info Current cuckoos nest information and status Info() []uint64 Start() }
BirdsNest Bird's Nest
type BirdsNestConfig ¶ added in v2.3.0
type BirdsNestConfig struct { // ChainId ChainId string `json:"chain_id,omitempty"` // Length cuckoo numbers Length uint32 `json:"length,omitempty"` // rules configuration Rules *RulesConfig `json:"rules,omitempty"` // Cuckoo configuration Cuckoo *CuckooConfig `json:"cuckoo,omitempty"` // Snapshot configuration Snapshot *SnapshotSerializerConfig `json:"snapshot,omitempty"` }
type BirdsNestImpl ¶
type BirdsNestImpl struct {
// contains filtered or unexported fields
}
BirdsNestImpl impl
func NewBirdsNest ¶
func NewBirdsNest(config *BirdsNestConfig, exitC chan struct{}, strategy Strategy, logger Logger) ( *BirdsNestImpl, error)
NewBirdsNest Create a BirdsNest
func NewBirdsNestByNumber ¶
func NewBirdsNestByNumber(config *BirdsNestConfig, exitC chan struct{}, strategy Strategy, logger Logger, number int) (*BirdsNestImpl, error)
NewBirdsNestByNumber Create a numbered BirdsNest
func (*BirdsNestImpl) AddsAndSetHeight ¶
func (b *BirdsNestImpl) AddsAndSetHeight(keys []Key, height uint64) (result error)
AddsAndSetHeight Add multiple Key and set height
func (*BirdsNestImpl) Contains ¶
func (b *BirdsNestImpl) Contains(key Key, rules ...RuleType) (bool, error)
Contains returns whether the filter exists sync
func (*BirdsNestImpl) Deserialize ¶
func (b *BirdsNestImpl) Deserialize() error
Deserialize deserialize Bird's nest
func (*BirdsNestImpl) GetHeight ¶
func (b *BirdsNestImpl) GetHeight() uint64
GetHeight get current height
func (*BirdsNestImpl) Info ¶
func (b *BirdsNestImpl) Info() []uint64
Info index 0 height index 1 cuckoo size index 2 current index index 3 total cuckoo size index 4 total space occupied by cuckoo
func (*BirdsNestImpl) Serialize ¶
func (b *BirdsNestImpl) Serialize(index int) error
Serialize cuckoo of index in current BirdsNest
func (*BirdsNestImpl) SetHeight ¶
func (b *BirdsNestImpl) SetHeight(height uint64)
SetHeight set height
func (*BirdsNestImpl) Start ¶
func (b *BirdsNestImpl) Start()
Start serialize monitor TODO Goroutinue should be turned off using context.Context here
func (*BirdsNestImpl) ValidateRule ¶
func (b *BirdsNestImpl) ValidateRule(key Key, rules ...RuleType) error
type BirdsNestSerialize ¶ added in v2.3.0
type BirdsNestSerialize struct { // Bird's Nest config Config *BirdsNestConfig `protobuf:"bytes,1,opt,name=config,proto3" json:"config,omitempty"` // The final height Height uint64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` // current index CurrentIndex uint32 `protobuf:"varint,3,opt,name=currentIndex,proto3" json:"currentIndex,omitempty"` // A group of cuckoos filter Filters []*CuckooFilterSerialize `protobuf:"bytes,4,rep,name=filters,proto3" json:"filters,omitempty"` }
BirdsNestSerialize Bird's nest serialize
type BlockHeightSerializeIntervalConfig ¶ added in v2.3.0
type BlockHeightSerializeIntervalConfig struct { // Block height Interval Interval uint64 `json:"interval,omitempty"` }
BlockHeightSerializeIntervalConfig Block height serialization interval configuration
type CuckooConfig ¶ added in v2.3.0
type CuckooConfig struct { // KeyType key type KeyType KeyType `json:"key_type,omitempty"` // TagsPerBucket num of tags for each bucket, which is b in paper. tag is fingerprint, which is f in paper. TagsPerBucket uint32 `json:"tags_per_bucket,omitempty"` // BitsPerItem num of bits for each item, which is length of tag(fingerprint) BitsPerItem uint32 `json:"bits_per_item,omitempty"` // MaxNumKeys num of keys that filter will store. this value should close to and lower // nextPow2(maxNumKeys/tagsPerBucket) * maxLoadFactor. cause table.NumBuckets is always a // power of two MaxNumKeys uint32 `json:"max_num_keys,omitempty"` // TableType has two constant parameters to choose from: // TableTypeSingle normal single table // TableTypePacked packed table, use semi-sort to save 1 bit per item TableType uint32 `json:"table_type,omitempty"` }
CuckooConfig Cuckoo configuration
type CuckooFilter ¶
type CuckooFilter interface { IsFull() bool Add(key Key) (bool, error) Contains(key Key) (bool, error) Encode() (FilterEncoder, error) Extension() FilterExtension Info() []uint64 }
func NewCuckooFilter ¶
func NewCuckooFilter(config *CuckooConfig) CuckooFilter
NewCuckooFilter Params: common.CuckooConfig.TableType : has two constant parameters to choose from: 1. TableTypeSingle normal single table 2. TableTypePacked packed table, use semi-sort to save 1 bit per item common.CuckooConfig.TagsPerBucket: num of tags for each bucket, which is b in paper. tag is fingerprint, which is f in paper. common.CuckooConfig.MaxNumKeys : num of keys that filter will store. this value should close to and lower nextPow2(maxNumKeys/tagsPerBucket) * maxLoadFactor. cause table.NumBuckets is always a power of two common.CuckooConfig.BitsPerItem : num of bits for each item, which is length of tag(fingerprint) common.CuckooConfig.TableType : common.CuckooConfig.KeyType : 0 TableTypeSingle normal single table 1 TableTypePacked packed table, use semi-sort to save 1 bit per item 1 is recommended Result: CuckooFilter
type CuckooFilterImpl ¶
type CuckooFilterImpl struct {
// contains filtered or unexported fields
}
CuckooFilterImpl Cuckoo Filter
func NewCuckooFilterByDecode ¶
func NewCuckooFilterByDecode(filter *birdsnestpb.CuckooFilter) (*CuckooFilterImpl, error)
func (*CuckooFilterImpl) Add ¶
func (c *CuckooFilterImpl) Add(key Key) (bool, error)
Add key to cuckoo filter
func (*CuckooFilterImpl) Config ¶
func (c *CuckooFilterImpl) Config() ([]byte, error)
func (*CuckooFilterImpl) Contains ¶
func (c *CuckooFilterImpl) Contains(key Key) (bool, error)
Contains Whether the cuckoo filter contains keys
func (*CuckooFilterImpl) Encode ¶
func (c *CuckooFilterImpl) Encode() (FilterEncoder, error)
func (*CuckooFilterImpl) Extension ¶
func (c *CuckooFilterImpl) Extension() FilterExtension
func (*CuckooFilterImpl) Info ¶
func (c *CuckooFilterImpl) Info() []uint64
Info index 0 cuckoo size index 1 Space occupied by cuckoo
type CuckooFilterSerialize ¶ added in v2.3.0
type CuckooFilterSerialize struct { // The field "cuckoo" is used to hold the serialized data of the cuckoo // Pb limit: The size of bytes cannot be larger than 4 GB Cuckoo []byte `protobuf:"bytes,1,opt,name=cuckoo,proto3" json:"cuckoo,omitempty"` // Carries the ID of the time Extension []byte `protobuf:"bytes,2,opt,name=extension,proto3" json:"extension,omitempty"` // cuckoo configuration Config []byte `protobuf:"bytes,3,opt,name=config,proto3" json:"config,omitempty"` }
type DefaultFilterExtension ¶
type DefaultFilterExtension struct { }
DefaultFilterExtension default filter extension
func NewDefaultFilterExtension ¶
func NewDefaultFilterExtension() *DefaultFilterExtension
NewDefaultFilterExtension new default filter extension
func (DefaultFilterExtension) Serialize ¶
func (d DefaultFilterExtension) Serialize() []byte
Serialize serialize
func (DefaultFilterExtension) Validate ¶
func (d DefaultFilterExtension) Validate(Key, bool) error
Validate validate
type FilterEncoder ¶
type FilterEncoder struct {
// contains filtered or unexported fields
}
type FilterExtension ¶
type FilterExtension interface { // Validate validate key Validate(key Key, full bool) error Store(key Key) error Serialize() []byte }
FilterExtension filter extension
func DeserializeDefault ¶
func DeserializeDefault() FilterExtension
DeserializeDefault new default deserialize extension
func ExtensionDeserialize ¶
func ExtensionDeserialize(bytes []byte) (FilterExtension, error)
ExtensionDeserialize Extension deserialize
func NewTimestampFilterExtension ¶
func NewTimestampFilterExtension() FilterExtension
NewTimestampFilterExtension new timestamp filter extension
type FilterExtensionType ¶ added in v2.3.0
type FilterExtensionType int32
FilterExtensionType filter extension type
const ( // FilterExtensionType_FETDefault default filter extension type FilterExtensionType_FETDefault FilterExtensionType = 0 // FilterExtensionType_FETTimestamp timestamp filter extension type FilterExtensionType_FETTimestamp FilterExtensionType = 1 )
type Key ¶
type Key interface { // Parse the key Parse() ([][]byte, error) // Key bytes Key() []byte // Len The length of the key Len() int // String key to string String() string // GetNano get nanosecond GetNano() int64 }
Key filter key
func GetTimestampKeyByNano ¶
GetTimestampKeyByNano get timestamp key by nanosecond
type Logger ¶
type RuleType ¶ added in v2.3.0
type RuleType int32
RuleType rule type
const ( // RuleType_AbsoluteExpireTime absolute expire time RuleType_AbsoluteExpireTime RuleType = 0 )
type RulesConfig ¶ added in v2.3.0
type RulesConfig struct { // absolute expire time second AbsoluteExpireTime int64 `json:"absolute_expire_time,omitempty"` }
RulesConfig rules configuration
type SerializeIntervalType ¶ added in v2.3.0
type SerializeIntervalType int32
SerializeIntervalType Serialize interval type
const ( // SerializeIntervalType_Height Timed serialize type SerializeIntervalType_Height SerializeIntervalType = 0 // SerializeIntervalType_Timed Timed serialize type SerializeIntervalType_Timed SerializeIntervalType = 1 // SerializeIntervalType_Exit Exit serialize type SerializeIntervalType_Exit SerializeIntervalType = 2 )
type Snapshot ¶ added in v2.3.0
type SnapshotSerializerConfig ¶ added in v2.3.0
type SnapshotSerializerConfig struct { // Type serialize interval type Type SerializeIntervalType `json:"type,omitempty"` // Timed serialize interval configuration Timed *TimedSerializeIntervalConfig `json:"timed,omitempty"` // BlockHeight serialize interval configuration BlockHeight *BlockHeightSerializeIntervalConfig `json:"block_height,omitempty"` // Path filepath Path string `json:"path,omitempty"` }
SnapshotSerializerConfig Snapshot serializer config
type SprintfError ¶
type SprintfError string
func NewError ¶
func NewError(s string, args ...interface{}) SprintfError
type TestLogger ¶
TestLogger test logger
func (TestLogger) Debugf ¶
func (t TestLogger) Debugf(format string, args ...interface{})
Debugf debug format
func (TestLogger) Errorf ¶
func (t TestLogger) Errorf(format string, args ...interface{})
Errorf error format
func (TestLogger) Infof ¶
func (t TestLogger) Infof(format string, args ...interface{})
Infof info format
func (TestLogger) Warnf ¶ added in v2.3.0
func (t TestLogger) Warnf(format string, args ...interface{})
Warnf warn format
type TimedSerializeIntervalConfig ¶ added in v2.3.0
type TimedSerializeIntervalConfig struct { // Timed Interval Interval int64 `json:"interval,omitempty"` }
TimedSerializeIntervalConfig Timed serialization interval configuration
type TimestampFilterExtension ¶
type TimestampFilterExtension struct {
// contains filtered or unexported fields
}
TimestampFilterExtension timestamp filter extension
func DeserializeTimestamp ¶
func DeserializeTimestamp(bytes []byte) (*TimestampFilterExtension, error)
DeserializeTimestamp deserialize timestamp by bytes
func (*TimestampFilterExtension) Serialize ¶
func (t *TimestampFilterExtension) Serialize() []byte
Serialize serialize
func (*TimestampFilterExtension) Store ¶
func (t *TimestampFilterExtension) Store(key Key) error
Store store start time and end time
func (*TimestampFilterExtension) Validate ¶
func (t *TimestampFilterExtension) Validate(key Key, full bool) error
Validate validate
type TimestampKey ¶
type TimestampKey []byte
TimestampKey Converting TxId directly using TimestampKey is not allowed, see ToTimestampKey
func ToTimestampKey ¶
func ToTimestampKey(txId string) (TimestampKey, error)
ToTimestampKey strings txid to timestamp key
func (TimestampKey) String ¶
func (k TimestampKey) String() string