bstore

package
v4.15.0+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 17, 2018 License: GPL-3.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	VSIZE           = 1024
	KFACTOR         = 64
	VBSIZE          = 2 + 9*VSIZE + 9*VSIZE + 2*VSIZE //Worst case with huffman
	CBSIZE          = 1 + KFACTOR*9*6
	DBSIZE          = VBSIZE
	PWFACTOR        = uint8(6) //1<<6 == 64
	RELOCATION_BASE = 0xFF00000000000000
	SUPERBLOCK_SIZE = 8 + 8 // address + walltime
)

Note to self, if you bump VSIZE such that the max blob goes past 2^16, make sure to adapt providers

View Source 
const ABSZERO = 1
View Source 
const FULLZERO = 2
View Source 
const FlagsMask uint8 = 3
View Source 
const LatestGeneration = uint64(^(uint64(0)))
View Source 
const SUPERBLOCK_CACHE_SIZE = 32768

Like roughly 1MB

View Source 
const SUPERBLOCK_PRUNE_SIZE = 1024

If the superblock cache exceeds the max above, randomly delete this many elements from it

View Source 
const VALUE = 0

These functions allow us to read/write the packed numbers in the datablocks These are huffman encoded in big endian 0xxx xxxx 7 0x00 10xx xxxx +1 14 0x80 1100 xxxx +2 20 0xC0 1101 xxxx +3 28 0xD0 1110 xxxx +4 36 0xE0 1111 00xx +5 42 0xF0 1111 01xx +6 50 0xF4 1111 10xx +7 58 0xF8 1111 1100 +8 64 0xFC 1111 1101 +0 ABSZERO (special symbol) 0xFD 1111 1110 +0 FULLZERO (special symbol) 0xFE

Variables

View Source 
var ErrDatablockNotFound = errors.New("Coreblock not found")
View Source 
var ErrGenerationNotFound = errors.New("Generation not found")

Functions

func CreateDatabase 

func CreateDatabase(cfg configprovider.Configuration, overwrite bool)

func LinkAndStore 

func LinkAndStore(uuid []byte, bs *BlockStore, bp bprovider.StorageProvider, vblocks []*Vectorblock, cblocks []*Coreblock) map[uint64]uint64

func UUIDToMapKey 

func UUIDToMapKey(id uuid.UUID) [16]byte

Types

type BlockStore 

type BlockStore struct {
	// contains filtered or unexported fields
}

func NewBlockStore 

func NewBlockStore(cfg configprovider.Configuration, rm *rez.RezManager) (*BlockStore, error)
func (bs *BlockStore) UnlinkGenerations(id uuid.UUID, sgen uint64, egen uint64) error {
	iter := bs.db.C("superblocks").Find(bson.M{"uuid": id.String(), "gen": bson.M{"$gte": sgen, "$lt": egen}, "unlinked": false}).Iter()
	rs := fake_sblock{}
	for iter.Next(&rs) {
		rs.Unlinked = true
		_, err := bs.db.C("superblocks").Upsert(bson.M{"uuid": id.String(), "gen": rs.Gen}, rs)
		if err != nil {
			lg.Panic(err)
		}
	}
	return nil
}

func (*BlockStore) FlushSuperblockFromCache 

func (bs *BlockStore) FlushSuperblockFromCache(uu uuid.UUID)

func (*BlockStore) FreeCoreblock 

func (bs *BlockStore) FreeCoreblock(cb **Coreblock)

func (*BlockStore) FreeVectorblock 

func (bs *BlockStore) FreeVectorblock(vb **Vectorblock)

func (*BlockStore) LASMetrics 

func (bs *BlockStore) LASMetrics(m *LASMetric)

func (*BlockStore) LoadSuperblock 

func (bs *BlockStore) LoadSuperblock(ctx context.Context, id uuid.UUID, generation uint64) (*Superblock, bte.BTE)

func (*BlockStore) LoadSuperblockFromCache 

func (bs *BlockStore) LoadSuperblockFromCache(uu uuid.UUID) *Superblock

func (*BlockStore) NotifyWriteLockLost 

func (bs *BlockStore) NotifyWriteLockLost()

This is called if our write lock changes. Need to invalidate caches

func (*BlockStore) ObtainGeneration 

func (bs *BlockStore) ObtainGeneration(ctx context.Context, id uuid.UUID) (*Generation, bte.BTE)

* This obtains a generation, blocking if necessary

func (*BlockStore) PutSuperblockInCache 

func (bs *BlockStore) PutSuperblockInCache(s *Superblock)

func (*BlockStore) ReadDatablock 

func (bs *BlockStore) ReadDatablock(ctx context.Context, uuid uuid.UUID, addr uint64, impl_Generation uint64, impl_Pointwidth uint8, impl_StartTime int64) (Datablock, bte.BTE)

func (*BlockStore) StorageProvider 

func (bs *BlockStore) StorageProvider() bprovider.StorageProvider

func (*BlockStore) StreamExists 

func (bs *BlockStore) StreamExists(ctx context.Context, id uuid.UUID) (bool, bte.BTE)

type BlockType 

type BlockType uint64
const (
	Vector BlockType = 1
	Core   BlockType = 2
	Bad    BlockType = 255
)

func DatablockGetBufferType 

func DatablockGetBufferType(buf []byte) BlockType

type CacheItem 

type CacheItem struct {
	// contains filtered or unexported fields
}

type Coreblock 

type Coreblock struct {

	//Metadata, not copied
	Identifier uint64 "metadata,implicit"
	Generation uint64 "metadata,implicit"

	//Payload, copied
	PointWidth  uint8 "implicit"
	StartTime   int64 "implicit"
	Addr        [KFACTOR]uint64
	Count       [KFACTOR]uint64
	Min         [KFACTOR]float64
	Mean        [KFACTOR]float64
	Max         [KFACTOR]float64
	CGeneration [KFACTOR]uint64
}

func (*Coreblock) CopyInto 

func (src *Coreblock) CopyInto(dst *Coreblock)

Copy a core block, only copying the payload, not the metadata

func (*Coreblock) Deserialize 

func (c *Coreblock) Deserialize(src []byte)

func (*Coreblock) GetDatablockType 

func (*Coreblock) GetDatablockType() BlockType

func (*Coreblock) Serialize 

func (c *Coreblock) Serialize(dst []byte) []byte

type Datablock 

type Datablock interface {
	GetDatablockType() BlockType
}

type Generation 

type Generation struct {
	Cur_SB *Superblock
	New_SB *Superblock
	// contains filtered or unexported fields
}

A generation stores all the information acquired during a write pass. * A superblock contains all the information required to navigate a tree.

func (*Generation) AllocateCoreblock 

func (gen *Generation) AllocateCoreblock() *Coreblock

func (*Generation) AllocateVectorblock 

func (gen *Generation) AllocateVectorblock() *Vectorblock

func (*Generation) Commit 

func (gen *Generation) Commit() (map[uint64]uint64, bte.BTE)

The returned address map is primarily for unit testing

func (*Generation) HintEvictReplaced 

func (g *Generation) HintEvictReplaced(addr uint64)

func (*Generation) Number 

func (g *Generation) Number() uint64

func (*Generation) UpdateRootAddr 

func (g *Generation) UpdateRootAddr(addr uint64)

func (*Generation) Uuid 

func (g *Generation) Uuid() *uuid.UUID

type LASMetric 

type LASMetric struct {
	// contains filtered or unexported fields
}

type Superblock 

type Superblock struct {
	// contains filtered or unexported fields
}

func DeserializeSuperblock 

func DeserializeSuperblock(id uuid.UUID, gen uint64, arr []byte) *Superblock

func NewSuperblock 

func NewSuperblock(id uuid.UUID) *Superblock

func (*Superblock) CloneInc 

func (s *Superblock) CloneInc() *Superblock

func (*Superblock) Gen 

func (s *Superblock) Gen() uint64

func (*Superblock) Root 

func (s *Superblock) Root() uint64

func (*Superblock) Serialize 

func (s *Superblock) Serialize() []byte

func (*Superblock) Uuid 

func (s *Superblock) Uuid() uuid.UUID

type Vectorblock 

type Vectorblock struct {

	//Metadata, not copied on clone
	Identifier uint64 "metadata,implicit"
	Generation uint64 "metadata,implicit"

	//Payload, copied on clone
	Len        uint16
	PointWidth uint8 "implicit"
	StartTime  int64 "implicit"
	Time       [VSIZE]int64
	Value      [VSIZE]float64
}

The leaf datablock type. The tags allow unit tests to work out if clone / serdes are working properly metadata is not copied when a node is cloned implicit is not serialised

func (*Vectorblock) CopyInto 

func (src *Vectorblock) CopyInto(dst *Vectorblock)

func (*Vectorblock) Deserialize 

func (v *Vectorblock) Deserialize(src []byte)

func (*Vectorblock) GetDatablockType 

func (*Vectorblock) GetDatablockType() BlockType

func (*Vectorblock) Serialize 

func (v *Vectorblock) Serialize(dst []byte) []byte

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.