storage

Documentation

Index

• func Decrypt(s string, key []byte) (string, error)
• func Encrypt(s string, key []byte) (string, error)
• func GetNextReverseLexicographicPath(evictionCount int, treeHeight int) (nextPath int)
• func GetRandomPathAndStorageID(treeHeight int, storageCount int) (path int, storageID int)
• type BlockInfo
• type BlockOffsetStatus
• type IntSet
  • func (s IntSet) Add(item int)
  • func (s IntSet) Contains(item int) bool
  • func (s IntSet) Remove(item int)
• type MockStorageHandler
  • func NewMockStorageHandler(levelCount int, maxAccessCount int) *MockStorageHandler
  • func (m *MockStorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)
  • func (m *MockStorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (offsets map[int]BlockOffsetStatus, err error)
  • func (m *MockStorageHandler) BatchReadBlock(offsets map[int]int, storageID int) (values map[int]string, err error)
  • func (m *MockStorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)
  • func (m *MockStorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, ...) (writtenBlocks map[string]string, err error)
  • func (m *MockStorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)
  • func (m *MockStorageHandler) GetMaxAccessCount() int
  • func (m *MockStorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)
  • func (m *MockStorageHandler) GetRandomStorageID() int
  • func (m *MockStorageHandler) LockStorage(storageID int)
  • func (m *MockStorageHandler) UnlockStorage(storageID int)
  • func (m *MockStorageHandler) WithCusomBatchGetBlockOffsetFunc(...) *MockStorageHandler
  • func (m *MockStorageHandler) WithCustomBatchGetAccessCountFunc(f func(bucketIDs []int, storageID int) (counts map[int]int, err error)) *MockStorageHandler
  • func (m *MockStorageHandler) WithCustomBatchReadBlockFunc(f func(offsets map[int]int, storageID int) (values map[int]string, err error)) *MockStorageHandler
  • func (m *MockStorageHandler) WithCustomBatchReadBucketFunc(...) *MockStorageHandler
  • func (m *MockStorageHandler) WithCustomBatchWriteBucketFunc(...) *MockStorageHandler
• type StorageHandler
  • func NewStorageHandler(treeHeight int, Z int, S int, shift int, redisEndpoints []config.RedisEndpoint) *StorageHandler
  • func (s *StorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)
  • func (s *StorageHandler) BatchGetAllMetaData(bucketIDs []int, storageID int) (map[int]map[string]int, error)
  • func (s *StorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (blockoffsetStatuses map[int]BlockOffsetStatus, err error)
  • func (s *StorageHandler) BatchPushDataAndMetadata(bucketId int, valueData []string, valueMetadata []string, pipe redis.Pipeliner) (dataCmd *redis.BoolCmd, metadataCmd *redis.BoolCmd)
  • func (s *StorageHandler) BatchReadBlock(bucketOffsets map[int]int, storageID int) (values map[int]string, err error)
  • func (s *StorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)
  • func (s *StorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, ...) (writtenBlocks map[string]string, err error)
  • func (s *StorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)
  • func (s *StorageHandler) GetMaxAccessCount() int
  • func (s *StorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)
  • func (s *StorageHandler) GetRandomStorageID() int
  • func (s *StorageHandler) InitDatabase() error
  • func (s *StorageHandler) LockStorage(storageID int)
  • func (s *StorageHandler) UnlockStorage(storageID int)

Functions

func Decrypt

func Decrypt(s string, key []byte) (string, error)

func Encrypt

func Encrypt(s string, key []byte) (string, error)

func GetNextReverseLexicographicPath

func GetNextReverseLexicographicPath(evictionCount int, treeHeight int) (nextPath int)

evictionCount starts from zero and goes forward

func GetRandomPathAndStorageID

func GetRandomPathAndStorageID(treeHeight int, storageCount int) (path int, storageID int)

It returns valid randomly chosen path and storageID.

Types

type BlockInfo

type BlockInfo struct {
	Value string
	Path  int
}

type BlockOffsetStatus

type BlockOffsetStatus struct {
	Offset     int
	IsReal     bool
	BlockFound string
}

type IntSet

type IntSet map[int]struct{}

func (IntSet) Add

func (s IntSet) Add(item int)

func (IntSet) Contains

func (s IntSet) Contains(item int) bool

func (IntSet) Remove

func (s IntSet) Remove(item int)

type MockStorageHandler

type MockStorageHandler struct {
	// contains filtered or unexported fields
}

func NewMockStorageHandler

func NewMockStorageHandler(levelCount int, maxAccessCount int) *MockStorageHandler

func (*MockStorageHandler) BatchGetAccessCount

func (m *MockStorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)

func (*MockStorageHandler) BatchGetBlockOffset

func (m *MockStorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (offsets map[int]BlockOffsetStatus, err error)

func (*MockStorageHandler) BatchReadBlock

func (m *MockStorageHandler) BatchReadBlock(offsets map[int]int, storageID int) (values map[int]string, err error)

func (*MockStorageHandler) BatchReadBucket

func (m *MockStorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)

func (*MockStorageHandler) BatchWriteBucket

func (m *MockStorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)

func (*MockStorageHandler) GetBucketsInPaths

func (m *MockStorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)

func (*MockStorageHandler) GetMaxAccessCount

func (m *MockStorageHandler) GetMaxAccessCount() int

func (*MockStorageHandler) GetMultipleReverseLexicographicPaths

func (m *MockStorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)

func (*MockStorageHandler) GetRandomStorageID

func (m *MockStorageHandler) GetRandomStorageID() int

func (*MockStorageHandler) LockStorage

func (m *MockStorageHandler) LockStorage(storageID int)

func (*MockStorageHandler) UnlockStorage

func (m *MockStorageHandler) UnlockStorage(storageID int)

func (*MockStorageHandler) WithCusomBatchGetBlockOffsetFunc

func (m *MockStorageHandler) WithCusomBatchGetBlockOffsetFunc(f func(bucketIDs []int, storageID int, blocks []string) (offsets map[int]BlockOffsetStatus, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchGetAccessCountFunc

func (m *MockStorageHandler) WithCustomBatchGetAccessCountFunc(f func(bucketIDs []int, storageID int) (counts map[int]int, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchReadBlockFunc

func (m *MockStorageHandler) WithCustomBatchReadBlockFunc(f func(offsets map[int]int, storageID int) (values map[int]string, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchReadBucketFunc

func (m *MockStorageHandler) WithCustomBatchReadBucketFunc(f func(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)) *MockStorageHandler

func (*MockStorageHandler) WithCustomBatchWriteBucketFunc

func (m *MockStorageHandler) WithCustomBatchWriteBucketFunc(f func(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)) *MockStorageHandler

type StorageHandler

type StorageHandler struct {
	Z int // the maximum number of real blocks in each bucket
	S int // the number of dummy blocks in each bucket
	// contains filtered or unexported fields
}

StorageHandler is responsible for handling one or multiple storage shards.

func NewStorageHandler

func NewStorageHandler(treeHeight int, Z int, S int, shift int, redisEndpoints []config.RedisEndpoint) *StorageHandler

func (*StorageHandler) BatchGetAccessCount

func (s *StorageHandler) BatchGetAccessCount(bucketIDs []int, storageID int) (counts map[int]int, err error)

It returns the number of times a bucket was accessed for multiple buckets. This is helpful to know when to do an early reshuffle.

func (*StorageHandler) BatchGetAllMetaData

func (s *StorageHandler) BatchGetAllMetaData(bucketIDs []int, storageID int) (map[int]map[string]int, error)

Returns a map of bucketID to a map of block to position. It returns all the valid real and dummy blocks in the bucket. The invalidated blocks are not returned.

func (*StorageHandler) BatchGetBlockOffset

func (s *StorageHandler) BatchGetBlockOffset(bucketIDs []int, storageID int, blocks []string) (blockoffsetStatuses map[int]BlockOffsetStatus, err error)

func (*StorageHandler) BatchPushDataAndMetadata

func (s *StorageHandler) BatchPushDataAndMetadata(bucketId int, valueData []string, valueMetadata []string, pipe redis.Pipeliner) (dataCmd *redis.BoolCmd, metadataCmd *redis.BoolCmd)

func (*StorageHandler) BatchReadBlock

func (s *StorageHandler) BatchReadBlock(bucketOffsets map[int]int, storageID int) (values map[int]string, err error)

It reads multiple blocks from multiple buckets and returns the values.

func (*StorageHandler) BatchReadBucket

func (s *StorageHandler) BatchReadBucket(bucketIDs []int, storageID int) (blocks map[int]map[string]string, err error)

It reads multiple buckets from a single storage shard.

func (*StorageHandler) BatchWriteBucket

func (s *StorageHandler) BatchWriteBucket(storageID int, readBucketBlocksList map[int]map[string]string, shardNodeBlocks map[string]BlockInfo) (writtenBlocks map[string]string, err error)

It writes blocks to multiple buckets in a single storage shard.

func (*StorageHandler) GetBucketsInPaths

func (s *StorageHandler) GetBucketsInPaths(paths []int) (bucketIDs []int, err error)

GetBucketsInPaths return all the bucket ids for the passed paths.

func (*StorageHandler) GetMaxAccessCount

func (s *StorageHandler) GetMaxAccessCount() int

func (*StorageHandler) GetMultipleReverseLexicographicPaths

func (s *StorageHandler) GetMultipleReverseLexicographicPaths(evictionCount int, count int) (paths []int)

func (*StorageHandler) GetRandomStorageID

func (s *StorageHandler) GetRandomStorageID() int

func (*StorageHandler) InitDatabase

func (s *StorageHandler) InitDatabase() error

func (*StorageHandler) LockStorage

func (s *StorageHandler) LockStorage(storageID int)

func (*StorageHandler) UnlockStorage

func (s *StorageHandler) UnlockStorage(storageID int)