metastore

package
v1.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 27, 2023 License: MIT Imports: 14 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	// MetaFlagValid flags that the meta is valid.
	// If the meta is invalid because of a former creation, some chunks may be created.
	// A new PUT should skip current version and advance to the next version.
	MetaFlagValid = int32(0x01)
	// MetaFlagCreated flags that the object is successfully created
	MetaFlagCreated = int32(0x02)
	// MetaFlagDeleted flags that the object is deleted
	MetaFlagDeleted = int32(0x04)
)
View Source 
const (

	// Report there's not enough space in storage
	// Parameter: *lambdastore.Instance
	EventInsufficientStorage = 300
)
View Source 
const (
	INIT_LRU_CAPACITY = 10000
)
View Source 
const (
	PlacerEventBeforePlacing = "before_placing"
)

Variables

View Source 
var (
	ErrConcurrentCreation = errors.New("concurrent creation")
)
View Source 
var (
	ErrPlacementConflict = errors.New("conflict on placing")
)
View Source 
var (
	InvalidPlacement = ^uint64(0)
)
View Source 
var (
	RegDelimiter = regexp.MustCompile(`@`)
)

Functions

func IsPlacementEmpty 

func IsPlacementEmpty(p Placement) bool

func MetaKeyByVersion 

func MetaKeyByVersion(key string, ver int) string

MetaKeyByVersion is the static version of Meta.KeyByVersion. Key will be santicized first.

Types

type ClusterManager 

type ClusterManager interface {
	// GetActiveInstances Request available instances with minimum number required.
	GetActiveInstances(int) lambdastore.InstanceEnumerator

	// GetSlice Get slice implementation if support
	GetSlice(int) Slice

	// Trigger Trigger event with code and parameter
	Trigger(int, ...interface{})
}

type DefaultPlacer 

type DefaultPlacer struct {
	*PlacerEvents
	// contains filtered or unexported fields
}

func NewDefaultPlacer 

func NewDefaultPlacer(store *MetaStore, cluster InstanceManager) *DefaultPlacer

func (*DefaultPlacer) Dispatch 

func (l *DefaultPlacer) Dispatch(ins *lambdastore.Instance, cmd types.Command) (err error)

func (*DefaultPlacer) Get 

func (l *DefaultPlacer) Get(key string, chunk int) (*Meta, bool)

func (*DefaultPlacer) GetByVersion 

func (l *DefaultPlacer) GetByVersion(key string, ver int, chunk int) (*Meta, bool)

func (*DefaultPlacer) InsertAndPlace 

func (l *DefaultPlacer) InsertAndPlace(key string, newMeta *Meta, cmd types.Command) (*Meta, MetaPostProcess, error)

func (*DefaultPlacer) MetaStats 

func (l *DefaultPlacer) MetaStats() types.MetaStoreStats

func (*DefaultPlacer) NewMeta 

func (l *DefaultPlacer) NewMeta(reqId string, key string, size int64, dChunks, pChunks, chunkId int, chunkSize int64, lambdaId uint64, sliceSize int) *Meta

func (*DefaultPlacer) Place 

func (l *DefaultPlacer) Place(meta *Meta, chunkId int, cmd types.Command) (*lambdastore.Instance, MetaPostProcess, error)

type InstanceManager 

type InstanceManager interface {
	lambdastore.InstanceManager
	ClusterManager
}

type LRUPlacer 

type LRUPlacer struct {
	*PlacerEvents
	// contains filtered or unexported fields
}

Placer implements a Clock LRU for object eviction. Because objects (or metas) are constantly added to the system in partial (chunks), a object (newer) can be appended to the list normally, while a later chunk of the object requires an eviction of another object (older). In this case, we evict the older in whole based on Clock LRU algorithm, and set the original position of the newer to nil, which a compact operation is needed later. We use a secondary array for online compact.

func NewLRUPlacer 

func NewLRUPlacer(store *MetaStore, cluster InstanceManager) *LRUPlacer

func (*LRUPlacer) AddObject 

func (p *LRUPlacer) AddObject(meta *Meta)

Object management implementation: Clock LRU

func (*LRUPlacer) Dispatch 

func (p *LRUPlacer) Dispatch(ins *lambdastore.Instance, cmd types.Command) error

func (*LRUPlacer) FindPlacement 

func (p *LRUPlacer) FindPlacement(meta *Meta, chunkId int) (*lambdastore.Instance, MetaPostProcess, error)

NewMeta will remap idx according to following logic:

  1. If an LRU relocation is present, remap according to "chunk" in relocation array.
  2. Base on the size of slice, remap to a instance in the group.
  3. If target instance is full, request an LRU relocation and restart from 0.
  4. If no Balancer relocation is available, request one.
  5. Remap to smaller "Size" of instance between target instance and remapped instance according to "chunk" in relocation array.

func (*LRUPlacer) Get 

func (p *LRUPlacer) Get(key string, chunk int) (*Meta, bool)

func (*LRUPlacer) GetByVersion 

func (p *LRUPlacer) GetByVersion(key string, ver int, chunk int) (*Meta, bool)

func (*LRUPlacer) Insert 

func (p *LRUPlacer) Insert(key string, newMeta *Meta) (*Meta, MetaPostProcess, error)

For test and simulation

func (*LRUPlacer) InsertAndPlace 

func (p *LRUPlacer) InsertAndPlace(key string, newMeta *Meta, cmd types.Command) (*Meta, MetaPostProcess, error)

func (*LRUPlacer) MetaStats 

func (p *LRUPlacer) MetaStats() types.MetaStoreStats

func (*LRUPlacer) NewMeta 

func (p *LRUPlacer) NewMeta(reqId string, key string, size int64, dChunks, pChunks, chunk int, chunkSize int64, lambdaId uint64, sliceSize int) *Meta

func (*LRUPlacer) NextAvailableObject 

func (p *LRUPlacer) NextAvailableObject(meta *Meta, candidate *Meta) (*Meta, bool)

func (*LRUPlacer) Place 

func (p *LRUPlacer) Place(meta *Meta, chunkId int, cmd types.Command) (*lambdastore.Instance, MetaPostProcess, error)

func (*LRUPlacer) TouchObject 

func (p *LRUPlacer) TouchObject(meta *Meta)

type LRUPlacerMeta 

type LRUPlacerMeta struct {
	// contains filtered or unexported fields
}

type Meta 

type Meta struct {
	Size    int64
	DChunks int
	PChunks int
	Placement
	ChunkSize int64
	// contains filtered or unexported fields
}

func NewMeta 

func NewMeta(reqId string, key string, size int64, d, p int, chunkSize int64) *Meta

func (*Meta) ChunkKey 

func (m *Meta) ChunkKey(chunkId int) string

func (*Meta) ConfirmCreated 

func (m *Meta) ConfirmCreated()

func (*Meta) Delete 

func (m *Meta) Delete()

func (*Meta) GetPlace 

func (m *Meta) GetPlace(chunkId int) uint64

func (*Meta) HasHistory 

func (m *Meta) HasHistory() bool

func (*Meta) Invalidate 

func (m *Meta) Invalidate()

func (*Meta) IsCreated 

func (m *Meta) IsCreated() bool

func (*Meta) IsDeleted 

func (m *Meta) IsDeleted() bool

func (*Meta) IsValid 

func (m *Meta) IsValid() bool

func (*Meta) Key 

func (m *Meta) Key() string

func (*Meta) KeyByVersion 

func (m *Meta) KeyByVersion(ver int) string

func (*Meta) NumChunks 

func (m *Meta) NumChunks() int

func (*Meta) PreviousVersion 

func (m *Meta) PreviousVersion() int

func (*Meta) ReviseBy 

func (m *Meta) ReviseBy(newMeta *Meta) *Meta

Revise prepares the new meta created from calling NewMeta with new version infomation

func (*Meta) ReviseFrom 

func (m *Meta) ReviseFrom(lastVer *Meta) *Meta

ReviseFrom updates the meta with new version information according to the specified meta of latest version.

func (*Meta) SetTimout 

func (m *Meta) SetTimout(timeout time.Duration)

func (*Meta) Validate 

func (m *Meta) Validate() bool

func (*Meta) Version 

func (m *Meta) Version() int

func (*Meta) VersioningKey 

func (m *Meta) VersioningKey() string

func (*Meta) Wait 

func (m *Meta) Wait()

type MetaDoPostProcess 

type MetaDoPostProcess func(*Meta)

type MetaDoPostProcess func(*Meta)

type MetaPostProcess 

type MetaPostProcess func(MetaDoPostProcess)

type MetaStore 

type MetaStore struct {
	// contains filtered or unexported fields
}

func New 

func New() *MetaStore

func NewWithCapacity 

func NewWithCapacity(size int) *MetaStore

func (*MetaStore) Get 

func (ms *MetaStore) Get(key string) (*Meta, bool)

func (*MetaStore) GetByVersion 

func (ms *MetaStore) GetByVersion(key string, ver int) (*Meta, bool)

func (*MetaStore) GetOrInsert 

func (ms *MetaStore) GetOrInsert(key string, insert *Meta) (*Meta, bool, error)

GetOrInsert get the meta if exists, otherwise insert a new one. The second return value is true if a existed meta is loaded.

func (*MetaStore) Len 

func (ms *MetaStore) Len() int

type Placement 

type Placement []uint64

type Placer 

type Placer interface {
	// Parameters: key, size, dChunks, pChunks, chunkId, chunkSize, lambdaId, sliceSize
	NewMeta(string, string, int64, int, int, int, int64, uint64, int) *Meta
	InsertAndPlace(key string, prepared *Meta, req types.Command) (meta *Meta, postProcess MetaPostProcess, err error)
	Place(meta *Meta, chunkId int, req types.Command) (ins *lambdastore.Instance, postProcess MetaPostProcess, err error)
	Get(string, int) (*Meta, bool)
	GetByVersion(string, int, int) (*Meta, bool)
	Dispatch(*lambdastore.Instance, types.Command) error
	MetaStats() types.MetaStoreStats
	RegisterHandler(event PlacerEvent, handler PlacerHandler)
}

type PlacerEvent 

type PlacerEvent string

type PlacerEvents 

type PlacerEvents struct {
	// contains filtered or unexported fields
}

func (*PlacerEvents) RegisterHandler 

func (e *PlacerEvents) RegisterHandler(event PlacerEvent, handler PlacerHandler)

type PlacerHandler 

type PlacerHandler func(meta *Meta, chunkId int, req types.Command)

type Slice 

type Slice interface {
	Size() int
	Reset(int)
	GetIndex(uint64) uint64
}

type SliceInitializer 

type SliceInitializer func(int) (int, int)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.