erasure_coding

package
v0.0.0-...-0b345dd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 6, 2022 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DataShardsCount             = 10
	ParityShardsCount           = 4
	TotalShardsCount            = DataShardsCount + ParityShardsCount
	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
)

Variables

View Source 
var (
	MarkNeedleDeleted = func(file *os.File, offset int64) error {
		b := make([]byte, types.SizeSize)
		types.SizeToBytes(b, types.TombstoneFileSize)
		n, err := file.WriteAt(b, offset+types.NeedleIdSize+types.OffsetSize)
		if err != nil {
			return fmt.Errorf("sorted needle write error: %v", err)
		}
		if n != types.SizeSize {
			return fmt.Errorf("sorted needle written %d bytes, expecting %d", n, types.SizeSize)
		}
		return nil
	}
)
View Source 
var (
	NotFoundError = errors.New("needle not found")
)

Functions

func EcShardBaseFileName 

func EcShardBaseFileName(collection string, id int) (baseFileName string)

func EcShardFileName 

func EcShardFileName(collection string, dir string, id int) (fileName string)

func FindDatFileSize 

func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64, err error)

FindDatFileSize calculate .dat file size from max offset entry there may be extra deletions after that entry but they are deletions anyway

func RebuildEcFiles 

func RebuildEcFiles(baseFileName string) ([]uint32, error)

func RebuildEcxFile 

func RebuildEcxFile(baseFileName string) error

func SearchNeedleFromSortedIndex 

func SearchNeedleFromSortedIndex(ecxFile *os.File, ecxFileSize int64, needleId types.NeedleId, processNeedleFn func(file *os.File, offset int64) error) (offset types.Offset, size types.Size, err error)

func ToExt 

func ToExt(ecIndex int) string

func WriteDatFile 

func WriteDatFile(baseFileName string, datFileSize int64) error

WriteDatFile generates .dat from from .ec00 ~ .ec09 files

func WriteEcFiles 

func WriteEcFiles(baseFileName string) error

WriteEcFiles generates .ec00 ~ .ec13 files

func WriteIdxFileFromEcIndex 

func WriteIdxFileFromEcIndex(baseFileName string) (err error)

write .idx file from .ecx and .ecj files

func WriteSortedFileFromIdx 

func WriteSortedFileFromIdx(baseFileName string, ext string) (e error)

WriteSortedFileFromIdx generates .ecx file from existing .idx file all keys are sorted in ascending order

Types

type EcVolume 

type EcVolume struct {
	VolumeId   needle.VolumeId
	Collection string

	Shards                    []*EcVolumeShard
	ShardLocations            map[ShardId][]pb.ServerAddress
	ShardLocationsRefreshTime time.Time
	ShardLocationsLock        sync.RWMutex
	Version                   needle.Version
	// contains filtered or unexported fields
}

func NewEcVolume 

func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error)

func (*EcVolume) AddEcVolumeShard 

func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool

func (*EcVolume) Close 

func (ev *EcVolume) Close()

func (*EcVolume) CreatedAt 

func (ev *EcVolume) CreatedAt() time.Time

func (*EcVolume) DataBaseFileName 

func (ev *EcVolume) DataBaseFileName() string

func (*EcVolume) DeleteEcVolumeShard 

func (ev *EcVolume) DeleteEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, deleted bool)

func (*EcVolume) DeleteNeedleFromEcx 

func (ev *EcVolume) DeleteNeedleFromEcx(needleId types.NeedleId) (err error)

func (*EcVolume) Destroy 

func (ev *EcVolume) Destroy()

func (*EcVolume) FileName 

func (ev *EcVolume) FileName(ext string) string

func (*EcVolume) FindEcVolumeShard 

func (ev *EcVolume) FindEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, found bool)

func (*EcVolume) FindNeedleFromEcx 

func (ev *EcVolume) FindNeedleFromEcx(needleId types.NeedleId) (offset types.Offset, size types.Size, err error)

func (*EcVolume) IndexBaseFileName 

func (ev *EcVolume) IndexBaseFileName() string

func (*EcVolume) LocateEcShardNeedle 

func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.Version) (offset types.Offset, size types.Size, intervals []Interval, err error)

func (*EcVolume) ShardIdList 

func (ev *EcVolume) ShardIdList() (shardIds []ShardId)

func (*EcVolume) ShardSize 

func (ev *EcVolume) ShardSize() uint64

func (*EcVolume) Size 

func (ev *EcVolume) Size() (size int64)

func (*EcVolume) ToVolumeEcShardInformationMessage 

func (ev *EcVolume) ToVolumeEcShardInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage)

type EcVolumeInfo 

type EcVolumeInfo struct {
	VolumeId   needle.VolumeId
	Collection string
	ShardBits  ShardBits
	DiskType   string
}

data structure used in master

func NewEcVolumeInfo 

func NewEcVolumeInfo(diskType string, collection string, vid needle.VolumeId, shardBits ShardBits) *EcVolumeInfo

func (*EcVolumeInfo) AddShardId 

func (ecInfo *EcVolumeInfo) AddShardId(id ShardId)

func (*EcVolumeInfo) HasShardId 

func (ecInfo *EcVolumeInfo) HasShardId(id ShardId) bool

func (*EcVolumeInfo) Minus 

func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo

func (*EcVolumeInfo) RemoveShardId 

func (ecInfo *EcVolumeInfo) RemoveShardId(id ShardId)

func (*EcVolumeInfo) ShardIdCount 

func (ecInfo *EcVolumeInfo) ShardIdCount() (count int)

func (*EcVolumeInfo) ShardIds 

func (ecInfo *EcVolumeInfo) ShardIds() (ret []ShardId)

func (*EcVolumeInfo) ToVolumeEcShardInformationMessage 

func (ecInfo *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage)

type EcVolumeShard 

type EcVolumeShard struct {
	VolumeId   needle.VolumeId
	ShardId    ShardId
	Collection string

	DiskType types.DiskType
	// contains filtered or unexported fields
}

func NewEcVolumeShard 

func NewEcVolumeShard(diskType types.DiskType, dirname string, collection string, id needle.VolumeId, shardId ShardId) (v *EcVolumeShard, e error)

func (*EcVolumeShard) Close 

func (shard *EcVolumeShard) Close()

func (*EcVolumeShard) Destroy 

func (shard *EcVolumeShard) Destroy()

func (*EcVolumeShard) FileName 

func (shard *EcVolumeShard) FileName() (fileName string)

func (*EcVolumeShard) ReadAt 

func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error)

func (*EcVolumeShard) Size 

func (shard *EcVolumeShard) Size() int64

func (*EcVolumeShard) String 

func (shard *EcVolumeShard) String() string

type Interval 

type Interval struct {
	BlockIndex          int
	InnerBlockOffset    int64
	Size                types.Size
	IsLargeBlock        bool
	LargeBlockRowsCount int
}

func LocateData 

func LocateData(largeBlockLength, smallBlockLength int64, datSize int64, offset int64, size types.Size) (intervals []Interval)

func (Interval) ToShardIdAndOffset 

func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64)

type ShardBits 

type ShardBits uint32 // use bits to indicate the shard id, use 32 bits just for possible future extension

func (ShardBits) AddShardId 

func (b ShardBits) AddShardId(id ShardId) ShardBits

func (ShardBits) HasShardId 

func (b ShardBits) HasShardId(id ShardId) bool

func (ShardBits) Minus 

func (b ShardBits) Minus(other ShardBits) ShardBits

func (ShardBits) MinusParityShards 

func (b ShardBits) MinusParityShards() ShardBits

func (ShardBits) Plus 

func (b ShardBits) Plus(other ShardBits) ShardBits

func (ShardBits) RemoveShardId 

func (b ShardBits) RemoveShardId(id ShardId) ShardBits

func (ShardBits) ShardIdCount 

func (b ShardBits) ShardIdCount() (count int)

func (ShardBits) ShardIds 

func (b ShardBits) ShardIds() (ret []ShardId)

func (ShardBits) ToUint32Slice 

func (b ShardBits) ToUint32Slice() (ret []uint32)

type ShardId 

type ShardId uint8

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.