Documentation ¶
Index ¶
- Constants
- Variables
- func PutExtentSlice(eks []BtreeItem)
- type APIResponse
- type BTree
- func (b *BTree) Ascend(fn func(i BtreeItem) bool)
- func (b *BTree) AscendGreaterOrEqual(pivot BtreeItem, iterator func(i BtreeItem) bool)
- func (b *BTree) AscendRange(greaterOrEqual, lessThan BtreeItem, iterator func(i BtreeItem) bool)
- func (b *BTree) CopyFind(key BtreeItem, fn func(i BtreeItem))
- func (b *BTree) CopyGet(key BtreeItem) (item BtreeItem)
- func (b *BTree) Delete(key BtreeItem) (item BtreeItem)
- func (b *BTree) Find(key BtreeItem, fn func(i BtreeItem))
- func (b *BTree) Get(key BtreeItem) (item BtreeItem)
- func (b *BTree) GetTree() *BTree
- func (b *BTree) Has(key BtreeItem) (ok bool)
- func (b *BTree) Len() (size int)
- func (b *BTree) MaxItem() BtreeItem
- func (b *BTree) ReplaceOrInsert(key BtreeItem, replace bool) (item BtreeItem, ok bool)
- func (b *BTree) Reset()
- type BtreeItem
- type CreateDentryReq
- type CreateInoReq
- type CreateInoResp
- type CreateMetaRangeReq
- type CreateMetaRangeResp
- type DataPartition
- type DataPartitionsView
- type DeleteDentryReq
- type DeleteDentryResp
- type Dentry
- func (d *Dentry) Copy() BtreeItem
- func (d *Dentry) Less(than BtreeItem) (less bool)
- func (d *Dentry) Marshal() (result []byte, err error)
- func (d *Dentry) MarshalKey() (k []byte)
- func (d *Dentry) MarshalValue() (k []byte)
- func (d *Dentry) Unmarshal(raw []byte) (err error)
- func (d *Dentry) UnmarshalKey(k []byte) (err error)
- func (d *Dentry) UnmarshalValue(val []byte) (err error)
- type DentryResponse
- type EvictInodeReq
- type ExtentsTree
- func (e *ExtentsTree) Append(key BtreeItem) (items []BtreeItem)
- func (e *ExtentsTree) Clone() *ExtentsTree
- func (e *ExtentsTree) Marshal() ([]byte, error)
- func (e *ExtentsTree) MarshalBinary() (data []byte, err error)
- func (e *ExtentsTree) MarshalJSON() ([]byte, error)
- func (e *ExtentsTree) Range(f func(item BtreeItem) bool)
- func (e *ExtentsTree) Size() (size uint64)
- func (e *ExtentsTree) String() string
- func (e *ExtentsTree) UnmarshalBinary(data []byte) (err error)
- type ExtentsTruncateReq
- type Inode
- func (i *Inode) AppendExtents(exts []BtreeItem, ct int64) (items []BtreeItem)
- func (i *Inode) Copy() BtreeItem
- func (i *Inode) DecNLink()
- func (i *Inode) DoReadFunc(fn func())
- func (i *Inode) DoWriteFunc(fn func())
- func (i *Inode) ExtentsTruncate(exts []BtreeItem, length uint64, ct int64)
- func (i *Inode) GetNLink() uint32
- func (i *Inode) IncNLink()
- func (i *Inode) IsEmptyDir() bool
- func (i *Inode) IsTempFile() bool
- func (i *Inode) Less(than BtreeItem) bool
- func (i *Inode) Marshal() (result []byte, err error)
- func (i *Inode) MarshalKey() (k []byte)
- func (i *Inode) MarshalToJSON() ([]byte, error)
- func (i *Inode) MarshalValue() (val []byte)
- func (i *Inode) SetAttr(valid, mode, uid, gid uint32)
- func (i *Inode) SetDeleteMark()
- func (i *Inode) ShouldDelete() bool
- func (i *Inode) String() string
- func (i *Inode) Unmarshal(raw []byte) (err error)
- func (i *Inode) UnmarshalKey(k []byte) (err error)
- func (i *Inode) UnmarshalValue(val []byte) (err error)
- type InodeGetReq
- type InodeGetReqBatch
- type InodeResponse
- type LinkInodeReq
- type LinkInodeResp
- type LookupReq
- type LookupResp
- type MetaItem
- type MetaItemIterator
- type MetaNode
- type MetaNodeInfo
- type MetaPartition
- type MetaPartitionConfig
- type MetadataManager
- type MetadataManagerConfig
- type NowTime
- type OpDentry
- type OpExtent
- type OpInode
- type OpMeta
- type OpPartition
- type Packet
- type ReadDirReq
- type ReadDirResp
- type SetattrRequest
- type UnlinkInoReq
- type UnlinkInoResp
- type UpdateDentryReq
- type UpdateDentryResp
- type UpdatePartitionReq
- type UpdatePartitionResp
- type Vol
Constants ¶
const ( StateStandby uint32 = iota StateStart StateRunning StateShutdown StateStopped )
const ( KB = 1 << (10 * iota) MB GB )
const ( ForceClosedConnect = true NoClosedConnect = false )
const ( AsyncDeleteInterval = 10 * time.Second UpdateVolTicket = 5 * time.Minute BatchCounts = 100 OpenRWAppendOpt = os.O_CREATE | os.O_RDWR | os.O_APPEND TempFileValidTime = 86400 //units: sec DeleteInodeFileExtension = "INODE_DEL" )
const (
DeleteMarkFlag = 1 << 0
)
const (
MaxUsedMemFactor = 1.1
)
const (
SnapshotSign = ".sign"
)
Variables ¶
var ( ErrNoLeader = errors.New("no leader") ErrNotALeader = errors.New("not a leader") )
var ( ErrIllegalHeartbeatAddress = errors.New("illegal heartbeat address") ErrIllegalReplicateAddress = errors.New("illegal replicate address") )
var (
ErrInodeIDOutOfRange = errors.New("inode ID out of range")
)
Errors
var Now = NewNowTime()
Now
Functions ¶
func PutExtentSlice ¶
func PutExtentSlice(eks []BtreeItem)
PutExtentSlice put an extent slice into the extent pool.
Types ¶
type APIResponse ¶
type APIResponse struct { Code int `json:"code"` Msg string `json:"msg"` Data interface{} `json:"data, omitempty"` }
APIResponse defines the structure of the response to an HTTP request
func NewAPIResponse ¶
func NewAPIResponse(code int, msg string) *APIResponse
NewAPIResponse returns a new API response.
func (*APIResponse) Marshal ¶
func (api *APIResponse) Marshal() ([]byte, error)
Marshal is a wrapper function of json.Marshal
type BTree ¶
BTree is the wrapper of Google's btree.
func (*BTree) Ascend ¶
Ascend is the wrapper of the google's btree Ascend. This function scans the entire btree. When the data is huge, it is not recommended to use this function online. Instead, it is recommended to call GetTree to obtain the snapshot of the current btree, and then do the scan on the snapshot.
func (*BTree) AscendGreaterOrEqual ¶
AscendGreaterOrEqual is the wrapper of the google's btree AscendGreaterOrEqual
func (*BTree) AscendRange ¶
AscendRange is the wrapper of the google's btree AscendRange.
func (*BTree) ReplaceOrInsert ¶
ReplaceOrInsert is the wrapper of google's btree ReplaceOrInsert.
type BtreeItem ¶
BtreeItem type alias google btree Item
func GetExtentSlice ¶
func GetExtentSlice() []BtreeItem
GetExtentSlice returns an extent slice in the extent pool.
type CreateDentryReq ¶
type CreateDentryReq = proto.CreateDentryRequest
Client -> MetaNode create Dentry request struct
type CreateInoReq ¶
type CreateInoReq = proto.CreateInodeRequest
Client -> MetaNode create Inode request
type CreateInoResp ¶
type CreateInoResp = proto.CreateInodeResponse
MetaNode -> Client create Inode response
type CreateMetaRangeReq ¶
type CreateMetaRangeReq = proto.CreateMetaPartitionRequest
Master -> MetaNode create metaPartition request
type CreateMetaRangeResp ¶
type CreateMetaRangeResp = proto.CreateMetaPartitionResponse
MetaNode -> Master create metaPartition response
type DataPartition ¶
type DataPartition struct { PartitionID uint64 Status int8 ReplicaNum uint8 PartitionType string Hosts []string }
DataPartition defines the struct of data partition that will be used on the meta node.
func (*DataPartition) GetAllAddrs ¶
func (dp *DataPartition) GetAllAddrs() (m string)
GetAllAddrs returns all addresses of the data partition.
type DataPartitionsView ¶
type DataPartitionsView struct {
DataPartitions []*DataPartition
}
DataPartitionsView defines the view of the data node.
func NewDataPartitionsView ¶
func NewDataPartitionsView() *DataPartitionsView
type DeleteDentryReq ¶
type DeleteDentryReq = proto.DeleteDentryRequest
Client -> MetaNode delete Dentry request
type DeleteDentryResp ¶
type DeleteDentryResp = proto.DeleteDentryResponse
MetaNode -> Client delete Dentry response
type Dentry ¶
type Dentry struct { ParentId uint64 // FileID value of the parent inode. Name string // Name of the current dentry. Inode uint64 // FileID value of the current inode. Type uint32 }
Dentry wraps necessary properties of the `dentry` information in file system. Marshal exporterKey:
+-------+----------+------+ | item | ParentId | Name | +-------+----------+------+ | bytes | 8 | rest | +-------+----------+------+
Marshal value:
+-------+-------+------+ | item | Inode | Type | +-------+-------+------+ | bytes | 8 | 4 | +-------+-------+------+
Marshal entity:
+-------+-----------+--------------+-----------+--------------+ | item | KeyLength | MarshaledKey | ValLength | MarshaledVal | +-------+-----------+--------------+-----------+--------------+ | bytes | 4 | KeyLength | 4 | ValLength | +-------+-----------+--------------+-----------+--------------+
func (*Dentry) Less ¶
Less tests whether the current dentry is less than the given one. This method is necessary fot B-Tree item implementation.
func (*Dentry) MarshalKey ¶
MarshalKey is the bytes version of the MarshalKey method which returns the byte slice result.
func (*Dentry) MarshalValue ¶
MarshalValue marshals the exporterKey to bytes.
func (*Dentry) UnmarshalKey ¶
UnmarshalKey unmarshals the exporterKey from bytes.
func (*Dentry) UnmarshalValue ¶
UnmarshalValue unmarshals the value from bytes.
type DentryResponse ¶
func NewDentryResponse ¶
func NewDentryResponse() *DentryResponse
type ExtentsTree ¶
type ExtentsTree struct {
*BTree
}
ExtentsTree defines the struct that includes a btree for the extents.
func (*ExtentsTree) Append ¶
func (e *ExtentsTree) Append(key BtreeItem) (items []BtreeItem)
Append appends a btree item to the extent tree.
func (*ExtentsTree) Clone ¶
func (e *ExtentsTree) Clone() *ExtentsTree
Clone returns a copy of the extent tree.
func (*ExtentsTree) Marshal ¶
func (e *ExtentsTree) Marshal() ([]byte, error)
Marshal marshals the ExtentsTree into a byte array.
func (*ExtentsTree) MarshalBinary ¶
func (e *ExtentsTree) MarshalBinary() (data []byte, err error)
MarshalBinary marshals the data in binary.
func (*ExtentsTree) MarshalJSON ¶
func (e *ExtentsTree) MarshalJSON() ([]byte, error)
MarshalJSON is the wrapper of e.Marshal().
func (*ExtentsTree) Range ¶
func (e *ExtentsTree) Range(f func(item BtreeItem) bool)
Range calls f sequentially for each exporterKey and value presented in the extent exporterKey collection. If f returns false, range stops the iteration.
func (*ExtentsTree) Size ¶
func (e *ExtentsTree) Size() (size uint64)
Size returns the size of the btree.
func (*ExtentsTree) String ¶
func (e *ExtentsTree) String() string
String returns the string format of the extent tree.
func (*ExtentsTree) UnmarshalBinary ¶
func (e *ExtentsTree) UnmarshalBinary(data []byte) (err error)
UnmarshalBinary unmarshals the data in binary.
type Inode ¶
type Inode struct { sync.RWMutex Inode uint64 // Inode ID Type uint32 Uid uint32 Gid uint32 Size uint64 Generation uint64 CreateTime int64 AccessTime int64 ModifyTime int64 LinkTarget []byte // SymLink target name NLink uint32 // NodeLink counts Flag int32 Reserved uint64 // reserved space Extents *ExtentsTree }
Inode wraps necessary properties of `Inode` information in the file system. Marshal exporterKey:
+-------+-------+ | item | Inode | +-------+-------+ | bytes | 8 | +-------+-------+
Marshal value:
+-------+------+------+-----+----+----+----+--------+------------------+ | item | Type | Size | Gen | CT | AT | MT | ExtLen | MarshaledExtents | +-------+------+------+-----+----+----+----+--------+------------------+ | bytes | 4 | 8 | 8 | 8 | 8 | 8 | 4 | ExtLen | +-------+------+------+-----+----+----+----+--------+------------------+
Marshal entity:
+-------+-----------+--------------+-----------+--------------+ | item | KeyLength | MarshaledKey | ValLength | MarshaledVal | +-------+-----------+--------------+-----------+--------------+ | bytes | 4 | KeyLength | 4 | ValLength | +-------+-----------+--------------+-----------+--------------+
func NewInode ¶
NewInode returns a new Inode instance with specified Inode ID, name and type. The AccessTime and ModifyTime will be set to the current time.
func (*Inode) AppendExtents ¶
AppendExtents append the extent to the btree.
func (*Inode) DoReadFunc ¶
func (i *Inode) DoReadFunc(fn func())
DoFunc executes the given function.
func (*Inode) DoWriteFunc ¶
func (i *Inode) DoWriteFunc(fn func())
func (*Inode) ExtentsTruncate ¶
ExtentsTruncate truncates the extents.
func (*Inode) IsEmptyDir ¶
func (*Inode) IsTempFile ¶
func (*Inode) Less ¶
Less tests whether the current Inode item is less than the given one. This method is necessary fot B-Tree item implementation.
func (*Inode) MarshalKey ¶
MarshalKey marshals the exporterKey to bytes.
func (*Inode) MarshalToJSON ¶
MarshalToJSON is the wrapper of json.Marshal.
func (*Inode) MarshalValue ¶
MarshalValue marshals the value to bytes.
func (*Inode) SetDeleteMark ¶
func (i *Inode) SetDeleteMark()
SetDeleteMark set the deleteMark flag. TODO markDelete or deleteMark? markDelete has been used in datanode.
func (*Inode) ShouldDelete ¶
ShouldDelete returns if the inode has been marked as deleted.
func (*Inode) UnmarshalKey ¶
UnmarshalKey unmarshals the exporterKey from bytes.
func (*Inode) UnmarshalValue ¶
UnmarshalValue unmarshals the value from bytes.
type InodeResponse ¶
func NewInodeResponse ¶
func NewInodeResponse() *InodeResponse
type LinkInodeReq ¶
type LinkInodeReq = proto.LinkInodeRequest
Client -> MetaNode create Link Request
type LinkInodeResp ¶
type LinkInodeResp = proto.LinkInodeResponse
MetaNode -> Client create Link Response
type MetaItem ¶
MetaItem defines the structure of the metadata operations.
func NewMetaItem ¶
NewMetaItem returns a new MetaItem.
func (*MetaItem) MarshalBinary ¶
MarshalBinary marshals MetaItem to binary data. Binary frame structure:
+------+----+------+------+------+------+ | Item | Op | LenK | K | LenV | V | +------+----+------+------+------+------+ | byte | 4 | 4 | LenK | 4 | LenV | +------+----+------+------+------+------+
func (*MetaItem) UnmarshalBinary ¶
MarshalBinary unmarshal this MetaItem entity from binary data. Binary frame structure:
+------+----+------+------+------+------+ | Item | Op | LenK | K | LenV | V | +------+----+------+------+------+------+ | byte | 4 | 4 | LenK | 4 | LenV | +------+----+------+------+------+------+
func (*MetaItem) UnmarshalJson ¶
UnmarshalJson unmarshals binary data to MetaItem.
type MetaItemIterator ¶
type MetaItemIterator struct {
// contains filtered or unexported fields
}
MetaItemIterator defines the iterator of the MetaItem.
func NewMetaItemIterator ¶
func NewMetaItemIterator(applyID uint64, ino, den *BTree, rootDir string, filelist []string) *MetaItemIterator
NewMetaItemIterator returns a new MetaItemIterator.
func (*MetaItemIterator) ApplyIndex ¶
func (si *MetaItemIterator) ApplyIndex() uint64
ApplyIndex returns the applyID of the iterator.
func (*MetaItemIterator) Next ¶
func (si *MetaItemIterator) Next() (data []byte, err error)
Next returns the next item.
type MetaNode ¶
type MetaNode struct {
// contains filtered or unexported fields
}
The MetaNode manages the dentry and inode information of the meta partitions on a meta node. The data consistency is ensured by Raft.
type MetaNodeInfo ¶ added in v1.4.0
type MetaPartition ¶
MetaPartition defines the interface for the meta partition operations.
func NewMetaPartition ¶
func NewMetaPartition(conf *MetaPartitionConfig, manager *metadataManager) MetaPartition
NewMetaPartition creates a new meta partition with the specified configuration.
type MetaPartitionConfig ¶
type MetaPartitionConfig struct { // Identity for raftStore group. RaftStore nodes in the same raftStore group must have the same groupID. PartitionId uint64 `json:"partition_id"` VolName string `json:"vol_name"` Start uint64 `json:"start"` // Minimal Inode ID of this range. (Required during initialization) End uint64 `json:"end"` // Maximal Inode ID of this range. (Required during initialization) Peers []proto.Peer `json:"peers"` // Peers information of the raftStore Cursor uint64 `json:"-"` // Cursor ID of the inode that have been assigned NodeId uint64 `json:"-"` RootDir string `json:"-"` BeforeStart func() `json:"-"` AfterStart func() `json:"-"` BeforeStop func() `json:"-"` AfterStop func() `json:"-"` RaftStore raftstore.RaftStore `json:"-"` ConnPool *util.ConnectPool `json:"-"` }
MetaPartitionConfig is used to create a meta partition.
type MetadataManager ¶
type MetadataManager interface { Start() error Stop() //CreatePartition(id string, start, end uint64, peers []proto.Peer) error HandleMetadataOperation(conn net.Conn, p *Packet, remoteAddr string) error GetPartition(id uint64) (MetaPartition, error) }
MetadataManager manages all the meta partitions.
func NewMetadataManager ¶
func NewMetadataManager(conf MetadataManagerConfig) MetadataManager
NewMetadataManager returns a new metadata manager.
type MetadataManagerConfig ¶
MetadataManagerConfig defines the configures in the metadata manager.
type NowTime ¶
NowTime defines the current time.
func (*NowTime) GetCurrentTime ¶
GetCurrentTime returns the current time.
func (*NowTime) UpdateTime ¶
UpdateTime updates the stored time.
type OpDentry ¶
type OpDentry interface { CreateDentry(req *CreateDentryReq, p *Packet) (err error) DeleteDentry(req *DeleteDentryReq, p *Packet) (err error) UpdateDentry(req *UpdateDentryReq, p *Packet) (err error) ReadDir(req *ReadDirReq, p *Packet) (err error) Lookup(req *LookupReq, p *Packet) (err error) GetDentryTree() *BTree }
OpDentry defines the interface for the dentry operations.
type OpExtent ¶
type OpExtent interface { ExtentAppend(req *proto.AppendExtentKeyRequest, p *Packet) (err error) ExtentsList(req *proto.GetExtentsRequest, p *Packet) (err error) ExtentsTruncate(req *ExtentsTruncateReq, p *Packet) (err error) }
OpExtent defines the interface for the extent operations.
type OpInode ¶
type OpInode interface { CreateInode(req *CreateInoReq, p *Packet) (err error) UnlinkInode(req *UnlinkInoReq, p *Packet) (err error) InodeGet(req *InodeGetReq, p *Packet) (err error) InodeGetBatch(req *InodeGetReqBatch, p *Packet) (err error) CreateInodeLink(req *LinkInodeReq, p *Packet) (err error) EvictInode(req *EvictInodeReq, p *Packet) (err error) SetAttr(reqData []byte, p *Packet) (err error) GetInodeTree() *BTree }
OpInode defines the interface for the inode operations.
type OpMeta ¶
type OpMeta interface { OpInode OpDentry OpExtent OpPartition }
OpMeta defines the interface for the metadata operations.
type OpPartition ¶
type OpPartition interface { IsLeader() (leaderAddr string, isLeader bool) GetCursor() uint64 GetBaseConfig() MetaPartitionConfig LoadSnapshotSign(p *Packet) (err error) PersistMetadata() (err error) ChangeMember(changeType raftproto.ConfChangeType, peer raftproto.Peer, context []byte) (resp interface{}, err error) DeletePartition() (err error) UpdatePartition(req *UpdatePartitionReq, resp *UpdatePartitionResp) (err error) DeleteRaft() error IsExsitPeer(peer proto.Peer) bool TryToLeader(groupID uint64) error CanRemoveRaftMember(peer proto.Peer) error }
OpPartition defines the interface for the partition operations.
type Packet ¶
func NewPacketToDeleteExtent ¶
func NewPacketToDeleteExtent(dp *DataPartition, ext *proto.ExtentKey) *Packet
NewPacketToDeleteExtent returns a new packet to delete the extent.
func NewPacketToFreeInodeOnRaftFollower ¶
NewPacketToDeleteExtent returns a new packet to delete the extent.
type UnlinkInoReq ¶
type UnlinkInoReq = proto.UnlinkInodeRequest
Client -> MetaNode delete Inode request struct
type UnlinkInoResp ¶
type UnlinkInoResp = proto.UnlinkInodeResponse
MetaNode -> Client delete Inode response
type UpdateDentryReq ¶
type UpdateDentryReq = proto.UpdateDentryRequest
Client -> MetaNode updateDentry request
type UpdateDentryResp ¶
type UpdateDentryResp = proto.UpdateDentryResponse
MetaNode -> Client updateDentry response
type UpdatePartitionReq ¶
type UpdatePartitionReq = proto.UpdateMetaPartitionRequest
Master -> MetaNode
type UpdatePartitionResp ¶
type UpdatePartitionResp = proto.UpdateMetaPartitionResponse
MetaNode -> Master
type Vol ¶
Vol defines the view of the data partition with the read/write lock.
func (*Vol) GetPartition ¶
func (v *Vol) GetPartition(partitionID uint64) *DataPartition
GetPartition returns the data partition based on the given partition ID.
func (*Vol) UpdatePartitions ¶
func (v *Vol) UpdatePartitions(partitions *DataPartitionsView)
UpdatePartitions updates the data partition.
Source Files ¶
- api_handler.go
- btree.go
- const.go
- datapartition.go
- dentry.go
- extents.go
- free_list.go
- inode.go
- manager.go
- manager_op.go
- manager_proxy.go
- manager_resp.go
- metanode.go
- packet.go
- partition.go
- partition_delete_extents.go
- partition_free_list.go
- partition_fsm.go
- partition_fsmop.go
- partition_fsmop_dentry.go
- partition_fsmop_inode.go
- partition_item.go
- partition_op_dentry.go
- partition_op_extent.go
- partition_op_inode.go
- partition_store.go
- partition_store_ticket.go
- raft_server.go
- server.go
- time.go