master

package
v2.2.2+incompatible Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 23, 2020 License: Apache-2.0 Imports: 43 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// the maximum number of tasks that can be handled each time
	MaxTaskNum = 30

	TaskWorkerInterval = time.Second * time.Duration(2)
)

const

View Source 
const (
	LRUCacheSize    = 3 << 30
	WriteBufferSize = 4 * util.MB
)
View Source 
const (
	EmptyCrcValue   uint32 = 4045511210
	DefaultZoneName        = proto.DefaultZoneName
)
View Source 
const (
	OpSyncAddToken    uint32 = 0x20
	OpSyncDelToken    uint32 = 0x21
	OpSyncUpdateToken uint32 = 0x22
)
View Source 
const (
	StatPeriod                 = time.Minute * time.Duration(1)
	MetricDataNodesUsedGB      = "dataNodes_used_GB"
	MetricDataNodesTotalGB     = "dataNodes_total_GB"
	MetricDataNodesIncreasedGB = "dataNodes_increased_GB"
	MetricMetaNodesUsedGB      = "metaNodes_used_GB"
	MetricMetaNodesTotalGB     = "metaNodes_total_GB"
	MetricMetaNodesIncreasedGB = "metaNodes_increased_GB"
	MetricDataNodesCount       = "dataNodes_count"
	MetricMetaNodesCount       = "metaNodes_count"
	MetricVolCount             = "vol_count"
	MetricVolTotalGB           = "vol_total_GB"
	MetricVolUsedGB            = "vol_used_GB"
	MetricVolUsageGB           = "vol_usage_ratio"
	MetricDiskError            = "disk_error"
	MetricDataNodesInactive    = "dataNodes_inactive"
	MetricMetaNodesInactive    = "metaNodes_inactive"
)

metrics

View Source 
const (
	ClusterName       = "clusterName"
	ID                = "id"
	IP                = "ip"
	Port              = "port"
	LogLevel          = "logLevel"
	WalDir            = "walDir"
	StoreDir          = "storeDir"
	GroupID           = 1
	ModuleName        = "master"
	CfgRetainLogs     = "retainLogs"
	DefaultRetainLogs = 20000

	SecretKey = "masterServiceKey"
)

configuration keys

View Source 
const (
	RootUserID          = "root"
	DefaultRootPasswd   = "ChubaoFSRoot"
	DefaultUserPassword = "ChubaoFSUser"
)
View Source 
const ADMIN permissionMode = permissionMode(1)
View Source 
const (
	NumberOfDataPartitionsToLoad = "numberOfDataPartitionsToLoad"
)

config key

View Source 
const (
	TokenPrefix = keySeparator + tokenAcronym + keySeparator
)
View Source 
const USER permissionMode = permissionMode(2)

Variables

View Source 
var AddrDatabase = make(map[uint64]string)

AddrDatabase is a map that stores the address of a given host (e.g., the leader)

Functions

func ErrResponse 

func ErrResponse(w http.ResponseWriter, err error)

func Warn 

func Warn(clusterID, msg string)

Warn provides warnings when exits

func WarnBySpecialKey 

func WarnBySpecialKey(key, msg string)

WarnBySpecialKey provides warnings when exits

Types

type AdminTaskManager 

type AdminTaskManager struct {
	TaskMap map[string]*proto.AdminTask
	sync.RWMutex
	// contains filtered or unexported fields
}

AdminTaskManager sends administration commands to the metaNode or dataNode.

func (*AdminTaskManager) AddTask 

func (sender *AdminTaskManager) AddTask(t *proto.AdminTask)

AddTask adds a new task to the task map.

func (*AdminTaskManager) DelTask 

func (sender *AdminTaskManager) DelTask(t *proto.AdminTask)

DelTask deletes the to-be-deleted tasks.

type AuthorizedVols 

type AuthorizedVols struct {
	Vol        string
	Authorized []string
}

type Cluster 

type Cluster struct {
	Name string

	BadDataPartitionIds *sync.Map
	BadMetaPartitionIds *sync.Map
	DisableAutoAllocate bool

	MasterSecretKey []byte
	// contains filtered or unexported fields
}

Cluster stores all the cluster-level information.

type ClusterService 

type ClusterService struct {
	// contains filtered or unexported fields
}

func (*ClusterService) Schema 

func (s *ClusterService) Schema() *graphql.Schema

type DataNode 

type DataNode struct {
	Total          uint64 `json:"TotalWeight"`
	Used           uint64 `json:"UsedWeight"`
	AvailableSpace uint64
	ID             uint64
	ZoneName       string `json:"Zone"`
	Addr           string
	ReportTime     time.Time

	sync.RWMutex              `graphql:"-"`
	UsageRatio                float64           // used / total space
	SelectedTimes             uint64            // number times that this datanode has been selected as the location for a data partition.
	Carry                     float64           // carry is a factor used in cacluate the node's weight
	TaskManager               *AdminTaskManager `graphql:"-"`
	DataPartitionReports      []*proto.PartitionReport
	DataPartitionCount        uint32
	NodeSetID                 uint64
	PersistenceDataPartitions []uint64
	BadDisks                  []string
	ToBeOffline               bool
	// contains filtered or unexported fields
}

DataNode stores all the information about a data node

func (*DataNode) GetAddr 

func (dataNode *DataNode) GetAddr() string

func (*DataNode) GetID 

func (dataNode *DataNode) GetID() uint64

func (*DataNode) SelectNodeForWrite 

func (dataNode *DataNode) SelectNodeForWrite()

SelectNodeForWrite implements "SelectNodeForWrite" in the Node interface

func (*DataNode) SetCarry 

func (dataNode *DataNode) SetCarry(carry float64)

SetCarry implements "SetCarry" in the Node interface

type DataPartition 

type DataPartition struct {
	PartitionID    uint64
	LastLoadedTime int64
	ReplicaNum     uint8
	Status         int8

	Replicas []*DataReplica
	Hosts    []string // host addresses
	Peers    []proto.Peer

	sync.RWMutex

	MissingNodes map[string]int64 // key: address of the missing node, value: when the node is missing
	VolName      string
	VolID        uint64

	OfflinePeerID           uint64
	FileInCoreMap           map[string]*FileInCore
	FilesWithMissingReplica map[string]int64 // key: file name, value: last time when a missing replica is found
	// contains filtered or unexported fields
}

DataPartition represents the structure of storing the file contents.

func (*DataPartition) ToProto added in v1.5.0 

func (partition *DataPartition) ToProto(c *Cluster) *proto.DataPartitionInfo

type DataPartitionMap 

type DataPartitionMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

DataPartitionMap stores all the data partitionMap

type DataReplica 

type DataReplica struct {
	proto.DataReplica
	// contains filtered or unexported fields
}

DataReplica represents the replica of a data partition

type FileCrc 

type FileCrc struct {
	// contains filtered or unexported fields
}

FileCrc defines the crc of a file

type FileInCore 

type FileInCore struct {
	proto.FileInCore
	MetadataArray []*FileMetadata
}

FileInCore define file in data partition

func (FileInCore) ToProto added in v1.5.0 

func (fc FileInCore) ToProto() proto.FileInCore

type FileMetadata 

type FileMetadata struct {
	proto.FileMetadata
	// contains filtered or unexported fields
}

FileMetadata defines the file metadata on a dataNode

func (*FileMetadata) String 

func (fm *FileMetadata) String() (msg string)

type GetCarryNodes 

type GetCarryNodes func(maxTotal uint64, excludeHosts []string, nodes *sync.Map) (weightedNodes SortedWeightedNodes, availCount int)

type GetMaxTotal 

type GetMaxTotal func(nodes *sync.Map) (maxTotal uint64)

type IDAllocator 

type IDAllocator struct {
	// contains filtered or unexported fields
}

IDAllocator generates and allocates ids

type InvalidNodeView 

type InvalidNodeView struct {
	Addr     string
	ID       uint64
	OldID    uint64
	NodeType string
}

NodeView provides the view of the data or meta node.

type LeaderInfo 

type LeaderInfo struct {
	// contains filtered or unexported fields
}

LeaderInfo represents the leader's information

type MasterInfo 

type MasterInfo struct {
	Index    string
	Addr     string
	IsLeader bool
}

type MetaNode 

type MetaNode struct {
	ID                uint64
	Addr              string
	IsActive          bool
	Sender            *AdminTaskManager `graphql:"-"`
	ZoneName          string            `json:"Zone"`
	MaxMemAvailWeight uint64            `json:"MaxMemAvailWeight"`
	Total             uint64            `json:"TotalWeight"`
	Used              uint64            `json:"UsedWeight"`
	Ratio             float64
	SelectCount       uint64
	Carry             float64
	Threshold         float32
	ReportTime        time.Time

	MetaPartitionCount        int
	NodeSetID                 uint64
	sync.RWMutex              `graphql:"-"`
	ToBeOffline               bool
	PersistenceMetaPartitions []uint64
	// contains filtered or unexported fields
}

MetaNode defines the structure of a meta node

func (*MetaNode) GetAddr 

func (metaNode *MetaNode) GetAddr() string

func (*MetaNode) GetID 

func (metaNode *MetaNode) GetID() uint64

func (*MetaNode) SelectNodeForWrite 

func (metaNode *MetaNode) SelectNodeForWrite()

SelectNodeForWrite implements the Node interface

func (*MetaNode) SetCarry 

func (metaNode *MetaNode) SetCarry(carry float64)

SetCarry implements the Node interface

type MetaPartition 

type MetaPartition struct {
	PartitionID uint64
	Start       uint64
	End         uint64
	MaxInodeID  uint64
	InodeCount  uint64
	DentryCount uint64
	Replicas    []*MetaReplica
	ReplicaNum  uint8
	Status      int8
	IsRecover   bool

	Hosts         []string
	Peers         []proto.Peer
	OfflinePeerID uint64
	MissNodes     map[string]int64
	LoadResponse  []*proto.MetaPartitionLoadResponse

	sync.RWMutex
	// contains filtered or unexported fields
}

MetaPartition defines the structure of a meta partition

type MetaReplica 

type MetaReplica struct {
	Addr string

	MaxInodeID  uint64
	InodeCount  uint64
	DentryCount uint64
	ReportTime  int64
	Status      int8 // unavailable, readOnly, readWrite
	IsLeader    bool
	// contains filtered or unexported fields
}

MetaReplica defines the replica of a meta partition

type MetadataFsm 

type MetadataFsm struct {
	// contains filtered or unexported fields
}

MetadataFsm represents the finite state machine of a metadata partition

func (*MetadataFsm) Apply 

func (mf *MetadataFsm) Apply(command []byte, index uint64) (resp interface{}, err error)

Apply implements the interface of raft.StateMachine

func (*MetadataFsm) ApplyMemberChange 

func (mf *MetadataFsm) ApplyMemberChange(confChange *proto.ConfChange, index uint64) (interface{}, error)

ApplyMemberChange implements the interface of raft.StateMachine

func (*MetadataFsm) ApplySnapshot 

func (mf *MetadataFsm) ApplySnapshot(peers []proto.Peer, iterator proto.SnapIterator) (err error)

ApplySnapshot implements the interface of raft.StateMachine

func (*MetadataFsm) HandleFatalEvent 

func (mf *MetadataFsm) HandleFatalEvent(err *raft.FatalError)

HandleFatalEvent implements the interface of raft.StateMachine

func (*MetadataFsm) HandleLeaderChange 

func (mf *MetadataFsm) HandleLeaderChange(leader uint64)

HandleLeaderChange implements the interface of raft.StateMachine

func (*MetadataFsm) Snapshot 

func (mf *MetadataFsm) Snapshot() (proto.Snapshot, error)

Snapshot implements the interface of raft.StateMachine

type MetadataSnapshot 

type MetadataSnapshot struct {
	// contains filtered or unexported fields
}

MetadataSnapshot represents the snapshot of a meta partition

func (*MetadataSnapshot) ApplyIndex 

func (ms *MetadataSnapshot) ApplyIndex() uint64

ApplyIndex implements the Snapshot interface

func (*MetadataSnapshot) Close 

func (ms *MetadataSnapshot) Close()

Close implements the Snapshot interface

func (*MetadataSnapshot) Next 

func (ms *MetadataSnapshot) Next() (data []byte, err error)

Next implements the Snapshot interface

type Node 

type Node interface {
	SetCarry(carry float64)
	SelectNodeForWrite()
	GetID() uint64
	GetAddr() string
}

Node defines an interface that needs to be implemented by weightedNode

type NodeSetView 

type NodeSetView struct {
	DataNodeLen int
	MetaNodeLen int
	MetaNodes   []proto.NodeView
	DataNodes   []proto.NodeView
}

type NodeView 

type NodeView struct {
	Addr       string
	Status     bool
	ID         uint64
	IsWritable bool
}

NodeView provides the view of the data or meta node.

type RaftCmd 

type RaftCmd struct {
	Op uint32 `json:"op"`
	K  string `json:"k"`
	V  []byte `json:"v"`
}

RaftCmd defines the Raft commands.

func (*RaftCmd) Marshal 

func (m *RaftCmd) Marshal() ([]byte, error)

Marshal converts the RaftCmd to a byte array.

func (*RaftCmd) Unmarshal 

func (m *RaftCmd) Unmarshal(data []byte) (err error)

Unmarshal converts the byte array to a RaftCmd.

type Server 

type Server struct {
	// contains filtered or unexported fields
}

Server represents the server in a cluster

func NewServer 

func NewServer() *Server

NewServer creates a new server

func (*Server) Shutdown 

func (m *Server) Shutdown()

Shutdown closes the server

func (*Server) Start 

func (m *Server) Start(cfg *config.Config) (err error)

Start starts a server

func (*Server) Sync 

func (m *Server) Sync()

Sync waits for the execution termination of the server

type SortedWeightedNodes 

type SortedWeightedNodes []*weightedNode

SortedWeightedNodes defines an array sorted by carry

func (SortedWeightedNodes) Len 

func (nodes SortedWeightedNodes) Len() int

func (SortedWeightedNodes) Less 

func (nodes SortedWeightedNodes) Less(i, j int) bool

func (SortedWeightedNodes) Swap 

func (nodes SortedWeightedNodes) Swap(i, j int)

type TokenValue 

type TokenValue struct {
	VolName   string
	Value     string
	TokenType int8
}

type TopologyView 

type TopologyView struct {
	Zones []*ZoneView
}

TopologyView provides the view of the topology view of the cluster

type User 

type User struct {
	AKStore sync.Map //K: ak, V: userID

	AKStoreMutex sync.RWMutex
	// contains filtered or unexported fields
}

type UserPermission 

type UserPermission struct {
	UserID string
	Access []string
	Edit   bool
}

type UserService 

type UserService struct {
	// contains filtered or unexported fields
}

func (*UserService) Schema 

func (s *UserService) Schema() *graphql.Schema

type UserStatistical 

type UserStatistical struct {
	Data               uint64
	VolumeCount        int32
	DataPartitionCount int32
	MetaPartitionCount int32
}

type UserUseSpace 

type UserUseSpace struct {
	Name  string
	Size  uint64
	Ratio float32
}

type Vol 

type Vol struct {
	ID           uint64
	Name         string
	Owner        string
	OSSAccessKey string
	OSSSecretKey string

	Status uint8

	Capacity           uint64 // GB
	NeedToLowerReplica bool
	FollowerRead       bool

	MetaPartitions map[uint64]*MetaPartition `graphql:"-"`

	sync.RWMutex
	// contains filtered or unexported fields
}

Vol represents a set of meta partitionMap and data partitionMap

func (*Vol) String 

func (vol *Vol) String() string

type VolVarargs 

type VolVarargs struct {
	// contains filtered or unexported fields
}

type VolumeService 

type VolumeService struct {
	// contains filtered or unexported fields
}

func (*VolumeService) Schema 

func (s *VolumeService) Schema() *graphql.Schema

type WarnMessage 

type WarnMessage struct {
	Time     string `json:"time"`
	Key      string `json:"key"`
	Hostname string `json:"hostname"`
	Type     string `json:"type"`
	Value    string `json:"value"`
	Detail   string `json:"detail"`
}

type Zone 

type Zone struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Zone stores all the zone related information

type ZoneView 

type ZoneView struct {
	Name    string
	Status  string
	NodeSet map[uint64]*NodeSetView
}

ZoneView define the view of zone

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.