master

package
v1.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 11, 2019 License: Apache-2.0 Imports: 32 Imported by: 2

Documentation

Index

Constants

View Source 
const (
	// the maximum number of tasks that can be handled each time
	MaxTaskNum = 30

	TaskWorkerInterval = time.Microsecond * time.Duration(200)
)

const

View Source 
const (
	LRUCacheSize    = 3 << 30
	WriteBufferSize = 4 * util.MB
)
View Source 
const (
	StatPeriod                 = time.Minute * time.Duration(1)
	MetricDataNodesUsedGB      = "dataNodes_used_GB"
	MetricDataNodesTotalGB     = "dataNodes_total_GB"
	MetricDataNodesIncreasedGB = "dataNodes_increased_GB"
	MetricMetaNodesUsedGB      = "metaNodes_used_GB"
	MetricMetaNodesTotalGB     = "metaNodes_total_GB"
	MetricMetaNodesIncreasedGB = "metaNodes_increased_GB"
	MetricDataNodesCount       = "dataNodes_count"
	MetricMetaNodesCount       = "metaNodes_count"
	MetricVolCount             = "vol_count"
	MetricVolTotalGB           = "vol_total_GB"
	MetricVolUsedGB            = "vol_used_GB"
	MetricVolUsageGB           = "vol_usage_ratio"
)

metrics

View Source 
const (
	ClusterName       = "clusterName"
	ID                = "id"
	IP                = "ip"
	Port              = "port"
	LogLevel          = "logLevel"
	WalDir            = "walDir"
	StoreDir          = "storeDir"
	GroupID           = 1
	ModuleName        = "master"
	CfgRetainLogs     = "retainLogs"
	DefaultRetainLogs = 20000
)

configuration keys

View Source 
const (
	EmptyCrcValue uint32 = 4045511210
)
View Source 
const (
	NumberOfDataPartitionsToLoad = "NumberOfDataPartitionsToLoad"
)

config key

Variables

View Source 
var AddrDatabase = make(map[uint64]string)

AddrDatabase is a map that stores the address of a given host (e.g., the leader)

Functions

func Warn 

func Warn(clusterID, msg string)

Warn provides warnings when exits

func WarnBySpecialKey 

func WarnBySpecialKey(key, msg string)

WarnBySpecialKey provides warnings when exits

Types

type AdminTaskManager 

type AdminTaskManager struct {
	TaskMap map[string]*proto.AdminTask
	sync.RWMutex
	// contains filtered or unexported fields
}

AdminTaskManager sends administration commands to the metaNode or dataNode.

func (*AdminTaskManager) AddTask 

func (sender *AdminTaskManager) AddTask(t *proto.AdminTask)

AddTask adds a new task to the task map.

func (*AdminTaskManager) DelTask 

func (sender *AdminTaskManager) DelTask(t *proto.AdminTask)

DelTask deletes the to-be-deleted tasks.

type Cluster 

type Cluster struct {
	Name string

	BadDataPartitionIds *sync.Map
	DisableAutoAllocate bool
	// contains filtered or unexported fields
}

Cluster stores all the cluster-level information.

type ClusterView 

type ClusterView struct {
	Name               string
	LeaderAddr         string
	DisableAutoAlloc   bool
	MetaNodeThreshold  float32
	Applied            uint64
	MaxDataPartitionID uint64
	MaxMetaNodeID      uint64
	MaxMetaPartitionID uint64
	DataNodeStatInfo   *nodeStatInfo
	MetaNodeStatInfo   *nodeStatInfo
	VolStatInfo        []*volStatInfo
	BadPartitionIDs    []badPartitionView
	MetaNodes          []NodeView
	DataNodes          []NodeView
}

ClusterView provides the view of a cluster.

type DataNode 

type DataNode struct {
	Total          uint64 `json:"TotalWeight"`
	Used           uint64 `json:"UsedWeight"`
	AvailableSpace uint64
	ID             uint64
	RackName       string `json:"Rack"`
	Addr           string
	ReportTime     time.Time

	sync.RWMutex
	UsageRatio    float64 // used / total space
	SelectedTimes uint64  // number times that this datanode has been selected as the location for a data partition.
	Carry         float64 // carry is a factor used in cacluate the node's weight
	TaskManager   *AdminTaskManager

	DataPartitionCount uint32
	NodeSetID          uint64
	// contains filtered or unexported fields
}

DataNode stores all the information about a data node

func (*DataNode) SelectNodeForWrite 

func (dataNode *DataNode) SelectNodeForWrite()

SelectNodeForWrite implements "SelectNodeForWrite" in the Node interface

func (*DataNode) SetCarry 

func (dataNode *DataNode) SetCarry(carry float64)

SetCarry implements "SetCarry" in the Node interface

type DataPartition 

type DataPartition struct {
	PartitionID    uint64
	LastLoadedTime int64
	ReplicaNum     uint8
	Status         int8

	Replicas []*DataReplica
	Hosts    []string // host addresses
	Peers    []proto.Peer
	sync.RWMutex

	MissingNodes map[string]int64 // key: address of the missing node, value: when the node is missing
	VolName      string
	VolID        uint64

	FileInCoreMap           map[string]*FileInCore
	FilesWithMissingReplica map[string]int64 // key: file name, value: last time when a missing replica is found
	// contains filtered or unexported fields
}

DataPartition represents the structure of storing the file contents.

type DataPartitionMap 

type DataPartitionMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

DataPartitionMap stores all the data partitionMap

type DataReplica 

type DataReplica struct {
	Addr string

	ReportTime int64
	FileCount  uint32

	Status          int8
	HasLoadResponse bool   // if there is any response when loading
	Total           uint64 `json:"TotalSize"`
	Used            uint64 `json:"UsedSize"`
	IsLeader        bool
	NeedsToCompare  bool
	DiskPath        string
	// contains filtered or unexported fields
}

DataReplica represents the replica of a data partition

type FileCrc 

type FileCrc struct {
	// contains filtered or unexported fields
}

FileCrc defines the crc of a file

type FileInCore 

type FileInCore struct {
	Name          string
	LastModify    int64
	MetadataArray []*FileMetadata
}

FileInCore define file in data partition

type FileMetadata 

type FileMetadata struct {
	Crc     uint32
	LocAddr string

	Size uint32
	// contains filtered or unexported fields
}

FileMetadata defines the file metadata on a dataNode

func (*FileMetadata) String 

func (fm *FileMetadata) String() (msg string)

type IDAllocator 

type IDAllocator struct {
	// contains filtered or unexported fields
}

IDAllocator generates and allocates ids

type LeaderInfo 

type LeaderInfo struct {
	// contains filtered or unexported fields
}

LeaderInfo represents the leader's information

type MetaNode 

type MetaNode struct {
	ID                uint64
	Addr              string
	IsActive          bool
	Sender            *AdminTaskManager
	RackName          string `json:"Rack"`
	MaxMemAvailWeight uint64 `json:"MaxMemAvailWeight"`
	Total             uint64 `json:"TotalWeight"`
	Used              uint64 `json:"UsedWeight"`
	Ratio             float64
	SelectCount       uint64
	Carry             float64
	Threshold         float32
	ReportTime        time.Time

	MetaPartitionCount int
	NodeSetID          uint64
	sync.RWMutex
	// contains filtered or unexported fields
}

MetaNode defines the structure of a meta node

func (*MetaNode) SelectNodeForWrite 

func (metaNode *MetaNode) SelectNodeForWrite()

SelectNodeForWrite implements the Node interface

func (*MetaNode) SetCarry 

func (metaNode *MetaNode) SetCarry(carry float64)

SetCarry implements the Node interface

type MetaPartition 

type MetaPartition struct {
	PartitionID uint64
	Start       uint64
	End         uint64
	MaxNodeID   uint64
	Replicas    []*MetaReplica
	ReplicaNum  uint8
	Status      int8

	Hosts        []string
	Peers        []proto.Peer
	MissNodes    map[string]int64
	LoadResponse []*proto.MetaPartitionLoadResponse
	sync.RWMutex
	// contains filtered or unexported fields
}

MetaPartition defines the structure of a meta partition

type MetaReplica 

type MetaReplica struct {
	Addr string

	ReportTime int64
	Status     int8 // unavailable, readOnly, readWrite
	IsLeader   bool
	// contains filtered or unexported fields
}

MetaReplica defines the replica of a meta partition

type MetadataFsm 

type MetadataFsm struct {
	// contains filtered or unexported fields
}

MetadataFsm represents the finite state machine of a metadata partition

func (*MetadataFsm) Apply 

func (mf *MetadataFsm) Apply(command []byte, index uint64) (resp interface{}, err error)

Apply implements the interface of raft.StateMachine

func (*MetadataFsm) ApplyMemberChange 

func (mf *MetadataFsm) ApplyMemberChange(confChange *proto.ConfChange, index uint64) (interface{}, error)

ApplyMemberChange implements the interface of raft.StateMachine

func (*MetadataFsm) ApplySnapshot 

func (mf *MetadataFsm) ApplySnapshot(peers []proto.Peer, iterator proto.SnapIterator) (err error)

ApplySnapshot implements the interface of raft.StateMachine

func (*MetadataFsm) Del 

func (mf *MetadataFsm) Del(key interface{}) (interface{}, error)

Del implements the interface of raft.StateMachine

func (*MetadataFsm) Get 

func (mf *MetadataFsm) Get(key interface{}) (interface{}, error)

Get implements the interface of raft.StateMachine

func (*MetadataFsm) HandleFatalEvent 

func (mf *MetadataFsm) HandleFatalEvent(err *raft.FatalError)

HandleFatalEvent implements the interface of raft.StateMachine

func (*MetadataFsm) HandleLeaderChange 

func (mf *MetadataFsm) HandleLeaderChange(leader uint64)

HandleLeaderChange implements the interface of raft.StateMachine

func (*MetadataFsm) Put 

func (mf *MetadataFsm) Put(key, val interface{}) (interface{}, error)

Put implements the interface of raft.StateMachine

func (*MetadataFsm) Snapshot 

func (mf *MetadataFsm) Snapshot() (proto.Snapshot, error)

Snapshot implements the interface of raft.StateMachine

type MetadataSnapshot 

type MetadataSnapshot struct {
	// contains filtered or unexported fields
}

MetadataSnapshot represents the snapshot of a meta partition

func (*MetadataSnapshot) ApplyIndex 

func (ms *MetadataSnapshot) ApplyIndex() uint64

ApplyIndex implements the Snapshot interface

func (*MetadataSnapshot) Close 

func (ms *MetadataSnapshot) Close()

Close implements the Snapshot interface

func (*MetadataSnapshot) Next 

func (ms *MetadataSnapshot) Next() (data []byte, err error)

Next implements the Snapshot interface

type Node 

type Node interface {
	SetCarry(carry float64)
	SelectNodeForWrite()
}

Node defines an interface that needs to be implemented by weightedNode

type NodeView 

type NodeView struct {
	Addr       string
	Status     bool
	ID         uint64
	IsWritable bool
}

NodeView provides the view of the data or meta node.

type Rack 

type Rack struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Rack stores all the rack related information

type RackView 

type RackView struct {
	Name      string
	DataNodes []NodeView
}

RackView define the view of rack

type RaftCmd 

type RaftCmd struct {
	Op uint32 `json:"op"`
	K  string `json:"k"`
	V  []byte `json:"v"`
}

RaftCmd defines the Raft commands.

func (*RaftCmd) Marshal 

func (m *RaftCmd) Marshal() ([]byte, error)

Marshal converts the RaftCmd to a byte array.

func (*RaftCmd) Unmarshal 

func (m *RaftCmd) Unmarshal(data []byte) (err error)

Unmarshal converts the byte array to a RaftCmd.

type Server 

type Server struct {
	// contains filtered or unexported fields
}

Server represents the server in a cluster

func NewServer 

func NewServer() *Server

NewServer creates a new server

func (*Server) ServeHTTP 

func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (*Server) Shutdown 

func (m *Server) Shutdown()

Shutdown closes the server

func (*Server) Start 

func (m *Server) Start(cfg *config.Config) (err error)

Start starts a server

func (*Server) Sync 

func (m *Server) Sync()

Sync waits for the execution termination of the server

type SortedWeightedNodes 

type SortedWeightedNodes []*weightedNode

SortedWeightedNodes defines an array sorted by carry

func (SortedWeightedNodes) Len 

func (nodes SortedWeightedNodes) Len() int

func (SortedWeightedNodes) Less 

func (nodes SortedWeightedNodes) Less(i, j int) bool

func (SortedWeightedNodes) Swap 

func (nodes SortedWeightedNodes) Swap(i, j int)

type TopologyView 

type TopologyView struct {
	NodeSet map[uint64]*nodeSetView
}

TopologyView provides the view of the topology view of the cluster

type Vol 

type Vol struct {
	ID    uint64
	Name  string
	Owner string

	Status uint8

	Capacity       uint64 // GB
	MetaPartitions map[uint64]*MetaPartition

	sync.RWMutex
	// contains filtered or unexported fields
}

Vol represents a set of meta partitionMap and data partitionMap

func (*Vol) String 

func (vol *Vol) String() string

type VolStatView 

type VolStatView struct {
	Name      string
	Total     uint64 `json:"TotalGB"`
	Used      uint64 `json:"UsedGB"`
	Increased uint64 `json:"IncreasedGB"`
}

VolStatView provides the view of the volume.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.