master

package
v1.5.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 20, 2020 License: Apache-2.0 Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// the maximum number of tasks that can be handled each time
	MaxTaskNum = 30

	TaskWorkerInterval = time.Second * time.Duration(2)
)

const

View Source 
const (
	LRUCacheSize    = 3 << 30
	WriteBufferSize = 4 * util.MB
)
View Source 
const (
	EmptyCrcValue   uint32 = 4045511210
	DefaultCellName        = "default"
)
View Source 
const (
	StatPeriod                 = time.Minute * time.Duration(1)
	MetricDataNodesUsedGB      = "dataNodes_used_GB"
	MetricDataNodesTotalGB     = "dataNodes_total_GB"
	MetricDataNodesIncreasedGB = "dataNodes_increased_GB"
	MetricMetaNodesUsedGB      = "metaNodes_used_GB"
	MetricMetaNodesTotalGB     = "metaNodes_total_GB"
	MetricMetaNodesIncreasedGB = "metaNodes_increased_GB"
	MetricDataNodesCount       = "dataNodes_count"
	MetricMetaNodesCount       = "metaNodes_count"
	MetricVolCount             = "vol_count"
	MetricVolTotalGB           = "vol_total_GB"
	MetricVolUsedGB            = "vol_used_GB"
	MetricVolUsageGB           = "vol_usage_ratio"
	MetricDiskError            = "disk_error"
	MetricDataNodesInactive    = "dataNodes_inactive"
	MetricMetaNodesInactive    = "metaNodes_inactive"
)

metrics

View Source 
const (
	ClusterName       = "clusterName"
	ID                = "id"
	IP                = "ip"
	Port              = "port"
	LogLevel          = "logLevel"
	WalDir            = "walDir"
	StoreDir          = "storeDir"
	GroupID           = 1
	ModuleName        = "master"
	CfgRetainLogs     = "retainLogs"
	DefaultRetainLogs = 20000

	SecretKey = "masterServiceKey"
)

configuration keys

View Source 
const (
	NumberOfDataPartitionsToLoad = "NumberOfDataPartitionsToLoad"
)

config key

Variables

View Source 
var AddrDatabase = make(map[uint64]string)

AddrDatabase is a map that stores the address of a given host (e.g., the leader)

Functions

func Warn 

func Warn(clusterID, msg string)

Warn provides warnings when exits

func WarnBySpecialKey 

func WarnBySpecialKey(key, msg string)

WarnBySpecialKey provides warnings when exits

Types

type AdminTaskManager 

type AdminTaskManager struct {
	TaskMap map[string]*proto.AdminTask
	sync.RWMutex
	// contains filtered or unexported fields
}

AdminTaskManager sends administration commands to the metaNode or dataNode.

func (*AdminTaskManager) AddTask 

func (sender *AdminTaskManager) AddTask(t *proto.AdminTask)

AddTask adds a new task to the task map.

func (*AdminTaskManager) DelTask 

func (sender *AdminTaskManager) DelTask(t *proto.AdminTask)

DelTask deletes the to-be-deleted tasks.

type Cell added in v1.5.0 

type Cell struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Cell stores all the cell related information

type CellView added in v1.5.0 

type CellView struct {
	Name      string
	DataNodes []NodeView
}

CellView define the view of cell

type Cluster 

type Cluster struct {
	Name string

	BadDataPartitionIds *sync.Map
	BadMetaPartitionIds *sync.Map
	DisableAutoAllocate bool

	MasterSecretKey []byte
	// contains filtered or unexported fields
}

Cluster stores all the cluster-level information.

type DataNode 

type DataNode struct {
	Total          uint64 `json:"TotalWeight"`
	Used           uint64 `json:"UsedWeight"`
	AvailableSpace uint64
	ID             uint64
	CellName       string `json:"Cell"`
	Addr           string
	ReportTime     time.Time

	sync.RWMutex
	UsageRatio                float64 // used / total space
	SelectedTimes             uint64  // number times that this datanode has been selected as the location for a data partition.
	Carry                     float64 // carry is a factor used in cacluate the node's weight
	TaskManager               *AdminTaskManager
	DataPartitionReports      []*proto.PartitionReport
	DataPartitionCount        uint32
	NodeSetID                 uint64
	PersistenceDataPartitions []uint64
	BadDisks                  []string
	// contains filtered or unexported fields
}

DataNode stores all the information about a data node

func (*DataNode) SelectNodeForWrite 

func (dataNode *DataNode) SelectNodeForWrite()

SelectNodeForWrite implements "SelectNodeForWrite" in the Node interface

func (*DataNode) SetCarry 

func (dataNode *DataNode) SetCarry(carry float64)

SetCarry implements "SetCarry" in the Node interface

type DataPartition 

type DataPartition struct {
	PartitionID    uint64
	LastLoadedTime int64
	ReplicaNum     uint8
	Status         int8

	Replicas []*DataReplica
	Hosts    []string // host addresses
	Peers    []proto.Peer
	sync.RWMutex

	MissingNodes map[string]int64 // key: address of the missing node, value: when the node is missing
	VolName      string
	VolID        uint64

	FileInCoreMap           map[string]*FileInCore
	FilesWithMissingReplica map[string]int64 // key: file name, value: last time when a missing replica is found
	// contains filtered or unexported fields
}

DataPartition represents the structure of storing the file contents.

func (*DataPartition) ToProto added in v1.5.0 

func (partition *DataPartition) ToProto() *proto.DataPartitionInfo

type DataPartitionMap 

type DataPartitionMap struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

DataPartitionMap stores all the data partitionMap

type DataReplica 

type DataReplica struct {
	proto.DataReplica
	// contains filtered or unexported fields
}

DataReplica represents the replica of a data partition

type FileCrc 

type FileCrc struct {
	// contains filtered or unexported fields
}

FileCrc defines the crc of a file

type FileInCore 

type FileInCore struct {
	proto.FileInCore
	MetadataArray []*FileMetadata
}

FileInCore define file in data partition

func (FileInCore) ToProto added in v1.5.0 

func (fc FileInCore) ToProto() proto.FileInCore

type FileMetadata 

type FileMetadata struct {
	proto.FileMetadata
	// contains filtered or unexported fields
}

FileMetadata defines the file metadata on a dataNode

func (*FileMetadata) String 

func (fm *FileMetadata) String() (msg string)

type IDAllocator 

type IDAllocator struct {
	// contains filtered or unexported fields
}

IDAllocator generates and allocates ids

type LeaderInfo 

type LeaderInfo struct {
	// contains filtered or unexported fields
}

LeaderInfo represents the leader's information

type MetaNode 

type MetaNode struct {
	ID                uint64
	Addr              string
	IsActive          bool
	Sender            *AdminTaskManager
	CellName          string `json:"Cell"`
	MaxMemAvailWeight uint64 `json:"MaxMemAvailWeight"`
	Total             uint64 `json:"TotalWeight"`
	Used              uint64 `json:"UsedWeight"`
	Ratio             float64
	SelectCount       uint64
	Carry             float64
	Threshold         float32
	ReportTime        time.Time

	MetaPartitionCount int
	NodeSetID          uint64
	sync.RWMutex
	PersistenceMetaPartitions []uint64
	// contains filtered or unexported fields
}

MetaNode defines the structure of a meta node

func (*MetaNode) SelectNodeForWrite 

func (metaNode *MetaNode) SelectNodeForWrite()

SelectNodeForWrite implements the Node interface

func (*MetaNode) SetCarry 

func (metaNode *MetaNode) SetCarry(carry float64)

SetCarry implements the Node interface

type MetaPartition 

type MetaPartition struct {
	PartitionID uint64
	Start       uint64
	End         uint64
	MaxInodeID  uint64
	Replicas    []*MetaReplica
	ReplicaNum  uint8
	Status      int8
	IsRecover   bool

	Hosts        []string
	Peers        []proto.Peer
	MissNodes    map[string]int64
	LoadResponse []*proto.MetaPartitionLoadResponse
	sync.RWMutex
	// contains filtered or unexported fields
}

MetaPartition defines the structure of a meta partition

type MetaReplica 

type MetaReplica struct {
	Addr string

	MaxInodeID uint64
	ReportTime int64
	Status     int8 // unavailable, readOnly, readWrite
	IsLeader   bool
	// contains filtered or unexported fields
}

MetaReplica defines the replica of a meta partition

type MetadataFsm 

type MetadataFsm struct {
	// contains filtered or unexported fields
}

MetadataFsm represents the finite state machine of a metadata partition

func (*MetadataFsm) Apply 

func (mf *MetadataFsm) Apply(command []byte, index uint64) (resp interface{}, err error)

Apply implements the interface of raft.StateMachine

func (*MetadataFsm) ApplyMemberChange 

func (mf *MetadataFsm) ApplyMemberChange(confChange *proto.ConfChange, index uint64) (interface{}, error)

ApplyMemberChange implements the interface of raft.StateMachine

func (*MetadataFsm) ApplySnapshot 

func (mf *MetadataFsm) ApplySnapshot(peers []proto.Peer, iterator proto.SnapIterator) (err error)

ApplySnapshot implements the interface of raft.StateMachine

func (*MetadataFsm) Del 

func (mf *MetadataFsm) Del(key interface{}) (interface{}, error)

Del implements the interface of raft.StateMachine

func (*MetadataFsm) Get 

func (mf *MetadataFsm) Get(key interface{}) (interface{}, error)

Get implements the interface of raft.StateMachine

func (*MetadataFsm) HandleFatalEvent 

func (mf *MetadataFsm) HandleFatalEvent(err *raft.FatalError)

HandleFatalEvent implements the interface of raft.StateMachine

func (*MetadataFsm) HandleLeaderChange 

func (mf *MetadataFsm) HandleLeaderChange(leader uint64)

HandleLeaderChange implements the interface of raft.StateMachine

func (*MetadataFsm) Put 

func (mf *MetadataFsm) Put(key, val interface{}) (interface{}, error)

Put implements the interface of raft.StateMachine

func (*MetadataFsm) Snapshot 

func (mf *MetadataFsm) Snapshot() (proto.Snapshot, error)

Snapshot implements the interface of raft.StateMachine

type MetadataSnapshot 

type MetadataSnapshot struct {
	// contains filtered or unexported fields
}

MetadataSnapshot represents the snapshot of a meta partition

func (*MetadataSnapshot) ApplyIndex 

func (ms *MetadataSnapshot) ApplyIndex() uint64

ApplyIndex implements the Snapshot interface

func (*MetadataSnapshot) Close 

func (ms *MetadataSnapshot) Close()

Close implements the Snapshot interface

func (*MetadataSnapshot) Next 

func (ms *MetadataSnapshot) Next() (data []byte, err error)

Next implements the Snapshot interface

type Node 

type Node interface {
	SetCarry(carry float64)
	SelectNodeForWrite()
}

Node defines an interface that needs to be implemented by weightedNode

type NodeView 

type NodeView struct {
	Addr       string
	Status     bool
	ID         uint64
	IsWritable bool
}

NodeView provides the view of the data or meta node.

type RaftCmd 

type RaftCmd struct {
	Op uint32 `json:"op"`
	K  string `json:"k"`
	V  []byte `json:"v"`
}

RaftCmd defines the Raft commands.

func (*RaftCmd) Marshal 

func (m *RaftCmd) Marshal() ([]byte, error)

Marshal converts the RaftCmd to a byte array.

func (*RaftCmd) Unmarshal 

func (m *RaftCmd) Unmarshal(data []byte) (err error)

Unmarshal converts the byte array to a RaftCmd.

type Server 

type Server struct {
	// contains filtered or unexported fields
}

Server represents the server in a cluster

func NewServer 

func NewServer() *Server

NewServer creates a new server

func (*Server) ServeHTTP 

func (m *Server) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (*Server) Shutdown 

func (m *Server) Shutdown()

Shutdown closes the server

func (*Server) Start 

func (m *Server) Start(cfg *config.Config) (err error)

Start starts a server

func (*Server) Sync 

func (m *Server) Sync()

Sync waits for the execution termination of the server

type SortedWeightedNodes 

type SortedWeightedNodes []*weightedNode

SortedWeightedNodes defines an array sorted by carry

func (SortedWeightedNodes) Len 

func (nodes SortedWeightedNodes) Len() int

func (SortedWeightedNodes) Less 

func (nodes SortedWeightedNodes) Less(i, j int) bool

func (SortedWeightedNodes) Swap 

func (nodes SortedWeightedNodes) Swap(i, j int)

type TopologyView 

type TopologyView struct {
	NodeSet map[uint64]*nodeSetView
}

TopologyView provides the view of the topology view of the cluster

type Vol 

type Vol struct {
	ID           uint64
	Name         string
	Owner        string
	OSSAccessKey string
	OSSSecretKey string

	Status uint8

	Capacity           uint64 // GB
	NeedToLowerReplica bool
	FollowerRead       bool

	MetaPartitions map[uint64]*MetaPartition

	sync.RWMutex
	// contains filtered or unexported fields
}

Vol represents a set of meta partitionMap and data partitionMap

func (*Vol) String 

func (vol *Vol) String() string

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.