Documentation ¶
Index ¶
- Constants
- Variables
- func Accept(state *state.State, gateway *Gateway, name, address string, ...) ([]db.RaftNode, error)
- func Assign(state *state.State, gateway *Gateway, nodes []db.RaftNode) error
- func Bootstrap(state *state.State, gateway *Gateway, serverName string) error
- func ClusterState(s *state.State, networkCert *shared.CertInfo, members ...db.NodeInfo) (map[string]api.ClusterMemberState, error)
- func Connect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo, ...) (lxd.InstanceServer, error)
- func ConnectIfInstanceIsRemote(s *state.State, projectName string, instName string, r *http.Request, ...) (lxd.InstanceServer, error)
- func Count(state *state.State) (int, error)
- func DatabaseReplaceFromTarball(tarballPath string, database *db.Node) error
- func DqliteLog(l client.LogLevel, format string, a ...any)
- func Enabled(node *db.Node) (bool, error)
- func EnsureServerCertificateTrusted(serverName string, serverCert *shared.CertInfo, tx *db.ClusterTx) error
- func EventHubPush(event api.Event)
- func EventListenerWait(ctx context.Context, address string) error
- func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, ...)
- func Handover(state *state.State, gateway *Gateway, address string) (string, []db.RaftNode, error)
- func HasConnectivity(networkCert *shared.CertInfo, serverCert *shared.CertInfo, address string) bool
- func HeartbeatNode(taskCtx context.Context, address string, networkCert *shared.CertInfo, ...) error
- func HeartbeatTask(gateway *Gateway) (task.Func, task.Schedule)
- func Join(state *state.State, gateway *Gateway, networkCert *shared.CertInfo, ...) error
- func Leave(state *state.State, gateway *Gateway, name string, force bool) (string, error)
- func ListDatabaseNodes(database *db.Node) ([]string, error)
- func LocalSysInfo() (*api.ClusterMemberSysInfo, error)
- func MaybeUpdate(state *state.State) error
- func MemberState(ctx context.Context, s *state.State) (*api.ClusterMemberState, error)
- func NotifyHeartbeat(state *state.State, gateway *Gateway)
- func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error
- func Purge(c *db.Cluster, name string) error
- func Rebalance(state *state.State, gateway *Gateway, unavailableMembers []string) (string, []db.RaftNode, error)
- func Reconfigure(database *db.Node, raftNodes []db.RaftNode) (string, error)
- func Recover(database *db.Node) error
- func RemoveRaftNode(gateway *Gateway, address string) error
- func ResolveTarget(ctx context.Context, s *state.State, targetMember string) (string, error)
- func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool
- func SetupTrust(serverCert *shared.CertInfo, clusterPut api.ClusterPut) error
- func UpdateTrust(serverCert *shared.CertInfo, serverName string, targetAddress string, ...) error
- func UpgradeMembersWithoutRole(gateway *Gateway, members []db.NodeInfo) error
- type APIHeartbeat
- type APIHeartbeatMember
- type APIHeartbeatVersion
- type EventMode
- type Gateway
- func (g *Gateway) Context() context.Context
- func (g *Gateway) DemoteOfflineNode(raftID uint64) error
- func (g *Gateway) DialFunc() client.DialFunc
- func (g *Gateway) HandlerFuncs(heartbeatHandler HeartbeatHandler, identityCache *identity.Cache) map[string]http.HandlerFunc
- func (g *Gateway) HearbeatCancelFunc() func()
- func (g *Gateway) HeartbeatRestart() bool
- func (g *Gateway) IsDqliteNode() bool
- func (g *Gateway) Kill()
- func (g *Gateway) LeaderAddress() (string, error)
- func (g *Gateway) NetworkUpdateCert(cert *shared.CertInfo)
- func (g *Gateway) NodeStore() client.NodeStore
- func (g *Gateway) Reset(networkCert *shared.CertInfo) error
- func (g *Gateway) Shutdown() error
- func (g *Gateway) Sync()
- func (g *Gateway) TransferLeadership() error
- func (g *Gateway) WaitLeadership() error
- func (g *Gateway) WaitUpgradeNotification()
- type HeartbeatHandler
- type HeartbeatHook
- type Notifier
- type NotifierPolicy
- type Option
Constants ¶
const RecoveryTarballName = "lxd_recovery_db.tar.gz"
RecoveryTarballName is the filename used for recovery tarballs.
Variables ¶
var ErrNodeIsNotClustered = fmt.Errorf("Server is not clustered")
ErrNodeIsNotClustered indicates the node is not clustered.
var ErrNotLeader = fmt.Errorf("Not leader")
ErrNotLeader signals that a node not the leader.
var SchemaVersion = cluster.SchemaVersion
SchemaVersion holds the version of the cluster database schema.
Functions ¶
func Accept ¶
func Accept(state *state.State, gateway *Gateway, name, address string, schema, api, arch int) ([]db.RaftNode, error)
Accept a new node and add it to the cluster.
This instance must already be clustered.
Return an updated list raft database nodes (possibly including the newly accepted node).
func Bootstrap ¶
Bootstrap turns a non-clustered LXD instance into the first (and leader) node of a new LXD cluster.
This instance must already have its cluster.https_address set and be listening on the associated network address.
func ClusterState ¶
func ClusterState(s *state.State, networkCert *shared.CertInfo, members ...db.NodeInfo) (map[string]api.ClusterMemberState, error)
ClusterState returns a map from clusterMemberName -> state for every member of the cluster. This requires an HTTP call to the rest of the cluster.
func Connect ¶
func Connect(address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo, r *http.Request, notify bool) (lxd.InstanceServer, error)
Connect is a convenience around lxd.ConnectLXD that configures the client with the correct parameters for node-to-node communication.
If 'notify' switch is true, then the user agent will be set to the special to the UserAgentNotifier value, which can be used in some cases to distinguish between a regular client request and an internal cluster request.
func ConnectIfInstanceIsRemote ¶
func ConnectIfInstanceIsRemote(s *state.State, projectName string, instName string, r *http.Request, instanceType instancetype.Type) (lxd.InstanceServer, error)
ConnectIfInstanceIsRemote figures out the address of the cluster member which is running the instance with the given name in the specified project. If it's not the local member will connect to it and return the connected client (configured with the specified project), otherwise it will just return nil.
func DatabaseReplaceFromTarball ¶
DatabaseReplaceFromTarball unpacks the tarball found at `tarballPath`, replaces the global database, updates the local database with any changed addresses, and writes a global patch file to update the global database with any changed addresses.
func EnsureServerCertificateTrusted ¶
func EnsureServerCertificateTrusted(serverName string, serverCert *shared.CertInfo, tx *db.ClusterTx) error
EnsureServerCertificateTrusted adds the serverCert to the DB trusted certificates store using the serverName. If a certificate with the same fingerprint is already in the trust store, but is of the wrong type or name then the existing certificate is updated to the correct type and name. If the existing certificate is the correct type but the wrong name then an error is returned. And if the existing certificate is the correct type and name then nothing more is done.
func EventHubPush ¶
EventHubPush pushes the event to the event hub members if local server is an event-hub client.
func EventListenerWait ¶
EventListenerWait waits for there to be listener connected to the specified address, or one of the event hubs if operating in event hub mode.
func EventsUpdateListeners ¶
func EventsUpdateListeners(endpoints *endpoints.Endpoints, cluster *db.Cluster, serverCert func() *shared.CertInfo, hbMembers map[int64]APIHeartbeatMember, inject events.InjectFunc)
EventsUpdateListeners refreshes the cluster event listener connections.
func Handover ¶
Handover looks for a non-voter member that can be promoted to replace a the member with the given address, which is shutting down. It returns the address of such member along with an updated list of nodes, with the ne role set.
It should be called only by the current leader.
func HasConnectivity ¶
func HasConnectivity(networkCert *shared.CertInfo, serverCert *shared.CertInfo, address string) bool
HasConnectivity probes the member with the given address for connectivity.
func HeartbeatNode ¶
func HeartbeatNode(taskCtx context.Context, address string, networkCert *shared.CertInfo, serverCert *shared.CertInfo, heartbeatData *APIHeartbeat) error
HeartbeatNode performs a single heartbeat request against the node with the given address.
func HeartbeatTask ¶
HeartbeatTask returns a task function that performs leader-initiated heartbeat checks against all LXD nodes in the cluster.
It will update the heartbeat timestamp column of the nodes table accordingly, and also notify them of the current list of database nodes.
func Join ¶
func Join(state *state.State, gateway *Gateway, networkCert *shared.CertInfo, serverCert *shared.CertInfo, name string, raftNodes []db.RaftNode) error
Join makes a non-clustered LXD node join an existing cluster.
It's assumed that Accept() was previously called against the leader node, which handed the raft server ID.
The cert parameter must contain the keypair/CA material of the cluster being joined.
func Leave ¶
Leave a cluster.
If the force flag is true, the node will leave even if it still has containers and images.
The node will only leave the raft cluster, and won't be removed from the database. That's done by Purge().
Upon success, return the address of the leaving node.
This function must be called by the cluster leader.
func ListDatabaseNodes ¶
ListDatabaseNodes returns a list of database node names.
func LocalSysInfo ¶
func LocalSysInfo() (*api.ClusterMemberSysInfo, error)
LocalSysInfo retrieves system information about a cluster member.
func MaybeUpdate ¶
MaybeUpdate Check this node's version and possibly run LXD_CLUSTER_UPDATE.
func MemberState ¶
MemberState retrieves state information about the cluster member.
func NotifyHeartbeat ¶
NotifyHeartbeat attempts to send a heartbeat to all other members to notify them of a new or changed member.
func NotifyUpgradeCompleted ¶
func NotifyUpgradeCompleted(state *state.State, networkCert *shared.CertInfo, serverCert *shared.CertInfo) error
NotifyUpgradeCompleted sends a notification to all other nodes in the cluster that any possible pending database update has been applied, and any nodes which was waiting for this node to be upgraded should re-check if it's okay to move forward.
func Rebalance ¶
func Rebalance(state *state.State, gateway *Gateway, unavailableMembers []string) (string, []db.RaftNode, error)
Rebalance the raft cluster, trying to see if we have a spare online node that we can promote to voter node if we are below membershipMaxRaftVoters, or to standby if we are below membershipMaxStandBys.
If there's such spare node, return its address as well as the new list of raft nodes.
func Reconfigure ¶
Reconfigure replaces the entire cluster configuration. Addresses and node roles may be updated. Node IDs are read-only. Returns the path to the new database state (recovery tarball).
func Recover ¶
Recover rebuilds the dqlite raft configuration leaving only the current member in the cluster. Use `Reconfigure` if more members should remain in the raft configuration.
func RemoveRaftNode ¶
RemoveRaftNode removes a raft node from the raft configuration.
func ResolveTarget ¶
ResolveTarget is a convenience for resolving a target member name to address. It returns the address of the given member, or the empty string if the given member is the local one.
func RoleInSlice ¶
func RoleInSlice(role db.ClusterRole, roles []db.ClusterRole) bool
RoleInSlice returns whether or not the rule is within the roles list.
func SetupTrust ¶
func SetupTrust(serverCert *shared.CertInfo, clusterPut api.ClusterPut) error
SetupTrust is a convenience around InstanceServer.CreateCertificate that adds the given server certificate to the trusted pool of the cluster at the given address, using the given token. The certificate is added as type CertificateTypeServer to allow intra-member communication. If a certificate with the same fingerprint already exists with a different name or type, then no error is returned.
func UpdateTrust ¶
func UpdateTrust(serverCert *shared.CertInfo, serverName string, targetAddress string, targetCert string) error
UpdateTrust ensures that the supplied certificate is stored in the target trust store with the correct name and type to ensure correct cluster operation. Should be called after SetupTrust. If a certificate with the same fingerprint is already in the trust store, but is of the wrong type or name then the existing certificate is updated to the correct type and name. If the existing certificate is the correct type but the wrong name then an error is returned. And if the existing certificate is the correct type and name then nothing more is done.
func UpgradeMembersWithoutRole ¶
UpgradeMembersWithoutRole assigns the Spare raft role to all cluster members that are not currently part of the raft configuration. It's used for upgrading a cluster from a version without roles support.
Types ¶
type APIHeartbeat ¶
type APIHeartbeat struct { sync.Mutex // Used to control access to Members maps. Members map[int64]APIHeartbeatMember Version APIHeartbeatVersion Time time.Time // Indicates if heartbeat contains a fresh set of node states. // This can be used to indicate to the receiving node that the state is fresh enough to // trigger node refresh activies (such as forkdns). FullStateList bool // contains filtered or unexported fields }
APIHeartbeat contains data sent to nodes in heartbeat.
func NewAPIHearbeat ¶
func NewAPIHearbeat(cluster *db.Cluster) *APIHeartbeat
NewAPIHearbeat returns initialised APIHeartbeat.
func (*APIHeartbeat) Send ¶
func (hbState *APIHeartbeat) Send(ctx context.Context, networkCert *shared.CertInfo, serverCert *shared.CertInfo, localAddress string, nodes []db.NodeInfo, spreadDuration time.Duration)
Send sends heartbeat requests to the nodes supplied and updates heartbeat state.
func (*APIHeartbeat) Update ¶
func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration)
Update updates an existing APIHeartbeat struct with the raft and all node states supplied. If allNodes provided is an empty set then this is considered a non-full state list.
type APIHeartbeatMember ¶
type APIHeartbeatMember struct { ID int64 // ID field value in nodes table. Address string // Host and Port of node. Name string // Name of cluster member. RaftID uint64 // ID field value in raft_nodes table, zero if non-raft node. RaftRole int // Node role in the raft cluster, from the raft_nodes table LastHeartbeat time.Time // Last time we received a successful response from node. Online bool // Calculated from offline threshold and LastHeatbeat time. Roles []db.ClusterRole // Supplementary non-database roles the member has. // contains filtered or unexported fields }
APIHeartbeatMember contains specific cluster node info.
type APIHeartbeatVersion ¶
APIHeartbeatVersion contains max versions for all nodes in cluster.
type EventMode ¶
type EventMode string
EventMode indicates the event distribution mode.
const EventModeFullMesh EventMode = "full-mesh"
EventModeFullMesh is when every cluster member connects to every other cluster member to pull events.
const EventModeHubClient EventMode = "hub-client"
EventModeHubClient is when the cluster is operating in event-hub mode and this member is designated as a hub client, meaning that it is expected to connect to the event-hub members.
const EventModeHubServer EventMode = "hub-server"
EventModeHubServer is when the cluster is operating in event-hub mode and this server is designated as a hub server, meaning that it will only connect to the other event-hub members and not other members.
func ServerEventMode ¶
func ServerEventMode() EventMode
ServerEventMode returns the event distribution mode that this local server is operating in.
type Gateway ¶
type Gateway struct { // Used for the heartbeat handler Cluster *db.Cluster HeartbeatNodeHook HeartbeatHook HeartbeatOfflineThreshold time.Duration HeartbeatLock sync.Mutex // contains filtered or unexported fields }
Gateway mediates access to the dqlite cluster using a gRPC SQL client, and possibly runs a dqlite replica on this LXD node (if we're configured to do so).
func NewGateway ¶
func NewGateway(shutdownCtx context.Context, db *db.Node, networkCert *shared.CertInfo, stateFunc func() *state.State, options ...Option) (*Gateway, error)
NewGateway creates a new Gateway for managing access to the dqlite cluster.
When a new gateway is created, the node-level database is queried to check what kind of role this node plays and if it's exposed over the network. It will initialize internal data structures accordingly, for example starting a local dqlite server if this node is a database node.
After creation, the Daemon is expected to expose whatever http handlers the HandlerFuncs method returns and to access the dqlite cluster using the dialer returned by the DialFunc method.
func (*Gateway) Context ¶
Context returns a cancellation context to pass to dqlite.NewDriver as option.
This context gets cancelled by Gateway.Kill() and at that point any connection failure won't be retried.
func (*Gateway) DemoteOfflineNode ¶
DemoteOfflineNode force demoting an offline node.
func (*Gateway) DialFunc ¶
DialFunc returns a dial function that can be used to connect to one of the dqlite nodes.
func (*Gateway) HandlerFuncs ¶
func (g *Gateway) HandlerFuncs(heartbeatHandler HeartbeatHandler, identityCache *identity.Cache) map[string]http.HandlerFunc
HandlerFuncs returns the HTTP handlers that should be added to the REST API endpoint in order to handle database-related requests.
There are two handlers, one for the /internal/raft endpoint and the other for /internal/db, which handle respectively raft and gRPC-SQL requests.
These handlers might return 404, either because this LXD node is a non-clustered node not available over the network or because it is not a database node part of the dqlite cluster.
func (*Gateway) HearbeatCancelFunc ¶
func (g *Gateway) HearbeatCancelFunc() func()
HearbeatCancelFunc returns the function that can be used to cancel an ongoing heartbeat. Returns nil if no ongoing heartbeat.
func (*Gateway) HeartbeatRestart ¶
HeartbeatRestart restarts cancels any ongoing heartbeat and restarts it. If there is no ongoing heartbeat then this is a no-op. Returns true if new heartbeat round was started.
func (*Gateway) IsDqliteNode ¶
IsDqliteNode returns true if this gateway is running a dqlite node.
func (*Gateway) Kill ¶
func (g *Gateway) Kill()
Kill is an API that the daemon calls before it actually shuts down and calls Shutdown(). It will abort any ongoing or new attempt to establish a SQL gRPC connection with the dialer (typically for running some pre-shutdown queries).
func (*Gateway) LeaderAddress ¶
LeaderAddress returns the address of the current raft leader.
func (*Gateway) NetworkUpdateCert ¶
NetworkUpdateCert sets a new network certificate for the gateway Use with Endpoints.NetworkUpdateCert() to fully update the API endpoint.
func (*Gateway) NodeStore ¶
NodeStore returns a dqlite server store that can be used to lookup the addresses of known database nodes.
func (*Gateway) Reset ¶
Reset the gateway, shutting it down.
This is used when disabling clustering on a node.
func (*Gateway) Shutdown ¶
Shutdown this gateway, stopping the gRPC server and possibly the raft factory.
func (*Gateway) Sync ¶
func (g *Gateway) Sync()
Sync dumps the content of the database to disk. This is useful for inspection purposes, and it's also needed by the activateifneeded command so it can inspect the database in order to decide whether to activate the daemon or not.
func (*Gateway) TransferLeadership ¶
TransferLeadership attempts to transfer leadership to another node.
func (*Gateway) WaitLeadership ¶
WaitLeadership waits for the raft node to become leader.
func (*Gateway) WaitUpgradeNotification ¶
func (g *Gateway) WaitUpgradeNotification()
WaitUpgradeNotification waits for a notification from another node that all nodes in the cluster should now have been upgraded and have matching schema and API versions.
type HeartbeatHandler ¶
type HeartbeatHandler func(w http.ResponseWriter, r *http.Request, isLeader bool, hbData *APIHeartbeat)
HeartbeatHandler represents a function that can be called when a heartbeat request arrives.
type HeartbeatHook ¶
type HeartbeatHook func(heartbeatData *APIHeartbeat, isLeader bool, unavailableMembers []string)
HeartbeatHook represents a function that can be called as the heartbeat hook.
type Notifier ¶
Notifier is a function that invokes `hook` against each node in the cluster, excluding the invoking one. The NodeInfo passed to `hook` describes the cluster member of InstanceServer.
type NotifierPolicy ¶
type NotifierPolicy int
NotifierPolicy can be used to tweak the behavior of NewNotifier in case of some nodes are down.
const ( NotifyAll NotifierPolicy = iota // Requires that all nodes are up. NotifyAlive // Only notifies nodes that are alive NotifyTryAll // Attempt to notify all nodes regardless of state. )
Possible notifcation policies.
type Option ¶
type Option func(*options)
Option to be passed to NewGateway to customize the resulting instance.
func Latency ¶
Latency is a coarse grain measure of how fast/reliable network links are. This is used to tweak the various timeouts parameters of the raft algorithm. See the raft.Config structure for more details. A value of 1.0 means use the default values from hashicorp's raft package. Values closer to 0 reduce the values of the various timeouts (useful when running unit tests in-memory).