kubescaler

README

Kubescaler

The kubescaler is an app to scale k8s cluster node pool to run a bunch of stateful dedicated servers (like dedicated game server using memory state) using kubernetes API inspired by Mark Mandel blog post . This application is a simple scaler and you can use Agones for more advanced features.

Scale Strategy

The scaler follows a simple rule to scale the nodes. There is a buffer size and if the resource request is more than the buffer size, needed new nodes are created and if the resource request decreases, the extra nodes are deleted.

The scaler runs the scale method in a loop. If the available resource is smaller than the buffer size, first check the unschedulable nodes and if the resource isn't enough, then resize the pool size to the needed nodes. Otherwise, if the available resource is greater than the buffer size, first calculate the extra nodes and then mark them as unschedulable to prevent scheduling new pods on them. at the end check the unschedulable nodes and deletes expired nodes with no dedicated server pods.

Permissions

kubescaler uses cluster config to manage nodes & pods, so permissions and roles must be applied to the kubescaler Deployment

Configs

The example config file exists as config.yaml.exmaple file. Also, you can set the configs as environment variables in uppercase and snail case format.

• cloud-provider : Currently only digitalocean is implemented as a cloud provider, but the app supports driver, so you can implement any other provider and only register it (or contribute to this repo and send a pull request). Cloud provider manages the node pool size and deletes extra nodes.
• cloud-provider-token : Access token for cloud API
• cluster-name : Cluster name
• node-pool-name : Node pool name
• node-selector : Kubernetes selector to target the dedicated servers nodes. (ex: "role=scalable"). This selector is used to find the nodes that host the dedicated server pods.
• cluster-kube-config-master-url : cluster kube config master URL to use k8s API (leave empty if using in-cluster config)
• cluster-kube-config-path : cluster kube config path (leave empty if using in-cluster config)
• minimum-node-pool-size : minimum node pool size
• maximum-node-pool-size : maximum node pool size
• server-pod-label-name : dedicated server pod label name (using to filter the dedicated server pods)
• server-pod-label-value : dedicated server pod label value (using to filter the dedicated server pods)
• buffer-slot-size : buffer slot size
• scale-loop-tick-sec : scale loop tick duration in seconds
• server-cpu-resource-request : dedicated server pod CPU resource request (in MilliValue)
• empty-node-expiration-sec : empty node expiration duration in seconds (delete node after this time if no pods scheduled)

TODOs

• Add kubernetes deployment
• Add health check

Documentation

Index

• Variables
• func ClientSet(masterURL, kubeConfigPath string) (kubernetes.Interface, error)
• func NewDefaultLogger(infoLogger *log.Logger, debugLogger *log.Logger, errorLogger *log.Logger) *defaultLogger
• func SortNodesByPods(nodes []*Node)
• func SortNodesByPodsDesc(nodes []*Node)
• type Config
• type K8S
  • func NewK8S(i kubernetes.Interface) *K8S
  • func NewK8SFromKubeConfig(masterURL, kubeConfigPath string) (*K8S, error)
  • func (k *K8S) NewPodWatcher(ctx context.Context, namespace, labelSelector string) (*PodWatcher, error)
  • func (k *K8S) NodePods(ctx context.Context, nodeName string) (*v1.PodList, error)
  • func (k *K8S) Nodes(ctx context.Context, selector string) (*NodeList, error)
  • func (k *K8S) UpdateNode(ctx context.Context, node *Node) error
• type Kubernetes
• type Logger
• type Node
  • func (n *Node) AvailableResource(resource v1.ResourceName) int64
  • func (n *Node) IsReady() bool
  • func (n *Node) IsSchedulable() bool
  • func (n *Node) MarkAsSchedulable() error
  • func (n *Node) MarkAsUnschedulable() error
  • func (n *Node) ResourceCapacity(resource v1.ResourceName) int64
  • func (n *Node) SchedulingMarkTimestamp() (time.Time, error)
  • func (n *Node) UsingResources(resource v1.ResourceName) int64
• type NodeList
  • func (n *NodeList) AvailableNodes() []*Node
  • func (n *NodeList) AvailableResource(resource v1.ResourceName) int64
  • func (n *NodeList) AvailableSlot(need Resource) int64
  • func (n *NodeList) SchedulableNodes() []*Node
  • func (n *NodeList) UnschedulableNodes() []*Node
• type PodWatcher
  • func (pw *PodWatcher) Stop()
  • func (pw *PodWatcher) Watch()
• type Resource
• type Scaler
  • func NewScaler(npm nodepoolmanager.Provider, k8s Kubernetes, config *Config) *Scaler
  • func (s *Scaler) Start() error
  • func (s *Scaler) Stop()

Variables

var (
	ErrNotEnoughResources = errors.New("not enough resources")
)

Functions

func ClientSet

func ClientSet(masterURL, kubeConfigPath string) (kubernetes.Interface, error)

func NewDefaultLogger

func NewDefaultLogger(infoLogger *log.Logger, debugLogger *log.Logger, errorLogger *log.Logger) *defaultLogger

func SortNodesByPods

func SortNodesByPods(nodes []*Node)

func SortNodesByPodsDesc

func SortNodesByPodsDesc(nodes []*Node)

Types

type Config

type Config struct {
	NodeSelector string

	MinimumNode int
	MaximumNode int

	PodCPURequest int64
	PodLabelName  string
	PodLabelValue string

	EmptyNodeExpiration time.Duration

	BufferSlotSize int64

	ScaleLoopDuration time.Duration

	Logger Logger
}

type K8S

type K8S struct {
	// contains filtered or unexported fields
}

func NewK8S

func NewK8S(i kubernetes.Interface) *K8S

func NewK8SFromKubeConfig

func NewK8SFromKubeConfig(masterURL, kubeConfigPath string) (*K8S, error)

func (*K8S) NewPodWatcher

func (k *K8S) NewPodWatcher(ctx context.Context, namespace, labelSelector string) (*PodWatcher, error)

func (*K8S) NodePods

func (k *K8S) NodePods(ctx context.Context, nodeName string) (*v1.PodList, error)

func (*K8S) Nodes

func (k *K8S) Nodes(ctx context.Context, selector string) (*NodeList, error)

func (*K8S) UpdateNode

func (k *K8S) UpdateNode(ctx context.Context, node *Node) error

type Kubernetes

type Kubernetes interface {
	Nodes(ctx context.Context, selector string) (*NodeList, error)
	UpdateNode(ctx context.Context, node *Node) error
	NewPodWatcher(ctx context.Context, namespace, labelSelector string) (*PodWatcher, error)
}

type Logger

type Logger interface {
	Info(v ...interface{})
	Infof(format string, v ...interface{})
	Debug(v ...interface{})
	Debugf(format string, v ...interface{})
	Error(v ...interface{})
	Errorf(format string, v ...interface{})
}

type Node

type Node struct {
	N    *v1.Node
	Pods []v1.Pod
}

func (*Node) AvailableResource

func (n *Node) AvailableResource(resource v1.ResourceName) int64

func (*Node) IsReady

func (n *Node) IsReady() bool

func (*Node) IsSchedulable

func (n *Node) IsSchedulable() bool

func (*Node) MarkAsSchedulable

func (n *Node) MarkAsSchedulable() error

func (*Node) MarkAsUnschedulable

func (n *Node) MarkAsUnschedulable() error

func (*Node) ResourceCapacity

func (n *Node) ResourceCapacity(resource v1.ResourceName) int64

func (*Node) SchedulingMarkTimestamp

func (n *Node) SchedulingMarkTimestamp() (time.Time, error)

func (*Node) UsingResources

func (n *Node) UsingResources(resource v1.ResourceName) int64

type NodeList

type NodeList struct {
	Nodes []*Node
}

func (*NodeList) AvailableNodes

func (n *NodeList) AvailableNodes() []*Node

func (*NodeList) AvailableResource

func (n *NodeList) AvailableResource(resource v1.ResourceName) int64

func (*NodeList) AvailableSlot

func (n *NodeList) AvailableSlot(need Resource) int64

func (*NodeList) SchedulableNodes

func (n *NodeList) SchedulableNodes() []*Node

func (*NodeList) UnschedulableNodes

func (n *NodeList) UnschedulableNodes() []*Node

type PodWatcher

type PodWatcher struct {
	Events chan watch.Event
	// contains filtered or unexported fields
}

func (*PodWatcher) Stop

func (pw *PodWatcher) Stop()

func (*PodWatcher) Watch

func (pw *PodWatcher) Watch()

type Resource

type Resource struct {
	Name  v1.ResourceName
	Value int64
}

type Scaler

type Scaler struct {
	// contains filtered or unexported fields
}

func NewScaler

func NewScaler(npm nodepoolmanager.Provider, k8s Kubernetes, config *Config) *Scaler

func (*Scaler) Start

func (s *Scaler) Start() error

func (*Scaler) Stop

func (s *Scaler) Stop()