gsh

README

gardener-scaling-history

[!NOTE] This is a prototype for PROOF OF CONCEPT only.

Setup

Please refer to the setup guide

Architecture

Design

Recorder

• The scaling-history-recorder monitors a list of clusters defined inside a config map scaling-history-recorder-config and setups up client-go Shared Informers on the data and control plane of those clusters monitoring Pod, Node, Events, MachineDeployments, MachineClasses and CA Deployment and ConfigMap.
• Recorded data is stored into a SQLite DB per cluster. Care is taken to minimize amount of data stored - skipping storage if no scaling relevant change is made

graph TB;
    sqlitedb[(ClusterDB)]
    recorder[<div style='text-align:left'>
    <h4 style='margin:0px;padding:0'>recorder</h4>
    Listens to event callbacks
    Persists to clusterDB
    ]
    shared_informers[<div style='text-align:left;margin-top:0;padding-top:0'>
    <h4 style='margin:0px;padding:0'>shared_informers</h4>
     Control Plane Informer for MCC, MCD, etc
     Data Plane Informers for Pods, Nodes, etc
    </div>]
    shared_informers-->recorder;
    recorder-->sqlitedb

Replayer

• The replayer can replay scenarios against either the VCA (Virtual Cluster Autoscaler) or the SR (scaling recommender)
• This is primarily meant for simulating autoscaling and hence ideally should be run against a virtual cluster like one setup by https://github.com/unmarshall/kvcl/

Virtual Cluster Autoscaler Replayer

• Virtual Cluster Autoscaler is a fork of the k8s cluster-autoscaler with a virtual provider implementation that scales virtual nodes on the virtual cluster

High level overview of the replay loop for the virtual CA replay run.

graph TB

subgraph vca["Virtual Cluster Autoscaler"]
CARunLoop
-->ScalingOrchestrator
-->IncreaseNodeGroupSize
-->ScalingDone
CloudProviderRefresh
-->ReadAutoScalerConfig
-->CreateVirtualNodes
end

BuildAutoScalerConfig-->ReadAutoScalerConfig
subgraph ca_replayer["CA Replayer Loop"]
sqlitedb-->GetScalingEvent
GetScalingEvent
-->GetClusterSnapshotAtScalingEventTime
-->BuildAutoScalerConfig
CARefreshed
-->DeployWorkload["<div>Deploy Pods, Namespaces</div>"]
-->WaitForVirtualScaling
-->DoneSignalReceived
-->CreateAndWriteScalingReport
-->RepeatLoop(((CALoopEnd)))
end


subgraph kvcl["Virtual Cluster"]
Pods
Nodes
end


IncreaseNodeGroupSize-->kvcl
CreateVirtualNodes-->kvcl
CreateVirtualNodes-->CARefreshed
ScalingDone-->DoneSignalReceived

Scaling Recommender Replayer

High level overview of the replay loop for the scaling recommender replay run

graph TB

subgraph vca["Scaling Recommender"]
ReceiveClusterSnapshot
-->GenerateScoresForGroupAndZone
-->FindWinner
-->GenerateScoresForGroupAndZone
-->UntilNoUnscheduledPods
-->ScaleVirtualNodes
-->SendRecommenderResponse


end

subgraph sr_replayer["SR Replayer Loop"]
ReadCAReport
-->LoadClusterSnapshotInCAReport
-->SynchronizeVirtualNodes
-->DeployWorkload["<div>Deploy Pods, Namespaces</div>"]
-->WaitForStabilizeInterval
-->PostClusterSnapshot
-->ReceiveRecommenderResponse
-->CreateAndWriteScalingReport
-->LoopEnd(((ReplayLoopEnd)))
end


subgraph kvcl["Virtual Cluster"]
Pods
Nodes
end

SendRecommenderResponse-->ReceiveRecommenderResponse
ScaleVirtualNodes-->kvcl
SynchronizeVirtualNodes-->kvcl
PostClusterSnapshot-->ReceiveClusterSnapshot

Deployment model

Documentation

Index

• Constants
• Variables
• func GardenerTimestampFromUnstructured(worker *unstructured.Unstructured) (gardenerTimeStamp time.Time, err error)
• func IsEqualQuantity(a, b resource.Quantity) bool
• func MachineDeploymentInfoFromUnstructured(mcd *unstructured.Unstructured, snapshotTime time.Time) (mcdInfo gsc.MachineDeploymentInfo, err error)
• func NodeInfoFromNode(n *corev1.Node, allocatableVolumes int) gsc.NodeInfo
• func ResourceListFromMap(input map[string]any) (corev1.ResourceList, error)
• func WorkerPoolInfosFromUnstructured(worker *unstructured.Unstructured) (map[string]gsc.WorkerPoolInfo, error)
• type ExecutionMode
• type FileInfo
• type FileInfos
• type MachineClassInfo
  • func MachineClassInfoFromUnstructured(mcc *unstructured.Unstructured, snapshotTime time.Time) (mccInfo MachineClassInfo, err error)
  • func (m MachineClassInfo) GetHash() string
  • func (m MachineClassInfo) String() string
• type PodInfoKey
• type PoolZone
  • func (pz PoolZone) String() string
• type Recorder
• type RecorderParams
  • func (rp RecorderParams) ShootLabel() string
  • func (rp RecorderParams) String() string
• type ReplayReport
• type Replayer
• type ReplayerParams
• type ResourceStats
• type ScalingResult
• type Scenario
  • func (report Scenario) GetResourceStat() (stats ResourceStats, err error)

Constants

const PoolLabel = "worker.gardener.cloud/pool"

const PoolLabelAlt = "worker_gardener_cloud_pool"

const PoolLabelAlt1 = "worker.gardener.cloud_pool"

Variables

var PoolLabels = []string{
	PoolLabel, PoolLabelAlt, PoolLabelAlt1,
}

Functions

func GardenerTimestampFromUnstructured

func GardenerTimestampFromUnstructured(worker *unstructured.Unstructured) (gardenerTimeStamp time.Time, err error)

func IsEqualQuantity

func IsEqualQuantity(a, b resource.Quantity) bool

func MachineDeploymentInfoFromUnstructured

func MachineDeploymentInfoFromUnstructured(mcd *unstructured.Unstructured, snapshotTime time.Time) (mcdInfo gsc.MachineDeploymentInfo, err error)

func NodeInfoFromNode

func NodeInfoFromNode(n *corev1.Node, allocatableVolumes int) gsc.NodeInfo

func ResourceListFromMap

func ResourceListFromMap(input map[string]any) (corev1.ResourceList, error)

func WorkerPoolInfosFromUnstructured

func WorkerPoolInfosFromUnstructured(worker *unstructured.Unstructured) (map[string]gsc.WorkerPoolInfo, error)

Types

type ExecutionMode

type ExecutionMode string

var (
	InUtilityClusterMode ExecutionMode = "in-utility-cluster"
	LocalMode            ExecutionMode = "local"
)

type FileInfo

type FileInfo struct {
	Name         string
	URL          string
	LastModified time.Time
	Size         uint64
	ReadableSize string
}

type FileInfos

type FileInfos struct {
	Items []FileInfo
}

type MachineClassInfo

type MachineClassInfo struct {
	gsc.SnapshotMeta

	// Instance type of the node belonging to nodeGroup
	InstanceType string

	// PoolName is the name of the gardener shoot worker pool that this machine class belongs to
	PoolName string

	// Region of the node belonging to nodeGroup
	Region string

	// Zone of the node that will be associated with this machine class
	Zone string

	// Labels is the machine class provider spec labels.
	Labels map[string]string

	// Capacity contains subfields to track all node resources required to scale nodegroup from zero
	Capacity corev1.ResourceList

	DeletionTimestamp time.Time
	Hash              string
}

func MachineClassInfoFromUnstructured

func MachineClassInfoFromUnstructured(mcc *unstructured.Unstructured, snapshotTime time.Time) (mccInfo MachineClassInfo, err error)

func (MachineClassInfo) GetHash

func (m MachineClassInfo) GetHash() string

func (MachineClassInfo) String

func (m MachineClassInfo) String() string

type PodInfoKey

type PodInfoKey struct {
	UID  string
	Name string
	Hash string
}

type PoolZone

type PoolZone struct {
	PoolName string
	Zone     string
}

func (PoolZone) String

func (pz PoolZone) String() string

type Recorder

type Recorder interface {
	io.Closer
	Start() error
	IsStarted() bool
}

Recorder monitors the cluster denoted by given kubeconfig and records events and cluster data into cluster database

type RecorderParams

type RecorderParams struct {
	Mode                ExecutionMode
	Landscape           string
	ProjectName         string
	ShootName           string
	SeedName            string
	ShootNameSpace      string
	ShootKubeConfigPath string
	SeedKubeConfigPath  string
	DBDir               string
	// contains filtered or unexported fields
}

func (RecorderParams) ShootLabel

func (rp RecorderParams) ShootLabel() string

func (RecorderParams) String

func (rp RecorderParams) String() string

type ReplayReport

type ReplayReport struct {
	StartTime time.Time
	Scenarios []Scenario
}

type Replayer

type Replayer interface {
	io.Closer
	Start() error
	GetRecordedClusterSnapshot(runMarkTime time.Time) (gsc.ClusterSnapshot, error)
	GetParams() ReplayerParams
}

type ReplayerParams

type ReplayerParams struct {
	Mode                         ExecutionMode
	InputDataPath                string
	ReportDir                    string
	VirtualAutoScalerConfigPath  string
	VirtualClusterKubeConfigPath string
	DeployParallel               int
	ReplayInterval               time.Duration
	AutoLaunchDependencies       bool
}

type ResourceStats

type ResourceStats struct {
	TotalUtilCPU  resource.Quantity
	TotalUtilMem  resource.Quantity
	AvailAllocCPU resource.Quantity
	AvailAllocMem resource.Quantity
}

type ScalingResult

type ScalingResult struct {
	ScaledUpNodeGroups     map[string]int
	ScaledUpNodes          []gsc.NodeInfo
	PendingUnscheduledPods []gsc.PodInfo
	NodesUtilization       map[string]corev1.ResourceList
	EmptyNodeNames         []string
	ScheduledPods          []gsc.PodInfo
}

type Scenario

type Scenario struct {
	BeginTime       time.Time
	ClusterSnapshot gsc.ClusterSnapshot
	ScalingResult   ScalingResult
}

func (Scenario) GetResourceStat

func (report Scenario) GetResourceStat() (stats ResourceStats, err error)