scalehist

README

scalehist

Kubernetes Scaling History Recorder and Analyzer

TODO

• pod_details table:

• when we get FailedScheduling event, we get the Pod name from regardingName / regardingUid, get the pod details and save it into the pod_details table

• when we get a Scheduled event, we should do the same. If the Pod already exists and the resource version is unchanged, we can skip...

• columns: name, uid, spec (jsonified),

• machine_deployments table:

• when we detect a TriggeredScaleUp event, we get

2 Node groups: A, B PODS: X1A, X2A, Y1A, Y2B

Check out

//t turns out that we could already modify the sharedInformer regardless of my patch, e.g.
sharedInformer.InformerFor(&corev1.Pod{}, func(cli kubernetes.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
// create and return custom informer
})
sharedInformer.Start(stopCh

Info Required from Seed

• Need the CA deployment
• Need the MCD object

NodeGroups Ordering

• MCD-NG does not have min and max replicas count.
• Keep a MCD table to store mcd objects.
• Populate nodegroup_history with CA-NG.

Scenario Report

• Represent a scale event for the nodegroup

Info required from nodes

• .status.allocatable.pods
• .status.capacity.pods

Analyzer

clusters.csv

landscape_name, shoot-namespace,shoot-yaml,seed-yaml

TODO

• TSC Case for PendingUnscheduledPods //After sceanario over //Unsch : p5 //Nodes: {n1,n2}

//{p6} -> {n3} -> trigger //{p5 -> n1}

Case1 : p6 -> n3 p5 -> n1 //assigned to TSC satisfied after p6

Case2 : p6 -> n3 p5 -> n1 //scheduled event emitted after new workload deployed

• Put triggerScalingEvents[] in a map with time

Documentation

Index

• Constants
• func CmpNodeInfoDescending(a, b NodeInfo) int
• func ContainsPod(podUID string, podInfos []PodInfo) bool
• func CumulatePodRequests(pod *corev1.Pod) corev1.ResourceList
• func IsEqualNodeInfo(a, b NodeInfo) bool
• func IsEqualPodInfo(a, b PodInfo) bool
• func IsEqualQuantity(a, b resource.Quantity) bool
• func IsEqualTaint(a, b corev1.Taint) bool
• func IsResourceListEqual(r1 corev1.ResourceList, r2 corev1.ResourceList) bool
• func ResourcesAsString(resources corev1.ResourceList) string
• func SumResources(resources []corev1.ResourceList) corev1.ResourceList
• type Analysis
• type Analyzer
• type CASettingsInfo
  • func (ca CASettingsInfo) GetHash() string
• type EventCASettingsAssoc
• type EventInfo
  • func (eI EventInfo) String() string
• type EventNodeGroupAssoc
  • func (a EventNodeGroupAssoc) String() string
• type NodeGroupInfo
  • func (ng NodeGroupInfo) GetHash() string
  • func (ng NodeGroupInfo) String() string
• type NodeInfo
  • func (n NodeInfo) GetHash() string
  • func (n NodeInfo) String() string
• type PodInfo
  • func (p PodInfo) GetHash() string
  • func (p PodInfo) String() string
• type PodInfoKey
• type PodScheduleStatus
• type Recorder
• type RecorderParams
  • func (rp RecorderParams) String() string
• type Reporter
• type ReporterParams
• type Scenario
• type WorkerPool
  • func (w WorkerPool) String() string

Constants

const PodScheduleCommited = 1

const PodScheduleNominated = -1

const PodSchedulePending = -2

const PodUnscheduled = 0

Functions

func CmpNodeInfoDescending

func CmpNodeInfoDescending(a, b NodeInfo) int

func ContainsPod

func ContainsPod(podUID string, podInfos []PodInfo) bool

func CumulatePodRequests

func CumulatePodRequests(pod *corev1.Pod) corev1.ResourceList

func IsEqualNodeInfo

func IsEqualNodeInfo(a, b NodeInfo) bool

func IsEqualPodInfo

func IsEqualPodInfo(a, b PodInfo) bool

func IsEqualQuantity

func IsEqualQuantity(a, b resource.Quantity) bool

func IsEqualTaint

func IsEqualTaint(a, b corev1.Taint) bool

func IsResourceListEqual

func IsResourceListEqual(r1 corev1.ResourceList, r2 corev1.ResourceList) bool

func ResourcesAsString

func ResourcesAsString(resources corev1.ResourceList) string

func SumResources

func SumResources(resources []corev1.ResourceList) corev1.ResourceList

Types

type Analysis

type Analysis struct {
	Name               string
	CoalesceInterval   time.Duration
	TolerationInterval time.Duration
	Scenarios          []Scenario
}

type Analyzer

type Analyzer interface {
	io.Closer
	Analyze(ctx context.Context) (Analysis, error)
}

type CASettingsInfo

type CASettingsInfo struct {
	Expander      string
	MaxNodesTotal int `db:"MaxNodesTotal"`
	// Priorities is the value of the `priorities` key in the `cluster-autoscaler-priority-expander` config map.
	// See https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/expander/priority/readme.md#configuration
	Priorities string
	Hash       string //primary key
}

func (CASettingsInfo) GetHash

func (ca CASettingsInfo) GetHash() string

type EventCASettingsAssoc

type EventCASettingsAssoc struct {
	EventUID       string `db:"EventUID"`
	CASettingsHash string `db:"CASettingsHash"`
}

type EventInfo

type EventInfo struct {
	UID                     string    `db:"UID"`
	EventTime               time.Time `db:"EventTime"`
	ReportingController     string    `db:"ReportingController"`
	Reason                  string    `db:"Reason"`
	Message                 string    `db:"Message"`
	InvolvedObjectKind      string    `db:"InvolvedObjectKind"`
	InvolvedObjectName      string    `db:"InvolvedObjectName"`
	InvolvedObjectNamespace string    `db:"InvolvedObjectNamespace"`
	InvolvedObjectUID       string    `db:"InvolvedObjectUID"`
}

func (EventInfo) String

func (eI EventInfo) String() string

type EventNodeGroupAssoc

type EventNodeGroupAssoc struct {
	EventUID       string `db:"EventUID"`
	NodeGroupRowID int64  `db:"NodeGroupRowID"`
	NodeGroupHash  string `db:"NodeGroupHash"`
}

func (EventNodeGroupAssoc) String

func (a EventNodeGroupAssoc) String() string

type NodeGroupInfo

type NodeGroupInfo struct {
	RowID             int64 `db:"RowID"` // give db tags only for mixed case fields
	Name              string
	CreationTimestamp time.Time `db:"CreationTimestamp"`
	CurrentSize       int       `db:"CurrentSize"`
	TargetSize        int       `db:"TargetSize"`
	MinSize           int       `db:"MinSize"`
	MaxSize           int       `db:"MaxSize"`
	Zone              string
	MachineType       string `db:"MachineType"`
	Architecture      string
	ShootGeneration   int64  `db:"ShootGeneration"`
	MCDGeneration     int64  `db:"MCDGeneration"`
	PoolName          string `db:"PoolName"`
	PoolMin           int    `db:"PoolMin"`
	PoolMax           int    `db:"PoolMax"`
	Hash              string
}

func (NodeGroupInfo) GetHash

func (ng NodeGroupInfo) GetHash() string

func (NodeGroupInfo) String

func (ng NodeGroupInfo) String() string

type NodeInfo

type NodeInfo struct {
	Name               string
	Namespace          string
	CreationTimestamp  time.Time
	ProviderID         string
	AllocatableVolumes int
	Labels             map[string]string
	Taints             []corev1.Taint
	Allocatable        corev1.ResourceList
	Capacity           corev1.ResourceList
	Hash               string
}

NodeInfo represents snapshot information captured about an active k8s Node in the cluster at a particular moment in time. The snapshot time is captured in CreationTimestamp. A NodeInfo snapshot is only captured if there is a change in the properties excepting for DeletionTimestamp, in which case the DeletionTimestamp is only updated.

func (NodeInfo) GetHash

func (n NodeInfo) GetHash() string

func (NodeInfo) String

func (n NodeInfo) String() string

type PodInfo

type PodInfo struct {
	Name              string
	Namespace         string
	UID               string
	CreationTimestamp time.Time
	NodeName          string
	NominatedNodeName string
	Labels            map[string]string
	Requests          corev1.ResourceList
	Spec              corev1.PodSpec
	PodScheduleStatus PodScheduleStatus
	Hash              string
}

func (PodInfo) GetHash

func (p PodInfo) GetHash() string

func (PodInfo) String

func (p PodInfo) String() string

type PodInfoKey

type PodInfoKey struct {
	UID  string
	Name string
	Hash string
}

type PodScheduleStatus

type PodScheduleStatus int

type Recorder

type Recorder interface {
	io.Closer
	Start(ctx context.Context) error
}

Recorder monitors the cluster denoted by given kubeconfig and records events and cluster data into cluster database

type RecorderParams

type RecorderParams struct {
	ShootKubeConfigPath string
	ShootNameSpace      string
	SeedKubeConfigPath  string
	DBDir               string
	SchedulerName       string
}

func (RecorderParams) String

func (rp RecorderParams) String() string

type Reporter

type Reporter interface {
	GenerateTextReport(analysis Analysis) (reportPath string, err error)
	GenerateJsonReport(analysis Analysis) (reportPath string, err error)
}

type ReporterParams

type ReporterParams struct {
	DBDir     string
	ReportDir string
}

type Scenario

type Scenario struct {
	StartTime                 time.Time
	EndTime                   time.Time
	SystemComponentRequests   corev1.ResourceList
	CriticalComponentRequests corev1.ResourceList
	CASettings                CASettingsInfo
	UnscheduledPods           []PodInfo
	NominatedPods             []PodInfo
	ScheduledPods             []PodInfo
	NodeGroups                []NodeGroupInfo
	ScaleUpEvents             []EventInfo
	Nodes                     []NodeInfo
}

type WorkerPool

type WorkerPool struct {
	Name            string
	MachineType     string
	Architecture    string
	Minimum         int
	Maximum         int
	MaxSurge        intstr.IntOrString // TODO: persist as string if needed.
	MaxUnavailable  intstr.IntOrString // TODO: persist as string if needed.
	ShootGeneration int64
	Zones           []string
}

func (WorkerPool) String

func (w WorkerPool) String() string