rules

Documentation

Index

• Variables

Variables

var IngressNeedsAnnotation = engine.NewRule(
	func(old runtime.Object, new runtime.Object, ctx *engine.RuleHandlerContext) {
		ingress := new.(*networkingv1.Ingress)
		logger := log.WithFields(log.Fields{"name": ingress.Name, "namespace": ingress.Namespace, "rule": "IngressNeedsAnnotation"})
		annotation := ingress.GetAnnotations()
		hasAnnotation := false
		for key := range annotation {
			if strings.HasPrefix(key, prefix) {
				logger.Debugf("Checking annotation %s", key)
				hasAnnotation = true
				break
			}
		}
		if !hasAnnotation {
			ctx.Alertf(new, "You don't have any alerts set up for your ingress: %s.%s. You may want to check https://github.com/uswitch/heimdall for more info.", ingress.Namespace, ingress.Name)
		}
	}, engine.WantIngress)

var RequireCronJobHistoryLimits = engine.NewRule(
	func(old runtime.Object, new runtime.Object, ctx *engine.RuleHandlerContext) {
		job := new.(*batchv1.CronJob)
		logger := log.WithFields(log.Fields{"rule": "RequireCronJobHistoryLimits", "namespace": job.GetNamespace(), "name": job.GetName()})

		logger.Debugf("checking for history limit requirement")

		messages := make([]string, 0)
		if job.Spec.SuccessfulJobsHistoryLimit == nil {
			message := fmt.Sprintf("CronJob `%s/%s` doesn't specify `.spec.successfulJobsHistoryLimit`. Must be 10 or under.", job.GetNamespace(), job.GetName())
			messages = append(messages, message)
		} else {
			if *job.Spec.SuccessfulJobsHistoryLimit > 10 {
				message := fmt.Sprintf("CronJob `%s/%s` `.spec.succcessfulJobsHistoryLimit` is too high: `%d`. Must be 10 or under.", job.GetNamespace(), job.GetName(), *job.Spec.SuccessfulJobsHistoryLimit)
				messages = append(messages, message)
			}
		}

		if job.Spec.FailedJobsHistoryLimit == nil {
			message := fmt.Sprintf("CronJob `%s/%s` doesn't specify `.spec.failedJobsHistoryLimit`. Must be 10 or under.", job.GetNamespace(), job.GetName())
			messages = append(messages, message)
		} else {
			if *job.Spec.FailedJobsHistoryLimit > 10 {
				message := fmt.Sprintf("CronJob `%s/%s` `.spec.failedJobsHistoryLimit` is too high: `%d`. Must be 10 or under.", job.GetNamespace(), job.GetName(), *job.Spec.FailedJobsHistoryLimit)
				messages = append(messages, message)
			}
		}

		for _, msg := range messages {
			ctx.Alert(job, msg)
		}
	},
	engine.WantCronJobs,
)

var ResourceAnnotationRule = engine.NewRule(
	func(old runtime.Object, new runtime.Object, ctx *engine.RuleHandlerContext) {
		deployment := new.(*appsv1.Deployment)
		logger := log.WithFields(log.Fields{"rule": "ResourceAnnotationRule", "name": deployment.Name, "namespace": deployment.Namespace})

		newInViolation := containersInViolation(deployment)

		if old == nil || !reflect.DeepEqual(containersInViolation(old.(*appsv1.Deployment)), newInViolation) {
			if len(newInViolation) == 0 {
				if old != nil {
					ctx.Alertf(new, "Thanks for sorting your resource requests and limits on %s.%s!", deployment.ObjectMeta.Namespace, podNameForDeployment(deployment))
				}
			} else {
				ctx.Alertf(new, "Please add resource requests and limits to the containers (%s) part of %s.%s", strings.Join(newInViolation, ", "), deployment.ObjectMeta.Namespace, podNameForDeployment(deployment))
			}
		} else {
			logger.Debugf("ResourceAnnotationRule: %s.%s hadn't changed", deployment.ObjectMeta.Namespace, podNameForDeployment(deployment))
		}
	},
	engine.WantDeployments,
)

var ScrapeNeedsPortsRule = engine.NewRule(
	func(old runtime.Object, new runtime.Object, ctx *engine.RuleHandlerContext) {
		deployment := new.(*appsv1.Deployment)
		logger := log.WithFields(log.Fields{"name": deployment.Name, "namespace": deployment.Namespace, "rule": "ScrapeNeedsPortsRule"})

		if old == nil || validScrapeAndPorts(old.(*appsv1.Deployment)) != validScrapeAndPorts(deployment) {
			podName := podNameForDeployment(deployment)

			if validScrapeAndPorts(deployment) {
				if old != nil {
					ctx.Alertf(new, "Thanks for sorting the ports for scraping on %s.%s", deployment.ObjectMeta.Namespace, podName)
				}
			} else {
				ctx.Alertf(new, "%s.%s wants to be scraped so it needs to expose some ports", deployment.ObjectMeta.Namespace, podName)
			}
		} else {
			logger.Debugf("ScrapeNeedsPortsRule: %s.%s hadn't changed", deployment.ObjectMeta.Namespace, podNameForDeployment(deployment))
		}
	},
	engine.WantDeployments,
)

var UnsuccessfulExitRule = engine.NewRule(
	func(old runtime.Object, newObj runtime.Object, ctx *engine.RuleHandlerContext) {
		pod := newObj.(*v1.Pod)

		logger := log.WithFields(log.Fields{"name": pod.Name, "namespace": pod.Namespace, "rule": "UnsuccessfulExitRule"})

		for _, c := range pod.Status.ContainerStatuses {
			logger = logger.WithFields(log.Fields{"container.name": c.Name, "container.id": c.ContainerID})
			contx := context.Background()
			if c.State.Terminated != nil {
				switch exitCode := c.State.Terminated.ExitCode; exitCode {
				case 0:
					break
				case 143:
					break
				case 137:
					if c.State.Terminated.Reason == "OOMKilled" {
						ctx.Alertf(newObj, "Pod `%s.%s` (container: `%s`) ran out of memory and was killed.", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, c.Name)
					} else {
						ctx.Alertf(newObj, "Pod `%s.%s` (container: `%s`) was killed by a SIGKILL. Please make sure you gracefully shut down in time or extend `terminationGracePeriodSeconds` on your pod.", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, c.Name)
					}
				default:
					tailLines := int64(20)
					opts := &v1.PodLogOptions{
						Container: c.Name,
						Follow:    false,
						Previous:  false,
						TailLines: &tailLines,
					}
					message := fmt.Sprintf("Pod `%s.%s` (container: `%s`) has failed with exit code: `%d`", pod.ObjectMeta.Namespace, pod.ObjectMeta.Name, c.Name, c.State.Terminated.ExitCode)

					result := ctx.Client().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, opts).Do(contx)
					if result.Error() != nil {
						logger.Errorf("error retrieving pod logs: %s", result.Error())
						ctx.Alert(newObj, message)
						return
					}

					bytes, err := result.Raw()
					if err != nil {
						logger.Errorf("error retrieving pod logs: %s", err.Error())
						ctx.Alert(newObj, message)
						return
					}

					logger.Debugf("log: \"%s\"", string(bytes))
					ctx.Alertf(newObj, "%s\n\n```%s```", message, string(bytes))
				}
			}
		}
	},
	engine.WantPods,
)