vnfs/DAaaS/prometheus-operator/templates/alertmanager/rules/kubernetes-apps.yaml

   1 # Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/coreos/prometheus-operator/master/contrib/kube-prometheus/manifests/prometheus-rules.yaml
   2 {{- if and .Values.defaultRules.create .Values.kubeStateMetrics.enabled }}
   3 apiVersion: {{ printf "%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }}
   4 kind: PrometheusRule
   5 metadata:
   6   name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }}
   7   labels:
   8     app: {{ template "prometheus-operator.name" . }}
   9 {{ include "prometheus-operator.labels" . | indent 4 }}
  10 {{- if .Values.defaultRules.labels }}
  11 {{ toYaml .Values.defaultRules.labels | indent 4 }}
  12 {{- end }}
  13 {{- if .Values.defaultRules.annotations }}
  14   annotations:
  15 {{ toYaml .Values.defaultRules.annotations | indent 4 }}
  16 {{- end }}
  17 spec:
  18   groups:
  19   - name: kubernetes-apps
  20     rules:
  21     - alert: KubePodCrashLooping
  22       annotations:
  23         message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} ({{`{{ $labels.container }}`}}) is restarting {{`{{ printf "%.2f" $value }}`}} times / 5 minutes.
  24         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
  25       expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[15m]) * 60 * 5 > 0
  26       for: 1h
  27       labels:
  28         severity: critical
  29     - alert: KubePodNotReady
  30       annotations:
  31         message: Pod {{`{{ $labels.namespace }}`}}/{{`{{ $labels.pod }}`}} has been in a non-ready state for longer than an hour.
  32         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
  33       expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}) > 0
  34       for: 1h
  35       labels:
  36         severity: critical
  37     - alert: KubeDeploymentGenerationMismatch
  38       annotations:
  39         message: Deployment generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} does not match, this indicates that the Deployment has failed but has not been rolled back.
  40         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
  41       expr: |-
  42         kube_deployment_status_observed_generation{job="kube-state-metrics"}
  43           !=
  44         kube_deployment_metadata_generation{job="kube-state-metrics"}
  45       for: 15m
  46       labels:
  47         severity: critical
  48     - alert: KubeDeploymentReplicasMismatch
  49       annotations:
  50         message: Deployment {{`{{ $labels.namespace }}`}}/{{`{{ $labels.deployment }}`}} has not matched the expected number of replicas for longer than an hour.
  51         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
  52       expr: |-
  53         kube_deployment_spec_replicas{job="kube-state-metrics"}
  54           !=
  55         kube_deployment_status_replicas_available{job="kube-state-metrics"}
  56       for: 1h
  57       labels:
  58         severity: critical
  59     - alert: KubeStatefulSetReplicasMismatch
  60       annotations:
  61         message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} has not matched the expected number of replicas for longer than 15 minutes.
  62         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
  63       expr: |-
  64         kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
  65           !=
  66         kube_statefulset_status_replicas{job="kube-state-metrics"}
  67       for: 15m
  68       labels:
  69         severity: critical
  70     - alert: KubeStatefulSetGenerationMismatch
  71       annotations:
  72         message: StatefulSet generation for {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
  73         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
  74       expr: |-
  75         kube_statefulset_status_observed_generation{job="kube-state-metrics"}
  76           !=
  77         kube_statefulset_metadata_generation{job="kube-state-metrics"}
  78       for: 15m
  79       labels:
  80         severity: critical
  81     - alert: KubeStatefulSetUpdateNotRolledOut
  82       annotations:
  83         message: StatefulSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.statefulset }}`}} update has not been rolled out.
  84         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
  85       expr: |-
  86         max without (revision) (
  87           kube_statefulset_status_current_revision{job="kube-state-metrics"}
  88             unless
  89           kube_statefulset_status_update_revision{job="kube-state-metrics"}
  90         )
  91           *
  92         (
  93           kube_statefulset_replicas{job="kube-state-metrics"}
  94             !=
  95           kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
  96         )
  97       for: 15m
  98       labels:
  99         severity: critical
 100     - alert: KubeDaemonSetRolloutStuck
 101       annotations:
 102         message: Only {{`{{ $value }}`}}% of the desired Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are scheduled and ready.
 103         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
 104       expr: |-
 105         kube_daemonset_status_number_ready{job="kube-state-metrics"}
 106           /
 107         kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} * 100 < 100
 108       for: 15m
 109       labels:
 110         severity: critical
 111     - alert: KubeDaemonSetNotScheduled
 112       annotations:
 113         message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are not scheduled.'
 114         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
 115       expr: |-
 116         kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
 117           -
 118         kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
 119       for: 10m
 120       labels:
 121         severity: warning
 122     - alert: KubeDaemonSetMisScheduled
 123       annotations:
 124         message: '{{`{{ $value }}`}} Pods of DaemonSet {{`{{ $labels.namespace }}`}}/{{`{{ $labels.daemonset }}`}} are running where they are not supposed to run.'
 125         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
 126       expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
 127       for: 10m
 128       labels:
 129         severity: warning
 130     - alert: KubeCronJobRunning
 131       annotations:
 132         message: CronJob {{`{{ $labels.namespace }}`}}/{{`{{ $labels.cronjob }}`}} is taking more than 1h to complete.
 133         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
 134       expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600
 135       for: 1h
 136       labels:
 137         severity: warning
 138     - alert: KubeJobCompletion
 139       annotations:
 140         message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} is taking more than one hour to complete.
 141         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
 142       expr: kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"}  > 0
 143       for: 1h
 144       labels:
 145         severity: warning
 146     - alert: KubeJobFailed
 147       annotations:
 148         message: Job {{`{{ $labels.namespace }}`}}/{{`{{ $labels.job_name }}`}} failed to complete.
 149         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
 150       expr: kube_job_status_failed{job="kube-state-metrics"}  > 0
 151       for: 1h
 152       labels:
 153         severity: warning
 154 {{- end }}