vnfs/DAaaS/prometheus-operator/templates/alertmanager/rules/etcd.yaml

   1 # Generated from 'etcd' group from https://raw.githubusercontent.com/paskal/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml
   2 {{- if and .Values.defaultRules.create .Values.kubeEtcd.enabled }}
   3 apiVersion: {{ printf "%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }}
   4 kind: PrometheusRule
   5 metadata:
   6   name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }}
   7   labels:
   8     app: {{ template "prometheus-operator.name" . }}
   9 {{ include "prometheus-operator.labels" . | indent 4 }}
  10 {{- if .Values.defaultRules.labels }}
  11 {{ toYaml .Values.defaultRules.labels | indent 4 }}
  12 {{- end }}
  13 {{- if .Values.defaultRules.annotations }}
  14   annotations:
  15 {{ toYaml .Values.defaultRules.annotations | indent 4 }}
  16 {{- end }}
  17 spec:
  18   groups:
  19   - name: etcd
  20     rules:
  21     - alert: etcdInsufficientMembers
  22       annotations:
  23         message: 'etcd cluster "{{`{{ $labels.job }}`}}": insufficient members ({{`{{ $value }}`}}).'
  24       expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
  25       for: 3m
  26       labels:
  27         severity: critical
  28     - alert: etcdNoLeader
  29       annotations:
  30         message: 'etcd cluster "{{`{{ $labels.job }}`}}": member {{`{{ $labels.instance }}`}} has no leader.'
  31       expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
  32       for: 1m
  33       labels:
  34         severity: critical
  35     - alert: etcdHighNumberOfLeaderChanges
  36       annotations:
  37         message: 'etcd cluster "{{`{{ $labels.job }}`}}": instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} leader changes within the last hour.'
  38       expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
  39       for: 15m
  40       labels:
  41         severity: warning
  42     - alert: etcdHighNumberOfFailedGRPCRequests
  43       annotations:
  44         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
  45       expr: |-
  46         100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
  47           /
  48         sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
  49           > 1
  50       for: 10m
  51       labels:
  52         severity: warning
  53     - alert: etcdHighNumberOfFailedGRPCRequests
  54       annotations:
  55         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
  56       expr: |-
  57         100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
  58           /
  59         sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
  60           > 5
  61       for: 5m
  62       labels:
  63         severity: critical
  64     - alert: etcdGRPCRequestsSlow
  65       annotations:
  66         message: 'etcd cluster "{{`{{ $labels.job }}`}}": gRPC requests to {{`{{ $labels.grpc_method }}`}} are taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  67       expr: |-
  68         histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
  69         > 0.15
  70       for: 10m
  71       labels:
  72         severity: critical
  73     - alert: etcdMemberCommunicationSlow
  74       annotations:
  75         message: 'etcd cluster "{{`{{ $labels.job }}`}}": member communication with {{`{{ $labels.To }}`}} is taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  76       expr: |-
  77         histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
  78         > 0.15
  79       for: 10m
  80       labels:
  81         severity: warning
  82     - alert: etcdHighNumberOfFailedProposals
  83       annotations:
  84         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}} proposal failures within the last hour on etcd instance {{`{{ $labels.instance }}`}}.'
  85       expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
  86       for: 15m
  87       labels:
  88         severity: warning
  89     - alert: etcdHighFsyncDurations
  90       annotations:
  91         message: 'etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile fync durations are {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  92       expr: |-
  93         histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
  94         > 0.5
  95       for: 10m
  96       labels:
  97         severity: warning
  98     - alert: etcdHighCommitDurations
  99       annotations:
 100         message: 'etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile commit durations {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
 101       expr: |-
 102         histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
 103         > 0.25
 104       for: 10m
 105       labels:
 106         severity: warning
 107     - alert: etcdHighNumberOfFailedHTTPRequests
 108       annotations:
 109         message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}'
 110       expr: |-
 111         sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
 112         BY (method) > 0.01
 113       for: 10m
 114       labels:
 115         severity: warning
 116     - alert: etcdHighNumberOfFailedHTTPRequests
 117       annotations:
 118         message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
 119       expr: |-
 120         sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
 121         BY (method) > 0.05
 122       for: 10m
 123       labels:
 124         severity: critical
 125     - alert: etcdHTTPRequestsSlow
 126       annotations:
 127         message: etcd instance {{`{{ $labels.instance }}`}} HTTP requests to {{`{{ $labels.method }}`}} are slow.
 128       expr: |-
 129         histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
 130         > 0.15
 131       for: 10m
 132       labels:
 133         severity: warning
 134 {{- end }}