vnfs/DAaaS/deploy/operator/charts/prometheus-operator/templates/prometheus/rules/etcd.yaml

   1 # Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml
   2 # Do not change in-place! In order to change this file first read following link:
   3 # https://github.com/helm/charts/tree/master/stable/prometheus-operator/hack
   4 {{- if and .Values.defaultRules.create .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd }}
   5 apiVersion: {{ printf "%s/v1" (.Values.prometheusOperator.crdApiGroup | default "monitoring.coreos.com") }}
   6 kind: PrometheusRule
   7 metadata:
   8   name: {{ printf "%s-%s" (include "prometheus-operator.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }}
   9   labels:
  10     app: {{ template "prometheus-operator.name" . }}
  11 {{ include "prometheus-operator.labels" . | indent 4 }}
  12 {{- if .Values.defaultRules.labels }}
  13 {{ toYaml .Values.defaultRules.labels | indent 4 }}
  14 {{- end }}
  15 {{- if .Values.defaultRules.annotations }}
  16   annotations:
  17 {{ toYaml .Values.defaultRules.annotations | indent 4 }}
  18 {{- end }}
  19 spec:
  20   groups:
  21   - name: etcd
  22     rules:
  23     - alert: etcdInsufficientMembers
  24       annotations:
  25         message: 'etcd cluster "{{`{{ $labels.job }}`}}": insufficient members ({{`{{ $value }}`}}).'
  26       expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
  27       for: 3m
  28       labels:
  29         severity: critical
  30     - alert: etcdNoLeader
  31       annotations:
  32         message: 'etcd cluster "{{`{{ $labels.job }}`}}": member {{`{{ $labels.instance }}`}} has no leader.'
  33       expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
  34       for: 1m
  35       labels:
  36         severity: critical
  37     - alert: etcdHighNumberOfLeaderChanges
  38       annotations:
  39         message: 'etcd cluster "{{`{{ $labels.job }}`}}": instance {{`{{ $labels.instance }}`}} has seen {{`{{ $value }}`}} leader changes within the last hour.'
  40       expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
  41       for: 15m
  42       labels:
  43         severity: warning
  44     - alert: etcdHighNumberOfFailedGRPCRequests
  45       annotations:
  46         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
  47       expr: |-
  48         100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
  49           /
  50         sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
  51           > 1
  52       for: 10m
  53       labels:
  54         severity: warning
  55     - alert: etcdHighNumberOfFailedGRPCRequests
  56       annotations:
  57         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}}% of requests for {{`{{ $labels.grpc_method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
  58       expr: |-
  59         100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method)
  60           /
  61         sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
  62           > 5
  63       for: 5m
  64       labels:
  65         severity: critical
  66     - alert: etcdGRPCRequestsSlow
  67       annotations:
  68         message: 'etcd cluster "{{`{{ $labels.job }}`}}": gRPC requests to {{`{{ $labels.grpc_method }}`}} are taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  69       expr: |-
  70         histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
  71         > 0.15
  72       for: 10m
  73       labels:
  74         severity: critical
  75     - alert: etcdMemberCommunicationSlow
  76       annotations:
  77         message: 'etcd cluster "{{`{{ $labels.job }}`}}": member communication with {{`{{ $labels.To }}`}} is taking {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  78       expr: |-
  79         histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
  80         > 0.15
  81       for: 10m
  82       labels:
  83         severity: warning
  84     - alert: etcdHighNumberOfFailedProposals
  85       annotations:
  86         message: 'etcd cluster "{{`{{ $labels.job }}`}}": {{`{{ $value }}`}} proposal failures within the last hour on etcd instance {{`{{ $labels.instance }}`}}.'
  87       expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
  88       for: 15m
  89       labels:
  90         severity: warning
  91     - alert: etcdHighFsyncDurations
  92       annotations:
  93         message: 'etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile fync durations are {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
  94       expr: |-
  95         histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
  96         > 0.5
  97       for: 10m
  98       labels:
  99         severity: warning
 100     - alert: etcdHighCommitDurations
 101       annotations:
 102         message: 'etcd cluster "{{`{{ $labels.job }}`}}": 99th percentile commit durations {{`{{ $value }}`}}s on etcd instance {{`{{ $labels.instance }}`}}.'
 103       expr: |-
 104         histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
 105         > 0.25
 106       for: 10m
 107       labels:
 108         severity: warning
 109     - alert: etcdHighNumberOfFailedHTTPRequests
 110       annotations:
 111         message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}'
 112       expr: |-
 113         sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
 114         BY (method) > 0.01
 115       for: 10m
 116       labels:
 117         severity: warning
 118     - alert: etcdHighNumberOfFailedHTTPRequests
 119       annotations:
 120         message: '{{`{{ $value }}`}}% of requests for {{`{{ $labels.method }}`}} failed on etcd instance {{`{{ $labels.instance }}`}}.'
 121       expr: |-
 122         sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
 123         BY (method) > 0.05
 124       for: 10m
 125       labels:
 126         severity: critical
 127     - alert: etcdHTTPRequestsSlow
 128       annotations:
 129         message: etcd instance {{`{{ $labels.instance }}`}} HTTP requests to {{`{{ $labels.method }}`}} are slow.
 130       expr: |-
 131         histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
 132         > 0.15
 133       for: 10m
 134       labels:
 135         severity: warning
 136 {{- end }}