From 8c9416ba16ce872d52622f7ba14bb0021b227b93 Mon Sep 17 00:00:00 2001 From: Sylvain Desbureaux Date: Wed, 5 May 2021 11:12:48 +0200 Subject: [PATCH] [SDC] Use Startup probes Instead of long initial delay on readiness and liveness probes, use startup probes and be more aggressive on readiness and liveness. Issue-ID: OOM-2743 Signed-off-by: Sylvain Desbureaux Change-Id: Ibde8d10dad150fcad5740533d0bdd0926a44ccc1 --- .../components/sdc-be/templates/deployment.yaml | 14 ++++++++++++++ kubernetes/sdc/components/sdc-be/values.yaml | 15 +++++++++++++-- .../components/sdc-fe/templates/deployment.yaml | 14 +++++++++++++- kubernetes/sdc/components/sdc-fe/values.yaml | 17 ++++++++++++++--- .../sdc-helm-validator/templates/deployment.yaml | 10 ++++++++++ .../sdc/components/sdc-helm-validator/values.yaml | 14 ++++++++++++-- .../sdc-onboarding-be/templates/deployment.yaml | 15 ++++++++++++++- .../sdc/components/sdc-onboarding-be/values.yaml | 19 +++++++++++++++---- .../sdc-wfd-be/templates/deployment.yaml | 11 +++++++++++ kubernetes/sdc/components/sdc-wfd-be/values.yaml | 22 ++++++++++++++++++++++ .../sdc-wfd-fe/templates/deployment.yaml | 11 +++++++++++ kubernetes/sdc/components/sdc-wfd-fe/values.yaml | 14 ++++++++++++-- 12 files changed, 161 insertions(+), 15 deletions(-) diff --git a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml index 44439869cc..28e9c1369c 100644 --- a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml @@ -124,6 +124,8 @@ spec: initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} {{ end }} readinessProbe: exec: @@ -132,6 +134,18 @@ spec: initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} + successThreshold: {{ .Values.readiness.successThreshold }} + failureThreshold: {{ .Values.readiness.failureThreshold }} + resources: {{ include "common.resources" . | nindent 12 }} + startupProbe: + exec: + command: + - "/var/lib/jetty/ready-probe.sh" + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + timeoutSeconds: {{ .Values.startup.timeoutSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} resources: {{ include "common.resources" . | nindent 12 }} env: - name: ENVNAME diff --git a/kubernetes/sdc/components/sdc-be/values.yaml b/kubernetes/sdc/components/sdc-be/values.yaml index 045136aef0..1277fc9176 100644 --- a/kubernetes/sdc/components/sdc-be/values.yaml +++ b/kubernetes/sdc/components/sdc-be/values.yaml @@ -83,18 +83,29 @@ affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 120 + initialDelaySeconds: 1 periodSeconds: 10 timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container port: api enabled: true readiness: - initialDelaySeconds: 60 + initialDelaySeconds: 1 periodSeconds: 10 timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + +startup: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 service: type: NodePort diff --git a/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml b/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml index 45c7bc85b6..0a5c0a3445 100644 --- a/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml @@ -117,13 +117,25 @@ spec: initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} {{ end }} readinessProbe: tcpSocket: port: {{ .Values.service.internalPort2 }} initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} + successThreshold: {{ .Values.readiness.successThreshold }} + failureThreshold: {{ .Values.readiness.failureThreshold }} + startupProbe: + tcpSocket: + port: {{ .Values.service.internalPort2 }} + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + timeoutSeconds: {{ .Values.startup.timeoutSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} resources: {{ include "common.resources" . | nindent 12 }} env: - name: ENVNAME diff --git a/kubernetes/sdc/components/sdc-fe/values.yaml b/kubernetes/sdc/components/sdc-fe/values.yaml index 7aad4c13cb..dcebc475ad 100644 --- a/kubernetes/sdc/components/sdc-fe/values.yaml +++ b/kubernetes/sdc/components/sdc-fe/values.yaml @@ -76,17 +76,28 @@ affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 10 - periodSeconds: 60 + initialDelaySeconds: 1 + periodSeconds: 10 timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 3 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true readiness: + initialDelaySeconds: 1 + periodSeconds: 10 + timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + +startup: initialDelaySeconds: 10 - periodSeconds: 60 + periodSeconds: 10 timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 60 service: #Example service definition with external, internal and node ports. diff --git a/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml b/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml index 08228ad99c..f736a174db 100644 --- a/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml @@ -40,5 +40,15 @@ spec: port: {{ .Values.liveness.port }} initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} + startupProbe: + httpGet: + path: {{ .Values.startup.path }} + port: {{ .Values.startup.port }} + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/sdc/components/sdc-helm-validator/values.yaml b/kubernetes/sdc/components/sdc-helm-validator/values.yaml index 9c0d90649f..ede80a6af0 100644 --- a/kubernetes/sdc/components/sdc-helm-validator/values.yaml +++ b/kubernetes/sdc/components/sdc-helm-validator/values.yaml @@ -31,14 +31,24 @@ service: port: *svc_port liveness: - initialDelaySeconds: 30 - periodSeconds: 30 + initialDelaySeconds: 1 + periodSeconds: 10 path: /actuator/health + successThreshold: 1 + failureThreshold: 3 port: *port # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true +startup: + initialDelaySeconds: 10 + periodSeconds: 10 + path: /actuator/health + successThreshold: 1 + failureThreshold: 12 + port: *port + flavor: small resources: small: diff --git a/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml index 5c530fea72..eb85465855 100644 --- a/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml @@ -128,6 +128,8 @@ spec: initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} {{ end }} readinessProbe: exec: @@ -135,7 +137,18 @@ spec: - "/var/lib/jetty/ready-probe.sh" initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} + timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} + successThreshold: {{ .Values.readiness.successThreshold }} + failureThreshold: {{ .Values.readiness.failureThreshold }} + startupProbe: + exec: + command: + - "/var/lib/jetty/ready-probe.sh" + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + timeoutSeconds: {{ .Values.startup.timeoutSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} resources: {{ include "common.resources" . | nindent 12 }} env: - name: ENVNAME diff --git a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml index 7e4dddd124..ddff1d3f1a 100644 --- a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml +++ b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml @@ -83,17 +83,28 @@ affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 120 - periodSeconds: 60 + initialDelaySeconds: 1 + periodSeconds: 10 timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 3 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true readiness: - initialDelaySeconds: 120 - periodSeconds: 60 + initialDelaySeconds: 1 + periodSeconds: 10 timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 3 + +startup: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 15 + successThreshold: 1 + failureThreshold: 60 service: type: ClusterIP diff --git a/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml index 9defb8e1ce..de75092f8a 100644 --- a/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml @@ -88,12 +88,23 @@ spec: port: {{ template "wfd-be.internalPort" . }} initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} {{ end }} readinessProbe: tcpSocket: port: {{ template "wfd-be.internalPort" . }} initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} + successThreshold: {{ .Values.readiness.successThreshold }} + failureThreshold: {{ .Values.readiness.failureThreshold }} + startupProbe: + tcpSocket: + port: {{ template "wfd-be.internalPort" . }} + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} env: - name: JAVA_OPTIONS value: {{ .Values.config.javaOptions }} diff --git a/kubernetes/sdc/components/sdc-wfd-be/values.yaml b/kubernetes/sdc/components/sdc-wfd-be/values.yaml index 4aebe7ab9a..0d75ae3c8b 100644 --- a/kubernetes/sdc/components/sdc-wfd-be/values.yaml +++ b/kubernetes/sdc/components/sdc-wfd-be/values.yaml @@ -101,6 +101,28 @@ readiness: initialDelaySeconds: 60 periodSeconds: 10 +# probe configuration parameters +liveness: + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + +startup: + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 60 + service: type: NodePort portName: sdc-wfd-be diff --git a/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml b/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml index 7a8cf8fb34..b8073d723d 100644 --- a/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml +++ b/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml @@ -105,12 +105,23 @@ spec: port: {{ template "wfd-fe.internalPort" . }} initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} + successThreshold: {{ .Values.liveness.successThreshold }} + failureThreshold: {{ .Values.liveness.failureThreshold }} {{ end }} readinessProbe: tcpSocket: port: {{ template "wfd-fe.internalPort" . }} initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} + successThreshold: {{ .Values.readiness.successThreshold }} + failureThreshold: {{ .Values.readiness.failureThreshold }} + startupProbe: + tcpSocket: + port: {{ template "wfd-fe.internalPort" . }} + initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }} + periodSeconds: {{ .Values.startup.periodSeconds }} + successThreshold: {{ .Values.startup.successThreshold }} + failureThreshold: {{ .Values.startup.failureThreshold }} env: - name: ENVNAME value: {{ .Values.env.name }} diff --git a/kubernetes/sdc/components/sdc-wfd-fe/values.yaml b/kubernetes/sdc/components/sdc-wfd-fe/values.yaml index ff8aebf6b2..235931bd7d 100644 --- a/kubernetes/sdc/components/sdc-wfd-fe/values.yaml +++ b/kubernetes/sdc/components/sdc-wfd-fe/values.yaml @@ -77,15 +77,25 @@ affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 60 + initialDelaySeconds: 1 periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true readiness: - initialDelaySeconds: 60 + initialDelaySeconds: 1 periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + +startup: + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 60 service: type: NodePort -- 2.16.6