From cabbb6f88df4e74f599920ebe2e7b0ae9f6ee2c8 Mon Sep 17 00:00:00 2001 From: Jozsef Csongvai Date: Fri, 4 Mar 2022 15:58:31 -0500 Subject: [PATCH] [COMMON][MARIADB] Fix backup job The backup job stopped working after upgrade to bitnami images. Mariabackup was not designed to work remotely, it is supposed to run on the database server. Because of this we need to mount the data pvc into the backup job pod. It will however connect to the database daemon using a hostname, so we need to connect to the first replica in the cluster. Also had to set readOnlyRootFilesystem=false and add emptyDir volumes to solve various permission issues. Issue-ID: OOM-2932 Signed-off-by: Jozsef Csongvai Change-Id: I776903f9ec541f8dc5818b2ba4c1292226ec2bc6 --- .../mariadb-galera/templates/backup/cronjob.yaml | 43 +++++++++++++++++----- kubernetes/common/mariadb-galera/values.yaml | 3 ++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml index 210fbd02ba..4248cfe85c 100644 --- a/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml +++ b/kubernetes/common/mariadb-galera/templates/backup/cronjob.yaml @@ -15,7 +15,7 @@ # limitations under the License. */}} -{{- if .Values.backup.enabled }} +{{- if and .Values.backup.enabled .Values.persistence.enabled }} apiVersion: batch/v1beta1 kind: CronJob metadata: @@ -37,7 +37,10 @@ spec: - name: mariadb-galera-backup-init image: {{ include "repositoryGenerator.image.mariadb" . }} imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - {{ include "common.containerSecurityContext" . | indent 14 | trim }} + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false command: - /bin/bash - -c @@ -52,7 +55,7 @@ spec: target_dir=/backup/backup-`date +%s` mkdir -p $target_dir - mysqlhost={{ include "common.servicename" . }}.{{ include "common.namespace" . }} + mysqlhost={{ include "common.fullname" . }}-0.{{ include "common.servicename" . }}-headless.{{ include "common.namespace" . }} mariabackup --backup --target-dir=$target_dir --user=root --password=$DB_PASS --host=$mysqlhost @@ -78,13 +81,18 @@ spec: volumeMounts: - name: backup-dir mountPath: /backup + - name: data + mountPath: /bitnami/mariadb containers: - name: mariadb-backup-validate image: {{ include "repositoryGenerator.image.mariadb" . }} imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - {{ include "common.containerSecurityContext" . | indent 14 | trim }} + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false env: - - name: MYSQL_ROOT_PASSWORD + - name: MARIADB_ROOT_PASSWORD {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" (include "common.mariadb.secret.rootPassUID" .) "key" "password") | indent 18 }} command: - /bin/bash @@ -105,17 +113,17 @@ spec: fi target_dir=$(ls -td -- /backup/backup-* | head -n 1) - cp -Ra $target_dir/* /var/lib/mysql/ + cp -Ra $target_dir/* /bitnami/mariadb/data - if [ ! "$(ls -A /var/lib/mysql)" ]; then + if [ ! "$(ls -A /bitnami/mariadb/data)" ]; then remove_dir $target_dir exit 0 fi - /docker-entrypoint.sh mysqld & + /opt/bitnami/scripts/mariadb/entrypoint.sh /opt/bitnami/scripts/mariadb/run.sh & count=0 - until mysql --user=root --password=$MYSQL_ROOT_PASSWORD -e "SELECT 1"; + until mysql --user=root --password=$MARIADB_ROOT_PASSWORD -e "SELECT 1"; do sleep 3; count=`expr $count + 1`; if [ $count -ge 30 ]; then @@ -124,7 +132,7 @@ spec: fi; done - mysqlcheck -A --user=root --password=$MYSQL_ROOT_PASSWORD > /tmp/output.log + mysqlcheck -A --user=root --password=$MARIADB_ROOT_PASSWORD > /tmp/output.log error_lines=`cat /tmp/output.log| grep -v "OK" | wc -l` cat /tmp/output.log @@ -142,6 +150,10 @@ spec: fi resources: {{ include "common.resources" . | nindent 12 }} volumeMounts: + - mountPath: /bitnami/mariadb/data + name: tmp-data + - mountPath: /opt/bitnami/mariadb/tmp + name: tmp - mountPath: /etc/localtime name: localtime readOnly: true @@ -153,7 +165,18 @@ spec: - name: localtime hostPath: path: /etc/localtime + - name: data + persistentVolumeClaim: + {{- if .Values.persistence.existingClaim }} + claimName: {{ .Values.persistence.existingClaim }} + {{- else }} + claimName: {{ include "common.fullname" . }}-{{ include "common.fullname" . }}-0 + {{- end }} - name: backup-dir persistentVolumeClaim: claimName: {{ include "common.fullname" . }}-backup-data + - name: tmp-data + emptyDir: {} + - name: tmp + emptyDir: {} {{- end }} diff --git a/kubernetes/common/mariadb-galera/values.yaml b/kubernetes/common/mariadb-galera/values.yaml index 9f7c882134..d65c4f7943 100644 --- a/kubernetes/common/mariadb-galera/values.yaml +++ b/kubernetes/common/mariadb-galera/values.yaml @@ -174,6 +174,8 @@ galera: # password: # externalSecret: +## The backup job will mount the mariadb data pvc in order to run mariabackup. +## For this reason the db data pvc needs to have accessMode: ReadWriteMany. backup: enabled: false cron: "00 00 * * *" @@ -458,6 +460,7 @@ persistence: ## annotations: ## Persistent Volume Access Mode + ## Use ReadWriteMany if backup is enabled, see backup section. ## accessMode: ReadWriteOnce ## Persistent Volume size -- 2.16.6