kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- if .Values.backup.enabled }}
  17 apiVersion: batch/v1beta1
  18 kind: CronJob
  19 metadata:
  20   name: {{ include "common.fullname" . }}-backup
  21   namespace: {{ include "common.namespace" . }}
  22   labels:
  23     app: {{ include "common.fullname" . }}
  24     chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
  25     release: {{ .Release.Name }}
  26     heritage: {{ .Release.Service }}
  27 spec:
  28   schedule: {{ .Values.backup.cron | quote }}
  29   concurrencyPolicy: Forbid
  30   startingDeadlineSeconds: 120
  31   jobTemplate:
  32     spec:
  33       template:
  34         spec:
  35           restartPolicy: Never
  36           initContainers:
  37             - command:
  38               - /root/ready.py
  39               args:
  40               - --container-name
  41               - {{ include "common.name" . }}
  42               env:
  43               - name: NAMESPACE
  44                 valueFrom:
  45                   fieldRef:
  46                     apiVersion: v1
  47                     fieldPath: metadata.namespace
  48               image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
  49               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  50               name: {{ include "common.name" . }}-readiness
  51             - name: "cassandra-backup-init"
  52               image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
  53               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  54               command:
  55               - /bin/bash
  56               - -c
  57               - |
  58                 clearSnapshot(){
  59                   curr_time=$1
  60                   echo "Clearing snapshots!!!"
  61                   command="nodetool clearsnapshot -t $curr_time"
  62                   /root/exec.py -p "cassandra" -c "$command"
  63                 }
  64                 {{ $root := . }}
  65                 curr_time=`date +%s`
  66                 pids=""
  67                 set -x
  68
  69                 echo "Copying data"
  70                 {{ range $i, $e := until (int .Values.replicaCount) }}
  71                   target_dir=/backup/temp/cassandra-{{ $i }}
  72                   mkdir -p $target_dir
  73                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  74                 {{- end }}
  75
  76                 echo "Executing cleanup!!"
  77                 command="nodetool cleanup"
  78                 /root/exec.py -p "cassandra" -c "$command"
  79                 echo "Cleaned Node!! Backing up database now!!!"
  80
  81                 command="nodetool snapshot -t $curr_time"
  82                 /root/exec.py -p "cassandra" -c "$command"
  83                 retCode=$?
  84                 if [ $retCode -ne 0 ]; then
  85                   echo "Backup Failed!!!"
  86                   rm -rf /backup/temp
  87                   clearSnapshot $curr_time
  88                   echo "Failed" > /backup/backup.log
  89                   exit 0
  90                 fi
  91
  92                 backup_dir=/backup/temp
  93                 {{ range $i, $e := until (int .Values.replicaCount) }}
  94                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
  95                     d=$(echo $d | sed 's:/*$::')
  96                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
  97                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
  98                       /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
  99                       pids="$pids $!"
 100                     fi
 101                   done
 102                 {{- end }}
 103
 104                 for p in $pids; do
 105                   wait $p
 106                   if [ $? -ne 0 ]; then
 107                     rm -rf /backup/temp
 108                     echo "Creation of Backup Failed!!!"
 109                     clearSnapshot $curr_time
 110                     echo "Failed" > /backup/backup.log
 111                     exit 0
 112                   fi
 113                 done
 114
 115                 clearSnapshot $curr_time
 116
 117                 exit_code=$?
 118                 if [ $exit_code -ne 0 ]; then
 119                   rm -rf /backup/temp
 120                   echo "Backup Failed!!!"
 121                   echo "Failed" > /backup/backup.log
 122                   exit 0
 123                 fi
 124
 125                 mv /backup/temp /backup/backup-${curr_time}
 126                 echo "Success" > /backup/backup.log
 127                 echo "Cassandra Backup Succeeded"
 128               env:
 129               - name: NAMESPACE
 130                 valueFrom:
 131                   fieldRef:
 132                     apiVersion: v1
 133                     fieldPath: metadata.namespace
 134               volumeMounts:
 135               - mountPath: /etc/localtime
 136                 name: localtime
 137                 readOnly: true
 138               - mountPath: /onap-data
 139                 name: data-dir
 140               - mountPath: /backup
 141                 name: backup-dir
 142               - name: scripts
 143                 mountPath: /root/restore.sh
 144                 subPath: restore.sh
 145               - name: scripts
 146                 mountPath: /root/exec.py
 147                 subPath: exec.py
 148           containers:
 149             - name: cassandra-backup-validate
 150               image: "{{ .Values.image }}"
 151               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 152               command:
 153               - /bin/bash
 154               - -c
 155               - |
 156                 remove_dir(){
 157                   dirToRemove=$1
 158                   rm -rf $dirToRemove
 159                 }
 160
 161                 backup_result=`cat /backup/backup.log`
 162                 rm -rf /backup/backup.log
 163
 164                 if [ "$backup_result" == "Failed" ]; then
 165                   echo "Backup Failed!!! So Validation Failed!!!";
 166                   exit 0
 167                 fi
 168
 169                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 170                 chown -R cassandra.cassandra $target_dir
 171                 {{- $root := . -}}
 172                 {{ range $i, $e := until (int .Values.replicaCount) }}
 173                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 174                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 175                   if [ $dbSize -lt $minDbSize ]; then
 176                     remove_dir $target_dir
 177                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 178                     exit 0
 179                   fi
 180                   rm -rf /var/lib/cassandra/*
 181                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 182                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 183                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 184                   CASS_PID=$!
 185                   sleep 45
 186
 187                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 188                       d=$(echo $d | sed 's:/*$::')
 189                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 190                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 191                         echo "Verifying the data for  $keyspace_name "
 192                         nodetool verify -e $keyspace_name
 193                         ret=$?
 194                         if [ $ret -ne 0 ]; then
 195                           remove_dir $target_dir
 196                           echo "Validation Failed!!!"
 197                           exit 0
 198                         fi
 199                       fi
 200                   done
 201                   kill -9 $CASS_PID
 202                 {{- end }}
 203                 echo "Validation Successful!!!"
 204                 cd /backup
 205                 totalFiles=`ls -t | grep "backup-" | wc -l`
 206                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 207                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 208                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 209                 fi
 210               env:
 211               - name: CASSANDRA_CLUSTER_NAME
 212                 value: {{ .Values.config.clusterName }}
 213               - name: MAX_HEAP_SIZE
 214                 value: {{ .Values.config.heap.max }}
 215               - name: HEAP_NEWSIZE
 216                 value: {{ .Values.config.heap.min }}
 217               - name: HOST_IP
 218                 valueFrom:
 219                   fieldRef:
 220                     fieldPath: status.podIP
 221               volumeMounts:
 222               - name: backup-dir
 223                 mountPath: /backup
 224               - name: localtime
 225                 mountPath: /etc/localtime
 226                 readOnly: true
 227           volumes:
 228           - name: localtime
 229             hostPath:
 230               path: /etc/localtime
 231           - name: scripts
 232             configMap:
 233               name: {{ include "common.fullname" $ }}-configmap
 234               defaultMode: 0755
 235           - name: data-dir
 236             persistentVolumeClaim:
 237               claimName: {{ include "common.fullname" . }}-db-data
 238           - name: backup-dir
 239             persistentVolumeClaim:
 240               claimName: {{ include "common.fullname" . }}-backup-data
 241 {{- end -}}
 242