kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- if .Values.backup.enabled }}
  17 {{- if .Values.global.cassandra.useOperator }}
  18 {{ else }}
  19 apiVersion: batch/v1beta1
  20 kind: CronJob
  21 metadata:
  22   name: {{ include "common.fullname" . }}-backup
  23   namespace: {{ include "common.namespace" . }}
  24   labels: {{- include "common.labels" . | nindent 4 }}
  25 spec:
  26   schedule: {{ .Values.backup.cron | quote }}
  27   concurrencyPolicy: Forbid
  28   startingDeadlineSeconds: 120
  29   jobTemplate:
  30     spec:
  31       template:
  32         metadata:
  33           labels: {{- include "common.labels" . | nindent 12 }}
  34           annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" .) | nindent 12 }}
  35           name: {{ include "common.name" . }}
  36         spec:
  37           restartPolicy: Never
  38           initContainers:
  39             - command:
  40               - /app/ready.py
  41               args:
  42               - --app-name
  43               - {{ include "common.name" . }}
  44               env:
  45               - name: NAMESPACE
  46                 valueFrom:
  47                   fieldRef:
  48                     apiVersion: v1
  49                     fieldPath: metadata.namespace
  50               image: {{ include "repositoryGenerator.image.readiness" . }}
  51               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  52               name: {{ include "common.name" . }}-readiness
  53             - name: "cassandra-backup-init"
  54               image: {{ include "repositoryGenerator.image.readiness" . }}
  55               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  56               command:
  57               - /bin/bash
  58               - -c
  59               - |
  60                 clearSnapshot(){
  61                   curr_time=$1
  62                   echo "Clearing snapshots!!!"
  63                   command="nodetool clearsnapshot -t $curr_time"
  64                   /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  65                 }
  66                 {{ $root := . }}
  67                 curr_time=`date +%s`
  68                 pids=""
  69                 set -x
  70
  71                 echo "Copying data"
  72                 {{ range $i, $e := until (int .Values.replicaCount) }}
  73                   target_dir=/backup/temp/cassandra-{{ $i }}
  74                   mkdir -p $target_dir
  75                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  76                 {{- end }}
  77
  78                 echo "Executing cleanup!!"
  79                 command="nodetool cleanup"
  80                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  81                 echo "Cleaned Node!! Backing up database now!!!"
  82
  83                 command="nodetool snapshot -t $curr_time"
  84                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  85                 retCode=$?
  86                 if [ $retCode -ne 0 ]; then
  87                   echo "Backup Failed!!!"
  88                   rm -rf /backup/temp
  89                   clearSnapshot $curr_time
  90                   echo "Failed" > /backup/backup.log
  91                   exit 0
  92                 fi
  93
  94                 backup_dir=/backup/temp
  95                 {{ range $i, $e := until (int .Values.replicaCount) }}
  96                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
  97                     d=$(echo $d | sed 's:/*$::')
  98                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
  99                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 100                       /app/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
 101                       pids="$pids $!"
 102                     fi
 103                   done
 104                 {{- end }}
 105
 106                 for p in $pids; do
 107                   wait $p
 108                   if [ $? -ne 0 ]; then
 109                     rm -rf /backup/temp
 110                     echo "Creation of Backup Failed!!!"
 111                     clearSnapshot $curr_time
 112                     echo "Failed" > /backup/backup.log
 113                     exit 0
 114                   fi
 115                 done
 116
 117                 clearSnapshot $curr_time
 118
 119                 exit_code=$?
 120                 if [ $exit_code -ne 0 ]; then
 121                   rm -rf /backup/temp
 122                   echo "Backup Failed!!!"
 123                   echo "Failed" > /backup/backup.log
 124                   exit 0
 125                 fi
 126
 127                 mv /backup/temp /backup/backup-${curr_time}
 128                 echo "Success" > /backup/backup.log
 129                 echo "Cassandra Backup Succeeded"
 130               env:
 131               - name: NAMESPACE
 132                 valueFrom:
 133                   fieldRef:
 134                     apiVersion: v1
 135                     fieldPath: metadata.namespace
 136               volumeMounts:
 137               - mountPath: /etc/localtime
 138                 name: localtime
 139                 readOnly: true
 140               {{- range $i := until (int .Values.replicaCount)}}
 141               - mountPath: /onap-data/cassandra-{{ $i }}
 142                 name: data-dir-{{ $i }}
 143               {{- end }}
 144               - mountPath: /backup
 145                 name: backup-dir
 146               - name: scripts
 147                 mountPath: /app/restore.sh
 148                 subPath: restore.sh
 149               - name: scripts
 150                 mountPath: /app/exec.py
 151                 subPath: exec.py
 152           containers:
 153             - name: cassandra-backup-validate
 154               image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.image }}
 155               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 156               command:
 157               - /bin/bash
 158               - -c
 159               - |
 160                 remove_dir(){
 161                   dirToRemove=$1
 162                   rm -rf $dirToRemove
 163                 }
 164
 165                 backup_result=`cat /backup/backup.log`
 166                 rm -rf /backup/backup.log
 167
 168                 if [ "$backup_result" == "Failed" ]; then
 169                   echo "Backup Failed!!! So Validation Failed!!!";
 170                   exit 0
 171                 fi
 172
 173                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 174                 chown -R cassandra.cassandra $target_dir
 175                 {{- $root := . -}}
 176                 {{ range $i, $e := until (int .Values.replicaCount) }}
 177                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 178                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 179                   if [ $dbSize -lt $minDbSize ]; then
 180                     remove_dir $target_dir
 181                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 182                     exit 0
 183                   fi
 184                   rm -rf /var/lib/cassandra/*
 185                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 186                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 187                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 188                   CASS_PID=$!
 189                   sleep 45
 190
 191                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 192                       d=$(echo $d | sed 's:/*$::')
 193                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 194                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 195                         echo "Verifying the data for  $keyspace_name "
 196                         nodetool verify -e $keyspace_name
 197                         ret=$?
 198                         if [ $ret -ne 0 ]; then
 199                           remove_dir $target_dir
 200                           echo "Validation Failed!!!"
 201                           exit 0
 202                         fi
 203                       fi
 204                   done
 205                   kill -9 $CASS_PID
 206                 {{- end }}
 207                 echo "Validation Successful!!!"
 208                 cd /backup
 209                 totalFiles=`ls -t | grep "backup-" | wc -l`
 210                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 211                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 212                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 213                 fi
 214               env:
 215               - name: CASSANDRA_CLUSTER_NAME
 216                 value: {{ .Values.config.clusterName }}
 217               - name: MAX_HEAP_SIZE
 218                 value: {{ .Values.config.heap.max }}
 219               - name: HEAP_NEWSIZE
 220                 value: {{ .Values.config.heap.min }}
 221               - name: HOST_IP
 222                 valueFrom:
 223                   fieldRef:
 224                     fieldPath: status.podIP
 225               volumeMounts:
 226               - name: backup-dir
 227                 mountPath: /backup
 228               - name: localtime
 229                 mountPath: /etc/localtime
 230                 readOnly: true
 231           volumes:
 232           - name: localtime
 233             hostPath:
 234               path: /etc/localtime
 235           - name: scripts
 236             configMap:
 237               name: {{ include "common.fullname" . }}-configmap
 238               defaultMode: 0755
 239           {{- range $i := until (int .Values.replicaCount)}}
 240           - name: data-dir-{{ $i }}
 241             persistentVolumeClaim:
 242               claimName: {{ include "common.fullname" . }}-data-{{ $i }}
 243           {{- end }}
 244           - name: backup-dir
 245             persistentVolumeClaim:
 246               claimName: {{ include "common.fullname" . }}-backup-data
 247 {{- end -}}
 248 {{- end -}}