kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- $global := . }}
  17 {{- if .Values.backup.enabled }}
  18 apiVersion: batch/v1beta1
  19 kind: CronJob
  20 metadata:
  21   name: {{ include "common.fullname" . }}-backup
  22   namespace: {{ include "common.namespace" . }}
  23   labels: {{- include "common.labels" . | nindent 4 }}
  24 spec:
  25   schedule: {{ .Values.backup.cron | quote }}
  26   concurrencyPolicy: Forbid
  27   startingDeadlineSeconds: 120
  28   jobTemplate:
  29     spec:
  30       template:
  31         metadata:
  32           labels: {{- include "common.labels" . | nindent 12 }}
  33           annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" .) | nindent 12 }}
  34           name: {{ include "common.name" . }}
  35         spec:
  36           restartPolicy: Never
  37           initContainers:
  38             - command:
  39               - /root/ready.py
  40               args:
  41               - --container-name
  42               - {{ include "common.name" . }}
  43               env:
  44               - name: NAMESPACE
  45                 valueFrom:
  46                   fieldRef:
  47                     apiVersion: v1
  48                     fieldPath: metadata.namespace
  49               image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
  50               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  51               name: {{ include "common.name" . }}-readiness
  52             - name: "cassandra-backup-init"
  53               image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
  54               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  55               command:
  56               - /bin/sh
  57               - -c
  58               - |
  59                 clearSnapshot(){
  60                   curr_time=$1
  61                   echo "Clearing snapshots!!!"
  62                   command="nodetool clearsnapshot -t $curr_time"
  63                   /root/exec.py -p "cassandra" -c "$command"
  64                 }
  65                 {{ $root := . }}
  66                 curr_time=`date +%s`
  67                 pids=""
  68                 set -x
  69
  70                 echo "Copying data"
  71                 {{ range $i, $e := until (int .Values.replicaCount) }}
  72                   target_dir=/backup/temp/cassandra-{{ $i }}
  73                   mkdir -p $target_dir
  74                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  75                 {{- end }}
  76
  77                 echo "Executing cleanup!!"
  78                 command="nodetool cleanup"
  79                 /root/exec.py -p "cassandra" -c "$command"
  80                 echo "Cleaned Node!! Backing up database now!!!"
  81
  82                 command="nodetool snapshot -t $curr_time"
  83                 /root/exec.py -p "cassandra" -c "$command"
  84                 retCode=$?
  85                 if [ $retCode -ne 0 ]; then
  86                   echo "Backup Failed!!!"
  87                   rm -rf /backup/temp
  88                   clearSnapshot $curr_time
  89                   echo "Failed" > /backup/backup.log
  90                   exit 0
  91                 fi
  92
  93                 backup_dir=/backup/temp
  94                 {{ range $i, $e := until (int .Values.replicaCount) }}
  95                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
  96                     d=$(echo $d | sed 's:/*$::')
  97                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
  98                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
  99                       /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
 100                       pids="$pids $!"
 101                     fi
 102                   done
 103                 {{- end }}
 104
 105                 for p in $pids; do
 106                   wait $p
 107                   if [ $? -ne 0 ]; then
 108                     rm -rf /backup/temp
 109                     echo "Creation of Backup Failed!!!"
 110                     clearSnapshot $curr_time
 111                     echo "Failed" > /backup/backup.log
 112                     exit 0
 113                   fi
 114                 done
 115
 116                 clearSnapshot $curr_time
 117
 118                 exit_code=$?
 119                 if [ $exit_code -ne 0 ]; then
 120                   rm -rf /backup/temp
 121                   echo "Backup Failed!!!"
 122                   echo "Failed" > /backup/backup.log
 123                   exit 0
 124                 fi
 125
 126                 mv /backup/temp /backup/backup-${curr_time}
 127                 echo "Success" > /backup/backup.log
 128                 echo "Cassandra Backup Succeeded"
 129               env:
 130               - name: NAMESPACE
 131                 valueFrom:
 132                   fieldRef:
 133                     apiVersion: v1
 134                     fieldPath: metadata.namespace
 135               volumeMounts:
 136               - mountPath: /etc/localtime
 137                 name: localtime
 138                 readOnly: true
 139               {{- range $i := until (int .Values.replicaCount)}}
 140               - mountPath: /onap-data/cassandra-{{ $i }}
 141                 name: data-dir-{{ $i }}
 142               {{- end }}
 143               - mountPath: /backup
 144                 name: backup-dir
 145               - name: scripts
 146                 mountPath: /root/restore.sh
 147                 subPath: restore.sh
 148               - name: scripts
 149                 mountPath: /root/exec.py
 150                 subPath: exec.py
 151           containers:
 152             - name: cassandra-backup-validate
 153               image: "{{ .Values.image }}"
 154               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 155               command:
 156               - /bin/bash
 157               - -c
 158               - |
 159                 remove_dir(){
 160                   dirToRemove=$1
 161                   rm -rf $dirToRemove
 162                 }
 163
 164                 backup_result=`cat /backup/backup.log`
 165                 rm -rf /backup/backup.log
 166
 167                 if [ "$backup_result" == "Failed" ]; then
 168                   echo "Backup Failed!!! So Validation Failed!!!";
 169                   exit 0
 170                 fi
 171
 172                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 173                 chown -R cassandra.cassandra $target_dir
 174                 {{- $root := . -}}
 175                 {{ range $i, $e := until (int .Values.replicaCount) }}
 176                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 177                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 178                   if [ $dbSize -lt $minDbSize ]; then
 179                     remove_dir $target_dir
 180                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 181                     exit 0
 182                   fi
 183                   rm -rf /var/lib/cassandra/*
 184                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 185                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 186                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 187                   CASS_PID=$!
 188                   sleep 45
 189
 190                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 191                       d=$(echo $d | sed 's:/*$::')
 192                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 193                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 194                         echo "Verifying the data for  $keyspace_name "
 195                         nodetool verify -e $keyspace_name
 196                         ret=$?
 197                         if [ $ret -ne 0 ]; then
 198                           remove_dir $target_dir
 199                           echo "Validation Failed!!!"
 200                           exit 0
 201                         fi
 202                       fi
 203                   done
 204                   kill -9 $CASS_PID
 205                 {{- end }}
 206                 echo "Validation Successful!!!"
 207                 cd /backup
 208                 totalFiles=`ls -t | grep "backup-" | wc -l`
 209                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 210                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 211                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 212                 fi
 213               env:
 214               - name: CASSANDRA_CLUSTER_NAME
 215                 value: {{ .Values.config.clusterName }}
 216               - name: MAX_HEAP_SIZE
 217                 value: {{ .Values.config.heap.max }}
 218               - name: HEAP_NEWSIZE
 219                 value: {{ .Values.config.heap.min }}
 220               - name: HOST_IP
 221                 valueFrom:
 222                   fieldRef:
 223                     fieldPath: status.podIP
 224               volumeMounts:
 225               - name: backup-dir
 226                 mountPath: /backup
 227               - name: localtime
 228                 mountPath: /etc/localtime
 229                 readOnly: true
 230           volumes:
 231           - name: localtime
 232             hostPath:
 233               path: /etc/localtime
 234           - name: scripts
 235             configMap:
 236               name: {{ include "common.fullname" . }}-configmap
 237               defaultMode: 0755
 238           {{- range $i := until (int .Values.replicaCount)}}
 239           - name: data-dir-{{ $i }}
 240             persistentVolumeClaim:
 241               claimName: {{ include "common.fullname" $global }}-data-{{ include "common.fullname" $global }}-{{ $i }}
 242           {{- end }}
 243           - name: backup-dir
 244             persistentVolumeClaim:
 245               claimName: {{ include "common.fullname" . }}-backup-data
 246 {{- end -}}