kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- if .Values.backup.enabled }}
  17 apiVersion: batch/v1beta1
  18 kind: CronJob
  19 metadata:
  20   name: {{ include "common.fullname" . }}-backup
  21   namespace: {{ include "common.namespace" . }}
  22   labels: {{- include "common.labels" . | nindent 4 }}
  23 spec:
  24   schedule: {{ .Values.backup.cron | quote }}
  25   concurrencyPolicy: Forbid
  26   startingDeadlineSeconds: 120
  27   jobTemplate:
  28     spec:
  29       template:
  30         metadata:
  31           labels: {{- include "common.labels" . | nindent 12 }}
  32           annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" .) | nindent 12 }}
  33           name: {{ include "common.name" . }}
  34         spec:
  35           restartPolicy: Never
  36           initContainers:
  37             - command:
  38               - /app/ready.py
  39               args:
  40               - --container-name
  41               - {{ include "common.name" . }}
  42               env:
  43               - name: NAMESPACE
  44                 valueFrom:
  45                   fieldRef:
  46                     apiVersion: v1
  47                     fieldPath: metadata.namespace
  48               image: {{ include "repositoryGenerator.image.readiness" . }}
  49               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  50               name: {{ include "common.name" . }}-readiness
  51             - name: "cassandra-backup-init"
  52               image: {{ include "repositoryGenerator.image.readiness" . }}
  53               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  54               command:
  55               - /bin/bash
  56               - -c
  57               - |
  58                 clearSnapshot(){
  59                   curr_time=$1
  60                   echo "Clearing snapshots!!!"
  61                   command="nodetool clearsnapshot -t $curr_time"
  62                   /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  63                 }
  64                 {{ $root := . }}
  65                 curr_time=`date +%s`
  66                 pids=""
  67                 set -x
  68
  69                 echo "Copying data"
  70                 {{ range $i, $e := until (int .Values.replicaCount) }}
  71                   target_dir=/backup/temp/cassandra-{{ $i }}
  72                   mkdir -p $target_dir
  73                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  74                 {{- end }}
  75
  76                 echo "Executing cleanup!!"
  77                 command="nodetool cleanup"
  78                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  79                 echo "Cleaned Node!! Backing up database now!!!"
  80
  81                 command="nodetool snapshot -t $curr_time"
  82                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  83                 retCode=$?
  84                 if [ $retCode -ne 0 ]; then
  85                   echo "Backup Failed!!!"
  86                   rm -rf /backup/temp
  87                   clearSnapshot $curr_time
  88                   echo "Failed" > /backup/backup.log
  89                   exit 0
  90                 fi
  91
  92                 backup_dir=/backup/temp
  93                 {{ range $i, $e := until (int .Values.replicaCount) }}
  94                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
  95                     d=$(echo $d | sed 's:/*$::')
  96                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
  97                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
  98                       /app/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
  99                       pids="$pids $!"
 100                     fi
 101                   done
 102                 {{- end }}
 103
 104                 for p in $pids; do
 105                   wait $p
 106                   if [ $? -ne 0 ]; then
 107                     rm -rf /backup/temp
 108                     echo "Creation of Backup Failed!!!"
 109                     clearSnapshot $curr_time
 110                     echo "Failed" > /backup/backup.log
 111                     exit 0
 112                   fi
 113                 done
 114
 115                 clearSnapshot $curr_time
 116
 117                 exit_code=$?
 118                 if [ $exit_code -ne 0 ]; then
 119                   rm -rf /backup/temp
 120                   echo "Backup Failed!!!"
 121                   echo "Failed" > /backup/backup.log
 122                   exit 0
 123                 fi
 124
 125                 mv /backup/temp /backup/backup-${curr_time}
 126                 echo "Success" > /backup/backup.log
 127                 echo "Cassandra Backup Succeeded"
 128               env:
 129               - name: NAMESPACE
 130                 valueFrom:
 131                   fieldRef:
 132                     apiVersion: v1
 133                     fieldPath: metadata.namespace
 134               volumeMounts:
 135               - mountPath: /etc/localtime
 136                 name: localtime
 137                 readOnly: true
 138               {{- range $i := until (int .Values.replicaCount)}}
 139               - mountPath: /onap-data/cassandra-{{ $i }}
 140                 name: data-dir-{{ $i }}
 141               {{- end }}
 142               - mountPath: /backup
 143                 name: backup-dir
 144               - name: scripts
 145                 mountPath: /app/restore.sh
 146                 subPath: restore.sh
 147               - name: scripts
 148                 mountPath: /app/exec.py
 149                 subPath: exec.py
 150           containers:
 151             - name: cassandra-backup-validate
 152               image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.image }}
 153               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 154               command:
 155               - /bin/bash
 156               - -c
 157               - |
 158                 remove_dir(){
 159                   dirToRemove=$1
 160                   rm -rf $dirToRemove
 161                 }
 162
 163                 backup_result=`cat /backup/backup.log`
 164                 rm -rf /backup/backup.log
 165
 166                 if [ "$backup_result" == "Failed" ]; then
 167                   echo "Backup Failed!!! So Validation Failed!!!";
 168                   exit 0
 169                 fi
 170
 171                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 172                 chown -R cassandra.cassandra $target_dir
 173                 {{- $root := . -}}
 174                 {{ range $i, $e := until (int .Values.replicaCount) }}
 175                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 176                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 177                   if [ $dbSize -lt $minDbSize ]; then
 178                     remove_dir $target_dir
 179                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 180                     exit 0
 181                   fi
 182                   rm -rf /var/lib/cassandra/*
 183                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 184                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 185                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 186                   CASS_PID=$!
 187                   sleep 45
 188
 189                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 190                       d=$(echo $d | sed 's:/*$::')
 191                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 192                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 193                         echo "Verifying the data for  $keyspace_name "
 194                         nodetool verify -e $keyspace_name
 195                         ret=$?
 196                         if [ $ret -ne 0 ]; then
 197                           remove_dir $target_dir
 198                           echo "Validation Failed!!!"
 199                           exit 0
 200                         fi
 201                       fi
 202                   done
 203                   kill -9 $CASS_PID
 204                 {{- end }}
 205                 echo "Validation Successful!!!"
 206                 cd /backup
 207                 totalFiles=`ls -t | grep "backup-" | wc -l`
 208                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 209                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 210                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 211                 fi
 212               env:
 213               - name: CASSANDRA_CLUSTER_NAME
 214                 value: {{ .Values.config.clusterName }}
 215               - name: MAX_HEAP_SIZE
 216                 value: {{ .Values.config.heap.max }}
 217               - name: HEAP_NEWSIZE
 218                 value: {{ .Values.config.heap.min }}
 219               - name: HOST_IP
 220                 valueFrom:
 221                   fieldRef:
 222                     fieldPath: status.podIP
 223               volumeMounts:
 224               - name: backup-dir
 225                 mountPath: /backup
 226               - name: localtime
 227                 mountPath: /etc/localtime
 228                 readOnly: true
 229           volumes:
 230           - name: localtime
 231             hostPath:
 232               path: /etc/localtime
 233           - name: scripts
 234             configMap:
 235               name: {{ include "common.fullname" . }}-configmap
 236               defaultMode: 0755
 237           {{- range $i := until (int .Values.replicaCount)}}
 238           - name: data-dir-{{ $i }}
 239             persistentVolumeClaim:
 240               claimName: {{ include "common.fullname" . }}-data-{{ $i }}
 241           {{- end }}
 242           - name: backup-dir
 243             persistentVolumeClaim:
 244               claimName: {{ include "common.fullname" . }}-backup-data
 245 {{- end -}}