kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- if .Values.backup.enabled }}
  17 {{- if .Values.global.cassandra.useOperator }}
  18 {{ else }}
  19 apiVersion: batch/v1beta1
  20 kind: CronJob
  21 metadata:
  22   name: {{ include "common.fullname" . }}-backup
  23   namespace: {{ include "common.namespace" . }}
  24   labels: {{- include "common.labels" . | nindent 4 }}
  25 spec:
  26   schedule: {{ .Values.backup.cron | quote }}
  27   concurrencyPolicy: Forbid
  28   startingDeadlineSeconds: 120
  29   jobTemplate:
  30     spec:
  31       template:
  32         metadata:
  33           labels: {{- include "common.labels" . | nindent 12 }}
  34           annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" .) | nindent 12 }}
  35           name: {{ include "common.name" . }}
  36         spec:
  37           restartPolicy: Never
  38           initContainers:
  39             - command:
  40               - /app/ready.py
  41               args:
  42               - --app-name
  43               - {{ include "common.name" . }}
  44               env:
  45               - name: NAMESPACE
  46                 valueFrom:
  47                   fieldRef:
  48                     apiVersion: v1
  49                     fieldPath: metadata.namespace
  50               image: {{ include "repositoryGenerator.image.readiness" . }}
  51               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  52               name: {{ include "common.name" . }}-readiness
  53               resources:
  54                 limits:
  55                   cpu: "100m"
  56                   memory: "500Mi"
  57                 requests:
  58                   cpu: "3m"
  59                   memory: "20Mi"
  60             - name: "cassandra-backup-init"
  61               image: {{ include "repositoryGenerator.image.readiness" . }}
  62               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  63               resources:
  64                 limits:
  65                   cpu: "100m"
  66                   memory: "500Mi"
  67                 requests:
  68                   cpu: "3m"
  69                   memory: "20Mi"
  70               command:
  71               - /bin/bash
  72               - -c
  73               - |
  74                 clearSnapshot(){
  75                   curr_time=$1
  76                   echo "Clearing snapshots!!!"
  77                   command="nodetool clearsnapshot -t $curr_time"
  78                   /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  79                 }
  80                 {{ $root := . }}
  81                 curr_time=`date +%s`
  82                 pids=""
  83                 set -x
  84
  85                 echo "Copying data"
  86                 {{ range $i, $e := until (int .Values.replicaCount) }}
  87                   target_dir=/backup/temp/cassandra-{{ $i }}
  88                   mkdir -p $target_dir
  89                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  90                 {{- end }}
  91
  92                 echo "Executing cleanup!!"
  93                 command="nodetool cleanup"
  94                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  95                 echo "Cleaned Node!! Backing up database now!!!"
  96
  97                 command="nodetool snapshot -t $curr_time"
  98                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  99                 retCode=$?
 100                 if [ $retCode -ne 0 ]; then
 101                   echo "Backup Failed!!!"
 102                   rm -rf /backup/temp
 103                   clearSnapshot $curr_time
 104                   echo "Failed" > /backup/backup.log
 105                   exit 0
 106                 fi
 107
 108                 backup_dir=/backup/temp
 109                 {{ range $i, $e := until (int .Values.replicaCount) }}
 110                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
 111                     d=$(echo $d | sed 's:/*$::')
 112                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 113                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 114                       /app/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
 115                       pids="$pids $!"
 116                     fi
 117                   done
 118                 {{- end }}
 119
 120                 for p in $pids; do
 121                   wait $p
 122                   if [ $? -ne 0 ]; then
 123                     rm -rf /backup/temp
 124                     echo "Creation of Backup Failed!!!"
 125                     clearSnapshot $curr_time
 126                     echo "Failed" > /backup/backup.log
 127                     exit 0
 128                   fi
 129                 done
 130
 131                 clearSnapshot $curr_time
 132
 133                 exit_code=$?
 134                 if [ $exit_code -ne 0 ]; then
 135                   rm -rf /backup/temp
 136                   echo "Backup Failed!!!"
 137                   echo "Failed" > /backup/backup.log
 138                   exit 0
 139                 fi
 140
 141                 mv /backup/temp /backup/backup-${curr_time}
 142                 echo "Success" > /backup/backup.log
 143                 echo "Cassandra Backup Succeeded"
 144               env:
 145               - name: NAMESPACE
 146                 valueFrom:
 147                   fieldRef:
 148                     apiVersion: v1
 149                     fieldPath: metadata.namespace
 150               volumeMounts:
 151               {{- range $i := until (int .Values.replicaCount)}}
 152               - mountPath: /onap-data/cassandra-{{ $i }}
 153                 name: data-dir-{{ $i }}
 154               {{- end }}
 155               - mountPath: /backup
 156                 name: backup-dir
 157               - name: scripts
 158                 mountPath: /app/restore.sh
 159                 subPath: restore.sh
 160               - name: scripts
 161                 mountPath: /app/exec.py
 162                 subPath: exec.py
 163           containers:
 164             - name: cassandra-backup-validate
 165               image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.image }}
 166               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 167               command:
 168               - /bin/bash
 169               - -c
 170               - |
 171                 remove_dir(){
 172                   dirToRemove=$1
 173                   rm -rf $dirToRemove
 174                 }
 175
 176                 backup_result=`cat /backup/backup.log`
 177                 rm -rf /backup/backup.log
 178
 179                 if [ "$backup_result" == "Failed" ]; then
 180                   echo "Backup Failed!!! So Validation Failed!!!";
 181                   exit 0
 182                 fi
 183
 184                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 185                 chown -R cassandra.cassandra $target_dir
 186                 {{- $root := . -}}
 187                 {{ range $i, $e := until (int .Values.replicaCount) }}
 188                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 189                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 190                   if [ $dbSize -lt $minDbSize ]; then
 191                     remove_dir $target_dir
 192                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 193                     exit 0
 194                   fi
 195                   rm -rf /var/lib/cassandra/*
 196                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 197                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 198                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 199                   CASS_PID=$!
 200                   sleep 45
 201
 202                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 203                       d=$(echo $d | sed 's:/*$::')
 204                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 205                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 206                         echo "Verifying the data for  $keyspace_name "
 207                         nodetool verify -e $keyspace_name
 208                         ret=$?
 209                         if [ $ret -ne 0 ]; then
 210                           remove_dir $target_dir
 211                           echo "Validation Failed!!!"
 212                           exit 0
 213                         fi
 214                       fi
 215                   done
 216                   kill -9 $CASS_PID
 217                 {{- end }}
 218                 echo "Validation Successful!!!"
 219                 cd /backup
 220                 totalFiles=`ls -t | grep "backup-" | wc -l`
 221                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 222                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 223                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 224                 fi
 225               env:
 226               - name: CASSANDRA_CLUSTER_NAME
 227                 value: {{ .Values.config.clusterName }}
 228               - name: MAX_HEAP_SIZE
 229                 value: {{ .Values.config.heap.max }}
 230               - name: HEAP_NEWSIZE
 231                 value: {{ .Values.config.heap.min }}
 232               - name: HOST_IP
 233                 valueFrom:
 234                   fieldRef:
 235                     fieldPath: status.podIP
 236               volumeMounts:
 237               - name: backup-dir
 238                 mountPath: /backup
 239           volumes:
 240           - name: scripts
 241             configMap:
 242               name: {{ include "common.fullname" . }}-configmap
 243               defaultMode: 0755
 244           {{- range $i := until (int .Values.replicaCount)}}
 245           - name: data-dir-{{ $i }}
 246             persistentVolumeClaim:
 247               claimName: {{ include "common.fullname" . }}-data-{{ $i }}
 248           {{- end }}
 249           - name: backup-dir
 250             persistentVolumeClaim:
 251               claimName: {{ include "common.fullname" . }}-backup-data
 252 {{- end -}}
 253 {{- end -}}