kubernetes/common/cassandra/templates/backup/cronjob.yaml

   1 {{/*
   2 # Copyright © 2019 Amdocs, Bell Canada
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #       http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 */}}
  16 {{- if .Values.backup.enabled }}
  17 {{- if .Values.global.cassandra.useOperator }}
  18 {{ else }}
  19 apiVersion: batch/v1beta1
  20 kind: CronJob
  21 metadata:
  22   name: {{ include "common.fullname" . }}-backup
  23   namespace: {{ include "common.namespace" . }}
  24   labels: {{- include "common.labels" . | nindent 4 }}
  25 spec:
  26   schedule: {{ .Values.backup.cron | quote }}
  27   concurrencyPolicy: Forbid
  28   startingDeadlineSeconds: 120
  29   jobTemplate:
  30     spec:
  31       template:
  32         metadata:
  33           labels: {{- include "common.labels" . | nindent 12 }}
  34           annotations: {{- include "common.tplValue" (dict "value" .Values.podAnnotations "context" .) | nindent 12 }}
  35           name: {{ include "common.name" . }}
  36         spec:
  37           restartPolicy: Never
  38           initContainers:
  39             - command:
  40               - /app/ready.py
  41               args:
  42               - --app-name
  43               - {{ include "common.name" . }}
  44               env:
  45               - name: NAMESPACE
  46                 valueFrom:
  47                   fieldRef:
  48                     apiVersion: v1
  49                     fieldPath: metadata.namespace
  50               image: {{ include "repositoryGenerator.image.readiness" . }}
  51               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  52               name: {{ include "common.name" . }}-readiness
  53               resources:
  54                 limits:
  55                   cpu: "100m"
  56                   memory: "500Mi"
  57                 requests:
  58                   cpu: "3m"
  59                   memory: "20Mi"
  60             - name: "cassandra-backup-init"
  61               image: {{ include "repositoryGenerator.image.readiness" . }}
  62               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
  63               resources:
  64                 limits:
  65                   cpu: "100m"
  66                   memory: "500Mi"
  67                 requests:
  68                   cpu: "3m"
  69                   memory: "20Mi"
  70               command:
  71               - /bin/bash
  72               - -c
  73               - |
  74                 clearSnapshot(){
  75                   curr_time=$1
  76                   echo "Clearing snapshots!!!"
  77                   command="nodetool clearsnapshot -t $curr_time"
  78                   /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  79                 }
  80                 {{ $root := . }}
  81                 curr_time=`date +%s`
  82                 pids=""
  83                 set -x
  84
  85                 echo "Copying data"
  86                 {{ range $i, $e := until (int .Values.replicaCount) }}
  87                   target_dir=/backup/temp/cassandra-{{ $i }}
  88                   mkdir -p $target_dir
  89                   cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
  90                 {{- end }}
  91
  92                 echo "Executing cleanup!!"
  93                 command="nodetool cleanup"
  94                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  95                 echo "Cleaned Node!! Backing up database now!!!"
  96
  97                 command="nodetool snapshot -t $curr_time"
  98                 /app/exec.py -p "{{ include "common.name" . }}" -c "$command"
  99                 retCode=$?
 100                 if [ $retCode -ne 0 ]; then
 101                   echo "Backup Failed!!!"
 102                   rm -rf /backup/temp
 103                   clearSnapshot $curr_time
 104                   echo "Failed" > /backup/backup.log
 105                   exit 0
 106                 fi
 107
 108                 backup_dir=/backup/temp
 109                 {{ range $i, $e := until (int .Values.replicaCount) }}
 110                   for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
 111                     d=$(echo $d | sed 's:/*$::')
 112                     keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 113                     if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 114                       /app/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
 115                       pids="$pids $!"
 116                     fi
 117                   done
 118                 {{- end }}
 119
 120                 for p in $pids; do
 121                   wait $p
 122                   if [ $? -ne 0 ]; then
 123                     rm -rf /backup/temp
 124                     echo "Creation of Backup Failed!!!"
 125                     clearSnapshot $curr_time
 126                     echo "Failed" > /backup/backup.log
 127                     exit 0
 128                   fi
 129                 done
 130
 131                 clearSnapshot $curr_time
 132
 133                 exit_code=$?
 134                 if [ $exit_code -ne 0 ]; then
 135                   rm -rf /backup/temp
 136                   echo "Backup Failed!!!"
 137                   echo "Failed" > /backup/backup.log
 138                   exit 0
 139                 fi
 140
 141                 mv /backup/temp /backup/backup-${curr_time}
 142                 echo "Success" > /backup/backup.log
 143                 echo "Cassandra Backup Succeeded"
 144               env:
 145               - name: NAMESPACE
 146                 valueFrom:
 147                   fieldRef:
 148                     apiVersion: v1
 149                     fieldPath: metadata.namespace
 150               volumeMounts:
 151               - mountPath: /etc/localtime
 152                 name: localtime
 153                 readOnly: true
 154               {{- range $i := until (int .Values.replicaCount)}}
 155               - mountPath: /onap-data/cassandra-{{ $i }}
 156                 name: data-dir-{{ $i }}
 157               {{- end }}
 158               - mountPath: /backup
 159                 name: backup-dir
 160               - name: scripts
 161                 mountPath: /app/restore.sh
 162                 subPath: restore.sh
 163               - name: scripts
 164                 mountPath: /app/exec.py
 165                 subPath: exec.py
 166           containers:
 167             - name: cassandra-backup-validate
 168               image: {{ include "repositoryGenerator.dockerHubRepository" . }}/{{ .Values.image }}
 169               imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
 170               command:
 171               - /bin/bash
 172               - -c
 173               - |
 174                 remove_dir(){
 175                   dirToRemove=$1
 176                   rm -rf $dirToRemove
 177                 }
 178
 179                 backup_result=`cat /backup/backup.log`
 180                 rm -rf /backup/backup.log
 181
 182                 if [ "$backup_result" == "Failed" ]; then
 183                   echo "Backup Failed!!! So Validation Failed!!!";
 184                   exit 0
 185                 fi
 186
 187                 target_dir=$(ls -td -- /backup/*/ | head -n 1)
 188                 chown -R cassandra.cassandra $target_dir
 189                 {{- $root := . -}}
 190                 {{ range $i, $e := until (int .Values.replicaCount) }}
 191                   dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
 192                   minDbSize={{ (int $root.Values.backup.dbSize) }}
 193                   if [ $dbSize -lt $minDbSize ]; then
 194                     remove_dir $target_dir
 195                     echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
 196                     exit 0
 197                   fi
 198                   rm -rf /var/lib/cassandra/*
 199                   cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
 200                   export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
 201                   /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
 202                   CASS_PID=$!
 203                   sleep 45
 204
 205                   for d in $target_dir/cassandra-{{ $i }}/data/*/; do
 206                       d=$(echo $d | sed 's:/*$::')
 207                       keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
 208                       if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
 209                         echo "Verifying the data for  $keyspace_name "
 210                         nodetool verify -e $keyspace_name
 211                         ret=$?
 212                         if [ $ret -ne 0 ]; then
 213                           remove_dir $target_dir
 214                           echo "Validation Failed!!!"
 215                           exit 0
 216                         fi
 217                       fi
 218                   done
 219                   kill -9 $CASS_PID
 220                 {{- end }}
 221                 echo "Validation Successful!!!"
 222                 cd /backup
 223                 totalFiles=`ls -t | grep "backup-" | wc -l`
 224                 if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
 225                   filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
 226                   ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
 227                 fi
 228               env:
 229               - name: CASSANDRA_CLUSTER_NAME
 230                 value: {{ .Values.config.clusterName }}
 231               - name: MAX_HEAP_SIZE
 232                 value: {{ .Values.config.heap.max }}
 233               - name: HEAP_NEWSIZE
 234                 value: {{ .Values.config.heap.min }}
 235               - name: HOST_IP
 236                 valueFrom:
 237                   fieldRef:
 238                     fieldPath: status.podIP
 239               volumeMounts:
 240               - name: backup-dir
 241                 mountPath: /backup
 242               - name: localtime
 243                 mountPath: /etc/localtime
 244                 readOnly: true
 245           volumes:
 246           - name: localtime
 247             hostPath:
 248               path: /etc/localtime
 249           - name: scripts
 250             configMap:
 251               name: {{ include "common.fullname" . }}-configmap
 252               defaultMode: 0755
 253           {{- range $i := until (int .Values.replicaCount)}}
 254           - name: data-dir-{{ $i }}
 255             persistentVolumeClaim:
 256               claimName: {{ include "common.fullname" . }}-data-{{ $i }}
 257           {{- end }}
 258           - name: backup-dir
 259             persistentVolumeClaim:
 260               claimName: {{ include "common.fullname" . }}-backup-data
 261 {{- end -}}
 262 {{- end -}}