--- /dev/null
+#!/usr/bin/python
+import getopt
+import logging
+import os
+import sys
+import time
+
+from kubernetes import config
+from kubernetes.client import Configuration
+from kubernetes.client.apis import core_v1_api
+from kubernetes.client.rest import ApiException
+from kubernetes.stream import stream
+
+from kubernetes import client
+
+# extract env variables.
+namespace = os.environ['NAMESPACE']
+cert = os.environ['CERT']
+host = os.environ['KUBERNETES_SERVICE_HOST']
+token_path = os.environ['TOKEN']
+
+with open(token_path, 'r') as token_file:
+ token = token_file.read().replace('\n', '')
+
+# setup logging
+log = logging.getLogger(__name__)
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
+handler.setLevel(logging.INFO)
+log.addHandler(handler)
+log.setLevel(logging.INFO)
+
+configuration = client.Configuration()
+configuration.host = "https://" + host
+configuration.ssl_ca_cert = cert
+configuration.api_key['authorization'] = token
+configuration.api_key_prefix['authorization'] = 'Bearer'
+configuration.assert_hostname = False
+coreV1Api = client.CoreV1Api(client.ApiClient(configuration))
+api_instance = client.CoreV1Api(client.ApiClient(configuration))
+
+def run_command( pod_name, command ):
+ try:
+ exec_command = [
+ '/bin/sh',
+ '-c',
+ command]
+ resp = stream(api_instance.connect_get_namespaced_pod_exec, pod_name, namespace,
+ command=exec_command,
+ stderr=True, stdin=False,
+ stdout=True, tty=False)
+ except ApiException as e:
+ print("Exception when calling CoreV1Api->connect_get_namespaced_pod_exec: %s\n" % e)
+ return False
+ print(resp)
+ return True
+
+def find_pod(container_name,command,pods):
+ ready = False
+ try:
+ response = coreV1Api.list_namespaced_pod(namespace=namespace, watch=False)
+ for i in response.items:
+ # container_statuses can be None, which is non-iterable.
+ if i.status.container_statuses is None:
+ continue
+ for s in i.status.container_statuses:
+ if s.name == container_name:
+ if pods == True:
+ print (i.metadata.name)
+ else:
+ ready = run_command(i.metadata.name,command)
+ else:
+ continue
+ except Exception as e:
+ log.error("Exception when calling list_namespaced_pod: %s\n" % e)
+
+ return ready
+
+
+DESCRIPTION = "Kubernetes container readiness check utility"
+USAGE = "Usage: ready.py [-t <timeout>] -c <container_name> [-c <container_name> ...]\n" \
+ "where\n" \
+ "<container_name> - name of the container to wait for\n"
+
+def main(argv):
+ pods = False
+ command = ""
+ container_name = ""
+ try:
+ opts, args = getopt.getopt(argv, "ghp:c:", ["pod-container-name=", "command=", "help","getpods"])
+ for opt, arg in opts:
+ if opt in ("-h", "--help"):
+ print("%s\n\n%s" % (DESCRIPTION, USAGE))
+ sys.exit()
+ elif opt in ("-p", "--pod-container-name"):
+ container_name = arg
+ elif opt in ("-c", "--command"):
+ command = arg
+ elif opt in ("-g", "--getpods"):
+ pods = True
+ except (getopt.GetoptError, ValueError) as e:
+ print("Error parsing input parameters: %s\n" % e)
+ print(USAGE)
+ sys.exit(2)
+ if container_name.__len__() == 0:
+ print("Missing required input parameter(s)\n")
+ print(USAGE)
+ sys.exit(2)
+
+ if pods == False:
+ if command.__len__() == 0:
+ print("Missing required input parameter(s)\n")
+ print(USAGE)
+ sys.exit(2)
+ ready = find_pod(container_name,command,pods)
+ if ready == False:
+ sys.exit(2)
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
+
+
--- /dev/null
+#!/bin/bash
+
+# Initialize variables
+ss_dir=""
+base_db_dir=""
+ss_name=""
+ss="snapshots"
+me=`basename $0`
+
+function find_target_table_name()
+{
+ dest_path=$1
+ keyspace_name=$2
+ src_table_name=$3
+ find_in_dir=$dest_path/$keyspace_name
+ tname_without_uuid=$(echo $src_table_name | cut -d '-' -f 1)
+ dest_table_name=$(ls -td -- $find_in_dir/$tname_without_uuid-* | head -n 1 | rev | cut -d'/' -f1 | rev)
+ printf $dest_table_name
+}
+
+function print_usage()
+{
+ echo "NAME"
+ echo " Script to restore Cassandra database from Nuvo/Cain snapshot"
+ echo "SYNOPSIS"
+ echo " $me [--help|-h] [--base_db_dir|-b] [--snapshot_dir|-s] [--keyspace|-k] [--tag|-t]"
+ echo " MUST OPTIONS: base_db_dir, snapshot_dir, keyspace_name"
+ echo "DESCRIPTION"
+ echo " --base_db_dir, -b"
+ echo " Location of running Cassandra database"
+ echo " --snapshot_dir, -s"
+ echo " Snapshot location of Cassandra database taken by Nuvo/Cain"
+ echo " --keyspace, -k"
+ echo " Name of the keyspace to restore"
+ echo "EXAMPLE"
+ echo " $me -b /var/lib/cassandra/data -s /root/data.ss -k DISCOVERY_SERVER -t 1234567"
+ exit
+}
+if [ $# -eq 0 ]
+then
+ print_usage
+fi
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+shift
+
+case $key in
+ -h|--help)
+ print_usage
+ ;;
+ -b|--base_db_dir)
+ base_db_dir="$1"
+ shift
+ ;;
+ -s|--snapshot_dir)
+ ss_dir="$1"
+ shift
+ ;;
+ -k|--keyspace)
+ keyspace_name="$1"
+ ;;
+ -t|--tag)
+ tag_name="$1"
+ ;;
+ --default)
+ DEFAULT=YES
+ shift
+ ;;
+ *)
+ # unknown option
+ ;;
+esac
+done
+
+# Validate inputs
+if [ "$base_db_dir" == "" ] || [ "$ss_dir" == "" ] || [ "$keyspace_name" == "" ]
+then
+ echo ""
+ echo ">>>>>>>>>>Not all inputs provided, please check usage >>>>>>>>>>"
+ echo ""
+ print_usage
+fi
+
+# Remove commit logs from current data dir
+#/var/lib/cassandra/commitlog/CommitLog*.log
+find $base_db_dir/../ -name "CommitLog*.log" -delete
+
+# Remove *.db from current data dir excluding skipped keyspaces
+find $base_db_dir/$keyspace_name -name "*.db" -delete
+
+# Copy snapshots to data dir
+echo "----------db files in snapshots--------------"
+dirs_to_be_restored=`ls $ss_dir`
+for i in ${dirs_to_be_restored}
+do
+ src_path=$ss_dir/$i/snapshots/$tag_name
+ # Find the destination
+ table_name=$i
+ dest_table=$(find_target_table_name $base_db_dir $keyspace_name $table_name)
+ dest_path=$base_db_dir/$keyspace_name/$dest_table
+ # Create keyspace/table directory if not exists
+ #if [ ! -d "$dest_path" ]; then
+ # mkdir -p $dest_path
+ #fi
+ db_files=$(ls $src_path/*.db 2> /dev/null | wc -l)
+ if [ $db_files -ne 0 ]
+ then
+ cp $src_path/*.db $dest_path
+ if [ $? -ne 0 ]
+ then
+ echo "=====ERROR: Unable to restore $src_path/*.db to $dest_path====="
+ exit 1
+ fi
+ echo "=======check $dest_path ==============="
+ ls $dest_path
+ fi
+done
--- /dev/null
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "common.fullname" . }}-configmap
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+data:
+{{ tpl (.Files.Glob "resources/restore.sh").AsConfig . | indent 2 }}
+{{ tpl (.Files.Glob "resources/exec.py").AsConfig . | indent 2 }}
+{{- end -}}
--- /dev/null
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+ name: {{ include "common.fullname" . }}-backup
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.fullname" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+spec:
+ schedule: {{ .Values.backup.cron | quote }}
+ concurrencyPolicy: Forbid
+ startingDeadlineSeconds: 120
+ jobTemplate:
+ spec:
+ template:
+ spec:
+ restartPolicy: Never
+ initContainers:
+ - command:
+ - /root/ready.py
+ args:
+ - --container-name
+ - {{ include "common.name" . }}
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+ imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+ name: {{ include "common.name" . }}-readiness
+ - name: "cassandra-backup-init"
+ image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
+ imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+ command:
+ - /bin/bash
+ - -c
+ - |
+ clearSnapshot(){
+ curr_time=$1
+ echo "Clearing snapshots!!!"
+ command="nodetool clearsnapshot -t $curr_time"
+ /root/exec.py -p "cassandra" -c "$command"
+ }
+ {{ $root := . }}
+ curr_time=`date +%s`
+ pids=""
+ set -x
+
+ echo "Copying data"
+ {{ range $i, $e := until (int .Values.replicaCount) }}
+ target_dir=/backup/temp/cassandra-{{ $i }}
+ mkdir -p $target_dir
+ cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
+ {{- end }}
+
+ echo "Executing cleanup!!"
+ command="nodetool cleanup"
+ /root/exec.py -p "cassandra" -c "$command"
+ echo "Cleaned Node!! Backing up database now!!!"
+
+ command="nodetool snapshot -t $curr_time"
+ /root/exec.py -p "cassandra" -c "$command"
+ retCode=$?
+ if [ $retCode -ne 0 ]; then
+ echo "Backup Failed!!!"
+ rm -rf /backup/temp
+ clearSnapshot $curr_time
+ echo "Failed" > /backup/backup.log
+ exit 0
+ fi
+
+ backup_dir=/backup/temp
+ {{ range $i, $e := until (int .Values.replicaCount) }}
+ for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
+ d=$(echo $d | sed 's:/*$::')
+ keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
+ if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
+ /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
+ pids="$pids $!"
+ fi
+ done
+ {{- end }}
+
+ for p in $pids; do
+ wait $p
+ if [ $? -ne 0 ]; then
+ rm -rf /backup/temp
+ echo "Creation of Backup Failed!!!"
+ clearSnapshot $curr_time
+ echo "Failed" > /backup/backup.log
+ exit 0
+ fi
+ done
+
+ clearSnapshot $curr_time
+
+ exit_code=$?
+ if [ $exit_code -ne 0 ]; then
+ rm -rf /backup/temp
+ echo "Backup Failed!!!"
+ echo "Failed" > /backup/backup.log
+ exit 0
+ fi
+
+ mv /backup/temp /backup/backup-${curr_time}
+ echo "Success" > /backup/backup.log
+ echo "Cassandra Backup Succeeded"
+ env:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - mountPath: /etc/localtime
+ name: localtime
+ readOnly: true
+ - mountPath: /onap-data
+ name: data-dir
+ - mountPath: /backup
+ name: backup-dir
+ - name: scripts
+ mountPath: /root/restore.sh
+ subPath: restore.sh
+ - name: scripts
+ mountPath: /root/exec.py
+ subPath: exec.py
+ containers:
+ - name: cassandra-backup-validate
+ image: "{{ .Values.image }}"
+ imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
+ command:
+ - /bin/bash
+ - -c
+ - |
+ remove_dir(){
+ dirToRemove=$1
+ rm -rf $dirToRemove
+ }
+
+ backup_result=`cat /backup/backup.log`
+ rm -rf /backup/backup.log
+
+ if [ "$backup_result" == "Failed" ]; then
+ echo "Backup Failed!!! So Validation Failed!!!";
+ exit 0
+ fi
+
+ target_dir=$(ls -td -- /backup/*/ | head -n 1)
+ chown -R cassandra.cassandra $target_dir
+ {{- $root := . -}}
+ {{ range $i, $e := until (int .Values.replicaCount) }}
+ dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
+ minDbSize={{ (int $root.Values.backup.dbSize) }}
+ if [ $dbSize -lt $minDbSize ]; then
+ remove_dir $target_dir
+ echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
+ exit 0
+ fi
+ rm -rf /var/lib/cassandra/*
+ cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
+ export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
+ /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
+ CASS_PID=$!
+ sleep 45
+
+ for d in $target_dir/cassandra-{{ $i }}/data/*/; do
+ d=$(echo $d | sed 's:/*$::')
+ keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
+ if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
+ echo "Verifying the data for $keyspace_name "
+ nodetool verify -e $keyspace_name
+ ret=$?
+ if [ $ret -ne 0 ]; then
+ remove_dir $target_dir
+ echo "Validation Failed!!!"
+ exit 0
+ fi
+ fi
+ done
+ kill -9 $CASS_PID
+ {{- end }}
+ echo "Validation Successful!!!"
+ cd /backup
+ totalFiles=`ls -t | grep "backup-" | wc -l`
+ if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
+ filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
+ ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
+ fi
+ env:
+ - name: CASSANDRA_CLUSTER_NAME
+ value: {{ .Values.config.clusterName }}
+ - name: MAX_HEAP_SIZE
+ value: {{ .Values.config.heap.max }}
+ - name: HEAP_NEWSIZE
+ value: {{ .Values.config.heap.min }}
+ - name: HOST_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ volumeMounts:
+ - name: backup-dir
+ mountPath: /backup
+ - name: localtime
+ mountPath: /etc/localtime
+ readOnly: true
+ volumes:
+ - name: localtime
+ hostPath:
+ path: /etc/localtime
+ - name: scripts
+ configMap:
+ name: {{ include "common.fullname" $ }}-configmap
+ defaultMode: 0755
+ - name: data-dir
+ persistentVolumeClaim:
+ claimName: {{ include "common.fullname" . }}-db-data
+ - name: backup-dir
+ persistentVolumeClaim:
+ claimName: {{ include "common.fullname" . }}-backup-data
+{{- end -}}
+
--- /dev/null
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada, AT&T
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+{{ if .Values.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ include "common.fullname" . }}-db-data
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ heritage: {{ .Release.Service }}
+ name: {{ include "common.fullname" . }}-db-data
+spec:
+ capacity:
+ storage: {{ .Values.persistence.size }}
+ accessModes:
+ - {{ .Values.persistence.accessMode }}
+ hostPath:
+ path: {{ .Values.global.persistence.mountPath | default .Values.persistence.mountPath }}/{{ .Release.Name }}
+ persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }}
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ include "common.fullname" . }}-backup-data
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ heritage: {{ .Release.Service }}
+ name: {{ include "common.fullname" . }}-backup-data
+spec:
+ capacity:
+ storage: {{ .Values.persistence.size }}
+ accessModes:
+ - {{ .Values.persistence.accessMode }}
+ hostPath:
+ path: {{ .Values.global.persistence.backup.mountPath | default .Values.persistence.backup.mountPath }}/{{ include "common.namespace" $ }}/{{ include "common.fullname" $ }}
+ persistentVolumeReclaimPolicy: {{ .Values.persistence.volumeReclaimPolicy }}
+{{ end }}
+{{- end -}}
+
--- /dev/null
+{{/*
+# Copyright © 2019 Amdocs, Bell Canada
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{- if .Values.backup.enabled }}
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) -}}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: {{ include "common.fullname" . }}-db-data
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}-backup
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ release: "{{ .Release.Name }}"
+ heritage: "{{ .Release.Service }}"
+{{- if .Values.persistence.annotations }}
+ annotations:
+{{ toYaml .Values.persistence.annotations | indent 4 }}
+{{- end }}
+spec:
+ selector:
+ matchLabels:
+ name: {{ include "common.fullname" . }}-db-data
+ accessModes:
+ - {{ .Values.persistence.accessMode }}
+ resources:
+ requests:
+ storage: {{ .Values.persistence.size }}
+{{- if .Values.persistence.storageClass }}
+{{- if (eq "-" .Values.persistence.storageClass) }}
+ storageClassName: ""
+{{- else }}
+ storageClassName: "{{ .Values.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+ name: {{ include "common.fullname" . }}-backup-data
+ namespace: {{ include "common.namespace" . }}
+ labels:
+ app: {{ include "common.name" . }}-backup
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ release: "{{ .Release.Name }}"
+ heritage: "{{ .Release.Service }}"
+{{- if .Values.persistence.annotations }}
+ annotations:
+{{ toYaml .Values.persistence.annotations | indent 4 }}
+{{- end }}
+spec:
+ selector:
+ matchLabels:
+ name: {{ include "common.fullname" . }}-backup-data
+ accessModes:
+ - {{ .Values.persistence.accessMode }}
+ resources:
+ requests:
+ storage: {{ .Values.persistence.size }}
+{{- if .Values.persistence.storageClass }}
+{{- if (eq "-" .Values.persistence.storageClass) }}
+ storageClassName: ""
+{{- else }}
+ storageClassName: "{{ .Values.persistence.storageClass }}"
+{{- end }}
+{{- end }}
+{{- end -}}
+{{- end -}}
# Declare variables to be passed into your templates.
global: # global defaults
nodePortPrefix: 302
-
+ persistence:
+ mountPath: /dockerdata-nfs
+ backup:
+ mountPath: /dockerdata-nfs/backup
+ repository: nexus3.onap.org:10001
+ readinessRepository: oomk8s
+ readinessImage: readiness-check:2.0.2
# application image
repository: nexus3.onap.org:10001
mountSubPath: cassandra
storageType: local
storageClass: ""
+ backup:
+ mountPath: /dockerdata-nfs/backup
configOverrides: {}
# requests:
# cpu: 2
# memory: 4Gi
+backup:
+ enabled: false
+ cron: "00 00 * * *"
+ retentionPeriod: 3
+ dbSize: 1
+ keyspacesToSkip:
+ - name: system_traces
+ - name: system_auth
+ - name: system_distributed
Code Review / oom.git / commitdiff