kud/tests/cmk.sh

   1 #!/bin/bash
   2 ENV=$(kubectl get nodes --all-namespaces | wc -l)
   3 if [[ $ENV -gt 2 ]]; then
   4     COMPUTE_NODE=$(kubectl get nodes --all-namespaces | grep -v master | awk 'NR==2{print $1}')
   5 else
   6     COMPUTE_NODE=$(kubectl get nodes --all-namespaces | grep master | awk 'NR==1{print $1}')
   7 fi
   8 cases=("exclusive ${COMPUTE_NODE} 1" "shared ${COMPUTE_NODE} -1")
   9 case=(null null 0)
  10 num=${#cases[*]}
  11 POOL=0
  12 NODE=1
  13 CORE=2
  14 DIR=/tmp
  15 pod_name=cmk-test-pod
  16
  17 function wait_for_pod_up {
  18     status_phase=""
  19     while [[ $status_phase != "Running" ]]; do
  20         new_phase=$(kubectl get pods "$@" | awk 'NR==2{print $3}')
  21         if [[ $new_phase != $status_phase ]]; then
  22             echo "$(date +%H:%M:%S) - $@ : $new_phase"
  23             status_phase=$new_phase
  24         fi
  25         if [[ $new_phase == "Running" ]]; then
  26             echo "Pod $@ is up and running.."
  27         fi
  28         if [[ $new_phase == "Err"* ]]; then
  29             exit 1
  30         fi
  31     done
  32 }
  33
  34
  35 function start_nginx_pod {
  36     kubectl delete deployment -n default nginx --ignore-not-found=true
  37     kubectl create deployment nginx --image=nginx
  38     sleep 2
  39     nginx_pod=$(kubectl get pods --all-namespaces| grep nginx | awk 'NR==1{print $2}')
  40     wait_for_pod_up $nginx_pod
  41     kubectl delete deployment -n default nginx --ignore-not-found=true
  42     pod_status="Running"
  43     until [[ $pod_status == "" ]]; do
  44         pod_status=$(kubectl get pod $nginx_pod --ignore-not-found=true | awk 'NR==2{print $3}')
  45     done
  46 }
  47
  48 rm -f $DIR/$pod_name.yaml
  49 kubectl delete pod $pod_name --ignore-not-found=true --now --wait
  50 echo
  51 echo "env is $ENV"
  52 echo
  53 for ((i=0;i<$num;i++)); do
  54     inner_case=(${cases[$i]})
  55     num_inner=${#inner_case[*]}
  56     for ((j=0;j<$num_inner;j++)); do
  57         case[$j]=${inner_case[$j]}
  58     done
  59     echo "##################################"
  60     if [ "${case[$POOL]}" == "exclusive" ]; then
  61         echo "TC: to allocate ${case[$CORE]} CPU(s) from pool of ${case[$POOL]} on node of ${case[$NODE]}"
  62         TOTAL=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.${case[$POOL]} | jq .cpuLists | awk -F  '{' '{print $(NF)}' | awk -F  '}' '{print $(NF)}' | awk -F  ',' '{print $(NF)}' | grep "\"tasks\": \[" | wc -l)
  63             echo "ready to generate yaml"
  64 cat << EOF > $DIR/$pod_name.yaml
  65     apiVersion: v1
  66     kind: Pod
  67     metadata:
  68       labels:
  69         app: cmk-test-pod
  70       name: cmk-test-pod
  71     spec:
  72       nodeName: ${case[$NODE]}
  73       containers:
  74       - args:
  75         - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=exclusive sleep -- 3900"
  76         command:
  77         - "sh"
  78         - "-c"
  79         env:
  80         - name: CMK_PROC_FS
  81           value: "/host/proc"
  82         - name: CMK_NUM_CORES
  83           value: "${case[$CORE]}"
  84         image: ubuntu:18.04
  85         imagePullPolicy: "IfNotPresent"
  86         name: cmk-test
  87         volumeMounts:
  88         - mountPath: "/host/proc"
  89           name: host-proc
  90         - mountPath: "/opt/bin"
  91           name: cmk-install-dir
  92         - mountPath: "/etc/cmk"
  93           name: cmk-conf-dir
  94       restartPolicy: Never
  95       volumes:
  96       - hostPath:
  97           path: "/opt/bin"
  98         name: cmk-install-dir
  99       - hostPath:
 100           path: "/proc"
 101         name: host-proc
 102       - hostPath:
 103           path: "/etc/cmk"
 104         name: cmk-conf-dir
 105 EOF
 106
 107         echo "ready to create pod"
 108         kubectl create -f $DIR/$pod_name.yaml --validate=false
 109         sleep 2
 110         echo "waiting for pod up"
 111         for pod in $pod_name; do
 112             wait_for_pod_up $pod
 113         done
 114         echo "waiting for CPU allocation finished ..."
 115         rest=$TOTAL
 116         until [[ $TOTAL -gt $rest ]]; do
 117             rest=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.exclusive | jq .cpuLists | awk -F  '{' '{print $(NF)}' | awk -F  '}' '{print $(NF)}' | awk -F  ',' '{print $(NF)}' | grep "\"tasks\": \[\]" | wc -l)
 118         done
 119         let allocated=`expr $TOTAL - $rest`
 120         echo "The allocated CPU amount is:" $allocated
 121         echo "deploy a nginx pod"
 122         start_nginx_pod
 123         if [[ $allocated == ${case[$CORE]} ]]; then
 124             echo "CPU was allocated as expected, TC passed !!"
 125         else
 126             echo "failed to allocate CPU, TC failed !!"
 127         fi
 128         rm -f $DIR/$pod_name.yaml
 129         echo "ready to delete pod"
 130         kubectl delete pod $pod_name --ignore-not-found=true --now --wait
 131         echo "Pod was deleted"
 132         echo "##################################"
 133         echo
 134         echo
 135     else
 136         echo "TC: to allocate CPU(s) from pool of ${case[$POOL]} on node of ${case[$NODE]}"
 137         echo "ready to generate yaml"
 138 cat << EOF > $DIR/$pod_name.yaml
 139 apiVersion: v1
 140 kind: Pod
 141 metadata:
 142   labels:
 143     app: cmk-test-pod
 144   name: cmk-test-pod
 145 spec:
 146   nodeName: ${case[$NODE]}
 147   containers:
 148   - name: share1
 149     args:
 150     - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=shared sleep -- 3900"
 151     command:
 152     - "sh"
 153     - "-c"
 154     env:
 155     - name: CMK_PROC_FS
 156       value: "/host/proc"
 157     - name: CMK_NUM_CORES
 158       value: "3"
 159     image: ubuntu:18.10
 160     imagePullPolicy: "IfNotPresent"
 161     volumeMounts:
 162     - mountPath: "/host/proc"
 163       name: host-proc
 164     - mountPath: "/opt/bin"
 165       name: cmk-install-dir
 166     - mountPath: "/etc/cmk"
 167       name: cmk-conf-dir
 168   - name: share2
 169     args:
 170     - "/opt/bin/cmk isolate --conf-dir=/etc/cmk --pool=shared sleep -- 3300"
 171     command:
 172     - "sh"
 173     - "-c"
 174     env:
 175     - name: CMK_PROC_FS
 176       value: "/host/proc"
 177     - name: CMK_NUM_CORES
 178       value: "3"
 179     image: ubuntu:18.10
 180     imagePullPolicy: "IfNotPresent"
 181     volumeMounts:
 182     - mountPath: "/host/proc"
 183       name: host-proc
 184     - mountPath: "/opt/bin"
 185       name: cmk-install-dir
 186     - mountPath: "/etc/cmk"
 187       name: cmk-conf-dir
 188   volumes:
 189   - hostPath:
 190       path: "/opt/bin"
 191     name: cmk-install-dir
 192   - hostPath:
 193       path: "/proc"
 194     name: host-proc
 195   - hostPath:
 196       path: "/etc/cmk"
 197     name: cmk-conf-dir
 198 EOF
 199
 200         echo "ready to create pod"
 201         kubectl create -f $DIR/$pod_name.yaml --validate=false
 202         sleep 2
 203         echo "waiting for pod up"
 204         for pod in $pod_name; do
 205             wait_for_pod_up $pod
 206         done
 207         echo "waiting for CPU allocation finished ..."
 208         rest=0
 209         timeout=0
 210         until [ $rest == 2 -o $timeout == 180 ]; do
 211             rest=$(kubectl get cmk-nodereport ${case[$NODE]} -o json | jq .spec.report.description.pools.shared | jq .cpuLists | awk -F  '{' '{print $(NF)}' | awk -F  '}' '{print $(NF)}' | grep -v "cpus" | grep "  "| grep -v "tasks"| grep -v "\]" | wc -l)
 212             sleep -- 1
 213             let timeout++
 214         done
 215         echo "The CPU allocated in shared pool for 2 tasks"
 216         echo "deploy a nginx pod"
 217         start_nginx_pod
 218         if [[ $rest == 2 ]]; then
 219             echo "CPU was allocated as expected, TC passed !!"
 220         else
 221             echo "failed to allocate CPU, TC failed !!"
 222         fi
 223         rm -f $DIR/$pod_name.yaml
 224         echo "ready to delete pod"
 225         kubectl delete pod $pod_name --ignore-not-found=true --now --wait
 226         echo "Pod was deleted"
 227         echo "##################################"
 228         echo
 229         echo
 230     fi
 231 done