kud/hosting_providers/containerized/installer.sh

   1 #!/bin/bash
   2 #SPDX-license-identifier: Apache-2.0
   3 ##############################################################################
   4 # Copyright (c) 2018
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 ##############################################################################
  10
  11 set -o errexit
  12 set -o nounset
  13 set -o pipefail
  14 set -ex
  15
  16 INSTALLER_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")"
  17
  18 function install_prerequisites {
  19 #install package for docker images
  20     echo "Removing ppa for jonathonf/python-3.6"
  21     ls /etc/apt/sources.list.d/ || true
  22     find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true
  23     apt-get update
  24     apt-get install -y curl vim wget git \
  25         software-properties-common python-pip sudo gettext-base
  26     add-apt-repository -y ppa:longsleep/golang-backports
  27     apt-get update
  28     apt-get install -y golang-go rsync
  29 }
  30
  31 # _install_ansible() - Install and Configure Ansible program
  32 function _install_ansible {
  33     local version=$(grep "ansible_version" ${kud_playbooks}/kud-vars.yml |
  34         awk -F ': ' '{print $2}')
  35     mkdir -p /etc/ansible/
  36     pip install --no-cache-dir ansible==$version
  37 }
  38
  39 function install_kubespray {
  40     echo "Deploying kubernetes"
  41     version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \
  42         awk -F ': ' '{print $2}')
  43     local_release_dir=$(grep "local_release_dir" \
  44         $kud_inventory_folder/group_vars/k8s-cluster.yml | \
  45         awk -F "\"" '{print $2}')
  46     local tarball=v$version.tar.gz
  47     # install make to run mitogen target & unzip is mitogen playbook dependency
  48     apt-get install -y sshpass make unzip
  49     _install_ansible
  50     wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball
  51     tar -C $dest_folder -xzf $tarball
  52     chown -R root:root $dest_folder/kubespray-$version
  53     mkdir -p ${local_release_dir}/containers
  54     rm $tarball
  55
  56     pushd $dest_folder/kubespray-$version/
  57     pip install --no-cache-dir -r ./requirements.txt
  58     make mitogen
  59     popd
  60     rm -f $kud_inventory_folder/group_vars/all.yml 2> /dev/null
  61     if [[ -n "${verbose:-}" ]]; then
  62         echo "kube_log_level: 5" | tee \
  63             $kud_inventory_folder/group_vars/all.yml
  64     else
  65         echo "kube_log_level: 2" | tee \
  66             $kud_inventory_folder/group_vars/all.yml
  67     fi
  68     echo "kubeadm_enabled: true" | \
  69         tee --append $kud_inventory_folder/group_vars/all.yml
  70     if [[ -n "${http_proxy:-}" ]]; then
  71         echo "http_proxy: \"$http_proxy\"" | tee --append \
  72             $kud_inventory_folder/group_vars/all.yml
  73     fi
  74     if [[ -n "${https_proxy:-}" ]]; then
  75         echo "https_proxy: \"$https_proxy\"" | tee --append \
  76             $kud_inventory_folder/group_vars/all.yml
  77     fi
  78 }
  79
  80 # install_k8s() - Install Kubernetes using kubespray tool including Kata
  81 function install_k8s {
  82     local cluster_name=$1
  83     ansible-playbook $verbose -i \
  84         $kud_inventory $kud_playbooks/preconfigure-kubespray.yml \
  85         --become --become-user=root | \
  86         tee $cluster_log/setup-kubernetes.log
  87     if [ "$container_runtime" == "docker" ]; then
  88         echo "Docker will be used as the container runtime interface"
  89         ansible-playbook $verbose -i \
  90             $kud_inventory $dest_folder/kubespray-$version/cluster.yml \
  91             -e cluster_name=$cluster_name --become --become-user=root | \
  92             tee $cluster_log/setup-kubernetes.log
  93     elif [ "$container_runtime" == "containerd" ]; then
  94         echo "Containerd will be used as the container runtime interface"
  95         ansible-playbook $verbose -i \
  96             $kud_inventory $dest_folder/kubespray-$version/cluster.yml \
  97             -e $kud_kata_override_variables -e cluster_name=$cluster_name \
  98             --become --become-user=root | \
  99             tee $cluster_log/setup-kubernetes.log
 100         #Install Kata Containers in containerd scenario
 101         ansible-playbook $verbose -i \
 102             $kud_inventory -e "base_dest=$HOME" \
 103             $kud_playbooks/configure-kata.yml | \
 104             tee $cluster_log/setup-kata.log
 105     else
 106         echo "Only Docker or Containerd are supported container runtimes"
 107         exit 1
 108     fi
 109
 110     # Configure environment
 111     # Requires kubeconfig_localhost and kubectl_localhost to be true
 112     # in inventory/group_vars/k8s-cluster.yml
 113     mkdir -p $HOME/.kube
 114     cp $kud_inventory_folder/artifacts/admin.conf $HOME/.kube/config
 115     if !(which kubectl); then
 116         cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/
 117     fi
 118 }
 119
 120 # install_addons() - Install Kubenertes AddOns
 121 function install_addons {
 122     if [ ${1:+1} ]; then
 123         local plugins_name="$1"
 124         echo "additional addons plugins $1"
 125     else
 126         local plugins_name=""
 127         echo "no additional addons pluigns"
 128     fi
 129
 130     source /etc/environment
 131     echo "Installing Kubernetes AddOns"
 132     ansible-galaxy install $verbose -r \
 133         $kud_infra_folder/galaxy-requirements.yml --ignore-errors
 134
 135     ansible-playbook $verbose -i \
 136         $kud_inventory -e "base_dest=$HOME" $kud_playbooks/configure-kud.yml \
 137         | tee $cluster_log/setup-kud.log
 138
 139     kud_addons="${KUD_ADDONS:-} ${plugins_name}"
 140
 141     for addon in ${kud_addons}; do
 142         echo "Deploying $addon using configure-$addon.yml playbook.."
 143         ansible-playbook $verbose -i \
 144             $kud_inventory -e "base_dest=$HOME" \
 145             $kud_playbooks/configure-${addon}.yml | \
 146             tee $cluster_log/setup-${addon}.log
 147     done
 148
 149     echo "Run the test cases if testing_enabled is set to true."
 150     if [[ "${testing_enabled}" == "true" ]]; then
 151         failed_kud_tests=""
 152         # Run Kata test first if Kata was installed
 153         if [ "$container_runtime" == "containerd" ]; then
 154             #Install Kata webhook for test pods
 155             ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" \
 156                 -e "kata_webhook_runtimeclass=$kata_webhook_runtimeclass" \
 157                 $kud_playbooks/configure-kata-webhook.yml \
 158                 --become --become-user=root | \
 159                 sudo tee $cluster_log/setup-kata-webhook.log
 160             kata_webhook_deployed=true
 161             pushd $kud_tests
 162             bash kata.sh || failed_kud_tests="${failed_kud_tests} kata"
 163             popd
 164         fi
 165         #Run other plugin tests
 166         for addon in ${kud_addons}; do
 167             pushd $kud_tests
 168             bash ${addon}.sh || failed_kud_tests="${failed_kud_tests} ${addon}"
 169             case $addon in
 170                 "onap4k8s" )
 171                     echo "Test the onap4k8s plugin installation"
 172                     for functional_test in plugin_edgex plugin_fw plugin_eaa; do
 173                         bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}"
 174                     done
 175                     ;;
 176                 "emco" )
 177                     echo "Test the emco plugin installation"
 178                     # TODO plugin_fw_v2 requires virtlet and a patched multus to succeed
 179                     # for functional_test in plugin_fw_v2; do
 180                     #     bash ${functional_test}.sh --external || failed_kud_tests="${failed_kud_tests} ${functional_test}"
 181                     # done
 182                     ;;
 183             esac
 184             popd
 185         done
 186         # Remove Kata webhook if user didn't want it permanently installed
 187         if ! [ "$enable_kata_webhook" == "true" ] && [ "$kata_webhook_deployed" == "true" ]; then
 188             ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" \
 189                 -e "kata_webhook_runtimeclass=$kata_webhook_runtimeclass" \
 190                 $kud_playbooks/configure-kata-webhook-reset.yml \
 191                 --become --become-user=root | \
 192                 sudo tee $cluster_log/kata-webhook-reset.log
 193             kata_webhook_deployed=false
 194         fi
 195         if [[ ! -z "$failed_kud_tests" ]]; then
 196             echo "Test cases failed:${failed_kud_tests}"
 197             return 1
 198         fi
 199     fi
 200
 201     # Check if Kata webhook should be installed and isn't already installed
 202     if [ "$enable_kata_webhook" == "true" ] && ! [ "$kata_webhook_deployed" == "true" ]; then
 203         ansible-playbook $verbose -i $kud_inventory -e "base_dest=$HOME" \
 204             -e "kata_webhook_runtimeclass=$kata_webhook_runtimeclass" \
 205             $kud_playbooks/configure-kata-webhook.yml \
 206             --become --become-user=root | \
 207             sudo tee $cluster_log/setup-kata-webhook.log
 208     fi
 209
 210     echo "Add-ons deployment complete..."
 211 }
 212
 213 function master_ip {
 214     kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}'
 215 }
 216
 217 # Copy installation artifacts to be usable in host running Ansible
 218 function install_host_artifacts {
 219     local -r cluster_name=$1
 220     local -r host_dir="/opt/kud/multi-cluster"
 221     local -r host_addons_dir="${host_dir}/addons"
 222     local -r host_artifacts_dir="${host_dir}/${cluster_name}/artifacts"
 223
 224     for addon in cdi cdi-operator cpu-manager kubevirt kubevirt-operator multus-cni node-feature-discovery ovn4nfv qat-device-plugin sriov-network sriov-network-operator; do
 225         mkdir -p ${host_addons_dir}/${addon}/{helm,profile}
 226         cp -r ${kud_infra_folder}/helm/${addon} ${host_addons_dir}/${addon}/helm
 227         cp -r ${kud_infra_folder}/profiles/${addon}/* ${host_addons_dir}/${addon}/profile
 228         tar -czf ${host_addons_dir}/${addon}.tar.gz -C ${host_addons_dir}/${addon}/helm .
 229         tar -czf ${host_addons_dir}/${addon}_profile.tar.gz -C ${host_addons_dir}/${addon}/profile .
 230     done
 231
 232     mkdir -p ${host_addons_dir}/tests
 233     for test in _common _common_test _functions topology-manager-sriov multus ovn4nfv nfd sriov-network qat cmk; do
 234         cp ${kud_tests}/${test}.sh ${host_addons_dir}/tests
 235     done
 236
 237     mkdir -p ${host_artifacts_dir}
 238     cp -rf ${kud_inventory_folder}/artifacts/* ${host_artifacts_dir}
 239
 240     mkdir -p ${host_artifacts_dir}/addons
 241     for yaml in ${kud_infra_folder}/emco/examples/*.yaml; do
 242         cp ${yaml} ${host_artifacts_dir}/addons
 243     done
 244     for template in addons/*.tmpl; do
 245         CLUSTER_NAME="${cluster_name}" \
 246         HOST_IP="$(master_ip)" \
 247         KUBE_PATH="${host_artifacts_dir}/admin.conf" \
 248         PACKAGES_PATH="${host_addons_dir}" \
 249         envsubst <${template} >${host_artifacts_dir}/${template%.tmpl}
 250     done
 251 }
 252
 253 # _print_kubernetes_info() - Prints the login Kubernetes information
 254 function _print_kubernetes_info {
 255     if ! $(kubectl version &>/dev/null); then
 256         return
 257     fi
 258
 259     # Expose Dashboard using NodePort
 260     node_port=30080
 261     KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" \
 262         kubectl -n kube-system edit service kubernetes-dashboard
 263     KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" \
 264         kubectl -n kube-system edit service kubernetes-dashboard
 265
 266     printf "Kubernetes Info\n===============\n" > $k8s_info_file
 267     echo "Dashboard URL: https://$(master_ip):$node_port" >> $k8s_info_file
 268     echo "Admin user: kube" >> $k8s_info_file
 269     echo "Admin password: secret" >> $k8s_info_file
 270 }
 271
 272 verbose=""
 273 if [[ -n "${KUD_DEBUG:-}" ]]; then
 274     set -o xtrace
 275     verbose="-vvv"
 276 fi
 277
 278 # Configuration values
 279 dest_folder=/opt
 280 kud_folder=${INSTALLER_DIR}
 281 kud_infra_folder=$kud_folder/../../deployment_infra
 282 kud_playbooks=$kud_infra_folder/playbooks
 283 kud_tests=$kud_folder/../../tests
 284 k8s_info_file=$kud_folder/k8s_info.log
 285 testing_enabled=${KUD_ENABLE_TESTS:-false}
 286 container_runtime=${CONTAINER_RUNTIME:-docker}
 287 enable_kata_webhook=${ENABLE_KATA_WEBHOOK:-false}
 288 kata_webhook_runtimeclass=${KATA_WEBHOOK_RUNTIMECLASS:-kata-qemu}
 289 kata_webhook_deployed=false
 290 # For containerd the etcd_deployment_type: docker is the default and doesn't work.
 291 # You have to use either etcd_kubeadm_enabled: true or etcd_deployment_type: host
 292 # See https://github.com/kubernetes-sigs/kubespray/issues/5713
 293 kud_kata_override_variables="container_manager=containerd \
 294     -e etcd_deployment_type=host -e kubelet_cgroup_driver=cgroupfs"
 295
 296 mkdir -p /opt/csar
 297 export CSAR_DIR=/opt/csar
 298
 299 function install_pkg {
 300 # Install dependencies
 301     apt-get update
 302     install_prerequisites
 303     install_kubespray
 304 }
 305
 306 function install_cluster {
 307     version=$(grep "kubespray_version" ${kud_playbooks}/kud-vars.yml | \
 308         awk -F ': ' '{print $2}')
 309     export ANSIBLE_CONFIG=$dest_folder/kubespray-$version/ansible.cfg
 310     install_k8s $1
 311     if [ ${2:+1} ]; then
 312         echo "install default addons and $2"
 313         install_addons "$2"
 314     else
 315         install_addons
 316     fi
 317     echo "installed the addons"
 318
 319     install_host_artifacts $1
 320
 321     _print_kubernetes_info
 322 }
 323
 324 function usage {
 325     echo "installer usage:"
 326     echo "./installer.sh --install_pkg - Install the required softwarepackage"
 327     echo "./installer.sh --cluster <cluster name> \
 328 - Install k8s cluster with default plugins"
 329     echo "./installer.sh --cluster <cluster name> \
 330 --plugins <plugin_1 plugin_2> - Install k8s cluster with default plugins \
 331 and additional plugins such as onap4k8s."
 332 }
 333
 334 if [ $# -eq 0 ]; then
 335     echo "Error: No arguments supplied"
 336     usage
 337     exit 1
 338 fi
 339
 340 if [ -z "$1" ]; then
 341     echo "Error: Null argument passed"
 342     usage
 343     exit 1
 344 fi
 345
 346 if [ "$1" == "--install_pkg" ]; then
 347     export kud_inventory_folder=$kud_folder/inventory
 348     kud_inventory=$kud_inventory_folder/hosts.ini
 349     install_pkg
 350     echo "install pkg"
 351     exit 0
 352 fi
 353
 354 if [ "$1" == "--cluster" ]; then
 355     if [ -z "${2-}"  ]; then
 356         echo "Error: Cluster name is null"
 357         usage
 358         exit 1
 359     fi
 360
 361     cluster_name=$2
 362     kud_multi_cluster_path=/opt/kud/multi-cluster
 363     cluster_path=$kud_multi_cluster_path/$cluster_name
 364     echo $cluster_path
 365     if [ ! -d "${cluster_path}" ]; then
 366         echo "Error: cluster_path ${cluster_path} doesn't exit"
 367         usage
 368         exit 1
 369     fi
 370
 371     cluster_log=$kud_multi_cluster_path/$cluster_name/log
 372     export kud_inventory_folder=$kud_folder/inventory/$cluster_name
 373     kud_inventory=$kud_inventory_folder/hosts.ini
 374
 375     mkdir -p $kud_inventory_folder
 376     mkdir -p $cluster_log
 377     cp $kud_multi_cluster_path/$cluster_name/hosts.ini $kud_inventory_folder/
 378     cp -rf $kud_folder/inventory/group_vars $kud_inventory_folder/
 379
 380     if [ ${3:+1} ]; then
 381         if [ "$3" == "--plugins" ]; then
 382             if [ -z "${4-}"  ]; then
 383                 echo "Error: plugins arguments is null; Refer the usage"
 384                 usage
 385                 exit 1
 386             fi
 387             plugins_name=${@:4:$#}
 388             install_cluster $cluster_name "$plugins_name"
 389             exit 0
 390         else
 391             echo "Error: cluster argument should have plugins; \
 392                 Refer the usage"
 393             usage
 394             exit 1
 395         fi
 396     fi
 397     install_cluster $cluster_name
 398     exit 0
 399 fi
 400
 401
 402 echo "Error: Refer the installer usage"
 403 usage
 404 exit 1