From 9f9f50031c10407d9f46623e52d504c11c411440 Mon Sep 17 00:00:00 2001 From: Fiete Ostkamp Date: Mon, 15 Sep 2025 15:55:13 +0200 Subject: [PATCH] Use k3s in pipeline - replace kubespray with k3s which is - much simpler to set up + does not need to be 'production-ready' to test deployment in pipeline - disable addon installation - enable k8s-plugin test run by verify-shell - do not enable the other plugin_ tests since that would be too much for this CR - pull docker images through nexus proxy to avoid rate limiting from dockerhub - bump snapshot version to 0.10.2-SNAPSHOT Issue-ID: MULTICLOUD-1513 Change-Id: I8ca40dff67ddf41153bdd2f7cfa07a22fb2b03f5 Signed-off-by: Fiete Ostkamp --- build/Dockerfile | 3 +- deployments/Dockerfile | 2 +- deployments/_functions.sh | 12 +++--- deployments/build.sh | 18 +++++++-- deployments/docker-compose.yml | 31 ++++++++++++++- docs/kud_architecture.rst | 2 +- kud/hosting_providers/vagrant/installer.sh | 47 +++++++++++++++-------- kud/tests/_common_test.sh | 6 +++ kud/tests/_functions.sh | 4 +- kud/tests/emco.sh | 2 +- kud/tests/negative_tests/_test_functions.sh | 5 +++ kud/tests/negative_tests/_test_variables_setup.sh | 2 +- kud/tests/plugin.sh | 14 +++++-- kud/tests/plugin_collection_v2.sh | 2 +- kud/tests/plugin_eaa.sh | 20 +++++----- kud/tests/plugin_edgex.sh | 10 +++-- kud/tests/plugin_fw.sh | 4 +- kud/tests/plugin_ncm_v2.sh | 4 +- src/k8splugin/Makefile | 4 +- src/k8splugin/go.mod | 2 + src/k8splugin/go.sum | 5 +++ src/orchestrator/scripts/docker-compose.yml | 2 +- 22 files changed, 142 insertions(+), 59 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index d7ba3c35..23787735 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -FROM golang:1.14.1 +FROM nexus3.onap.org:10001/golang:1.14.1 WORKDIR /go/src/github.com/onap/multicloud-k8s COPY ./ ./ @@ -41,4 +41,3 @@ COPY --chown=emco --from=0 /go/src/github.com/onap/multicloud-k8s/src/ovnaction/ USER emco ENTRYPOINT ["./entrypoint"] - diff --git a/deployments/Dockerfile b/deployments/Dockerfile index 842345c2..abe37ad1 100644 --- a/deployments/Dockerfile +++ b/deployments/Dockerfile @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -FROM ubuntu:18.04 +FROM nexus3.onap.org:10001/ubuntu:18.04 ARG HTTP_PROXY=${HTTP_PROXY} ARG HTTPS_PROXY=${HTTPS_PROXY} diff --git a/deployments/_functions.sh b/deployments/_functions.sh index c0feed0d..3d4e0336 100755 --- a/deployments/_functions.sh +++ b/deployments/_functions.sh @@ -9,12 +9,12 @@ ############################################################################## function stop_all { - docker-compose kill - docker-compose down + docker compose kill + docker compose down } function start_mongo { - docker-compose up -d mongo + docker compose up -d mongo export DATABASE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aqf "name=mongo")) export no_proxy=${no_proxy:-},${DATABASE_IP} export NO_PROXY=${NO_PROXY:-},${DATABASE_IP} @@ -32,16 +32,18 @@ EOF } function start_all { - docker-compose up -d + docker compose up -d } function wait_for_service { - for try in {0..59}; do + for try in {0..29}; do echo "$(date +%H:%M:%S) - Waiting for service up" sleep 1 if $(curl http://localhost:9015/v1 &>/dev/null); then return 0 fi done + echo Wait failed + docker compose logs exit 1 } diff --git a/deployments/build.sh b/deployments/build.sh index f3d195f3..b1e0771a 100755 --- a/deployments/build.sh +++ b/deployments/build.sh @@ -13,13 +13,17 @@ set -o pipefail k8s_path="$(git rev-parse --show-toplevel)" -VERSION="0.10.1-SNAPSHOT" +VERSION="0.10.2-SNAPSHOT" export IMAGE_NAME="nexus3.onap.org:10003/onap/multicloud/k8s" function _compile_src { echo "Compiling source code" + go version + ls pushd $k8s_path/src/k8splugin/ - make + pwd + # mount directory and build in container (thus not relying on the state of the runner) + docker run --rm -v "$PWD":/usr/src/myapp -w /usr/src/myapp golang:1.14 make popd } @@ -37,11 +41,19 @@ function _cleanup { if [[ -n ${image} ]]; then docker images ${image#*:} -q | xargs docker rmi -f fi - docker ps -a --filter "status=exited" -q | xargs docker rm + + exited_containers=$(docker ps -a --filter "status=exited" -q) + if [[ -n "$exited_containers" ]]; then + echo "Removing exited containers..." + echo "$exited_containers" | xargs docker rm + else + echo "Nothing to remove" + fi } function _build_docker { echo "Building docker image" + apt-get update && apt-get install -y docker-compose-plugin docker-compose build --no-cache } diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 3fa82fe5..af8c9906 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -25,14 +25,41 @@ services: - HTTPS_PROXY=${HTTPS_PROXY} - NO_PROXY=${NO_PROXY},mongo depends_on: - - mongo + mongo: + condition: service_healthy network_mode: host + ports: + - 9015:9015 volumes: - /opt/csar:/opt/csar - ${PWD}/k8sconfig.json:/opt/multicloud/k8splugin/k8sconfig.json:ro + restart: always + healthcheck: + test: ["CMD", "curl", "http://localhost:9015/v1"] mongo: - image: mongo + image: nexus3.onap.org:10001/mongo:5.0.28 environment: - HTTP_PROXY=${HTTP_PROXY} - HTTPS_PROXY=${HTTPS_PROXY} - NO_PROXY=${NO_PROXY} + healthcheck: + test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + ports: + - 27017:27017 + restart: always + etcd: + image: nexus3.onap.org:10001/bitnami/etcd:3 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - NO_PROXY=${NO_PROXY} + volumes: + - etcd_data:/bitnami/etcd + ports: + - 2379:2379 + - 2380:2380 + +volumes: + etcd_data: + driver: local diff --git a/docs/kud_architecture.rst b/docs/kud_architecture.rst index 2f724410..428d04a4 100644 --- a/docs/kud_architecture.rst +++ b/docs/kud_architecture.rst @@ -130,7 +130,7 @@ This bash script is used for the installation and configuration of dependencies required for the usage of the KUD via Virtual Machines. Some of this dependencies are: - - `Vagrant `_, + - Vagrant, - `Libvirt `_ or `VirtualBox `_ The *-p* argument determines the Virtualization provider to be used diff --git a/kud/hosting_providers/vagrant/installer.sh b/kud/hosting_providers/vagrant/installer.sh index b69a0b3c..5a7ea210 100755 --- a/kud/hosting_providers/vagrant/installer.sh +++ b/kud/hosting_providers/vagrant/installer.sh @@ -30,7 +30,7 @@ function _install_go { return fi - wget https://dl.google.com/go/$tarball + wget -nv https://dl.google.com/go/$tarball sudo tar -C /usr/local -xzf $tarball rm $tarball @@ -60,9 +60,9 @@ function _set_environment_file { OVN_CENTRAL_INTERFACE="${OVN_CENTRAL_INTERFACE:-$(ip addr show | awk '/inet.*brd/{print $NF; exit}')}" echo "export OVN_CENTRAL_INTERFACE=${OVN_CENTRAL_INTERFACE}" | sudo tee --append /etc/environment echo "export OVN_CENTRAL_ADDRESS=$(get_ovn_central_address)" | sudo tee --append /etc/environment - echo "export KUBE_CONFIG_DIR=/opt/kubeconfig" | sudo tee --append /etc/environment + echo "export KUBE_CONFIG_DIR=$HOME/.kube" | sudo tee --append /etc/environment echo "export CSAR_DIR=/opt/csar" | sudo tee --append /etc/environment - echo "export ANSIBLE_CONFIG=${ANSIBLE_CONFIG}" | sudo tee --append /etc/environment + # echo "export ANSIBLE_CONFIG=${ANSIBLE_CONFIG}" | sudo tee --append /etc/environment } # install_k8s() - Install Kubernetes using kubespray tool @@ -141,6 +141,18 @@ function install_k8s { sudo cp $kud_inventory_folder/artifacts/kubectl /usr/local/bin/ } +function install_k3s { + echo "Installing k3s..." + + curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" sh - + + systemctl status k3s + + sudo kubectl get all -n kube-system + + sudo kubectl cluster-info +} + # install_addons() - Install Kubenertes AddOns function install_addons { source /etc/environment @@ -219,21 +231,25 @@ function install_addons { # install_plugin() - Install ONAP Multicloud Kubernetes plugin function install_plugin { echo "Installing multicloud/k8s plugin" - sudo -E pip install --no-cache-dir docker-compose sudo mkdir -p /opt/{kubeconfig,consul/config} - sudo cp $HOME/.kube/config /opt/kubeconfig/kud + # sudo cp $HOME/.kube/config /opt/kubeconfig/kud + mkdir -p $WORKSPACE/.kube + kubeconfig_path="$WORKSPACE/.kube/config" + sudo cat /etc/rancher/k3s/k3s.yaml > $kubeconfig_path pushd $kud_folder/../../../deployments sudo ./build.sh if [[ "${testing_enabled}" == "true" ]]; then sudo ./start.sh pushd $kud_tests - plugin_tests="plugin plugin_edgex plugin_fw plugin_eaa" + # plugin_tests="plugin plugin_edgex plugin_fw plugin_eaa" + plugin_tests="plugin" # TODO: re-enable plugin_edgex plugin_fw plugin_eaa if [[ $kubespray_version == "2.16.0" ]]; then plugin_tests=${plugin_tests//plugin_fw/}; fi for functional_test in ${plugin_tests}; do + echo "running $functional_test.sh" bash ${functional_test}.sh done popd @@ -243,15 +259,15 @@ function install_plugin { # _print_kubernetes_info() - Prints the login Kubernetes information function _print_kubernetes_info { - if ! $(kubectl version &>/dev/null); then + if ! $(sudo kubectl version &>/dev/null); then return fi # Expose Dashboard using NodePort node_port=30080 - KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard - KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard + # KUBE_EDITOR="sed -i \"s|type\: ClusterIP|type\: NodePort|g\"" kubectl -n kube-system edit service kubernetes-dashboard + # KUBE_EDITOR="sed -i \"s|nodePort\: .*|nodePort\: $node_port|g\"" kubectl -n kube-system edit service kubernetes-dashboard - master_ip=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}') + master_ip=$(sudo kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' | awk -F '[:/]' '{print $4}') printf "Kubernetes Info\n===============\n" > $k8s_info_file echo "Dashboard URL: https://$master_ip:$node_port" >> $k8s_info_file @@ -294,7 +310,7 @@ kud_inventory=$kud_inventory_folder/hosts.ini kud_playbooks=$kud_infra_folder/playbooks kud_tests=$kud_folder/../../tests k8s_info_file=$kud_folder/k8s_info.log -testing_enabled=${KUD_ENABLE_TESTS:-false} +testing_enabled=${KUD_ENABLE_TESTS:-true} container_runtime=${CONTAINER_RUNTIME:-docker} enable_kata_webhook=${ENABLE_KATA_WEBHOOK:-false} kata_webhook_runtimeclass=${KATA_WEBHOOK_RUNTIMECLASS:-kata-clh} @@ -326,10 +342,11 @@ sudo ls /etc/apt/sources.list.d/ || true sudo find /etc/apt/sources.list.d -maxdepth 1 -name '*jonathonf*' -delete || true sudo apt-get update _install_go -install_k8s +# install_k8s +install_k3s _set_environment_file -install_addons -if ${KUD_PLUGIN_ENABLED:-false}; then +# install_addons +if ${KUD_PLUGIN_ENABLED:-true}; then install_plugin fi -_print_kubernetes_info +# _print_kubernetes_info diff --git a/kud/tests/_common_test.sh b/kud/tests/_common_test.sh index 0ca606b6..a00391cb 100755 --- a/kud/tests/_common_test.sh +++ b/kud/tests/_common_test.sh @@ -20,6 +20,12 @@ function install_deps { } install_packages "" ubuntu_deps "" fi + if ! $(helm --version &>/dev/null); then + function ubuntu_deps { + sudo snap install --classic helm + } + install_packages "" ubuntu_deps "" + fi } # install_ipcalc() - Install ipcalc for tests diff --git a/kud/tests/_functions.sh b/kud/tests/_functions.sh index 4c261953..6d24602d 100755 --- a/kud/tests/_functions.sh +++ b/kud/tests/_functions.sh @@ -186,8 +186,8 @@ function _destroy { local name=$2 echo "$(date +%H:%M:%S) - $name : Destroying $type" - kubectl delete $type $name --ignore-not-found=true --now - while kubectl get $type $name &>/dev/null; do + sudo kubectl delete $type $name --ignore-not-found=true --now + while sudo kubectl get $type $name &>/dev/null; do echo "$(date +%H:%M:%S) - $name : Destroying $type" done } diff --git a/kud/tests/emco.sh b/kud/tests/emco.sh index 109f5638..5b2c3d38 100755 --- a/kud/tests/emco.sh +++ b/kud/tests/emco.sh @@ -32,7 +32,7 @@ app1_profile_path="$CSAR_DIR/$csar_id/prometheus-operator_profile.tar.gz" app2_helm_path="$CSAR_DIR/$csar_id/collectd.tar.gz" app2_profile_path="$CSAR_DIR/$csar_id/collectd_profile.tar.gz" -kubeconfig_path="$HOME/.kube/config" +kubeconfig_path="$WORKSPACE/.kube/config" function populate_CSAR_composite_app_helm { _checks_args "$1" diff --git a/kud/tests/negative_tests/_test_functions.sh b/kud/tests/negative_tests/_test_functions.sh index f05c21fc..25e243c3 100755 --- a/kud/tests/negative_tests/_test_functions.sh +++ b/kud/tests/negative_tests/_test_functions.sh @@ -50,6 +50,11 @@ function call_api_negative { return 2 else echo "[INFO] Server replied with status: ${status}" >&2 + if [[ "${status}" -gt 400 ]]; then + echo "[DEBUG] curl_response_file path: ${curl_response_file}" + echo "[DEBUG] Listing contents of /tmp:" + ls -lh /tmp + fi cat "${curl_response_file}" rm "${curl_response_file}" return_status=$status diff --git a/kud/tests/negative_tests/_test_variables_setup.sh b/kud/tests/negative_tests/_test_variables_setup.sh index 1630c039..03642097 100755 --- a/kud/tests/negative_tests/_test_variables_setup.sh +++ b/kud/tests/negative_tests/_test_variables_setup.sh @@ -15,7 +15,7 @@ # */ base_url=${base_url:-"http://localhost:9015/v2"} -kubeconfig_path="$HOME/.kube/config" +kubeconfig_path="$WORKSPACE/.kube/config" csar_id=cb009bfe-bbee-11e8-9766-525400435678 project_name="test_project" diff --git a/kud/tests/plugin.sh b/kud/tests/plugin.sh index aff04128..816a4d87 100755 --- a/kud/tests/plugin.sh +++ b/kud/tests/plugin.sh @@ -18,7 +18,7 @@ source _common_test.sh source _functions.sh base_url="http://localhost:9015/v1" -kubeconfig_path="$HOME/.kube/config" +kubeconfig_path="$WORKSPACE/.kube/config" #Will resolve to file $KUBE_CONFIG_DIR/kud cloud_region_id="kud" cloud_region_owner="test_owner" @@ -94,7 +94,7 @@ if [[ "$rb_list" != *"${rb_name}"* ]]; then fi print_msg "Create Resource Bundle Profile Metadata" -kubeversion=$(kubectl version | grep 'Server Version' | awk -F '"' '{print $6}') +kubeversion=$(sudo kubectl version | grep 'Server Version' | awk -F '"' '{print $6}') payload=" { \"profile-name\": \"${profile_name}\", @@ -121,7 +121,9 @@ if [[ "$rbp_ret" != *"${profile_name}"* ]]; then exit 1 fi + print_msg "Setup cloud data" + payload="$(cat <