From 0c5dd89fb241bafccd139a3bb9fa348bb820f19d Mon Sep 17 00:00:00 2001 From: "waqas.ikram" Date: Tue, 28 Feb 2023 14:21:57 +0000 Subject: [PATCH] [SO] Creation of kind cluster for CNFM CSIT Change-Id: Icdad72e8bbcbd48cb92f630be0c39fb4af4bb47e Issue-ID: SO-4076 Signed-off-by: waqas.ikram Kind cluster is required to facilitate testing of remote cluster deployment using CNFM component Change-Id: I56c06832c35794dd6bbdd4aaf839c3c086360cb8 Signed-off-by: waqas.ikram --- .../so-simulators/package/docker/pom.xml | 19 ++++- .../kind-cluster/Dockerfile.kind-cluster | 28 +++++++ .../kind-cluster/entrypoint-original.sh | 95 ++++++++++++++++++++++ .../docker/docker-files/kind-cluster/entrypoint.sh | 72 ++++++++++++++++ .../docker-files/kind-cluster/kind-config.yaml | 47 +++++++++++ 5 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/Dockerfile.kind-cluster create mode 100644 plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint-original.sh create mode 100644 plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint.sh create mode 100644 plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/kind-config.yaml diff --git a/plans/so/integration-etsi-testing/so-simulators/package/docker/pom.xml b/plans/so/integration-etsi-testing/so-simulators/package/docker/pom.xml index 9173439f..2340579d 100644 --- a/plans/so/integration-etsi-testing/so-simulators/package/docker/pom.xml +++ b/plans/so/integration-etsi-testing/so-simulators/package/docker/pom.xml @@ -9,16 +9,22 @@ docker pom ${project.artifactId} + + + true + 0.28.0 + 1.23 + ${project.artifactId}-${project.version} io.fabric8 docker-maven-plugin - 0.28.0 + ${version-fabric8} true - 1.23 + ${version-apiVersion} ${docker.pull.registry} ${docker.push.registry} @@ -145,6 +151,15 @@ + + kind-cluster + + try + docker-files/kind-cluster + Dockerfile.kind-cluster + ${skip-kind-cluster} + + diff --git a/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/Dockerfile.kind-cluster b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/Dockerfile.kind-cluster new file mode 100644 index 00000000..be2bcbb5 --- /dev/null +++ b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/Dockerfile.kind-cluster @@ -0,0 +1,28 @@ +FROM library/alpine:3.8 + +RUN apk -U upgrade && apk add ca-certificates util-linux gnupg curl make kmod procps bash && gpg --keyserver https://download.docker.com/linux/debian/gpg --recv-keys && apk update && apk add docker openrc && rm -rf /var/lib/apt/lists/* +RUN update-alternatives --set iptables /usr/sbin/iptables-legacy || true && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy || true && update-alternatives --set arptables /usr/sbin/arptables-legacy || true +RUN addgroup --system dockremap && adduser --system --ingroup dockremap dockremap && echo 'dockremap:165536:65536' >> /etc/subuid && echo 'dockremap:165536:65536' >> /etc/subgid + +VOLUME /var/lib/docker +VOLUME /var/log/docker +ENV container=docker + +COPY entrypoint.sh /entrypoint.sh +RUN chmod 777 /entrypoint.sh + +ARG KUBECTL_VERSION=v1.21.2 +ARG KIND_VERSION=v0.11.1 + +RUN curl -Lso /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && chmod +x /usr/bin/kubectl && curl -Lso /usr/bin/kind "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" && chmod +x /usr/bin/kind # buildkit + +COPY kind-config.yaml /kind-config.yaml +RUN chmod 777 /kind-config.yaml + +EXPOSE 30001 + +COPY entrypoint-original.sh /entrypoint-original.sh +RUN chmod 777 /entrypoint-original.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint-original.sh b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint-original.sh new file mode 100644 index 00000000..488f7543 --- /dev/null +++ b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint-original.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# ============LICENSE_START======================================================= +# Copyright (C) 2023 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# Script copied from https://hub.docker.com/r/devopps/kind-cluster-buster + +set -o errexit +set -o nounset +set -o pipefail + +# This is copied from official dind script: +# https://raw.githubusercontent.com/docker/docker/master/hack/dind +if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then + mount -t securityfs none /sys/kernel/security || { + echo >&2 'Could not mount /sys/kernel/security.' + echo >&2 'AppArmor detection and --privileged mode might break.' + } +fi + +# Mount /tmp (conditionally) +if ! mountpoint -q /tmp; then + mount -t tmpfs none /tmp +fi + +# Check cgroupfs. +# Verify the filesystem. +if [ ! -d /sys/fs/cgroup/ ]; then + echo >&2 'Cgroupfs is not mounted' + exit 1 +fi + +# Determine cgroup parent for docker daemon. +# We need to make sure cgroups created by the docker daemon do not +# interfere with other cgroups on the host, and do not leak after this +# container is terminated. +if [ -f /sys/fs/cgroup/systemd/release_agent ]; then + # This means the user has bind mounted host /sys/fs/cgroup to the + # same location in the container (e.g., using the following docker + # run flags: `-v /sys/fs/cgroup:/sys/fs/cgroup`). In this case, we + # need to make sure the docker daemon in the container does not + # pollute the host cgroups hierarchy. + # Note that `release_agent` file is only created at the root of a + # cgroup hierarchy. + CGROUP_PARENT="$(grep systemd /proc/self/cgroup | cut -d: -f3)/docker" +else + CGROUP_PARENT="/docker" + # For each cgroup subsystem, Docker does a bind mount from the + # current cgroup to the root of the cgroup subsystem. For instance: + # /sys/fs/cgroup/memory/docker/ -> /sys/fs/cgroup/memory + # + # This will confuse some system software that manipulate cgroups + # (e.g., kubelet/cadvisor, etc.) sometimes because + # `/proc//cgroup` is not affected by the bind mount. The + # following is a workaround to recreate the original cgroup + # environment by doing another bind mount for each subsystem. + CURRENT_CGROUP=$(grep systemd /proc/self/cgroup | cut -d: -f3) + CGROUP_SUBSYSTEMS=$(findmnt -lun -o source,target -t cgroup | grep "${CURRENT_CGROUP}" | awk '{print $2}') + + echo "${CGROUP_SUBSYSTEMS}" | + while IFS= read -r SUBSYSTEM; do + mkdir -p "${SUBSYSTEM}${CURRENT_CGROUP}" + mount --bind "${SUBSYSTEM}" "${SUBSYSTEM}${CURRENT_CGROUP}" + done +fi + +setsid dockerd \ + --cgroup-parent="${CGROUP_PARENT}" \ + --bip="${DOCKERD_BIP:-172.17.1.1/24}" \ + --mtu="${DOCKERD_MTU:-1400}" \ + --raw-logs \ + ${DOCKER_ARGS:-} >/var/log/docker/dockerd.log 2>&1 & + +# Wait until dockerd is ready. +until docker ps >/dev/null 2>&1 +do + echo "Waiting for dockerd..." + sleep 1 +done + +exec "$@" diff --git a/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint.sh b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint.sh new file mode 100644 index 00000000..4e4e75ec --- /dev/null +++ b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/entrypoint.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# ============LICENSE_START======================================================= +# Copyright (C) 2023 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# Script copied from https://hub.docker.com/r/devopps/kind-cluster-buster + +set -o errexit +set -o nounset +set -o pipefail + +# Startup Docker daemon and wait for it to be ready. +echo "Running entrypoint-original.sh .." +/entrypoint-original.sh bash -c "touch /dockerd-ready && sleep infinity" & +while [ ! -f /dockerd-ready ]; do sleep 10; done + +echo "Setting up KIND cluster" + +# Startup a KIND cluster. +API_SERVER_ADDRESS=${API_SERVER_ADDRESS:-$(hostname -i)} +echo "hostname: ${API_SERVER_ADDRESS}" +sed -ri "s/^(\s*)(apiServerAddress\s*:\s*apiServerAddress\s*$)/\1apiServerAddress: ${API_SERVER_ADDRESS}/" kind-config.yaml + +CERT_SANS=(${CERT_SANS:-""}) +CERT_SANS+=(${API_SERVER_ADDRESS}) +CERT_SANS+=($(hostname -i)) +CERT_SANS+=(localhost) +CERT_SANS+=(127.0.0.1) + +for node in $(kubectl get nodes -o wide --no-headers | awk '{print $6}'); do +echo "node: $node" +CERT_SANS+=(node) +done + +UNIQUE_CERT_SANS=($(echo "${CERT_SANS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + +for hostname in "${UNIQUE_CERT_SANS[@]}"; do +cat <> kind-config.yaml +- group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: ${hostname} +EOF +done + +kind create cluster --config=kind-config.yaml --image=${KIND_NODE_IMAGE-"devopps/kind-node:v1.21.1"} --wait=900s + +while read -r line; +do + echo "$line"; +done < "$HOME/.kube/config" + +CONFIG_ADDRESS=$HOME/.kube/config + +exec "$@" diff --git a/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/kind-config.yaml b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/kind-config.yaml new file mode 100644 index 00000000..164afa21 --- /dev/null +++ b/plans/so/integration-etsi-testing/so-simulators/package/docker/src/main/docker/docker-files/kind-cluster/kind-config.yaml @@ -0,0 +1,47 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +networking: + apiServerPort: 30001 + apiServerAddress: apiServerAddress + podSubnet: "10.245.0.0/16" + serviceSubnet: "10.246.0.0/16" +kubeadmConfigPatches: +- | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: JoinConfiguration + metadata: + name: config + nodeRegistration: + kubeletExtraArgs: + cgroup-root: "/kubelet" +- | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: InitConfiguration + metadata: + name: config + nodeRegistration: + kubeletExtraArgs: + cgroup-root: "/kubelet" +kubeadmConfigPatchesJSON6902: +- group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: 127.0.0.1 +- group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: 172.17.0.2 +- group: kubeadm.k8s.io + version: v1beta2 + kind: ClusterConfiguration + patch: | + - op: add + path: /apiServer/certSANs/- + value: localhost + -- 2.16.6