echo "Cleaning previous execution"
docker-compose kill
image=$(grep "image.*k8plugin" docker-compose.yml)
- docker images ${image#*:} -q | xargs docker rmi -f
+ if [[ -n ${image} ]]; then
+ docker images ${image#*:} -q | xargs docker rmi -f
+ fi
docker ps -a --filter "status=exited" -q | xargs docker rm
}
.. code-block:: bash
- # sed -i '/andrewrothstein.kubectl/d' playbooks/configure-*.ymlb
+ # sed -i '/andrewrothstein.kubectl/d' playbooks/configure-*.yml
Ansible uses SSH protocol for executing remote instructions. The following
instructions create and register ssh keys which avoid the usage of passwords.
# Inventory file creation
File.open(File.dirname(__FILE__) + "/inventory/hosts.ini", "w") do |inventory_file|
- inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\nansible_ssh_pass=vagrant\n\n[all]")
+ inventory_file.puts("[all:vars]\nansible_connection=ssh\nansible_ssh_user=vagrant\n[all]")
nodes.each do |node|
inventory_file.puts("#{node['name']}\tansible_ssh_host=#{node['ip']} ansible_ssh_port=22")
end
Vagrant.configure("2") do |config|
config.vm.box = box[provider][:name]
config.vm.box_version = box[provider][:version]
+ config.ssh.insert_key = false
if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil
if Vagrant.has_plugin?('vagrant-proxyconf')
config.vm.define :installer, primary: true, autostart: false do |installer|
installer.vm.hostname = "multicloud"
installer.vm.network :private_network, :ip => "10.10.10.2", :type => :static
- installer.vm.synced_folder '../', '/root/go/src/k8-plugin-multicloud/', type: sync_type
- installer.vm.provision 'shell' do |sh|
+ installer.vm.synced_folder '../', '/home/vagrant/multicloud-k8s/', type: sync_type
+ installer.vm.provision 'shell', privileged: false do |sh|
sh.env = {'KRD_PLUGIN_ENABLED': 'true'}
- sh.path = "main.sh"
+ sh.inline = <<-SHELL
+ cp /vagrant/insecure_keys/key /home/vagrant/.ssh/id_rsa
+ chown vagrant /home/vagrant/.ssh/id_rsa
+ chmod 400 /home/vagrant/.ssh/id_rsa
+ cd /home/vagrant/multicloud-k8s/vagrant/ && ./installer.sh | tee krd_installer.log
+ SHELL
end
end
end
--- /dev/null
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----
fi
wget https://dl.google.com/go/$tarball
- tar -C /usr/local -xzf $tarball
+ sudo tar -C /usr/local -xzf $tarball
rm $tarball
export PATH=$PATH:/usr/local/go/bin
- sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment
- export INSTALL_DIRECTORY=/usr/local/bin
- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+ sudo sed -i "s|^PATH=.*|PATH=\"$PATH\"|" /etc/environment
}
# _install_pip() - Install Python Package Manager
function _install_pip {
if $(pip --version &>/dev/null); then
- return
+ sudo apt-get install -y python-dev
+ curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python
+ else
+ sudo -E pip install --upgrade pip
fi
- apt-get install -y python-dev
- curl -sL https://bootstrap.pypa.io/get-pip.py | python
- pip install --upgrade pip
}
# _install_ansible() - Install and Configure Ansible program
function _install_ansible {
- mkdir -p /etc/ansible/
+ sudo mkdir -p /etc/ansible/
if $(ansible --version &>/dev/null); then
return
fi
_install_pip
- pip install ansible
+ sudo -E pip install ansible
}
# _install_docker() - Download and install docker-engine
if $(docker version &>/dev/null); then
return
fi
- apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
- add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
- apt-get update
- apt-get install -y docker-ce
+ sudo apt-get install -y software-properties-common linux-image-extra-$(uname -r) linux-image-extra-virtual apt-transport-https ca-certificates curl
+ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+ sudo apt-get update
+ sudo apt-get install -y docker-ce
- mkdir -p /etc/systemd/system/docker.service.d
+ sudo mkdir -p /etc/systemd/system/docker.service.d
if [ $http_proxy ]; then
- cat <<EOL > /etc/systemd/system/docker.service.d/http-proxy.conf
-[Service]
-Environment="HTTP_PROXY=$http_proxy"
-EOL
+ echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf
+ echo "Environment=\"HTTP_PROXY=$http_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/http-proxy.conf
fi
if [ $https_proxy ]; then
- cat <<EOL > /etc/systemd/system/docker.service.d/https-proxy.conf
-[Service]
-Environment="HTTPS_PROXY=$https_proxy"
-EOL
+ echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/https-proxy.conf
+ echo "Environment=\"HTTPS_PROXY=$https_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/https-proxy.conf
fi
if [ $no_proxy ]; then
- cat <<EOL > /etc/systemd/system/docker.service.d/no-proxy.conf
-[Service]
-Environment="NO_PROXY=$no_proxy"
-EOL
+ echo "[Service]" | sudo tee /etc/systemd/system/docker.service.d/no-proxy.conf
+ echo "Environment=\"NO_PROXY=$no_proxy\"" | sudo tee --append /etc/systemd/system/docker.service.d/no-proxy.conf
+ fi
+ sudo systemctl daemon-reload
+ echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | sudo tee --append /etc/default/docker
+ if [[ -z $(groups | grep docker) ]]; then
+ sudo usermod -aG docker $USER
+ newgrp docker
fi
- systemctl daemon-reload
- echo "DOCKER_OPTS=\"-H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock --max-concurrent-downloads $max_concurrent_downloads \"" | tee --append /etc/default/docker
- usermod -aG docker $USER
- systemctl restart docker
+ sudo systemctl restart docker
sleep 10
}
echo "Deploying kubernetes"
local dest_folder=/opt
version=$(grep "kubespray_version" ${krd_playbooks}/krd-vars.yml | awk -F ': ' '{print $2}')
+ local_release_dir=$(grep "local_release_dir" $krd_inventory_folder/group_vars/k8s-cluster.yml | awk -F "\"" '{print $2}')
local tarball=v$version.tar.gz
- apt-get install -y sshpass
+ sudo apt-get install -y sshpass
+ _install_docker
_install_ansible
wget https://github.com/kubernetes-incubator/kubespray/archive/$tarball
- tar -C $dest_folder -xzf $tarball
- mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
+ sudo tar -C $dest_folder -xzf $tarball
+ sudo mv $dest_folder/kubespray-$version/ansible.cfg /etc/ansible/ansible.cfg
+ sudo chown -R $USER $dest_folder/kubespray-$version
+ sudo mkdir -p ${local_release_dir}/containers
rm $tarball
- pip install -r $dest_folder/kubespray-$version/requirements.txt
+ sudo -E pip install -r $dest_folder/kubespray-$version/requirements.txt
rm -f $krd_inventory_folder/group_vars/all.yml 2> /dev/null
if [[ -n "${verbose}" ]]; then
echo "kube_log_level: 5" | tee $krd_inventory_folder/group_vars/all.yml
if [[ -n "${https_proxy}" ]]; then
echo "https_proxy: \"$https_proxy\"" | tee --append $krd_inventory_folder/group_vars/all.yml
fi
- ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml -b | tee $log_folder/setup-kubernetes.log
+ ansible-playbook $verbose -i $krd_inventory $dest_folder/kubespray-$version/cluster.yml --become --become-user=root | sudo tee $log_folder/setup-kubernetes.log
# Configure environment
mkdir -p $HOME/.kube
- mv $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config
+ cp $krd_inventory_folder/artifacts/admin.conf $HOME/.kube/config
}
# install_addons() - Install Kubenertes AddOns
function install_addons {
echo "Installing Kubernetes AddOns"
_install_ansible
- ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors
+ sudo ansible-galaxy install $verbose -r $krd_folder/galaxy-requirements.yml --ignore-errors
- ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | tee $log_folder/setup-krd.log
+ ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-krd.yml | sudo tee $log_folder/setup-krd.log
for addon in ${KRD_ADDONS:-virtlet ovn-kubernetes multus}; do
echo "Deploying $addon using configure-$addon.yml playbook.."
- ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | tee $log_folder/setup-${addon}.log
+ ansible-playbook $verbose -i $krd_inventory $krd_playbooks/configure-${addon}.yml | sudo tee $log_folder/setup-${addon}.log
if [[ "${testing_enabled}" == "true" ]]; then
pushd $krd_tests
bash ${addon}.sh
echo "Installing multicloud/k8s plugin"
_install_go
_install_docker
- pip install docker-compose
+ sudo -E pip install docker-compose
- mkdir -p /opt/{kubeconfig,consul/config}
- cp $HOME/.kube/config /opt/kubeconfig/krd
+ sudo mkdir -p /opt/{kubeconfig,consul/config}
+ sudo cp $HOME/.kube/config /opt/kubeconfig/krd
export KUBE_CONFIG_DIR=/opt/kubeconfig
- echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" >> /etc/environment
-
- GOPATH=$(go env GOPATH)
- pushd $GOPATH/src/k8-plugin-multicloud/deployments
- ./build.sh
+ echo "export KUBE_CONFIG_DIR=${KUBE_CONFIG_DIR}" | sudo tee --append /etc/environment
+ pushd $krd_folder/../deployments
+ sudo ./build.sh
if [[ "${testing_enabled}" == "true" ]]; then
docker-compose up -d
pushd $krd_tests
# Configuration values
log_folder=/var/log/krd
krd_folder=$(pwd)
-krd_inventory_folder=$krd_folder/inventory
+export krd_inventory_folder=$krd_folder/inventory
krd_inventory=$krd_inventory_folder/hosts.ini
krd_playbooks=$krd_folder/playbooks
krd_tests=$krd_folder/tests
k8s_info_file=$krd_folder/k8s_info.log
testing_enabled=${KRD_ENABLE_TESTS:-false}
-mkdir -p $log_folder
-mkdir -p /opt/csar
+sudo mkdir -p $log_folder
+sudo mkdir -p /opt/csar
export CSAR_DIR=/opt/csar
-echo "export CSAR_DIR=${CSAR_DIR}" | tee --append /etc/environment
+echo "export CSAR_DIR=${CSAR_DIR}" | sudo tee --append /etc/environment
# Install dependencies
# Setup proxy variables
if [ -f $krd_folder/sources.list ]; then
- mv /etc/apt/sources.list /etc/apt/sources.list.backup
- cp $krd_folder/sources.list /etc/apt/sources.list
+ sudo mv /etc/apt/sources.list /etc/apt/sources.list.backup
+ sudo cp $krd_folder/sources.list /etc/apt/sources.list
fi
-apt-get update
+sudo apt-get update
install_k8s
install_addons
if [[ "${KRD_PLUGIN_ENABLED:-false}" ]]; then
# works in the kernel space
# https://kubernetes.io/docs/concepts/services-networking/service/#proxy-mode-ipvs
#kube_proxy_mode: ipvs
+
+# Download container images only once then push to cluster nodes in batches
+download_run_once: true
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+
+# Makes the installer node a delegate for pushing images while running
+# the deployment with ansible. This maybe the case if cluster nodes
+# cannot access each over via ssh or you want to use local docker
+# images as a cache for multiple clusters.
+download_localhost: true
+++ /dev/null
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-##############################################################################
-# Copyright (c) 2018
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-set -o nounset
-set -o pipefail
-
-cd ~/go/src/k8-plugin-multicloud/vagrant
-sudo -H -E bash ./installer.sh | tee krd_installer.log
##############################################################################
- hosts: localhost
- become: yes
pre_tasks:
- name: Load krd variables
include_vars:
dest: "{{ istio_dest }}"
remote_src: yes
- name: copy istioctl binary to usr/local/bin folder
+ become: yes
command: "mv {{ istio_dest }}/istio-{{ istio_version }}/bin/istioctl /usr/local/bin/"
when: istio_source_type == "tarball"
- name: create network objects
tasks:
- name: copy admin.conf file to kube-nodes
copy:
- src: "{{ ansible_env.HOME}}/.kube/config"
+ src: "{{ lookup('env','krd_inventory_folder') }}/artifacts/admin.conf"
dest: "/etc/kubernetes/admin.conf"
- node-feature-discovery-daemonset.json.template
- hosts: localhost
- become: yes
+ pre_tasks:
+ - name: Load krd variables
+ include_vars:
+ file: krd-vars.yml
roles:
- role: andrewrothstein.kubectl
- kubectl_ver: "v{{ kubectl_version }}
+ kubectl_ver: "v{{ kubectl_version }}"
tasks:
- name: create service accounts
command: "/usr/local/bin/kubectl apply -f /tmp/{{ item }}"
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
- hosts: localhost
- become: yes
vars:
images_file: /tmp/images.yaml
pre_tasks:
- name: configure proxy values for docker service
block:
- name: create docker config folder
+ become: yes
file:
state: directory
path: "/etc/systemd/system/docker.service.d"
- name: Configure docker service to use http_proxy env value
+ become: yes
blockinfile:
dest: "/etc/systemd/system/docker.service.d/http-proxy.conf"
create: yes
when:
- lookup('env','http_proxy') != "fooproxy"
- name: Configure docker service to use https_proxy env value
+ become: yes
blockinfile:
dest: "/etc/systemd/system/docker.service.d/https-proxy.conf"
create: yes
when:
- lookup('env','https_proxy') != "fooproxy"
- name: Configure docker service to use no_proxy env value
+ become: yes
blockinfile:
dest: "/etc/systemd/system/docker.service.d/no-proxy.conf"
create: yes
when:
- lookup('env','no_proxy') != "fooproxy"
- name: reload systemd
+ become: yes
command: systemctl daemon-reload
- name: restart docker service
+ become: yes
service:
name: docker
state: restarted
delay: 10
- hosts: virtlet
- become: yes
tasks:
- name: Load krd variables
include_vars:
state: directory
path: "{{ criproxy_dest }}"
- name: disable AppArmor in all nodes
+ become: yes
service:
name: apparmor
state: stopped
enabled: no
when: ansible_os_family == "Debian"
- name: modify args for kubelet service
+ become: yes
lineinfile:
dest: /etc/systemd/system/kubelet.service
line: " --container-runtime=remote --container-runtime-endpoint=unix:///run/criproxy.sock --image-service-endpoint=unix:///run/criproxy.sock --enable-controller-attach-detach=false \\"
insertafter: '^ExecStart=/usr/local/bin/kubelet *'
state: present
- name: create dockershim service
+ become: yes
blockinfile:
path: /etc/systemd/system/dockershim.service
create: yes
path: "{{ criproxy_dest }}/criproxy"
mode: "+x"
- name: create criproxy service
+ become: yes
blockinfile:
path: /etc/systemd/system/criproxy.service
create: yes
[Install]
WantedBy=kubelet.service
- name: start criproxy and dockershim services
+ become: yes
service:
name: "{{ item }}"
state: started
- dockershim
- criproxy
- name: restart kubelet services
+ become: yes
service:
name: kubelet
state: restarted
Code Review / multicloud / k8s.git / commitdiff