From fc206b0b14de7b7dc7048ea3500de97b9a2b55a7 Mon Sep 17 00:00:00 2001 From: Jessica Wagantall Date: Mon, 19 Mar 2018 13:41:33 -0700 Subject: [PATCH 1/1] Upgrade to common-packer - Enable builder.json - Relove unused provision scripts - Update ci-management-packer-jobs - Update builder.yaml Change-Id: I0c9abcd33fd3acb84690863dca966a01731dc64c Issue-ID: CIMAN-33 Signed-off-by: Jessica Wagantall --- jjb/ci-management/ci-management-packer-jobs.yaml | 71 ++++++++++++++ jjb/ci-management/ci-management.yaml | 23 ----- jjb/global-defaults.yaml | 3 + packer/README.markdown | 4 +- packer/provision/docker.sh | 119 ----------------------- packer/provision/local-builder.yaml | 17 ++++ packer/provision/local-docker.yaml | 17 ++++ packer/templates/basebuild.json | 53 ---------- packer/templates/builder.json | 1 + packer/templates/docker.json | 55 +---------- packer/vars/cloud-env.json.example | 1 + packer/vars/ubuntu-14.04.json | 4 +- packer/vars/ubuntu-16.04.json | 4 +- 13 files changed, 117 insertions(+), 255 deletions(-) create mode 100644 jjb/ci-management/ci-management-packer-jobs.yaml delete mode 100644 packer/provision/docker.sh create mode 100644 packer/provision/local-builder.yaml create mode 100644 packer/provision/local-docker.yaml delete mode 100644 packer/templates/basebuild.json create mode 120000 packer/templates/builder.json mode change 100644 => 120000 packer/templates/docker.json diff --git a/jjb/ci-management/ci-management-packer-jobs.yaml b/jjb/ci-management/ci-management-packer-jobs.yaml new file mode 100644 index 000000000..41a3853ef --- /dev/null +++ b/jjb/ci-management/ci-management-packer-jobs.yaml @@ -0,0 +1,71 @@ +--- +- project: + name: packer-builder-jobs + jobs: + - gerrit-packer-merge + + project: ci-management + project-name: ci-management + branch: master + archive-artifacts: '**/*.log' + + build-node: ubuntu1604-builder-2c-1g + build-timeout: 90 + + platforms: + - centos-7 + - ubuntu-16.04 + templates: builder + +- project: + name: packer-docker-jobs + jobs: + - gerrit-packer-merge + + project: ci-management + project-name: ci-management + branch: master + archive-artifacts: '**/*.log' + + build-node: ubuntu1604-builder-2c-1g + build-timeout: 60 + + platforms: + - centos-7 + - ubuntu-16.04 + templates: docker + +- project: + name: packer-memcached-jobs + jobs: + - gerrit-packer-merge + + project: ci-management + project-name: ci-management + branch: master + archive-artifacts: '**/*.log' + + build-node: ubuntu1604-builder-2c-1g + build-timeout: 90 + + platforms: + - centos-7 + - ubuntu-16.04 + templates: memcached + +- project: + name: packer-redis-jobs + jobs: + - gerrit-packer-merge + + project: ci-management + project-name: ci-management + branch: master + archive-artifacts: '**/*.log' + + build-node: ubuntu1604-builder-2c-1g + + platforms: + - centos-7 + - ubuntu-16.04 + templates: redis diff --git a/jjb/ci-management/ci-management.yaml b/jjb/ci-management/ci-management.yaml index 528639e7d..184bf894d 100644 --- a/jjb/ci-management/ci-management.yaml +++ b/jjb/ci-management/ci-management.yaml @@ -12,29 +12,6 @@ build-node: 'centos7-builder-2c-1g' jjb-version: 2.0.0 -- project: - name: packer-jobs - # packer jobs templates are defined in global-jjb - jobs: - - '{project-name}-packer-jobs' - project: 'ci-management' - project-name: 'ci-management' - branch: master - archive-artifacts: '**/*.log' - build-node: 'centos7-builder-2c-1g' - platforms: - - centos - - ubuntu-16.04 - templates: - - builder - - docker - - memcached - - redis - exclude: - # only build the docker image on ubuntu at present - - platforms: centos - templates: docker - - job-template: name: '{project}-verify-jjb' diff --git a/jjb/global-defaults.yaml b/jjb/global-defaults.yaml index 64f6a74eb..51cf6c6dd 100644 --- a/jjb/global-defaults.yaml +++ b/jjb/global-defaults.yaml @@ -14,6 +14,9 @@ # lf-infra defaults lftools-version: <1.0.0 + # packer_version + packer-version: 1.1.3 + # default gerrit server definition server-name: 'Primary' gerrit-server-name: 'Primary' diff --git a/packer/README.markdown b/packer/README.markdown index 4972affae..1ef176a62 100644 --- a/packer/README.markdown +++ b/packer/README.markdown @@ -12,10 +12,10 @@ You'll need to [install Packer][2], of course. Open-O's Packer configuration is divided into build-specific variables, output-specific templates and a set of shared provisioning scripts. To do a specific build, combine the template for the desired output artifact type with -a variable file. To build a new basebuild instance the following would be done: +a variable file. To build a new builder instance the following would be done: ``` -packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/basebuild.json +packer build -var-file=vars/cloud-env.json -var-file=vars/centos.json templates/builder.json ``` **NOTE:** vars/cloud-env.json is a gitignored file as it contains private diff --git a/packer/provision/docker.sh b/packer/provision/docker.sh deleted file mode 100644 index 9d34eb079..000000000 --- a/packer/provision/docker.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# vim: ts=4 sw=4 sts=4 et tw=72 : - -# force any errors to cause the script and job to end in failure -set -xeu -o pipefail - -rh_systems() { - # Assumes that python is already installed by basebuild - - # Install dependencies for robotframework and robotframework-sshlibrary - yum install -y yum-utils unzip sshuttle nc libffi-devel openssl-devel - - # Install docker - yum install -y docker supervisor bridge-utils - systemctl enable docker - - # configure docker networking so that it does not conflict with LF - # internal networks - cat < /etc/sysconfig/docker-network -# /etc/sysconfig/docker-network -DOCKER_NETWORK_OPTIONS='--bip=10.250.0.254/24' -EOL - # configure docker daemon to listen on port 5555 enabling remote - # managment - sed -i -e "s#='--selinux-enabled'#='--selinux-enabled -H unix:///var/run/docker.sock -H tcp://0.0.0.0:5555'#g" /etc/sysconfig/docker - - # docker group doesn't get created by default for some reason - groupadd docker -} - -ubuntu_docker_mtu_fix(){ - echo "---> Fixing docker's mtu settings" - systemctl stop docker - cat <<'EOL' > /etc/systemd/system/docker.service -[Unit] -Description=Docker Application Container Engine -Documentation=https://docs.docker.com -After=network.target docker.socket -Requires=docker.socket - -[Service] -Type=notify -# the default is not to use systemd for cgroups because the delegate issues still -# exists and systemd currently does not support the cgroup feature set required -# for containers run by docker -ExecStart= -ExecStart=/usr/bin/dockerd --mtu 1454 -H tcp://0.0.0.0:2375 -H unix:///var/run/docker.sock -ExecReload=/bin/kill -s HUP $MAINPID -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -# Uncomment TasksMax if your systemd version supports it. -# Only systemd 226 and above support this version. -TasksMax=infinity -TimeoutStartSec=0 -# set delegate yes so that systemd does not reset the cgroups of docker containers -Delegate=yes -# kill only the docker process, not all processes in the cgroup -KillMode=process - -[Install] -WantedBy=multi-user.target -EOL - systemctl start docker - echo "---> Checking MTU" - docker network inspect bridge - echo "---> MTU set to 1454" -} - -ubuntu_systems() { - # Assumes that python is already installed by basebuild - - # Install Python3.6 - sudo add-apt-repository -y ppa:jonathonf/python-3.6 - sudo apt-get update - sudo apt-get install -y python3.6 python3.6-dev - - # Install dependencies for robotframework and robotframework-sshlibrary - apt install -y unzip sshuttle netcat libffi-dev libssl-dev - wget https://github.com/mozilla/geckodriver/releases/download/v0.18.0/geckodriver-v0.18.0-linux64.tar.gz - tar xvzf geckodriver-v0.18.0-linux64.tar.gz -C /usr/local/bin - export PATH=$PATH:/usr/local/bin - - # Install docker - apt install -y docker.io - - # Fixing Docker MTU settings - ubuntu_docker_mtu_fix -} - -all_systems() { - # Install docker-compose - curl -o /usr/local/bin/docker-compose -L "https://github.com/docker/compose/releases/download/1.15.0/docker-compose-$(uname -s)-$(uname -m)" - chmod +x /usr/local/bin/docker-compose - docker-compose -v -} - -echo "---> Detecting OS" -ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') - -case "${ORIGIN}" in - fedora|centos|redhat) - echo "---> RH type system detected" - rh_systems - ;; - ubuntu) - echo "---> Ubuntu system detected" - ubuntu_systems - ;; - *) - echo "---> Unknown operating system" - ;; -esac - -# execute steps for all systems -all_systems diff --git a/packer/provision/local-builder.yaml b/packer/provision/local-builder.yaml new file mode 100644 index 000000000..a54fc0220 --- /dev/null +++ b/packer/provision/local-builder.yaml @@ -0,0 +1,17 @@ +--- +- import_playbook: ../common-packer/provision/baseline.yaml + +- hosts: all + become_user: root + become_method: sudo + + pre_tasks: + - include_role: name=system-update + + roles: + - protobuf-install + + post_tasks: + - name: System Reseal + script: ../common-packer/provision/system-reseal.sh + become: true diff --git a/packer/provision/local-docker.yaml b/packer/provision/local-docker.yaml new file mode 100644 index 000000000..fce32a404 --- /dev/null +++ b/packer/provision/local-docker.yaml @@ -0,0 +1,17 @@ +--- +- import_playbook: ../common-packer/provision/docker.yaml + +- hosts: all + become_user: root + become_method: sudo + + pre_tasks: + - include_role: name=system-update + + roles: + - protobuf-install + + post_tasks: + - name: System Reseal + script: ../common-packer/provision/system-reseal.sh + become: true diff --git a/packer/templates/basebuild.json b/packer/templates/basebuild.json deleted file mode 100644 index ff84a72d6..000000000 --- a/packer/templates/basebuild.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "variables": { - "stack_tenant": null, - "stack_user": null, - "stack_pass": null, - "stack_network": null, - "base_image": null, - "cloud_user": null, - "distro": null, - "cloud_user_data": null - }, - "builders": [ - { - "type": "openstack", - "identity_endpoint": "https://auth.vexxhost.net/v2.0/", - "tenant_name": "{{user `stack_tenant`}}", - "username": "{{user `stack_user`}}", - "password": "{{user `stack_pass`}}", - "region": "ca-ymq-1", - "ssh_username": "{{user `cloud_user`}}", - "image_name": "{{user `distro`}} - basebuild - {{isotime \"20060102-1504\"}}", - "source_image_name": "{{user `base_image`}}", - "flavor": "v1-standard-1", - "availability_zone": "ca-ymq-2", - "networks": [ - "{{user `stack_network`}}" - ], - "user_data_file": "{{user `cloud_user_data`}}" - } - ], - "provisioners": [ - { - "type": "shell", - "inline": [ - "mkdir -p /tmp/packer" - ] - }, - { - "type": "file", - "source": "provision/basebuild/", - "destination": "/tmp/packer" - }, - { - "type": "shell", - "scripts": [ - "provision/baseline.sh", - "provision/basebuild.sh", - "provision/system_reseal.sh" - ], - "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" - } - ] -} diff --git a/packer/templates/builder.json b/packer/templates/builder.json new file mode 120000 index 000000000..bd227be50 --- /dev/null +++ b/packer/templates/builder.json @@ -0,0 +1 @@ +../common-packer/templates/builder.json \ No newline at end of file diff --git a/packer/templates/docker.json b/packer/templates/docker.json deleted file mode 100644 index 1c8d3ec56..000000000 --- a/packer/templates/docker.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "variables": { - "stack_tenant": null, - "stack_user": null, - "stack_pass": null, - "stack_network": null, - "base_image": null, - "cloud_user": null, - "distro": null, - "cloud_user_data": null - }, - "builders": [ - { - "type": "openstack", - "identity_endpoint": "https://auth.vexxhost.net/v2.0/", - "tenant_name": "{{user `stack_tenant`}}", - "username": "{{user `stack_user`}}", - "password": "{{user `stack_pass`}}", - "region": "ca-ymq-1", - "ssh_username": "{{user `cloud_user`}}", - "image_name": "{{user `distro`}} - docker - {{isotime \"20060102-1504\"}}", - "source_image_name": "{{user `base_image`}}", - "flavor": "v1-standard-1", - "availability_zone": "ca-ymq-2", - "networks": [ - "{{user `stack_network`}}" - ], - "user_data_file": "{{user `cloud_user_data`}}" - } - ], - "provisioners": [ - { - "type": "shell", - "inline": [ - "mkdir -p /tmp/packer" - ] - }, - { - "type": "file", - "source": "provision/basebuild/", - "destination": "/tmp/packer" - }, - { - "type": "shell", - "scripts": [ - "provision/baseline.sh", - "provision/basebuild.sh", - "provision/docker.sh", - "provision/system_reseal.sh" - ], - "execute_command": "chmod +x {{ .Path }}; if [ \"$UID\" == \"0\" ]; then {{ .Vars }} '{{ .Path }}'; else {{ .Vars }} sudo -E '{{ .Path }}'; fi" - } - ] -} diff --git a/packer/templates/docker.json b/packer/templates/docker.json new file mode 120000 index 000000000..edea6d285 --- /dev/null +++ b/packer/templates/docker.json @@ -0,0 +1 @@ +../common-packer/templates/docker.json \ No newline at end of file diff --git a/packer/vars/cloud-env.json.example b/packer/vars/cloud-env.json.example index 3a2ea9d2b..451c5dab9 100644 --- a/packer/vars/cloud-env.json.example +++ b/packer/vars/cloud-env.json.example @@ -3,4 +3,5 @@ "stack_user": "USERID", "stack_pass": "PASSWD", "stack_network": "NETID", + "ssh_proxy_host": "" } diff --git a/packer/vars/ubuntu-14.04.json b/packer/vars/ubuntu-14.04.json index dd81e2594..882d9598d 100644 --- a/packer/vars/ubuntu-14.04.json +++ b/packer/vars/ubuntu-14.04.json @@ -1,6 +1,6 @@ { - "base_image": "Ubuntu 14.04 (2017-02-01) - LF upload", - "cloud_user": "ubuntu", + "base_image": "LF - Ubuntu 14.04 LTS (2017-12-01)", "distro": "Ubuntu 14.04", + "ssh_user": "ubuntu", "cloud_user_data": "provision/null_data.sh" } diff --git a/packer/vars/ubuntu-16.04.json b/packer/vars/ubuntu-16.04.json index 12f9430ed..b55f96b4b 100644 --- a/packer/vars/ubuntu-16.04.json +++ b/packer/vars/ubuntu-16.04.json @@ -1,6 +1,6 @@ { - "base_image": "Ubuntu 16.04 (2017-02-01) - LF upload", - "cloud_user": "ubuntu", + "base_image": "LF - Ubuntu 16.04 LTS (2017-12-01)", "distro": "Ubuntu 16.04", + "ssh_user": "ubuntu", "cloud_user_data": "provision/null_data.sh" } -- 2.16.6