From c27a91f7f050c29557d87e108b77706eef16f509 Mon Sep 17 00:00:00 2001
From: Antoine Legrand <2t.antoine@gmail.com>
Date: Wed, 10 Oct 2018 04:14:33 +0200
Subject: [PATCH] Split deploy steps in separate playbooks: part1 (#3451)

* Fix bootstrap_os/ubuntu idempotency

* Update bastion role

* move container_engine in sub-roles

* requires ansible 2.5

* ubuntu18 as first CI job
---
 .gitlab-ci.yml                                | 20 ++++++-----
 cluster.yml                                   |  8 ++---
 requirements.txt                              |  2 +-
 roles/bastion-ssh-config/tasks/main.yml       | 11 ++-----
 .../templates/ssh-bastion.conf                |  4 +--
 roles/bootstrap-os/defaults/main.yml          |  5 +--
 roles/bootstrap-os/tasks/bootstrap-ubuntu.yml | 21 +++++++-----
 roles/bootstrap-os/tasks/main.yml             | 33 ++++++++-----------
 .../cri-o/defaults/main.yml                   |  0
 .../cri-o/files/mounts.conf                   |  0
 .../cri-o/tasks/main.yaml                     |  0
 .../cri-o/templates/crio.conf.j2              |  0
 .../cri-o/vars/fedora.yml                     |  0
 .../cri-o/vars/redhat.yml                     |  0
 roles/container-engine/defaults/main.yml      |  6 ++++
 .../docker/defaults/main.yml                  |  1 +
 .../docker/docker-storage/defaults/main.yml   |  0
 .../files/install_container_storage_setup.sh  |  0
 .../docker/docker-storage/tasks/main.yml      |  0
 .../templates/docker-storage-setup.j2         |  0
 .../docker/handlers/main.yml                  |  0
 .../docker/meta/main.yml                      |  2 +-
 .../docker/tasks/main.yml                     | 14 ++++++--
 .../docker/tasks/pre-upgrade.yml              |  0
 .../docker/tasks/set_facts_dns.yml            | 16 ++++-----
 .../docker/tasks/systemd.yml                  |  0
 .../apt_preferences.d/debian_docker.j2        |  0
 .../docker/templates/docker-dns.conf.j2       |  0
 .../docker/templates/docker-options.conf.j2   |  0
 .../docker/templates/docker.service.j2        |  0
 .../docker/templates/fedora_docker.repo.j2    |  0
 .../docker/templates/http-proxy.conf.j2       |  0
 .../docker/templates/rh_docker.repo.j2        |  0
 .../docker/vars/debian.yml                    |  0
 .../docker/vars/fedora.yml                    |  0
 .../docker/vars/redhat-aarch64.yml            |  0
 .../docker/vars/redhat.yml                    |  0
 .../docker/vars/suse.yml                      |  0
 .../docker/vars/ubuntu-amd64.yml              |  0
 .../docker/vars/ubuntu-arm64.yml              |  0
 .../docker/vars/ubuntu-bionic.yml             |  3 --
 roles/container-engine/meta/main.yml          | 23 +++++++++++++
 .../rkt/defaults/main.yml                     |  0
 .../rkt/files/rkt-gc.sh                       |  0
 .../rkt/tasks/install.yml                     |  0
 .../{ => container-engine}/rkt/tasks/main.yml |  0
 .../rkt/vars/debian.yml                       |  0
 .../rkt/vars/fedora.yml                       |  0
 .../rkt/vars/redhat.yml                       |  0
 .../{ => container-engine}/rkt/vars/suse.yml  |  0
 roles/kubespray-defaults/defaults/main.yaml   |  8 +++++
 scale.yml                                     |  7 +---
 ...m.yml => gce_centos-weave-kubeadm-sep.yml} |  1 +
 tests/files/gce_ubuntu18-flannel-aio.yml      |  4 +--
 upgrade-cluster.yml                           |  8 ++---
 55 files changed, 109 insertions(+), 88 deletions(-)
 rename roles/{ => container-engine}/cri-o/defaults/main.yml (100%)
 rename roles/{ => container-engine}/cri-o/files/mounts.conf (100%)
 rename roles/{ => container-engine}/cri-o/tasks/main.yaml (100%)
 rename roles/{ => container-engine}/cri-o/templates/crio.conf.j2 (100%)
 rename roles/{ => container-engine}/cri-o/vars/fedora.yml (100%)
 rename roles/{ => container-engine}/cri-o/vars/redhat.yml (100%)
 create mode 100644 roles/container-engine/defaults/main.yml
 rename roles/{ => container-engine}/docker/defaults/main.yml (98%)
 rename roles/{ => container-engine}/docker/docker-storage/defaults/main.yml (100%)
 rename roles/{ => container-engine}/docker/docker-storage/files/install_container_storage_setup.sh (100%)
 rename roles/{ => container-engine}/docker/docker-storage/tasks/main.yml (100%)
 rename roles/{ => container-engine}/docker/docker-storage/templates/docker-storage-setup.j2 (100%)
 rename roles/{ => container-engine}/docker/handlers/main.yml (100%)
 rename roles/{ => container-engine}/docker/meta/main.yml (65%)
 rename roles/{ => container-engine}/docker/tasks/main.yml (96%)
 rename roles/{ => container-engine}/docker/tasks/pre-upgrade.yml (100%)
 rename roles/{ => container-engine}/docker/tasks/set_facts_dns.yml (84%)
 rename roles/{ => container-engine}/docker/tasks/systemd.yml (100%)
 rename roles/{ => container-engine}/docker/templates/apt_preferences.d/debian_docker.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/docker-dns.conf.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/docker-options.conf.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/docker.service.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/fedora_docker.repo.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/http-proxy.conf.j2 (100%)
 rename roles/{ => container-engine}/docker/templates/rh_docker.repo.j2 (100%)
 rename roles/{ => container-engine}/docker/vars/debian.yml (100%)
 rename roles/{ => container-engine}/docker/vars/fedora.yml (100%)
 rename roles/{ => container-engine}/docker/vars/redhat-aarch64.yml (100%)
 rename roles/{ => container-engine}/docker/vars/redhat.yml (100%)
 rename roles/{ => container-engine}/docker/vars/suse.yml (100%)
 rename roles/{ => container-engine}/docker/vars/ubuntu-amd64.yml (100%)
 rename roles/{ => container-engine}/docker/vars/ubuntu-arm64.yml (100%)
 rename roles/{ => container-engine}/docker/vars/ubuntu-bionic.yml (90%)
 create mode 100644 roles/container-engine/meta/main.yml
 rename roles/{ => container-engine}/rkt/defaults/main.yml (100%)
 rename roles/{ => container-engine}/rkt/files/rkt-gc.sh (100%)
 rename roles/{ => container-engine}/rkt/tasks/install.yml (100%)
 rename roles/{ => container-engine}/rkt/tasks/main.yml (100%)
 rename roles/{ => container-engine}/rkt/vars/debian.yml (100%)
 rename roles/{ => container-engine}/rkt/vars/fedora.yml (100%)
 rename roles/{ => container-engine}/rkt/vars/redhat.yml (100%)
 rename roles/{ => container-engine}/rkt/vars/suse.yml (100%)
 rename tests/files/{gce_centos-weave-kubeadm.yml => gce_centos-weave-kubeadm-sep.yml} (99%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 98bc57c20..77a9fd41a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -42,7 +42,7 @@ before_script:
   tags:
     - kubernetes
     - docker
-  image: quay.io/kubespray/kubespray:latest
+  image: quay.io/kubespray/kubespray:v2.7
 
 .docker_service: &docker_service
   services:
@@ -323,12 +323,13 @@ before_script:
 
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 ### PR JOBS PART1
-gce_coreos-calico-aio:
+
+gce_ubuntu18-flannel-aio:
   stage: deploy-part1
   <<: *job
   <<: *gce
   variables:
-    <<: *coreos_calico_aio_variables
+    <<: *ubuntu18_flannel_aio_variables
     <<: *gce_variables
   when: on_success
   except: ['triggers']
@@ -336,14 +337,14 @@ gce_coreos-calico-aio:
 
 ### PR JOBS PART2
 
-gce_ubuntu18-flannel-aio:
+gce_coreos-calico-aio:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
-    <<: *ubuntu18_flannel_aio_variables
+    <<: *coreos_calico_aio_variables
     <<: *gce_variables
-  when: manual
+  when: on_success
   except: ['triggers']
   only: [/^pr-.*$/]
 
@@ -358,7 +359,7 @@ gce_centos7-flannel-addons:
   except: ['triggers']
   only: [/^pr-.*$/]
 
-gce_centos-weave-kubeadm:
+gce_centos-weave-kubeadm-sep:
   stage: deploy-part2
   <<: *job
   <<: *gce
@@ -369,6 +370,8 @@ gce_centos-weave-kubeadm:
   except: ['triggers']
   only: [/^pr-.*$/]
 
+### MANUAL JOBS
+
 gce_ubuntu-weave-sep:
   stage: deploy-part2
   <<: *job
@@ -376,11 +379,10 @@ gce_ubuntu-weave-sep:
   variables:
     <<: *gce_variables
     <<: *ubuntu_weave_sep_variables
-  when: on_success
+  when: manual
   except: ['triggers']
   only: [/^pr-.*$/]
 
-### MANUAL JOBS
 gce_coreos-calico-sep-triggers:
   stage: deploy-part2
   <<: *job
diff --git a/cluster.yml b/cluster.yml
index 249a04f1d..5b748da59 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -1,5 +1,5 @@
 ---
-- hosts: localhost
+- hosts: bastion[0]
   gather_facts: False
   roles:
     - { role: kubespray-defaults}
@@ -33,11 +33,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+    - { role: "container-engine", tags: "container-engine" }
     - { role: download, tags: download, when: "not skip_downloads" }
   environment: "{{proxy_env}}"
 
diff --git a/requirements.txt b/requirements.txt
index 01ff9f23c..59034dff5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-ansible>=2.4.0
+ansible>=2.5.0
 jinja2>=2.9.6
 netaddr
 pbr>=1.6
diff --git a/roles/bastion-ssh-config/tasks/main.yml b/roles/bastion-ssh-config/tasks/main.yml
index a3cacdc3b..71c96db21 100644
--- a/roles/bastion-ssh-config/tasks/main.yml
+++ b/roles/bastion-ssh-config/tasks/main.yml
@@ -1,21 +1,16 @@
 ---
 - set_fact:
-    has_bastion: "{{ 'bastion' in groups['all'] }}"
-
-- set_fact:
-    bastion_ip: "{{ hostvars['bastion']['ansible_host'] }}"
-  when: has_bastion
+    bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}"
+  delegate_to: localhost
 
 # As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly
 # To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user
 - set_fact:
     real_user: "{{ ansible_user }}"
-  delegate_to: bastion
-  when: has_bastion
 
 - name: create ssh bastion conf
   become: false
+  delegate_to: localhost
   template:
     src: ssh-bastion.conf
     dest: "{{ playbook_dir }}/ssh-bastion.conf"
-  when: has_bastion
diff --git a/roles/bastion-ssh-config/templates/ssh-bastion.conf b/roles/bastion-ssh-config/templates/ssh-bastion.conf
index 3f2a69ef1..814545c1d 100644
--- a/roles/bastion-ssh-config/templates/ssh-bastion.conf
+++ b/roles/bastion-ssh-config/templates/ssh-bastion.conf
@@ -1,9 +1,8 @@
-{% if has_bastion %}
 {% set vars={'hosts': ''} %}
 {% set user='' %}
 
 {% for h in groups['all'] %}
-{% if h != 'bastion' %}
+{% if h not in groups['bastion'] %}
 {% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %}
 {% endif %}
 {% endfor %}
@@ -17,4 +16,3 @@ Host {{ bastion_ip }}
 
 Host {{ vars['hosts'] }}
   ProxyCommand ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}
-{% endif %}
diff --git a/roles/bootstrap-os/defaults/main.yml b/roles/bootstrap-os/defaults/main.yml
index 5d2f7321a..c0974cfbf 100644
--- a/roles/bootstrap-os/defaults/main.yml
+++ b/roles/bootstrap-os/defaults/main.yml
@@ -1,9 +1,10 @@
 ---
+bootstrap_os:
+os_family: "{{bootstrap_os}}"
+
 pip_python_coreos_modules:
   - httplib2
   - six
 
 override_system_hostname: true
-
-
 coreos_auto_upgrade: true
diff --git a/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml
index baf3a433d..9c088b2c7 100644
--- a/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml
+++ b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml
@@ -1,25 +1,30 @@
 ---
 #  raw: cat /etc/issue.net | grep '{{ bootstrap_versions }}'
 
+- name: List ubuntu_packages
+  set_fact:
+    ubuntu_packages:
+      - python
+      - python-apt
+      - python-pip
+      - dbus
+
 - name: Bootstrap | Check if bootstrap is needed
-  raw: which "{{ item }}"
+  raw: dpkg -l | cut -d' ' -f3 |grep -e ^{{item}}$
   register: need_bootstrap
   failed_when: false
   changed_when: false
-  with_items:
-    - python
-    - python-apt
-    - pip
-    - dbus-daemon
+  with_items: "{{ubuntu_packages}}"
+
   tags:
     - facts
 
 - name: Bootstrap | Install python 2.x and pip
   raw:
     apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal python-pip dbus
+    DEBIAN_FRONTEND=noninteractive apt-get install -y {{ubuntu_packages | join(" ")}}
   when:
-    "{{ need_bootstrap.results | map(attribute='rc') | sort | last | bool }}"
+    - need_bootstrap.results | map(attribute='rc') | sort | last | bool
 
 - set_fact:
     ansible_python_interpreter: "/usr/bin/python"
diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml
index f05ab6614..af49faefe 100644
--- a/roles/bootstrap-os/tasks/main.yml
+++ b/roles/bootstrap-os/tasks/main.yml
@@ -1,32 +1,25 @@
 ---
-- import_tasks: bootstrap-ubuntu.yml
-  when: bootstrap_os == "ubuntu"
 
-- import_tasks: bootstrap-debian.yml
-  when: bootstrap_os == "debian"
+- include_tasks: bootstrap-ubuntu.yml
+  when: os_family == "ubuntu"
 
-- import_tasks: bootstrap-coreos.yml
-  when: bootstrap_os == "coreos"
+- include_tasks: bootstrap-debian.yml
+  when: os_family == "debian"
 
-- import_tasks: bootstrap-fedora.yml
-  when: bootstrap_os == "fedora"
+- include_tasks: bootstrap-coreos.yml
+  when: os_family == "coreos"
 
-- import_tasks: bootstrap-centos.yml
-  when: bootstrap_os == "centos"
+- include_tasks: bootstrap-fedora.yml
+  when: os_family == "fedora"
 
-- import_tasks: bootstrap-opensuse.yml
-  when: bootstrap_os == "opensuse"
+- include_tasks: bootstrap-centos.yml
+  when: os_family == "centos"
+
+- include_tasks: bootstrap-opensuse.yml
+  when: os_family == "opensuse"
 
 - import_tasks: setup-pipelining.yml
 
-- name: check if atomic host
-  stat:
-    path: /run/ostree-booted
-  register: ostree
-
-- set_fact:
-    is_atomic: "{{ ostree.stat.exists }}"
-
 - name: Gather nodes hostnames
   setup:
     gather_subset: '!all'
diff --git a/roles/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml
similarity index 100%
rename from roles/cri-o/defaults/main.yml
rename to roles/container-engine/cri-o/defaults/main.yml
diff --git a/roles/cri-o/files/mounts.conf b/roles/container-engine/cri-o/files/mounts.conf
similarity index 100%
rename from roles/cri-o/files/mounts.conf
rename to roles/container-engine/cri-o/files/mounts.conf
diff --git a/roles/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml
similarity index 100%
rename from roles/cri-o/tasks/main.yaml
rename to roles/container-engine/cri-o/tasks/main.yaml
diff --git a/roles/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2
similarity index 100%
rename from roles/cri-o/templates/crio.conf.j2
rename to roles/container-engine/cri-o/templates/crio.conf.j2
diff --git a/roles/cri-o/vars/fedora.yml b/roles/container-engine/cri-o/vars/fedora.yml
similarity index 100%
rename from roles/cri-o/vars/fedora.yml
rename to roles/container-engine/cri-o/vars/fedora.yml
diff --git a/roles/cri-o/vars/redhat.yml b/roles/container-engine/cri-o/vars/redhat.yml
similarity index 100%
rename from roles/cri-o/vars/redhat.yml
rename to roles/container-engine/cri-o/vars/redhat.yml
diff --git a/roles/container-engine/defaults/main.yml b/roles/container-engine/defaults/main.yml
new file mode 100644
index 000000000..2dbb43b14
--- /dev/null
+++ b/roles/container-engine/defaults/main.yml
@@ -0,0 +1,6 @@
+---
+## DNS
+dns_domain: cluster.local
+dns_servers: []
+upstream_dns_servers: []
+searchdomains: []
diff --git a/roles/docker/defaults/main.yml b/roles/container-engine/docker/defaults/main.yml
similarity index 98%
rename from roles/docker/defaults/main.yml
rename to roles/container-engine/docker/defaults/main.yml
index bd8d293c1..ec819b24a 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/container-engine/docker/defaults/main.yml
@@ -43,3 +43,4 @@ dockerproject_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/
 dockerproject_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'
 dockerproject_apt_repo_base_url: 'https://apt.dockerproject.org/repo'
 dockerproject_apt_repo_gpgkey: 'https://apt.dockerproject.org/gpg'
+docker_bin_dir: "/usr/bin"
diff --git a/roles/docker/docker-storage/defaults/main.yml b/roles/container-engine/docker/docker-storage/defaults/main.yml
similarity index 100%
rename from roles/docker/docker-storage/defaults/main.yml
rename to roles/container-engine/docker/docker-storage/defaults/main.yml
diff --git a/roles/docker/docker-storage/files/install_container_storage_setup.sh b/roles/container-engine/docker/docker-storage/files/install_container_storage_setup.sh
similarity index 100%
rename from roles/docker/docker-storage/files/install_container_storage_setup.sh
rename to roles/container-engine/docker/docker-storage/files/install_container_storage_setup.sh
diff --git a/roles/docker/docker-storage/tasks/main.yml b/roles/container-engine/docker/docker-storage/tasks/main.yml
similarity index 100%
rename from roles/docker/docker-storage/tasks/main.yml
rename to roles/container-engine/docker/docker-storage/tasks/main.yml
diff --git a/roles/docker/docker-storage/templates/docker-storage-setup.j2 b/roles/container-engine/docker/docker-storage/templates/docker-storage-setup.j2
similarity index 100%
rename from roles/docker/docker-storage/templates/docker-storage-setup.j2
rename to roles/container-engine/docker/docker-storage/templates/docker-storage-setup.j2
diff --git a/roles/docker/handlers/main.yml b/roles/container-engine/docker/handlers/main.yml
similarity index 100%
rename from roles/docker/handlers/main.yml
rename to roles/container-engine/docker/handlers/main.yml
diff --git a/roles/docker/meta/main.yml b/roles/container-engine/docker/meta/main.yml
similarity index 65%
rename from roles/docker/meta/main.yml
rename to roles/container-engine/docker/meta/main.yml
index 2adfe16ca..d04b24088 100644
--- a/roles/docker/meta/main.yml
+++ b/roles/container-engine/docker/meta/main.yml
@@ -1,4 +1,4 @@
 ---
 dependencies:
-  - role: docker/docker-storage
+  - role: container-engine/docker/docker-storage
     when: docker_container_storage_setup and ansible_os_family == "RedHat"
diff --git a/roles/docker/tasks/main.yml b/roles/container-engine/docker/tasks/main.yml
similarity index 96%
rename from roles/docker/tasks/main.yml
rename to roles/container-engine/docker/tasks/main.yml
index 0ba076deb..de4556dcf 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/container-engine/docker/tasks/main.yml
@@ -1,4 +1,12 @@
 ---
+- name: check if atomic host
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- set_fact:
+    is_atomic: "{{ ostree.stat.exists }}"
+
 - name: gather os specific variables
   include_vars: "{{ item }}"
   with_first_found:
@@ -52,7 +60,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic)
 
@@ -73,7 +81,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ dockerproject_repo_key_info.repo_keys }}"
   when:
     - not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic)
@@ -128,7 +136,7 @@
   register: docker_task_result
   until: docker_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_package_info.pkgs }}"
   notify: restart docker
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0)
diff --git a/roles/docker/tasks/pre-upgrade.yml b/roles/container-engine/docker/tasks/pre-upgrade.yml
similarity index 100%
rename from roles/docker/tasks/pre-upgrade.yml
rename to roles/container-engine/docker/tasks/pre-upgrade.yml
diff --git a/roles/docker/tasks/set_facts_dns.yml b/roles/container-engine/docker/tasks/set_facts_dns.yml
similarity index 84%
rename from roles/docker/tasks/set_facts_dns.yml
rename to roles/container-engine/docker/tasks/set_facts_dns.yml
index 8303e63ea..b80a7753c 100644
--- a/roles/docker/tasks/set_facts_dns.yml
+++ b/roles/container-engine/docker/tasks/set_facts_dns.yml
@@ -2,16 +2,11 @@
 
 - name: set dns server for docker
   set_fact:
-    docker_dns_servers: |-
-      {%- if dns_mode in ['kubedns', 'coredns'] -%}
-        {{ [ skydns_server ] }}
-      {%- elif dns_mode == 'coredns_dual' -%}
-        {{ [ skydns_server ] + [ skydns_server_secondary ] }}
-      {%- elif dns_mode == 'dnsmasq_kubedns' -%}
-        {{ [ dnsmasq_dns_server ] }}
-      {%- elif dns_mode == 'manual' -%}
-        {{ [ manual_dns_server ] }}
-      {%- endif -%}
+    docker_dns_servers: "{{dns_servers}}"
+
+- name: show docker_dns_servers
+  debug:
+    msg: "{{docker_dns_servers}}"
 
 - name: set base docker dns facts
   set_fact:
@@ -23,6 +18,7 @@
       - timeout:2
       - attempts:2
 
+
 - name: add upstream dns servers (only when dnsmasq is not used)
   set_fact:
     docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}"
diff --git a/roles/docker/tasks/systemd.yml b/roles/container-engine/docker/tasks/systemd.yml
similarity index 100%
rename from roles/docker/tasks/systemd.yml
rename to roles/container-engine/docker/tasks/systemd.yml
diff --git a/roles/docker/templates/apt_preferences.d/debian_docker.j2 b/roles/container-engine/docker/templates/apt_preferences.d/debian_docker.j2
similarity index 100%
rename from roles/docker/templates/apt_preferences.d/debian_docker.j2
rename to roles/container-engine/docker/templates/apt_preferences.d/debian_docker.j2
diff --git a/roles/docker/templates/docker-dns.conf.j2 b/roles/container-engine/docker/templates/docker-dns.conf.j2
similarity index 100%
rename from roles/docker/templates/docker-dns.conf.j2
rename to roles/container-engine/docker/templates/docker-dns.conf.j2
diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/container-engine/docker/templates/docker-options.conf.j2
similarity index 100%
rename from roles/docker/templates/docker-options.conf.j2
rename to roles/container-engine/docker/templates/docker-options.conf.j2
diff --git a/roles/docker/templates/docker.service.j2 b/roles/container-engine/docker/templates/docker.service.j2
similarity index 100%
rename from roles/docker/templates/docker.service.j2
rename to roles/container-engine/docker/templates/docker.service.j2
diff --git a/roles/docker/templates/fedora_docker.repo.j2 b/roles/container-engine/docker/templates/fedora_docker.repo.j2
similarity index 100%
rename from roles/docker/templates/fedora_docker.repo.j2
rename to roles/container-engine/docker/templates/fedora_docker.repo.j2
diff --git a/roles/docker/templates/http-proxy.conf.j2 b/roles/container-engine/docker/templates/http-proxy.conf.j2
similarity index 100%
rename from roles/docker/templates/http-proxy.conf.j2
rename to roles/container-engine/docker/templates/http-proxy.conf.j2
diff --git a/roles/docker/templates/rh_docker.repo.j2 b/roles/container-engine/docker/templates/rh_docker.repo.j2
similarity index 100%
rename from roles/docker/templates/rh_docker.repo.j2
rename to roles/container-engine/docker/templates/rh_docker.repo.j2
diff --git a/roles/docker/vars/debian.yml b/roles/container-engine/docker/vars/debian.yml
similarity index 100%
rename from roles/docker/vars/debian.yml
rename to roles/container-engine/docker/vars/debian.yml
diff --git a/roles/docker/vars/fedora.yml b/roles/container-engine/docker/vars/fedora.yml
similarity index 100%
rename from roles/docker/vars/fedora.yml
rename to roles/container-engine/docker/vars/fedora.yml
diff --git a/roles/docker/vars/redhat-aarch64.yml b/roles/container-engine/docker/vars/redhat-aarch64.yml
similarity index 100%
rename from roles/docker/vars/redhat-aarch64.yml
rename to roles/container-engine/docker/vars/redhat-aarch64.yml
diff --git a/roles/docker/vars/redhat.yml b/roles/container-engine/docker/vars/redhat.yml
similarity index 100%
rename from roles/docker/vars/redhat.yml
rename to roles/container-engine/docker/vars/redhat.yml
diff --git a/roles/docker/vars/suse.yml b/roles/container-engine/docker/vars/suse.yml
similarity index 100%
rename from roles/docker/vars/suse.yml
rename to roles/container-engine/docker/vars/suse.yml
diff --git a/roles/docker/vars/ubuntu-amd64.yml b/roles/container-engine/docker/vars/ubuntu-amd64.yml
similarity index 100%
rename from roles/docker/vars/ubuntu-amd64.yml
rename to roles/container-engine/docker/vars/ubuntu-amd64.yml
diff --git a/roles/docker/vars/ubuntu-arm64.yml b/roles/container-engine/docker/vars/ubuntu-arm64.yml
similarity index 100%
rename from roles/docker/vars/ubuntu-arm64.yml
rename to roles/container-engine/docker/vars/ubuntu-arm64.yml
diff --git a/roles/docker/vars/ubuntu-bionic.yml b/roles/container-engine/docker/vars/ubuntu-bionic.yml
similarity index 90%
rename from roles/docker/vars/ubuntu-bionic.yml
rename to roles/container-engine/docker/vars/ubuntu-bionic.yml
index 6f4119ad0..99a72d052 100644
--- a/roles/docker/vars/ubuntu-bionic.yml
+++ b/roles/container-engine/docker/vars/ubuntu-bionic.yml
@@ -1,9 +1,6 @@
 ---
 docker_kernel_min_version: '3.10'
 
-# overide defaults, missing 17.03 for Ubuntu 18.04
-docker_version: '18.06'
-
 use_docker_engine: false
 
 docker_versioned_pkg:
diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml
new file mode 100644
index 000000000..106e44719
--- /dev/null
+++ b/roles/container-engine/meta/main.yml
@@ -0,0 +1,23 @@
+---
+dependencies:
+  - role: container-engine/cri-o
+    when:
+      - container_manager == 'crio'
+    tags:
+      - container-engine
+      - crio
+
+  - role: container-engine/rkt
+    when:
+      - container_manager == 'rkt'
+    tags:
+      - container-engine
+      - rkt
+
+  - role: container-engine/docker
+    when:
+      - container_manager == 'docker' or container_manager == "rkt"
+    tags:
+      - container-engine
+      - docker
+      - rkt
diff --git a/roles/rkt/defaults/main.yml b/roles/container-engine/rkt/defaults/main.yml
similarity index 100%
rename from roles/rkt/defaults/main.yml
rename to roles/container-engine/rkt/defaults/main.yml
diff --git a/roles/rkt/files/rkt-gc.sh b/roles/container-engine/rkt/files/rkt-gc.sh
similarity index 100%
rename from roles/rkt/files/rkt-gc.sh
rename to roles/container-engine/rkt/files/rkt-gc.sh
diff --git a/roles/rkt/tasks/install.yml b/roles/container-engine/rkt/tasks/install.yml
similarity index 100%
rename from roles/rkt/tasks/install.yml
rename to roles/container-engine/rkt/tasks/install.yml
diff --git a/roles/rkt/tasks/main.yml b/roles/container-engine/rkt/tasks/main.yml
similarity index 100%
rename from roles/rkt/tasks/main.yml
rename to roles/container-engine/rkt/tasks/main.yml
diff --git a/roles/rkt/vars/debian.yml b/roles/container-engine/rkt/vars/debian.yml
similarity index 100%
rename from roles/rkt/vars/debian.yml
rename to roles/container-engine/rkt/vars/debian.yml
diff --git a/roles/rkt/vars/fedora.yml b/roles/container-engine/rkt/vars/fedora.yml
similarity index 100%
rename from roles/rkt/vars/fedora.yml
rename to roles/container-engine/rkt/vars/fedora.yml
diff --git a/roles/rkt/vars/redhat.yml b/roles/container-engine/rkt/vars/redhat.yml
similarity index 100%
rename from roles/rkt/vars/redhat.yml
rename to roles/container-engine/rkt/vars/redhat.yml
diff --git a/roles/rkt/vars/suse.yml b/roles/container-engine/rkt/vars/suse.yml
similarity index 100%
rename from roles/rkt/vars/suse.yml
rename to roles/container-engine/rkt/vars/suse.yml
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index d45929bc7..6d8f8c497 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -59,6 +59,14 @@ skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('addres
 skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
+kube_dns_servers:
+  kubedns: ["{{skydns_server}}"]
+  coredns: ["{{skydns_server}}"]
+  coredns_dual: "{{[skydns_server] + [ skydns_server_secondary ]}}"
+  manual: ["{{manual_dns_server}}"]
+  dnsmasq_kubedns: ["{{dnsmasq_dns_server}}"]
+
+dns_servers: "{{kube_dns_servers[dns_mode]}}"
 
 # Kubernetes configuration dirs and system namespace.
 # Those are where all the additional config stuff goes
diff --git a/scale.yml b/scale.yml
index c4cd117f0..862674754 100644
--- a/scale.yml
+++ b/scale.yml
@@ -35,12 +35,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+    - { role: container-engine, tags: "container-engine"}
     - { role: download, tags: download, when: "not skip_downloads" }
     - { role: etcd, tags: etcd, etcd_cluster_setup: false }
     - { role: vault, tags: vault, when: "cert_management == 'vault'"}
diff --git a/tests/files/gce_centos-weave-kubeadm.yml b/tests/files/gce_centos-weave-kubeadm-sep.yml
similarity index 99%
rename from tests/files/gce_centos-weave-kubeadm.yml
rename to tests/files/gce_centos-weave-kubeadm-sep.yml
index 199fa437c..c7efd2806 100644
--- a/tests/files/gce_centos-weave-kubeadm.yml
+++ b/tests/files/gce_centos-weave-kubeadm-sep.yml
@@ -3,6 +3,7 @@ cloud_image_family: centos-7
 cloud_machine_type: "n1-standard-1"
 cloud_region: us-central1-b
 mode: ha
+
 startup_script: ""
 
 # Deployment settings
diff --git a/tests/files/gce_ubuntu18-flannel-aio.yml b/tests/files/gce_ubuntu18-flannel-aio.yml
index 595fbf358..9df1fa7ed 100644
--- a/tests/files/gce_ubuntu18-flannel-aio.yml
+++ b/tests/files/gce_ubuntu18-flannel-aio.yml
@@ -5,10 +5,10 @@ cloud_machine_type: "n1-standard-1"
 mode: aio
 
 # Deployment settings
-
+kubeadm_enabled: false
 bootstrap_os: ubuntu
 kube_network_plugin: flannel
-
+dynamic_kubelet_configuration: true
 deploy_netchecker: true
 kubedns_min_replicas: 1
 cloud_provider: gce
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index dd705439e..69e312794 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -1,5 +1,5 @@
 ---
-- hosts: localhost
+- hosts: bastion[0]
   gather_facts: False
   roles:
     - { role: kubespray-defaults}
@@ -34,11 +34,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+    - { role: container-engine, tags: "container-engine"}
     - { role: download, tags: download, when: "not skip_downloads" }
   environment: "{{proxy_env}}"