From 7c79e09424c6d0776a7957ddf6057eea7a46910b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Mon, 20 Feb 2017 17:07:53 -0500
Subject: [PATCH 01/80] common: fix "disable transparent hugepage"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

To configure kernel the task is using "command" module which is not
respect operator ">". So this task just print to "stdout": "never >
/sys/kernel/mm/transparent_hugepage/enabled"

fix: #1319

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 roles/ceph-common/tasks/misc/system_tuning.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/roles/ceph-common/tasks/misc/system_tuning.yml b/roles/ceph-common/tasks/misc/system_tuning.yml
index 8da585dae..cef219ec0 100644
--- a/roles/ceph-common/tasks/misc/system_tuning.yml
+++ b/roles/ceph-common/tasks/misc/system_tuning.yml
@@ -5,7 +5,8 @@
   failed_when: false
 
 - name: disable transparent hugepage
-  command: "echo never > /sys/kernel/mm/transparent_hugepage/enabled"
+  shell: |
+    echo never > /sys/kernel/mm/transparent_hugepage/enabled
   changed_when: false
   failed_when: false
   when: disable_transparent_hugepage

From effefe91d51d8e59dcd2bf33ad432c9d873a839e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 12:00:37 -0500
Subject: [PATCH 02/80] common: add doc for rgw on ipv6
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

See: https://bugzilla.redhat.com/show_bug.cgi?id=1424799

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/all.yml.sample           | 2 +-
 roles/ceph-common/defaults/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 2cf51f8bb..4e1ab81ca 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -294,7 +294,7 @@ dummy:
 #radosgw_dns_name: your.subdomain.tld # subdomains used by radosgw. See http://ceph.com/docs/master/radosgw/config/#enabling-subdomain-s3-calls
 #radosgw_resolve_cname: false # enable for radosgw to resolve DNS CNAME based bucket names
 #radosgw_civetweb_port: 8080 # on Infernalis we get: "set_ports_option: cannot bind to 80: 13 (Permission denied)"
-#radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}"
+#radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}" # when using ipv6 enclose with brackets: "[{{ ansible_default_ipv6.address }}]"
 #radosgw_civetweb_num_threads: 50
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at
diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml
index c5a564326..0573d9017 100644
--- a/roles/ceph-common/defaults/main.yml
+++ b/roles/ceph-common/defaults/main.yml
@@ -286,7 +286,7 @@ mds_max_mds: 3
 #radosgw_dns_name: your.subdomain.tld # subdomains used by radosgw. See http://ceph.com/docs/master/radosgw/config/#enabling-subdomain-s3-calls
 radosgw_resolve_cname: false # enable for radosgw to resolve DNS CNAME based bucket names
 radosgw_civetweb_port: 8080 # on Infernalis we get: "set_ports_option: cannot bind to 80: 13 (Permission denied)"
-radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}"
+radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}" # when using ipv6 enclose with brackets: "[{{ ansible_default_ipv6.address }}]"
 radosgw_civetweb_num_threads: 50
 # For additional civetweb configuration options available such as SSL, logging,
 # keepalive, and timeout settings, please see the civetweb docs at

From 9e7fbbd6c5873366a71f282f91773712f7be8f2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 12:01:08 -0500
Subject: [PATCH 03/80] docker-common: sync group_vars file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/docker-commons.yml.sample | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 group_vars/docker-commons.yml.sample

diff --git a/group_vars/docker-commons.yml.sample b/group_vars/docker-commons.yml.sample
new file mode 100644
index 000000000..69bcc8401
--- /dev/null
+++ b/group_vars/docker-commons.yml.sample
@@ -0,0 +1,12 @@
+---
+# Variables here are applicable to all host groups NOT roles
+
+# This sample file generated by generate_group_vars_sample.sh
+
+# Dummy variable to avoid error because ansible does not recognize the
+# file as a good configuration file when no variable in it.
+dummy:
+
+
+#ceph_docker_registry: docker.io
+

From 4db4de52b03b46b6c1b063b130342bc376f5a661 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Tue, 21 Feb 2017 09:49:40 -0700
Subject: [PATCH 04/80] group_vars: rm Infernalis comment

As of Infernalis, the Ceph daemons run as an unprivileged "ceph" UID,
and this is by design.

Commit f19b765f7995dab4aca46e1885320dc440485ead altered the default
civetweb port from 80 to 8080 with a comment in the commit log about
"until this gets solved"

Remove the comment about permissions on Infernalis, because this is
always going to be the case on the Ceph versions we support, and it
is just confusing.

If users want to expose civetweb to s3 clients using privileged TCP
ports, they can redirect traffic with iptables, or use a reverse proxy
application like HAproxy.
---
 group_vars/all.yml.sample | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 4e1ab81ca..c33c60869 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -293,7 +293,7 @@ dummy:
 #
 #radosgw_dns_name: your.subdomain.tld # subdomains used by radosgw. See http://ceph.com/docs/master/radosgw/config/#enabling-subdomain-s3-calls
 #radosgw_resolve_cname: false # enable for radosgw to resolve DNS CNAME based bucket names
-#radosgw_civetweb_port: 8080 # on Infernalis we get: "set_ports_option: cannot bind to 80: 13 (Permission denied)"
+#radosgw_civetweb_port: 8080
 #radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}" # when using ipv6 enclose with brackets: "[{{ ansible_default_ipv6.address }}]"
 #radosgw_civetweb_num_threads: 50
 # For additional civetweb configuration options available such as SSL, logging,

From 920bd9cf2db404ee3b93b8521451df2e7332cda4 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Fri, 17 Feb 2017 14:27:15 -0600
Subject: [PATCH 05/80] ceph-common: use yum_repository when adding the
 ceph_stable repo

This gives us more flexibility than installing the ceph-release package
as we can easily use different mirrors. Also, I noticed an issue when
upgrading from jewel -> kraken as the ceph-release package for those
releases both have the same version number and yum doesn't know to
update anything.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 .../tasks/installs/redhat_ceph_repository.yml    | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml b/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
index a90309653..b0f285b3e 100644
--- a/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
+++ b/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
@@ -6,19 +6,15 @@
   when: ceph_stable
 
 - name: add ceph stable repository
-  package:
-    name: "{{ ceph_mirror }}/rpm-{{ ceph_stable_release }}/{{ ceph_stable_redhat_distro }}/noarch/ceph-release-1-0.{{ ceph_stable_redhat_distro|replace('rhel', 'el') }}.noarch.rpm"
+  yum_repository:
+    name: ceph_stable
+    description: Ceph Stable repo
+    gpgcheck: yes
     state: present
-  changed_when: false
+    gpgkey: "{{ ceph_stable_key }}"
+    baseurl: "{{ ceph_mirror }}/rpm-{{ ceph_stable_release }}/{{ ceph_stable_redhat_distro }}/$basearch"
   when: ceph_stable
 
-- name: change download url to ceph mirror
-  replace:
-    name: /etc/yum.repos.d/ceph.repo
-    regexp: http://download.ceph.com
-    replace: "{{ ceph_mirror }}"
-  when: ceph_mirror != "http://download.ceph.com"
-
 # we must use curl instead of ansible's uri module because SNI support in
 # Python is only available in 2.7.9 and later, and most supported distributions
 # don't have that version, so a request to https fails.

From 1579642e3f2bbf2fb830858e1c4fd28490654de0 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Fri, 17 Feb 2017 14:29:45 -0600
Subject: [PATCH 06/80] ceph-common: do not get current fsid when performing a
 rolling_update

This avoids a situation where during a rolling_update we try to talk to
a mon to get the fsid and if that mon is down the playbook hangs
indefinitely.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 roles/ceph-common/defaults/main.yml |  4 ++++
 roles/ceph-common/tasks/facts.yml   | 14 +++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml
index 0573d9017..b1905f78b 100644
--- a/roles/ceph-common/defaults/main.yml
+++ b/roles/ceph-common/defaults/main.yml
@@ -401,3 +401,7 @@ mon_containerized_default_ceph_conf_with_kv: false
 nfs_file_gw: true
 # Set this to true to enable Object access via NFS. Requires an RGW role.
 nfs_obj_gw: false
+
+# this is only here for usage with the rolling_update.yml playbook
+# do not ever change this here
+rolling_update: false
diff --git a/roles/ceph-common/tasks/facts.yml b/roles/ceph-common/tasks/facts.yml
index 6a05cc097..0e444e00f 100644
--- a/roles/ceph-common/tasks/facts.yml
+++ b/roles/ceph-common/tasks/facts.yml
@@ -5,6 +5,9 @@
   always_run: yes
   register: ceph_version
 
+# this task shouldn't run in a rolling_update situation
+# because it blindly picks a mon, which may be down because
+# of the rolling update
 - name: is ceph running already?
   command: ceph --connect-timeout 3 --cluster {{ cluster }} fsid
   changed_when: false
@@ -12,6 +15,14 @@
   always_run: yes
   register: ceph_current_fsid
   delegate_to: "{{ groups[mon_group_name][0] }}"
+  when: not rolling_update
+
+# set this as a default when performing a rolling_update
+# so the rest of the tasks here will succeed
+- set_fact:
+    ceph_current_fsid:
+      rc: 1
+  when: rolling_update
 
 - name: create a local fetch directory if it does not exist
   local_action: file path={{ fetch_directory }} state=directory
@@ -22,7 +33,8 @@
 
 - set_fact:
     fsid: "{{ ceph_current_fsid.stdout }}"
-  when: ceph_current_fsid.rc == 0
+  when:
+    - ceph_current_fsid.rc == 0
 
 - set_fact:
     monitor_name: "{{ ansible_hostname }}"

From 5622c94e8bd7821f94f1f016563b052a3587c9f5 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Fri, 17 Feb 2017 14:31:25 -0600
Subject: [PATCH 07/80] rolling-update: do not use upstart to stop mons when
 using systemd

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 infrastructure-playbooks/rolling_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index 9623d392c..a1f4b3e5a 100644
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -78,7 +78,7 @@
         name: ceph-mon
         state: stopped
         args: id={{ ansible_hostname }}
-      when: ansible_service_mgr == 'systemd'
+      when: ansible_service_mgr == 'upstart'
 
     - name: stop ceph mons with sysvinit
       service:

From 0cdc6fb79a0f025c95292f47d91218e751608518 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Fri, 17 Feb 2017 14:33:30 -0600
Subject: [PATCH 08/80] tests: adds a new ansible2.2-update_dmcrypt scenario

This performs a rolling update on a cluster using dmcrypt dedicated
journals.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 30fb7a4a1..0b4725a3b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt}
+envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,update_dmcrypt}
 skipsdist = True
 
 # extra commands for purging clusters
@@ -15,6 +15,16 @@ commands=
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
+# extra commands for performing a rolling update
+# currently this hardcodes the release to kraken
+# as we're still installing jewel by default
+[update]
+commands=
+  cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch"
+
+  testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
+
 [testenv]
 whitelist_externals =
     vagrant
@@ -48,6 +58,7 @@ changedir=
   docker_cluster: {toxinidir}/tests/functional/centos/7/docker-cluster
   purge_cluster: {toxinidir}/tests/functional/ubuntu/16.04/cluster
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
+  update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
 commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
@@ -59,5 +70,6 @@ commands=
 
   purge_cluster: {[purge]commands}
   purge_dmcrypt: {[purge]commands}
+  update_dmcrypt: {[update]commands}
 
   vagrant destroy --force

From 6cf842eb396d0a26882db9658cf7010ff84a7733 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 21 Feb 2017 12:35:00 -0600
Subject: [PATCH 09/80] ceph-common: remove infernalis comment on
 radosgw_civetweb_port

As of Infernalis, the Ceph daemons run as an unprivileged "ceph" UID,
and this is by design.

Commit f19b765 altered the default
civetweb port from 80 to 8080 with a comment in the commit log about
"until this gets solved"

Remove the comment about permissions on Infernalis, because this is
always going to be the case on the Ceph versions we support, and it
is just confusing.

If users want to expose civetweb to s3 clients using privileged TCP
ports, they can redirect traffic with iptables, or use a reverse proxy
application like HAproxy.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 roles/ceph-common/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml
index b1905f78b..3f5f2fe2d 100644
--- a/roles/ceph-common/defaults/main.yml
+++ b/roles/ceph-common/defaults/main.yml
@@ -285,7 +285,7 @@ mds_max_mds: 3
 #
 #radosgw_dns_name: your.subdomain.tld # subdomains used by radosgw. See http://ceph.com/docs/master/radosgw/config/#enabling-subdomain-s3-calls
 radosgw_resolve_cname: false # enable for radosgw to resolve DNS CNAME based bucket names
-radosgw_civetweb_port: 8080 # on Infernalis we get: "set_ports_option: cannot bind to 80: 13 (Permission denied)"
+radosgw_civetweb_port: 8080
 radosgw_civetweb_bind_ip: "{{ ansible_default_ipv4.address }}" # when using ipv6 enclose with brackets: "[{{ ansible_default_ipv6.address }}]"
 radosgw_civetweb_num_threads: 50
 # For additional civetweb configuration options available such as SSL, logging,

From f3a1c6464cb9a6e39d1b43f6661f848a8f16caec Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 21 Feb 2017 12:36:54 -0600
Subject: [PATCH 10/80] update group_vars sample for rolling_update variable

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 group_vars/all.yml.sample | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index c33c60869..3c09f7c4d 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -410,3 +410,7 @@ dummy:
 # Set this to true to enable Object access via NFS. Requires an RGW role.
 #nfs_obj_gw: false
 
+# this is only here for usage with the rolling_update.yml playbook
+# do not ever change this here
+#rolling_update: false
+

From dd548c6034d191c00e5bc55b952ae93a98d87923 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 7 Feb 2017 13:45:31 +0100
Subject: [PATCH 11/80] docker: osd, do not skip on failure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If the systemd unit file can not be generated we should fail, same for
systemd enable and reload.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 roles/ceph-osd/tasks/docker/start_docker_osd.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index e1ea76a84..a694523f2 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -75,18 +75,15 @@
     owner: "root"
     group: "root"
     mode: "0644"
-  failed_when: false
 
 - name: enable systemd unit file for osd instance
   shell: systemctl enable ceph-osd@{{ item | basename }}.service
-  failed_when: false
   changed_when: false
   with_items: "{{ ceph_osd_docker_devices }}"
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
   changed_when: false
-  failed_when: false
 
 - name: systemd start osd container
   service:

From a002508a915c310a9a0b7afd0b26c3410acf2471 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 7 Feb 2017 21:55:36 +0100
Subject: [PATCH 12/80] purge-docker: also purge journal devices
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../purge-docker-cluster.yml                     | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/infrastructure-playbooks/purge-docker-cluster.yml b/infrastructure-playbooks/purge-docker-cluster.yml
index ec53beeff..5d601b42c 100644
--- a/infrastructure-playbooks/purge-docker-cluster.yml
+++ b/infrastructure-playbooks/purge-docker-cluster.yml
@@ -293,14 +293,18 @@
       privileged: yes
       env: "CEPH_DAEMON=zap_device,OSD_DEVICE={{ item }}"
       volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
-    with_items: "{{ ceph_osd_docker_devices }}"
+    with_items:
+      - "{{ ceph_osd_docker_devices }}"
+      - "{{ raw_journal_devices }}"
 
   - name: remove ceph osd zap disk container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
       name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
       state: absent
-    with_items: "{{ ceph_osd_docker_devices }}"
+    with_items:
+      - "{{ ceph_osd_docker_devices }}"
+      - "{{ raw_journal_devices }}"
 
   # zap twice
   - name: zap ceph osd disk
@@ -313,14 +317,18 @@
       privileged: yes
       env: "CEPH_DAEMON=zap_device,OSD_DEVICE={{ item }}"
       volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
-    with_items: "{{ ceph_osd_docker_devices }}"
+    with_items:
+      - "{{ ceph_osd_docker_devices }}"
+      - "{{ raw_journal_devices }}"
 
   - name: remove ceph osd zap disk container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
       name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
       state: absent
-    with_items: "{{ ceph_osd_docker_devices }}"
+    with_items:
+      - "{{ ceph_osd_docker_devices }}"
+      - "{{ raw_journal_devices }}"
 
   - name: remove ceph osd service
     file:

From 73cf0378c20f52a6e408e5be84c2826627e570fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 7 Feb 2017 22:00:53 +0100
Subject: [PATCH 13/80] docker: osd, do not use priviledged container anymore
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Oh yeah! This patch adds more fine grained control on how we run the
activation osd container. We now use --device to give a read, write and
mknodaccess to a specific device to be consumed by Ceph. We also use
SYS_ADMIN cap to allow mount operations, ceph-disk needs to temporary
mount the osd data directory during the activation sequence.

This patch also enables the support of dedicated journal devices when
deploying ceph-docker with ceph-ansible.

Depends on https://github.com/ceph/ceph-docker/pull/478

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/osds.yml.sample                    | 10 ++++
 roles/ceph-osd/defaults/main.yml              | 10 ++++
 .../tasks/docker/start_docker_osd.yml         | 17 +++++--
 roles/ceph-osd/templates/ceph-osd-run.sh.j2   | 50 +++++++++++++++++++
 roles/ceph-osd/templates/ceph-osd.service.j2  | 21 ++------
 5 files changed, 86 insertions(+), 22 deletions(-)
 create mode 100644 roles/ceph-osd/templates/ceph-osd-run.sh.j2

diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample
index 7ee4363f9..20a234c91 100644
--- a/group_vars/osds.yml.sample
+++ b/group_vars/osds.yml.sample
@@ -134,6 +134,11 @@ dummy:
 #  - /dev/sdf
 #  - /dev/sdg
 #  - /dev/sdg
+#
+# NOTE(leseb):
+# On a containerized scenario we only support A SINGLE journal
+# for all the OSDs on a given machine. If you don't, bad things will happen
+# This is a limitation we plan to fix at some point.
 #raw_journal_devices: []
 
 
@@ -176,6 +181,11 @@ dummy:
 #kv_type: etcd
 #kv_endpoint: 127.0.0.1
 #kv_port: 4001
+
+# Add -e OSD_JOURNAL={{ raw_journal_devices }} to configure a journal device to ceph_osd_docker_prepare_env variable
+# make sure you only pass a single device, otherwise this will fail horribly.
+#
+# Add -e OSD_DMCRYPT=1 to use the collocated dmcrypt scenario to the ceph_osd_docker_prepare_env and ceph_osd_docker_extra_env variables
 #ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
 #ceph_docker_image: "ceph/daemon"
 #ceph_docker_image_tag: latest
diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index b0e00ce0f..8672d9ba8 100644
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -126,6 +126,11 @@ raw_multi_journal: false
 #  - /dev/sdf
 #  - /dev/sdg
 #  - /dev/sdg
+#
+# NOTE(leseb):
+# On a containerized scenario we only support A SINGLE journal
+# for all the OSDs on a given machine. If you don't, bad things will happen
+# This is a limitation we plan to fix at some point.
 raw_journal_devices: []
 
 
@@ -168,6 +173,11 @@ osd_containerized_deployment_with_kv: false
 kv_type: etcd
 kv_endpoint: 127.0.0.1
 kv_port: 4001
+
+# Add -e OSD_JOURNAL={{ raw_journal_devices }} to configure a journal device to ceph_osd_docker_prepare_env variable
+# make sure you only pass a single device, otherwise this will fail horribly.
+#
+# Add -e OSD_DMCRYPT=1 to use the collocated dmcrypt scenario to the ceph_osd_docker_prepare_env and ceph_osd_docker_extra_env variables
 ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
 ceph_docker_image: "ceph/daemon"
 ceph_docker_image_tag: latest
diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index a694523f2..973569e7c 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -24,13 +24,13 @@
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 |
-    regex_replace('/', '') }}" \
+    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}" \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
     -v /etc/localtime:/etc/localtime:ro \
     -e "OSD_DEVICE={{ item.0 }}" \
+    -e "OSD_JOURNAL_UUID=$(python -c "import uuid; print uuid.uuid5(uuid.NAMESPACE_DNS, '{{ ansible_machine_id }}{{ item.0 }}')")" \
     -e CEPH_DAEMON=OSD_CEPH_DISK_PREPARE \
     {{ ceph_osd_docker_prepare_env }} \
     "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
@@ -47,11 +47,11 @@
     docker run  --net=host   \
     --pid=host \
     --privileged=true \
-    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 |
-    regex_replace('/', '') }}" \
+    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}" \
     -v /dev:/dev \
     -v /etc/localtime:/etc/localtime:ro \
     -e "OSD_DEVICE={{ item.0 }}" \
+    -e "OSD_JOURNAL_UUID=$(python -c "import uuid; print uuid.uuid5(uuid.NAMESPACE_DNS, '{{ ansible_machine_id }}{{ item.0 }}')")" \
     -e "{{ ceph_osd_docker_prepare_env }}" \
     -e CEPH_DAEMON=OSD_CEPH_DISK_PREPARE \
     -e KV_TYPE={{kv_type}} \
@@ -67,6 +67,15 @@
     - ceph_osd_docker_prepare_env is defined
     - osd_containerized_deployment_with_kv
 
+- name: generate ceph osd docker run script
+  become: true
+  template:
+    src: "{{ role_path }}/templates/ceph-osd-run.sh.j2"
+    dest: /usr/share/ceph-osd-run.sh
+    owner: "root"
+    group: "root"
+    mode: "0744"
+
 - name: generate systemd unit file
   become: true
   template:
diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
new file mode 100644
index 000000000..0689be9e0
--- /dev/null
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -0,0 +1,50 @@
+#!/bin/bash
+# {{ ansible_managed }}
+
+if [[ "$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}* | wc -l)" -gt 0 ]] ; then
+  for part in /dev/${1}*; do
+    if [[ "$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID ${part} | wc -l)" -gt 0 ]]; then
+DEVICES="${DEVICES} --device=/dev/disk/by-partuuid/$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID ${part}) "
+    fi
+  done
+  # we test if the dm exist, if it does we add it to --device list
+  # if not we don't add it, the first activation will fail
+  # however the dm will be created, on the second run it'll added to the device list
+  # the second run will succeed
+  blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}1
+  # make sure blkid returns 0 otherwise we will test /dev/mapper/ which always exists
+	if [[ -e /dev/mapper/$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}1) && "$?" -eq 0 ]]; then
+    DEVICES="${DEVICES} --device=/dev/disk/by-partuuid/$(blkid -t PARTLABEL="ceph lockbox" -o value -s PARTUUID /dev/${1}3) --device=/dev/${1}3 --device=/dev/mapper/control --device=/dev/mapper/$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}2) --device=/dev/mapper/$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}1)"
+  else
+    DEVICES="${DEVICES} --device=/dev/disk/by-partuuid/$(blkid -t PARTLABEL="ceph lockbox" -o value -s PARTUUID /dev/${1}3) --device=/dev/${1}3 --device=/dev/mapper/control --device=/dev/mapper/$(blkid -t TYPE=crypto_LUKS -o value -s PARTUUID /dev/${1}2)"
+  fi
+fi
+
+/usr/bin/docker run \
+  --rm \
+  --net=host \
+  --cap-add SYS_ADMIN \
+  --pid=host \
+  {% if not osd_containerized_deployment_with_kv -%}
+  -v /var/lib/ceph:/var/lib/ceph \
+  -v /etc/ceph:/etc/ceph \
+  {% else -%}
+  -e KV_TYPE={{kv_type}} \
+  -e KV_IP={{kv_endpoint}} \
+  -e KV_PORT={{kv_port}} \
+  {% endif -%}
+  -v /etc/localtime:/etc/localtime:ro \
+  --device=/dev/${1} \
+  --device=/dev/${1}1 \
+  {% if raw_journal_devices|length > 0 -%}
+  -e OSD_JOURNAL={{ raw_journal_devices[0] }} \
+  --device={{ raw_journal_devices[0] }} \
+  {% else -%}
+  --device=/dev/${1}2 \
+  {% endif -%}
+  --device=/dev/disk/by-partuuid/$(python -c "import uuid; f = open('/etc/machine-id', 'r').read(); print uuid.uuid5(uuid.NAMESPACE_DNS, f.strip() + '/dev/$1')") ${DEVICES} \
+  -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
+  -e OSD_DEVICE=/dev/${1} \
+  {{ ceph_osd_docker_extra_env }} \
+  --name={{ ansible_hostname }}-osd-dev${1} \
+  {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
diff --git a/roles/ceph-osd/templates/ceph-osd.service.j2 b/roles/ceph-osd/templates/ceph-osd.service.j2
index 46bee9ff6..cd55833a5 100644
--- a/roles/ceph-osd/templates/ceph-osd.service.j2
+++ b/roles/ceph-osd/templates/ceph-osd.service.j2
@@ -1,3 +1,4 @@
+# {{ ansible_managed }}
 [Unit]
 Description=Ceph OSD
 After=docker.service
@@ -5,24 +6,8 @@ After=docker.service
 [Service]
 EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
-ExecStartPre=-/usr/bin/docker rm -f  {{ ansible_hostname }}-osd-dev%i
-ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
-   {% if not osd_containerized_deployment_with_kv -%}
-   -v /var/lib/ceph:/var/lib/ceph \
-   -v /etc/ceph:/etc/ceph \
-   {% else -%}
-   -e KV_TYPE={{kv_type}} \
-   -e KV_IP={{kv_endpoint}} \
-   -e KV_PORT={{kv_port}} \
-   {% endif -%}
-   -v /etc/localtime:/etc/localtime:ro \
-   -v /dev:/dev \
-   --privileged \
-   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
-   -e OSD_DEVICE=/dev/%i \
-   {{ ceph_osd_docker_extra_env }} \
-   --name={{ ansible_hostname }}-osd-dev%i \
-   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
+ExecStartPre=-/usr/bin/docker rm -f {{ ansible_hostname }}-osd-dev%i
+ExecStart=/usr/share/ceph-osd-run.sh %i
 ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
 Restart=always
 RestartSec=10s

From 7aabbc931d9792584c7fde092ef2e0fb01fa42bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 7 Feb 2017 22:08:47 +0100
Subject: [PATCH 14/80] tests: add scenario for dedicated-journal on docker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../Vagrantfile                               |  1 +
 .../group_vars/all                            | 25 ++++++++
 .../7/docker-cluster-dedicated-journal/hosts  |  5 ++
 .../vagrant_variables.yml                     | 61 +++++++++++++++++++
 4 files changed, 92 insertions(+)
 create mode 120000 tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
 create mode 100644 tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
 create mode 100644 tests/functional/centos/7/docker-cluster-dedicated-journal/hosts
 create mode 100644 tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml

diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile b/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
new file mode 120000
index 000000000..dfd7436c9
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
@@ -0,0 +1 @@
+../../../../../Vagrantfile
\ No newline at end of file
diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
new file mode 100644
index 000000000..9230cfcec
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
@@ -0,0 +1,25 @@
+---
+# this is only here to let the CI tests know
+# that this scenario is using docker
+docker: True
+
+ceph_stable: True
+mon_containerized_deployment: True
+osd_containerized_deployment: True
+mds_containerized_deployment: True
+rgw_containerized_deployment: True
+cluster: test
+ceph_mon_docker_interface: eth1
+ceph_mon_docker_subnet: "{{ public_network }}"
+journal_size: 100
+ceph_docker_on_openstack: False
+public_network: "192.168.15.0/24"
+cluster_network: "192.168.16.0/24"
+journal_collocation: true
+ceph_rgw_civetweb_port: 8080
+ceph_osd_docker_devices: "{{ devices }}"
+devices:
+  - /dev/sda
+raw_journal_devices:
+  - /dev/sdb
+ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/hosts b/tests/functional/centos/7/docker-cluster-dedicated-journal/hosts
new file mode 100644
index 000000000..f6a265ab3
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/hosts
@@ -0,0 +1,5 @@
+[mons]
+mon0
+
+[osds]
+osd0
diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml
new file mode 100644
index 000000000..0c54f6978
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml
@@ -0,0 +1,61 @@
+---
+
+# DEPLOY CONTAINERIZED DAEMONS
+docker: True
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 1
+mds_vms: 0
+rgw_vms: 0
+nfs_vms: 0
+rbd_mirror_vms: 0
+client_vms: 0
+iscsi_gw_vms: 0
+
+# Deploy RESTAPI on each of the Monitors
+restapi: true
+
+# SUBNETS TO USE FOR THE VMS
+public_subnet: 192.168.15
+cluster_subnet: 192.168.16
+
+# MEMORY
+# set 1024 for CentOS
+memory: 1024
+
+# Disks
+# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]"
+# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]"
+disks: "[ '/dev/sda', '/dev/sdb' ]"
+
+# VAGRANT BOX
+# Ceph boxes are *strongly* suggested. They are under better control and will
+# not get updated frequently unless required for build systems. These are (for
+# now):
+#
+# * ceph/ubuntu-xenial
+#
+# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64
+# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet
+# libvirt CentOS: centos/7
+# parallels Ubuntu: parallels/ubuntu-14.04
+# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller'
+# For more boxes have a look at:
+#   - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q=
+#   - https://download.gluster.org/pub/gluster/purpleidea/vagrant/
+vagrant_box: centos/atomic-host
+#ssh_private_key_path: "~/.ssh/id_rsa"
+# The sync directory changes based on vagrant box
+# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
+#vagrant_sync_dir: /home/vagrant/sync
+#vagrant_sync_dir: /
+# Disables synced folder creation. Not needed for testing, will skip mounting
+# the vagrant directory on the remote box regardless of the provider.
+vagrant_disable_synced_folder: true
+# VAGRANT URL
+# This is a URL to download an image from an alternate location.  vagrant_box
+# above should be set to the filename of the image.
+# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box
+# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box

From 36eaca693bbbf119691f933c3bdcf2b3ce8eaf65 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Fri, 10 Feb 2017 10:37:52 -0600
Subject: [PATCH 15/80] tests: enable the docker_dedicated_journal scenario

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 0b4725a3b..c877a9ca7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,update_dmcrypt}
+envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,update_dmcrypt}
 skipsdist = True
 
 # extra commands for purging clusters
@@ -38,6 +38,7 @@ setenv=
   # only available for ansible >= 2.2
   ANSIBLE_STDOUT_CALLBACK = debug
   docker_cluster: PLAYBOOK = site-docker.yml.sample
+  docker_dedicated_journal: PLAYBOOK = site-docker.yml.sample
 deps=
   ansible1.9: ansible==1.9.4
   ansible2.1: ansible==2.1
@@ -56,6 +57,7 @@ changedir=
   centos7_cluster: {toxinidir}/tests/functional/centos/7/cluster
   # tests a 1 mon, 1 osd, 1 mds and 1 rgw centos7 cluster using docker
   docker_cluster: {toxinidir}/tests/functional/centos/7/docker-cluster
+  docker_dedicated_journal: {toxinidir}/tests/functional/centos/7/docker-cluster-dedicated-journal
   purge_cluster: {toxinidir}/tests/functional/ubuntu/16.04/cluster
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal

From 7b216aa8e03688ae9bf8909f98946d573a63bd4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 16 Feb 2017 21:25:29 +0100
Subject: [PATCH 16/80] ci: add docker-cluster-dmcrypt-journal-collocation
 scenario
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../Vagrantfile                               | 498 ++++++++++++++++++
 .../group_vars/all                            |  26 +
 .../hosts                                     |   5 +
 .../vagrant_variables.yml                     |  51 ++
 tox.ini                                       |   4 +-
 5 files changed, 583 insertions(+), 1 deletion(-)
 create mode 100644 tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/Vagrantfile
 create mode 100644 tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/group_vars/all
 create mode 100644 tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/hosts
 create mode 100644 tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml

diff --git a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/Vagrantfile b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/Vagrantfile
new file mode 100644
index 000000000..d232095b3
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/Vagrantfile
@@ -0,0 +1,498 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require 'yaml'
+require 'time'
+VAGRANTFILE_API_VERSION = '2'
+
+DEBUG = false
+
+config_file=File.expand_path(File.join(File.dirname(__FILE__), 'vagrant_variables.yml'))
+settings=YAML.load_file(config_file)
+
+LABEL_PREFIX   = settings['label_prefix'] ? settings['label_prefix'] + "-" : ""
+NMONS          = settings['mon_vms']
+NOSDS          = settings['osd_vms']
+NMDSS          = settings['mds_vms']
+NRGWS          = settings['rgw_vms']
+NNFSS          = settings['nfs_vms']
+RESTAPI        = settings['restapi']
+NRBD_MIRRORS   = settings['rbd_mirror_vms']
+CLIENTS        = settings['client_vms']
+NISCSI_GWS     = settings['iscsi_gw_vms']
+PUBLIC_SUBNET  = settings['public_subnet']
+CLUSTER_SUBNET = settings['cluster_subnet']
+BOX            = settings['vagrant_box']
+BOX_URL        = settings['vagrant_box_url']
+SYNC_DIR       = settings['vagrant_sync_dir']
+MEMORY         = settings['memory']
+ETH            = settings['eth']
+DOCKER         = settings['docker']
+USER           = settings['ssh_username']
+
+ASSIGN_STATIC_IP = !(BOX == 'openstack' or BOX == 'linode')
+DISABLE_SYNCED_FOLDER = settings.fetch('vagrant_disable_synced_folder', false)
+DISK_UUID = Time.now.utc.to_i
+
+
+ansible_provision = proc do |ansible|
+  if DOCKER then
+    ansible.playbook = 'site-docker.yml'
+    if settings['skip_tags']
+      ansible.skip_tags = settings['skip_tags']
+    end
+  else
+    ansible.playbook = 'site.yml'
+  end
+
+  # Note: Can't do ranges like mon[0-2] in groups because
+  # these aren't supported by Vagrant, see
+  # https://github.com/mitchellh/vagrant/issues/3539
+  ansible.groups = {
+    'mons'            => (0..NMONS - 1).map { |j| "#{LABEL_PREFIX}mon#{j}" },
+    'osds'            => (0..NOSDS - 1).map { |j| "#{LABEL_PREFIX}osd#{j}" },
+    'mdss'            => (0..NMDSS - 1).map { |j| "#{LABEL_PREFIX}mds#{j}" },
+    'rgws'            => (0..NRGWS - 1).map { |j| "#{LABEL_PREFIX}rgw#{j}" },
+    'nfss'            => (0..NNFSS - 1).map { |j| "#{LABEL_PREFIX}nfs#{j}" },
+    'rbd_mirrors'     => (0..NRBD_MIRRORS - 1).map { |j| "#{LABEL_PREFIX}rbd_mirror#{j}" },
+    'clients'         => (0..CLIENTS - 1).map { |j| "#{LABEL_PREFIX}client#{j}" },
+    'iscsi_gw'        => (0..NISCSI_GWS - 1).map { |j| "#{LABEL_PREFIX}iscsi_gw#{j}" }
+  }
+
+  if RESTAPI then
+    ansible.groups['restapis'] = (0..NMONS - 1).map { |j| "#{LABEL_PREFIX}mon#{j}" }
+  end
+
+  ansible.extra_vars = {
+      cluster_network: "#{CLUSTER_SUBNET}.0/24",
+      journal_size: 100,
+      public_network: "#{PUBLIC_SUBNET}.0/24",
+  }
+
+  # In a production deployment, these should be secret
+  if DOCKER then
+    ansible.extra_vars = ansible.extra_vars.merge({
+      mon_containerized_deployment: 'true',
+      osd_containerized_deployment: 'true',
+      mds_containerized_deployment: 'true',
+      rgw_containerized_deployment: 'true',
+      nfs_containerized_deployment: 'true',
+      restapi_containerized_deployment: 'true',
+      rbd_mirror_containerized_deployment: 'true',
+      ceph_mon_docker_interface: ETH,
+      ceph_mon_docker_subnet: "#{PUBLIC_SUBNET}.0/24",
+      ceph_osd_docker_devices: settings['disks'],
+      devices: settings['disks'],
+      ceph_docker_on_openstack: BOX == 'openstack',
+      ceph_rgw_civetweb_port: 8080,
+      generate_fsid: 'true',
+    })
+  else
+    ansible.extra_vars = ansible.extra_vars.merge({
+      devices: settings['disks'],
+      journal_collocation: 'true',
+      monitor_interface: ETH,
+      os_tuning_params: settings['os_tuning_params'],
+      pool_default_size: '2',
+    })
+  end
+
+  if BOX == 'linode' then
+    ansible.sudo = true
+    # Use monitor_address_block instead of monitor_interface:
+    ansible.extra_vars.delete(:monitor_interface)
+    ansible.extra_vars = ansible.extra_vars.merge({
+      cluster_network: "#{CLUSTER_SUBNET}.0/16",
+      devices: ['/dev/sdc'], # hardcode leftover disk
+      journal_collocation: 'true',
+      monitor_address_block: "#{PUBLIC_SUBNET}.0/16",
+      public_network: "#{PUBLIC_SUBNET}.0/16",
+    })
+  end
+
+  if DEBUG then
+    ansible.verbose = '-vvv'
+  end
+  ansible.limit = 'all'
+end
+
+def create_vmdk(name, size)
+  dir = Pathname.new(__FILE__).expand_path.dirname
+  path = File.join(dir, '.vagrant', name + '.vmdk')
+  `vmware-vdiskmanager -c -s #{size} -t 0 -a scsi #{path} \
+   2>&1 > /dev/null` unless File.exist?(path)
+end
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = BOX
+  config.vm.box_url = BOX_URL
+  config.ssh.insert_key = false # workaround for https://github.com/mitchellh/vagrant/issues/5048
+  config.ssh.private_key_path = settings['ssh_private_key_path']
+  config.ssh.username = USER
+
+  # Faster bootup. Disables mounting the sync folder for libvirt and virtualbox
+  if DISABLE_SYNCED_FOLDER
+    config.vm.provider :virtualbox do |v,override|
+      override.vm.synced_folder '.', SYNC_DIR, disabled: true
+    end
+    config.vm.provider :libvirt do |v,override|
+      override.vm.synced_folder '.', SYNC_DIR, disabled: true
+    end
+  end
+
+  if BOX == 'openstack'
+    # OpenStack VMs
+    config.vm.provider :openstack do |os|
+      config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true
+      config.ssh.pty = true
+      os.openstack_auth_url = settings['os_openstack_auth_url']
+      os.username = settings['os_username']
+      os.password = settings['os_password']
+      os.tenant_name = settings['os_tenant_name']
+      os.region = settings['os_region']
+      os.flavor = settings['os_flavor']
+      os.image = settings['os_image']
+      os.keypair_name = settings['os_keypair_name']
+      os.security_groups = ['default']
+
+      if settings['os.networks'] then
+        os.networks = settings['os_networks']
+      end
+
+      if settings['os.floating_ip_pool'] then
+        os.floating_ip_pool = settings['os_floating_ip_pool']
+      end
+
+      config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell"
+    end
+  elsif BOX == 'linode'
+    config.vm.provider :linode do |provider, override|
+      provider.token = ENV['LINODE_API_KEY']
+      provider.distribution = settings['cloud_distribution'] # 'Ubuntu 16.04 LTS'
+      provider.datacenter = settings['cloud_datacenter']
+      provider.plan = MEMORY.to_s
+      provider.private_networking = true
+      # root install generally takes <1GB
+      provider.xvda_size = 4*1024
+      # add some swap as the Linode distros require it
+      provider.swap_size = 128
+    end
+  end
+
+  (0..CLIENTS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}client#{i}" do |client|
+      client.vm.hostname = "#{LABEL_PREFIX}ceph-client#{i}"
+      if ASSIGN_STATIC_IP
+        client.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.4#{i}"
+      end
+      # Virtualbox
+      client.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      client.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      client.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      client.vm.provider "parallels" do |prl|
+        prl.name = "ceph-client#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      client.vm.provider :linode do |provider|
+        provider.label = client.vm.hostname
+      end
+    end
+  end
+
+  (0..NRGWS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}rgw#{i}" do |rgw|
+      rgw.vm.hostname = "#{LABEL_PREFIX}ceph-rgw#{i}"
+      if ASSIGN_STATIC_IP
+        rgw.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.5#{i}"
+      end
+
+      # Virtualbox
+      rgw.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      rgw.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      rgw.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      rgw.vm.provider "parallels" do |prl|
+        prl.name = "ceph-rgw#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      rgw.vm.provider :linode do |provider|
+        provider.label = rgw.vm.hostname
+      end
+    end
+  end
+
+  (0..NNFSS - 1).each do |i|
+    config.vm.define "nfs#{i}" do |nfs|
+      nfs.vm.hostname = "ceph-nfs#{i}"
+      if ASSIGN_STATIC_IP
+        nfs.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.6#{i}"
+      end
+
+      # Virtualbox
+      nfs.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      nfs.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      nfs.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      nfs.vm.provider "parallels" do |prl|
+        prl.name = "ceph-nfs#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      nfs.vm.provider :linode do |provider|
+        provider.label = nfs.vm.hostname
+      end
+    end
+  end
+
+  (0..NMDSS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}mds#{i}" do |mds|
+      mds.vm.hostname = "#{LABEL_PREFIX}ceph-mds#{i}"
+      if ASSIGN_STATIC_IP
+        mds.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.7#{i}"
+      end
+      # Virtualbox
+      mds.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      mds.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      mds.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      mds.vm.provider "parallels" do |prl|
+        prl.name = "ceph-mds#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      mds.vm.provider :linode do |provider|
+        provider.label = mds.vm.hostname
+      end
+    end
+  end
+
+  (0..NRBD_MIRRORS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}rbd_mirror#{i}" do |rbd_mirror|
+      rbd_mirror.vm.hostname = "#{LABEL_PREFIX}ceph-rbd-mirror#{i}"
+      if ASSIGN_STATIC_IP
+        rbd_mirror.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.8#{i}"
+      end
+      # Virtualbox
+      rbd_mirror.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      rbd_mirror.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      rbd_mirror.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      rbd_mirror.vm.provider "parallels" do |prl|
+        prl.name = "ceph-rbd-mirror#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      rbd_mirror.vm.provider :linode do |provider|
+        provider.label = rbd_mirror.vm.hostname
+      end
+    end
+  end
+
+  (0..NISCSI_GWS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}iscsi_gw#{i}" do |iscsi_gw|
+      iscsi_gw.vm.hostname = "#{LABEL_PREFIX}ceph-iscsi-gw#{i}"
+      if ASSIGN_STATIC_IP
+        iscsi_gw.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.9#{i}"
+      end
+      # Virtualbox
+      iscsi_gw.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      iscsi_gw.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      iscsi_gw.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      iscsi_gw.vm.provider "parallels" do |prl|
+        prl.name = "ceph-iscsi-gw#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      iscsi_gw.vm.provider :linode do |provider|
+        provider.label = iscsi_gw.vm.hostname
+      end
+    end
+  end
+
+  (0..NMONS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}mon#{i}" do |mon|
+      mon.vm.hostname = "#{LABEL_PREFIX}ceph-mon#{i}"
+      if ASSIGN_STATIC_IP
+        mon.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.1#{i}"
+      end
+      # Virtualbox
+      mon.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      mon.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      mon.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = false
+      end
+
+      # Parallels
+      mon.vm.provider "parallels" do |prl|
+        prl.name = "ceph-mon#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      mon.vm.provider :linode do |provider|
+        provider.label = mon.vm.hostname
+      end
+    end
+  end
+
+  (0..NOSDS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}osd#{i}" do |osd|
+      osd.vm.hostname = "#{LABEL_PREFIX}ceph-osd#{i}"
+      if ASSIGN_STATIC_IP
+        osd.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.10#{i}"
+        osd.vm.network :private_network,
+          ip: "#{CLUSTER_SUBNET}.20#{i}"
+      end
+      # Virtualbox
+      osd.vm.provider :virtualbox do |vb|
+        # Create our own controller for consistency and to remove VM dependency
+        vb.customize ['storagectl', :id,
+                      '--name', 'OSD Controller',
+                      '--add', 'scsi']
+        (0..1).each do |d|
+          vb.customize ['createhd',
+                        '--filename', "disk-#{i}-#{d}",
+                        '--size', '11000'] unless File.exist?("disk-#{i}-#{d}.vdi")
+          vb.customize ['storageattach', :id,
+                        '--storagectl', 'OSD Controller',
+                        '--port', 3 + d,
+                        '--device', 0,
+                        '--type', 'hdd',
+                        '--medium', "disk-#{i}-#{d}.vdi"]
+        end
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      osd.vm.provider :vmware_fusion do |v|
+        (0..1).each do |d|
+          v.vmx["scsi0:#{d + 1}.present"] = 'TRUE'
+          v.vmx["scsi0:#{d + 1}.fileName"] =
+            create_vmdk("disk-#{i}-#{d}", '11000MB')
+        end
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      driverletters = ('a'..'z').to_a
+      osd.vm.provider :libvirt do |lv|
+        # always make /dev/sd{a/b/c} so that CI can ensure that
+        # virtualbox and libvirt will have the same devices to use for OSDs
+        (0..2).each do |d|
+          lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => '12G', :bus => "ide"
+        end
+        lv.memory = MEMORY
+        lv.random_hostname = false
+      end
+
+      # Parallels
+      osd.vm.provider "parallels" do |prl|
+        prl.name = "ceph-osd#{i}"
+        prl.memory = "#{MEMORY}"
+        (0..1).each do |d|
+          prl.customize ["set", :id,
+                         "--device-add",
+                         "hdd",
+                         "--iface",
+                         "sata"]
+        end
+      end
+
+      osd.vm.provider :linode do |provider|
+        provider.label = osd.vm.hostname
+      end
+
+      # Run the provisioner after the last machine comes up
+      osd.vm.provision 'ansible', &ansible_provision if i == (NOSDS - 1)
+    end
+  end
+end
diff --git a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/group_vars/all b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/group_vars/all
new file mode 100644
index 000000000..a9178552f
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/group_vars/all
@@ -0,0 +1,26 @@
+---
+# this is only here to let the CI tests know
+# that this scenario is using docker
+docker: True
+
+ceph_stable: True
+mon_containerized_deployment: True
+osd_containerized_deployment: True
+mds_containerized_deployment: True
+rgw_containerized_deployment: True
+cluster: ceph
+ceph_mon_docker_interface: eth1
+ceph_mon_docker_subnet: "{{ public_network }}"
+journal_size: 100
+ceph_docker_on_openstack: False
+public_network: "192.168.15.0/24"
+cluster_network: "192.168.16.0/24"
+journal_collocation: true
+ceph_rgw_civetweb_port: 8080
+ceph_osd_docker_devices: "{{ devices }}"
+devices:
+  - /dev/sda
+raw_journal_devices:
+  - /dev/sdb
+ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1
+ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1
diff --git a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/hosts b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/hosts
new file mode 100644
index 000000000..f6a265ab3
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/hosts
@@ -0,0 +1,5 @@
+[mons]
+mon0
+
+[osds]
+osd0
diff --git a/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml
new file mode 100644
index 000000000..f77870126
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation/vagrant_variables.yml
@@ -0,0 +1,51 @@
+---
+
+# DEPLOY CONTAINERIZED DAEMONS
+docker: True
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 1
+mds_vms: 0
+rgw_vms: 0
+nfs_vms: 0
+rbd_mirror_vms: 0
+client_vms: 0
+iscsi_gw_vms: 0
+
+# Deploy RESTAPI on each of the Monitors
+restapi: true
+
+# SUBNETS TO USE FOR THE VMS
+public_subnet: 192.168.15
+cluster_subnet: 192.168.16
+
+# MEMORY
+# set 1024 for CentOS
+memory: 1024
+
+# Disks
+# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]"
+# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]"
+disks: "[ '/dev/sda', '/dev/sdb' ]"
+
+# VAGRANT BOX
+# Ceph boxes are *strongly* suggested. They are under better control and will
+# not get updated frequently unless required for build systems. These are (for
+# now):
+#
+# * ceph/ubuntu-xenial
+#
+# NOTE(leseb): we use centos for this scenario since we at least need Docker version 1.12.5
+# which is not available in Atomic Host.
+# There are bug like this one: https://github.com/docker/docker/issues/12694
+vagrant_box: centos/7
+
+#ssh_private_key_path: "~/.ssh/id_rsa"
+# The sync directory changes based on vagrant box
+# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
+#vagrant_sync_dir: /home/vagrant/sync
+#vagrant_sync_dir: /
+# Disables synced folder creation. Not needed for testing, will skip mounting
+# the vagrant directory on the remote box regardless of the provider.
+vagrant_disable_synced_folder: true
diff --git a/tox.ini b/tox.ini
index c877a9ca7..9fcb5f6be 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,update_dmcrypt}
+envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt}
 skipsdist = True
 
 # extra commands for purging clusters
@@ -39,6 +39,7 @@ setenv=
   ANSIBLE_STDOUT_CALLBACK = debug
   docker_cluster: PLAYBOOK = site-docker.yml.sample
   docker_dedicated_journal: PLAYBOOK = site-docker.yml.sample
+  docker_dmcrypt_journal_collocation = site-docker.yml.sample
 deps=
   ansible1.9: ansible==1.9.4
   ansible2.1: ansible==2.1
@@ -58,6 +59,7 @@ changedir=
   # tests a 1 mon, 1 osd, 1 mds and 1 rgw centos7 cluster using docker
   docker_cluster: {toxinidir}/tests/functional/centos/7/docker-cluster
   docker_dedicated_journal: {toxinidir}/tests/functional/centos/7/docker-cluster-dedicated-journal
+  docker_dmcrypt_journal_collocation: {toxinidir}/tests/functional/centos/7/docker-cluster-dmcrypt-journal-collocation
   purge_cluster: {toxinidir}/tests/functional/ubuntu/16.04/cluster
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal

From b91d227b995a9b87b0864211a74529675b9e17f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 16 Feb 2017 22:13:26 +0100
Subject: [PATCH 17/80] docker: make ceph docker osd script path
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Since distro will not allow /usr/share to be writable (e.g: atomic) so
we let the operator decide where to put that script.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/osds.yml.sample                              | 1 +
 roles/ceph-osd/defaults/main.yml                        | 1 +
 roles/ceph-osd/tasks/docker/start_docker_osd.yml        | 2 +-
 roles/ceph-osd/templates/ceph-osd.service.j2            | 2 +-
 tests/functional/centos/7/docker-cluster/group_vars/all | 1 +
 5 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample
index 20a234c91..1d154f4a1 100644
--- a/group_vars/osds.yml.sample
+++ b/group_vars/osds.yml.sample
@@ -193,4 +193,5 @@ dummy:
 #ceph_osd_docker_devices: "{{ devices }}"
 #ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME
+#ceph_osd_docker_run_script_path: "/usr/share"
 
diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index 8672d9ba8..c2713274a 100644
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -185,3 +185,4 @@ ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK
 ceph_osd_docker_devices: "{{ devices }}"
 ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME
+ceph_osd_docker_run_script_path: "/usr/share"
diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index 973569e7c..7d1760c19 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -71,7 +71,7 @@
   become: true
   template:
     src: "{{ role_path }}/templates/ceph-osd-run.sh.j2"
-    dest: /usr/share/ceph-osd-run.sh
+    dest: "{{ ceph_osd_docker_run_script_path }}/ceph-osd-run.sh"
     owner: "root"
     group: "root"
     mode: "0744"
diff --git a/roles/ceph-osd/templates/ceph-osd.service.j2 b/roles/ceph-osd/templates/ceph-osd.service.j2
index cd55833a5..e1b3f43f8 100644
--- a/roles/ceph-osd/templates/ceph-osd.service.j2
+++ b/roles/ceph-osd/templates/ceph-osd.service.j2
@@ -7,7 +7,7 @@ After=docker.service
 EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
 ExecStartPre=-/usr/bin/docker rm -f {{ ansible_hostname }}-osd-dev%i
-ExecStart=/usr/share/ceph-osd-run.sh %i
+ExecStart={{ ceph_osd_docker_run_script_path }}/ceph-osd-run.sh %i
 ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
 Restart=always
 RestartSec=10s
diff --git a/tests/functional/centos/7/docker-cluster/group_vars/all b/tests/functional/centos/7/docker-cluster/group_vars/all
index 11c3d413f..56cf5cbde 100644
--- a/tests/functional/centos/7/docker-cluster/group_vars/all
+++ b/tests/functional/centos/7/docker-cluster/group_vars/all
@@ -21,3 +21,4 @@ ceph_osd_docker_devices: "{{ devices }}"
 devices:
   - /dev/sda
   - /dev/sdb
+ceph_osd_docker_run_script_path: /var/tmp

From 3b633d5ddcd7578d907908549a12291e11fa58b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Sat, 18 Feb 2017 02:54:47 +0100
Subject: [PATCH 18/80] purge-docker: re-implement zap devices
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We now run the container and waits until it dies. Prior to this we were
stopping it before completion so not all the devices where zapped.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../purge-docker-cluster.yml                  | 50 +++++++------------
 1 file changed, 17 insertions(+), 33 deletions(-)

diff --git a/infrastructure-playbooks/purge-docker-cluster.yml b/infrastructure-playbooks/purge-docker-cluster.yml
index 5d601b42c..66cb1d768 100644
--- a/infrastructure-playbooks/purge-docker-cluster.yml
+++ b/infrastructure-playbooks/purge-docker-cluster.yml
@@ -283,43 +283,27 @@
     with_items: "{{ ceph_osd_docker_devices }}"
     ignore_errors: true
 
-  - name: zap ceph osd disk
-    docker:
-      image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
-      net: host
-      pid: host
-      state: started
-      privileged: yes
-      env: "CEPH_DAEMON=zap_device,OSD_DEVICE={{ item }}"
-      volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
+  - name: zap ceph osd disks
+    shell: |
+      docker run \
+      --privileged=true \
+      --name {{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }} \
+      -v /dev/:/dev/ \
+      -e OSD_DEVICE={{ item }} \
+      {{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
+      zap_device
     with_items:
       - "{{ ceph_osd_docker_devices }}"
       - "{{ raw_journal_devices }}"
 
-  - name: remove ceph osd zap disk container
-    docker:
-      image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
-      state: absent
-    with_items:
-      - "{{ ceph_osd_docker_devices }}"
-      - "{{ raw_journal_devices }}"
-
-  # zap twice
-  - name: zap ceph osd disk
-    docker:
-      image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
-      net: host
-      pid: host
-      state: started
-      privileged: yes
-      env: "CEPH_DAEMON=zap_device,OSD_DEVICE={{ item }}"
-      volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
-    with_items:
-      - "{{ ceph_osd_docker_devices }}"
-      - "{{ raw_journal_devices }}"
+  - name: wait until the zap containers die
+    shell: |
+      docker ps | grep -sq {{ ansible_hostname }}-osd-zap
+    register: zap_alive
+    failed_when: false
+    until: zap_alive.rc != 0
+    retries: 5
+    delay: 10
 
   - name: remove ceph osd zap disk container
     docker:

From 72b17d248046c74c1ddd1f47be0eb6a552ba6f10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Sat, 18 Feb 2017 14:48:18 +0100
Subject: [PATCH 19/80] docker: osd, clarify variable usage for scenarii
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/osds.yml.sample       | 34 ++++++++++++++++++++++----------
 roles/ceph-osd/defaults/main.yml | 34 ++++++++++++++++++++++----------
 2 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample
index 1d154f4a1..2fbae8006 100644
--- a/group_vars/osds.yml.sample
+++ b/group_vars/osds.yml.sample
@@ -181,17 +181,31 @@ dummy:
 #kv_type: etcd
 #kv_endpoint: 127.0.0.1
 #kv_port: 4001
-
-# Add -e OSD_JOURNAL={{ raw_journal_devices }} to configure a journal device to ceph_osd_docker_prepare_env variable
-# make sure you only pass a single device, otherwise this will fail horribly.
-#
-# Add -e OSD_DMCRYPT=1 to use the collocated dmcrypt scenario to the ceph_osd_docker_prepare_env and ceph_osd_docker_extra_env variables
-#ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
 #ceph_docker_image: "ceph/daemon"
 #ceph_docker_image_tag: latest
-#ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
-#ceph_osd_docker_devices: "{{ devices }}"
-#ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME
-#ceph_osd_docker_run_script_path: "/usr/share"
+#ceph_docker_on_openstack: false
+
+# PREPARE DEVICE
+# Make sure you only pass a single device to raw_journal_devices, otherwise this will fail horribly.
+# This is why we use [0] in the example.
+#
+# WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above
+#
+# Examples:
+# Journal collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+# Dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1
+# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+#
+#ceph_osd_docker_devices: "{{ devices }}"
+#ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+
+# ACTIVATE DEVICE
+# Examples:
+# Journal collocated or Dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1
+#
+#ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+#ceph_osd_docker_run_script_path: "/usr/share" # script called by systemd to run the docker command
 
diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index c2713274a..d32625113 100644
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -173,16 +173,30 @@ osd_containerized_deployment_with_kv: false
 kv_type: etcd
 kv_endpoint: 127.0.0.1
 kv_port: 4001
-
-# Add -e OSD_JOURNAL={{ raw_journal_devices }} to configure a journal device to ceph_osd_docker_prepare_env variable
-# make sure you only pass a single device, otherwise this will fail horribly.
-#
-# Add -e OSD_DMCRYPT=1 to use the collocated dmcrypt scenario to the ceph_osd_docker_prepare_env and ceph_osd_docker_extra_env variables
-ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
 ceph_docker_image: "ceph/daemon"
 ceph_docker_image_tag: latest
-ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
-ceph_osd_docker_devices: "{{ devices }}"
-ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME
-ceph_osd_docker_run_script_path: "/usr/share"
+ceph_docker_on_openstack: false
+
+# PREPARE DEVICE
+# Make sure you only pass a single device to raw_journal_devices, otherwise this will fail horribly.
+# This is why we use [0] in the example.
+#
+# WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above
+#
+# Examples:
+# Journal collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+# Dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1
+# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+#
+ceph_osd_docker_devices: "{{ devices }}"
+ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+
+# ACTIVATE DEVICE
+# Examples:
+# Journal collocated or Dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1
+#
+ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+ceph_osd_docker_run_script_path: "/usr/share" # script called by systemd to run the docker command

From 458a9ad5c3557ee2a1e6c801c2db6721883dbd46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 10:33:40 -0500
Subject: [PATCH 20/80] mon: docker, ability to enable centos extra repo
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/mons.yml.sample                    | 1 +
 roles/ceph-mon/defaults/main.yml              | 1 +
 roles/ceph-mon/tasks/docker/pre_requisite.yml | 1 +
 3 files changed, 3 insertions(+)

diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample
index 16bd40746..4dc8b7c99 100644
--- a/group_vars/mons.yml.sample
+++ b/group_vars/mons.yml.sample
@@ -88,4 +88,5 @@ dummy:
 #mon_docker_privileged: false
 #mon_docker_net_host: true
 #ceph_config_keys: [] # DON'T TOUCH ME
+#ceph_mon_docker_enable_centos_extra_repo: false
 
diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index a9a837ec0..e6d102e54 100644
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -80,3 +80,4 @@ ceph_docker_on_openstack: false
 mon_docker_privileged: false
 mon_docker_net_host: true
 ceph_config_keys: [] # DON'T TOUCH ME
+ceph_mon_docker_enable_centos_extra_repo: false
diff --git a/roles/ceph-mon/tasks/docker/pre_requisite.yml b/roles/ceph-mon/tasks/docker/pre_requisite.yml
index 5a8135ed5..41f4b2bb6 100644
--- a/roles/ceph-mon/tasks/docker/pre_requisite.yml
+++ b/roles/ceph-mon/tasks/docker/pre_requisite.yml
@@ -41,6 +41,7 @@
     enabled: yes
   when:
     - ansible_distribution == 'CentOS'
+    - ceph_mon_docker_enable_centos_extra_repo
   tags:
     with_pkg
 

From e9311bcc7463648f1042fb7557a81156ddbd2194 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 13:39:47 -0500
Subject: [PATCH 21/80] ci: do not generate random hostname for
 ansible2.2-docker_dedicated_journal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes the error: Call to virDomainCreateWithFlags failed: internal
error: Monitor path
/var/lib/libvirt/qemu/domain-docker-cluster-dedicated-journal_osd0_1487692576_dbfc21d851071d3e2cd2/monitor.sock
too big for destination

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../Vagrantfile                               | 499 +++++++++++++++++-
 1 file changed, 498 insertions(+), 1 deletion(-)
 mode change 120000 => 100644 tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile

diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile b/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
deleted file mode 120000
index dfd7436c9..000000000
--- a/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
+++ /dev/null
@@ -1 +0,0 @@
-../../../../../Vagrantfile
\ No newline at end of file
diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile b/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
new file mode 100644
index 000000000..d232095b3
--- /dev/null
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/Vagrantfile
@@ -0,0 +1,498 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+require 'yaml'
+require 'time'
+VAGRANTFILE_API_VERSION = '2'
+
+DEBUG = false
+
+config_file=File.expand_path(File.join(File.dirname(__FILE__), 'vagrant_variables.yml'))
+settings=YAML.load_file(config_file)
+
+LABEL_PREFIX   = settings['label_prefix'] ? settings['label_prefix'] + "-" : ""
+NMONS          = settings['mon_vms']
+NOSDS          = settings['osd_vms']
+NMDSS          = settings['mds_vms']
+NRGWS          = settings['rgw_vms']
+NNFSS          = settings['nfs_vms']
+RESTAPI        = settings['restapi']
+NRBD_MIRRORS   = settings['rbd_mirror_vms']
+CLIENTS        = settings['client_vms']
+NISCSI_GWS     = settings['iscsi_gw_vms']
+PUBLIC_SUBNET  = settings['public_subnet']
+CLUSTER_SUBNET = settings['cluster_subnet']
+BOX            = settings['vagrant_box']
+BOX_URL        = settings['vagrant_box_url']
+SYNC_DIR       = settings['vagrant_sync_dir']
+MEMORY         = settings['memory']
+ETH            = settings['eth']
+DOCKER         = settings['docker']
+USER           = settings['ssh_username']
+
+ASSIGN_STATIC_IP = !(BOX == 'openstack' or BOX == 'linode')
+DISABLE_SYNCED_FOLDER = settings.fetch('vagrant_disable_synced_folder', false)
+DISK_UUID = Time.now.utc.to_i
+
+
+ansible_provision = proc do |ansible|
+  if DOCKER then
+    ansible.playbook = 'site-docker.yml'
+    if settings['skip_tags']
+      ansible.skip_tags = settings['skip_tags']
+    end
+  else
+    ansible.playbook = 'site.yml'
+  end
+
+  # Note: Can't do ranges like mon[0-2] in groups because
+  # these aren't supported by Vagrant, see
+  # https://github.com/mitchellh/vagrant/issues/3539
+  ansible.groups = {
+    'mons'            => (0..NMONS - 1).map { |j| "#{LABEL_PREFIX}mon#{j}" },
+    'osds'            => (0..NOSDS - 1).map { |j| "#{LABEL_PREFIX}osd#{j}" },
+    'mdss'            => (0..NMDSS - 1).map { |j| "#{LABEL_PREFIX}mds#{j}" },
+    'rgws'            => (0..NRGWS - 1).map { |j| "#{LABEL_PREFIX}rgw#{j}" },
+    'nfss'            => (0..NNFSS - 1).map { |j| "#{LABEL_PREFIX}nfs#{j}" },
+    'rbd_mirrors'     => (0..NRBD_MIRRORS - 1).map { |j| "#{LABEL_PREFIX}rbd_mirror#{j}" },
+    'clients'         => (0..CLIENTS - 1).map { |j| "#{LABEL_PREFIX}client#{j}" },
+    'iscsi_gw'        => (0..NISCSI_GWS - 1).map { |j| "#{LABEL_PREFIX}iscsi_gw#{j}" }
+  }
+
+  if RESTAPI then
+    ansible.groups['restapis'] = (0..NMONS - 1).map { |j| "#{LABEL_PREFIX}mon#{j}" }
+  end
+
+  ansible.extra_vars = {
+      cluster_network: "#{CLUSTER_SUBNET}.0/24",
+      journal_size: 100,
+      public_network: "#{PUBLIC_SUBNET}.0/24",
+  }
+
+  # In a production deployment, these should be secret
+  if DOCKER then
+    ansible.extra_vars = ansible.extra_vars.merge({
+      mon_containerized_deployment: 'true',
+      osd_containerized_deployment: 'true',
+      mds_containerized_deployment: 'true',
+      rgw_containerized_deployment: 'true',
+      nfs_containerized_deployment: 'true',
+      restapi_containerized_deployment: 'true',
+      rbd_mirror_containerized_deployment: 'true',
+      ceph_mon_docker_interface: ETH,
+      ceph_mon_docker_subnet: "#{PUBLIC_SUBNET}.0/24",
+      ceph_osd_docker_devices: settings['disks'],
+      devices: settings['disks'],
+      ceph_docker_on_openstack: BOX == 'openstack',
+      ceph_rgw_civetweb_port: 8080,
+      generate_fsid: 'true',
+    })
+  else
+    ansible.extra_vars = ansible.extra_vars.merge({
+      devices: settings['disks'],
+      journal_collocation: 'true',
+      monitor_interface: ETH,
+      os_tuning_params: settings['os_tuning_params'],
+      pool_default_size: '2',
+    })
+  end
+
+  if BOX == 'linode' then
+    ansible.sudo = true
+    # Use monitor_address_block instead of monitor_interface:
+    ansible.extra_vars.delete(:monitor_interface)
+    ansible.extra_vars = ansible.extra_vars.merge({
+      cluster_network: "#{CLUSTER_SUBNET}.0/16",
+      devices: ['/dev/sdc'], # hardcode leftover disk
+      journal_collocation: 'true',
+      monitor_address_block: "#{PUBLIC_SUBNET}.0/16",
+      public_network: "#{PUBLIC_SUBNET}.0/16",
+    })
+  end
+
+  if DEBUG then
+    ansible.verbose = '-vvv'
+  end
+  ansible.limit = 'all'
+end
+
+def create_vmdk(name, size)
+  dir = Pathname.new(__FILE__).expand_path.dirname
+  path = File.join(dir, '.vagrant', name + '.vmdk')
+  `vmware-vdiskmanager -c -s #{size} -t 0 -a scsi #{path} \
+   2>&1 > /dev/null` unless File.exist?(path)
+end
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  config.vm.box = BOX
+  config.vm.box_url = BOX_URL
+  config.ssh.insert_key = false # workaround for https://github.com/mitchellh/vagrant/issues/5048
+  config.ssh.private_key_path = settings['ssh_private_key_path']
+  config.ssh.username = USER
+
+  # Faster bootup. Disables mounting the sync folder for libvirt and virtualbox
+  if DISABLE_SYNCED_FOLDER
+    config.vm.provider :virtualbox do |v,override|
+      override.vm.synced_folder '.', SYNC_DIR, disabled: true
+    end
+    config.vm.provider :libvirt do |v,override|
+      override.vm.synced_folder '.', SYNC_DIR, disabled: true
+    end
+  end
+
+  if BOX == 'openstack'
+    # OpenStack VMs
+    config.vm.provider :openstack do |os|
+      config.vm.synced_folder ".", "/home/#{USER}/vagrant", disabled: true
+      config.ssh.pty = true
+      os.openstack_auth_url = settings['os_openstack_auth_url']
+      os.username = settings['os_username']
+      os.password = settings['os_password']
+      os.tenant_name = settings['os_tenant_name']
+      os.region = settings['os_region']
+      os.flavor = settings['os_flavor']
+      os.image = settings['os_image']
+      os.keypair_name = settings['os_keypair_name']
+      os.security_groups = ['default']
+
+      if settings['os.networks'] then
+        os.networks = settings['os_networks']
+      end
+
+      if settings['os.floating_ip_pool'] then
+        os.floating_ip_pool = settings['os_floating_ip_pool']
+      end
+
+      config.vm.provision "shell", inline: "true", upload_path: "/home/#{USER}/vagrant-shell"
+    end
+  elsif BOX == 'linode'
+    config.vm.provider :linode do |provider, override|
+      provider.token = ENV['LINODE_API_KEY']
+      provider.distribution = settings['cloud_distribution'] # 'Ubuntu 16.04 LTS'
+      provider.datacenter = settings['cloud_datacenter']
+      provider.plan = MEMORY.to_s
+      provider.private_networking = true
+      # root install generally takes <1GB
+      provider.xvda_size = 4*1024
+      # add some swap as the Linode distros require it
+      provider.swap_size = 128
+    end
+  end
+
+  (0..CLIENTS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}client#{i}" do |client|
+      client.vm.hostname = "#{LABEL_PREFIX}ceph-client#{i}"
+      if ASSIGN_STATIC_IP
+        client.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.4#{i}"
+      end
+      # Virtualbox
+      client.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      client.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      client.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      client.vm.provider "parallels" do |prl|
+        prl.name = "ceph-client#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      client.vm.provider :linode do |provider|
+        provider.label = client.vm.hostname
+      end
+    end
+  end
+
+  (0..NRGWS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}rgw#{i}" do |rgw|
+      rgw.vm.hostname = "#{LABEL_PREFIX}ceph-rgw#{i}"
+      if ASSIGN_STATIC_IP
+        rgw.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.5#{i}"
+      end
+
+      # Virtualbox
+      rgw.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      rgw.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      rgw.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      rgw.vm.provider "parallels" do |prl|
+        prl.name = "ceph-rgw#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      rgw.vm.provider :linode do |provider|
+        provider.label = rgw.vm.hostname
+      end
+    end
+  end
+
+  (0..NNFSS - 1).each do |i|
+    config.vm.define "nfs#{i}" do |nfs|
+      nfs.vm.hostname = "ceph-nfs#{i}"
+      if ASSIGN_STATIC_IP
+        nfs.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.6#{i}"
+      end
+
+      # Virtualbox
+      nfs.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      nfs.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      nfs.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+
+      # Parallels
+      nfs.vm.provider "parallels" do |prl|
+        prl.name = "ceph-nfs#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      nfs.vm.provider :linode do |provider|
+        provider.label = nfs.vm.hostname
+      end
+    end
+  end
+
+  (0..NMDSS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}mds#{i}" do |mds|
+      mds.vm.hostname = "#{LABEL_PREFIX}ceph-mds#{i}"
+      if ASSIGN_STATIC_IP
+        mds.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.7#{i}"
+      end
+      # Virtualbox
+      mds.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      mds.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      mds.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      mds.vm.provider "parallels" do |prl|
+        prl.name = "ceph-mds#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      mds.vm.provider :linode do |provider|
+        provider.label = mds.vm.hostname
+      end
+    end
+  end
+
+  (0..NRBD_MIRRORS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}rbd_mirror#{i}" do |rbd_mirror|
+      rbd_mirror.vm.hostname = "#{LABEL_PREFIX}ceph-rbd-mirror#{i}"
+      if ASSIGN_STATIC_IP
+        rbd_mirror.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.8#{i}"
+      end
+      # Virtualbox
+      rbd_mirror.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      rbd_mirror.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      rbd_mirror.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      rbd_mirror.vm.provider "parallels" do |prl|
+        prl.name = "ceph-rbd-mirror#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      rbd_mirror.vm.provider :linode do |provider|
+        provider.label = rbd_mirror.vm.hostname
+      end
+    end
+  end
+
+  (0..NISCSI_GWS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}iscsi_gw#{i}" do |iscsi_gw|
+      iscsi_gw.vm.hostname = "#{LABEL_PREFIX}ceph-iscsi-gw#{i}"
+      if ASSIGN_STATIC_IP
+        iscsi_gw.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.9#{i}"
+      end
+      # Virtualbox
+      iscsi_gw.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      iscsi_gw.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      iscsi_gw.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = true
+      end
+      # Parallels
+      iscsi_gw.vm.provider "parallels" do |prl|
+        prl.name = "ceph-iscsi-gw#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      iscsi_gw.vm.provider :linode do |provider|
+        provider.label = iscsi_gw.vm.hostname
+      end
+    end
+  end
+
+  (0..NMONS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}mon#{i}" do |mon|
+      mon.vm.hostname = "#{LABEL_PREFIX}ceph-mon#{i}"
+      if ASSIGN_STATIC_IP
+        mon.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.1#{i}"
+      end
+      # Virtualbox
+      mon.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      mon.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      mon.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+        lv.random_hostname = false
+      end
+
+      # Parallels
+      mon.vm.provider "parallels" do |prl|
+        prl.name = "ceph-mon#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+
+      mon.vm.provider :linode do |provider|
+        provider.label = mon.vm.hostname
+      end
+    end
+  end
+
+  (0..NOSDS - 1).each do |i|
+    config.vm.define "#{LABEL_PREFIX}osd#{i}" do |osd|
+      osd.vm.hostname = "#{LABEL_PREFIX}ceph-osd#{i}"
+      if ASSIGN_STATIC_IP
+        osd.vm.network :private_network,
+          ip: "#{PUBLIC_SUBNET}.10#{i}"
+        osd.vm.network :private_network,
+          ip: "#{CLUSTER_SUBNET}.20#{i}"
+      end
+      # Virtualbox
+      osd.vm.provider :virtualbox do |vb|
+        # Create our own controller for consistency and to remove VM dependency
+        vb.customize ['storagectl', :id,
+                      '--name', 'OSD Controller',
+                      '--add', 'scsi']
+        (0..1).each do |d|
+          vb.customize ['createhd',
+                        '--filename', "disk-#{i}-#{d}",
+                        '--size', '11000'] unless File.exist?("disk-#{i}-#{d}.vdi")
+          vb.customize ['storageattach', :id,
+                        '--storagectl', 'OSD Controller',
+                        '--port', 3 + d,
+                        '--device', 0,
+                        '--type', 'hdd',
+                        '--medium', "disk-#{i}-#{d}.vdi"]
+        end
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      osd.vm.provider :vmware_fusion do |v|
+        (0..1).each do |d|
+          v.vmx["scsi0:#{d + 1}.present"] = 'TRUE'
+          v.vmx["scsi0:#{d + 1}.fileName"] =
+            create_vmdk("disk-#{i}-#{d}", '11000MB')
+        end
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      driverletters = ('a'..'z').to_a
+      osd.vm.provider :libvirt do |lv|
+        # always make /dev/sd{a/b/c} so that CI can ensure that
+        # virtualbox and libvirt will have the same devices to use for OSDs
+        (0..2).each do |d|
+          lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => '12G', :bus => "ide"
+        end
+        lv.memory = MEMORY
+        lv.random_hostname = false
+      end
+
+      # Parallels
+      osd.vm.provider "parallels" do |prl|
+        prl.name = "ceph-osd#{i}"
+        prl.memory = "#{MEMORY}"
+        (0..1).each do |d|
+          prl.customize ["set", :id,
+                         "--device-add",
+                         "hdd",
+                         "--iface",
+                         "sata"]
+        end
+      end
+
+      osd.vm.provider :linode do |provider|
+        provider.label = osd.vm.hostname
+      end
+
+      # Run the provisioner after the last machine comes up
+      osd.vm.provision 'ansible', &ansible_provision if i == (NOSDS - 1)
+    end
+  end
+end

From 55bde0336f406b914e0ee16ce670e4018e3011b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 14:04:58 -0500
Subject: [PATCH 22/80] ci: set a different directory for ceph osd docker run
 script
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

/usr/share is not writable on Atomic Host so we use /var/tmp instead.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../centos/7/docker-cluster-dedicated-journal/group_vars/all     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
index 9230cfcec..6763e06de 100644
--- a/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
@@ -23,3 +23,4 @@ devices:
 raw_journal_devices:
   - /dev/sdb
 ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+ceph_osd_docker_run_script_path: /var/tmp

From 51b759fc16aa5c9d0fd474497a98961a7cbcc6f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 15:39:42 -0500
Subject: [PATCH 23/80] ci: do not use atomic host for
 ansible2.2-docker_dedicated_journal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Switch to CentOS since Atomic host does not have the right Docker
version.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../vagrant_variables.yml                     | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml
index 0c54f6978..f77870126 100644
--- a/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/vagrant_variables.yml
@@ -36,15 +36,11 @@ disks: "[ '/dev/sda', '/dev/sdb' ]"
 #
 # * ceph/ubuntu-xenial
 #
-# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64
-# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet
-# libvirt CentOS: centos/7
-# parallels Ubuntu: parallels/ubuntu-14.04
-# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller'
-# For more boxes have a look at:
-#   - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q=
-#   - https://download.gluster.org/pub/gluster/purpleidea/vagrant/
-vagrant_box: centos/atomic-host
+# NOTE(leseb): we use centos for this scenario since we at least need Docker version 1.12.5
+# which is not available in Atomic Host.
+# There are bug like this one: https://github.com/docker/docker/issues/12694
+vagrant_box: centos/7
+
 #ssh_private_key_path: "~/.ssh/id_rsa"
 # The sync directory changes based on vagrant box
 # Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
@@ -53,9 +49,3 @@ vagrant_box: centos/atomic-host
 # Disables synced folder creation. Not needed for testing, will skip mounting
 # the vagrant directory on the remote box regardless of the provider.
 vagrant_disable_synced_folder: true
-# VAGRANT URL
-# This is a URL to download an image from an alternate location.  vagrant_box
-# above should be set to the filename of the image.
-# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
-# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box
-# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box

From e22acb81e6eb3e5f599568f83855eac7e77b3ea8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 21 Feb 2017 16:14:39 -0500
Subject: [PATCH 24/80] ci: fix issue on ansible2.2-docker_dedicated_journal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

journal_collocation was enabled so the test suite was testing this
scenario and obviously failed since there is no second partition to
verify.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../centos/7/docker-cluster-dedicated-journal/group_vars/all     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
index 6763e06de..9608fb6ee 100644
--- a/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
+++ b/tests/functional/centos/7/docker-cluster-dedicated-journal/group_vars/all
@@ -15,7 +15,6 @@ journal_size: 100
 ceph_docker_on_openstack: False
 public_network: "192.168.15.0/24"
 cluster_network: "192.168.16.0/24"
-journal_collocation: true
 ceph_rgw_civetweb_port: 8080
 ceph_osd_docker_devices: "{{ devices }}"
 devices:

From 503ec9be578847c715996da424d99cb66bc6fd31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Wed, 22 Feb 2017 09:21:23 -0500
Subject: [PATCH 25/80] ci: decorate the tests to not run on docker scenario
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Certain scenario won't work on containerized deployment. So we decorate
them so they can be skipped.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 tests/functional/tests/mon/test_mons.py | 2 ++
 tests/functional/tests/test_install.py  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/tests/functional/tests/mon/test_mons.py b/tests/functional/tests/mon/test_mons.py
index e5df2d264..73229f0a0 100644
--- a/tests/functional/tests/mon/test_mons.py
+++ b/tests/functional/tests/mon/test_mons.py
@@ -22,6 +22,7 @@ class TestMons(object):
         )
         assert Service(service_name).is_enabled
 
+    @pytest.mark.no_docker
     def test_can_get_cluster_health(self, node, Command):
         cmd = "sudo ceph --cluster={} --connect-timeout 5 -s".format(node["cluster_name"])
         output = Command.check_output(cmd)
@@ -30,6 +31,7 @@ class TestMons(object):
 
 class TestOSDs(object):
 
+    @pytest.mark.no_docker
     def test_all_osds_are_up_and_in(self, node, Command):
         cmd = "sudo ceph --cluster={} --connect-timeout 5 -s".format(node["cluster_name"])
         output = Command.check_output(cmd)
diff --git a/tests/functional/tests/test_install.py b/tests/functional/tests/test_install.py
index aea5dde37..478581bb1 100644
--- a/tests/functional/tests/test_install.py
+++ b/tests/functional/tests/test_install.py
@@ -1,3 +1,4 @@
+import pytest
 
 class TestInstall(object):
 
@@ -13,6 +14,7 @@ class TestInstall(object):
     def test_ceph_conf_is_a_file(self, File, node):
         assert File(node["conf_path"]).is_file
 
+    @pytest.mark.no_docker
     def test_ceph_command_exists(self, Command):
         assert Command.exists("ceph")
 

From a551ad97bb6cf86e3a39592db6646276c0e52b54 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Feb 2017 13:43:59 -0600
Subject: [PATCH 26/80] tests: when using pytest mark decorators ensure all
 fixtures are defined

Decorating a test method directly with a pytest mark seems to break if
the test function does not explicitly define all pytest fixtures it
expects to recieve.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/tests/test_install.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/tests/test_install.py b/tests/functional/tests/test_install.py
index 478581bb1..44436b3b8 100644
--- a/tests/functional/tests/test_install.py
+++ b/tests/functional/tests/test_install.py
@@ -15,7 +15,7 @@ class TestInstall(object):
         assert File(node["conf_path"]).is_file
 
     @pytest.mark.no_docker
-    def test_ceph_command_exists(self, Command):
+    def test_ceph_command_exists(self, Command, node):
         assert Command.exists("ceph")
 
 

From 931027e6f7aee324a0fcc582e4daa07f8bbe5c61 Mon Sep 17 00:00:00 2001
From: Tobias Florek <tob@butter.sh>
Date: Mon, 20 Feb 2017 10:03:49 +0100
Subject: [PATCH 27/80] harmonize docker names

Created containers now are named more or less in the form of

    <ansible role>-<ansible_hostname>
---
 .../purge-docker-cluster.yml                  | 22 +++++++++----------
 infrastructure-playbooks/rolling_update.yml   |  8 +++----
 ...inerized-to-containerized-ceph-daemons.yml | 12 +++++-----
 roles/ceph-mds/templates/ceph-mds.service.j2  |  9 ++++----
 roles/ceph-mon/tasks/docker/main.yml          |  6 ++---
 roles/ceph-mon/templates/ceph-mon.service.j2  |  6 ++---
 .../ceph-nfs/tasks/docker/create_configs.yml  |  4 ++--
 roles/ceph-nfs/templates/ceph-nfs.service.j2  |  6 ++---
 .../tasks/docker/start_docker_osd.yml         |  4 ++--
 roles/ceph-osd/templates/ceph-osd-run.sh.j2   |  2 +-
 roles/ceph-osd/templates/ceph-osd.service.j2  |  4 ++--
 .../templates/ceph-rbd-mirror.service.j2      |  9 ++++----
 .../tasks/docker/start_docker_restapi.yml     |  2 +-
 roles/ceph-rgw/templates/ceph-rgw.service.j2  |  8 +++----
 14 files changed, 50 insertions(+), 52 deletions(-)

diff --git a/infrastructure-playbooks/purge-docker-cluster.yml b/infrastructure-playbooks/purge-docker-cluster.yml
index 66cb1d768..7ce6283f4 100644
--- a/infrastructure-playbooks/purge-docker-cluster.yml
+++ b/infrastructure-playbooks/purge-docker-cluster.yml
@@ -72,7 +72,7 @@
   - name: remove ceph mds container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-mds-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
@@ -119,7 +119,7 @@
   - name: remove ceph rgw container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-rgw-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
@@ -166,7 +166,7 @@
   - name: remove ceph rbd-mirror container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-rbd-mirror-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
@@ -213,7 +213,7 @@
   - name: remove ceph nfs container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-nfs-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
@@ -270,7 +270,7 @@
   - name: remove ceph osd prepare container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-prepare-{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
       state: absent
     with_items: "{{ ceph_osd_docker_devices }}"
     ignore_errors: true
@@ -278,7 +278,7 @@
   - name: remove ceph osd container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
       state: absent
     with_items: "{{ ceph_osd_docker_devices }}"
     ignore_errors: true
@@ -287,7 +287,7 @@
     shell: |
       docker run \
       --privileged=true \
-      --name {{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }} \
+      --name ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }} \
       -v /dev/:/dev/ \
       -e OSD_DEVICE={{ item }} \
       {{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
@@ -298,7 +298,7 @@
 
   - name: wait until the zap containers die
     shell: |
-      docker ps | grep -sq {{ ansible_hostname }}-osd-zap
+      docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}-dev
     register: zap_alive
     failed_when: false
     until: zap_alive.rc != 0
@@ -308,7 +308,7 @@
   - name: remove ceph osd zap disk container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-zap-{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
       state: absent
     with_items:
       - "{{ ceph_osd_docker_devices }}"
@@ -361,14 +361,14 @@
   - name: remove ceph mon container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-mon-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
   - name: remove restapi container
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-ceph-restapi"
+      name: "ceph-restapi-{{ ansible_hostname }}"
       state: absent
     ignore_errors: true
 
diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index a1f4b3e5a..c276372c2 100644
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -158,7 +158,7 @@
 
     - name: waiting for the containerized monitor to join the quorum...
       shell: |
-        docker exec {{ hostvars[mon_host]['ansible_hostname'] }} ceph -s --cluster {{ cluster }} | grep quorum | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
+        docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph -s --cluster {{ cluster }} | grep quorum | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
       register: result
       until: result.rc == 0
       retries: "{{ health_mon_check_retries }}"
@@ -193,7 +193,7 @@
 
     - name: set containerized osd flags
       command: |
-          docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}
+          docker exec ceph-osd-{{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph osd set {{ item }} --cluster {{ cluster }}
       with_items:
         - noout
         - noscrub
@@ -283,7 +283,7 @@
 
     - name: container - waiting for clean pgs...
       shell: |
-        test "$(docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
+        test "$(docker exec ceph-osd-{{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph pg stat --cluster {{ cluster }} | sed 's/pgs.*//;s/^.*://;s/ //')" && docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
       register: result
       until: result.rc == 0
       retries: "{{ health_osd_check_retries }}"
@@ -302,7 +302,7 @@
 
     - name: unset containerized osd flags
       command: |
-          docker exec {{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph osd unset {{ item }} --cluster {{ cluster }}
+          docker exec ceph-osd-{{ hostvars[groups.mons[0]]['ansible_hostname'] }} ceph osd unset {{ item }} --cluster {{ cluster }}
       with_items:
         - noout
         - noscrub
diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
index 26edb8866..0b09c203e 100644
--- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
@@ -163,7 +163,7 @@
   - name: start ceph mon container image
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-mon-{{ ansible_hostname }}"
       net: "host"
       state: "running"
       privileged: "{{ mon_docker_privileged }}"
@@ -266,7 +266,7 @@
   - name: start ceph osd container image(s)
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
       net: host
       pid: host
       state: started
@@ -339,7 +339,7 @@
   - name: start ceph metadata container image
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: ceph-{{ ansible_hostname }}-mds
+      name: ceph-mds-{{ ansible_hostname }}
       net: host
       state: running
       env: "CEPH_DAEMON=MDS,CEPHFS_CREATE=1,{{ ceph_mds_docker_extra_env }}"
@@ -399,7 +399,7 @@
   - name: start ceph rados gateway container image
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: ceph-{{ ansible_hostname }}-rgw
+      name: ceph-rgw-{{ ansible_hostname }}
       expose: "{{ ceph_rgw_civetweb_port }}"
       ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
       state: running
@@ -460,7 +460,7 @@
   - name: start ceph rbd mirror container image
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-rbd-mirror-{{ ansible_hostname }}"
       net: host
       state: running
       volumes: "/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
@@ -519,7 +519,7 @@
   - name: start ceph nfs container image
     docker:
       image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "{{ ansible_hostname }}"
+      name: "ceph-nfs-{{ ansible_hostname }}"
       net: "host"
       state: "running"
       privileged: true
diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2
index 15a35aad4..dcf9ff2bf 100644
--- a/roles/ceph-mds/templates/ceph-mds.service.j2
+++ b/roles/ceph-mds/templates/ceph-mds.service.j2
@@ -4,8 +4,8 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}
-ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker rm ceph-mds-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not mds_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
@@ -15,13 +15,12 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    -e KV_IP={{kv_endpoint}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
-   --privileged \
    -e CEPH_DAEMON=MDS \
    -e CEPHFS_CREATE=1 \
    {{ ceph_mds_docker_extra_env }} \
-   --name={{ ansible_hostname }} \
+   --name=ceph-mds-{{ ansible_hostname }} \
    {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}
+ExecStopPost=-/usr/bin/docker stop ceph-mds-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index f1bc6dc11..7f323c690 100644
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -59,14 +59,14 @@
 - include: start_docker_monitor.yml
 
 - name: wait for monitor socket to exist
-  command: docker exec {{ ansible_hostname }} stat /var/run/ceph/{{ cluster }}-mon.{{ ansible_hostname }}.asok
+  command: docker exec ceph-mon-{{ ansible_hostname }} stat /var/run/ceph/{{ cluster }}-mon.{{ ansible_fqdn }}.asok
   register: monitor_socket
   retries: 5
   delay: 10
   until: monitor_socket.rc == 0
 
 - name: force peer addition as potential bootstrap peer for cluster bringup
-  command: docker exec {{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ ansible_hostname }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + ceph_mon_docker_interface].ipv4.address }}
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ ansible_fqdn }}.asok add_bootstrap_peer_hint {{ hostvars[item]['ansible_' + ceph_mon_docker_interface].ipv4.address }}
   with_items: "{{ groups.mons }}"
   changed_when: false
   failed_when: false
@@ -78,7 +78,7 @@
   when: not mon_containerized_deployment_with_kv
 
 - name: create ceph rest api keyring when mon is containerized
-  command: docker exec {{ ansible_hostname }} ceph --cluster {{ cluster }} auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/{{ cluster }}.client.restapi.keyring
+  command: docker exec ceph-mon-{{ ansible_hostname }} ceph --cluster {{ cluster }} auth get-or-create client.restapi osd 'allow *' mon 'allow *' -o /etc/ceph/{{ cluster }}.client.restapi.keyring
   args:
      creates: /etc/ceph/{{ cluster }}.client.restapi.keyring
   changed_when: false
diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2
index 373d46e76..f231d2bfb 100644
--- a/roles/ceph-mon/templates/ceph-mon.service.j2
+++ b/roles/ceph-mon/templates/ceph-mon.service.j2
@@ -4,9 +4,9 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker rm %i
+ExecStartPre=-/usr/bin/docker rm ceph-mon-%i
 ExecStartPre=$(command -v mkdir) -p /etc/ceph /var/lib/ceph/mon
-ExecStart=/usr/bin/docker run --rm --name %i --net=host \
+ExecStart=/usr/bin/docker run --rm --name ceph-mon-%i --net=host \
    {% if not mon_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
    -v /etc/ceph:/etc/ceph \
@@ -27,7 +27,7 @@ ExecStart=/usr/bin/docker run --rm --name %i --net=host \
    -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
    {{ ceph_mon_docker_extra_env }} \
    {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop %i
+ExecStopPost=-/usr/bin/docker stop ceph-mon-%i
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
diff --git a/roles/ceph-nfs/tasks/docker/create_configs.yml b/roles/ceph-nfs/tasks/docker/create_configs.yml
index 28962251b..c30dd7c91 100644
--- a/roles/ceph-nfs/tasks/docker/create_configs.yml
+++ b/roles/ceph-nfs/tasks/docker/create_configs.yml
@@ -10,7 +10,7 @@
 - name: create the nfs rgw user
   docker:
     image: "{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-    name: ceph-{{ ansible_hostname }}-rgw-user
+    name: ceph-rgw-user-{{ ansible_hostname }}
     hostname: "{{ ansible_hostname }}"
     expose: "{{ ceph_rgw_civetweb_port }}"
     ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
@@ -20,7 +20,7 @@
   when: nfs_obj_gw
 
 - name: get user create output
-  command: docker logs ceph-{{ ansible_hostname }}-rgw-user
+  command: docker logs ceph-rgw-user-{{ ansible_hostname }}
   always_run: true
   register: rgwuser
 
diff --git a/roles/ceph-nfs/templates/ceph-nfs.service.j2 b/roles/ceph-nfs/templates/ceph-nfs.service.j2
index 43cf0a5dd..d7471c61e 100644
--- a/roles/ceph-nfs/templates/ceph-nfs.service.j2
+++ b/roles/ceph-nfs/templates/ceph-nfs.service.j2
@@ -5,7 +5,7 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker rm %i
+ExecStartPre=-/usr/bin/docker rm ceph-nfs-%i
 ExecStartPre=/usr/bin/mkdir -p /etc/ceph /etc/ganesha /var/lib/nfs/ganesha
 ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not mon_containerized_deployment_with_kv -%}
@@ -19,9 +19,9 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    --privileged \
    -e CEPH_DAEMON=NFS \
    {{ ceph_nfs_docker_extra_env }} \
-   --name=nfs-{{ ansible_hostname }} \
+   --name=ceph-nfs-{{ ansible_hostname }} \
    {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop %i
+ExecStopPost=-/usr/bin/docker stop ceph-nfs-%i
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
diff --git a/roles/ceph-osd/tasks/docker/start_docker_osd.yml b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
index 7d1760c19..3423a9452 100644
--- a/roles/ceph-osd/tasks/docker/start_docker_osd.yml
+++ b/roles/ceph-osd/tasks/docker/start_docker_osd.yml
@@ -24,7 +24,7 @@
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}" \
+    --name="ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.0 | regex_replace('/', '') }}" \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -47,7 +47,7 @@
     docker run  --net=host   \
     --pid=host \
     --privileged=true \
-    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}" \
+    --name="ceph-osd-prepare-{{ ansible_hostname }}-dev-{{ item.0 | regex_replace('/', '') }}" \
     -v /dev:/dev \
     -v /etc/localtime:/etc/localtime:ro \
     -e "OSD_DEVICE={{ item.0 }}" \
diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
index 0689be9e0..8ddd5fff7 100644
--- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2
+++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2
@@ -46,5 +46,5 @@ fi
   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
   -e OSD_DEVICE=/dev/${1} \
   {{ ceph_osd_docker_extra_env }} \
-  --name={{ ansible_hostname }}-osd-dev${1} \
+  --name=ceph-osd-{{ ansible_hostname }}-dev${1} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
diff --git a/roles/ceph-osd/templates/ceph-osd.service.j2 b/roles/ceph-osd/templates/ceph-osd.service.j2
index e1b3f43f8..3822b75a8 100644
--- a/roles/ceph-osd/templates/ceph-osd.service.j2
+++ b/roles/ceph-osd/templates/ceph-osd.service.j2
@@ -6,9 +6,9 @@ After=docker.service
 [Service]
 EnvironmentFile=-/etc/environment
 ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
-ExecStartPre=-/usr/bin/docker rm -f {{ ansible_hostname }}-osd-dev%i
+ExecStartPre=-/usr/bin/docker rm -f ceph-osd-{{ ansible_hostname }}-dev%i
 ExecStart={{ ceph_osd_docker_run_script_path }}/ceph-osd-run.sh %i
-ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i
+ExecStop=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-dev%i
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
index d0cb40871..6cf8069ea 100644
--- a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
+++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
@@ -4,8 +4,8 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}
-ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker stop ceph-rdb-mirror-{{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker rm ceph-rdb-mirror-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not rbd_mirror_containerized_deployment_with_kv -%}
    -v /etc/ceph:/etc/ceph \
@@ -14,11 +14,10 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    -e KV_IP={{kv_endpoint}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
-   --privileged \
    -e CEPH_DAEMON=RBD_MIRROR \
-   --name={{ ansible_hostname }} \
+   --name=ceph-rdb-mirror-{{ ansible_hostname }} \
    {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}
+ExecStopPost=-/usr/bin/docker stop ceph-rdb-mirror-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
diff --git a/roles/ceph-restapi/tasks/docker/start_docker_restapi.yml b/roles/ceph-restapi/tasks/docker/start_docker_restapi.yml
index d3205ef22..9c4b261e9 100644
--- a/roles/ceph-restapi/tasks/docker/start_docker_restapi.yml
+++ b/roles/ceph-restapi/tasks/docker/start_docker_restapi.yml
@@ -2,7 +2,7 @@
 - name: run the ceph rest api docker image
   docker:
     image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-    name: "{{ ansible_hostname }}-ceph-restapi"
+    name: "ceph-restapi-{{ ansible_hostname }}"
     net: host
     expose: "{{ ceph_restapi_port }}"
     state: running
diff --git a/roles/ceph-rgw/templates/ceph-rgw.service.j2 b/roles/ceph-rgw/templates/ceph-rgw.service.j2
index 621f65a17..0573aa2e2 100644
--- a/roles/ceph-rgw/templates/ceph-rgw.service.j2
+++ b/roles/ceph-rgw/templates/ceph-rgw.service.j2
@@ -4,8 +4,8 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}
-ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker stop ceph-rgw-{{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker rm ceph-rgw-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not rgw_containerized_deployment_with_kv -%}
    -v /var/lib/ceph:/var/lib/ceph \
@@ -18,9 +18,9 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    --privileged \
    -e CEPH_DAEMON=RGW \
    {{ ceph_rgw_docker_extra_env }} \
-   --name={{ ansible_hostname }} \
+   --name=ceph-rgw-{{ ansible_hostname }} \
    {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}
+ExecStopPost=-/usr/bin/docker stop ceph-rgw-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120

From 4639d89231dc35c743ded29ed0f962f36a4b0574 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 23 Feb 2017 15:39:55 -0500
Subject: [PATCH 28/80] infra: fix cluster name detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous command was returning /etc/ceph/ceph.conf, we only need
'ceph' to be returned.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 infrastructure-playbooks/take-over-existing-cluster.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure-playbooks/take-over-existing-cluster.yml b/infrastructure-playbooks/take-over-existing-cluster.yml
index 08b31bd59..1e3df7697 100644
--- a/infrastructure-playbooks/take-over-existing-cluster.yml
+++ b/infrastructure-playbooks/take-over-existing-cluster.yml
@@ -28,7 +28,7 @@
 
     - name: get the name of the existing ceph cluster
       shell: |
-        grep -lE '\[global\]|fsid' /etc/ceph/*.conf
+        basename $(grep -R fsid /etc/ceph/ | egrep -o '^[^.]*')
       changed_when: false
       register: ceph_conf
 

From e00b745a6c3998ff5a679444037204449fcbdde6 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Thu, 23 Feb 2017 16:06:04 +0100
Subject: [PATCH 29/80] ceph-mon: Check if ceph filesystem exists

Check if ceph filesystem already exists before creating it.
If the ceph filesystem doesn't exist, execute the task only on one node.

Fix: #1314
Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
---
 .../ceph-mon/tasks/create_mds_filesystems.yml | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/roles/ceph-mon/tasks/create_mds_filesystems.yml b/roles/ceph-mon/tasks/create_mds_filesystems.yml
index d9a89ca77..6e7b82d98 100644
--- a/roles/ceph-mon/tasks/create_mds_filesystems.yml
+++ b/roles/ceph-mon/tasks/create_mds_filesystems.yml
@@ -10,21 +10,31 @@
     - cephfs_metadata
   changed_when: false
 
+- name: check if ceph filesystem already exists
+  command: ceph --cluster {{ cluster }} fs get {{ cephfs }}
+  register: check_existing_cephfs
+  changed_when: false
+  failed_when: false
+  when: inventory_hostname == groups.mons|last
+
 - name: create ceph filesystem
   command: ceph --cluster {{ cluster }} fs new {{ cephfs }} {{ cephfs_metadata }} {{ cephfs_data }}
   changed_when: false
+  when:
+    - inventory_hostname == groups.mons|last
+    - check_existing_cephfs.rc != 0
 
 - name: allow multimds
   command: ceph --cluster {{ cluster }} fs set {{ cephfs }} allow_multimds true --yes-i-really-mean-it
   changed_when: false
   when:
-  - ceph_release_num.{{ ceph_release }} >= ceph_release_num.jewel
-  - mds_allow_multimds
+    - ceph_release_num.{{ ceph_release }} >= ceph_release_num.jewel
+    - mds_allow_multimds
 
 - name: set max_mds
   command: ceph --cluster {{ cluster }} fs set {{ cephfs }} max_mds {{ mds_max_mds }}
   changed_when: false
   when:
-  - ceph_release_num.{{ ceph_release }} >= ceph_release_num.jewel
-  - mds_allow_multimds
-  - mds_max_mds > 1
+    - ceph_release_num.{{ ceph_release }} >= ceph_release_num.jewel
+    - mds_allow_multimds
+    - mds_max_mds > 1

From 8b36cbac647ef3fd13d5ff9236b1d2b108b0afad Mon Sep 17 00:00:00 2001
From: yangyimincn <yangyimincn@163.com>
Date: Tue, 28 Feb 2017 16:56:02 +0800
Subject: [PATCH 30/80] Update rolling_update.yml

The task waiting for the monitor to join the quorum... , the result for ceph -s | grep monmap only contain monmap, not included quorum:

# ceph -s --cluster ceph | grep monmap
     monmap e1: 3 mons at {sh-office-ceph-1=10.12.10.34:6789/0,sh-office-ceph-2=10.12.10.35:6789/0,sh-office-ceph-3=10.12.10.36:6789/0}

If want to get monitor, should use this:

# ceph -s --cluster ceph | grep election
            election epoch 80, quorum 0,1 sh-office-ceph-1,sh-office-ceph-2

ceph verison: 10.2.5
---
 infrastructure-playbooks/rolling_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index a1f4b3e5a..d258020af 100644
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -148,7 +148,7 @@
 
     - name: waiting for the monitor to join the quorum...
       shell: |
-        ceph -s --cluster {{ cluster }} | grep monmap | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
+        ceph -s --cluster {{ cluster }} | grep election | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
       register: result
       until: result.rc == 0
       retries: "{{ health_mon_check_retries }}"

From 56d8367339edae49c3c6dd457396952b337ce981 Mon Sep 17 00:00:00 2001
From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 28 Feb 2017 12:48:08 -0500
Subject: [PATCH 31/80] send fs commands to one mon

Add-on to #1329.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
---
 roles/ceph-mon/tasks/create_mds_filesystems.yml | 5 +----
 roles/ceph-mon/tasks/main.yml                   | 1 +
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/roles/ceph-mon/tasks/create_mds_filesystems.yml b/roles/ceph-mon/tasks/create_mds_filesystems.yml
index 6e7b82d98..29bdb8efa 100644
--- a/roles/ceph-mon/tasks/create_mds_filesystems.yml
+++ b/roles/ceph-mon/tasks/create_mds_filesystems.yml
@@ -15,14 +15,11 @@
   register: check_existing_cephfs
   changed_when: false
   failed_when: false
-  when: inventory_hostname == groups.mons|last
 
 - name: create ceph filesystem
   command: ceph --cluster {{ cluster }} fs new {{ cephfs }} {{ cephfs_metadata }} {{ cephfs_data }}
   changed_when: false
-  when:
-    - inventory_hostname == groups.mons|last
-    - check_existing_cephfs.rc != 0
+  when: check_existing_cephfs.rc != 0
 
 - name: allow multimds
   command: ceph --cluster {{ cluster }} fs set {{ cephfs }} allow_multimds true --yes-i-really-mean-it
diff --git a/roles/ceph-mon/tasks/main.yml b/roles/ceph-mon/tasks/main.yml
index cf3d1f40f..9d7c5c81a 100644
--- a/roles/ceph-mon/tasks/main.yml
+++ b/roles/ceph-mon/tasks/main.yml
@@ -15,6 +15,7 @@
     - not mon_containerized_deployment
     - groups[mds_group_name] is defined
     - "{{ groups[mds_group_name]|length > 0 }}"
+    - inventory_hostname == groups.mons|last
 
 - include: secure_cluster.yml
   when:

From 88c6296fa20761b75348cdbeb7cdeabf6f77296d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 2 Mar 2017 14:59:21 +0100
Subject: [PATCH 32/80] rgw: remove sudoers file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This was needed for Hammer and older version, not needed anymore since
we have a 'ceph' user to run ceph processes.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/rgws.yml.sample             | 5 -----
 roles/ceph-rgw/defaults/main.yml       | 5 -----
 roles/ceph-rgw/tasks/pre_requisite.yml | 9 ---------
 roles/ceph-rgw/templates/ceph.j2       | 2 --
 4 files changed, 21 deletions(-)
 delete mode 100644 roles/ceph-rgw/templates/ceph.j2

diff --git a/group_vars/rgws.yml.sample b/group_vars/rgws.yml.sample
index 61152debc..21ea8d9cf 100644
--- a/group_vars/rgws.yml.sample
+++ b/group_vars/rgws.yml.sample
@@ -25,11 +25,6 @@ dummy:
 #
 #cephx: true
 
-# Used for the sudo exception while starting the radosgw process
-# a new entry /etc/sudoers.d/ceph will be created
-# allowing root to not require tty
-#radosgw_user: root
-
 # Multi-site remote pull URL variables
 #rgw_pull_port: "{{ radosgw_civetweb_port }}"
 #rgw_pull_proto: "http"
diff --git a/roles/ceph-rgw/defaults/main.yml b/roles/ceph-rgw/defaults/main.yml
index fb07dd456..599870f46 100644
--- a/roles/ceph-rgw/defaults/main.yml
+++ b/roles/ceph-rgw/defaults/main.yml
@@ -17,11 +17,6 @@ copy_admin_key: false
 #
 cephx: true
 
-# Used for the sudo exception while starting the radosgw process
-# a new entry /etc/sudoers.d/ceph will be created
-# allowing root to not require tty
-radosgw_user: root
-
 # Multi-site remote pull URL variables
 rgw_pull_port: "{{ radosgw_civetweb_port }}"
 rgw_pull_proto: "http"
diff --git a/roles/ceph-rgw/tasks/pre_requisite.yml b/roles/ceph-rgw/tasks/pre_requisite.yml
index 57a1bf3b0..f80d14a4c 100644
--- a/roles/ceph-rgw/tasks/pre_requisite.yml
+++ b/roles/ceph-rgw/tasks/pre_requisite.yml
@@ -40,12 +40,3 @@
     group: "ceph"
     mode: "0600"
   when: cephx
-
-- name: generate rados gateway sudoers file
-  template:
-    src: ceph.j2
-    dest: /etc/sudoers.d/ceph
-    owner: root
-    group: root
-    mode: 0400
-  when: ansible_distribution != "Ubuntu"
diff --git a/roles/ceph-rgw/templates/ceph.j2 b/roles/ceph-rgw/templates/ceph.j2
deleted file mode 100644
index d0d31dc00..000000000
--- a/roles/ceph-rgw/templates/ceph.j2
+++ /dev/null
@@ -1,2 +0,0 @@
-# {{ ansible_managed }}
-Defaults:{{ radosgw_user }} !requiretty

From 66b59ea9c6ace0434dab9535748881c779369307 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Tue, 21 Feb 2017 17:58:24 +0100
Subject: [PATCH 33/80] docker: Fix #1303

Install package from official repos rather than pip when using RHEL.

This commit fix https://bugzilla.redhat.com/show_bug.cgi?id=1420855

Also this commit Refact all `roles/ceph-*/tasks/docker/pre_requisite.yml`
to avoid a lot of duplicated code.

Fix: #1303
Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
---
 roles/ceph-docker-common/defaults/main.yml    |   1 +
 roles/ceph-docker-common/tasks/main.yml       |  13 ++
 .../pre_requisites/debian_prerequisites.yml   |  67 ++++++++
 .../tasks/pre_requisites/prerequisites.yml    |  80 ++++++++++
 roles/ceph-mds/tasks/docker/main.yml          |  12 --
 roles/ceph-mds/tasks/docker/pre_requisite.yml | 144 -----------------
 roles/ceph-mon/defaults/main.yml              |   1 -
 roles/ceph-mon/tasks/docker/main.yml          |  12 --
 roles/ceph-mon/tasks/docker/pre_requisite.yml | 149 ------------------
 roles/ceph-nfs/tasks/docker/main.yml          |  12 --
 roles/ceph-nfs/tasks/docker/pre_requisite.yml | 117 --------------
 roles/ceph-osd/tasks/docker/main.yml          |  12 --
 roles/ceph-osd/tasks/docker/pre_requisite.yml | 136 ----------------
 roles/ceph-rbd-mirror/tasks/docker/main.yml   |  12 --
 .../tasks/docker/pre_requisite.yml            | 144 -----------------
 roles/ceph-restapi/tasks/docker/main.yml      |  12 --
 .../tasks/docker/pre_requisite.yml            | 142 -----------------
 roles/ceph-rgw/tasks/docker/main.yml          |  12 --
 roles/ceph-rgw/tasks/docker/pre_requisite.yml | 130 ---------------
 19 files changed, 161 insertions(+), 1047 deletions(-)
 create mode 100644 roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
 create mode 100644 roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
 delete mode 100644 roles/ceph-mds/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-mon/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-nfs/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-osd/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-restapi/tasks/docker/pre_requisite.yml
 delete mode 100644 roles/ceph-rgw/tasks/docker/pre_requisite.yml

diff --git a/roles/ceph-docker-common/defaults/main.yml b/roles/ceph-docker-common/defaults/main.yml
index adc21ceb1..66408683f 100644
--- a/roles/ceph-docker-common/defaults/main.yml
+++ b/roles/ceph-docker-common/defaults/main.yml
@@ -1,3 +1,4 @@
 ---
 
 ceph_docker_registry: docker.io
+ceph_mon_docker_enable_centos_extra_repo: false
diff --git a/roles/ceph-docker-common/tasks/main.yml b/roles/ceph-docker-common/tasks/main.yml
index 14e987cde..7e63d3295 100644
--- a/roles/ceph-docker-common/tasks/main.yml
+++ b/roles/ceph-docker-common/tasks/main.yml
@@ -1,2 +1,15 @@
 ---
 - include: system_checks.yml
+
+
+- name: check if it is atomic host
+  stat: path=/run/ostree-booted
+  register: stat_ostree
+  always_run: true
+
+- name: set fact for using atomic host
+  set_fact:
+    is_atomic: '{{ stat_ostree.stat.exists }}'
+
+- include: ./pre_requisites/prerequisites.yml
+  when: not is_atomic
diff --git a/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
new file mode 100644
index 000000000..effe2a346
--- /dev/null
+++ b/roles/ceph-docker-common/tasks/pre_requisites/debian_prerequisites.yml
@@ -0,0 +1,67 @@
+---
+# To install docker on debian
+- name: allow apt to use a repository over https (debian)
+  package:
+    name: "{{ item }}"
+    state: present
+    update_cache: yes
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+    - curl
+    - software-properties-common
+  when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
+
+- name: add docker's gpg key
+  apt_key:
+    url: https://apt.dockerproject.org/gpg
+    state: present
+  when: ansible_distribution == 'Debian'
+
+- name: add docker and debian testing repository
+  apt_repository:
+    repo: "{{ item }}"
+    state: present
+  with_items:
+    - "deb https://apt.dockerproject.org/repo/ debian-{{ ansible_distribution_release }} main"
+    - "deb http://http.us.debian.org/debian/ testing contrib main"
+  when: ansible_distribution == 'Debian'
+
+- name: install pip from testing on debian
+  package:
+    name: python-pip
+    state: present
+    default_release: testing
+    update_cache: yes
+  when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
+
+- name: install docker-py via pip for debian
+  pip:
+    name: docker-py
+    state: latest
+  tags:
+    with_pkg
+  when: ansible_distribution == 'Debian'
+
+- name: install docker on debian
+  package:
+    name: docker-engine
+    state: present
+    update_cache: yes
+  when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six via pip
+  pip:
+    name: six
+    version: 1.9.0
+  when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
diff --git a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
new file mode 100644
index 000000000..c337a6fa1
--- /dev/null
+++ b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
@@ -0,0 +1,80 @@
+---
+# Manage debian in a separate file because of specificities
+- include: debian_prerequisites.yml
+  when: ansible_distribution == 'Debian'
+
+- name: install docker on ubuntu
+  package:
+    name: docker.io
+    state: present
+    update_cache: yes
+  when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
+
+# ensure extras enabled for docker
+- name: enable extras on centos
+  yum_repository:
+    name: extras
+    state: present
+    enabled: yes
+  when:
+    - ansible_distribution == 'CentOS'
+    - ceph_mon_docker_enable_centos_extra_repo
+  tags:
+    with_pkg
+
+- name: install python-six
+  package:
+    name: python-six
+    state: present
+    update_cache: yes
+  when: ansible_distribution != 'Debian'
+  tags:
+    with_pkg
+
+- name: install python-docker-py on red hat / centos
+  package:
+    name: python-docker-py
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+- name: install python-docker on ubuntu
+  package:
+    name: python-docker
+    state: present
+  when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
+
+- name: install docker on red hat / centos
+  package:
+    name: docker
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+- name: pause after docker install before starting (on openstack vms)
+  pause: seconds=5
+  when: ceph_docker_on_openstack
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg
+
+- name: install ntp
+  package:
+    name: ntp
+    state: present
+  when: ntp_service_enabled
+  tags:
+    with_pkg
diff --git a/roles/ceph-mds/tasks/docker/main.yml b/roles/ceph-mds/tasks/docker/main.yml
index 507aadfa8..75a954e21 100644
--- a/roles/ceph-mds/tasks/docker/main.yml
+++ b/roles/ceph-mds/tasks/docker/main.yml
@@ -6,23 +6,11 @@
   failed_when: false
   always_run: true
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  register: stat_ostree
-  always_run: true
-
-- name: set fact for using Atomic host
-  set_fact:
-    is_atomic: '{{ stat_ostree.stat.exists }}'
-
 - include: checks.yml
   when:
     - ceph_health.rc != 0
     - not "{{ rolling_update | default(false) }}"
 
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-mds/tasks/docker/pre_requisite.yml b/roles/ceph-mds/tasks/docker/pre_requisite.yml
deleted file mode 100644
index 24924e357..000000000
--- a/roles/ceph-mds/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,144 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-# install epel for pip
-- name: install epel on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - epel-release
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-- name: enable extras repo for centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when: ansible_distribution == 'CentOS'
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-  
-# docker package could be docker-enginer or docker  
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
-
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index e6d102e54..a9a837ec0 100644
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -80,4 +80,3 @@ ceph_docker_on_openstack: false
 mon_docker_privileged: false
 mon_docker_net_host: true
 ceph_config_keys: [] # DON'T TOUCH ME
-ceph_mon_docker_enable_centos_extra_repo: false
diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index f1bc6dc11..af0dc33a4 100644
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -6,24 +6,12 @@
   failed_when: false
   always_run: true
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  register: stat_ostree
-  always_run: true
-
-- name: set fact for using Atomic host
-  set_fact:
-      is_atomic: '{{ stat_ostree.stat.exists }}'
-
 - include: checks.yml
   when:
     - ceph_health.rc != 0
     - not mon_containerized_deployment_with_kv
     - not "{{ rolling_update | default(false) }}"
 
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-mon/tasks/docker/pre_requisite.yml b/roles/ceph-mon/tasks/docker/pre_requisite.yml
deleted file mode 100644
index 41f4b2bb6..000000000
--- a/roles/ceph-mon/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,149 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-# install epel for pip
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-
-# ensure extras enabled for docker
-- name: enable extras on centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when:
-    - ansible_distribution == 'CentOS'
-    - ceph_mon_docker_enable_centos_extra_repo
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-- name: install pip and docker on redhat (dnf)
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-- name: pause after docker install before starting (on openstack vms)
-  pause: seconds=5
-  when: ceph_docker_on_openstack
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
-
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-nfs/tasks/docker/main.yml b/roles/ceph-nfs/tasks/docker/main.yml
index d80a4c975..2bff17fa7 100644
--- a/roles/ceph-nfs/tasks/docker/main.yml
+++ b/roles/ceph-nfs/tasks/docker/main.yml
@@ -6,23 +6,11 @@
   failed_when: false
   always_run: true
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  register: stat_ostree
-  always_run: true
-
-- name: set fact for using Atomic host
-  set_fact:
-      is_atomic: '{{ stat_ostree.stat.exists }}'
-
 - include: checks.yml
   when:
     ceph_health.rc != 0 and
     not mon_containerized_deployment_with_kv
 
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-nfs/tasks/docker/pre_requisite.yml b/roles/ceph-nfs/tasks/docker/pre_requisite.yml
deleted file mode 100644
index 8b24c58ac..000000000
--- a/roles/ceph-nfs/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,117 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-- name: enable extras repo for centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when: ansible_distribution == 'CentOS'
-  tags:
-    with_pkg
-
-- name: install pip and docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    ansible_os_family == 'RedHat' and
-    ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    ansible_os_family == 'RedHat' and
-    ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
- 
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: pause after docker install before starting (on openstack vms)
-  pause: seconds=5
-  when: ceph_docker_on_openstack
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-osd/tasks/docker/main.yml b/roles/ceph-osd/tasks/docker/main.yml
index 9771bcd14..23ecd54e3 100644
--- a/roles/ceph-osd/tasks/docker/main.yml
+++ b/roles/ceph-osd/tasks/docker/main.yml
@@ -12,18 +12,6 @@
     - not osd_containerized_deployment_with_kv
     - not "{{ rolling_update | default(false) }}"
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  register: stat_ostree
-  always_run: true
-
-- name: set fact for using Atomic host
-  set_fact:
-    is_atomic: '{{ stat_ostree.stat.exists }}'
-
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-osd/tasks/docker/pre_requisite.yml b/roles/ceph-osd/tasks/docker/pre_requisite.yml
deleted file mode 100644
index 85a1047f0..000000000
--- a/roles/ceph-osd/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,136 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-- name: pause after docker install before starting (on openstack vms)
-  pause: seconds=5
-  when: ceph_docker_on_openstack
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
-
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-rbd-mirror/tasks/docker/main.yml b/roles/ceph-rbd-mirror/tasks/docker/main.yml
index 20da41b5c..dbbc87eea 100644
--- a/roles/ceph-rbd-mirror/tasks/docker/main.yml
+++ b/roles/ceph-rbd-mirror/tasks/docker/main.yml
@@ -6,21 +6,9 @@
   failed_when: false
   always_run: true
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  always_run: true
-  register: stat_ostree
-
-- name: set fact for using Atomic host
-  set_fact:
-      is_atomic='{{ stat_ostree.stat.exists }}'
-
 - include: checks.yml
   when: ceph_health.rc != 0
 
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml b/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml
deleted file mode 100644
index edaeb6715..000000000
--- a/roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,144 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-# install epel for pip
-- name: install epel on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - epel-release
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-- name: enable extras repo for centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when: ansible_distribution == 'CentOS'
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# docker package could be docker-enginer or docker
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
-
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-restapi/tasks/docker/main.yml b/roles/ceph-restapi/tasks/docker/main.yml
index 22e7a5f63..c0143f9cc 100644
--- a/roles/ceph-restapi/tasks/docker/main.yml
+++ b/roles/ceph-restapi/tasks/docker/main.yml
@@ -1,16 +1,4 @@
 ---
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  register: stat_ostree
-  always_run: true
-
-- name: set fact for using Atomic host
-  set_fact:
-      is_atomic: '{{ stat_ostree.stat.exists }}'
-
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-restapi/tasks/docker/pre_requisite.yml b/roles/ceph-restapi/tasks/docker/pre_requisite.yml
deleted file mode 100644
index c6ebb9e2e..000000000
--- a/roles/ceph-restapi/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,142 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-
-- name: enable extras repo on centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when: ansible_distribution == 'CentOS'
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
- 
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: pause after docker install before starting (on openstack vms)
-  pause: seconds=5
-  when: ceph_docker_on_openstack
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg
diff --git a/roles/ceph-rgw/tasks/docker/main.yml b/roles/ceph-rgw/tasks/docker/main.yml
index f3b9b4ebe..5779e805e 100644
--- a/roles/ceph-rgw/tasks/docker/main.yml
+++ b/roles/ceph-rgw/tasks/docker/main.yml
@@ -6,23 +6,11 @@
   failed_when: false
   always_run: true
 
-- name: check if it is Atomic host
-  stat: path=/run/ostree-booted
-  always_run: true
-  register: stat_ostree
-
-- name: set fact for using Atomic host
-  set_fact:
-    is_atomic: '{{ stat_ostree.stat.exists }}'
-
 - include: checks.yml
   when:
     - ceph_health.rc != 0
     - not "{{ rolling_update | default(false) }}"
 
-- include: pre_requisite.yml
-  when: not is_atomic
-
 - include: "{{ playbook_dir }}/roles/ceph-common/tasks/misc/ntp_atomic.yml"
   when:
     - is_atomic
diff --git a/roles/ceph-rgw/tasks/docker/pre_requisite.yml b/roles/ceph-rgw/tasks/docker/pre_requisite.yml
deleted file mode 100644
index e66c0f49c..000000000
--- a/roles/ceph-rgw/tasks/docker/pre_requisite.yml
+++ /dev/null
@@ -1,130 +0,0 @@
----
-- name: install pip and docker on ubuntu
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker
-    - docker.io
-  when: ansible_distribution == 'Ubuntu'
-  tags:
-    with_pkg
-
-- name: install pip and docker on debian
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - python-pip
-    - docker-engine
-  when: ansible_distribution == 'Debian'
-  tags:
-    with_pkg
-
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-    
-- name: enable extras repo on centos
-  yum_repository:
-    name: extras
-    state: present
-    enabled: yes
-  when: ansible_distribution == 'CentOS'
-  tags:
-    with_pkg
-
-- name: install pip on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install docker-engine on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# for CentOS
-- name: install docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - docker
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-  failed_when: false
-
-# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
-# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
-- name: install six
-  pip:
-    name: six
-    version: 1.9.0
-  tags:
-    with_pkg
-
-# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
-- name: install docker-py
-  pip:
-    name: docker-py
-    version: 1.1.0
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
- 
-- name: install docker-py
-  pip:
-    name: docker-py
-    state: latest
-  tags:
-    with_pkg
-  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')
-
-- name: pause after docker install before starting (on openstack vms)
-  pause: seconds=5
-  when: ceph_docker_on_openstack
-  tags:
-    with_pkg
-
-- name: start docker service
-  service:
-    name: docker
-    state: started
-    enabled: yes
-  tags:
-    with_pkg
-
-- name: install ntp
-  package:
-    name: ntp
-    state: present
-  when:
-    - ntp_service_enabled
-  tags:
-    with_pkg

From f9a317ef30d689c2b9f39ddb5d6eaaa1570aa616 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Fri, 3 Mar 2017 09:56:30 -0700
Subject: [PATCH 34/80] check_system: improve RHCS support message and version
 comparison

"red hat storage" -> "red hat ceph storage"

"RHEL 7.1" -> "RHEL 7"

and make the version number check verify the very latest version

https://bugzilla.redhat.com/show_bug.cgi?id=1354059
---
 roles/ceph-common/tasks/checks/check_system.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/ceph-common/tasks/checks/check_system.yml b/roles/ceph-common/tasks/checks/check_system.yml
index e198e1f18..985c387e3 100644
--- a/roles/ceph-common/tasks/checks/check_system.yml
+++ b/roles/ceph-common/tasks/checks/check_system.yml
@@ -14,12 +14,12 @@
     msg: "Distribution not supported {{ ansible_os_family }}"
   when: "'{{ ansible_os_family }}' not in ['Debian', 'RedHat']"
 
-- name: fail on unsupported distribution for red hat storage
+- name: fail on unsupported distribution for red hat ceph storage
   fail:
-    msg: "Distribution not supported {{ ansible_distribution_version }} by Red Hat Ceph Storage, only RHEL 7.1"
+    msg: "Distribution not supported {{ ansible_distribution_version }} by Red Hat Ceph Storage, only RHEL 7"
   when:
     - ceph_rhcs
-    - ansible_distribution_version | version_compare('7.1', '<')
+    - ansible_distribution_version | version_compare('7.3', '<')
 
 - name: fail on unsupported distribution for ubuntu cloud archive
   fail:

From a77cd4be5e2365efd60047d7eb6dd1972e200f83 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Fri, 3 Mar 2017 11:20:05 -0700
Subject: [PATCH 35/80] avoid setting vfs_cache_pressure

From Josh Durgin, "I'd recommend not setting vfs_cache_pressure in
ceph-ansible. The syncfs issue is still there, and has caused real
problems in the past, whereas there hasn't been good data showing lower
vfs_cache_pressure is very helpful - the only cases I'm aware of have
shown it makes little difference to performance."

https://bugzilla.redhat.com/show_bug.cgi?id=1395451
---
 group_vars/all.yml.sample           | 1 -
 roles/ceph-common/defaults/main.yml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample
index 3c09f7c4d..c56ba8063 100644
--- a/group_vars/all.yml.sample
+++ b/group_vars/all.yml.sample
@@ -383,7 +383,6 @@ dummy:
 #  - { name: kernel.pid_max, value: 4194303 }
 #  - { name: fs.file-max, value: 26234859 }
 #  - { name: vm.zone_reclaim_mode, value: 0 }
-#  - { name: vm.vfs_cache_pressure, value: 50 }
 #  - { name: vm.swappiness, value: 10 }
 #  - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 
diff --git a/roles/ceph-common/defaults/main.yml b/roles/ceph-common/defaults/main.yml
index 3f5f2fe2d..c3753cba8 100644
--- a/roles/ceph-common/defaults/main.yml
+++ b/roles/ceph-common/defaults/main.yml
@@ -375,7 +375,6 @@ os_tuning_params:
   - { name: kernel.pid_max, value: 4194303 }
   - { name: fs.file-max, value: 26234859 }
   - { name: vm.zone_reclaim_mode, value: 0 }
-  - { name: vm.vfs_cache_pressure, value: 50 }
   - { name: vm.swappiness, value: 10 }
   - { name: vm.min_free_kbytes, value: "{{ vm_min_free_kbytes }}" }
 

From db241751db1c8ffc5c4bf4c73a059dbe33428875 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Tue, 7 Mar 2017 11:11:53 -0700
Subject: [PATCH 36/80] Makefile: generate tagged RPMs with "-1" release value

Prior to this change, if we built an RPM from a tagged commit, the RPM
would have a release value of "0".

This causes some problems with RPM version comparisons when comparing
version numbers with "rc" tags/packages, which should also have a
less-than-one Release value.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 120be428d..214615019 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ RELEASE := $(shell git describe --tags --match 'v*' \
              | sed 's/^[^-]*-//' \
              | sed 's/-/./')
 ifeq ($(VERSION),$(RELEASE))
-  RELEASE = 0
+  RELEASE = 1
 endif
 NVR := $(NAME)-$(VERSION)-$(RELEASE).el7
 

From 08a87e59e47e86aa7478b956ffcbc7fea99664b4 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Tue, 7 Mar 2017 11:01:15 -0700
Subject: [PATCH 37/80] Makefile: handle "rc" version tags

When "git describe" returns an "rc" tag (eg "2.2.0rc1"), insert the RC
number into the RPM's "Release" field.

Add comments with some git-describe values and NVRs.
---
 Makefile | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/Makefile b/Makefile
index 214615019..659fccd5a 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,19 @@
 # Try "make" (for SRPMS) or "make rpm"
 
 NAME = ceph-ansible
+
+# Set the RPM package NVR from "git describe".
+# Examples:
+#
+#  A "git describe" value of "v2.2.0rc1" would create an NVR
+#  "ceph-ansible-2.2.0-0.rc1.1.el7"
+#
+#  A "git describe" value of "v2.2.0rc1-1-gc465f85" would create an NVR
+#  "ceph-ansible-2.2.0-0.rc1.1.gc465f85.el7"
+#
+#  A "git describe" value of "v2.2.0" creates an NVR
+#  "ceph-ansible-2.2.0-1.el7"
+
 VERSION := $(shell git describe --tags --abbrev=0 --match 'v*' | sed 's/^v//')
 COMMIT := $(shell git rev-parse HEAD)
 SHORTCOMMIT := $(shell echo $(COMMIT) | cut -c1-7)
@@ -12,6 +25,11 @@ RELEASE := $(shell git describe --tags --match 'v*' \
 ifeq ($(VERSION),$(RELEASE))
   RELEASE = 1
 endif
+ifneq (,$(findstring rc,$(VERSION)))
+    RC := $(shell echo $(VERSION) | sed 's/.*rc/rc/')
+    RELEASE := 0.$(RC).$(RELEASE)
+    VERSION := $(subst $(RC),,$(VERSION))
+endif
 NVR := $(NAME)-$(VERSION)-$(RELEASE).el7
 
 all: srpm

From 4fe66070041fb8df49f4608e1ab1f8a402ecfd3f Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 10:16:09 -0600
Subject: [PATCH 38/80] purge-cluster: do not set group name vars at playbook
 level

This has the behavior of overriding custom values set in group_vars.
I've added defaults to the rest of the group names so that if they are
not overridden in group_vars then defaults will be used.

See: https://bugzilla.redhat.com/show_bug.cgi?id=1354700

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 infrastructure-playbooks/purge-cluster.yml | 66 +++++++---------------
 1 file changed, 20 insertions(+), 46 deletions(-)

diff --git a/infrastructure-playbooks/purge-cluster.yml b/infrastructure-playbooks/purge-cluster.yml
index 8bd1471eb..df68836b1 100644
--- a/infrastructure-playbooks/purge-cluster.yml
+++ b/infrastructure-playbooks/purge-cluster.yml
@@ -33,23 +33,14 @@
 
 - name: gather facts on all hosts
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-    client_group_name:       clients
-
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
-    - "{{ rbdmirror_group_name }}"
-    - "{{ nfs_group_name }}"
-    - "{{ client_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ nfs_group_name|default('nfss') }}"
+    - "{{ client_group_name|default('clients') }}"
 
   become: true
 
@@ -62,7 +53,7 @@
     mds_group_name: mdss
 
   hosts:
-    - "{{ mds_group_name }}"
+    - "{{ mds_group_name|default('mdss') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -93,7 +84,7 @@
     rgw_group_name: rgws
 
   hosts:
-    - "{{ rgw_group_name }}"
+    - "{{ rgw_group_name|default('rgws') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -124,7 +115,7 @@
     rbdmirror_group_name: rbd-mirrors
 
   hosts:
-    - "{{ rbdmirror_group_name }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -150,7 +141,7 @@
     nfs_group_name: nfss
 
   hosts:
-    - "{{ nfs_group_name }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -180,7 +171,7 @@
     osd_group_name: osds
 
   hosts:
-    - "{{ osd_group_name }}"
+    - "{{ osd_group_name|default('osds') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -358,7 +349,7 @@
     restapi_group_name:   restapis
 
   hosts:
-    - "{{ mon_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -390,14 +381,6 @@
 - name: final cleanup - check any running ceph, purge ceph packages, purge config and remove data
 
   vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-    client_group_name:       clients
-
     # When set to true both groups of packages are purged.
     # This can cause problem with qemu-kvm
     purge_all_packages: true
@@ -423,13 +406,13 @@
       - python-rbd
 
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
-    - "{{ rbdmirror_group_name }}"
-    - "{{ nfs_group_name }}"
-    - "{{ client_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ nfs_group_name|default('nfss') }}"
+    - "{{ client_group_name|default('clients') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -554,15 +537,6 @@
 
 - name: purge fetch directory
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbdmirrors
-    nfs_group_name:       nfss
-    restapi_group_name:   restapis
-
   hosts:
     - localhost
 

From 46f26bec1374ab957bec88e0b44878a95ac15248 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 13:16:12 -0600
Subject: [PATCH 39/80] rolling-update: do not set group name vars at playbook
 level

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 infrastructure-playbooks/rolling_update.yml | 25 +++++++--------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index d258020af..1d3c9f281 100644
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -34,17 +34,12 @@
 
 
 - name: gather facts and check the init system
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
 
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
 
   become: True
   tasks:
@@ -55,13 +50,12 @@
 - name: upgrade ceph mon cluster
 
   vars:
-    mon_group_name:       mons
     health_mon_check_retries: 5
     health_mon_check_delay: 10
     upgrade_ceph_packages: True
 
   hosts:
-    - "{{ mon_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
 
   serial: 1
   become: True
@@ -170,13 +164,12 @@
 - name: upgrade ceph osds cluster
 
   vars:
-    osd_group_name: osds
     health_osd_check_retries: 40
     health_osd_check_delay: 30
     upgrade_ceph_packages: True
 
   hosts:
-    - "{{ osd_group_name }}"
+    - "{{ osd_group_name|default('osds') }}"
 
   serial: 1
   become: True
@@ -314,11 +307,10 @@
 - name: upgrade ceph mdss cluster
 
   vars:
-    mds_group_name: mdss
     upgrade_ceph_packages: True
 
   hosts:
-    - "{{ mds_group_name }}"
+    - "{{ mds_group_name|default('mdss') }}"
 
   serial: 1
   become: True
@@ -387,11 +379,10 @@
 - name: upgrade ceph rgws cluster
 
   vars:
-    rgw_group_name: rgws
     upgrade_ceph_packages: True
 
   hosts:
-    - "{{ rgw_group_name }}"
+    - "{{ rgw_group_name|default('rgws') }}"
 
   serial: 1
   become: True

From 7289acb6b35c7e28593f67982aa6bdb88ed6477e Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 13:23:40 -0600
Subject: [PATCH 40/80] purge-docker-cluster: do not set group names vars at
 playbook level

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 .../purge-docker-cluster.yml                  | 80 +++++--------------
 1 file changed, 18 insertions(+), 62 deletions(-)

diff --git a/infrastructure-playbooks/purge-docker-cluster.yml b/infrastructure-playbooks/purge-docker-cluster.yml
index 66cb1d768..bd5469f6d 100644
--- a/infrastructure-playbooks/purge-docker-cluster.yml
+++ b/infrastructure-playbooks/purge-docker-cluster.yml
@@ -46,11 +46,8 @@
 
 - name: purge ceph mds cluster
 
-  vars:
-    mds_group_name: mdss
-
   hosts:
-    - "{{ mds_group_name }}"
+    - "{{ mds_group_name|default('mdss') }}"
 
   become: true
 
@@ -93,11 +90,8 @@
 
 - name: purge ceph rgw cluster
 
-  vars:
-    rgw_group_name: rgws
-
   hosts:
-    - "{{ rgw_group_name }}"
+    - "{{ rgw_group_name|default('rgws') }}"
 
   become: true
 
@@ -140,11 +134,8 @@
 
 - name: purge ceph rbd-mirror cluster
 
-  vars:
-    rbdmirror_group_name: rbd-mirrors
-
   hosts:
-    - "{{ rbdmirror_group_name }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
   become: true
 
@@ -187,11 +178,8 @@
 
 - name: purge ceph nfs cluster
 
-  vars:
-    nfs_group_name: nfss
-
   hosts:
-    - "{{ nfs_group_name }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   become: true
 
@@ -243,11 +231,8 @@
 
 - name: purge ceph osd cluster
 
-  vars:
-    osd_group_name: osds
-
   hosts:
-    - "{{ osd_group_name }}"
+    - "{{ osd_group_name|default('osds') }}"
 
   become: true
 
@@ -331,12 +316,8 @@
 
 - name: purge ceph mon cluster
 
-  vars:
-    mon_group_name:       mons
-    restapi_group_name:   restapis
-
   hosts:
-    - "{{ mon_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
 
   become: true
 
@@ -389,21 +370,13 @@
 
 - name: remove installed packages
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
-    - "{{ rbdmirror_group_name }}"
-    - "{{ nfs_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   become: true
 
@@ -559,21 +532,13 @@
 
 - name: purge ceph directories
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
-    - "{{ rbdmirror_group_name }}"
-    - "{{ nfs_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   gather_facts: false # Already gathered previously
 
@@ -592,15 +557,6 @@
 
 - name: purge fetch directory
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-    restapi_group_name:   restapis
-
   hosts:
     - localhost
 

From aef54d89d998b714d1b8f14f998c4271b8664bf5 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 13:32:57 -0600
Subject: [PATCH 41/80] switch-to-containers: do not set group name vars at
 playbook level

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 ...inerized-to-containerized-ceph-daemons.yml | 44 +++++--------------
 1 file changed, 12 insertions(+), 32 deletions(-)

diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
index 26edb8866..b9df8bd4a 100644
--- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
@@ -28,21 +28,13 @@
 
 - name: make sure docker is present and started
 
-  vars:
-    mon_group_name:       mons
-    osd_group_name:       osds
-    mds_group_name:       mdss
-    rgw_group_name:       rgws
-    rbdmirror_group_name: rbd-mirrors
-    nfs_group_name:       nfss
-
   hosts:
-    - "{{ mon_group_name }}"
-    - "{{ osd_group_name }}"
-    - "{{ mds_group_name }}"
-    - "{{ rgw_group_name }}"
-    - "{{ rbdmirror_group_name }}"
-    - "{{ nfs_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
+    - "{{ osd_group_name|default('osds') }}"
+    - "{{ mds_group_name|default('mdss') }}"
+    - "{{ rgw_group_name|default('rgws') }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   become: true
 
@@ -97,7 +89,7 @@
     restapi_group_name:   restapis
 
   hosts:
-    - "{{ mon_group_name }}"
+    - "{{ mon_group_name|default('mons') }}"
 
   serial: 1
   become: true
@@ -187,7 +179,7 @@
     osd_group_name: osds
 
   hosts:
-    - "{{ osd_group_name }}"
+    - "{{ osd_group_name|default('osds') }}"
 
   serial: 1
   become: true
@@ -288,11 +280,8 @@
 
 - name: switching from non-containerized to containerized ceph mds
 
-  vars:
-    mds_group_name: mdss
-
   hosts:
-    - "{{ mds_group_name }}"
+    - "{{ mds_group_name|default('mdss') }}"
 
   serial: 1
   become: true
@@ -348,11 +337,8 @@
 
 - name: switching from non-containerized to containerized ceph rgw
 
-  vars:
-    rgw_group_name: rgws
-
   hosts:
-    - "{{ rgw_group_name }}"
+    - "{{ rgw_group_name|default('rgws') }}"
 
   serial: 1
   become: true
@@ -409,11 +395,8 @@
 
 - name: switching from non-containerized to containerized ceph rbd-mirror
 
-  vars:
-    rbdmirror_group_name: rbd-mirrors
-
   hosts:
-    - "{{ rbdmirror_group_name }}"
+    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
 
   serial: 1
   become: true
@@ -468,11 +451,8 @@
 
 - name: switching from non-containerized to containerized ceph nfs
 
-  vars:
-    nfs_group_name: nfss
-
   hosts:
-    - "{{ nfs_group_name }}"
+    - "{{ nfs_group_name|default('nfss') }}"
 
   serial: 1
   become: true

From cf702b05cfbcdf76f4738a98ce2c779b53a224fc Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 13:45:22 -0600
Subject: [PATCH 42/80] purge-docker-cluster: do not include role defaults or
 group vars

Doing so at playbook level overrides whatever values might be set for
these in the user's group_vars directory that's relative to their
inventory.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 .../purge-docker-cluster.yml                  | 61 ++-----------------
 1 file changed, 5 insertions(+), 56 deletions(-)

diff --git a/infrastructure-playbooks/purge-docker-cluster.yml b/infrastructure-playbooks/purge-docker-cluster.yml
index bd5469f6d..d51d831c3 100644
--- a/infrastructure-playbooks/purge-docker-cluster.yml
+++ b/infrastructure-playbooks/purge-docker-cluster.yml
@@ -52,12 +52,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-mds/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mds_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph mds service
     service:
@@ -96,12 +90,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-rgw/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rgw_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph rgw service
     service:
@@ -140,12 +128,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-rbd-mirror/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rbdmirror_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph rbd-mirror service
     service:
@@ -184,12 +166,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-nfs/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ nfs_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph nfs service
     service:
@@ -237,12 +213,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-osd/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ osd_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph osd service
     service:
@@ -322,15 +292,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-mon/defaults/main.yml
-  - include_vars: roles/ceph-restapi/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mon_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ restapi_group_name }}.yml
-    failed_when: false
 
   - name: disable ceph mon service
     service:
@@ -563,23 +524,11 @@
   gather_facts: false
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mds_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rgw_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rbdmirror_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ nfs_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ osd_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mon_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ restapi_group_name }}.yml
-    failed_when: false
+
+  - name: set fetch_directory value if not set
+    set_fact:
+      fetch_directory: "fetch/"
+    when: fetch_directory is not defined
 
   - name: purge fetch directory for localhost
     file:

From e81d690aa02f8c531592bc26865d322af92f2adc Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 13:51:48 -0600
Subject: [PATCH 43/80] switch-to-containers: do not include group vars or role
 defaults

Doing so will override any values set for these in the group_vars
directory relative to the users inventory.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 ...inerized-to-containerized-ceph-daemons.yml | 39 -------------------
 1 file changed, 39 deletions(-)

diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
index b9df8bd4a..10bf98a79 100644
--- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
+++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml
@@ -95,15 +95,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-mon/defaults/main.yml
-  - include_vars: roles/ceph-restapi/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mon_group_name }}.yml
-    failed_when: false
-  - include_vars: group_vars/{{ restapi_group_name }}.yml
-    failed_when: false
 
   - name: select a running monitor
     set_fact: mon_host={{ item }}
@@ -185,12 +176,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-osd/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ osd_group_name }}.yml
-    failed_when: false
 
   - name: collect osd ids
     shell: |
@@ -287,12 +272,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-mds/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ mds_group_name }}.yml
-    failed_when: false
 
   - name: stop ceph mds service
     service:
@@ -344,12 +323,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-rgw/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rgw_group_name }}.yml
-    failed_when: false
 
   - name: stop ceph rgw service
     service:
@@ -402,12 +375,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-rbd-mirror/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ rbdmirror_group_name }}.yml
-    failed_when: false
 
   - name: stop ceph rbd mirror service
     service:
@@ -458,12 +425,6 @@
   become: true
 
   tasks:
-  - include_vars: roles/ceph-common/defaults/main.yml
-  - include_vars: roles/ceph-nfs/defaults/main.yml
-  - include_vars: group_vars/all.yml
-    failed_when: false
-  - include_vars: group_vars/{{ nfs_group_name }}.yml
-    failed_when: false
 
   - name: stop ceph nfs service
     service:

From 04c113b05fa0f2fde4a0b6374128f87b681d4eb0 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 7 Mar 2017 14:05:03 -0600
Subject: [PATCH 44/80] tests: adds an update_cluster testing scenario

This updates the centos 7 cluster scenario from jewel to kraken using
the rolling_update playbook.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 9fcb5f6be..57f455ac0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt}
+envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster}
 skipsdist = True
 
 # extra commands for purging clusters
@@ -63,6 +63,7 @@ changedir=
   purge_cluster: {toxinidir}/tests/functional/ubuntu/16.04/cluster
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
+  update_cluster: {toxinidir}/tests/functional/centos/7/cluster
 commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
@@ -75,5 +76,6 @@ commands=
   purge_cluster: {[purge]commands}
   purge_dmcrypt: {[purge]commands}
   update_dmcrypt: {[update]commands}
+  update_cluster: {[update]commands}
 
   vagrant destroy --force

From 3c6a7a60ebbdbdd93b2ab0298e3cf215a4be2931 Mon Sep 17 00:00:00 2001
From: Boris Ranto <branto@redhat.com>
Date: Mon, 13 Mar 2017 13:46:38 +0100
Subject: [PATCH 45/80] calamari: Add debugging option

This patch introduces calamari_debug option which will turn on debugging
for calamari before initializing and running it.

Signed-off-by: Boris Ranto <branto@redhat.com>
---
 group_vars/mons.yml.sample        |  3 +++
 roles/ceph-mon/defaults/main.yml  |  3 +++
 roles/ceph-mon/tasks/calamari.yml | 11 +++++++++++
 3 files changed, 17 insertions(+)

diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample
index 4dc8b7c99..43341f3ae 100644
--- a/group_vars/mons.yml.sample
+++ b/group_vars/mons.yml.sample
@@ -42,6 +42,9 @@ dummy:
 # Enable the Calamari-backed REST API on a Monitor
 #calamari: false
 
+# Enable debugging for Calamari
+#calamari_debug: false
+
 #############
 # OPENSTACK #
 #############
diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index a9a837ec0..6013bda4e 100644
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -34,6 +34,9 @@ secure_cluster_flags:
 # Enable the Calamari-backed REST API on a Monitor
 calamari: false
 
+# Enable debugging for Calamari
+calamari_debug: false
+
 #############
 # OPENSTACK #
 #############
diff --git a/roles/ceph-mon/tasks/calamari.yml b/roles/ceph-mon/tasks/calamari.yml
index 30cb08015..14d9f0542 100644
--- a/roles/ceph-mon/tasks/calamari.yml
+++ b/roles/ceph-mon/tasks/calamari.yml
@@ -6,5 +6,16 @@
   tags:
     - package-install
 
+- name: increase calamari logging level when debug is on
+  ini_file:
+    dest: /etc/calamari/calamari.conf
+    section: "{{ item }}"
+    option: log_level
+    value: DEBUG
+  with_items:
+    - cthulhu
+    - calamari_web
+  when: calamari_debug
+
 - name: initialize the calamari server api
   command: calamari-ctl initialize

From bf57efaf3cf198f0e84c56988ae35ef055fe9ae3 Mon Sep 17 00:00:00 2001
From: Ken Dreyer <kdreyer@redhat.com>
Date: Mon, 13 Mar 2017 09:34:35 -0600
Subject: [PATCH 46/80] ceph-common: install nfs-ganesha FSALs on Debian

Prior to this change, ceph-ansible would install the main NFS Ganesha
server daemon on Ubuntu, but it would skip the Ceph FSALs.

Running "apt-get install nfs-ganesha" will only install the main NFS Ganesha
server. It does *not* pull in the RGW FSAL
(/usr/lib/x86_64-linux-gnu/ganesha/libfsalrgw.so)

Running "apt-get install nfs-ganesha-fsal" will install the RGW FSAL as
well as the main NFS Ganesha server package.

Signed-off-by: Ken Dreyer <kdreyer@redhat.com>
---
 roles/ceph-common/tasks/installs/install_on_debian.yml          | 2 +-
 .../ceph-common/tasks/installs/install_rh_storage_on_debian.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/ceph-common/tasks/installs/install_on_debian.yml b/roles/ceph-common/tasks/installs/install_on_debian.yml
index 0f8648216..c6ab701ac 100644
--- a/roles/ceph-common/tasks/installs/install_on_debian.yml
+++ b/roles/ceph-common/tasks/installs/install_on_debian.yml
@@ -53,7 +53,7 @@
 
 - name: install NFS gateway
   apt:
-    pkg: nfs-ganesha
+    pkg: nfs-ganesha-fsal
     state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
     update_cache: yes
   when: nfs_group_name in group_names
diff --git a/roles/ceph-common/tasks/installs/install_rh_storage_on_debian.yml b/roles/ceph-common/tasks/installs/install_rh_storage_on_debian.yml
index 26a0c1825..1551638f3 100644
--- a/roles/ceph-common/tasks/installs/install_rh_storage_on_debian.yml
+++ b/roles/ceph-common/tasks/installs/install_rh_storage_on_debian.yml
@@ -87,6 +87,6 @@
 
 - name: install red hat storage nfs gateway
   apt:
-    name: nfs-ganesha
+    name: nfs-ganesha-fsal
     state: "{{ (upgrade_ceph_packages|bool) | ternary('latest','present') }}"
   when: nfs_group_name in group_names

From 8567609b7e32638d1733af029a32b42e7473c07e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Tue, 14 Mar 2017 22:46:09 +0100
Subject: [PATCH 47/80] kv scenario: add KV_PORT option
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This option was missing for rrgw, mds, rbd mirror and nfs making these
daemon impossible to run on a kv deployment with containers.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 roles/ceph-mds/templates/ceph-mds.service.j2               | 1 +
 roles/ceph-nfs/templates/ceph-nfs.service.j2               | 1 +
 roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 | 1 +
 roles/ceph-rgw/templates/ceph-rgw.service.j2               | 1 +
 4 files changed, 4 insertions(+)

diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2
index dcf9ff2bf..3ea1a32cb 100644
--- a/roles/ceph-mds/templates/ceph-mds.service.j2
+++ b/roles/ceph-mds/templates/ceph-mds.service.j2
@@ -13,6 +13,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    {% else -%}
    -e KV_TYPE={{kv_type}} \
    -e KV_IP={{kv_endpoint}} \
+   -e KV_PORT={{kv_port}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
    -e CEPH_DAEMON=MDS \
diff --git a/roles/ceph-nfs/templates/ceph-nfs.service.j2 b/roles/ceph-nfs/templates/ceph-nfs.service.j2
index d7471c61e..3b228b479 100644
--- a/roles/ceph-nfs/templates/ceph-nfs.service.j2
+++ b/roles/ceph-nfs/templates/ceph-nfs.service.j2
@@ -14,6 +14,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    {% else -%}
    -e KV_TYPE={{kv_type}} \
    -e KV_IP={{kv_endpoint}}\
+   -e KV_PORT={{kv_port}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
    --privileged \
diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
index 6cf8069ea..7bf77061e 100644
--- a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
+++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
@@ -12,6 +12,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    {% else -%}
    -e KV_TYPE={{kv_type}} \
    -e KV_IP={{kv_endpoint}} \
+   -e KV_PORT={{kv_port}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
    -e CEPH_DAEMON=RBD_MIRROR \
diff --git a/roles/ceph-rgw/templates/ceph-rgw.service.j2 b/roles/ceph-rgw/templates/ceph-rgw.service.j2
index 0573aa2e2..b63c60878 100644
--- a/roles/ceph-rgw/templates/ceph-rgw.service.j2
+++ b/roles/ceph-rgw/templates/ceph-rgw.service.j2
@@ -13,6 +13,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    {% else -%}
    -e KV_TYPE={{kv_type}} \
    -e KV_IP={{kv_endpoint}} \
+   -e KV_PORT={{kv_port}} \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
    --privileged \

From 77edd3d40ad0c8fcbf2bd4fc240a646282a1181a Mon Sep 17 00:00:00 2001
From: Daniel Marks <daniel.marks@codecentric.de>
Date: Wed, 15 Mar 2017 14:15:15 +0100
Subject: [PATCH 48/80] Fixing tabs that are breaking the syntax check

With the merge of PR #1336 the syntax check fails. This commit replaces
the tabs with proper indentation.
---
 infrastructure-playbooks/make-osd-partitions.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/infrastructure-playbooks/make-osd-partitions.yml b/infrastructure-playbooks/make-osd-partitions.yml
index c4ba15a5d..d348cb380 100644
--- a/infrastructure-playbooks/make-osd-partitions.yml
+++ b/infrastructure-playbooks/make-osd-partitions.yml
@@ -31,15 +31,15 @@
   - "{{ osd_group_name }}"
 
   tasks:
-  
+
   - name: load a variable file for devices partition
-    include_vars: "{{ item }}"
-    with_first_found:
+    include_vars: "{{ item }}"
+    with_first_found:
       - files:
           - "host_vars/{{ ansible_hostname }}.yml"
           - "host_vars/default.yml"
         skip: true
-        
+
   - name: exit playbook, if devices not defined
     fail:
       msg: "devices must be define in host_vars/default.yml or host_vars/{{ ansible_hostname }}.yml"

From 1f40347bd2ad516ba94bf6013c814036a4e8ae5c Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 14:32:42 -0500
Subject: [PATCH 49/80] tests: when running test set USE_RHCS=true to install
 set ceph_rhcs=true

When invoking the tests if USE_RHCS=true is set then all tests will be
run with ceph_rhcs=True.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 57f455ac0..90c226cd5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,9 +9,11 @@ skipsdist = True
 [purge]
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml \
+      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
   # set up the cluster again
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars="fetch_directory={changedir}/fetch"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample \
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -21,7 +23,8 @@ commands=
 [update]
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml \
+      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -68,7 +71,8 @@ commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars="fetch_directory={changedir}/fetch"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From 635f204001813568515b9d0d239b6be562741dda Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 14:53:42 -0500
Subject: [PATCH 50/80] tests: use CEPH_ORIGIN to control the value of
 ceph_origin

For example, the following would run the journal collocation test and
would install ceph from the repos already on the nodes:

  CEPH_ORIGIN=distro tox -rve ansible2.2-journal_collocation

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 90c226cd5..561dc05f4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,10 +10,10 @@ skipsdist = True
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml \
-      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
+      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -24,7 +24,7 @@ commands=
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml \
-      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
+      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -72,7 +72,7 @@ commands=
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From fbe2aca9af25b7ab118b50b25c9d2676b1c35a89 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 15:01:32 -0500
Subject: [PATCH 51/80] tests: add the ability to run tests with shaman repos

To run tests that deploy shaman repos set CEPH_DEV=true and optionally
use CEPH_DEV_BRANCH and CEPH_DEV_SHA1 to define with branch and sha1 to
test. CEPH_DEV_BRANCH defaults to master and CEPH_DEV_SHA1 defaults to
latest.

For example, this would run the journal_collocation test with the latest
build of the master branch:

  CEPH_DEV=true tox -rve ansible2.2-journal_collocation

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 561dc05f4..bad1fae04 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,10 +10,10 @@ skipsdist = True
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml \
-      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
+      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -24,7 +24,7 @@ commands=
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml \
-      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
+      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -72,7 +72,7 @@ commands=
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From 19a5362868d61550af4e957aa679f4994b5ff9cb Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 15:08:39 -0500
Subject: [PATCH 52/80] tests: adds the ability to set the ceph_stable_release
 value

Use CEPH_STABLE_RELEASE to set the name of the ceph release you plan to
install. When testing an upgrade scenario you'll also need to set
UPGRADE_CEPH_STABLE_RELEASE.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index bad1fae04..7118385ce 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,10 +10,10 @@ skipsdist = True
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml \
-      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -24,7 +24,7 @@ commands=
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml \
-      --extra-vars="ceph_stable_release=kraken ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
+      --extra-vars="ceph_stable_release={env:UPDATE_CEPH_STABLE_RELEASE:kraken} ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -72,7 +72,7 @@ commands=
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest}"
+      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From b1f57986076669b02dd0de42c1293068af49c3f8 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 16:06:00 -0500
Subject: [PATCH 53/80] tests: reformat tox.ini for better readability

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 42 ++++++++++++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 8 deletions(-)

diff --git a/tox.ini b/tox.ini
index 7118385ce..bc4ba3d61 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,11 +9,24 @@ skipsdist = True
 [purge]
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml \
-      --extra-vars="ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml --extra-vars="\
+      ireallymeanit=yes \
+      fetch_directory={changedir}/fetch \
+      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_origin={env:CEPH_ORIGIN:upstream} \
+      ceph_dev={env:CEPH_DEV:false} \
+      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   # set up the cluster again
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars="\
+      fetch_directory={changedir}/fetch \
+      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_origin={env:CEPH_ORIGIN:upstream} \
+      ceph_dev={env:CEPH_DEV:false} \
+      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -23,8 +36,15 @@ commands=
 [update]
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml \
-      --extra-vars="ceph_stable_release={env:UPDATE_CEPH_STABLE_RELEASE:kraken} ireallymeanit=yes fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml --extra-vars="\
+      ceph_stable_release={env:UPDATE_CEPH_STABLE_RELEASE:kraken} \
+      ireallymeanit=yes \
+      fetch_directory={changedir}/fetch \
+      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_origin={env:CEPH_ORIGIN:upstream} \
+      ceph_dev={env:CEPH_DEV:false} \
+      ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -71,8 +91,14 @@ commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
-      --extra-vars="fetch_directory={changedir}/fetch ceph_rhcs={env:USE_CEPH_RHCS:false} ceph_origin={env:CEPH_ORIGIN:upstream} ceph_dev={env:CEPH_DEV:false} ceph_dev_branch={env:CEPH_DEV_BRANCH:master} ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars="\
+      fetch_directory={changedir}/fetch \
+      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_origin={env:CEPH_ORIGIN:upstream} \
+      ceph_dev={env:CEPH_DEV:false} \
+      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
+      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From e3c8f7db4e74b6cdc778749fc7533e7e19aec29d Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 16:10:08 -0500
Subject: [PATCH 54/80] tests: use CEPH_STABLE to set the value for ceph_stable

When using CEPH_DEV=true you'll need to set CEPH_STABLE=false so that
that an upstream repo file doesn't get created.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index bc4ba3d61..61124376a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,7 +17,9 @@ commands=
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
+      ceph_stable={env:CEPH_STABLE:true} \
+  "
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars="\
       fetch_directory={changedir}/fetch \
@@ -26,7 +28,9 @@ commands=
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
+      ceph_stable={env:CEPH_STABLE:true} \
+  "
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -44,7 +48,9 @@ commands=
       ceph_origin={env:CEPH_ORIGIN:upstream} \
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
-      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest}"
+      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
+      ceph_stable={env:CEPH_STABLE:true} \
+  "
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -98,7 +104,9 @@ commands=
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
       ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel}"
+      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
+      ceph_stable={env:CEPH_STABLE:true} \
+  "
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From d76b1d960977d4e5339a17de5daacdbad33fc6e0 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 15 Mar 2017 16:28:50 -0500
Subject: [PATCH 55/80] tests: change USE_CEPH_RHCS to CEPH_RHCS

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 61124376a..6fd660e42 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,7 +12,7 @@ commands=
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml --extra-vars="\
       ireallymeanit=yes \
       fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_rhcs={env:CEPH_RHCS:false} \
       ceph_origin={env:CEPH_ORIGIN:upstream} \
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
@@ -23,7 +23,7 @@ commands=
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars="\
       fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_rhcs={env:CEPH_RHCS:false} \
       ceph_origin={env:CEPH_ORIGIN:upstream} \
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
@@ -44,7 +44,7 @@ commands=
       ceph_stable_release={env:UPDATE_CEPH_STABLE_RELEASE:kraken} \
       ireallymeanit=yes \
       fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_rhcs={env:CEPH_RHCS:false} \
       ceph_origin={env:CEPH_ORIGIN:upstream} \
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
@@ -99,7 +99,7 @@ commands=
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars="\
       fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:USE_CEPH_RHCS:false} \
+      ceph_rhcs={env:CEPH_RHCS:false} \
       ceph_origin={env:CEPH_ORIGIN:upstream} \
       ceph_dev={env:CEPH_DEV:false} \
       ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \

From 349b9ab3e72f188c50b5f84f3ce5ee2fddcfbf80 Mon Sep 17 00:00:00 2001
From: Christian Zunker <christian.zunker@codecentric.de>
Date: Wed, 15 Mar 2017 13:32:30 +0100
Subject: [PATCH 56/80] Make ceph-common aware off osd config fragments

This removes the implicit order requirement when using OSD fragments.
When you use OSD fragments and ceph-osd role is not the last one,
the fragments get removed from ceph.conf by ceph-common.

It is not nice to have this code at two locations, but this is
necessary to prevent problems, when ceph-osd is the last role as
ceph-common gets executed before ceph-osd.

This could be prevented when ceph-common would be explicitly called
at the end of the playbook.

Signed-off-by: Christian Zunker <christian.zunker@codecentric.de>
---
 roles/ceph-common/tasks/generate_ceph_conf.yml | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/roles/ceph-common/tasks/generate_ceph_conf.yml b/roles/ceph-common/tasks/generate_ceph_conf.yml
index 1766a5383..12b0bcfe5 100644
--- a/roles/ceph-common/tasks/generate_ceph_conf.yml
+++ b/roles/ceph-common/tasks/generate_ceph_conf.yml
@@ -1,22 +1,33 @@
 ---
-- name: create ceph conf directory
+- name: create ceph conf directory and assemble directory
   file:
-    path: /etc/ceph
+    path: "{{ item }}"
     state: directory
     owner: "ceph"
     group: "ceph"
     mode: "0755"
+  with_items:
+    - /etc/ceph/
+    - /etc/ceph/ceph.d/
 
 - name: "generate ceph configuration file: {{ cluster }}.conf"
   action: config_template
   args:
     src: ceph.conf.j2
-    dest: /etc/ceph/{{ cluster }}.conf
+    dest: /etc/ceph/ceph.d/{{ cluster }}.conf
     owner: "ceph"
     group: "ceph"
     mode: "0644"
     config_overrides: "{{ ceph_conf_overrides }}"
     config_type: ini
+
+- name: assemble {{ cluster }}.conf and fragments
+  assemble:
+    src: /etc/ceph/ceph.d/
+    dest: /etc/ceph/{{ cluster }}.conf
+    owner: "ceph"
+    group: "ceph"
+    mode: "0644"
   notify:
     - restart ceph mons
     - restart ceph osds

From 8c6079e23e3cad79cb802e5b1aa227d312562075 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 16 Mar 2017 10:18:31 +0100
Subject: [PATCH 57/80] rbd-mirror: fix spelling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It is RBD and NOT RDB.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 .../ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2 b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
index 7bf77061e..fe5ee0bb3 100644
--- a/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
+++ b/roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2
@@ -4,8 +4,8 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker stop ceph-rdb-mirror-{{ ansible_hostname }}
-ExecStartPre=-/usr/bin/docker rm ceph-rdb-mirror-{{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker rm ceph-rbd-mirror-{{ ansible_hostname }}
 ExecStart=/usr/bin/docker run --rm --net=host \
    {% if not rbd_mirror_containerized_deployment_with_kv -%}
    -v /etc/ceph:/etc/ceph \
@@ -16,9 +16,9 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    {% endif -%}
    -v /etc/localtime:/etc/localtime:ro \
    -e CEPH_DAEMON=RBD_MIRROR \
-   --name=ceph-rdb-mirror-{{ ansible_hostname }} \
+   --name=ceph-rbd-mirror-{{ ansible_hostname }} \
    {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/docker stop ceph-rdb-mirror-{{ ansible_hostname }}
+ExecStopPost=-/usr/bin/docker stop ceph-rbd-mirror-{{ ansible_hostname }}
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120

From ea68fbaaaee38b1a39b1f093e0faf5f897a466b0 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Fri, 24 Feb 2017 10:22:16 +0100
Subject: [PATCH 58/80] ceph-mon: Create openstack pools and keys even for
 containerized deployments

Add the possibility to create openstack pools and keys even for containerized deployments

Fix: #1321
Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
---
 group_vars/docker-commons.yml.sample          |  1 +
 group_vars/mons.yml.sample                    |  3 +-
 roles/ceph-docker-common/tasks/main.yml       |  9 ++++
 roles/ceph-mon/defaults/main.yml              |  2 +-
 roles/ceph-mon/tasks/ceph_keys.yml            | 43 ++-----------------
 roles/ceph-mon/tasks/docker/main.yml          | 12 ++++++
 roles/ceph-mon/tasks/openstack_config.yml     |  4 +-
 .../tasks/set_osd_pool_default_pg_num.yml     | 38 ++++++++++++++++
 8 files changed, 68 insertions(+), 44 deletions(-)
 create mode 100644 roles/ceph-mon/tasks/set_osd_pool_default_pg_num.yml

diff --git a/group_vars/docker-commons.yml.sample b/group_vars/docker-commons.yml.sample
index 69bcc8401..c6ef74d4b 100644
--- a/group_vars/docker-commons.yml.sample
+++ b/group_vars/docker-commons.yml.sample
@@ -9,4 +9,5 @@ dummy:
 
 
 #ceph_docker_registry: docker.io
+#ceph_mon_docker_enable_centos_extra_repo: false
 
diff --git a/group_vars/mons.yml.sample b/group_vars/mons.yml.sample
index 43341f3ae..5e28a72e5 100644
--- a/group_vars/mons.yml.sample
+++ b/group_vars/mons.yml.sample
@@ -77,7 +77,7 @@ dummy:
 ##########
 # DOCKER #
 ##########
-
+#docker_exec_cmd:
 #mon_containerized_deployment: false
 #mon_containerized_deployment_with_kv: false
 # This is currently in ceph-common defaults because it is shared with ceph-nfs
@@ -91,5 +91,4 @@ dummy:
 #mon_docker_privileged: false
 #mon_docker_net_host: true
 #ceph_config_keys: [] # DON'T TOUCH ME
-#ceph_mon_docker_enable_centos_extra_repo: false
 
diff --git a/roles/ceph-docker-common/tasks/main.yml b/roles/ceph-docker-common/tasks/main.yml
index 7e63d3295..a01824b16 100644
--- a/roles/ceph-docker-common/tasks/main.yml
+++ b/roles/ceph-docker-common/tasks/main.yml
@@ -13,3 +13,12 @@
 
 - include: ./pre_requisites/prerequisites.yml
   when: not is_atomic
+
+# NOTE(guits): would be nice to refact this block with L39-45 in roles/ceph-common/tasks/facts.yml
+- set_fact:
+    monitor_name: "{{ ansible_hostname }}"
+  when: not mon_use_fqdn
+
+- set_fact:
+    monitor_name: "{{ ansible_fqdn }}"
+  when: mon_use_fqdn
diff --git a/roles/ceph-mon/defaults/main.yml b/roles/ceph-mon/defaults/main.yml
index 6013bda4e..4fcaa88df 100644
--- a/roles/ceph-mon/defaults/main.yml
+++ b/roles/ceph-mon/defaults/main.yml
@@ -69,7 +69,7 @@ openstack_keys:
 ##########
 # DOCKER #
 ##########
-
+docker_exec_cmd:
 mon_containerized_deployment: false
 mon_containerized_deployment_with_kv: false
 # This is currently in ceph-common defaults because it is shared with ceph-nfs
diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml
index 4feba09e9..5bd61e51d 100644
--- a/roles/ceph-mon/tasks/ceph_keys.yml
+++ b/roles/ceph-mon/tasks/ceph_keys.yml
@@ -42,44 +42,7 @@
     - cephx
     - groups[restapi_group_name] is defined
 
-# NOTE(leseb): we add a conditional for backward compatibility
-# so people that had 'pool_default_pg_num' declared will get
-# the same behaviour
-#
-- name: check if does global key exist in ceph_conf_overrides
-  set_fact:
-    global_in_ceph_conf_overrides: "{{ 'global' in ceph_conf_overrides }}"
-
-- name: check if ceph_conf_overrides.global.osd_pool_default_pg_num is set
-  set_fact:
-    osd_pool_default_pg_num_in_overrides: "{{ 'osd_pool_default_pg_num' in ceph_conf_overrides.global }}"
-  when: global_in_ceph_conf_overrides
-
-- name: get default value for osd_pool_default_pg_num
-  shell: |
-    ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config get osd_pool_default_pg_num | grep -Po '(?<="osd_pool_default_pg_num": ")[^"]*'
-  failed_when: false
-  changed_when: false
-  run_once: true
-  register: default_pool_default_pg_num
-  when: (pool_default_pg_num is not defined or not global_in_ceph_conf_overrides)
-
-- set_fact:
-    osd_pool_default_pg_num: "{{ pool_default_pg_num }}"
-  when: pool_default_pg_num is defined
-
-- set_fact:
-    osd_pool_default_pg_num: "{{ default_pool_default_pg_num.stdout }}"
-  when:
-    - pool_default_pg_num is not defined
-    - default_pool_default_pg_num.rc == 0
-    - (osd_pool_default_pg_num_in_overrides is not defined or not osd_pool_default_pg_num_in_overrides)
-
-- set_fact:
-    osd_pool_default_pg_num: "{{ ceph_conf_overrides.global.osd_pool_default_pg_num }}"
-  when:
-    - global_in_ceph_conf_overrides
-    - ceph_conf_overrides.global.osd_pool_default_pg_num is defined
+- include: set_osd_pool_default_pg_num.yml
 
 - name: test if rbd exists
   command: ceph --cluster {{ cluster }} osd pool stats rbd
@@ -103,7 +66,9 @@
     - ceph_conf_overrides.global.osd_pool_default_size is defined
 
 - include: openstack_config.yml
-  when: openstack_config
+  when:
+    - openstack_config
+    - inventory_hostname == groups.mons|last
 
 - name: find ceph keys
   shell: ls -1 /etc/ceph/*.keyring
diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index ce2ad9324..6b1a53a21 100644
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -44,6 +44,10 @@
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'
 
+- name: set docker_exec_cmd fact
+  set_fact:
+    docker_exec_cmd: "docker exec ceph-mon-{{ ansible_hostname }}"
+
 - include: start_docker_monitor.yml
 
 - name: wait for monitor socket to exist
@@ -76,3 +80,11 @@
     - groups[restapi_group_name] is defined
     - inventory_hostname == groups.mons|last
     - not mon_containerized_deployment_with_kv
+
+- include: "{{ playbook_dir }}/roles/ceph-mon/tasks/set_osd_pool_default_pg_num.yml"
+
+# create openstack pools only when all mons are up.
+- include: "{{ playbook_dir }}/roles/ceph-mon/tasks/openstack_config.yml"
+  when:
+    - openstack_config
+    - inventory_hostname == groups.mons|last
diff --git a/roles/ceph-mon/tasks/openstack_config.yml b/roles/ceph-mon/tasks/openstack_config.yml
index 90894dc4c..5c45b9970 100644
--- a/roles/ceph-mon/tasks/openstack_config.yml
+++ b/roles/ceph-mon/tasks/openstack_config.yml
@@ -1,12 +1,12 @@
 ---
 - name: create openstack pool
-  command: ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }}
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} osd pool create {{ item.name }} {{ item.pg_num }}"
   with_items: "{{ openstack_pools | unique }}"
   changed_when: false
   failed_when: false
 
 - name: create openstack keys
-  command: ceph --cluster {{ cluster }} auth get-or-create {{ item.name }} {{ item.value }} -o /etc/ceph/{{ cluster }}.{{ item.name }}.keyring
+  command: "{{ docker_exec_cmd }} ceph --cluster {{ cluster }} auth get-or-create {{ item.name }} {{ item.value }} -o /etc/ceph/{{ cluster }}.{{ item.name }}.keyring"
   args:
     creates: /etc/ceph/{{ cluster }}.{{ item.name }}.keyring
   with_items: "{{ openstack_keys }}"
diff --git a/roles/ceph-mon/tasks/set_osd_pool_default_pg_num.yml b/roles/ceph-mon/tasks/set_osd_pool_default_pg_num.yml
new file mode 100644
index 000000000..e873410ab
--- /dev/null
+++ b/roles/ceph-mon/tasks/set_osd_pool_default_pg_num.yml
@@ -0,0 +1,38 @@
+# NOTE(leseb): we add a conditional for backward compatibility
+# so people that had 'pool_default_pg_num' declared will get
+# the same behaviour
+#
+- name: check if does global key exist in ceph_conf_overrides
+  set_fact:
+    global_in_ceph_conf_overrides: "{{ 'global' in ceph_conf_overrides }}"
+
+- name: check if ceph_conf_overrides.global.osd_pool_default_pg_num is set
+  set_fact:
+    osd_pool_default_pg_num_in_overrides: "{{ 'osd_pool_default_pg_num' in ceph_conf_overrides.global }}"
+  when: global_in_ceph_conf_overrides
+
+- name: get default value for osd_pool_default_pg_num
+  shell: |
+   {{ docker_exec_cmd }} ceph --cluster {{ cluster }} daemon mon.{{ monitor_name }} config get osd_pool_default_pg_num | grep -Po '(?<="osd_pool_default_pg_num": ")[^"]*'
+  failed_when: false
+  changed_when: false
+  run_once: true
+  register: default_pool_default_pg_num
+  when: pool_default_pg_num is not defined or not global_in_ceph_conf_overrides
+
+- set_fact:
+    osd_pool_default_pg_num: "{{ pool_default_pg_num }}"
+  when: pool_default_pg_num is defined
+
+- set_fact:
+    osd_pool_default_pg_num: "{{ default_pool_default_pg_num.stdout }}"
+  when:
+    - pool_default_pg_num is not defined
+    - default_pool_default_pg_num.rc == 0
+    - (osd_pool_default_pg_num_in_overrides is not defined or not osd_pool_default_pg_num_in_overrides)
+
+- set_fact:
+    osd_pool_default_pg_num: "{{ ceph_conf_overrides.global.osd_pool_default_pg_num }}"
+  when:
+    - global_in_ceph_conf_overrides
+    - ceph_conf_overrides.global.osd_pool_default_pg_num is defined

From 60a56b5d4ad978117821d0705557d7199cc4f337 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 16 Mar 2017 11:16:49 +0100
Subject: [PATCH 59/80] mon: increase wait for monitor socket timeout
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sometimes the socket appears during the 5th attempt and sometimes not so
increasing the timeout a little bit.

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 roles/ceph-mon/tasks/docker/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/ceph-mon/tasks/docker/main.yml b/roles/ceph-mon/tasks/docker/main.yml
index ce2ad9324..c7fb9b468 100644
--- a/roles/ceph-mon/tasks/docker/main.yml
+++ b/roles/ceph-mon/tasks/docker/main.yml
@@ -50,7 +50,7 @@
   command: docker exec ceph-mon-{{ ansible_hostname }} stat /var/run/ceph/{{ cluster }}-mon.{{ ansible_fqdn }}.asok
   register: monitor_socket
   retries: 5
-  delay: 10
+  delay: 15
   until: monitor_socket.rc == 0
 
 - name: force peer addition as potential bootstrap peer for cluster bringup

From 777913e9dad7d006cfe0dd3c8e1cf0ddd648aa0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Han?= <seb@redhat.com>
Date: Thu, 16 Mar 2017 10:57:13 +0100
Subject: [PATCH 60/80] docker: change centos extra repo var name
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This is not only for monitors, but also mds, rgw and rbd mirror so
making the var name more generic:
ceph_docker_enable_centos_extra_repo

Signed-off-by: Sébastien Han <seb@redhat.com>
---
 group_vars/docker-commons.yml.sample                            | 2 +-
 roles/ceph-docker-common/defaults/main.yml                      | 2 +-
 roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/group_vars/docker-commons.yml.sample b/group_vars/docker-commons.yml.sample
index c6ef74d4b..8515116ee 100644
--- a/group_vars/docker-commons.yml.sample
+++ b/group_vars/docker-commons.yml.sample
@@ -9,5 +9,5 @@ dummy:
 
 
 #ceph_docker_registry: docker.io
-#ceph_mon_docker_enable_centos_extra_repo: false
+#ceph_docker_enable_centos_extra_repo: false
 
diff --git a/roles/ceph-docker-common/defaults/main.yml b/roles/ceph-docker-common/defaults/main.yml
index 66408683f..073aec00c 100644
--- a/roles/ceph-docker-common/defaults/main.yml
+++ b/roles/ceph-docker-common/defaults/main.yml
@@ -1,4 +1,4 @@
 ---
 
 ceph_docker_registry: docker.io
-ceph_mon_docker_enable_centos_extra_repo: false
+ceph_docker_enable_centos_extra_repo: false
diff --git a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
index c337a6fa1..cb0951c86 100644
--- a/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
+++ b/roles/ceph-docker-common/tasks/pre_requisites/prerequisites.yml
@@ -20,7 +20,7 @@
     enabled: yes
   when:
     - ansible_distribution == 'CentOS'
-    - ceph_mon_docker_enable_centos_extra_repo
+    - ceph_docker_enable_centos_extra_repo
   tags:
     with_pkg
 

From 000110d2a2331dc17b7ab54762aa480837183cc2 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Thu, 16 Mar 2017 06:16:09 -0500
Subject: [PATCH 61/80] tests: convert extra-vars to use json

This will prevent ansible from misreading any of these values. There
were failures with xenial deployments because the value set for
``ceph_rhcs`` was being treated as a boolean True even though I'd set
the value to false. This is because boolean values passed in with
--extra-vars must use the json format.

The formatting of the json is very important as you need a '\' to escape
the starting and ending json to make tox happy. Also, each line needs to
end with '\' if it's a multi-line command.

Another thing to note is that if you want to use extra vars at the
command line to respond to a vars_prompt it must be in key/value format.
This is why we have a -e and a --extra-vars on the purge and update
tests.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 87 +++++++++++++++++++++++++++++----------------------------
 1 file changed, 45 insertions(+), 42 deletions(-)

diff --git a/tox.ini b/tox.ini
index 6fd660e42..4d9186a50 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,28 +9,29 @@ skipsdist = True
 [purge]
 commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml --extra-vars="\
-      ireallymeanit=yes \
-      fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:CEPH_RHCS:false} \
-      ceph_origin={env:CEPH_ORIGIN:upstream} \
-      ceph_dev={env:CEPH_DEV:false} \
-      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
-      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
-      ceph_stable={env:CEPH_STABLE:true} \
-  "
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml -e ireallymeanit=yes --extra-vars '\
+    \{\
+     "fetch_directory":"{changedir}/fetch",\
+     "ceph_rhcs":{env:CEPH_RHCS:false},\
+     "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
+     "ceph_dev":{env:CEPH_DEV:false},\
+     "ceph_dev_branch":"{env:CEPH_DEV_BRANCH:master}",\
+     "ceph_dev_sha1":"{env:CEPH_DEV_SHA1:latest}",\
+     "ceph_stable_release":"{env:CEPH_STABLE_RELEASE:jewel}",\
+     "ceph_stable":{env:CEPH_STABLE:true}\
+    \}'
   # set up the cluster again
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars="\
-      fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:CEPH_RHCS:false} \
-      ceph_origin={env:CEPH_ORIGIN:upstream} \
-      ceph_dev={env:CEPH_DEV:false} \
-      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
-      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
-      ceph_stable={env:CEPH_STABLE:true} \
-  "
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars '\
+    \{\
+     "fetch_directory":"{changedir}/fetch",\
+     "ceph_rhcs":{env:CEPH_RHCS:false},\
+     "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
+     "ceph_dev":{env:CEPH_DEV:false},\
+     "ceph_dev_branch":"{env:CEPH_DEV_BRANCH:master}",\
+     "ceph_dev_sha1":"{env:CEPH_DEV_SHA1:latest}",\
+     "ceph_stable_release":"{env:CEPH_STABLE_RELEASE:jewel}",\
+     "ceph_stable":{env:CEPH_STABLE:true}\
+    \}'
   # test that the cluster can be redeployed in a healthy state
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -40,17 +41,17 @@ commands=
 [update]
 commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml --extra-vars="\
-      ceph_stable_release={env:UPDATE_CEPH_STABLE_RELEASE:kraken} \
-      ireallymeanit=yes \
-      fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:CEPH_RHCS:false} \
-      ceph_origin={env:CEPH_ORIGIN:upstream} \
-      ceph_dev={env:CEPH_DEV:false} \
-      ceph_dev_branch={env:UPDATE_CEPH_DEV_BRANCH:master} \
-      ceph_dev_sha1={env:UPDATE_CEPH_DEV_SHA1:latest} \
-      ceph_stable={env:CEPH_STABLE:true} \
-  "
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml -e ireallymeanit=yes --extra-vars '\
+    \{\
+      "fetch_directory":"{changedir}/fetch",\
+      "ceph_rhcs":{env:CEPH_RHCS:false},\
+      "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
+      "ceph_dev":{env:CEPH_DEV:false},\
+      "ceph_dev_branch":"{env:UPDATE_CEPH_DEV_BRANCH:master}",\
+      "ceph_dev_sha1":"{env:UPDATE_CEPH_DEV_SHA1:latest}",\
+      "ceph_stable_release":"{env:UPDATE_CEPH_STABLE_RELEASE:kraken}",\
+      "ceph_stable":{env:CEPH_STABLE:true}\
+    \}'
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
 
@@ -97,16 +98,18 @@ commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars="\
-      fetch_directory={changedir}/fetch \
-      ceph_rhcs={env:CEPH_RHCS:false} \
-      ceph_origin={env:CEPH_ORIGIN:upstream} \
-      ceph_dev={env:CEPH_DEV:false} \
-      ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
-      ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} \
-      ceph_stable_release={env:CEPH_STABLE_RELEASE:jewel} \
-      ceph_stable={env:CEPH_STABLE:true} \
-  "
+  ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars '\
+    \{\
+     "fetch_directory":"{changedir}/fetch",\
+     "ceph_rhcs":{env:CEPH_RHCS:false},\
+     "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
+     "ceph_dev":{env:CEPH_DEV:false},\
+     "ceph_dev_branch":"{env:CEPH_DEV_BRANCH:master}",\
+     "ceph_dev_sha1":"{env:CEPH_DEV_SHA1:latest}",\
+     "ceph_stable_release":"{env:CEPH_STABLE_RELEASE:jewel}",\
+     "ceph_stable":{env:CEPH_STABLE:true}\
+    \}'
+
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/setup.yml
 
   testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests

From ec5ce818840f277d9f2fbe595e998f82e6517196 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Thu, 16 Mar 2017 16:31:25 -0500
Subject: [PATCH 62/80] ceph-mon: always call ceph-create-keys

After the jewel release the mon startup does not generate keys, but it's
still harmless to call ceph-create-keys with jewel because this task has
a 'creates' argument that will cause it not to run if the keys already
exist.

Removing this when condition also allows the downstream CI tests to
install kraken or luminous without resetting ceph_stable_release, which does not
pertain to rhcs.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 roles/ceph-mon/tasks/ceph_keys.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/ceph-mon/tasks/ceph_keys.yml b/roles/ceph-mon/tasks/ceph_keys.yml
index 5bd61e51d..67a6823e3 100644
--- a/roles/ceph-mon/tasks/ceph_keys.yml
+++ b/roles/ceph-mon/tasks/ceph_keys.yml
@@ -1,5 +1,5 @@
 ---
-- name: collect admin and bootstrap keys (for or after kraken release)
+- name: collect admin and bootstrap keys
   command: ceph-create-keys --cluster {{ cluster }} -i {{ monitor_name }}
   args:
     creates: /etc/ceph/{{ cluster }}.client.admin.keyring
@@ -8,7 +8,7 @@
   always_run: true
   when:
     - cephx
-    - ceph_release_num.{{ ceph_release }} > ceph_release_num.jewel
+
 # NOTE (leseb): wait for mon discovery and quorum resolution
 # the admin key is not instantaneously created so we have to wait a bit
 - name: "wait for {{ cluster }}.client.admin.keyring exists"

From 2ed94d914c3912031ca65e98cb5c6ae8ce0c5658 Mon Sep 17 00:00:00 2001
From: Daniel Marks <daniel.marks@codecentric.de>
Date: Thu, 16 Mar 2017 23:16:30 +0100
Subject: [PATCH 63/80] Use ansible uri module instead of shell module with
 curl

This fixes issue #1299. According to @ktdreyer s comment in the ticket,
he fixed the web server config so also older (non-SNI) python clients
can use the uri module here.
---
 .../tasks/installs/debian_ceph_repository.yml         | 11 +++++------
 .../tasks/installs/redhat_ceph_repository.yml         |  9 ++++-----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/roles/ceph-common/tasks/installs/debian_ceph_repository.yml b/roles/ceph-common/tasks/installs/debian_ceph_repository.yml
index 00bedaafb..45991f153 100644
--- a/roles/ceph-common/tasks/installs/debian_ceph_repository.yml
+++ b/roles/ceph-common/tasks/installs/debian_ceph_repository.yml
@@ -18,17 +18,16 @@
   changed_when: false
   when: ceph_stable
 
-# we must use curl instead of ansible's uri module because SNI support in
-# Python is only available in 2.7.9 and later, and most supported distributions
-# don't have that version, so a request to https fails.
-- name : fetch ceph development repository sources list file
-  command: "curl -L https://shaman.ceph.com/api/repos/ceph/{{ ceph_dev_branch }}/{{ ceph_dev_sha1 }}/{{ ansible_distribution | lower }}/{{ ansible_lsb.codename }}/repo"
+- name: fetch ceph development repository sources list file
+  uri:
+    url: https://shaman.ceph.com/api/repos/ceph/{{ ceph_dev_branch }}/{{ ceph_dev_sha1 }}/{{ ansible_distribution | lower }}/{{ ansible_lsb.codename }}/repo
+    return_content: yes
   register: ceph_dev_deb_repo
   when: ceph_dev
 
 - name: add ceph development repository
   apt_repository:
-    repo: "{{ ceph_dev_deb_repo.stdout }}"
+    repo: "{{ ceph_dev_deb_repo.content }}"
     state: present
   changed_when: false
   when: ceph_dev
diff --git a/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml b/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
index b0f285b3e..4d8ea056e 100644
--- a/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
+++ b/roles/ceph-common/tasks/installs/redhat_ceph_repository.yml
@@ -15,17 +15,16 @@
     baseurl: "{{ ceph_mirror }}/rpm-{{ ceph_stable_release }}/{{ ceph_stable_redhat_distro }}/$basearch"
   when: ceph_stable
 
-# we must use curl instead of ansible's uri module because SNI support in
-# Python is only available in 2.7.9 and later, and most supported distributions
-# don't have that version, so a request to https fails.
 - name: fetch ceph development repo file
-  command: 'curl -L https://shaman.ceph.com/api/repos/ceph/{{ ceph_dev_branch }}/{{ ceph_dev_sha1 }}/{{ ansible_distribution | lower }}/{{ ansible_distribution_major_version }}/repo'
+  uri:
+    url: https://shaman.ceph.com/api/repos/ceph/{{ ceph_dev_branch }}/{{ ceph_dev_sha1 }}/{{ ansible_distribution | lower }}/{{ ansible_distribution_major_version }}/repo
+    return_content: yes
   register: ceph_dev_yum_repo
   when: ceph_dev
 
 - name: add ceph development repository
   copy:
-    content: "{{ ceph_dev_yum_repo.stdout }}"
+    content: "{{ ceph_dev_yum_repo.content }}"
     dest: /etc/yum.repos.d/ceph-dev.repo
     owner: root
     group: root

From 09646041eee7cbe973dc6839856a6cc44c693b81 Mon Sep 17 00:00:00 2001
From: Christian Zunker <christian.zunker@codecentric.de>
Date: Thu, 16 Mar 2017 14:25:15 +0100
Subject: [PATCH 64/80] Fix osd_crush_location to prevent systemd error message

With ' in osd_crush_location, systemd will show this error:
ceph-osd-prestart.sh[2931]: Invalid command:  invalid chars ' in 'root=

Signed-off-by: Christian Zunker <christian.zunker@codecentric.de>
---
 group_vars/osds.yml.sample       | 4 ++--
 roles/ceph-osd/defaults/main.yml | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample
index 2fbae8006..911103e1e 100644
--- a/group_vars/osds.yml.sample
+++ b/group_vars/osds.yml.sample
@@ -59,7 +59,7 @@ dummy:
 # osd0 ceph_crush_root=foo ceph_crush_rack=bar
 
 #crush_location: false
-#osd_crush_location: "'root={{ ceph_crush_root }} rack={{ ceph_crush_rack }} host={{ ansible_hostname }}'"
+#osd_crush_location: "\"root={{ ceph_crush_root }} rack={{ ceph_crush_rack }} host={{ ansible_hostname }}\""
 
 
 ##############
@@ -119,7 +119,7 @@ dummy:
 
 # II.  Second scenario:  N journal devices for N OSDs
 # Use 'true' for 'raw_multi_journal' to enable this scenario
-# List devices under 'devices' variable above and 
+# List devices under 'devices' variable above and
 # write journal devices for those under 'raw_journal_devices'
 # In the following example:
 # * sdb and sdc will get sdf as a journal
diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml
index d32625113..6c55d9696 100644
--- a/roles/ceph-osd/defaults/main.yml
+++ b/roles/ceph-osd/defaults/main.yml
@@ -51,7 +51,7 @@ copy_admin_key: false
 # osd0 ceph_crush_root=foo ceph_crush_rack=bar
 
 crush_location: false
-osd_crush_location: "'root={{ ceph_crush_root }} rack={{ ceph_crush_rack }} host={{ ansible_hostname }}'"
+osd_crush_location: "\"root={{ ceph_crush_root }} rack={{ ceph_crush_rack }} host={{ ansible_hostname }}\""
 
 
 ##############
@@ -111,7 +111,7 @@ journal_collocation: false
 
 # II.  Second scenario:  N journal devices for N OSDs
 # Use 'true' for 'raw_multi_journal' to enable this scenario
-# List devices under 'devices' variable above and 
+# List devices under 'devices' variable above and
 # write journal devices for those under 'raw_journal_devices'
 # In the following example:
 # * sdb and sdc will get sdf as a journal

From aa90fe64176bff0e3d637336fc35a7601e3cee72 Mon Sep 17 00:00:00 2001
From: yuliyang <yuliyang@cmss.chinamobile.com>
Date: Fri, 17 Mar 2017 17:40:45 +0800
Subject: [PATCH 65/80] the $? of pgrep ceph-rest-api command return is 1 when
 ceph-rest-api already run.  use ps instead

---
 roles/ceph-restapi/tasks/start_restapi.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/ceph-restapi/tasks/start_restapi.yml b/roles/ceph-restapi/tasks/start_restapi.yml
index 357c5d7a2..68fd25bd8 100644
--- a/roles/ceph-restapi/tasks/start_restapi.yml
+++ b/roles/ceph-restapi/tasks/start_restapi.yml
@@ -1,6 +1,6 @@
 ---
 - name: check if ceph rest api is already started
-  shell: "pgrep ceph-rest-api"
+  shell: "pgrep -f ceph-rest-api"
   changed_when: false
   failed_when: false
   always_run: true

From 6c4911276ea0849b520cb8b15bbed079ca03c0d3 Mon Sep 17 00:00:00 2001
From: Anthony D'Atri <anthony.datri@gmail.com>
Date: Sun, 19 Mar 2017 00:23:26 -0700
Subject: [PATCH 66/80] Enhance clean PG check to catch active+clean+scrubbing
 and active+clean+scrubbing+deep

Signed-off-by: Anthony D'Atri <anthony.datri@gmail.com>
---
 infrastructure-playbooks/rolling_update.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/infrastructure-playbooks/rolling_update.yml b/infrastructure-playbooks/rolling_update.yml
index 24d1253ef..b351b66d3 100644
--- a/infrastructure-playbooks/rolling_update.yml
+++ b/infrastructure-playbooks/rolling_update.yml
@@ -266,7 +266,7 @@
 
     - name: waiting for clean pgs...
       shell: |
-        test "$(ceph pg stat --cluster {{ cluster }} | sed 's/^.*pgs://;s/active+clean.*//;s/ //')" -eq "$(ceph pg stat --cluster {{ cluster }}  | sed 's/pgs.*//;s/^.*://;s/ //')" && ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
+        test "$(ceph pg stat --cluster {{ cluster }} | tr , '\n'  | sed 's/^.*pgs: //' | awk '/active.clean/ { SUM += $1 } END {print SUM}')" -eq "$(ceph pg stat --cluster {{ cluster }}  | sed 's/pgs.*//;s/^.*://;s/ //')" && ceph health --cluster {{ cluster }}  | egrep -sq "HEALTH_OK|HEALTH_WARN"
       register: result
       until: result.rc == 0
       retries: "{{ health_osd_check_retries }}"

From 2967023ea92e4d43ca86cd7dc24700e7f73e06fd Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Mon, 20 Mar 2017 16:22:07 -0500
Subject: [PATCH 67/80] tests: allow setting fetch_directory with an
 environment variable

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 4d9186a50..40cf2db06 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,7 @@ commands=
   cp {toxinidir}/infrastructure-playbooks/purge-cluster.yml {toxinidir}/purge-cluster.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/purge-cluster.yml -e ireallymeanit=yes --extra-vars '\
     \{\
-     "fetch_directory":"{changedir}/fetch",\
+     "fetch_directory":"{env:FETCH_DIRECTORY:{changedir}/fetch}",\
      "ceph_rhcs":{env:CEPH_RHCS:false},\
      "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
      "ceph_dev":{env:CEPH_DEV:false},\
@@ -23,7 +23,7 @@ commands=
   # set up the cluster again
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/site.yml.sample --extra-vars '\
     \{\
-     "fetch_directory":"{changedir}/fetch",\
+     "fetch_directory":"{env:FETCH_DIRECTORY:{changedir}/fetch}",\
      "ceph_rhcs":{env:CEPH_RHCS:false},\
      "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
      "ceph_dev":{env:CEPH_DEV:false},\
@@ -43,7 +43,7 @@ commands=
   cp {toxinidir}/infrastructure-playbooks/rolling_update.yml {toxinidir}/rolling_update.yml
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/rolling_update.yml -e ireallymeanit=yes --extra-vars '\
     \{\
-      "fetch_directory":"{changedir}/fetch",\
+      "fetch_directory":"{env:FETCH_DIRECTORY:{changedir}/fetch}",\
       "ceph_rhcs":{env:CEPH_RHCS:false},\
       "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
       "ceph_dev":{env:CEPH_DEV:false},\
@@ -100,7 +100,7 @@ commands=
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars '\
     \{\
-     "fetch_directory":"{changedir}/fetch",\
+     "fetch_directory":"{env:FETCH_DIRECTORY:{changedir}/fetch}",\
      "ceph_rhcs":{env:CEPH_RHCS:false},\
      "ceph_origin":"{env:CEPH_ORIGIN:upstream}",\
      "ceph_dev":{env:CEPH_DEV:false},\

From 589d6812cae6271d72eb21781e107a8f999f0623 Mon Sep 17 00:00:00 2001
From: Guillaume Abrioux <gabrioux@redhat.com>
Date: Fri, 3 Mar 2017 14:58:56 +0100
Subject: [PATCH 68/80] ceph-docker: fix bootstrap directories permissions

Make bootstrap directories permissions work for both RedHat and Debian
os families.

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
Fix: #1338
---
 group_vars/docker-commons.yml.sample                    | 7 ++++++-
 roles/ceph-docker-common/defaults/main.yml              | 7 ++++++-
 roles/ceph-docker-common/tasks/main.yml                 | 1 -
 roles/ceph-mds/tasks/docker/dirs_permissions.yml        | 4 ++--
 roles/ceph-mon/tasks/docker/dirs_permissions.yml        | 4 ++--
 roles/ceph-nfs/tasks/docker/dirs_permissions.yml        | 4 ++--
 roles/ceph-osd/tasks/docker/dirs_permissions.yml        | 4 ++--
 roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml | 4 ++--
 roles/ceph-restapi/tasks/docker/dirs_permissions.yml    | 4 ++--
 roles/ceph-rgw/tasks/docker/dirs_permissions.yml        | 4 ++--
 10 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/group_vars/docker-commons.yml.sample b/group_vars/docker-commons.yml.sample
index 8515116ee..56eefb621 100644
--- a/group_vars/docker-commons.yml.sample
+++ b/group_vars/docker-commons.yml.sample
@@ -7,7 +7,12 @@
 # file as a good configuration file when no variable in it.
 dummy:
 
-
 #ceph_docker_registry: docker.io
 #ceph_docker_enable_centos_extra_repo: false
 
+# Set uid/gid to default '64045' for bootstrap directories.
+# '64045' is used for debian based distros. It must be set to 167 in case of rhel based distros.
+# These values have to be set according to the base OS used by the container image, NOT the host.
+#bootstrap_dirs_owner: "64045"
+#bootstrap_dirs_group: "64045"
+
diff --git a/roles/ceph-docker-common/defaults/main.yml b/roles/ceph-docker-common/defaults/main.yml
index 073aec00c..395acc406 100644
--- a/roles/ceph-docker-common/defaults/main.yml
+++ b/roles/ceph-docker-common/defaults/main.yml
@@ -1,4 +1,9 @@
 ---
-
 ceph_docker_registry: docker.io
 ceph_docker_enable_centos_extra_repo: false
+
+# Set uid/gid to default '64045' for bootstrap directories.
+# '64045' is used for debian based distros. It must be set to 167 in case of rhel based distros.
+# These values have to be set according to the base OS used by the container image, NOT the host.
+bootstrap_dirs_owner: "64045"
+bootstrap_dirs_group: "64045"
diff --git a/roles/ceph-docker-common/tasks/main.yml b/roles/ceph-docker-common/tasks/main.yml
index a01824b16..357573a8b 100644
--- a/roles/ceph-docker-common/tasks/main.yml
+++ b/roles/ceph-docker-common/tasks/main.yml
@@ -1,7 +1,6 @@
 ---
 - include: system_checks.yml
 
-
 - name: check if it is atomic host
   stat: path=/run/ostree-booted
   register: stat_ostree
diff --git a/roles/ceph-mds/tasks/docker/dirs_permissions.yml b/roles/ceph-mds/tasks/docker/dirs_permissions.yml
index 7ab4d2cc8..40f159410 100644
--- a/roles/ceph-mds/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-mds/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-mon/tasks/docker/dirs_permissions.yml b/roles/ceph-mon/tasks/docker/dirs_permissions.yml
index 5f00ecbf4..65997fd08 100644
--- a/roles/ceph-mon/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-mon/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-nfs/tasks/docker/dirs_permissions.yml b/roles/ceph-nfs/tasks/docker/dirs_permissions.yml
index 9f07c3a71..0fc40a1ce 100644
--- a/roles/ceph-nfs/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-nfs/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-osd/tasks/docker/dirs_permissions.yml b/roles/ceph-osd/tasks/docker/dirs_permissions.yml
index 615a36c9c..82b6d6b2b 100644
--- a/roles/ceph-osd/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-osd/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml b/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml
index 4e6510b10..0b446c639 100644
--- a/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-restapi/tasks/docker/dirs_permissions.yml b/roles/ceph-restapi/tasks/docker/dirs_permissions.yml
index 5f00ecbf4..65997fd08 100644
--- a/roles/ceph-restapi/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-restapi/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/
diff --git a/roles/ceph-rgw/tasks/docker/dirs_permissions.yml b/roles/ceph-rgw/tasks/docker/dirs_permissions.yml
index 3d115557a..07f36cec6 100644
--- a/roles/ceph-rgw/tasks/docker/dirs_permissions.yml
+++ b/roles/ceph-rgw/tasks/docker/dirs_permissions.yml
@@ -3,8 +3,8 @@
   file:
     path: "{{ item }}"
     state: directory
-    owner: "64045"
-    group: "64045"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
     mode: "0755"
   with_items:
     - /etc/ceph/

From 274c76f897a13d3b336faac679a59f7ff008b4e0 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 08:49:49 -0500
Subject: [PATCH 69/80] tests: adds a 'rhcs-' prefix to the testing scenarios
 matrix

This allows for us to have a copy of the existing testing scenarios with
a 'rhcs-' prefix. We can use that in the tox.ini to take actions we need
to properly test Red Hat Ceph Storage.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 40cf2db06..f603de8e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = {ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster}
+envlist = {,rhcs-}{ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster}
 skipsdist = True
 
 # extra commands for purging clusters

From 1fce790128fb6e1a489b3b74e81e8bc73c8f4433 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 09:16:06 -0500
Subject: [PATCH 70/80] test: docker dmcrypt tests should use the
 site.docker.yml playbook

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index f603de8e3..f90f71b6a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -69,7 +69,7 @@ setenv=
   ANSIBLE_STDOUT_CALLBACK = debug
   docker_cluster: PLAYBOOK = site-docker.yml.sample
   docker_dedicated_journal: PLAYBOOK = site-docker.yml.sample
-  docker_dmcrypt_journal_collocation = site-docker.yml.sample
+  docker_dmcrypt_journal_collocation: PLAYBOOK = site-docker.yml.sample
 deps=
   ansible1.9: ansible==1.9.4
   ansible2.1: ansible==2.1

From d261da66790ff253f5c686bb4064c87d914a088e Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 10:13:51 -0500
Subject: [PATCH 71/80] tests: adds a rhcs_setup.yml playbook

This is used to configure the test nodes for testing Ret Had Ceph
Storage.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 70 +++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 tests/functional/rhcs_setup.yml

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
new file mode 100644
index 000000000..4752749f5
--- /dev/null
+++ b/tests/functional/rhcs_setup.yml
@@ -0,0 +1,70 @@
+---
+- hosts: local
+  gather_facts: false
+  become: yes
+  tags:
+    - vagrant_setup
+  tasks:
+
+    - name: change centos/7 vagrant box name to rhel7
+      replace:
+        regexp: "centos/7"
+        replace: "rhel7"
+        path: "{{ change_dir }}/vagrant_variables.yml"
+      when: change_dir is defined
+
+- hosts: all
+  gather_facts: true
+  become: yes
+  tasks:
+
+    - name: check if it is Atomic host
+      stat: path=/run/ostree-booted
+      register: stat_ostree
+      always_run: true
+
+    - name: set fact for using Atomic host
+      set_fact:
+        is_atomic: '{{ stat_ostree.stat.exists }}'
+
+- hosts: mons
+  gather_facts: false
+  become: yes
+  tasks:
+
+    - name: install ceph mon repo
+      yum_repository:
+        name: ceph-mon
+        description: repo for rhcs ceph-mon
+        baseurl: "{{ repo_url }}/MON/x86_64/os/"
+        gpgcheck: no
+      when:
+        - not is_atomic
+
+- hosts: osds
+  gather_facts: false
+  become: yes
+  tasks:
+
+    - name: install ceph osd repo
+      yum_repository:
+        name: ceph-osd
+        description: repo for rhcs ceph-osd
+        baseurl: "{{ repo_url }}/OSD/x86_64/os/"
+        gpgcheck: no
+      when:
+        - not is_atomic
+
+- hosts: mdss:rgws:clients 
+  gather_facts: false
+  become: yes
+  tasks:
+
+    - name: install ceph tools repo
+      yum_repository:
+        name: ceph-osd
+        description: repo for rhcs ceph tools
+        baseurl: "{{ repo_url }}/TOOLS/x86_64/os/"
+        gpgcheck: no
+      when:
+        - not is_atomic

From 28f1f2475901e424157dbdee471f944727abf28b Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 10:14:41 -0500
Subject: [PATCH 72/80] tests: use the rhcs_setup.yml playbook in tox.ini

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tox.ini b/tox.ini
index f90f71b6a..be778e157 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,6 +70,7 @@ setenv=
   docker_cluster: PLAYBOOK = site-docker.yml.sample
   docker_dedicated_journal: PLAYBOOK = site-docker.yml.sample
   docker_dmcrypt_journal_collocation: PLAYBOOK = site-docker.yml.sample
+  rhcs: CEPH_RHCS = true
 deps=
   ansible1.9: ansible==1.9.4
   ansible2.1: ansible==2.1
@@ -95,9 +96,13 @@ changedir=
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
 commands=
+  rhcs: ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "change_dir={changedir}" --tags "vagrant_setup"
+
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
+  rhcs: ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "repo_url={env:REPO_URL:}" --skip-tags "vagrant_setup"
+
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars '\
     \{\
      "fetch_directory":"{env:FETCH_DIRECTORY:{changedir}/fetch}",\

From b0caeed682b96fa4cdab2d7f01573affcf136b04 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 12:38:12 -0500
Subject: [PATCH 73/80] tests: fix task in rhcs_setup that changes vagrant box
 to rhel7

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 4 ++--
 tox.ini                         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 4752749f5..700633781 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -1,5 +1,5 @@
 ---
-- hosts: local
+- hosts: all
   gather_facts: false
   become: yes
   tags:
@@ -10,7 +10,7 @@
       replace:
         regexp: "centos/7"
         replace: "rhel7"
-        path: "{{ change_dir }}/vagrant_variables.yml"
+        dest: "{{ change_dir }}/vagrant_variables.yml"
       when: change_dir is defined
 
 - hosts: all
diff --git a/tox.ini b/tox.ini
index be778e157..f9b2d5b57 100644
--- a/tox.ini
+++ b/tox.ini
@@ -96,7 +96,7 @@ changedir=
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
 commands=
-  rhcs: ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "change_dir={changedir}" --tags "vagrant_setup"
+  rhcs: ansible-playbook -vv -i "localhost," -c local {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "change_dir={changedir}" --tags "vagrant_setup"
 
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}

From 408cd614838f84036797406bade4b0da4db5e80c Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 15:42:17 -0500
Subject: [PATCH 74/80] tests: enable the downstream rhcs repos

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 700633781..3d8a73294 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -38,6 +38,7 @@
         description: repo for rhcs ceph-mon
         baseurl: "{{ repo_url }}/MON/x86_64/os/"
         gpgcheck: no
+        enabled: yes
       when:
         - not is_atomic
 
@@ -52,6 +53,7 @@
         description: repo for rhcs ceph-osd
         baseurl: "{{ repo_url }}/OSD/x86_64/os/"
         gpgcheck: no
+        enabled: yes
       when:
         - not is_atomic
 
@@ -66,5 +68,6 @@
         description: repo for rhcs ceph tools
         baseurl: "{{ repo_url }}/TOOLS/x86_64/os/"
         gpgcheck: no
+        enabled: yes
       when:
         - not is_atomic

From ead8dcebcfdbbe780b1348c879bdaba0d0342ed1 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 16:02:50 -0500
Subject: [PATCH 75/80] tests: fix ceph tools baseurl

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 3d8a73294..21bc63683 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -66,7 +66,7 @@
       yum_repository:
         name: ceph-osd
         description: repo for rhcs ceph tools
-        baseurl: "{{ repo_url }}/TOOLS/x86_64/os/"
+        baseurl: "{{ repo_url }}/Tools/x86_64/os/"
         gpgcheck: no
         enabled: yes
       when:

From 66f3f317023d0b61c6d37623943d5c3763924694 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 16:18:57 -0500
Subject: [PATCH 76/80] tests: adds a task to download a repo file for nightly
 rhel7 packages

This is a url to an actual repo file, not a baseurl to use in a repo.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 8 ++++++++
 tox.ini                         | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 21bc63683..93055b435 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -27,6 +27,14 @@
       set_fact:
         is_atomic: '{{ stat_ostree.stat.exists }}'
 
+    - name: install nightly rhel7 repo
+      get_url:
+        url: "{{ rhel7_repo_url }}"
+        dest: /etc/yum.repos.d
+        owner: root
+        group: root
+      when: not is_atomic
+
 - hosts: mons
   gather_facts: false
   become: yes
diff --git a/tox.ini b/tox.ini
index f9b2d5b57..4d595db56 100644
--- a/tox.ini
+++ b/tox.ini
@@ -101,7 +101,7 @@ commands=
   vagrant up --no-provision {posargs:--provider=virtualbox}
   bash {toxinidir}/tests/scripts/generate_ssh_config.sh {changedir}
 
-  rhcs: ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "repo_url={env:REPO_URL:}" --skip-tags "vagrant_setup"
+  rhcs: ansible-playbook -vv -i {changedir}/hosts {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "repo_url={env:REPO_URL:} rhel7_repo_url={env:RHEL7_REPO_URL:}" --skip-tags "vagrant_setup"
 
   ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars '\
     \{\

From a4f05e49262b891c0f95cb45463f9ae4bb33acb0 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 16:44:29 -0500
Subject: [PATCH 77/80] tests: set MTU to 1400 on test node interfaces

In the environment we were testing on, MTU was set to 1500 which causes
download failures of our yum repos. There might be a better way to set
this instead of doing it here in ansible.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 93055b435..4618a72a6 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -35,6 +35,12 @@
         group: root
       when: not is_atomic
 
+    - name: set MTU on eth0
+      command: "ifconfig eth0 mtu 1400 up"
+
+    - name: set MTU on eth1
+      command: "ifconfig eth1 mtu 1400 up"
+
 - hosts: mons
   gather_facts: false
   become: yes
@@ -65,6 +71,9 @@
       when:
         - not is_atomic
 
+    - name: set MTU on eth2
+      command: "ifconfig eth2 mtu 1400 up"
+
 - hosts: mdss:rgws:clients 
   gather_facts: false
   become: yes

From 23ab14c105f6ba741abde83e464100ccda68260a Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Wed, 22 Mar 2017 19:40:34 -0500
Subject: [PATCH 78/80] tests: change hosts in first play in rhcs_setup to
 localhost

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 4618a72a6..658f51481 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -1,5 +1,5 @@
 ---
-- hosts: all
+- hosts: localhost
   gather_facts: false
   become: yes
   tags:

From 5c34c9ccf121cccd01bc4bfb59f79b6529f028a5 Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Thu, 23 Mar 2017 06:42:43 -0500
Subject: [PATCH 79/80] tests: when testing rhcs default CEPH_STABLE=false

This will keep ceph-ansible from installing the upstream repo file on
the testing nodes.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 4d595db56..e19a28bb5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -71,6 +71,7 @@ setenv=
   docker_dedicated_journal: PLAYBOOK = site-docker.yml.sample
   docker_dmcrypt_journal_collocation: PLAYBOOK = site-docker.yml.sample
   rhcs: CEPH_RHCS = true
+  rhcs: CEPH_STABLE = false
 deps=
   ansible1.9: ansible==1.9.4
   ansible2.1: ansible==2.1

From f2aaaa49702c9ecd1dbef38dba0e67e8594a061d Mon Sep 17 00:00:00 2001
From: Andrew Schoen <aschoen@redhat.com>
Date: Thu, 23 Mar 2017 08:43:11 -0500
Subject: [PATCH 80/80] tests: change ceph/ubuntu-xenial boxes to rhel7

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---
 tests/functional/rhcs_setup.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/functional/rhcs_setup.yml b/tests/functional/rhcs_setup.yml
index 658f51481..f066fbd6a 100644
--- a/tests/functional/rhcs_setup.yml
+++ b/tests/functional/rhcs_setup.yml
@@ -13,6 +13,13 @@
         dest: "{{ change_dir }}/vagrant_variables.yml"
       when: change_dir is defined
 
+    - name: change ceph/ubuntu-xenial vagrant box name to rhel7
+      replace:
+        regexp: "ceph/ubuntu-xenial"
+        replace: "rhel7"
+        dest: "{{ change_dir }}/vagrant_variables.yml"
+      when: change_dir is defined
+
 - hosts: all
   gather_facts: true
   become: yes