Merge pull request #812 from rootfs/fix

some fixes for container deployment

group_vars/all.docker.sample

@@ -32,7 +32,7 @@ dummy:
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE" # comma separated variables
-#ceph_osd_docker_prepare_env: "CEPH_DAEMON=OSD_CEPH_DISK_PREPARE,OSD_FORCE_ZAP=1" # comma separated variables
+#ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1" # comma separated variables
 #ceph_osd_docker_devices:
 # - /dev/sdb
 # - /dev/sdc

group_vars/osds.sample

@@ -131,7 +131,7 @@ dummy:
 #osd_containerized_deployment_with_kv: false
 #kv_type: etcd
 #kv_endpoint: 127.0.0.1
-#ceph_osd_docker_prepare_env: ""
+#ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1"
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon
 #ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables

roles/ceph-common/templates/ceph.conf.j2

@@ -30,7 +30,19 @@ host = {{ hostvars[host]['ansible_fqdn'] }}
 host = {{ hostvars[host]['ansible_hostname'] }}
 {% endif %}
 # we need to check if monitor_interface is defined in the inventory per host or if it's set in a group_vars file
-{% if (hostvars[host]['monitor_interface'] is defined and hostvars[host]['monitor_interface'] != "interface") or monitor_interface != "interface" %}
+{% if mon_containerized_deployment %}
+{% set interface = ["ansible_",ceph_mon_docker_interface]|join %}
+{% if interface in hostvars[host] and 'ipv4' in hostvars[host][interface] %}
+# user address from interface {{ ceph_mon_docker_interface }}
+mon addr = {{ hostvars[host][interface]['ipv4']['address'] }}
+ {% elif hostvars[host]['monitor_address'] is defined %}
+ # use host monitor address
+mon addr = {{ hostvars[host]['monitor_address'] }}
+ {% elif monitor_address != "0.0.0.0" %}
+ # use group_var monitor address
+mon addr = monitor_address
+{% endif %}
+{% elif (hostvars[host]['monitor_interface'] is defined and hostvars[host]['monitor_interface'] != "interface") or monitor_interface != "interface" %}
 {% include 'mon_addr_interface.j2' %}
 {% else %}
 {% include 'mon_addr_address.j2' %}

roles/ceph-mds/tasks/docker/pre_requisite.yml

@@ -24,19 +24,60 @@
   tags:
     with_pkg
 
-- name: install pip and docker on redhat
+# install epel for pip
+- name: install epel on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - epel-release
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+- name: install pip on redhat
   yum:
     name: "{{ item }}"
     state: present
   with_items:
     - python-pip
-    - docker-engine
   when:
     - ansible_os_family == 'RedHat'
     - ansible_pkg_mgr == "yum"
   tags:
     with_pkg
 
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+  
+# docker package could be docker-enginer or docker  
 - name: install pip and docker on redhat
   dnf:
     name: "{{ item }}"

roles/ceph-mon/tasks/docker/pre_requisite.yml

@@ -24,32 +24,7 @@
   tags:
     with_pkg
 
-- name: install pip and docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
+# install epel for pip
 - name: install epel-release on redhat
   yum:
     name: epel-release
@@ -58,6 +33,60 @@
   tags:
     with_pkg
 
+- name: install pip on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+- name: install pip and docker on redhat (dnf)
+  dnf:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
+
+
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
 - name: install six

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -49,26 +49,26 @@
     owner: "root"
     group: "root"
     mode: "0644"
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when:  ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: link systemd unit file for mon instance
   file:
     src: /var/lib/ceph/ceph-mon@.service
     dest: /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
     state: link
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for mon instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-mon@{{ ansible_hostname }}.service
   failed_when: false
   changed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
   changed_when: false
   failed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: systemd start mon container
   service:
@@ -76,12 +76,12 @@
     state: started
     enabled: yes
   changed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: wait for ceph.conf exists
   wait_for:
       path: /etc/ceph/ceph.conf
-  when: is_atomic
+  when: ansible_os_family == 'RedHat'
 
 - name: run the ceph monitor docker image
   docker:
@@ -93,7 +93,7 @@
     env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},CEPH_FSID={{ fsid }},{{ ceph_mon_extra_envs }}"
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
   when:
-    - not is_atomic
+    - ansible_os_family != 'RedHat'
     - ansible_os_family != 'CoreOS'
     - not mon_containerized_deployment_with_kv
 
@@ -106,6 +106,6 @@
     privileged: "{{ mon_docker_privileged }}"
     env: "KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
   when:
-    - not is_atomic
+    - ansible_os_family != 'RedHat'
     - ansible_os_family != 'CoreOS'
     - mon_containerized_deployment_with_kv

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -24,7 +24,6 @@ ExecStart=/usr/bin/docker run --rm --name %i --net=host \
    -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \
    -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
    -e MON_NAME={{ ansible_hostname }} \
-   --name={{ ansible_hostname }} \
     {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}
 ExecStopPost=-/usr/bin/docker stop %i
 Restart=always

roles/ceph-osd/defaults/main.yml

@@ -123,7 +123,7 @@ osd_containerized_deployment: false
 osd_containerized_deployment_with_kv: false
 kv_type: etcd
 kv_endpoint: 127.0.0.1
-ceph_osd_docker_prepare_env: ""
+ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1"
 ceph_osd_docker_username: ceph
 ceph_osd_docker_imagename: daemon
 ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK" # comma separated variables

roles/ceph-osd/tasks/docker/pre_requisite.yml

@@ -24,18 +24,52 @@
   tags:
     with_pkg
 
-- name: install pip and docker on redhat
+- name: install epel-release on redhat
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+- name: install pip on redhat
   yum:
     name: "{{ item }}"
     state: present
   with_items:
     - python-pip
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
     - docker-engine
   when:
     - ansible_os_family == 'RedHat'
     - ansible_pkg_mgr == "yum"
   tags:
     with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
 
 - name: install pip and docker on redhat
   dnf:
@@ -44,19 +78,13 @@
   with_items:
     - python-pip
     - docker-engine
+    - docker
   when:
     - ansible_os_family == 'RedHat'
     - ansible_pkg_mgr == "dnf"
   tags:
     with_pkg
 
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
 
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -15,16 +15,23 @@
   with_items: ceph_osd_docker_devices
   register: osd_prepared
 
+# use shell rather than docker module
+# to ensure osd disk prepare finishes before
+# starting the next task
 - name: prepare ceph osd disk
-  docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}"
-    name: "{{ ansible_hostname }}-osd-prepare-{{ item.0 | regex_replace('/', '') }}"
-    net: host
-    pid: host
-    state: started
-    privileged: yes
-    env: "OSD_DEVICE={{ item.0 }},{{ ceph_osd_docker_prepare_env }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
+  shell: |   
+    docker run  --net=host \
+    --pid=host \
+    --privileged=true \
+    --name="{{ ansible_hostname }}-osd-prepare-{{ item.0 |
+    regex_replace('/', '') }}" \
+    -v /etc/ceph:/etc/ceph \
+    -v /var/lib/ceph/:/var/lib/ceph/ \
+    -v /dev:/dev \
+    -e "OSD_DEVICE={{ item.0 }}" \
+    -e "{{ ceph_osd_docker_prepare_env }}" \
+    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \
+    OSD_CEPH_DISK_PREPARE
   with_together:
     - ceph_osd_docker_devices
     - osd_prepared.results
@@ -32,8 +39,6 @@
     - item.1.get("rc", 0) != 0
     - ceph_osd_docker_prepare_env is defined
     - not osd_containerized_deployment_with_kv
-    - not is_atomic
-    - not ansible_os_family == 'CoreOS'
 
 - name: prepare ceph osd disk with kv_store
   docker:
@@ -53,54 +58,8 @@
     - item.1.get("rc", 0) != 0
     - ceph_osd_docker_prepare_env is defined
     - osd_containerized_deployment_with_kv
-    - not is_atomic
-    - not ansible_os_family == 'CoreOS'
 
-- name: prepare ceph osd disk for container operating systems
-  shell: |
-    docker run -d --net=host \
-    --pid=host \
-    --privileged=true \
-    -v /etc/ceph:/etc/ceph \
-    -v /var/lib/ceph/:/var/lib/ceph/ \
-    -v {{ item.0 }}:{{ item.0 }} \
-    -e OSD_DEVICE="{{ item.0 }}" \
-    -e "{{ ceph_osd_docker_prepare_env }}" \
-    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \
-    osd_ceph_disk_prepare
-  failed_when: false
-  changed_when: false
-  with_together:
-    - ceph_osd_docker_devices
-    - osd_prepared.results
-  when:
-    - item.1.get("rc", 0) != 0
-    - is_atomic or ansible_os_family == 'CoreOS'
-    - not osd_containerized_deployment_with_kv
-
-- name: prepare ceph osd disk for container operating systems with kv_store
-  shell: |
-    docker run -d --net=host \
-    --pid=host \
-    --privileged=true \
-    -e KV_TYPE={{ kv_type }} \
-    -e KV_IP={{ kv_endpoint }} \
-    -v {{ item.0 }}:{{ item.0 }} \
-    -e OSD_DEVICE={{ item.0 }} \
-    -e "{{ ceph_osd_docker_prepare_env }}" \
-    "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}" \
-    osd_ceph_disk_prepare
-  failed_when: false
-  changed_when: false
-  with_together:
-    - ceph_osd_docker_devices
-    - osd_prepared.results
-  when:
-    - item.1.get("rc", 0) != 0
-    - is_atomic or ansible_os_family == 'CoreOS'
-    - osd_containerized_deployment_with_kv
-
-# Use systemd to manage container on Atomic host
+# Use systemd to manage container on appropriate host
 - name: generate systemd unit file
   become: true
   template:
@@ -110,7 +69,7 @@
     group: "root"
     mode: "0644"
   failed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: link systemd unit file for osd instance
   file:
@@ -118,20 +77,20 @@
     dest: /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
     state: link
   with_items: ceph_osd_docker_devices
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: enable systemd unit file for osd instance
   shell: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-osd@{{ item | basename }}.service
   failed_when: false
   changed_when: false
   with_items: ceph_osd_docker_devices
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: reload systemd unit files
   shell: systemctl daemon-reload
   changed_when: false
   failed_when: false
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: systemd start osd container
   service:
@@ -140,7 +99,7 @@
     enabled: yes
   changed_when: false
   with_items: ceph_osd_docker_devices
-  when: is_atomic or ansible_os_family == 'CoreOS'
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
 - name: run the ceph osd docker image
   docker:
@@ -154,7 +113,7 @@
     volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/dev:/dev,/run:/run"
   with_items: ceph_osd_docker_devices
   when:
-    - not is_atomic
+    - ansible_os_family != 'RedHat'
     - ansible_os_family != 'CoreOS'
     - not osd_containerized_deployment_with_kv
 
@@ -170,6 +129,6 @@
     volumes: "/dev/:/dev/"
   with_items: ceph_osd_docker_devices
   when:
-   - not is_atomic
+   - ansible_os_family != 'RedHat'
    - ansible_os_family != 'CoreOS'
    - osd_containerized_deployment_with_kv

roles/ceph-restapi/tasks/docker/pre_requisite.yml

@@ -24,19 +24,52 @@
   tags:
     with_pkg
 
-- name: install pip and docker on redhat
+- name: install epel-release on redhat
+  yum:
+    name: epel-release
+    state: present
+  when: ansible_os_family == 'RedHat'
+  tags:
+    with_pkg
+
+- name: install pip on redhat
   yum:
     name: "{{ item }}"
     state: present
   with_items:
     - python-pip
-    - docker-engine
   when:
     - ansible_os_family == 'RedHat'
     - ansible_pkg_mgr == "yum"
   tags:
     with_pkg
 
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
 - name: install pip and docker on redhat
   dnf:
     name: "{{ item }}"
@@ -50,14 +83,6 @@
   tags:
     with_pkg
 
-- name: install epel-release on redhat
-  yum:
-    name: epel-release
-    state: present
-  when: ansible_os_family == 'RedHat'
-  tags:
-    with_pkg
-
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
 - name: install six

roles/ceph-rgw/tasks/docker/pre_requisite.yml

@@ -24,32 +24,6 @@
   tags:
     with_pkg
 
-- name: install pip and docker on redhat
-  yum:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "yum"
-  tags:
-    with_pkg
-
-- name: install pip and docker on redhat
-  dnf:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - python-pip
-    - docker-engine
-  when:
-    - ansible_os_family == 'RedHat'
-    - ansible_pkg_mgr == "dnf"
-  tags:
-    with_pkg
-
 - name: install epel-release on redhat
   yum:
     name: epel-release
@@ -57,6 +31,45 @@
   when: ansible_os_family == 'RedHat'
   tags:
     with_pkg
+    
+- name: install pip on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
 
 # NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
 # re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined