docker: only use systemd to manage containers

Prior to this patch we had several ways to runs containers, we could use
ansible's docker module on some distro and on containers distros we were
using systemd. We strongly believe threating containers as services with
systemd is the right approach so this patch generalizes to all the
distros. These days most of the distros are running systemd so it's fair
assumption.

Signed-off-by: Sébastien Han <seb@redhat.com>

group_vars/mdss.yml.sample

@@ -35,7 +35,7 @@ dummy:
 #ceph_mds_docker_username: ceph
 #ceph_mds_docker_imagename: daemon
 #ceph_mds_docker_image_tag: latest
-#ceph_mds_docker_extra_env: "CLUSTER={{ cluster }},MDS_NAME={{ ansible_hostname }}" # comma separated variables
+#ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }}
 #ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME
 

group_vars/mons.yml.sample

@@ -85,7 +85,7 @@ dummy:
 #ceph_mon_docker_username: ceph
 #ceph_mon_docker_imagename: daemon
 #ceph_mon_docker_image_tag: latest
-#ceph_mon_extra_envs: "CLUSTER={{ cluster }},FSID={{ fsid }},MON_NAME={{ ansible_hostname }}" # comma separated variables
+#ceph_mon_docker_extra_env: -e CLUSTER={{ cluster }} -e FSID={{ fsid }} -e MON_NAME={{ ansible_hostname }}
 #ceph_docker_on_openstack: false
 #mon_docker_privileged: false
 #mon_docker_net_host: true

group_vars/nfss.yml.sample

@@ -88,7 +88,7 @@ dummy:
 #ceph_nfs_docker_username: ceph
 #ceph_nfs_docker_imagename: ganesha
 #ceph_nfs_docker_image_tag: latest
-#ceph_nfs_docker_extra_env: "GANESHA_EPOCH={{ ganesha_epoch }}" # comma separated variables
+#ceph_nfs_docker_extra_env: -e GANESHA_EPOCH={{ ganesha_epoch }}
 #ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME
 

group_vars/osds.yml.sample

@@ -179,7 +179,7 @@ dummy:
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon
 #ceph_osd_docker_image_tag: latest
-#ceph_osd_docker_extra_env: "CLUSTER={{ cluster }},CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE,OSD_JOURNAL_SIZE={{ journal_size }}" # comma separated variables
+#ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
 #ceph_osd_docker_devices: "{{ devices }}"
 #ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME

group_vars/rgws.yml.sample

@@ -42,7 +42,7 @@ dummy:
 #ceph_rgw_docker_username: ceph
 #ceph_rgw_docker_imagename: daemon
 #ceph_rgw_docker_image_tag: latest
-#ceph_rgw_docker_extra_env: "CLUSTER={{ cluster }},RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+#ceph_rgw_docker_extra_env: -e CLUSTER={{ cluster }} -e RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}
 #ceph_docker_on_openstack: false
 #ceph_config_keys: [] # DON'T TOUCH ME
 #rgw_config_keys: "/" # DON'T TOUCH ME

roles/ceph-mds/defaults/main.yml

@@ -27,6 +27,6 @@ kv_endpoint: 127.0.0.1
 ceph_mds_docker_username: ceph
 ceph_mds_docker_imagename: daemon
 ceph_mds_docker_image_tag: latest
-ceph_mds_docker_extra_env: "CLUSTER={{ cluster }},MDS_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_mds_docker_extra_env: -e CLUSTER={{ cluster }} -e MDS_NAME={{ ansible_hostname }}
 ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME

roles/ceph-mds/tasks/docker/start_docker_mds.yml

@@ -35,14 +35,3 @@
     state: started
     enabled: yes
   changed_when: false
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
-- name: run the ceph metadata docker image
-  docker:
-    image: "{{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}:{{ ceph_mds_docker_image_tag }}"
-    name: ceph-{{ ansible_hostname }}-mds
-    net: host
-    state: running
-    env: "CEPH_DAEMON=MDS,CEPHFS_CREATE=1,{{ ceph_mds_docker_extra_env }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
-  when: ansible_os_family != 'RedHat' and ansible_os_family != 'CoreOS'

roles/ceph-mds/templates/ceph-mds.service.j2

@@ -18,7 +18,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    --privileged \
    -e CEPH_DAEMON=MDS \
    -e CEPHFS_CREATE=1 \
-   -e {{ ceph_mds_docker_extra_env }} \
+   {{ ceph_mds_docker_extra_env }} \
    --name={{ ansible_hostname }} \
    {{ ceph_mds_docker_username }}/{{ ceph_mds_docker_imagename }}:{{ ceph_mds_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}

roles/ceph-mon/defaults/main.yml

@@ -77,7 +77,7 @@ ceph_mon_docker_subnet: "{{ public_network }}"# subnet of the ceph_mon_docker_in
 ceph_mon_docker_username: ceph
 ceph_mon_docker_imagename: daemon
 ceph_mon_docker_image_tag: latest
-ceph_mon_extra_envs: "CLUSTER={{ cluster }},FSID={{ fsid }},MON_NAME={{ ansible_hostname }}" # comma separated variables
+ceph_mon_docker_extra_env: -e CLUSTER={{ cluster }} -e FSID={{ fsid }} -e MON_NAME={{ ansible_hostname }}
 ceph_docker_on_openstack: false
 mon_docker_privileged: false
 mon_docker_net_host: true

roles/ceph-mon/tasks/docker/start_docker_monitor.yml

@@ -51,7 +51,6 @@
     mode: "0644"
   when:  ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
 
-
 - name: enable systemd unit file for mon instance
   shell: systemctl enable ceph-mon@{{ ansible_hostname }}.service
   failed_when: false
@@ -70,32 +69,3 @@
     state: started
     enabled: yes
   changed_when: false
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
-- name: run the ceph monitor docker image
-  docker:
-    image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}:{{ ceph_mon_docker_image_tag }}"
-    name: "{{ ansible_hostname }}"
-    net: "host"
-    state: "running"
-    privileged: "{{ mon_docker_privileged }}"
-    env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},CEPH_FSID={{ fsid }},{{ ceph_mon_extra_envs }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
-  when:
-    - ansible_os_family != 'RedHat'
-    - ansible_os_family != 'CoreOS'
-    - not mon_containerized_deployment_with_kv
-
-- name: run the ceph monitor docker image with kv
-  docker:
-    image: "{{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}:{{ ceph_mon_docker_image_tag }}"
-    name: "{{ ansible_hostname }}"
-    net: "host"
-    state: "running"
-    privileged: "{{ mon_docker_privileged }}"
-    env: "KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }},{{ ceph_mon_extra_envs }}"
-    volumes: "/etc/localtime:/etc/localtime:ro"
-  when:
-    - ansible_os_family != 'RedHat'
-    - ansible_os_family != 'CoreOS'
-    - mon_containerized_deployment_with_kv

roles/ceph-mon/templates/ceph-mon.service.j2

@@ -25,7 +25,7 @@ ExecStart=/usr/bin/docker run --rm --name %i --net=host \
    -e CEPH_DAEMON=MON \
    -e MON_IP={{ hostvars[inventory_hostname]['ansible_' + ceph_mon_docker_interface]['ipv4']['address'] }} \
    -e CEPH_PUBLIC_NETWORK={{ ceph_mon_docker_subnet }} \
-   -e MON_NAME={{ ansible_hostname }} \
+   {{ ceph_mon_docker_extra_env }} \
     {{ ceph_mon_docker_username }}/{{ ceph_mon_docker_imagename }}:{{ ceph_mon_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop %i
 Restart=always

roles/ceph-nfs/defaults/main.yml

@@ -80,6 +80,6 @@ kv_endpoint: 127.0.0.1
 ceph_nfs_docker_username: ceph
 ceph_nfs_docker_imagename: ganesha
 ceph_nfs_docker_image_tag: latest
-#ceph_nfs_docker_extra_env: "GANESHA_EPOCH={{ ganesha_epoch }}" # comma separated variables
+#ceph_nfs_docker_extra_env: -e GANESHA_EPOCH={{ ganesha_epoch }}
 ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME

roles/ceph-nfs/tasks/docker/start_docker_nfs.yml

@@ -43,40 +43,8 @@
     state: started
     enabled: yes
   changed_when: false
-  when:
-    is_atomic or
-    ansible_os_family == 'CoreOS'
 
 - name: wait for ceph.conf exists
   wait_for:
       path: "/etc/ceph/{{ cluster }}.conf"
   when: is_atomic
-
-- name: run the ceph NFS docker image
-  docker:
-    image: "{{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }}"
-    name: "{{ ansible_hostname }}"
-    net: "host"
-    state: "running"
-    privileged: true
-    ports: "{{ ceph_nfs_port }}:{{ ceph_nfs_port }},111:111"
-    env: "CEPH_DAEMON=NFS,CEPH_PUBLIC_NETWORK={{ ceph_nfs_docker_subnet }},{{ ceph_nfs_extra_envs }}"
-    volumes: "/etc/ceph:/etc/ceph,/etc/ganesha:/etc/ganesha,/etc/localtime:/etc/localtime:ro"
-  when:
-    not is_atomic and
-    ansible_os_family != 'CoreOS' and
-    not mon_containerized_deployment_with_kv
-
-- name: run the ceph NFS docker image with kv
-  docker:
-    image: "{{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }}"
-    name: "{{ ansible_hostname }}"
-    net: "host"
-    state: "running"
-    privileged: true
-    env: "CEPH_DAEMON=NFS,CEPH_PUBLIC_NETWORK={{ ceph_nfs_docker_subnet }},{{ ceph_nfs_extra_envs }}"
-    volumes: "/etc/ganesha:/etc/ganesha,/etc/localtime:/etc/localtime:ro"
-  when:
-    not is_atomic and
-    ansible_os_family != 'CoreOS' and
-    mon_containerized_deployment_with_kv

roles/ceph-nfs/templates/ceph-nfs.service.j2

@@ -18,6 +18,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    -v /etc/localtime:/etc/localtime:ro \
    --privileged \
    -e CEPH_DAEMON=NFS \
+   {{ ceph_nfs_docker_extra_env }} \
    --name=nfs-{{ ansible_hostname }} \
     {{ ceph_nfs_docker_username }}/{{ ceph_nfs_docker_imagename }}:{{ ceph_nfs_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop %i

roles/ceph-osd/defaults/main.yml

@@ -171,7 +171,7 @@ ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ jou
 ceph_osd_docker_username: ceph
 ceph_osd_docker_imagename: daemon
 ceph_osd_docker_image_tag: latest
-ceph_osd_docker_extra_env: "CLUSTER={{ cluster }},CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE,OSD_JOURNAL_SIZE={{ journal_size }}" # comma separated variables
+ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
 ceph_osd_docker_devices: "{{ devices }}"
 ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -57,6 +57,7 @@
     -e KV_TYPE={{kv_type}} \
     -e KV_IP={{kv_endpoint}} \
     -e KV_PORT={{kv_port}} \
+    {{ ceph_osd_docker_prepare_env }} \
     "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}" \
   with_together:
     - "{{ ceph_osd_docker_devices }}"
@@ -98,37 +99,3 @@
     enabled: yes
   changed_when: false
   with_items: "{{ ceph_osd_docker_devices }}"
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
-- name: run the ceph osd docker image
-  docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
-    name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
-    net: host
-    pid: host
-    state: started
-    privileged: yes
-    env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro,/dev:/dev,/run:/run"
-  with_items: "{{ ceph_osd_docker_devices }}"
-  when:
-    - ansible_os_family != 'RedHat'
-    - ansible_os_family != 'CoreOS'
-    - not osd_containerized_deployment_with_kv
-
-- name: run the ceph osd docker image with kv
-  docker:
-    image: "{{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}"
-    name: "{{ ansible_hostname }}-osd-{{ item | regex_replace('/', '') }}"
-    net: host
-    pid: host
-    state: running
-    privileged: yes
-    env: "KV_TYPE={{kv_type}},KV_IP={{kv_endpoint}},OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}"
-    volumes: "/etc/localtime:/etc/localtime:ro,/dev/:/dev/"
-  with_items: "{{ ceph_osd_docker_devices }}"
-  when:
-   - ansible_os_family != 'RedHat'
-   - ansible_os_family != 'CoreOS'
-   - osd_containerized_deployment_with_kv
-

roles/ceph-osd/templates/ceph-osd.service.j2

@@ -20,6 +20,7 @@ ExecStart=/usr/bin/docker run --rm --net=host --pid=host\
    --privileged \
    -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
    -e OSD_DEVICE=/dev/%i \
+   {{ ceph_osd_docker_extra_env }} \
    --name={{ ansible_hostname }}-osd-dev%i \
    {{ ceph_osd_docker_username }}/{{ ceph_osd_docker_imagename }}:{{ ceph_osd_docker_image_tag }}
 ExecStop=-/usr/bin/docker stop {{ ansible_hostname }}-osd-dev%i

roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml

@@ -28,13 +28,3 @@
     state: started
     enabled: yes
   changed_when: false
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
-- name: run the ceph rbd mirror docker image
-  docker:
-    image: "{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}:{{ ceph_rbd_mirror_docker_image_tag }}"
-    name: "{{ ansible_hostname }}"
-    net: host
-    state: running
-    volumes: "/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
-  when: ansible_os_family != 'RedHat' and ansible_os_family != 'CoreOS'

roles/ceph-rgw/defaults/main.yml

@@ -34,7 +34,7 @@ ceph_rgw_civetweb_port: "{{ radosgw_civetweb_port }}"
 ceph_rgw_docker_username: ceph
 ceph_rgw_docker_imagename: daemon
 ceph_rgw_docker_image_tag: latest
-ceph_rgw_docker_extra_env: "CLUSTER={{ cluster }},RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}" # comma separated variables
+ceph_rgw_docker_extra_env: -e CLUSTER={{ cluster }} -e RGW_CIVETWEB_PORT={{ ceph_rgw_civetweb_port }}
 ceph_docker_on_openstack: false
 ceph_config_keys: [] # DON'T TOUCH ME
 rgw_config_keys: "/" # DON'T TOUCH ME

roles/ceph-rgw/tasks/docker/start_docker_rgw.yml

@@ -35,15 +35,3 @@
     state: started
     enabled: yes
   changed_when: false
-  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
-
-- name: run the rados gateway docker image
-  docker:
-    image: "{{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}:{{ ceph_rgw_docker_image_tag }}"
-    name: ceph-{{ ansible_hostname }}-rgw
-    expose: "{{ ceph_rgw_civetweb_port }}"
-    ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
-    state: running
-    env: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}"
-    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
-  when: ansible_os_family != 'RedHat' and ansible_os_family != 'CoreOS'

roles/ceph-rgw/templates/ceph-rgw.service.j2

@@ -17,6 +17,7 @@ ExecStart=/usr/bin/docker run --rm --net=host \
    -v /etc/localtime:/etc/localtime:ro \
    --privileged \
    -e CEPH_DAEMON=RGW \
+   {{ ceph_rgw_docker_extra_env }} \
    --name={{ ansible_hostname }} \
    {{ ceph_rgw_docker_username }}/{{ ceph_rgw_docker_imagename }}:{{ ceph_rgw_docker_image_tag }}
 ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}

tests/functional/centos/7/docker-cluster/group_vars/all

@@ -10,17 +10,14 @@ mds_containerized_deployment: True
 rgw_containerized_deployment: True
 cluster: test
 ceph_mon_docker_interface: eth1
-ceph_mon_docker_subnet: "192.168.15.0/24"
+ceph_mon_docker_subnet: "{{ public_network }}"
 journal_size: 100
-ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE,OSD_JOURNAL_SIZE={{ journal_size }}"
 ceph_docker_on_openstack: False
 public_network: "192.168.15.0/24"
 cluster_network: "192.168.16.0/24"
 journal_collocation: true
 ceph_rgw_civetweb_port: 8080
-ceph_osd_docker_devices:
-  - '/dev/sda'
-  - '/dev/sdb'
+ceph_osd_docker_devices: "{{ devices }}"
 devices:
-  - '/dev/sda'
-  - '/dev/sdb'
+  - /dev/sda
+  - /dev/sdb