rbd-mirror: container deployment support

add the ability to deploy a containerized version of the rbd mirror
daemon.

Signed-off-by: Sébastien Han <seb@redhat.com>

Vagrantfile

@@ -13,6 +13,7 @@ NMDSS      = settings['mds_vms']
 NRGWS          = settings['rgw_vms']
 NNFSS          = settings['nfs_vms']
 RESTAPI        = settings['restapi']
+NRBD_MIRRORS   = settings['rbd_mirror_vms']
 CLIENTS        = settings['client_vms']
 SUBNET         = settings['subnet']
 BOX            = settings['vagrant_box']
@@ -52,6 +53,7 @@ ansible_provision = proc do |ansible|
     'mdss'            => (0..NMDSS - 1).map { |j| "#{OSPREFIX}mds#{j}" },
     'rgws'            => (0..NRGWS - 1).map { |j| "#{OSPREFIX}rgw#{j}" },
     'nfss'            => (0..NNFSS - 1).map { |j| "#{OSPREFIX}nfs#{j}" },
+    'rbd_mirrors'     => (0..NRBD_MIRRORS - 1).map { |j| "#{OSPREFIX}rbd_mirror#{j}" },
     'clients'         => (0..CLIENTS - 1).map { |j| "#{OSPREFIX}client#{j}" }
   }
 
@@ -69,6 +71,7 @@ ansible_provision = proc do |ansible|
       rgw_containerized_deployment: 'true',
       nfs_containerized_deployment: 'true',
       restapi_containerized_deployment: 'true',
+      rbd_mirror_containerized_deployment: 'true',
       ceph_mon_docker_interface: ETH,
       ceph_mon_docker_subnet: "#{SUBNET}.0/24",
       ceph_osd_docker_extra_env: "CEPH_DAEMON=OSD_CEPH_DISK,OSD_JOURNAL_SIZE=100",
@@ -249,6 +252,34 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
+  (0..NRBD_MIRRORS - 1).each do |i|
+    config.vm.define "#{OSPREFIX}rbd_mirror#{i}" do |rbd_mirror|
+      rbd_mirror.vm.hostname = "#{OSPREFIX}ceph-rbd-mirror#{i}"
+      if !OSVM
+        rbd_mirror.vm.network :private_network, ip: "#{SUBNET}.8#{i}"
+      end
+      # Virtualbox
+      rbd_mirror.vm.provider :virtualbox do |vb|
+        vb.customize ['modifyvm', :id, '--memory', "#{MEMORY}"]
+      end
+
+      # VMware
+      rbd_mirror.vm.provider :vmware_fusion do |v|
+        v.vmx['memsize'] = "#{MEMORY}"
+      end
+
+      # Libvirt
+      rbd_mirror.vm.provider :libvirt do |lv|
+        lv.memory = MEMORY
+      end
+      # Parallels
+      rbd_mirror.vm.provider "parallels" do |prl|
+        prl.name = "ceph-rbd-mirror#{i}"
+        prl.memory = "#{MEMORY}"
+      end
+    end
+  end
+
   (0..NMONS - 1).each do |i|
     config.vm.define "#{OSPREFIX}mon#{i}" do |mon|
       mon.vm.hostname = "#{OSPREFIX}ceph-mon#{i}"

group_vars/all.docker.sample

@@ -4,6 +4,7 @@ dummy:
 ##########
 # GLOBAL #
 ##########
+#docker: true
 #ceph_docker_dev_image: false
 
 #######
@@ -42,7 +43,7 @@ dummy:
 #######
 # MDS #
 #######
-#mds_containerized_deployment: false
+#mds_containerized_deployment: true
 #mds_containerized_deployment_with_kv: false
 #kv_type: etcd
 #kv_endpoint: 127.0.0.1
@@ -67,10 +68,24 @@ dummy:
 ###########
 # RESTAPI #
 ###########
-#restapi_containerized_deployment: false
+#restapi_containerized_deployment: true
 #ceph_restapi_docker_interface: eth0
 #ceph_restapi_port: 5000
 #ceph_restapi_docker_username: ceph
 #ceph_restapi_docker_imagename: daemon
 #ceph_restapi_docker_image_tag: latest
 #ceph_restapi_docker_extra_env: "RESTAPI_IP=0.0.0.0" # comma separated variables
+
+
+##############
+# RBD MIRROR #
+##############
+#rbd_mirror_containerized_deployment: true
+#rbd_mirror_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#ceph_rbd_mirror_docker_username: ceph
+#ceph_rbd_mirror_docker_imagename: daemon
+#ceph_rbd_mirror_docker_image_tag: latest
+#ceph_docker_on_openstack: false
+

group_vars/osds.sample

@@ -131,6 +131,7 @@ dummy:
 #osd_containerized_deployment_with_kv: false
 #kv_type: etcd
 #kv_endpoint: 127.0.0.1
+#kv_port: 4001
 #ceph_osd_docker_prepare_env: "OSD_FORCE_ZAP=1"
 #ceph_osd_docker_username: ceph
 #ceph_osd_docker_imagename: daemon

group_vars/rbd-mirrors.sample

@@ -28,6 +28,7 @@ dummy:
 # Thus if cluster names are identical we can not have them under /etc/ceph
 #ceph_rbd_mirror_remote_cluster: ""
 
+
 #################
 # CONFIGURATION #
 #################
@@ -35,3 +36,17 @@ dummy:
 #ceph_rbd_mirror_configure: false
 #ceph_rbd_mirror_pool: ""
 
+
+##########
+# DOCKER #
+##########
+
+#rbd_mirror_containerized_deployment: false
+#rbd_mirror_containerized_deployment_with_kv: false
+#kv_type: etcd
+#kv_endpoint: 127.0.0.1
+#ceph_rbd_mirror_docker_username: ceph
+#ceph_rbd_mirror_docker_imagename: daemon
+#ceph_rbd_mirror_docker_image_tag: latest
+#ceph_docker_on_openstack: false
+

roles/ceph-rbd-mirror/defaults/main.yml

@@ -20,9 +20,24 @@ ceph_rbd_mirror_remote_user: admin
 # Thus if cluster names are identical we can not have them under /etc/ceph
 ceph_rbd_mirror_remote_cluster: ""
 
+
 #################
 # CONFIGURATION #
 #################
 
 ceph_rbd_mirror_configure: false
 ceph_rbd_mirror_pool: ""
+
+
+##########
+# DOCKER #
+##########
+
+rbd_mirror_containerized_deployment: false
+rbd_mirror_containerized_deployment_with_kv: false
+kv_type: etcd
+kv_endpoint: 127.0.0.1
+ceph_rbd_mirror_docker_username: ceph
+ceph_rbd_mirror_docker_imagename: daemon
+ceph_rbd_mirror_docker_image_tag: latest
+ceph_docker_on_openstack: false

roles/ceph-rbd-mirror/meta/main.yml

@@ -14,4 +14,4 @@ galaxy_info:
   categories:
     - system
 dependencies:
-  - { role: ceph.ceph-common }
+  - { role: ceph.ceph-common, when: not rbd_mirror_containerized_deployment }

roles/ceph-rbd-mirror/tasks/docker/checks.yml

@@ -0,0 +1,27 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/ceph.client.admin.keyring
+      - /etc/ceph/ceph.conf
+      - /etc/ceph/monmap
+      - /etc/ceph/ceph.mon.keyring
+      - /var/lib/ceph/bootstrap-osd/ceph.keyring
+      - /var/lib/ceph/bootstrap-rgw/ceph.keyring
+      - /var/lib/ceph/bootstrap-mds/ceph.keyring
+
+- name: stat for ceph config and keys
+  stat:
+    path: "{{ item }}"
+  with_items: ceph_config_keys
+  changed_when: false
+  failed_when: false
+  register: statleftover
+
+- name: fail if we find existing cluster files
+  fail:
+    msg: "looks like no cluster is running but ceph files are present, please remove them"
+  with_together:
+    - ceph_config_keys
+    - statleftover.results
+  when: item.1.stat.exists == true

roles/ceph-rbd-mirror/tasks/docker/dirs_permissions.yml

@@ -0,0 +1,35 @@
+---
+# NOTE (leseb): we can not use docker inspect with 'format filed' because of
+# https://github.com/ansible/ansible/issues/10156
+- name: inspect ceph version
+  shell: docker inspect "docker.io/{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}:{{ ceph_rbd_mirror_docker_image_tag }}" | awk -F '=' '/CEPH_VERSION/ { gsub ("\",", "", $2); print $2 }' | uniq
+  changed_when: false
+  failed_when: false
+  run_once: true
+  register: ceph_version
+
+- set_fact:
+    after_hammer=True
+  when: ceph_version.stdout not in ['firefly','giant', 'hammer']
+
+- name: create bootstrap directories (for or before hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+  when: not after_hammer
+
+- name: create bootstrap directories (after hammer)
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "64045"
+    group: "64045"
+    mode: "0755"
+  with_items:
+    - /etc/ceph/
+  when: after_hammer

roles/ceph-rbd-mirror/tasks/docker/fetch_configs.yml

@@ -0,0 +1,29 @@
+---
+# NOTE (leseb): the mds container needs the admin key
+# so it can create the mds pools for cephfs
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/ceph.conf
+      - /etc/ceph/ceph.client.admin.keyring
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/docker_mon_files/{{ item }}
+  with_items: ceph_config_keys
+  changed_when: false
+  become: false
+  failed_when: false
+  register: statconfig
+
+- name: try to fetch ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/docker_mon_files/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - ceph_config_keys
+    - statconfig.results
+  when: item.1.stat.exists == true

roles/ceph-rbd-mirror/tasks/docker/main.yml

@@ -0,0 +1,31 @@
+---
+- name: check if a cluster is already running
+  shell: "docker ps | grep -sq 'ceph/daemon'"
+  register: ceph_health
+  changed_when: false
+  failed_when: false
+
+- name: check if it is Atomic host
+  stat: path=/run/ostree-booted
+  register: stat_ostree
+
+- name: set fact for using Atomic host
+  set_fact:
+      is_atomic='{{ stat_ostree.stat.exists }}'
+
+- include: checks.yml
+  when: ceph_health.rc != 0
+
+- include: pre_requisite.yml
+- include: "{{ playbook_dir }}/roles/ceph-common/tasks/docker/fetch_image.yml"
+  vars:
+    ceph_docker_username: "{{ ceph_rbd_mirror_docker_username }}"
+    ceph_docker_imagename: "{{ ceph_rbd_mirror_docker_imagename }}"
+    ceph_docker_image_tag: "{{ ceph_rbd_mirror_docker_image_tag }}"
+- include: dirs_permissions.yml
+- include: fetch_configs.yml
+
+- include: selinux.yml
+  when: ansible_os_family == 'RedHat'
+
+- include: start_docker_rbd_mirror.yml

roles/ceph-rbd-mirror/tasks/docker/pre_requisite.yml

@@ -0,0 +1,126 @@
+---
+- name: install pip and docker on ubuntu
+  apt:
+    name: "{{ item }}"
+    state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker
+    - docker.io
+  when: ansible_distribution == 'Ubuntu'
+  tags:
+    with_pkg
+
+- name: install pip and docker on debian
+  apt:
+    name: "{{ item }}"
+    state: present
+    update_cache: yes
+  with_items:
+    - python-pip
+    - docker-engine
+  when: ansible_distribution == 'Debian'
+  tags:
+    with_pkg
+
+# install epel for pip
+- name: install epel on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - epel-release
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+- name: install pip on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+
+- name: install docker-engine on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# for CentOS
+- name: install docker on redhat
+  yum:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "yum"
+  tags:
+    with_pkg
+  failed_when: false
+
+# docker package could be docker-enginer or docker
+- name: install pip and docker on redhat
+  dnf:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - python-pip
+    - docker-engine
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_pkg_mgr == "dnf"
+  tags:
+    with_pkg
+
+- name: start docker service
+  service:
+    name: docker
+    state: started
+    enabled: yes
+  tags:
+    with_pkg
+
+# NOTE (jimcurtis): need at least version 1.9.0 of six or we get:
+# re:NameError: global name 'DEFAULT_DOCKER_API_VERSION' is not defined
+- name: install six
+  pip:
+    name: six
+    version: 1.9.0
+  tags:
+    with_pkg
+
+# NOTE (leseb): for version 1.1.0 because https://github.com/ansible/ansible-modules-core/issues/1227
+- name: install docker-py
+  pip:
+    name: docker-py
+    version: 1.1.0
+  tags:
+    with_pkg
+  when: ansible_version['full'] | version_compare('2.1.0.0', '<')
+
+- name: install docker-py
+  pip:
+    name: docker-py
+    state: latest
+  tags:
+    with_pkg
+  when: ansible_version['full'] | version_compare('2.1.0.0', '>=')

roles/ceph-rbd-mirror/tasks/docker/selinux.yml

@@ -0,0 +1,13 @@
+---
+- name: check if selinux is enabled
+  command: getenforce
+  register: sestatus
+  changed_when: false
+
+- name: set selinux permissions
+  shell: chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - /etc/ceph
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-rbd-mirror/tasks/docker/start_docker_rbd_mirror.yml

@@ -0,0 +1,47 @@
+---
+# Use systemd to manage container on Atomic host
+- name: generate systemd unit file
+  become: true
+  template:
+    src: "{{ role_path }}/templates/ceph-rbd-mirror.service.j2"
+    dest: /var/lib/ceph/ceph-rbd-mirror@.service
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
+
+- name: link systemd unit file for rbd mirror instance
+  file:
+    src: /var/lib/ceph/ceph-rbd-mirror@.service
+    dest: /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror@{{ ansible_hostname }}.service
+    state: link
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
+
+- name: enable systemd unit file for rbd mirror instance
+  command: systemctl enable /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror@{{ ansible_hostname }}.service
+  failed_when: false
+  changed_when: false
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
+
+- name: reload systemd unit files
+  command: systemctl daemon-reload
+  changed_when: false
+  failed_when: false
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
+
+- name: systemd start rbd mirror container
+  service:
+    name: ceph-rbd-mirror@{{ ansible_hostname }}
+    state: started
+    enabled: yes
+  changed_when: false
+  when: ansible_os_family == 'RedHat' or ansible_os_family == 'CoreOS'
+
+- name: run the ceph rbd mirror docker image
+  docker:
+    image: "{{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}"
+    name: ceph-{{ ansible_hostname }}-rbd-mirror
+    net: host
+    state: running
+    volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph"
+  when: ansible_os_family != 'RedHat' and ansible_os_family != 'CoreOS'

roles/ceph-rbd-mirror/tasks/main.yml

@@ -1,5 +1,14 @@
 ---
 - include: pre_requisite.yml
+  when: not rbd_mirror_containerized_deployment
+
 - include: start_rbd_mirror.yml
+  when: not rbd_mirror_containerized_deployment
+
 - include: configure_mirroring.yml
-  when: ceph_rbd_mirror_configure
+  when:
+    - ceph_rbd_mirror_configure
+    - not rbd_mirror_containerized_deployment
+
+- include: ./docker/main.yml
+  when: rbd_mirror_containerized_deployment

roles/ceph-rbd-mirror/templates/ceph-rbd-mirror.service.j2

@@ -0,0 +1,28 @@
+[Unit]
+Description=Ceph RBD mirror
+After=docker.service
+
+[Service]
+EnvironmentFile=-/etc/environment
+ExecStartPre=-/usr/bin/docker stop {{ ansible_hostname }}
+ExecStartPre=-/usr/bin/docker rm {{ ansible_hostname }}
+ExecStart=/usr/bin/docker run --rm --net=host \
+   {% if not rbd_mirror_containerized_deployment_with_kv -%}
+   -v /var/lib/ceph:/var/lib/ceph \
+   -v /etc/ceph:/etc/ceph \
+   {% else -%}
+   -e KV_TYPE={{kv_type}} \
+   -e KV_IP={{kv_endpoint}} \
+   {% endif -%}
+   --privileged \
+   -e CEPH_DAEMON=RBD_MIRROR \
+   --name={{ ansible_hostname }} \
+   {{ ceph_rbd_mirror_docker_username }}/{{ ceph_rbd_mirror_docker_imagename }}
+ExecStopPost=-/usr/bin/docker stop {{ ansible_hostname }}
+Restart=always
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=15
+
+[Install]
+WantedBy=multi-user.target

vagrant_variables.yml.sample

@@ -9,6 +9,7 @@ osd_vms: 3
 mds_vms: 0
 rgw_vms: 0
 nfs_vms: 0
+rbd_mirror_vms: 0
 client_vms: 0
 
 # Deploy RESTAPI on each of the Monitors