mon: destroy and recreate rbd pool if necessary

Users reported that pool_default_pg_num is not honoured for the default
pool 'rbd'. So now we check the pg num value for the RBD pool and if it
does not match pool_default_pg_num then we delete and recreate it.
We also make sure the pool is empty first, just in case someone changed
the value manually and didn't reflect the change in ceph-ansible.

The only issue with this patch is that the pool ID will not be 0 anymore
but more likely 1.

Signed-off-by: Sébastien Han <seb@redhat.com>

roles/ceph-mon/tasks/ceph_keys.yml

@@ -15,6 +15,9 @@
     - cephx
     - groups[restapi_group_name] is defined
 
+- include: rbd_pool.yml
+  when: ceph_conf_overrides.global.osd_pool_default_pg_num is defined
+
 - include: openstack_config.yml
   when: openstack_config
 

roles/ceph-mon/tasks/rbd_pool.yml

@@ -0,0 +1,24 @@
+---
+- name: check rbd pool usage
+  shell: |
+    ceph --connect-timeout 5 --cluster {{ cluster }} df | awk '/rbd/ {print $3}'
+  changed_when: false
+  failed_when: false
+  register: rbd_pool_df
+
+- name: check pg num for rbd pool
+  shell: |
+    ceph --connect-timeout 5 --cluster {{ cluster }} osd pool get rbd pg_num | awk '{print $2}'
+  changed_when: false
+  failed_when: false
+  register: rbd_pool_pgs
+
+- name: destroy and recreate rbd pool if osd_pool_default_pg_num is not honoured
+  shell: |
+    ceph --connect-timeout 5 --cluster {{ cluster }} osd pool rm rbd rbd --yes-i-really-really-mean-it
+    ceph --connect-timeout 5 --cluster {{ cluster }} osd pool create rbd {{ ceph_conf_overrides.global.osd_pool_default_pg_num }}
+  changed_when: false
+  failed_when: false
+  when:
+    - rbd_pool_df.stdout == "0"
+    - rbd_pool_pgs.stdout != "{{ ceph_conf_overrides.global.osd_pool_default_pg_num }}"