rgw: update rgw multisite tasks

- remove destroy tasks
- cleanup conditionals and syntax
- remove unnecessary realm pulls
- enable multisite to be tested in automated
testing infra
- add multisite related vars to main.yml and
group_vars
- update README-MULTISITE
- ensure all `radosgw-admin` commands are being run
on a mon

Signed-off-by: Ali Maredia <amaredia@redhat.com>

README-MULTISITE.md

@@ -1,5 +1,5 @@
-RGW Multisite (Experimental)
-============================
+RGW Multisite
+=============
 
 Directions for configuring the RGW Multisite support in ceph-ansible
 
@@ -13,68 +13,65 @@ More details:
 
 * Can configure a Master and Secondary realm/zonegroup/zone on 2 separate clusters.
 
-
 ## Configuring the Master Zone in the Primary Cluster
 
 This will setup the realm, zonegroup and master zone and make them the defaults.  It will also reconfigure the specified RGW for use with the zone.
 
-1. Edit the Inventory File
-
-```
-[rgws]
-cluster0-rgw0 rgw_zone=us-east rgw_zonemaster=true
-```
+``
 1. Generate System Access and System Secret Keys
 
 ```
-echo system_access_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) > multi-site-keys.sh
-echo system_secret_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 40 | head -n 1) >> multi-site-keys.sh
+echo system_access_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) > multi-site-keys.txt
+echo system_secret_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 40 | head -n 1) >> multi-site-keys.txt
 ```
-1. Edit the RGW Group Vars
+2. Edit the RGW Group Vars
 
 ```
 copy_admin_key: true
 # Enable Multisite support
 rgw_multisite: true
-rgw_realm: gold
-rgw_zonegroup: us
+rgw_zone: jupiter
+rgw_zonemaster: true
+rgw_zonesecondary: false
+rgw_zonegroup: solarsystem
+rgw_zone_user: zone.user
+rgw_realm: milkyway
 system_access_key: 6kWkikvapSnHyE22P7nO
 system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
 ```
 
 **Note:** replace the system_access_key and system_secret_key values with the ones you generated
+**Note:** rgw_zonemaster should have the value of true and rgw_zonesecondary should be false
 
-1. Run the ceph-ansible playbook on your 1st cluster
-
-Note: If you have already installed a cluster with ceph-ansible, you can use the `rgw-configure.yml` playbook as a shortcut (Only runs the ceph-rgw role)
+3. Run the ceph-ansible playbook on your 1st cluster
 
 ## Configuring the Secondary Zone in a Separate Cluster
 
-```
-[rgws]
-cluster1-rgw0 rgw_zone=us-west rgw_zonesecondary=true
-```
-
-1. Edit the RGW Group Vars
+4. Edit the RGW Group Vars
 
 ```
 copy_admin_key: true
 # Enable Multisite support
 rgw_multisite: true
-rgw_realm: gold
-rgw_zonegroup: us
-rgw_pullhost: cluster1-rgw0.fqdn
+rgw_zone: mars
+rgw_zonemaster: false
+rgw_zonesecondary: true
+rgw_zonegroup: solarsystem
+rgw_zone_user: zone.user
+rgw_realm: milkyway
 system_access_key: 6kWkikvapSnHyE22P7nO
 system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
+rgw_pull_proto: http
+rgw_pull_port: 8080
+rgw_pullhost: cluster0-rgw0
 ```
 
 **Note:** pullhost should be the host of the RGW that is configured as the Zone Master
-**Note:** system_access_key and system_secret_key should match what you used in the 1st cluster
+**Note:** rgw_zone_user, system_access_key, and system_secret_key should match what you used in the master cluster
+**Note:** rgw_zonemaster should have the value of false and rgw_zonesecondary should be true
 
 
-1. Run the ceph-ansible playbook on your 2nd cluster
-
-Note: If you have already installed a cluster with ceph-ansible, you can use the `rgw-configure.yml` playbook as a shortcut (Only runs the ceph-rgw role)
+5. Run the ceph-ansible playbook on your 2nd cluster
 
 ## Conclusion
 

group_vars/rgws.yml.sample

@@ -19,11 +19,6 @@ dummy:
 # will copy the admin key to the /etc/ceph/ directory
 #copy_admin_key: false
 
-# Multi-site remote pull URL variables
-#rgw_pull_port: "{{ radosgw_frontend_port }}"
-#rgw_pull_proto: "http"
-
-
 ##########
 # TUNING #
 ##########
@@ -54,6 +49,30 @@ dummy:
 #    size: ""
 
 
+#############
+# MULTISITE #
+#############
+
+#rgw_multisite: false
+
+# The following Multi-site related variables should be set by the user.
+# rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site
+# If multisite is configured rgw_zone should not be set to "default". See README-MULTISITE.md for an example.
+#rgw_zone: default
+
+#rgw_zonemaster: true
+#rgw_zonesecondary: false
+#rgw_zonegroup: dummy # should be set by the user
+#rgw_zone_user: zone.user
+#rgw_realm: dummy # should be set by the user
+#system_access_key: 6kWkikvapSnHyE22P7nO
+#system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
+
+# Multi-site remote pull URL variables
+#rgw_pull_port: "{{ radosgw_civetweb_port }}"
+#rgw_pull_proto: "http"
+#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary
+
 ##########
 # DOCKER #
 ##########
@@ -69,15 +88,6 @@ dummy:
 #ceph_config_keys: [] # DON'T TOUCH ME
 #rgw_config_keys: "/" # DON'T TOUCH ME
 
-
-#############
-# MULTISITE #
-#############
-#rgw_multisite: false
-#rgw_zonegroup: ""
-#rgw_zone: ""
-
-
 ###########
 # SYSTEMD #
 ###########

roles/ceph-rgw/defaults/main.yml

@@ -11,11 +11,6 @@
 # will copy the admin key to the /etc/ceph/ directory
 copy_admin_key: false
 
-# Multi-site remote pull URL variables
-rgw_pull_port: "{{ radosgw_frontend_port }}"
-rgw_pull_proto: "http"
-
-
 ##########
 # TUNING #
 ##########
@@ -46,6 +41,30 @@ rgw_pull_proto: "http"
 #    size: ""
 
 
+#############
+# MULTISITE #
+#############
+
+rgw_multisite: false
+
+# The following Multi-site related variables should be set by the user.
+# rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site
+# If multisite is configured rgw_zone should not be set to "default". See README-MULTISITE.md for an example.
+rgw_zone: default
+
+rgw_zonemaster: true
+rgw_zonesecondary: false
+#rgw_zonegroup: dummy # should be set by the user
+#rgw_zone_user: zone.user
+#rgw_realm: dummy # should be set by the user
+#system_access_key: 6kWkikvapSnHyE22P7nO
+#system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
+
+# Multi-site remote pull URL variables
+rgw_pull_port: "{{ radosgw_civetweb_port }}"
+rgw_pull_proto: "http"
+#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary
+
 ##########
 # DOCKER #
 ##########
@@ -61,15 +80,6 @@ ceph_rgw_docker_extra_env:
 ceph_config_keys: [] # DON'T TOUCH ME
 rgw_config_keys: "/" # DON'T TOUCH ME
 
-
-#############
-# MULTISITE #
-#############
-rgw_multisite: false
-rgw_zonegroup: ""
-rgw_zone: ""
-
-
 ###########
 # SYSTEMD #
 ###########

roles/ceph-rgw/tasks/main.yml

@@ -17,8 +17,8 @@
 - name: include_tasks multisite/main.yml
   include_tasks: multisite/main.yml
   when:
-    - rgw_zone != ""
     - rgw_multisite
+    - ceph_release_num[ceph_release] >= ceph_release_num.jewel
 
 - name: include_tasks docker/main.yml
   include_tasks: docker/main.yml

roles/ceph-rgw/tasks/multisite/checks.yml

@@ -1,6 +1,7 @@
 ---
 - name: check if the realm already exists
   command: radosgw-admin realm get --rgw-realm={{ rgw_realm }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   register: realmcheck
   failed_when: False
   changed_when: False
@@ -8,6 +9,7 @@
 
 - name: check if the zonegroup already exists
   command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   register: zonegroupcheck
   failed_when: False
   changed_when: False
@@ -15,13 +17,15 @@
 
 - name: check if the zone already exists
   command: radosgw-admin zone get --rgw-zone={{ rgw_zone }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   register: zonecheck
   failed_when: False
   changed_when: False
   check_mode: no
 
 - name: check if the system user already exists
-  command: radosgw-admin user info --uid=zone.user
+  command: radosgw-admin user info --uid={{ rgw_zone_user }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   register: usercheck
   failed_when: False
   changed_when: False

roles/ceph-rgw/tasks/multisite/destroy.yml

@@ -1,38 +0,0 @@
----
-- name: delete the zone user
-  command: radosgw-admin user rm --uid=zone.user
-  run_once: true
-  failed_when: false
-
-- name: remove zone from zonegroup
-  command: radosgw-admin zonegroup remove --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }}
-  run_once: true
-  failed_when: false
-  notify:
-    - update period
-
-- name: delete the zone
-  command: radosgw-admin zone delete --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }}
-  run_once: true
-  failed_when: false
-
-- name: delete the zonegroup
-  command: radosgw-admin zonegroup delete --rgw-zonegroup={{ rgw_zonegroup }}
-  run_once: true
-  failed_when: false
-
-- name: delete the realm
-  command: radosgw-admin realm delete --rgw-realm={{ rgw_realm }}
-  run_once: true
-  failed_when: false
-
-- name: delete zone from rgw stanza in ceph.conf
-  lineinfile:
-    dest: "/etc/ceph/{{ cluster }}.conf"
-    regexp: "rgw_zone = {{ rgw_zonegroup }}-{{ rgw_zone }}"
-    state: absent
-  when:
-    - rgw_zone is defined
-    - rgw_zonegroup is defined
-  notify:
-    - restart rgw

roles/ceph-rgw/tasks/multisite/main.yml

@@ -6,13 +6,13 @@
 - name: include_tasks master.yml
   include_tasks: master.yml
   when:
-    - rgw_zonemaster is defined
     - rgw_zonemaster
+    - not rgw_zonesecondary
 
 - name: include_tasks secondary.yml
   include_tasks: secondary.yml
   when:
-    - rgw_zonesecondary is defined
+    - not rgw_zonemaster
     - rgw_zonesecondary
 
 # Continue with common tasks

roles/ceph-rgw/tasks/multisite/master.yml

@@ -1,33 +1,28 @@
 ---
 - name: create the realm
   command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
     - "'No such file or directory' in realmcheck.stderr"
-    - rgw_zonemaster
-  notify:
-    - update period
 
 - name: create the zonegroup
-  command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} --master --default
+  command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_frontend_port }} --master --default
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
     - "'No such file or directory' in zonegroupcheck.stderr"
-    - rgw_zonemaster
-  notify:
-    - update period
 
 - name: create the zone
-  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master
+  command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
     - "'No such file or directory' in zonecheck.stderr"
-    - rgw_zonemaster
-  notify:
-    - update period
 
 - name: create the zone user
-  command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system
+  command: radosgw-admin user create --uid={{ rgw_zone_user }} --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
     - "'could not fetch user info: no user info saved' in usercheck.stderr"

roles/ceph-rgw/tasks/multisite/secondary.yml

@@ -1,36 +1,33 @@
 ---
 - name: fetch the realm
   command: radosgw-admin realm pull --url={{ rgw_pull_proto }}://{{ rgw_pullhost }}:{{ rgw_pull_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
-    - 'No such file or directory' in realmcheck.stderr
-  notify:
-    - update period
+    - "'No such file or directory' in realmcheck.stderr"
 
 - name: fetch the period
   command: radosgw-admin period pull --url={{ rgw_pull_proto }}://{{ rgw_pullhost }}:{{ rgw_pull_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
-    - 'No such file or directory' in realmcheck.stderr
-  notify:
-    - update period
+    - "'No such file or directory' in realmcheck.stderr"
 
 - name: set default realm
   command: radosgw-admin realm default --rgw-realm={{ rgw_realm }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  notify:
-    - update period # Might not need to update period here
 
 - name: set default zonegroup
   command: radosgw-admin zonegroup default --rgw-zonegroup={{ rgw_zonegroup }}
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  notify:
-    - update period # Might not need to update period here
 
 - name: create the zone
   command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default
+  delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
-    - 'No such file or directory' in zonecheck.stderr
+    - "'No such file or directory' in zonecheck.stderr"
   notify:
     - update period