rgw: add multi-instances support when deploying multisite

This commit adds the multi-instances when deploying rgw multisite

Signed-off-by: Guillaume Abrioux <gabrioux@redhat.com>
Co-authored-by: Dimitri Savineau <dsavinea@redhat.com>

roles/ceph-config/templates/ceph.conf.j2

@@ -115,7 +115,7 @@ osd memory target = {{ _osd_memory_target | default(osd_memory_target) }}
 host = {{ _rgw_hostname }}
 keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}/keyring
 log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] + '.' + instance['instance_name'] }}.log
-{% set _rgw_binding_socket = instance['radosgw_address']|string + ':' + instance['radosgw_frontend_port']|string %}
+{% set _rgw_binding_socket = instance['radosgw_address'] | default(_radosgw_address) | string + ':' + instance['radosgw_frontend_port'] | default(radosgw_frontend_port) | string %}
 {%- macro frontend_line(frontend_type) -%}
 {%- if frontend_type == 'civetweb' -%}
 {{ radosgw_frontend_type }} port={{ _rgw_binding_socket }}{{ 's ssl_certificate='+radosgw_frontend_ssl_certificate if radosgw_frontend_ssl_certificate else '' }}

roles/ceph-facts/tasks/set_radosgw_address.yml

@@ -52,8 +52,19 @@
 
 - name: set_fact rgw_instances with rgw multisite
   set_fact:
-    rgw_instances: "{{ rgw_instances|default([]) | union([{'instance_name': 'rgw' + item|string, 'radosgw_address': _radosgw_address, 'radosgw_frontend_port': radosgw_frontend_port|int, 'rgw_realm': rgw_realm|string, 'rgw_zonegroup': rgw_zonegroup|string, 'rgw_zone': rgw_zone|string}]) }}"
+    rgw_instances: "{{ rgw_instances|default([]) | union([{'instance_name': 'rgw' + item | string, 'radosgw_address': _radosgw_address, 'radosgw_frontend_port': radosgw_frontend_port | int, 'rgw_realm': rgw_realm | string, 'rgw_zonegroup': rgw_zonegroup | string, 'rgw_zone': rgw_zone | string, 'system_access_key': system_access_key, 'system_secret_key': 'system_secret_key', 'rgw_zone_user': rgw_zone_user, 'rgw_zone_user_display_name': rgw_zone_user_display_name, 'endpoint': (rgw_pull_proto + '://' + rgw_pullhost + ':' + rgw_pull_port | string) if not rgw_zonemaster | bool and rgw_zonesecondary | bool else omit }]) }}"
   with_sequence: start=0 end={{ radosgw_num_instances|int - 1 }}
+  when:
+    - inventory_hostname in groups.get(rgw_group_name, [])
+    - rgw_instances is undefined
+    - rgw_multisite | bool
+
+- name: set_fact rgw_instances_all
+  set_fact:
+    rgw_instances_all: '{{ rgw_instances_all | default([]) | union([item.1 | combine({"host": item.0})]) }}'
+  with_nested:
+    - "{{ groups.get(rgw_group_name, []) }}"
+    - "{{ rgw_instances }}"
   when:
     - inventory_hostname in groups.get(rgw_group_name, [])
     - rgw_multisite | bool

roles/ceph-rgw/tasks/multisite/checks.yml

@@ -1,24 +1,33 @@
 ---
 - name: check if the realm already exists
-  command: "{{ container_exec_cmd }} radosgw-admin realm get --cluster={{ cluster }} --rgw-realm={{ rgw_realm }}"
+  command: "{{ container_exec_cmd }} radosgw-admin realm get --cluster={{ cluster }} --rgw-realm={{ item }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   register: realmcheck
   failed_when: False
   changed_when: False
   check_mode: no
+  run_once: True
+  loop: "{{ realms }}"
+  when: realms is defined
 
 - name: check if the zonegroup already exists
-  command: "{{ container_exec_cmd }} radosgw-admin zonegroup get --cluster={{ cluster }} --rgw-realm={{ rgw_realm }} --rgw-zonegroup={{ rgw_zonegroup }}"
+  command: "{{ container_exec_cmd }} radosgw-admin zonegroup get --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   register: zonegroupcheck
   failed_when: False
   changed_when: False
   check_mode: no
+  run_once: True
+  loop: "{{ zonegroups }}"
+  when: zonegroups is defined
 
 - name: check if the zone already exists
-  command: "{{ container_exec_cmd }} radosgw-admin zone get --rgw-realm={{ rgw_realm }} --cluster={{ cluster }} --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }}"
+  command: "{{ container_exec_cmd }} radosgw-admin zone get --rgw-realm={{ item.realm }} --cluster={{ cluster }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   register: zonecheck
   failed_when: False
   changed_when: False
   check_mode: no
+  run_once: True
+  loop: "{{ zones }}"
+  when: zones is defined

roles/ceph-rgw/tasks/multisite/create_realm_zonegroup_zone_lists.yml

@@ -1,93 +1,39 @@
-- name: create list realms
+---
+- name: set_fact realms
   set_fact:
-    realms: "{{ realms | default([]) + [{ 'realm': hostvars[item]['rgw_realm'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    realms: '{{ realms | default([]) | union([item.rgw_realm]) }}'
+  loop: "{{ rgw_instances_all }}"
   run_once: true
-  when:
-    - hostvars[item]['rgw_zonemaster'] | bool
-    - "'No such file or directory' in hostvars[item]['realmcheck'].stderr"
-
-- name: make all items in realms unique
-  set_fact:
-    realms: "{{ realms | unique }}"
-  run_once: true
-  when:
-    - realms is defined
 
 - name: create list secondary_realms
   set_fact:
-    secondary_realms: "{{ secondary_realms | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'is_master': hostvars[item]['rgw_zonemaster'], 'endpoint': hostvars[item]['rgw_pull_proto'] + '://' + hostvars[item]['rgw_pullhost'] + ':' + hostvars[item]['rgw_pull_port']|string, 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    secondary_realms: "{{ secondary_realms | default([]) | union([{ 'realm': item.rgw_realm, 'endpoint': item.endpoint, 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key, 'is_master': hostvars[item.host]['rgw_zonemaster'] }]) }}"
+  loop: "{{ rgw_instances_all }}"
   run_once: true
-  when:
-    - not hostvars[item]['rgw_zonemaster'] | bool
-
-- name: make all items in secondary_realms unique
-  set_fact:
-    secondary_realms: "{{ secondary_realms | unique }}"
-  run_once: true
-  when:
-    - secondary_realms is defined
+  when: not hostvars[item.host]['rgw_zonemaster'] | bool
 
 - name: create list zonegroups
   set_fact:
-    zonegroups: "{{ zonegroups | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'is_master': hostvars[item]['rgw_zonegroupmaster'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    zonegroups: "{{ zonegroups | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'is_master': hostvars[item.host]['rgw_zonegroupmaster'] }]) }}"
+  loop: "{{ rgw_instances_all }}"
   run_once: true
   when:
-    - hostvars[item]['rgw_zonemaster'] | bool
-    - "'No such file or directory' in hostvars[item]['zonegroupcheck'].stderr"
-
-- name: make all items in zonegroups unique
-  set_fact:
-    zonegroups: "{{ zonegroups | unique }}"
-  run_once: true
-  when:
-    - zonegroups is defined
+    - hostvars[item.host]['rgw_zonemaster'] | bool
 
 - name: create list zones
   set_fact:
-    zones: "{{ zones | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'zone': hostvars[item]['rgw_zone'], 'is_master': hostvars[item]['rgw_zonemaster'], 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    zones: "{{ zones | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'zone': item.rgw_zone, 'is_master': hostvars[item.host]['rgw_zonemaster'], 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key }]) }}"
+  loop: "{{ rgw_instances_all }}"
   run_once: true
-  when: "'No such file or directory' in hostvars[item]['zonecheck'].stderr"
-
-- name: make all items in zones unique
-  set_fact:
-    zones: "{{ zones | unique }}"
-  run_once: true
-  when:
-    - zones is defined
 
 - name: create a list of dicts with each rgw endpoint and it's zone
   set_fact:
-    zone_endpoint_pairs: "{{ zone_endpoint_pairs | default([]) + [{ 'endpoint': hostvars[item]['rgw_multisite_proto'] + '://' + hostvars[item]['_radosgw_address'] + ':' + radosgw_frontend_port|string, 'rgw_zone': hostvars[item]['rgw_zone'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    zone_endpoint_pairs: "{{ zone_endpoint_pairs | default([]) | union([{ 'endpoint': hostvars[item.host]['rgw_multisite_proto'] + '://' + item.radosgw_address + ':' + item.radosgw_frontend_port | string, 'rgw_zone': item.rgw_zone, 'rgw_realm': item.rgw_realm, 'rgw_zonegroup': item.rgw_zonegroup, 'rgw_zonemaster': hostvars[item.host]['rgw_zonemaster']}]) }}"
+  loop: "{{ rgw_instances_all }}"
   run_once: true
 
-- name: create string of all the endpoints in the same rgw_zone
-  set_fact:
-    zone_endpoints_string: "{{ zone_endpoints_string | default('') + item.endpoint + ',' }}"
-  with_items: "{{ zone_endpoint_pairs }}"
-  when: item.rgw_zone == rgw_zone
-
-- name: remove ',' after last endpoint in a endpoints string
-  set_fact:
-    zone_endpoints_string: "{{ zone_endpoints_string[:-1] }}"
-  when:
-    - endpoints_string is defined
-    - endpoints_string[-1] == ','
-
 - name: create a list of zones and all their endpoints
   set_fact:
-    zone_endpoints_list: "{{ zone_endpoints_list | default([]) + [{ 'endpoints': hostvars[item]['zone_endpoints_string'], 'zone': hostvars[item]['rgw_zone'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'realm': hostvars[item]['rgw_realm'], 'is_master': hostvars[item]['rgw_zonemaster'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
+    zone_endpoints_list: "{{ zone_endpoints_list | default([]) | union([{'zone': item.rgw_zone, 'zonegroup': item.rgw_zonegroup, 'realm': item.rgw_realm, 'is_master': item.rgw_zonemaster, 'endpoints': ','.join(zone_endpoint_pairs | selectattr('rgw_zone','==',item.rgw_zone) | selectattr('rgw_realm','==',item.rgw_realm) | selectattr('rgw_zonegroup', '==', item.rgw_zonegroup) | map(attribute='endpoint'))}]) }}"
+  loop: "{{ zone_endpoint_pairs }}"
   run_once: true
-  when: hostvars[item]['zone_endpoints_string'] is defined
-
-- name: make all items in zone_endpoints_list unique
-  set_fact:
-    zone_endpoints_list: "{{ zone_endpoints_list | unique }}"
-  run_once: true
-  when:
-    - zone_endpoints_list is defined

roles/ceph-rgw/tasks/multisite/create_zone_user.yml

@@ -1,31 +1,28 @@
+---
+- name: create list zone_users
+  set_fact:
+    zone_users: "{{ zone_users | default([]) | union([{ 'realm': item.rgw_realm, 'zonegroup': item.rgw_zonegroup, 'zone': item.rgw_zone, 'system_access_key': item.system_access_key, 'system_secret_key': item.system_secret_key, 'user': item.rgw_zone_user, 'display_name': item.rgw_zone_user_display_name }]) }}"
+  loop: "{{ rgw_instances_all }}"
+  run_once: true
+  when:
+    - hostvars[item.host]['rgw_zonemaster'] | bool
+    - hostvars[item.host]['rgw_zonegroupmaster'] | bool
+
 - name: check if the realm system user already exists
-  command: "{{ container_exec_cmd }} radosgw-admin user info --cluster={{ cluster }} --rgw-realm={{ rgw_realm }} --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --uid={{ rgw_zone_user }}"
+  command: "{{ container_exec_cmd }} radosgw-admin user info --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --uid={{ item.user }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   register: usercheck
   failed_when: False
   changed_when: False
   check_mode: no
-
-- name: create list zone_users
-  set_fact:
-    zone_users: "{{ zone_users | default([]) + [{ 'realm': hostvars[item]['rgw_realm'], 'zonegroup': hostvars[item]['rgw_zonegroup'], 'zone': hostvars[item]['rgw_zone'], 'access_key': hostvars[item]['system_access_key'], 'secret_key': hostvars[item]['system_secret_key'], 'user': hostvars[item]['rgw_zone_user'], 'display_name': hostvars[item]['rgw_zone_user_display_name'] }] }}"
-  with_items: "{{ groups.get(rgw_group_name, []) }}"
-  run_once: true
-  when:
-    - hostvars[item]['rgw_zonemaster'] | bool
-    - hostvars[item]['rgw_zonegroupmaster'] | bool
-    - "'could not fetch user info: no user info saved' in hostvars[item]['usercheck'].stderr"
-
-- name: make all items in zone_users unique
-  set_fact:
-    zone_users: "{{ zone_users | unique }}"
-  run_once: true
-  when:
-    - zone_users is defined
+  run_once: True
+  loop: "{{ zone_users }}"
 
 - name: create the zone user(s)
-  command: "{{ container_exec_cmd }} radosgw-admin user create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --uid={{ item.user }} --display-name='{{ item.display_name }}' --access-key={{ item.access_key }} --secret={{ item.secret_key }} --system"
+  command: "{{ container_exec_cmd }} radosgw-admin user create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --uid={{ item.item.user }} --display-name='{{ item.item.display_name }}' --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} --system"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zone_users }}"
-  when: zone_users is defined
+  loop: "{{ usercheck.results }}"
+  when:
+    - zone_users is defined
+    - "'could not fetch user info: no user info saved' in item.stderr"

roles/ceph-rgw/tasks/multisite/main.yml

@@ -1,10 +1,10 @@
 ---
-- name: include multisite checks
-  include_tasks: checks.yml
-
 - name: include_tasks create_realm_zonegroup_zone_lists.yml
   include_tasks: create_realm_zonegroup_zone_lists.yml
 
+- name: include multisite checks
+  include_tasks: checks.yml
+
 # Include the tasks depending on the zone type
 - name: include_tasks master.yml
   include_tasks: master.yml

roles/ceph-rgw/tasks/multisite/master.yml

@@ -1,85 +1,39 @@
 ---
-- name: create default realm
-  command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --default"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ realms }}"
-  when:
-    - realms is defined
-    - realms | length == 1
-
 - name: create the realm(s)
-  command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.realm }}"
+  command: "{{ container_exec_cmd }} radosgw-admin realm create --cluster={{ cluster }} --rgw-realm={{ item.item }} {{ '--default' if realms | length == 1 else '' }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ realms }}"
+  loop: "{{ realmcheck.results }}"
   when:
     - realms is defined
-    - realms | length > 1
+    - realms | length > 0
+    - "'No such file or directory' in item.stderr"
 
-- name: create default master zonegroup(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --default --master"
+- name: create zonegroup(s)
+  command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} {{ '--default' if zonegroups | length == 1 else '' }} {{ '--master' if item.item.is_master | bool else '' }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zonegroups }}"
+  loop: "{{ zonegroupcheck.results }}"
   when:
     - zonegroups is defined
-    - zonegroups | length == 1
-    - item.is_master | bool
+    - zonegroups | length > 0
+    - item.item.is_master | bool
+    - "'No such file or directory' in item.stderr"
 
-- name: create default zonegroup(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --default"
+- name: create the master zone
+  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} {{ '--default' if zones | length == 1 else '' }} --master"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zonegroups }}"
-  when:
-    - zonegroups is defined
-    - zonegroups | length == 1
-    - not item.is_master | bool
-
-- name: create master zonegroup(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --master"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ zonegroups }}"
-  when:
-    - zonegroups is defined
-    - zonegroups | length > 1
-    - item.is_master | bool
-
-- name: create non-master non-default zonegroup(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zonegroup create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ zonegroups }}"
-  when:
-    - zonegroups is defined
-    - zonegroups | length > 1
-    - not item.is_master | bool
-
-- name: create the default master zone
-  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --master --default"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ zones }}"
+  loop: "{{ zonecheck.results }}"
   when:
     - zones is defined
-    - zones | length == 1
-    - item.is_master | bool
-
-- name: create the master zone(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --master"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ zones }}"
-  when:
-    - zones is defined
-    - zones | length > 1
-    - item.is_master | bool
+    - zones | length > 0
+    - item.item.is_master | bool
+    - "'No such file or directory' in item.stderr"
 
 - name: add endpoints to their zone(s)
   command: "{{ container_exec_cmd }} radosgw-admin zone modify --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --endpoints {{ item.endpoints }}"
-  with_items: "{{ zone_endpoints_list }}"
+  loop: "{{ zone_endpoints_list }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
@@ -90,7 +44,7 @@
   command: "{{ container_exec_cmd }} radosgw-admin --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} period update --commit"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zone_endpoints_list }}"
+  loop: "{{ zone_endpoints_list }}"
   when:
     - zone_endpoints_list is defined
     - item.is_master | bool

roles/ceph-rgw/tasks/multisite/secondary.yml

@@ -1,41 +1,32 @@
 ---
 - name: fetch the realm(s)
-  command: "{{ container_exec_cmd }} radosgw-admin realm pull --cluster={{ cluster }} --rgw-realm={{ item.realm }} --url={{ item.endpoint }} --access-key={{ item.access_key }} --secret={{ item.secret_key }}"
+  command: "{{ container_exec_cmd }} radosgw-admin realm pull --cluster={{ cluster }} --rgw-realm={{ item.realm }} --url={{ item.endpoint }} --access-key={{ item.system_access_key }} --secret={{ item.system_secret_key }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ secondary_realms }}"
+  loop: "{{ secondary_realms }}"
   when: secondary_realms is defined
 
 - name: get the period(s)
   command: "{{ container_exec_cmd }} radosgw-admin period get --cluster={{ cluster }} --rgw-realm={{ item.realm }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ secondary_realms }}"
+  loop: "{{ secondary_realms }}"
   when: secondary_realms is defined
 
-- name: create the default zone
-  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }} --default"
+- name: create the zone
+  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.item.realm }} --rgw-zonegroup={{ item.item.zonegroup }} --rgw-zone={{ item.item.zone }} --access-key={{ item.item.system_access_key }} --secret={{ item.item.system_secret_key }} {{ '--default' if zones | length == 1 else '' }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zones }}"
+  loop: "{{ zonecheck.results }}"
   when:
     - zones is defined
-    - zones | length == 1
-    - not item.is_master | bool
-
-- name: create the non-master non-default zone(s)
-  command: "{{ container_exec_cmd }} radosgw-admin zone create --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --access-key={{ item.access_key }} --secret={{ item.secret_key }}"
-  delegate_to: "{{ groups[mon_group_name][0] }}"
-  run_once: true
-  with_items: "{{ zones }}"
-  when:
-    - zones is defined
-    - zones | length > 1
-    - not item.is_master | bool
+    - zones | length > 0
+    - not item.item.is_master | bool
+    - "'No such file or directory' in item.stderr"
 
 - name: add endpoints to their zone(s)
   command: "{{ container_exec_cmd }} radosgw-admin zone modify --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} --endpoints {{ item.endpoints }}"
-  with_items: "{{ zone_endpoints_list }}"
+  loop: "{{ zone_endpoints_list }}"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
   when:
@@ -46,7 +37,7 @@
   command: "{{ container_exec_cmd }} radosgw-admin --cluster={{ cluster }} --rgw-realm={{ item.realm }} --rgw-zonegroup={{ item.zonegroup }} --rgw-zone={{ item.zone }} period update --commit"
   delegate_to: "{{ groups[mon_group_name][0] }}"
   run_once: true
-  with_items: "{{ zone_endpoints_list }}"
+  loop: "{{ zone_endpoints_list }}"
   when:
     - zone_endpoints_list is defined
     - not item.is_master | bool

roles/ceph-validate/tasks/check_rgw_multisite.yml

@@ -2,7 +2,8 @@
 - name: fail if rgw_zone is default
   fail:
     msg: "rgw_zone cannot be named 'default'"
-  when: rgw_zone == 'default' or rgw_zone is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.rgw_zone is undefined or item.rgw_zone == 'default'
 
 - name: fail if either rgw_zonemaster or rgw_zonesecondary is undefined
   fail:
@@ -19,55 +20,44 @@
 - name: fail if rgw_zonegroup is not set
   fail:
     msg: "rgw_zonegroup has not been set by the user"
-  when: rgw_zonegroup is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.rgw_zonegroup is undefined
 
 - name: fail if rgw_zone_user is not set
   fail:
     msg: "rgw_zone_user has not been set by the user"
-  when: rgw_zone_user is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.rgw_zone_user is undefined
 
 - name: fail if rgw_zone_user_display_name is not set
   fail:
     msg: "rgw_zone_user_display_name has not been set by the user"
-  when: rgw_zone_user_display_name is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.rgw_zone_user_display_name is undefined
 
 - name: fail if rgw_realm is not set
   fail:
     msg: "rgw_realm has not been set by the user"
-  when: rgw_realm is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.rgw_realm is undefined
 
 - name: fail if system_access_key is not set
   fail:
     msg: "system_access_key has not been set by the user"
-  when: system_access_key is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.system_access_key is undefined
 
 - name: fail if system_secret_key is not set
   fail:
     msg: "system_secret_key has not been set by the user"
-  when: system_secret_key is undefined
+  loop: "{{ rgw_instances }}"
+  when: item.system_secret_key is undefined
 
-- name: fail if rgw_pull_port is not set
+- name: fail if endpoint is not set
   fail:
-    msg: "rgw_pull_port has not been set by the user"
+    msg: "endpoint has not been set by the user"
+  loop: "{{ rgw_instances }}"
   when:
     - rgw_zonesecondary | bool
-    - rgw_pull_port is undefined
-
-- name: fail if rgw_pull_proto is not set
-  fail:
-    msg: "rgw_pull_proto has not been set by the user"
-  when:
-    - rgw_zonesecondary | bool
-    - rgw_pull_proto is undefined
-
-- name: fail if rgw_pullhost is not set
-  fail:
-    msg: "rgw_pullhost has not been set by the user"
-  when:
-    - rgw_zonesecondary | bool
-    - rgw_pullhost is undefined
-
-- name: fail if radosgw_num_instances is not 1
-  fail:
-    msg: "radosgw_num_instances cannot be more than 1"
-  when: radosgw_num_instances|int > 1
+    - rgw_pull_port is undefined and rgw_pullhost is undefined and rgw_pull_proto is undefined
+    - item.endpoint is undefined

tests/functional/rgw-multisite/container/group_vars/rgws

@@ -2,17 +2,6 @@
 copy_admin_key: true
 # Enable Multisite support
 rgw_multisite: true
-rgw_zone: jupiter
-rgw_zonemaster: true
-rgw_zonesecondary: false
-rgw_zonegroup: solarsystem
-rgw_zonegroupmaster: True
-rgw_zone_user: zone.user
-rgw_zone_user_display_name: "Zone User"
-rgw_multisite_proto: http
-rgw_realm: milkyway
-system_access_key: 6kWkikvapSnHyE22P7nO
-system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
 rgw_create_pools:
   foo:
     pg_num: 16
@@ -21,3 +10,7 @@ rgw_create_pools:
     pg_num: 16
 rgw_override_bucket_index_max_shards: 16
 rgw_bucket_default_quota_max_objects: 1638400
+rgw_zonemaster: True
+rgw_zonesecondary: False
+rgw_zonegroupmaster: True
+rgw_multisite_proto: http

tests/functional/rgw-multisite/container/host_vars/osd0

@@ -0,0 +1,24 @@
+rgw_instances:
+  - instance_name: 'rgw0'
+    rgw_realm: 'france'
+    rgw_zonegroup: 'idf'
+    rgw_zone: 'paris'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8080
+    rgw_zone_user: jacques.chirac
+    rgw_zone_user_display_name: "Jacques Chirac"
+    system_access_key: P9Eb6S8XNyo4dtZZUUMy
+    system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB
+  - instance_name: 'rgw1'
+    rgw_realm: 'usa'
+    rgw_zonegroup: 'alaska'
+    rgw_zone: 'juneau'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8081
+    rgw_zone_user: edward.lewis
+    rgw_zone_user_display_name: "Edward Lewis"
+    system_access_key: yu17wkvAx3B8Wyn08XoF
+    system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY
+# functional testing
+rgw_multisite_endpoint_addr: 192.168.105.11
+radosgw_num_instances: 2

tests/functional/rgw-multisite/container/hosts

@@ -5,4 +5,4 @@ mon0
 osd0
 
 [rgws]
-osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.105.11
+osd0

tests/functional/rgw-multisite/container/secondary/group_vars/rgws

@@ -1,4 +1,6 @@
 ---
+# Enable Multisite support
+rgw_multisite: true
 rgw_create_pools:
   foo:
     pg_num: 16
@@ -7,3 +9,7 @@ rgw_create_pools:
     pg_num: 16
 rgw_override_bucket_index_max_shards: 16
 rgw_bucket_default_quota_max_objects: 1638400
+rgw_zonemaster: False
+rgw_zonesecondary: True
+rgw_zonegroupmaster: True
+rgw_multisite_proto: http

tests/functional/rgw-multisite/container/secondary/host_vars/osd0

@@ -0,0 +1,26 @@
+rgw_instances:
+  - instance_name: 'rgw0'
+    rgw_realm: 'france'
+    rgw_zonegroup: 'idf'
+    rgw_zone: 'versailles'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8080
+    rgw_zone_user: jacques.chirac
+    rgw_zone_user_display_name: "Jacques Chirac"
+    system_access_key: P9Eb6S8XNyo4dtZZUUMy
+    system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB
+    endpoint: http://192.168.105.11:8080
+  - instance_name: 'rgw1'
+    rgw_realm: 'usa'
+    rgw_zonegroup: 'alaska'
+    rgw_zone: 'anchorage'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8081
+    rgw_zone_user: edward.lewis
+    rgw_zone_user_display_name: "Edward Lewis"
+    system_access_key: yu17wkvAx3B8Wyn08XoF
+    system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY
+    endpoint: http://192.168.105.11:8081
+# functional testing
+rgw_multisite_endpoint_addr: 192.168.107.11
+radosgw_num_instances: 2

tests/functional/rgw-multisite/container/secondary/hosts

@@ -5,4 +5,4 @@ mon0
 osd0
 
 [rgws]
-osd0 copy_admin_key=True rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zonegroupmaster=True rgw_zone_user=zone.user rgw_zone_user_display_name="Zone User" rgw_multisite_proto=http rgw_realm=milkyway system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.105.11 rgw_multisite_endpoint_addr=192.168.107.11
+osd0

tests/functional/rgw-multisite/group_vars/rgws

@@ -2,17 +2,6 @@
 copy_admin_key: true
 # Enable Multisite support
 rgw_multisite: true
-rgw_zone: jupiter
-rgw_zonemaster: true
-rgw_zonesecondary: false
-rgw_zonegroup: solarsystem
-rgw_zonegroupmaster: True
-rgw_zone_user: zone.user
-rgw_zone_user_display_name: "Zone User"
-rgw_multisite_proto: http
-rgw_realm: milkyway
-system_access_key: 6kWkikvapSnHyE22P7nO
-system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt
 rgw_create_pools:
   foo:
     pg_num: 16
@@ -21,3 +10,7 @@ rgw_create_pools:
     pg_num: 16
 rgw_override_bucket_index_max_shards: 16
 rgw_bucket_default_quota_max_objects: 1638400
+rgw_zonemaster: True
+rgw_zonesecondary: False
+rgw_zonegroupmaster: True
+rgw_multisite_proto: http

tests/functional/rgw-multisite/host_vars/osd0

@@ -0,0 +1,24 @@
+rgw_instances:
+  - instance_name: 'rgw0'
+    rgw_realm: 'france'
+    rgw_zonegroup: 'idf'
+    rgw_zone: 'paris'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8080
+    rgw_zone_user: jacques.chirac
+    rgw_zone_user_display_name: "Jacques Chirac"
+    system_access_key: P9Eb6S8XNyo4dtZZUUMy
+    system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB
+  - instance_name: 'rgw1'
+    rgw_realm: 'usa'
+    rgw_zonegroup: 'alaska'
+    rgw_zone: 'juneau'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8081
+    rgw_zone_user: edward.lewis
+    rgw_zone_user_display_name: "Edward Lewis"
+    system_access_key: yu17wkvAx3B8Wyn08XoF
+    system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY
+# functional testing
+rgw_multisite_endpoint_addr: 192.168.101.11
+radosgw_num_instances: 2

tests/functional/rgw-multisite/hosts

@@ -5,4 +5,4 @@ mon0
 osd0
 
 [rgws]
-osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.101.11
+osd0

tests/functional/rgw-multisite/secondary/group_vars/rgws

@@ -1,4 +1,6 @@
 ---
+# Enable Multisite support
+rgw_multisite: true
 rgw_create_pools:
   foo:
     pg_num: 16
@@ -7,3 +9,7 @@ rgw_create_pools:
     pg_num: 16
 rgw_override_bucket_index_max_shards: 16
 rgw_bucket_default_quota_max_objects: 1638400
+rgw_zonemaster: False
+rgw_zonesecondary: True
+rgw_zonegroupmaster: True
+rgw_multisite_proto: http

tests/functional/rgw-multisite/secondary/host_vars/osd0

@@ -0,0 +1,26 @@
+rgw_instances:
+  - instance_name: 'rgw0'
+    rgw_realm: 'france'
+    rgw_zonegroup: 'idf'
+    rgw_zone: 'versailles'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8080
+    rgw_zone_user: jacques.chirac
+    rgw_zone_user_display_name: "Jacques Chirac"
+    system_access_key: P9Eb6S8XNyo4dtZZUUMy
+    system_secret_key: qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB
+    endpoint: http://192.168.101.11:8080
+  - instance_name: 'rgw1'
+    rgw_realm: 'usa'
+    rgw_zonegroup: 'alaska'
+    rgw_zone: 'anchorage'
+    radosgw_address: "{{ _radosgw_address }}"
+    radosgw_frontend_port: 8081
+    rgw_zone_user: edward.lewis
+    rgw_zone_user_display_name: "Edward Lewis"
+    system_access_key: yu17wkvAx3B8Wyn08XoF
+    system_secret_key: 5YZfaSUPqxSNIkZQQA3lBZ495hnIV6k2HAz710BY
+    endpoint: http://192.168.101.11:8081
+# functional testing
+rgw_multisite_endpoint_addr: 192.168.103.11
+radosgw_num_instances: 2

tests/functional/rgw-multisite/secondary/hosts

@@ -5,4 +5,4 @@ mon0
 osd0
 
 [rgws]
-osd0 rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zonegroupmaster=True rgw_zone_user=zone.user rgw_zone_user_display_name="Zone User" rgw_realm=milkyway rgw_multisite_proto=http system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.101.11 rgw_multisite_endpoint_addr=192.168.103.11
+osd0

tests/functional/rgw_multisite.yml

@@ -3,7 +3,7 @@
   gather_facts: True
   become: True
   vars:
-    s3cmd_cmd: "s3cmd --no-ssl --access_key={{ system_access_key }} --secret_key={{ system_secret_key }} --host={{ rgw_multisite_endpoint_addr }}:8080 --host-bucket={{ rgw_multisite_endpoint_addr }}:8080"
+    s3cmd_cmd: "s3cmd --no-ssl --access_key=P9Eb6S8XNyo4dtZZUUMy --secret_key=qqHCUtfdNnpHq3PZRHW5un9l0bEBM812Uhow0XfB --host={{ rgw_multisite_endpoint_addr }}:8080 --host-bucket={{ rgw_multisite_endpoint_addr }}:8080"
   tasks:
     - name: install s3cmd
       package: