From 37f46a8c5de9585c2639cc4741ee8f62bc2c854b Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Thu, 31 Jan 2019 20:43:21 +0000 Subject: [PATCH] rgw multisite: add more than 1 rgw to the master or secondary zone Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1664869 Signed-off-by: Ali Maredia --- README-MULTISITE.md | 31 ++++++++++++++----- group_vars/all.yml.sample | 13 ++++++-- group_vars/rhcs.yml.sample | 13 ++++++-- roles/ceph-defaults/defaults/main.yml | 13 ++++++-- roles/ceph-rgw/tasks/multisite/master.yml | 13 ++++++-- roles/ceph-rgw/tasks/multisite/secondary.yml | 11 ++++++- .../rgw-multisite/container/group_vars/rgws | 3 +- .../functional/rgw-multisite/container/hosts | 2 +- .../rgw-multisite/container/secondary/hosts | 2 +- .../functional/rgw-multisite/group_vars/rgws | 3 +- tests/functional/rgw-multisite/hosts | 2 +- .../functional/rgw-multisite/secondary/hosts | 2 +- 12 files changed, 86 insertions(+), 22 deletions(-) diff --git a/README-MULTISITE.md b/README-MULTISITE.md index a921a35b8..ef4dc4b4d 100644 --- a/README-MULTISITE.md +++ b/README-MULTISITE.md @@ -33,7 +33,9 @@ rgw_multisite: true rgw_zone: jupiter rgw_zonemaster: true rgw_zonesecondary: false +rgw_multisite_proto: "http" rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" +rgw_multisite_endpoints_list: "{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}" rgw_zonegroup: solarsystem rgw_zone_user: zone.user rgw_realm: milkyway @@ -41,9 +43,17 @@ system_access_key: 6kWkikvapSnHyE22P7nO system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt ``` -**Note:** rgw_zonemaster should have the value of true and rgw_zonesecondary should be false -**Note:** replace the system_access_key and system_secret_key values with the ones you generated -**Note:** `ansible_fqdn` domain name assigned to rgw_multisite_endpoint_addr must be resolvable from the secondary Ceph cluster's mon and rgw node(s) +**Note:** `rgw_zonemaster` should have the value of `true` and `rgw_zonesecondary` should be `false` + +**Note:** replace the `system_access_key` and `system_secret_key` values with the ones you generated + +**Note:** `ansible_fqdn` domain name assigned to `rgw_multisite_endpoint_addr` must be resolvable from the secondary Ceph clusters mon and rgw node(s) + +**Note:** if there is more than 1 RGW in the cluster, `rgw_multisite_endpoints` needs to be set.
+`rgw_multisite_endpoints` is a comma seperated list, with no spaces, of the RGW endpoints in the format:
+`{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}`
+for example: `rgw_multisite_endpoints: http://foo.example.com:8080,http://bar.example.com:8080,http://baz.example.com:8080` + 3. Run the ceph-ansible playbook on your 1st cluster @@ -58,7 +68,9 @@ rgw_multisite: true rgw_zone: mars rgw_zonemaster: false rgw_zonesecondary: true +rgw_multisite_proto: "http" rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" +rgw_multisite_endpoints_list: "{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}" rgw_zonegroup: solarsystem rgw_zone_user: zone.user rgw_realm: milkyway @@ -69,10 +81,15 @@ rgw_pull_port: 8080 rgw_pullhost: cluster0-rgw0 ``` -**Note:** rgw_zonemaster should have the value of false and rgw_zonesecondary should be true -**Note:** pullhost should be the rgw_multisite_endpoint_addr of the RGW that is configured as the Zone Master -**Note:** rgw_zone_user, system_access_key, and system_secret_key should match what you used in the master cluster -**Note:** `ansible_fqdn` domain name assigned to rgw_multisite_endpoint_addr must be resolvable from the master Ceph cluster's mon and rgw node(s) +**Note:** `rgw_zonemaster` should have the value of `false` and `rgw_zonesecondary` should be `true` + +**Note:** `rgw_pullhost` should be the `rgw_multisite_endpoint_addr` of the RGW that is configured in the Primary Cluster + +**Note:** `rgw_zone_user`, `system_access_key`, and `system_secret_key` should match what you used in the Primary Cluster + +**Note:** `ansible_fqdn` domain name assigned to `rgw_multisite_endpoint_addr` must be resolvable from the Primary Ceph cluster's mon and rgw node(s) + +**Note:** if there is more than 1 RGW in the Secondary Cluster, `rgw_multisite_endpoints` needs to be set with the RGWs in the Secondary Cluster just like it was set in the Primary Cluster 5. Run the ceph-ansible playbook on your 2nd cluster diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index b28c78034..bdaeea978 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -478,13 +478,22 @@ dummy: #rgw_multisite: false # The following Multi-site related variables should be set by the user. +# +# If there is more than 1 RGW in a master or secondary cluster than rgw_multisite_endpoints needs to be a comma seperated list (with no spaces) of the RGW endpoints in the format: +# {{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} +# ex: rgw_multisite_endpoints: http://foo.example.com:8080,http://bar.example.com:8080,http://baz.example.com:8080 +# +# If there is only 1 RGW in the inventory, rgw_multisite_endpoints does not need to change +# # rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site # If multisite is configured rgw_zone should not be set to "default". See README-MULTISITE.md for an example. #rgw_zone: default #rgw_zonemaster: true #rgw_zonesecondary: false +#rgw_multisite_proto: "http" #rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" +#rgw_multisite_endpoints_list: "{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}" #rgw_zonegroup: solarsystem # should be set by the user #rgw_zone_user: zone.user #rgw_realm: milkyway # should be set by the user @@ -493,8 +502,8 @@ dummy: # Multi-site remote pull URL variables #rgw_pull_port: "{{ radosgw_civetweb_port }}" -#rgw_pull_proto: "http" -#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master cluster +#rgw_pull_proto: "http" # should be the same as rgw_multisite_proto for the master zone cluster +#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master zone cluster ################### diff --git a/group_vars/rhcs.yml.sample b/group_vars/rhcs.yml.sample index 28b745834..124f3e4c4 100644 --- a/group_vars/rhcs.yml.sample +++ b/group_vars/rhcs.yml.sample @@ -478,13 +478,22 @@ ceph_rhcs_version: 3 #rgw_multisite: false # The following Multi-site related variables should be set by the user. +# +# If there is more than 1 RGW in a master or secondary cluster than rgw_multisite_endpoints needs to be a comma seperated list (with no spaces) of the RGW endpoints in the format: +# {{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} +# ex: rgw_multisite_endpoints: http://foo.example.com:8080,http://bar.example.com:8080,http://baz.example.com:8080 +# +# If there is only 1 RGW in the inventory, rgw_multisite_endpoints does not need to change +# # rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site # If multisite is configured rgw_zone should not be set to "default". See README-MULTISITE.md for an example. #rgw_zone: default #rgw_zonemaster: true #rgw_zonesecondary: false +#rgw_multisite_proto: "http" #rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" +#rgw_multisite_endpoints_list: "{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}" #rgw_zonegroup: solarsystem # should be set by the user #rgw_zone_user: zone.user #rgw_realm: milkyway # should be set by the user @@ -493,8 +502,8 @@ ceph_rhcs_version: 3 # Multi-site remote pull URL variables #rgw_pull_port: "{{ radosgw_civetweb_port }}" -#rgw_pull_proto: "http" -#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master cluster +#rgw_pull_proto: "http" # should be the same as rgw_multisite_proto for the master zone cluster +#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master zone cluster ################### diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 798e90649..db56b0877 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -470,13 +470,22 @@ nfs_obj_gw: true rgw_multisite: false # The following Multi-site related variables should be set by the user. +# +# If there is more than 1 RGW in a master or secondary cluster than rgw_multisite_endpoints needs to be a comma seperated list (with no spaces) of the RGW endpoints in the format: +# {{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }} +# ex: rgw_multisite_endpoints: http://foo.example.com:8080,http://bar.example.com:8080,http://baz.example.com:8080 +# +# If there is only 1 RGW in the inventory, rgw_multisite_endpoints does not need to change +# # rgw_zone is set to "default" to enable compression for clusters configured without rgw multi-site # If multisite is configured rgw_zone should not be set to "default". See README-MULTISITE.md for an example. rgw_zone: default rgw_zonemaster: true rgw_zonesecondary: false +rgw_multisite_proto: "http" rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" +#rgw_multisite_endpoints_list: "{{ rgw_multisite_proto }}://{{ ansible_fqdn }}:{{ radosgw_frontend_port }}" #rgw_zonegroup: solarsystem # should be set by the user #rgw_zone_user: zone.user #rgw_realm: milkyway # should be set by the user @@ -485,8 +494,8 @@ rgw_multisite_endpoint_addr: "{{ ansible_fqdn }}" # Multi-site remote pull URL variables rgw_pull_port: "{{ radosgw_civetweb_port }}" -rgw_pull_proto: "http" -#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master cluster +rgw_pull_proto: "http" # should be the same as rgw_multisite_proto for the master zone cluster +#rgw_pullhost: localhost # rgw_pullhost only needs to be declared if there is a zone secondary. It should be the same as rgw_multisite_endpoint_addr for the master zone cluster ################### diff --git a/roles/ceph-rgw/tasks/multisite/master.yml b/roles/ceph-rgw/tasks/multisite/master.yml index 8982e960e..26449f3f3 100644 --- a/roles/ceph-rgw/tasks/multisite/master.yml +++ b/roles/ceph-rgw/tasks/multisite/master.yml @@ -7,14 +7,14 @@ - "'No such file or directory' in realmcheck.stderr" - name: create the zonegroup - command: "{{ docker_exec_cmd }} radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --master --default" + command: "{{ docker_exec_cmd }} radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints={{ rgw_multisite_proto }}://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --master --default" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: - "'No such file or directory' in zonegroupcheck.stderr" - name: create the zone - command: "{{ docker_exec_cmd }} radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master" + command: "{{ docker_exec_cmd }} radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints={{ rgw_multisite_proto }}://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: @@ -28,3 +28,12 @@ - "'could not fetch user info: no user info saved' in usercheck.stderr" notify: - update period + +- name: add other endpoints to the zone + command: "{{ docker_exec_cmd }} radosgw-admin zone modify --rgw-zone={{ rgw_zone }} --endpoints {{ rgw_multisite_endpoints_list }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true + when: + - rgw_multisite_endpoints_list is defined + notify: + - update period diff --git a/roles/ceph-rgw/tasks/multisite/secondary.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml index 413749c9d..24a44cf46 100644 --- a/roles/ceph-rgw/tasks/multisite/secondary.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -26,10 +26,19 @@ run_once: true - name: create the zone - command: "{{ docker_exec_cmd }} radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default" + command: "{{ docker_exec_cmd }} radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints={{ rgw_multisite_proto }}://{{ rgw_multisite_endpoint_addr }}:{{ radosgw_frontend_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default" delegate_to: "{{ groups[mon_group_name][0] }}" run_once: true when: - "'No such file or directory' in zonecheck.stderr" notify: - update period + +- name: add other endpoints to the zone + command: "{{ docker_exec_cmd }} radosgw-admin zone modify --rgw-zone={{ rgw_zone }} --endpoints {{ rgw_multisite_endpoints_list }}" + delegate_to: "{{ groups[mon_group_name][0] }}" + run_once: true + when: + - rgw_multisite_endpoints_list is defined + notify: + - update period diff --git a/tests/functional/rgw-multisite/container/group_vars/rgws b/tests/functional/rgw-multisite/container/group_vars/rgws index 6ecc36f69..23b6f80d8 100644 --- a/tests/functional/rgw-multisite/container/group_vars/rgws +++ b/tests/functional/rgw-multisite/container/group_vars/rgws @@ -5,6 +5,7 @@ rgw_multisite: true rgw_zone: jupiter rgw_zonemaster: true rgw_zonesecondary: false +rgw_multisite_proto: http rgw_zonegroup: solarsystem rgw_zone_user: zone.user rgw_realm: milkyway @@ -16,4 +17,4 @@ rgw_create_pools: bar: pg_num: 19 rgw_override_bucket_index_max_shards: 16 -rgw_bucket_default_quota_max_objects: 1638400 \ No newline at end of file +rgw_bucket_default_quota_max_objects: 1638400 diff --git a/tests/functional/rgw-multisite/container/hosts b/tests/functional/rgw-multisite/container/hosts index 3901c5a3f..6c57190d7 100644 --- a/tests/functional/rgw-multisite/container/hosts +++ b/tests/functional/rgw-multisite/container/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite_endpoint_addr=192.168.105.100 +osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.105.100 diff --git a/tests/functional/rgw-multisite/container/secondary/hosts b/tests/functional/rgw-multisite/container/secondary/hosts index e2e0c30e4..7ba05df9a 100644 --- a/tests/functional/rgw-multisite/container/secondary/hosts +++ b/tests/functional/rgw-multisite/container/secondary/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 copy_admin_key=True rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zone_user=zone.user rgw_realm=milkyway rgw_multisite_endpoint_addr=192.168.107.100 system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.105.100 +osd0 copy_admin_key=True rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zone_user=zone.user rgw_realm=milkyway rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.107.100 system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.105.100 diff --git a/tests/functional/rgw-multisite/group_vars/rgws b/tests/functional/rgw-multisite/group_vars/rgws index 6ecc36f69..23b6f80d8 100644 --- a/tests/functional/rgw-multisite/group_vars/rgws +++ b/tests/functional/rgw-multisite/group_vars/rgws @@ -5,6 +5,7 @@ rgw_multisite: true rgw_zone: jupiter rgw_zonemaster: true rgw_zonesecondary: false +rgw_multisite_proto: http rgw_zonegroup: solarsystem rgw_zone_user: zone.user rgw_realm: milkyway @@ -16,4 +17,4 @@ rgw_create_pools: bar: pg_num: 19 rgw_override_bucket_index_max_shards: 16 -rgw_bucket_default_quota_max_objects: 1638400 \ No newline at end of file +rgw_bucket_default_quota_max_objects: 1638400 diff --git a/tests/functional/rgw-multisite/hosts b/tests/functional/rgw-multisite/hosts index 245062489..48c12701d 100644 --- a/tests/functional/rgw-multisite/hosts +++ b/tests/functional/rgw-multisite/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite_endpoint_addr=192.168.101.100 +osd0 rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.101.100 diff --git a/tests/functional/rgw-multisite/secondary/hosts b/tests/functional/rgw-multisite/secondary/hosts index 2e1a40ad3..1e119a24f 100644 --- a/tests/functional/rgw-multisite/secondary/hosts +++ b/tests/functional/rgw-multisite/secondary/hosts @@ -5,4 +5,4 @@ mon0 osd0 [rgws] -osd0 rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zone_user=zone.user rgw_realm=milkyway rgw_multisite_endpoint_addr=192.168.103.100 system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.101.100 +osd0 rgw_multisite=True rgw_zone=mars rgw_zonemaster=False rgw_zonesecondary=True rgw_zonegroup=solarsystem rgw_zone_user=zone.user rgw_realm=milkyway rgw_multisite_proto=http rgw_multisite_endpoint_addr=192.168.103.100 system_access_key=6kWkikvapSnHyE22P7nO system_secret_key=MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt rgw_pull_proto=http rgw_pull_port=8080 rgw_pullhost=192.168.101.100