From 183c6eae9fb2e3c6f85013d28559e93ee3de851c Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Sun, 7 Aug 2016 23:16:15 -0400 Subject: [PATCH 1/6] Added initial multisite support --- roles/ceph-rgw/handlers/main.yml | 3 ++ roles/ceph-rgw/handlers/multisite.yml | 3 ++ roles/ceph-rgw/tasks/main.yml | 3 ++ roles/ceph-rgw/tasks/multisite.yml | 52 +++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 roles/ceph-rgw/handlers/main.yml create mode 100644 roles/ceph-rgw/handlers/multisite.yml create mode 100644 roles/ceph-rgw/tasks/multisite.yml diff --git a/roles/ceph-rgw/handlers/main.yml b/roles/ceph-rgw/handlers/main.yml new file mode 100644 index 000000000..a448e19bd --- /dev/null +++ b/roles/ceph-rgw/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- include: multisite.yml + when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) diff --git a/roles/ceph-rgw/handlers/multisite.yml b/roles/ceph-rgw/handlers/multisite.yml new file mode 100644 index 000000000..ccb822659 --- /dev/null +++ b/roles/ceph-rgw/handlers/multisite.yml @@ -0,0 +1,3 @@ +--- +- name: update period + command: radosgw-admin period update --commit diff --git a/roles/ceph-rgw/tasks/main.yml b/roles/ceph-rgw/tasks/main.yml index 91c8c9f79..f79ee73aa 100644 --- a/roles/ceph-rgw/tasks/main.yml +++ b/roles/ceph-rgw/tasks/main.yml @@ -5,6 +5,9 @@ - include: openstack-keystone.yml when: radosgw_keystone +- include: multisite.yml + when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) + - include: start_radosgw.yml when: not rgw_containerized_deployment diff --git a/roles/ceph-rgw/tasks/multisite.yml b/roles/ceph-rgw/tasks/multisite.yml new file mode 100644 index 000000000..d4afe069e --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite.yml @@ -0,0 +1,52 @@ +--- +- name: Check if the realm already exists + command: radosgw-admin realm get --rgw-realm={{ rgw_realm }} + register: realmcheck + failed_when: False + changed_when: False + +- name: Create the realm + command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default + run_once: true + when: ('No such file or directory' in realmcheck.stderr) + notify: + - update period + +- name: Check if the zonegroup already exists + command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }} + register: zonegroupcheck + failed_when: False + changed_when: False + +- name: Create the zonegroup + command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default + run_once: true + when: ('No such file or directory' in zonegroupcheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Check if the zone already exists + command: radosgw-admin zone get --rgw-zone={{ rgw_zone }} + register: zonecheck + failed_when: False + changed_when: False + +- name: Create the zone + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master + run_once: true + when: ('No such file or directory' in zonecheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Check if the system user already exists + command: radosgw-admin user info --uid=zone.user + register: usercheck + failed_when: False + changed_when: False + +- name: Create the zone user + command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system + run_once: true + when: "'could not fetch user info: no user info saved' in usercheck.stderr" + notify: + - update period From 402a597fe6d06aa83d82e8751263e82e8fb80d23 Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Sun, 7 Aug 2016 23:16:15 -0400 Subject: [PATCH 2/6] Added initial multisite support --- roles/ceph-rgw/handlers/main.yml | 3 ++ roles/ceph-rgw/handlers/multisite.yml | 3 ++ roles/ceph-rgw/tasks/main.yml | 3 ++ roles/ceph-rgw/tasks/multisite.yml | 52 +++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 roles/ceph-rgw/handlers/main.yml create mode 100644 roles/ceph-rgw/handlers/multisite.yml create mode 100644 roles/ceph-rgw/tasks/multisite.yml diff --git a/roles/ceph-rgw/handlers/main.yml b/roles/ceph-rgw/handlers/main.yml new file mode 100644 index 000000000..a448e19bd --- /dev/null +++ b/roles/ceph-rgw/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- include: multisite.yml + when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) diff --git a/roles/ceph-rgw/handlers/multisite.yml b/roles/ceph-rgw/handlers/multisite.yml new file mode 100644 index 000000000..ccb822659 --- /dev/null +++ b/roles/ceph-rgw/handlers/multisite.yml @@ -0,0 +1,3 @@ +--- +- name: update period + command: radosgw-admin period update --commit diff --git a/roles/ceph-rgw/tasks/main.yml b/roles/ceph-rgw/tasks/main.yml index 91c8c9f79..f79ee73aa 100644 --- a/roles/ceph-rgw/tasks/main.yml +++ b/roles/ceph-rgw/tasks/main.yml @@ -5,6 +5,9 @@ - include: openstack-keystone.yml when: radosgw_keystone +- include: multisite.yml + when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) + - include: start_radosgw.yml when: not rgw_containerized_deployment diff --git a/roles/ceph-rgw/tasks/multisite.yml b/roles/ceph-rgw/tasks/multisite.yml new file mode 100644 index 000000000..d4afe069e --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite.yml @@ -0,0 +1,52 @@ +--- +- name: Check if the realm already exists + command: radosgw-admin realm get --rgw-realm={{ rgw_realm }} + register: realmcheck + failed_when: False + changed_when: False + +- name: Create the realm + command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default + run_once: true + when: ('No such file or directory' in realmcheck.stderr) + notify: + - update period + +- name: Check if the zonegroup already exists + command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }} + register: zonegroupcheck + failed_when: False + changed_when: False + +- name: Create the zonegroup + command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default + run_once: true + when: ('No such file or directory' in zonegroupcheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Check if the zone already exists + command: radosgw-admin zone get --rgw-zone={{ rgw_zone }} + register: zonecheck + failed_when: False + changed_when: False + +- name: Create the zone + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master + run_once: true + when: ('No such file or directory' in zonecheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Check if the system user already exists + command: radosgw-admin user info --uid=zone.user + register: usercheck + failed_when: False + changed_when: False + +- name: Create the zone user + command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system + run_once: true + when: "'could not fetch user info: no user info saved' in usercheck.stderr" + notify: + - update period From a4c7ef3cc581c2185f8376198fffe2a171f1a30c Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Wed, 10 Aug 2016 17:05:27 -0400 Subject: [PATCH 3/6] Initial support for RGW Multisite --- roles/ceph-common/templates/ceph.conf.j2 | 3 ++ roles/ceph-rgw/handlers/main.yml | 4 +-- roles/ceph-rgw/tasks/main.yml | 2 +- roles/ceph-rgw/tasks/multisite/checks.yml | 24 +++++++++++++++ roles/ceph-rgw/tasks/multisite/destroy.yml | 29 +++++++++++++++++++ roles/ceph-rgw/tasks/multisite/main.yml | 19 ++++++++++++ roles/ceph-rgw/tasks/multisite/master.yml | 28 ++++++++++++++++++ .../secondary.yml} | 28 ++++++++++++++++-- 8 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 roles/ceph-rgw/tasks/multisite/checks.yml create mode 100644 roles/ceph-rgw/tasks/multisite/destroy.yml create mode 100644 roles/ceph-rgw/tasks/multisite/main.yml create mode 100644 roles/ceph-rgw/tasks/multisite/master.yml rename roles/ceph-rgw/tasks/{multisite.yml => multisite/secondary.yml} (61%) diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index aa383c217..14c5e0bb3 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -104,6 +104,9 @@ rgw keystone revocation interval = {{ radosgw_keystone_revocation_internal }} rgw s3 auth use keystone = {{ radosgw_s3_auth_use_keystone }} nss db path = {{ radosgw_nss_db_path }} {% endif %} +{% if rgw_zonegroup is defined and rgw_zone is defined %} +rgw zone = {{ rgw_zonegroup }}-{{ rgw_zone }} +{% endif %} {% endif %} {% endfor %} {% endif %} diff --git a/roles/ceph-rgw/handlers/main.yml b/roles/ceph-rgw/handlers/main.yml index a448e19bd..ccb822659 100644 --- a/roles/ceph-rgw/handlers/main.yml +++ b/roles/ceph-rgw/handlers/main.yml @@ -1,3 +1,3 @@ --- -- include: multisite.yml - when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) +- name: update period + command: radosgw-admin period update --commit diff --git a/roles/ceph-rgw/tasks/main.yml b/roles/ceph-rgw/tasks/main.yml index f79ee73aa..2388a5124 100644 --- a/roles/ceph-rgw/tasks/main.yml +++ b/roles/ceph-rgw/tasks/main.yml @@ -5,7 +5,7 @@ - include: openstack-keystone.yml when: radosgw_keystone -- include: multisite.yml +- include: ./multisite/main.yml when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) - include: start_radosgw.yml diff --git a/roles/ceph-rgw/tasks/multisite/checks.yml b/roles/ceph-rgw/tasks/multisite/checks.yml new file mode 100644 index 000000000..849e1ff33 --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite/checks.yml @@ -0,0 +1,24 @@ +--- +- name: Check if the realm already exists + command: radosgw-admin realm get --rgw-realm={{ rgw_realm }} + register: realmcheck + failed_when: False + changed_when: False + +- name: Check if the zonegroup already exists + command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }} + register: zonegroupcheck + failed_when: False + changed_when: False + +- name: Check if the zone already exists + command: radosgw-admin zone get --rgw-zone={{ rgw_zone }} + register: zonecheck + failed_when: False + changed_when: False + +- name: Check if the system user already exists + command: radosgw-admin user info --uid=zone.user + register: usercheck + failed_when: False + changed_when: False diff --git a/roles/ceph-rgw/tasks/multisite/destroy.yml b/roles/ceph-rgw/tasks/multisite/destroy.yml new file mode 100644 index 000000000..baae33a16 --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite/destroy.yml @@ -0,0 +1,29 @@ +--- +- name: Delete the zone user + command: radosgw-admin user rm --uid=zone.user + run_once: true + failed_when: false + notify: + - update period + +- name: Delete the zone + command: radosgw-admin zone delete --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} + run_once: true + failed_when: false + notify: + - update period + +- name: Delete the zonegroup + command: radosgw-admin zonegroup delete --rgw-zonegroup={{ rgw_zonegroup }} + run_once: true + failed_when: false + notify: + - update period + +- name: Delete the realm + command: radosgw-admin realm delete --rgw-realm={{ rgw_realm }} + run_once: true + failed_when: false + notify: + - update period + diff --git a/roles/ceph-rgw/tasks/multisite/main.yml b/roles/ceph-rgw/tasks/multisite/main.yml new file mode 100644 index 000000000..44c696b90 --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite/main.yml @@ -0,0 +1,19 @@ +--- +- include: checks.yml + +# Include the tasks depending on the zone type +- include: master.yml + when: "{{ rgw_zonemaster }} is defined and {{ rgw_zonemaster }}" + +- include: secondary.yml + when: "{{ rgw_zonesecondary }} is defined and {{ rgw_zonesecondary }}" + +## This is currently handled in the ceph-common ceph.conf template +## Probably should do here so we can use a restart handler for the rgw +#- name: Add zone to RGW stanza in ceph.conf +# lineinfile: +# dest: /etc/ceph/ceph.conf +# regexp: "{{ ansible_host }}" +# insertafter: "^[client.rgw.{{ ansible_host }}]" +# line: "rgw_zone={{ rgw_zonegroup }}-{{ rgw_zone }}" +# state: present diff --git a/roles/ceph-rgw/tasks/multisite/master.yml b/roles/ceph-rgw/tasks/multisite/master.yml new file mode 100644 index 000000000..ccfcf962b --- /dev/null +++ b/roles/ceph-rgw/tasks/multisite/master.yml @@ -0,0 +1,28 @@ +--- +- name: Create the realm + command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default + run_once: true + when: ("No such file or directory" in realmcheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Create the zonegroup + command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default + run_once: true + when: ("No such file or directory" in zonegroupcheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Create the zone + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master + run_once: true + when: ("No such file or directory" in zonecheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Create the zone user + command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system + run_once: true + when: "'could not fetch user info: no user info saved' in usercheck.stderr" + notify: + - update period diff --git a/roles/ceph-rgw/tasks/multisite.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml similarity index 61% rename from roles/ceph-rgw/tasks/multisite.yml rename to roles/ceph-rgw/tasks/multisite/secondary.yml index d4afe069e..3c53f63cd 100644 --- a/roles/ceph-rgw/tasks/multisite.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -8,7 +8,21 @@ - name: Create the realm command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default run_once: true - when: ('No such file or directory' in realmcheck.stderr) + when: ("No such file or directory" in realmcheck.stderr) and rgw_zonemaster + notify: + - update period + +- name: Fetch the realm (Secondary) + command: radosgw-admin realm pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} + run_once: true + when: ("No such file or directory" in realmcheck.stderr) and rgw_zonesecondary + notify: + - update period + +- name: Fetch the realm (Secondary) + command: radosgw-admin period pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} + run_once: true + when: ("No such file or directory" in realmcheck.stderr) and rgw_zonesecondary notify: - update period @@ -21,7 +35,7 @@ - name: Create the zonegroup command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default run_once: true - when: ('No such file or directory' in zonegroupcheck.stderr) and rgw_zonemaster + when: ("No such file or directory" in zonegroupcheck.stderr) and rgw_zonemaster notify: - update period @@ -34,7 +48,7 @@ - name: Create the zone command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master run_once: true - when: ('No such file or directory' in zonecheck.stderr) and rgw_zonemaster + when: ("No such file or directory" in zonecheck.stderr) and rgw_zonemaster notify: - update period @@ -50,3 +64,11 @@ when: "'could not fetch user info: no user info saved' in usercheck.stderr" notify: - update period + +#- name: Add zone to RGW stanza in ceph.conf +# lineinfile: +# dest: /etc/ceph/ceph.conf +# regexp: "{{ ansible_host }}" +# insertafter: "^[client.rgw.{{ ansible_host }}]" +# line: "rgw_zone={{ rgw_zonegroup }}-{{ rgw_zone }}" +# state: present From 052e4cc6876d6101088acc1244a0769dafb0adcc Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Wed, 10 Aug 2016 18:03:09 -0400 Subject: [PATCH 4/6] code cleanup --- roles/ceph-rgw/handlers/multisite.yml | 3 -- roles/ceph-rgw/tasks/multisite/secondary.yml | 57 +++++--------------- 2 files changed, 12 insertions(+), 48 deletions(-) delete mode 100644 roles/ceph-rgw/handlers/multisite.yml diff --git a/roles/ceph-rgw/handlers/multisite.yml b/roles/ceph-rgw/handlers/multisite.yml deleted file mode 100644 index ccb822659..000000000 --- a/roles/ceph-rgw/handlers/multisite.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- name: update period - command: radosgw-admin period update --commit diff --git a/roles/ceph-rgw/tasks/multisite/secondary.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml index 3c53f63cd..fac7a703b 100644 --- a/roles/ceph-rgw/tasks/multisite/secondary.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -1,67 +1,34 @@ --- -- name: Check if the realm already exists - command: radosgw-admin realm get --rgw-realm={{ rgw_realm }} - register: realmcheck - failed_when: False - changed_when: False - -- name: Create the realm - command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default - run_once: true - when: ("No such file or directory" in realmcheck.stderr) and rgw_zonemaster - notify: - - update period - - name: Fetch the realm (Secondary) command: radosgw-admin realm pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true - when: ("No such file or directory" in realmcheck.stderr) and rgw_zonesecondary + when: ("No such file or directory" in realmcheck.stderr) notify: - update period - name: Fetch the realm (Secondary) command: radosgw-admin period pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true - when: ("No such file or directory" in realmcheck.stderr) and rgw_zonesecondary + when: ("No such file or directory" in realmcheck.stderr) notify: - update period -- name: Check if the zonegroup already exists - command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }} - register: zonegroupcheck - failed_when: False - changed_when: False - -- name: Create the zonegroup - command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default +- name: Set default realm + command: radosgw-admin realm default --rgw-realm={{ rgw_realm }} run_once: true - when: ("No such file or directory" in zonegroupcheck.stderr) and rgw_zonemaster notify: - - update period + - update period # Might not need to update period here -- name: Check if the zone already exists - command: radosgw-admin zone get --rgw-zone={{ rgw_zone }} - register: zonecheck - failed_when: False - changed_when: False +- name: Set default zonegroup + command: radosgw-admin zonegroup default --rgw-zonegroup={{ rgw_zonegroup }} + run_once: true + notify: + - update period # Might not need to update period here - name: Create the zone - command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default run_once: true - when: ("No such file or directory" in zonecheck.stderr) and rgw_zonemaster - notify: - - update period - -- name: Check if the system user already exists - command: radosgw-admin user info --uid=zone.user - register: usercheck - failed_when: False - changed_when: False - -- name: Create the zone user - command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system - run_once: true - when: "'could not fetch user info: no user info saved' in usercheck.stderr" + when: ("No such file or directory" in zonecheck.stderr) notify: - update period From d8609ec6878cc47a3cdd71af6040e6c07bcee9c5 Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Tue, 16 Aug 2016 14:57:01 -0400 Subject: [PATCH 5/6] Experimental RGW Multisite Support --- README-MULTISITE.md | 81 ++++++++++++++++++++ purge-cluster.yml | 1 + purge-multisite.yml | 11 +++ rgw-standalone.yml | 10 +++ roles/ceph-common/templates/ceph.conf.j2 | 3 - roles/ceph-rgw/handlers/main.yml | 5 ++ roles/ceph-rgw/tasks/main.yml | 17 +++- roles/ceph-rgw/tasks/multisite/destroy.yml | 16 ++-- roles/ceph-rgw/tasks/multisite/main.yml | 36 +++++---- roles/ceph-rgw/tasks/multisite/master.yml | 4 +- roles/ceph-rgw/tasks/multisite/secondary.yml | 18 ++--- 11 files changed, 159 insertions(+), 43 deletions(-) create mode 100644 README-MULTISITE.md create mode 100644 purge-multisite.yml create mode 100644 rgw-standalone.yml diff --git a/README-MULTISITE.md b/README-MULTISITE.md new file mode 100644 index 000000000..2056a6b66 --- /dev/null +++ b/README-MULTISITE.md @@ -0,0 +1,81 @@ +RGW Multisite (Experimental) +============================ + +Directions for configuring the RGW Multisite support in ceph-ansible + +## Requirements + +* At least 2 Ceph clusters +* 1 RGW per cluster +* Jewel or newer + +More details: + +* Can configure a Master and Secondary realm/zonegroup/zone on 2 separate clusters. + + +## Configuring the Master Zone in the Primary Cluster + +This will setup the realm, zonegroup and master zone and make them the defaults. It will also reconfigure the specified RGW for use with the zone. + +1. Edit the Inventory File + +``` +[rgws] +cluster0-rgw0 rgw_zone=us-east rgw_zonemaster=true +``` +1. Generate System Access and System Secret Keys + +``` +echo system_access_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 20 | head -n 1) > multi-site-keys.sh +echo system_secret_key: $(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 40 | head -n 1) >> multi-site-keys.sh +``` +1. Edit the RGW Group Vars + +``` +copy_admin_key: true +# Enable Multisite support +rgw_multisite: true +rgw_realm: gold +rgw_zonegroup: us +system_access_key: 6kWkikvapSnHyE22P7nO +system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt +``` + +**Note:** replace the system_access_key and system_secret_key values with the ones you generated + +1. Run the ceph-ansible playbook on your 1st cluster + +Note: If you have already installed a cluster with ceph-ansible, you can use the `rgw-configure.yml` playbook as a shortcut (Only runs the ceph-rgw role) + +## Configuring the Secondary Zone in a Separate Cluster + +``` +[rgws] +cluster1-rgw0 rgw_zone=us-west rgw_zonesecondary=true +``` + +1. Edit the RGW Group Vars + +``` +copy_admin_key: true +# Enable Multisite support +rgw_multisite: true +rgw_realm: gold +rgw_zonegroup: us +rgw_pullhost: cluster1-rgw0.fqdn +system_access_key: 6kWkikvapSnHyE22P7nO +system_secret_key: MGecsMrWtKZgngOHZdrd6d3JxGO5CPWgT2lcnpSt +``` + +**Note:** pullhost should be the host of the RGW that is configured as the Zone Master +**Note:** system_access_key and system_secret_key should match what you used in the 1st cluster + + +1. Run the ceph-ansible playbook on your 2nd cluster + +Note: If you have already installed a cluster with ceph-ansible, you can use the `rgw-configure.yml` playbook as a shortcut (Only runs the ceph-rgw role) + +## Conclusion + +You should now have a master zone on cluster0 and a secondary zone on cluster1 in an Active-Active mode. diff --git a/purge-cluster.yml b/purge-cluster.yml index b2c8c9473..b0c8f48d0 100644 --- a/purge-cluster.yml +++ b/purge-cluster.yml @@ -423,6 +423,7 @@ - name: request data removal local_action: shell echo requesting data removal + become: false notify: - remove data diff --git a/purge-multisite.yml b/purge-multisite.yml new file mode 100644 index 000000000..0677a1491 --- /dev/null +++ b/purge-multisite.yml @@ -0,0 +1,11 @@ +--- +# Nukes a multisite config +- hosts: rgws + become: True + tasks: + - include: ~/ceph-ansible-master/roles/ceph-rgw/tasks/multisite/destroy.yml + + handlers: + - include: ~/ceph-ansible-master/roles/ceph-rgw/handlers/main.yml + # Ansible 2.1.0 bug will ignore included handlers without this + static: True diff --git a/rgw-standalone.yml b/rgw-standalone.yml new file mode 100644 index 000000000..10f1c3582 --- /dev/null +++ b/rgw-standalone.yml @@ -0,0 +1,10 @@ +--- +# Run ceph-rgw role standalone + +# Need to load the facts from mons because ceph-common need them to generate the ceph.conf +- hosts: mons + +- hosts: rgws + become: True + roles: + - ceph-rgw diff --git a/roles/ceph-common/templates/ceph.conf.j2 b/roles/ceph-common/templates/ceph.conf.j2 index 14c5e0bb3..aa383c217 100644 --- a/roles/ceph-common/templates/ceph.conf.j2 +++ b/roles/ceph-common/templates/ceph.conf.j2 @@ -104,9 +104,6 @@ rgw keystone revocation interval = {{ radosgw_keystone_revocation_internal }} rgw s3 auth use keystone = {{ radosgw_s3_auth_use_keystone }} nss db path = {{ radosgw_nss_db_path }} {% endif %} -{% if rgw_zonegroup is defined and rgw_zone is defined %} -rgw zone = {{ rgw_zonegroup }}-{{ rgw_zone }} -{% endif %} {% endif %} {% endfor %} {% endif %} diff --git a/roles/ceph-rgw/handlers/main.yml b/roles/ceph-rgw/handlers/main.yml index ccb822659..3955c8243 100644 --- a/roles/ceph-rgw/handlers/main.yml +++ b/roles/ceph-rgw/handlers/main.yml @@ -1,3 +1,8 @@ --- - name: update period command: radosgw-admin period update --commit + +- name: restart rgw + service: + name: ceph-radosgw@rgw.{{ ansible_host }} + state: restarted diff --git a/roles/ceph-rgw/tasks/main.yml b/roles/ceph-rgw/tasks/main.yml index 2388a5124..fd6e3108e 100644 --- a/roles/ceph-rgw/tasks/main.yml +++ b/roles/ceph-rgw/tasks/main.yml @@ -1,15 +1,26 @@ --- - include: pre_requisite.yml when: not rgw_containerized_deployment + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False - include: openstack-keystone.yml when: radosgw_keystone - -- include: ./multisite/main.yml - when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False - include: start_radosgw.yml when: not rgw_containerized_deployment + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False + +- name: Include rgw multisite playbooks + include: multisite/main.yml + when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False - include: ./docker/main.yml when: rgw_containerized_deployment + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False diff --git a/roles/ceph-rgw/tasks/multisite/destroy.yml b/roles/ceph-rgw/tasks/multisite/destroy.yml index baae33a16..8b608e0e2 100644 --- a/roles/ceph-rgw/tasks/multisite/destroy.yml +++ b/roles/ceph-rgw/tasks/multisite/destroy.yml @@ -3,27 +3,27 @@ command: radosgw-admin user rm --uid=zone.user run_once: true failed_when: false - notify: - - update period - name: Delete the zone command: radosgw-admin zone delete --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} run_once: true failed_when: false - notify: - - update period - name: Delete the zonegroup command: radosgw-admin zonegroup delete --rgw-zonegroup={{ rgw_zonegroup }} run_once: true failed_when: false - notify: - - update period - name: Delete the realm command: radosgw-admin realm delete --rgw-realm={{ rgw_realm }} run_once: true failed_when: false - notify: - - update period +- name: Delete zone from RGW stanza in ceph.conf + lineinfile: + dest: /etc/ceph/ceph.conf + regexp: "rgw_zone = {{ rgw_zonegroup }}-{{ rgw_zone }}" + state: absent + when: "rgw_zone is defined and rgw_zonegroup is defined" + notify: + - restart rgw diff --git a/roles/ceph-rgw/tasks/multisite/main.yml b/roles/ceph-rgw/tasks/multisite/main.yml index 44c696b90..b58073061 100644 --- a/roles/ceph-rgw/tasks/multisite/main.yml +++ b/roles/ceph-rgw/tasks/multisite/main.yml @@ -1,19 +1,27 @@ --- -- include: checks.yml +- name: Include multisite checks + include: checks.yml # Include the tasks depending on the zone type -- include: master.yml - when: "{{ rgw_zonemaster }} is defined and {{ rgw_zonemaster }}" +- name: Include master multisite tasks + include: master.yml + when: "rgw_zonemaster is defined and rgw_zonemaster" + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False -- include: secondary.yml - when: "{{ rgw_zonesecondary }} is defined and {{ rgw_zonesecondary }}" +- name: Include secondary multisite tasks + include: secondary.yml + when: "rgw_zonesecondary is defined and rgw_zonesecondary" + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False -## This is currently handled in the ceph-common ceph.conf template -## Probably should do here so we can use a restart handler for the rgw -#- name: Add zone to RGW stanza in ceph.conf -# lineinfile: -# dest: /etc/ceph/ceph.conf -# regexp: "{{ ansible_host }}" -# insertafter: "^[client.rgw.{{ ansible_host }}]" -# line: "rgw_zone={{ rgw_zonegroup }}-{{ rgw_zone }}" -# state: present +# Continue with common tasks +- name: Add zone to RGW stanza in ceph.conf + lineinfile: + dest: /etc/ceph/ceph.conf + regexp: "{{ ansible_host }}" + insertafter: "^[client.rgw.{{ ansible_host }}]" + line: "rgw_zone = {{ rgw_zone }}" + state: present + notify: + - restart rgw diff --git a/roles/ceph-rgw/tasks/multisite/master.yml b/roles/ceph-rgw/tasks/multisite/master.yml index ccfcf962b..94b5c5e91 100644 --- a/roles/ceph-rgw/tasks/multisite/master.yml +++ b/roles/ceph-rgw/tasks/multisite/master.yml @@ -7,14 +7,14 @@ - update period - name: Create the zonegroup - command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_hostname }}:8080 --master --default + command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --master --default run_once: true when: ("No such file or directory" in zonegroupcheck.stderr) and rgw_zonemaster notify: - update period - name: Create the zone - command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master run_once: true when: ("No such file or directory" in zonecheck.stderr) and rgw_zonemaster notify: diff --git a/roles/ceph-rgw/tasks/multisite/secondary.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml index fac7a703b..dd0f671ee 100644 --- a/roles/ceph-rgw/tasks/multisite/secondary.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -1,13 +1,13 @@ --- -- name: Fetch the realm (Secondary) - command: radosgw-admin realm pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} +- name: Fetch the realm + command: radosgw-admin realm pull --url=http://{{ rgw_pullhost }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true when: ("No such file or directory" in realmcheck.stderr) notify: - update period -- name: Fetch the realm (Secondary) - command: radosgw-admin period pull --url=http://{{ rgw_pullhost }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} +- name: Fetch the period + command: radosgw-admin period pull --url=http://{{ rgw_pullhost }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true when: ("No such file or directory" in realmcheck.stderr) notify: @@ -26,16 +26,8 @@ - update period # Might not need to update period here - name: Create the zone - command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:8080 --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default + command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default run_once: true when: ("No such file or directory" in zonecheck.stderr) notify: - update period - -#- name: Add zone to RGW stanza in ceph.conf -# lineinfile: -# dest: /etc/ceph/ceph.conf -# regexp: "{{ ansible_host }}" -# insertafter: "^[client.rgw.{{ ansible_host }}]" -# line: "rgw_zone={{ rgw_zonegroup }}-{{ rgw_zone }}" -# state: present From 6d55d57fa66da139e0f2a3cf084a0d3e281666ac Mon Sep 17 00:00:00 2001 From: James Saint-Rossy Date: Tue, 16 Aug 2016 17:57:49 -0400 Subject: [PATCH 6/6] fixed name: capitilization --- roles/ceph-rgw/tasks/main.yml | 2 +- roles/ceph-rgw/tasks/multisite/checks.yml | 8 ++++---- roles/ceph-rgw/tasks/multisite/destroy.yml | 10 +++++----- roles/ceph-rgw/tasks/multisite/main.yml | 8 ++++---- roles/ceph-rgw/tasks/multisite/master.yml | 8 ++++---- roles/ceph-rgw/tasks/multisite/secondary.yml | 10 +++++----- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/roles/ceph-rgw/tasks/main.yml b/roles/ceph-rgw/tasks/main.yml index fd6e3108e..ca2f9ea78 100644 --- a/roles/ceph-rgw/tasks/main.yml +++ b/roles/ceph-rgw/tasks/main.yml @@ -14,7 +14,7 @@ # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) static: False -- name: Include rgw multisite playbooks +- name: include rgw multisite playbooks include: multisite/main.yml when: rgw_zone is defined and rgw_multisite and ( is_jewel or is_after_jewel ) # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) diff --git a/roles/ceph-rgw/tasks/multisite/checks.yml b/roles/ceph-rgw/tasks/multisite/checks.yml index 849e1ff33..7bc6bbb8d 100644 --- a/roles/ceph-rgw/tasks/multisite/checks.yml +++ b/roles/ceph-rgw/tasks/multisite/checks.yml @@ -1,23 +1,23 @@ --- -- name: Check if the realm already exists +- name: check if the realm already exists command: radosgw-admin realm get --rgw-realm={{ rgw_realm }} register: realmcheck failed_when: False changed_when: False -- name: Check if the zonegroup already exists +- name: check if the zonegroup already exists command: radosgw-admin zonegroup get --rgw-zonegroup={{ rgw_zonegroup }} register: zonegroupcheck failed_when: False changed_when: False -- name: Check if the zone already exists +- name: check if the zone already exists command: radosgw-admin zone get --rgw-zone={{ rgw_zone }} register: zonecheck failed_when: False changed_when: False -- name: Check if the system user already exists +- name: check if the system user already exists command: radosgw-admin user info --uid=zone.user register: usercheck failed_when: False diff --git a/roles/ceph-rgw/tasks/multisite/destroy.yml b/roles/ceph-rgw/tasks/multisite/destroy.yml index 8b608e0e2..db1226130 100644 --- a/roles/ceph-rgw/tasks/multisite/destroy.yml +++ b/roles/ceph-rgw/tasks/multisite/destroy.yml @@ -1,25 +1,25 @@ --- -- name: Delete the zone user +- name: delete the zone user command: radosgw-admin user rm --uid=zone.user run_once: true failed_when: false -- name: Delete the zone +- name: delete the zone command: radosgw-admin zone delete --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} run_once: true failed_when: false -- name: Delete the zonegroup +- name: delete the zonegroup command: radosgw-admin zonegroup delete --rgw-zonegroup={{ rgw_zonegroup }} run_once: true failed_when: false -- name: Delete the realm +- name: delete the realm command: radosgw-admin realm delete --rgw-realm={{ rgw_realm }} run_once: true failed_when: false -- name: Delete zone from RGW stanza in ceph.conf +- name: delete zone from rgw stanza in ceph.conf lineinfile: dest: /etc/ceph/ceph.conf regexp: "rgw_zone = {{ rgw_zonegroup }}-{{ rgw_zone }}" diff --git a/roles/ceph-rgw/tasks/multisite/main.yml b/roles/ceph-rgw/tasks/multisite/main.yml index b58073061..cc874ba00 100644 --- a/roles/ceph-rgw/tasks/multisite/main.yml +++ b/roles/ceph-rgw/tasks/multisite/main.yml @@ -1,22 +1,22 @@ --- -- name: Include multisite checks +- name: include multisite checks include: checks.yml # Include the tasks depending on the zone type -- name: Include master multisite tasks +- name: include master multisite tasks include: master.yml when: "rgw_zonemaster is defined and rgw_zonemaster" # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) static: False -- name: Include secondary multisite tasks +- name: include secondary multisite tasks include: secondary.yml when: "rgw_zonesecondary is defined and rgw_zonesecondary" # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) static: False # Continue with common tasks -- name: Add zone to RGW stanza in ceph.conf +- name: add zone to rgw stanza in ceph.conf lineinfile: dest: /etc/ceph/ceph.conf regexp: "{{ ansible_host }}" diff --git a/roles/ceph-rgw/tasks/multisite/master.yml b/roles/ceph-rgw/tasks/multisite/master.yml index 94b5c5e91..ff818465b 100644 --- a/roles/ceph-rgw/tasks/multisite/master.yml +++ b/roles/ceph-rgw/tasks/multisite/master.yml @@ -1,26 +1,26 @@ --- -- name: Create the realm +- name: create the realm command: radosgw-admin realm create --rgw-realm={{ rgw_realm }} --default run_once: true when: ("No such file or directory" in realmcheck.stderr) and rgw_zonemaster notify: - update period -- name: Create the zonegroup +- name: create the zonegroup command: radosgw-admin zonegroup create --rgw-zonegroup={{ rgw_zonegroup }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --master --default run_once: true when: ("No such file or directory" in zonegroupcheck.stderr) and rgw_zonemaster notify: - update period -- name: Create the zone +- name: create the zone command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_fqdn }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default --master run_once: true when: ("No such file or directory" in zonecheck.stderr) and rgw_zonemaster notify: - update period -- name: Create the zone user +- name: create the zone user command: radosgw-admin user create --uid=zone.user --display-name="Zone User" --access-key={{ system_access_key }} --secret={{ system_secret_key }} --system run_once: true when: "'could not fetch user info: no user info saved' in usercheck.stderr" diff --git a/roles/ceph-rgw/tasks/multisite/secondary.yml b/roles/ceph-rgw/tasks/multisite/secondary.yml index dd0f671ee..90dd03225 100644 --- a/roles/ceph-rgw/tasks/multisite/secondary.yml +++ b/roles/ceph-rgw/tasks/multisite/secondary.yml @@ -1,31 +1,31 @@ --- -- name: Fetch the realm +- name: fetch the realm command: radosgw-admin realm pull --url=http://{{ rgw_pullhost }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true when: ("No such file or directory" in realmcheck.stderr) notify: - update period -- name: Fetch the period +- name: fetch the period command: radosgw-admin period pull --url=http://{{ rgw_pullhost }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} run_once: true when: ("No such file or directory" in realmcheck.stderr) notify: - update period -- name: Set default realm +- name: set default realm command: radosgw-admin realm default --rgw-realm={{ rgw_realm }} run_once: true notify: - update period # Might not need to update period here -- name: Set default zonegroup +- name: set default zonegroup command: radosgw-admin zonegroup default --rgw-zonegroup={{ rgw_zonegroup }} run_once: true notify: - update period # Might not need to update period here -- name: Create the zone +- name: create the zone command: radosgw-admin zone create --rgw-zonegroup={{ rgw_zonegroup }} --rgw-zone={{ rgw_zone }} --endpoints=http://{{ ansible_hostname }}:{{ radosgw_civetweb_port }} --access-key={{ system_access_key }} --secret={{ system_secret_key }} --default run_once: true when: ("No such file or directory" in zonecheck.stderr)