From cdf61540d89ba6a9495b5abce868b1566b0f43af Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 1 Jul 2020 10:47:45 +0200 Subject: [PATCH] rgw: fix multi instances scaleout When rgw and osd are collocated, the current workflow prevents from scaling out the radosgw_num_instances parameter when rerunning the playbook. The environment file used in the rgw systemd template is rendered when executing the `ceph-rgw` role but during a new run of the playbook (in order to scale out rgw instances), handlers are triggered from `ceph-osd` role which is run before `ceph-rgw`, therefore it tries to start the new rgw daemon whereas its corresponding environment file hasn't been rendered yet and fails like following: ``` ceph-radosgw@rgw.ceph4osd3.rgw1.service failed to run 'start-pre' task: No such file or directory ``` This commit moves the tasks generating this file in `ceph-config` role so it is generated early. Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1851906 Signed-off-by: Guillaume Abrioux (cherry picked from commit 7dd68b9ac1086e3c15cfad3812f1c141eadd80c0) --- ...inerized-to-containerized-ceph-daemons.yml | 4 ++++ roles/ceph-config/tasks/main.yml | 4 ++++ .../tasks/rgw_systemd_environment_file.yml | 23 +++++++++++++++++++ roles/ceph-rgw/tasks/common.yml | 10 -------- roles/ceph-rgw/tasks/start_docker_rgw.yml | 11 --------- 5 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 roles/ceph-config/tasks/rgw_systemd_environment_file.yml diff --git a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml index b15d5e54b..b5ffcf102 100644 --- a/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml +++ b/infrastructure-playbooks/switch-from-non-containerized-to-containerized-ceph-daemons.yml @@ -410,6 +410,10 @@ - import_role: name: ceph-facts + - import_role: + name: ceph-config + tasks_from: rgw_systemd_environment_file.yml + # NOTE: changed from file module to raw find command for performance reasons # The file module has to run checks on current ownership of all directories and files. This is unnecessary # as in this case we know we want all owned by ceph user diff --git a/roles/ceph-config/tasks/main.yml b/roles/ceph-config/tasks/main.yml index 6550f437f..bb406b477 100644 --- a/roles/ceph-config/tasks/main.yml +++ b/roles/ceph-config/tasks/main.yml @@ -3,6 +3,10 @@ include_tasks: create_ceph_initial_dirs.yml when: containerized_deployment | bool +- name: include_tasks rgw_systemd_environment_file.yml + include_tasks: rgw_systemd_environment_file.yml + when: inventory_hostname in groups.get(rgw_group_name, []) + - name: config file operations related to OSDs when: - inventory_hostname in groups.get(osd_group_name, []) diff --git a/roles/ceph-config/tasks/rgw_systemd_environment_file.yml b/roles/ceph-config/tasks/rgw_systemd_environment_file.yml new file mode 100644 index 000000000..21a77850a --- /dev/null +++ b/roles/ceph-config/tasks/rgw_systemd_environment_file.yml @@ -0,0 +1,23 @@ +--- +- name: create rados gateway instance directories + file: + path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}" + state: directory + owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}" + group: "{{ ceph_uid if containerized_deployment else 'ceph' }}" + mode: "{{ ceph_directories_mode | default('0755') }}" + with_items: "{{ rgw_instances }}" + +- name: generate environment file + copy: + dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/EnvironmentFile" + owner: "root" + group: "root" + mode: "0644" + content: | + INST_NAME={{ item.instance_name }} + INST_PORT={{ item.radosgw_frontend_port }} + with_items: "{{ rgw_instances }}" + when: + - containerized_deployment | bool + - rgw_instances is defined \ No newline at end of file diff --git a/roles/ceph-rgw/tasks/common.yml b/roles/ceph-rgw/tasks/common.yml index 87fb3d66e..5e0520c69 100644 --- a/roles/ceph-rgw/tasks/common.yml +++ b/roles/ceph-rgw/tasks/common.yml @@ -8,16 +8,6 @@ mode: "{{ ceph_directories_mode }}" with_items: "{{ rbd_client_admin_socket_path }}" -- name: create rados gateway instance directories - file: - path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}" - state: directory - owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}" - group: "{{ ceph_uid if containerized_deployment else 'ceph' }}" - mode: "{{ ceph_directories_mode }}" - with_items: "{{ rgw_instances }}" - when: rgw_instances is defined - - name: get keys from monitors command: "{{ container_exec_cmd }} ceph --cluster {{ cluster }} auth get {{ item.name }}" register: _rgw_keys diff --git a/roles/ceph-rgw/tasks/start_docker_rgw.yml b/roles/ceph-rgw/tasks/start_docker_rgw.yml index 7cb370214..d0d96314e 100644 --- a/roles/ceph-rgw/tasks/start_docker_rgw.yml +++ b/roles/ceph-rgw/tasks/start_docker_rgw.yml @@ -1,15 +1,4 @@ --- -- name: generate environment file - copy: - dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/EnvironmentFile" - owner: "root" - group: "root" - mode: "0644" - content: | - INST_NAME={{ item.instance_name }} - INST_PORT={{ item.radosgw_frontend_port }} - with_items: "{{ rgw_instances }}" - - name: include_task systemd.yml include_tasks: systemd.yml