Merge pull request #1761 from ceph/split_copy_keys

docker: split the task 'copy ceph configs&keys'

roles/ceph-docker-common/tasks/main.yml

@@ -89,19 +89,5 @@
 
 - include: create_configs.yml
 
-# Only include 'fetch_configs.yml' when:
-# - we are deploying containers without kv AND host is either a mon OR a nfs OR an osd
-# OR
-# - host is either a mdss OR mgrs OR rgws
-- include: fetch_configs.yml
-  when:
-    - (not containerized_deployment_with_kv and
-        ((inventory_hostname in groups.get(mon_group_name, [])) or
-          (inventory_hostname in groups.get(nfs_group_name, [])) or
-          (inventory_hostname in groups.get(osd_group_name, [])))) or
-      (inventory_hostname in groups.get('mdss', [])) or
-      (inventory_hostname in groups.get('mgrs', [])) or
-      (inventory_hostname in groups.get('rgws', []))
-
 - include: selinux.yml
   when: ansible_os_family == 'RedHat'

roles/ceph-docker-common/tasks/selinux.yml

@@ -4,12 +4,3 @@
   register: sestatus
   changed_when: false
   always_run: true
-
-- name: set selinux permissions
-  shell: |
-    chcon -Rt svirt_sandbox_file_t {{ item }}
-  with_items:
-    - "{{ ceph_conf_key_directory }}"
-    - /var/lib/ceph
-  changed_when: false
-  when: sestatus.stdout != 'Disabled'

roles/ceph-mds/tasks/docker/copy_configs.yml

@@ -0,0 +1,38 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/{{ cluster }}.conf
+      - /etc/ceph/{{ cluster }}.client.admin.keyring
+      - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+
+- name: try to fetch ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - "{{ ceph_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == true
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-mds/tasks/docker/main.yml

@@ -3,5 +3,6 @@
   set_fact:
     docker_exec_cmd: "docker exec ceph-mds-{{ ansible_hostname }}"
 
+- include: copy_configs.yml
 - include: start_docker_mds.yml
 - include: enable_multimds.yml

roles/ceph-mgr/tasks/docker/copy_configs.yml

@@ -0,0 +1,52 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/{{ cluster }}.conf
+      - /etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+
+- name: try to fetch ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - "{{ ceph_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == true
+
+- name: "copy mgr key to /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring"
+  command: cp /etc/ceph/{{ cluster }}.mgr.{{ ansible_hostname }}.keyring /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring
+  changed_when: false
+  always_run: true
+  with_items: "{{ statconfig.results }}"
+  when: item.stat.exists == true
+
+- name: set ceph mgr key permission
+  file:
+    path: "/var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}/keyring"
+    owner: "{{ bootstrap_dirs_owner }}"
+    group: "{{ bootstrap_dirs_group }}"
+    mode: "0600"
+  when: cephx
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-mgr/tasks/docker/main.yml

@@ -1,2 +1,19 @@
 ---
+- name: create mgr directory
+  file:
+    path: /var/lib/ceph/mgr/
+    state: directory
+    owner: "root"
+    group: "root"
+    mode: "0755"
+
+- name: create mgr directory
+  file:
+    path: /var/lib/ceph/mgr/{{ cluster }}-{{ ansible_hostname }}
+    state: directory
+    owner: "root"
+    group: "root"
+    mode: "0755"
+
+- include: copy_configs.yml
 - include: start_docker_mgr.yml

roles/ceph-mon/tasks/docker/copy_configs.yml

@@ -1,10 +1,59 @@
 ---
-- name: push ceph files to the ansible server
+- name: set config and keys paths
-  fetch:
+  set_fact:
-    src: "{{ item.0 }}"
+    ceph_config_keys:
-    dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+      - /etc/ceph/{{ cluster }}.conf
-    flat: yes
+      - /etc/ceph/{{ cluster }}.client.admin.keyring
+      - /etc/ceph/monmap-{{ cluster }}
+      - /etc/ceph/{{ cluster }}.mon.keyring
+      - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring
+      - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
+      - /var/lib/ceph/bootstrap-mds/{{ cluster }}.keyring
+
+- name: add mgr keys to config and keys paths
+  set_fact:
+    tmp_ceph_mgr_keys: /etc/ceph/{{ cluster }}.mgr.{{ hostvars[item]['ansible_hostname'] }}.keyring
+  with_items: "{{ groups.get(mgr_group_name, []) }}"
+  register: tmp_ceph_mgr_keys_result
+  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+
+- name: convert mgr keys to an array
+  set_fact:
+    ceph_mgr_keys: "{{ tmp_ceph_mgr_keys_result.results | map(attribute='ansible_facts.tmp_ceph_mgr_keys') | list }}"
+  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+
+- name: merge mgr keys to config and keys paths
+  set_fact:
+    ceph_config_keys: "{{ ceph_config_keys + ceph_mgr_keys }}"
+  when: "{{ groups.get(mgr_group_name, []) | length > 0 }}"
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  register: statconfig
+  always_run: true
+
+- name: try to copy ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
   with_together:
     - "{{ ceph_config_keys }}"
     - "{{ statconfig.results }}"
-  when: item.1.stat.exists == false
+  when: item.1.stat.exists == true
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-mon/tasks/docker/fetch_configs.yml

@@ -0,0 +1,10 @@
+---
+- name: push ceph files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    flat: yes
+  with_together:
+    - "{{ ceph_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == false

roles/ceph-mon/tasks/docker/main.yml

@@ -1,4 +1,7 @@
 ---
+- include: copy_configs.yml
+  when: not containerized_deployment_with_kv
+
 - include: start_docker_monitor.yml
 
 - name: wait for monitor socket to exist
@@ -68,7 +71,7 @@
     - hostvars[groups[mon_group_name][0]]['monitor_address_block'] is defined
     - hostvars[groups[mon_group_name][0]]['monitor_address_block'] | length > 0
 
-- include: copy_configs.yml
+- include: ./fetch_configs.yml
   when: not containerized_deployment_with_kv
 
 - name: create ceph rest api keyring when mon is containerized
@@ -116,7 +119,7 @@
     when:
       - "{{ groups.get(mgr_group_name, []) | length > 0 }}"
 
-  - name: push ceph mgr key(s)
+  - name: fetch ceph mgr key(s)
     fetch:
       src: "{{ ceph_conf_key_directory }}/{{ cluster }}.mgr.{{ hostvars[item.item]['ansible_hostname'] }}.keyring"
       dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.stat.path }}"

roles/ceph-nfs/tasks/docker/copy_configs.yml

@@ -27,3 +27,12 @@
     - "{{ ceph_config_keys }}"
     - "{{ statconfig.results }}"
   when: item.1.stat.exists == true
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-nfs/tasks/docker/copy_ganesha_configs.yml

@@ -0,0 +1,37 @@
+---
+- name: set config paths
+  set_fact:
+    nfs_config_keys:
+      - /etc/ganesha/ganesha.conf
+
+- name: stat for config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ nfs_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+
+- name: try to fetch config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - "{{ nfs_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == true
+
+- name: push ganesha files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    flat: yes
+  with_together:
+    - "{{ nfs_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == false

roles/ceph-nfs/tasks/docker/main.yml

@@ -1,6 +1,9 @@
 ---
 # Copy Ceph configs to host
 - include: copy_configs.yml
+# Copy Ganesha Ceph configs to host
+- include: copy_ganesha_configs.yml
+  when: not containerized_deployment_with_kv
 
 - include: start_docker_nfs.yml
 

roles/ceph-osd/tasks/copy_configs.yml

@@ -0,0 +1,39 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/{{ cluster }}.conf
+      - /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring
+
+- name: wait for ceph.conf and keys
+  local_action: wait_for path={{ fetch_directory }}/{{ fsid }}/{{ item.0 }}
+  become: false
+  with_items: "{{ ceph_config_keys }}"
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+
+- name: try to copy ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_items: "{{ ceph_config_keys }}"
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-osd/tasks/main.yml

@@ -21,6 +21,11 @@
 
 - include: check_devices.yml
 
+- include: copy_configs.yml
+  when:
+    - containerized_deployment
+    - not containerized_deployment_with_kv
+
 - include: ./scenarios/collocated.yml
   when:
     - osd_scenario == 'collocated'

roles/ceph-rbd-mirror/tasks/docker/copy_configs.yml

@@ -0,0 +1,43 @@
+---
+- name: register rbd bootstrap key
+  set_fact:
+    bootstrap_rbd_keyring: "/var/lib/ceph/bootstrap-rbd/{{ cluster }}.keyring"
+  when: ceph_release_num.{{ ceph_release }} >= ceph_release_num.luminous
+
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/{{ cluster }}.conf
+      - /etc/ceph/{{ cluster }}.client.admin.keyring
+      - "{{ bootstrap_rbd_keyring | default([]) }}"
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+
+- name: try to fetch ceph config and keys
+  copy:
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - "{{ ceph_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == true
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-rbd-mirror/tasks/docker/main.yml

@@ -1,2 +1,3 @@
 ---
+- include: copy_configs.yml
 - include: start_docker_rbd_mirror.yml

roles/ceph-restapi/tasks/docker/copy_configs.yml

@@ -0,0 +1,37 @@
+---
+- name: set config and keys paths
+  set_fact:
+    ceph_config_keys:
+      - /etc/ceph/{{ cluster }}.conf
+      - /etc/ceph/{{ cluster }}.client.admin.keyring
+
+- name: stat for ceph config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ ceph_config_keys }}"
+  changed_when: false
+  become: false
+  ignore_errors: true
+  register: statconfig
+  always_run: true
+
+- name: try to fetch ceph config and keys
+  copy:
+    src: "{{ playbook_dir }}/{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
+    dest: "{{ item.0 }}"
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
+  with_together:
+    - "{{ ceph_config_keys }}"
+    - "{{ statconfig.results }}"
+  when: item.1.stat.exists == true
+
+- name: set selinux permissions
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-restapi/tasks/docker/main.yml

@@ -1,2 +1,3 @@
 ---
+- include: copy_configs.yml
 - include: start_docker_restapi.yml

roles/ceph-rgw/tasks/docker/copy_configs.yml

@@ -1,37 +1,37 @@
 ---
 - name: set config and keys paths
   set_fact:
-    rgw_config_keys:
+    ceph_config_keys:
-      - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
+      - /etc/ceph/{{ cluster }}.conf
-  when: nfs_obj_gw
+      - /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring
 
-- name: wait for rgw keyring
+- name: stat for ceph config and keys
-  wait_for: path="/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
-  when:
-    - nfs_obj_gw
-    - inventory_hostname == groups[rgw_group_name][0]
-
-- name: stat for config and keys
   local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
-  with_items: "{{ rgw_config_keys }}"
+  with_items: "{{ ceph_config_keys }}"
   changed_when: false
   become: false
-  failed_when: false
+  ignore_errors: true
   always_run: true
   register: statconfig
-  when:
-    - nfs_obj_gw
-    - inventory_hostname == groups[rgw_group_name][0]
 
-- name: push ceph files to the ansible server
+- name: try to fetch ceph config and keys
-  fetch:
+  copy:
-    src: "{{ item.0 }}"
+    src: "{{ fetch_directory }}/{{ fsid }}/{{ item.0 }}"
-    dest: "{{ fetch_directory }}/{{ fsid }}/var/lib/ceph/radosgw/keyring"
+    dest: "{{ item.0 }}"
-    flat: yes
+    owner: root
+    group: root
+    mode: 0644
+  changed_when: false
   with_together:
-    - "{{ rgw_config_keys }}"
+    - "{{ ceph_config_keys }}"
     - "{{ statconfig.results }}"
-  when:
+  when: item.1.stat.exists == true
-    - nfs_obj_gw
+
-    - item.1.stat.exists == false
+- name: set selinux permissions
-    - inventory_hostname == groups[rgw_group_name][0]
+  shell: |
+    chcon -Rt svirt_sandbox_file_t {{ item }}
+  with_items:
+    - "{{ ceph_conf_key_directory }}"
+    - /var/lib/ceph
+  changed_when: false
+  when: sestatus.stdout != 'Disabled'

roles/ceph-rgw/tasks/docker/copy_rgw_configs.yml

@@ -0,0 +1,38 @@
+---
+- name: set config and keys paths
+  set_fact:
+    rgw_config_keys:
+      - "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
+  when: nfs_obj_gw
+
+- name: wait for rgw keyring
+  wait_for:
+    path: "/var/lib/ceph/radosgw/{{ ansible_hostname }}/keyring"
+  when:
+    - nfs_obj_gw
+    - inventory_hostname == groups[rgw_group_name][0]
+
+- name: stat for config and keys
+  local_action: stat path={{ fetch_directory }}/{{ fsid }}/{{ item }}
+  with_items: "{{ rgw_config_keys }}"
+  changed_when: false
+  become: false
+  failed_when: false
+  always_run: true
+  register: statconfig
+  when:
+    - nfs_obj_gw
+    - inventory_hostname == groups[rgw_group_name][0]
+
+- name: push ceph files to the ansible server
+  fetch:
+    src: "{{ item.0 }}"
+    dest: "{{ fetch_directory }}/{{ fsid }}/var/lib/ceph/radosgw/keyring"
+    flat: yes
+  with_together:
+    - "{{ rgw_config_keys }}"
+    - "{{ statconfig.results }}"
+  when:
+    - nfs_obj_gw
+    - item.1.stat.exists == false
+    - inventory_hostname == groups[rgw_group_name][0]

roles/ceph-rgw/tasks/docker/main.yml

@@ -1,4 +1,5 @@
 ---
+- include: copy_configs.yml
 - include: start_docker_rgw.yml
 
-- include: copy_configs.yml
+- include: copy_rgw_configs.yml