osd: container use id instead of dev name

Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1494127
Signed-off-by: Sébastien Han <seb@redhat.com>

infrastructure-playbooks/purge-docker-cluster.yml

@@ -278,10 +278,21 @@
     with_items: "{{ devices }}"
     ignore_errors: true
 
+  - name: resolve device link
+    command: readlink -f {{ item }}
+    changed_when: false
+    with_items: "{{ devices }}"
+    register: purge_devices_links
+
+  - name: set_fact devices generate device list when osd_auto_discovery
+    set_fact:
+      devices: "{{ devices | default([]) + [ item.stdout ] }}"
+    with_items: "{{ purge_devices_links.results }}"
+
   - name: remove ceph osd prepare container
     docker:
       image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-prepare-{{ ansible_hostname }}{{ item | regex_replace('/dev/', '') }}"
       state: absent
     with_items: "{{ devices }}"
     ignore_errors: true
@@ -289,7 +300,7 @@
   - name: remove ceph osd container
     docker:
       image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "ceph-osd-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }}"
       state: absent
     with_items: "{{ devices }}"
     ignore_errors: true
@@ -298,7 +309,7 @@
     shell: |
       docker run \
       --privileged=true \
-      --name ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }} \
+      --name ceph-osd-zap-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }} \
       -v /dev/:/dev/ \
       -e OSD_DEVICE={{ item }} \
       {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
@@ -309,7 +320,7 @@
 
   - name: wait until the zap containers die
     shell: |
-      docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}-dev
+      docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}
     register: zap_alive
     failed_when: false
     until: zap_alive.rc != 0
@@ -319,7 +330,7 @@
   - name: remove ceph osd zap disk container
     docker:
       image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
+      name: "ceph-osd-zap-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }}"
       state: absent
     with_items:
       - "{{ devices }}"

roles/ceph-defaults/tasks/check_socket.yml

@@ -263,7 +263,7 @@
 
 - name: check for a ceph socket in containerized deployment (osds)
   shell: |
-    docker exec ceph-osd-"{{ ansible_hostname }}"-"{{ item | replace('/', '') }}" bash -c 'stat --printf=%n /var/run/ceph/*.asok'
+    docker exec ceph-osd-"{{ ansible_hostname }}"-"{{ item | replace('/dev/', '') }}" bash -c 'stat --printf=%n /var/run/ceph/*.asok'
   changed_when: false
   failed_when: false
   always_run: true

roles/ceph-defaults/tasks/facts.yml

@@ -130,3 +130,24 @@
   when:
     - rbd_client_directory_mode is not defined
       or not rbd_client_directory_mode
+
+- name: resolve device link(s)
+  command: readlink -f {{ item }}
+  changed_when: false
+  with_items: "{{ devices }}"
+  register: devices_prepare_canonicalize
+  when:
+    - inventory_hostname in groups.get(osd_group_name, [])
+
+- name: set_fact build devices from resolved symlinks
+  set_fact:
+    devices: "{{ devices | default([]) + [ item.stdout ] }}"
+  with_items: "{{ devices_prepare_canonicalize.results }}"
+  when:
+    - inventory_hostname in groups.get(osd_group_name, [])
+
+- name: set_fact build final devices list
+  set_fact:
+    devices: "{{ devices | reject('search','/dev/disk') | list | unique }}"
+  when:
+    - inventory_hostname in groups.get(osd_group_name, [])

roles/ceph-osd/tasks/check_devices.yml

@@ -10,6 +10,27 @@
 # allow 2-digit partition numbers so fast SSDs can be shared by > 9 disks
 # for SSD journals.
 
+- name: resolve dedicated device link(s)
+  command: readlink -f {{ item }}
+  changed_when: false
+  with_items: "{{ dedicated_devices }}"
+  register: dedicated_devices_prepare_canonicalize
+  when:
+    - osd_scenario == 'non-collocated'
+
+- name: set_fact build dedicated_devices from resolved symlinks
+  set_fact:
+    dedicated_devices: "{{ dedicated_devices | default([]) + [ item.stdout ] }}"
+  with_items: "{{ dedicated_devices_prepare_canonicalize.results }}"
+  when:
+    - osd_scenario == 'non-collocated'
+
+- name: set_fact build final dedicated_devices list
+  set_fact:
+    dedicated_devices: "{{ dedicated_devices | reject('search','/dev/disk') | list | unique }}"
+  when:
+    - osd_scenario == 'non-collocated'
+
 - name: include check_devices_static.yml
   include: check_devices_static.yml
   # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
@@ -33,10 +54,10 @@
     - "{{ dedicated_devices|unique }}"
   changed_when: false
   when:
+    - not containerized_deployment
+    - osd_scenario == 'non-collocated'
     - not item.0.get("skipped")
     - item.0.get("rc", 0) != 0
-    - osd_scenario == 'non-collocated'
-    - not containerized_deployment
 
 - name: create gpt disk label of the journal device(s)
   command: parted --script {{ item.1 }} mklabel gpt
@@ -45,7 +66,7 @@
     - "{{ dedicated_devices|unique }}"
   changed_when: false
   when:
+    - containerized_deployment
+    - osd_scenario == 'non-collocated'
     - not item.0.get("skipped")
     - item.0.get("rc", 0) != 0
-    - osd_scenario == 'non-collocated'
-    - containerized_deployment

roles/ceph-osd/tasks/check_devices_static.yml

@@ -34,12 +34,3 @@
     - not item.0.get("skipped")
     - item.0.get("rc", 0) != 0
     - containerized_deployment
-
-- name: check if a partition named 'ceph' exists
-  shell: "parted --script {{ item }} print | egrep -sq '^ 1.*ceph'"
-  with_items:
-    - "{{ devices }}"
-  changed_when: false
-  failed_when: false
-  always_run: true
-  register: parted_results

roles/ceph-osd/tasks/docker/start_docker_osd.yml

@@ -30,7 +30,7 @@
 
 - name: systemd start osd container
   systemd:
-    name: ceph-osd@{{ item | basename }}
+    name: ceph-osd@{{ item | regex_replace('/dev/', '') }}
     state: started
     enabled: yes
     daemon_reload: yes

roles/ceph-osd/tasks/main.yml

@@ -17,14 +17,25 @@
     devices: "{{ devices | default([]) + [ item.key | regex_replace('^', '/dev/') ] }}"
   with_dict: "{{ ansible_devices }}"
   when:
+    - osd_auto_discovery
     - ansible_devices is defined
     - item.value.removable == "0"
     - item.value.partitions|count == 0
     - item.value.holders|count == 0
-    - osd_auto_discovery
 
 - name: include check_devices.yml
   include: check_devices.yml
+  when:
+    - not osd_auto_discovery
+
+- name: check if a partition named 'ceph' exists
+  shell: "parted --script {{ item }} print | egrep -sq '^ 1.*ceph'"
+  with_items:
+    - "{{ devices }}"
+  changed_when: false
+  failed_when: false
+  always_run: true
+  register: parted_results
 
 - name: include copy_configs.yml
   include: copy_configs.yml
@@ -42,6 +53,7 @@
 - name: include scenarios/non-collocated.yml
   include: scenarios/non-collocated.yml
   when:
+    - not osd_auto_discovery
     - osd_scenario == 'non-collocated'
   # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
   static: False

roles/ceph-osd/tasks/scenarios/collocated.yml

@@ -7,7 +7,7 @@
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
+    --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -22,17 +22,17 @@
     - "{{ parted_results.results | default([]) }}"
     - "{{ devices }}"
   when:
-    - not item.0.get("skipped")
-    - item.0.get("rc", 0) != 0
     - not osd_auto_discovery
     - containerized_deployment
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
 
 - name: automatic prepare ceph containerized osd disk collocated
   shell: |
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name=ceph-osd-prepare-{{ ansible_hostname }}-devdev{{ item.key }} \
+    --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.key }} \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -45,12 +45,12 @@
     {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
   with_dict: "{{ ansible_devices }}"
   when:
+    - osd_auto_discovery
+    - containerized_deployment
     - ansible_devices is defined
     - item.value.removable == "0"
     - item.value.partitions|count == 0
     - item.value.holders|count == 0
-    - osd_auto_discovery
-    - containerized_deployment
 
 - name: manually prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) with collocated osd data and journal
   command: "ceph-disk prepare {{ ceph_disk_cli_options }} {{ item.1 }}"
@@ -58,6 +58,6 @@
     - "{{ parted_results.results | default([]) }}"
     - "{{ devices }}"
   when:
+    - not containerized_deployment
     - not item.0.get("skipped")
     - item.0.get("rc", 0) != 0
-    - not containerized_deployment

roles/ceph-osd/tasks/scenarios/non-collocated.yml

@@ -7,7 +7,7 @@
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
+    --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -24,18 +24,17 @@
     - "{{ devices }}"
     - "{{ dedicated_devices }}"
   when:
-    - not item.0.get("skipped")
-    - item.0.get("rc", 0) != 0
-    - not osd_auto_discovery
     - containerized_deployment
     - osd_objectstore == 'filestore'
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
 
 - name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal
   shell: |
     docker run --net=host \
     --pid=host \
     --privileged=true \
-    --name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
+    --name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
     -v /etc/ceph:/etc/ceph \
     -v /var/lib/ceph/:/var/lib/ceph/ \
     -v /dev:/dev \
@@ -54,11 +53,10 @@
     - "{{ dedicated_devices }}"
     - "{{ bluestore_wal_devices }}"
   when:
-    - not item.0.get("skipped")
-    - item.0.get("rc", 0) != 0
-    - not osd_auto_discovery
     - containerized_deployment
     - osd_objectstore == 'bluestore'
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
 
 - name: prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) non-collocated
   command: "ceph-disk prepare {{ ceph_disk_cli_options }} {{ item.1 }} {{ item.2 }}"
@@ -68,10 +66,10 @@
     - "{{ dedicated_devices }}"
   changed_when: false
   when:
-    - not item.0.get("skipped")
-    - item.0.get("rc", 0) != 0
     - osd_objectstore == 'filestore'
     - not containerized_deployment
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
 
 - name: manually prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) with a dedicated device for db and wal
   command: "ceph-disk prepare {{ ceph_disk_cli_options }} --block.db {{ item.1 }} --block.wal {{ item.2 }} {{ item.3 }}"
@@ -79,9 +77,9 @@
     - "{{ parted_results.results | default([]) }}"
     - "{{ dedicated_devices }}"
     - "{{ bluestore_wal_devices }}"
-    - "{{ devices }}"
+    - "{{ devices | unique }}"
   when:
-    - not item.0.get("skipped")
-    - item.0.get("rc", 0) != 0
     - osd_objectstore == 'bluestore'
     - not containerized_deployment
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0

roles/ceph-osd/templates/ceph-osd-run.sh.j2

@@ -64,5 +64,5 @@ expose_partitions "$1"
   -e OSD_DEVICE=/dev/${1} \
   -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
   {{ ceph_osd_docker_extra_env }} \
-  --name=ceph-osd-{{ ansible_hostname }}-dev${1} \
+  --name=ceph-osd-{{ ansible_hostname }}-${1} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}

roles/ceph-osd/templates/ceph-osd.service.j2

@@ -5,10 +5,10 @@ After=docker.service
 
 [Service]
 EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-dev%i
-ExecStartPre=-/usr/bin/docker rm -f ceph-osd-{{ ansible_hostname }}-dev%i
+ExecStartPre=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-%i
+ExecStartPre=-/usr/bin/docker rm -f ceph-osd-{{ ansible_hostname }}-%i
 ExecStart={{ ceph_osd_docker_run_script_path }}/ceph-osd-run.sh %i
-ExecStop=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-dev%i
+ExecStop=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-%i
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120

tests/conftest.py

@@ -16,7 +16,7 @@ def node(host, request):
     # tox will pass in this environment variable. we need to do it this way
     # because testinfra does not collect and provide ansible config passed in
     # from using --extra-vars
-    ceph_stable_release = os.environ.get("CEPH_STABLE_RELEASE", "kraken")
+    ceph_stable_release = os.environ.get("CEPH_STABLE_RELEASE", "luminous")
     node_type = ansible_vars["group_names"][0]
     docker = ansible_vars.get("docker")
     osd_auto_discovery = ansible_vars.get("osd_auto_discovery")