From e4965e9ea9b68075d33f7c67a2130c45d391b229 Mon Sep 17 00:00:00 2001 From: Dimitri Savineau Date: Mon, 20 Jan 2020 16:40:58 -0500 Subject: [PATCH] filestore-to-bluestore: --destroy with raw devices We still need --destroy when using a raw device otherwise we won't be able to recreate the lvm stack on that device with bluestore. Running command: /usr/sbin/vgcreate -s 1G --force --yes ceph-bdc67a84-894a-4687-b43f-bcd76317580a /dev/sdd stderr: Physical volume '/dev/sdd' is already in volume group 'ceph-b7801d50-e827-4857-95ec-3291ad6f0151' Unable to add physical volume '/dev/sdd' to volume group 'ceph-b7801d50-e827-4857-95ec-3291ad6f0151' /dev/sdd: physical volume not initialized. --> Was unable to complete a new OSD, will rollback changes Closes: https://bugzilla.redhat.com/show_bug.cgi?id=1792227 Signed-off-by: Dimitri Savineau (cherry picked from commit f995b079a6a4f936da04fce3d55449361b2109e3) --- .../filestore-to-bluestore.yml | 18 +++++++++++++++--- .../filestore-to-bluestore/container/hosts | 4 +++- .../container/vagrant_variables.yml | 2 +- tests/functional/filestore-to-bluestore/hosts | 4 +++- .../vagrant_variables.yml | 2 +- tests/functional/lvm_setup.yml | 2 ++ tox-filestore_to_bluestore.ini | 5 +++-- 7 files changed, 28 insertions(+), 9 deletions(-) diff --git a/infrastructure-playbooks/filestore-to-bluestore.yml b/infrastructure-playbooks/filestore-to-bluestore.yml index 9b2415824..72bdd3a4c 100644 --- a/infrastructure-playbooks/filestore-to-bluestore.yml +++ b/infrastructure-playbooks/filestore-to-bluestore.yml @@ -179,15 +179,15 @@ - name: set_fact osd_fsid_list set_fact: - osd_fsid_list: "{{ osd_fsid_list | default([]) + [item.tags['ceph.osd_fsid']] }}" + osd_fsid_list: "{{ osd_fsid_list | default([]) + [{'osd_fsid': item.tags['ceph.osd_fsid'], 'destroy': (item.lv_name.startswith('osd-data-') and item.vg_name.startswith('ceph-')) | ternary(true, false)}] }}" with_items: "{{ _lvm_list }}" when: item.type == 'data' - name: zap ceph-volume prepared OSDs ceph_volume: action: "zap" - osd_fsid: "{{ item }}" - destroy: False + osd_fsid: "{{ item.osd_fsid }}" + destroy: "{{ item.destroy }}" environment: CEPH_VOLUME_DEBUG: 1 CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}" @@ -203,6 +203,14 @@ - item['lv_path'] is defined # Do not close mappers for non 'lvm batch' devices - devices | default([]) | length > 0 + + - name: ensure all pv are removed + command: "pvremove --yes {{ item.devices[0] }}" + with_items: "{{ _lvm_list }}" + when: + - item.type == 'data' + - item.lv_name.startswith('osd-data-') | bool + - item.vg_name.startswith('ceph-') | bool when: _lvm_list is defined - name: set_fact osd_ids @@ -238,6 +246,10 @@ - import_role: name: ceph-container-common when: containerized_deployment | bool + - import_role: + name: ceph-config + vars: + osd_objectstore: bluestore - import_role: name: ceph-osd vars: diff --git a/tests/functional/filestore-to-bluestore/container/hosts b/tests/functional/filestore-to-bluestore/container/hosts index c249f3588..170ea69cb 100644 --- a/tests/functional/filestore-to-bluestore/container/hosts +++ b/tests/functional/filestore-to-bluestore/container/hosts @@ -4,4 +4,6 @@ mon0 [osds] osd0 osd_objectstore=filestore lvm_volumes="[{'data': 'data-lv1', 'journal': '/dev/sdc1', 'data_vg': 'test_group'},{'data': 'data-lv2', 'data_vg': 'test_group', 'journal': 'journal1', 'journal_vg': 'journals'}]" osd1 osd_objectstore=filestore lvm_volumes="[{'data': 'data-lv1', 'journal': '/dev/sdc1', 'data_vg': 'test_group'},{'data': 'data-lv2', 'data_vg': 'test_group', 'journal': 'journal1', 'journal_vg': 'journals'}]" dmcrypt=true -osd2 osd_objectstore=filestore devices="['/dev/sda', '/dev/sdb']" dedicated_devices="['/dev/sdc']" journal_size=1024 \ No newline at end of file +osd2 osd_objectstore=filestore devices="['/dev/sda', '/dev/sdb']" dedicated_devices="['/dev/sdc']" journal_size=1024 +osd3 osd_objectstore=filestore lvm_volumes="[{'data': '/dev/sda', 'journal': '/dev/sdc1'},{'data': '/dev/sdb', 'journal': '/dev/sdc2'}]" +osd4 osd_objectstore=filestore lvm_volumes="[{'data': '/dev/sda', 'journal': '/dev/sdc1'},{'data': '/dev/sdb', 'journal': '/dev/sdc2'}]" dmcrypt=true \ No newline at end of file diff --git a/tests/functional/filestore-to-bluestore/container/vagrant_variables.yml b/tests/functional/filestore-to-bluestore/container/vagrant_variables.yml index 1f1c43df3..dc8d321ce 100644 --- a/tests/functional/filestore-to-bluestore/container/vagrant_variables.yml +++ b/tests/functional/filestore-to-bluestore/container/vagrant_variables.yml @@ -5,7 +5,7 @@ docker: true # DEFINE THE NUMBER OF VMS TO RUN mon_vms: 1 -osd_vms: 3 +osd_vms: 5 mds_vms: 0 rgw_vms: 0 nfs_vms: 0 diff --git a/tests/functional/filestore-to-bluestore/hosts b/tests/functional/filestore-to-bluestore/hosts index c249f3588..170ea69cb 100644 --- a/tests/functional/filestore-to-bluestore/hosts +++ b/tests/functional/filestore-to-bluestore/hosts @@ -4,4 +4,6 @@ mon0 [osds] osd0 osd_objectstore=filestore lvm_volumes="[{'data': 'data-lv1', 'journal': '/dev/sdc1', 'data_vg': 'test_group'},{'data': 'data-lv2', 'data_vg': 'test_group', 'journal': 'journal1', 'journal_vg': 'journals'}]" osd1 osd_objectstore=filestore lvm_volumes="[{'data': 'data-lv1', 'journal': '/dev/sdc1', 'data_vg': 'test_group'},{'data': 'data-lv2', 'data_vg': 'test_group', 'journal': 'journal1', 'journal_vg': 'journals'}]" dmcrypt=true -osd2 osd_objectstore=filestore devices="['/dev/sda', '/dev/sdb']" dedicated_devices="['/dev/sdc']" journal_size=1024 \ No newline at end of file +osd2 osd_objectstore=filestore devices="['/dev/sda', '/dev/sdb']" dedicated_devices="['/dev/sdc']" journal_size=1024 +osd3 osd_objectstore=filestore lvm_volumes="[{'data': '/dev/sda', 'journal': '/dev/sdc1'},{'data': '/dev/sdb', 'journal': '/dev/sdc2'}]" +osd4 osd_objectstore=filestore lvm_volumes="[{'data': '/dev/sda', 'journal': '/dev/sdc1'},{'data': '/dev/sdb', 'journal': '/dev/sdc2'}]" dmcrypt=true \ No newline at end of file diff --git a/tests/functional/filestore-to-bluestore/vagrant_variables.yml b/tests/functional/filestore-to-bluestore/vagrant_variables.yml index e127e8413..08fa705db 100644 --- a/tests/functional/filestore-to-bluestore/vagrant_variables.yml +++ b/tests/functional/filestore-to-bluestore/vagrant_variables.yml @@ -5,7 +5,7 @@ docker: false # DEFINE THE NUMBER OF VMS TO RUN mon_vms: 1 -osd_vms: 3 +osd_vms: 5 mds_vms: 0 rgw_vms: 0 nfs_vms: 0 diff --git a/tests/functional/lvm_setup.yml b/tests/functional/lvm_setup.yml index 2c28fc363..1c5191937 100644 --- a/tests/functional/lvm_setup.yml +++ b/tests/functional/lvm_setup.yml @@ -49,6 +49,7 @@ unit: '%' label: gpt state: present + tags: partitions - name: partition /dev/sdc for journals parted: device: /dev/sdc @@ -58,6 +59,7 @@ unit: '%' state: present label: gpt + tags: partitions - name: create journals vg from /dev/sdc2 lvg: vg: journals diff --git a/tox-filestore_to_bluestore.ini b/tox-filestore_to_bluestore.ini index 8b7424123..1d0983ece 100644 --- a/tox-filestore_to_bluestore.ini +++ b/tox-filestore_to_bluestore.ini @@ -43,7 +43,8 @@ commands= ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/setup.yml - ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/lvm_setup.yml --limit 'all:!osd2' + ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/lvm_setup.yml --limit 'osd0:osd1' + ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/tests/functional/lvm_setup.yml --limit 'osd3:osd4' --tags partitions # deploy the cluster ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/{env:PLAYBOOK:site.yml.sample} --extra-vars "\ @@ -54,7 +55,7 @@ commands= ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} \ ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest-nautilus} \ " - ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/filestore-to-bluestore.yml --limit osd0,osd1,osd2 + ansible-playbook -vv -i {changedir}/{env:INVENTORY} {toxinidir}/infrastructure-playbooks/filestore-to-bluestore.yml --limit osds bash -c "CEPH_STABLE_RELEASE={env:UPDATE_CEPH_STABLE_RELEASE:nautilus} py.test --reruns 5 --reruns-delay 1 -n 8 --durations=0 --sudo -v --connection=ansible --ansible-inventory={changedir}/{env:INVENTORY} --ssh-config={changedir}/vagrant_ssh_config {toxinidir}/tests/functional/tests"