From 04f02910a9486fa62479f8d633f9dd3fba3c3125 Mon Sep 17 00:00:00 2001 From: Andrew Schoen Date: Mon, 13 Nov 2017 14:29:37 -0600 Subject: [PATCH 1/3] lvm: ensure the data_vg exists before using it Signed-off-by: Andrew Schoen --- roles/ceph-osd/tasks/scenarios/lvm.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/ceph-osd/tasks/scenarios/lvm.yml b/roles/ceph-osd/tasks/scenarios/lvm.yml index d488ae6bb..009bc76ff 100644 --- a/roles/ceph-osd/tasks/scenarios/lvm.yml +++ b/roles/ceph-osd/tasks/scenarios/lvm.yml @@ -14,6 +14,7 @@ with_items: "{{ lvm_volumes }}" when: - item.journal_vg is not defined + - item.data_vg is defined - osd_objectstore == 'filestore' - "'{{ item.data_vg }}/{{ item.data }}' not in ceph_volume_lvm_list.stdout" @@ -24,6 +25,7 @@ with_items: "{{ lvm_volumes }}" when: - item.journal_vg is defined + - item.data_vg is defined - osd_objectstore == 'filestore' - "'{{ item.data_vg }}/{{ item.data }}' not in ceph_volume_lvm_list.stdout" @@ -34,6 +36,7 @@ with_items: "{{ lvm_volumes }}" when: - osd_objectstore == 'bluestore' + - item.data_vg is defined - item.wal is defined - item.wal_vg is defined - item.db is defined @@ -47,6 +50,7 @@ with_items: "{{ lvm_volumes }}" when: - osd_objectstore == 'bluestore' + - item.data_vg is defined - item.wal is not defined - item.wal_vg is not defined - item.db is defined @@ -61,6 +65,7 @@ when: - osd_objectstore == 'bluestore' - item.wal is defined + - item.data_vg is defined - item.wal_vg is defined - item.db is not defined - item.db_vg is not defined @@ -74,6 +79,7 @@ when: - osd_objectstore == 'bluestore' - item.wal is not defined + - item.data_vg is defined - item.wal_vg is not defined - item.db is not defined - item.db_vg is not defined From 3c604f1115ef0e404c5cff68322872eec5019e06 Mon Sep 17 00:00:00 2001 From: Andrew Schoen Date: Mon, 13 Nov 2017 14:56:51 -0600 Subject: [PATCH 2/3] lvm: support --data as a raw device or partition in ceph-volume Signed-off-by: Andrew Schoen --- docs/source/osds/scenarios.rst | 22 +++++--- roles/ceph-osd/defaults/main.yml | 12 +++- roles/ceph-osd/tasks/scenarios/lvm.yml | 78 ++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/docs/source/osds/scenarios.rst b/docs/source/osds/scenarios.rst index 7843fccf5..42dff3e21 100644 --- a/docs/source/osds/scenarios.rst +++ b/docs/source/osds/scenarios.rst @@ -174,9 +174,9 @@ is only available when the ceph release is Luminous or newer. ``lvm_volumes`` is the config option that needs to be defined to configure the mappings for devices to be deployed. It is a list of dictionaries which expects a volume name and a volume group for logical volumes, but can also accept -a device in the case of ``filestore`` for the ``journal``. +a partition in the case of ``filestore`` for the ``journal``. -The ``data`` key represents the logical volume name that is to be used for your +The ``data`` key represents the logical volume name, raw device or partition that is to be used for your OSD data. The ``data_vg`` key represents the volume group name that your ``data`` logical volume resides on. This key is required for purging of OSDs created by this scenario. @@ -196,17 +196,17 @@ There is filestore support which can be enabled with:: To configure this scenario use the ``lvm_volumes`` config option. ``lvm_volumes`` is a list of dictionaries which expects a volume name and -a volume group for logical volumes, but can also accept a device in the case of +a volume group for logical volumes, but can also accept a parition in the case of ``filestore`` for the ``journal``. The following keys are accepted for a ``filestore`` deployment: * ``data`` -* ``data_vg`` +* ``data_vg`` (not required if ``data`` is a raw device or partition) * ``journal`` -* ``journal_vg`` (not required if ``journal`` is a device and not a logical volume) +* ``journal_vg`` (not required if ``journal`` is a partition and not a logical volume) -The ``journal`` key represents the logical volume name, device or partition that will be used for your OSD journal. +The ``journal`` key represents the logical volume name or partition that will be used for your OSD journal. For example, a configuration to use the ``lvm`` osd scenario would look like:: @@ -223,19 +223,24 @@ For example, a configuration to use the ``lvm`` osd scenario would look like:: - data: data-lv3 journal: /dev/sdb1 data_vg: vg2 + - data: /dev/sda + journal: /dev/sdb1 + - data: /dev/sda1 + journal: journal-lv1 + journal_vg: vg2 ``bluestore`` ^^^^^^^^^^^^^ This scenario allows a combination of devices to be used in an OSD. ``bluestore`` can work just with a single "block" device (specified by the -``data`` and ``data_vg``) or additionally with a ``block.wal`` and ``block.db`` +``data`` and optionally ``data_vg``) or additionally with a ``block.wal`` and ``block.db`` (interchangeably) The following keys are accepted for a ``bluestore`` deployment: * ``data`` (required) -* ``data_vg`` (required) +* ``data_vg`` (not required if ``data`` is a raw device or partition) * ``db`` (optional for ``block.db``) * ``db_vg`` (optional for ``block.db``) * ``wal`` (optional for ``block.wal``) @@ -263,3 +268,4 @@ could look like:: db_vg: vg4 wal: wal-lv4 wal_vg: vg4 + - data: /dev/sda diff --git a/roles/ceph-osd/defaults/main.yml b/roles/ceph-osd/defaults/main.yml index 562fe95dc..7d3c81dee 100644 --- a/roles/ceph-osd/defaults/main.yml +++ b/roles/ceph-osd/defaults/main.yml @@ -195,9 +195,9 @@ bluestore_wal_devices: "{{ dedicated_devices }}" # # Filestore: Each dictionary must contain a data, journal and vg_name key. Any # logical volume or logical group used must be a name and not a path. data -# must be a logical volume. journal can be either a lv, device or partition. +# can be a logical volume, device or partition. journal can be either a lv or partition. # You can not use the same journal for many data lvs. -# data_vg must be the volume group name of the data lv +# data_vg must be the volume group name of the data lv, only applicable when data is an lv. # journal_vg is optional and must be the volume group name of the journal lv, if applicable. # For example: # lvm_volumes: @@ -206,11 +206,15 @@ bluestore_wal_devices: "{{ dedicated_devices }}" # journal: journal-lv1 # journal_vg: vg2 # - data: data-lv2 -# journal: /dev/sda +# journal: /dev/sda1 # data_vg: vg1 # - data: data-lv3 # journal: /dev/sdb1 # data_vg: vg2 +# - data: /dev/sda +# journal: /dev/sdb1 +# - data: /dev/sda1 +# journal: /dev/sdb1 # # Bluestore: Each dictionary must contain at least data. When defining wal or # db, it must have both the lv name and vg group (db and wal are not required). @@ -232,6 +236,8 @@ bluestore_wal_devices: "{{ dedicated_devices }}" # db_vg: vg3 # - data: data-lv4 # data_vg: vg4 +# - data: /dev/sda +# - data: /dev/sdb1 lvm_volumes: [] diff --git a/roles/ceph-osd/tasks/scenarios/lvm.yml b/roles/ceph-osd/tasks/scenarios/lvm.yml index 009bc76ff..c5037fe53 100644 --- a/roles/ceph-osd/tasks/scenarios/lvm.yml +++ b/roles/ceph-osd/tasks/scenarios/lvm.yml @@ -84,3 +84,81 @@ - item.db is not defined - item.db_vg is not defined - "'{{ item.data_vg }}/{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create filestore osds with dedicated journals and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --filestore --data {{ item.data }} --journal {{ item.journal }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - item.journal_vg is not defined + - item.data_vg is not defined + - osd_objectstore == 'filestore' + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create filestore osds with dedicated lv journals and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --filestore --data {{ item.data }} --journal {{item.journal_vg }}/{{ item.journal }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - item.journal_vg is defined + - item.data_vg is not defined + - osd_objectstore == 'filestore' + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create bluestore osds with db and wal and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data {{ item.data }} --block.wal {{ item.wal_vg }}/{{ item.wal }} --block.db {{ item.db_vg }}/{{ item.db }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - osd_objectstore == 'bluestore' + - item.data_vg is not defined + - item.wal is defined + - item.wal_vg is defined + - item.db is defined + - item.db_vg is defined + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create bluestore osds with db only and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data {{ item.data }} --block.db {{ item.db_vg }}/{{ item.db }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - osd_objectstore == 'bluestore' + - item.data_vg is not defined + - item.wal is not defined + - item.wal_vg is not defined + - item.db is defined + - item.db_vg is defined + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create bluestore osds with wal only and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data {{ item.data }} --block.wal {{ item.wal_vg }}/{{ item.wal }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - osd_objectstore == 'bluestore' + - item.wal is defined + - item.data_vg is not defined + - item.wal_vg is defined + - item.db is not defined + - item.db_vg is not defined + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" + +- name: use ceph-volume to create bluestore osds with just a data device and a raw device or partition for data + command: "ceph-volume --cluster {{ cluster }} lvm create --bluestore --data {{ item.data }}" + environment: + CEPH_VOLUME_DEBUG: 1 + with_items: "{{ lvm_volumes }}" + when: + - osd_objectstore == 'bluestore' + - item.wal is not defined + - item.data_vg is not defined + - item.wal_vg is not defined + - item.db is not defined + - item.db_vg is not defined + - "'{{ item.data }}' not in ceph_volume_lvm_list.stdout" From 6dc090a4c8e0da753b6755765cc71db8e8312674 Mon Sep 17 00:00:00 2001 From: Andrew Schoen Date: Tue, 14 Nov 2017 13:19:03 -0600 Subject: [PATCH 3/3] update group_vars/osds.yml.sample for the new lvm_volumes examples Signed-off-by: Andrew Schoen --- group_vars/osds.yml.sample | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/group_vars/osds.yml.sample b/group_vars/osds.yml.sample index ee1845eb2..eabdb51ec 100644 --- a/group_vars/osds.yml.sample +++ b/group_vars/osds.yml.sample @@ -203,9 +203,9 @@ dummy: # # Filestore: Each dictionary must contain a data, journal and vg_name key. Any # logical volume or logical group used must be a name and not a path. data -# must be a logical volume. journal can be either a lv, device or partition. +# can be a logical volume, device or partition. journal can be either a lv or partition. # You can not use the same journal for many data lvs. -# data_vg must be the volume group name of the data lv +# data_vg must be the volume group name of the data lv, only applicable when data is an lv. # journal_vg is optional and must be the volume group name of the journal lv, if applicable. # For example: # lvm_volumes: @@ -214,11 +214,15 @@ dummy: # journal: journal-lv1 # journal_vg: vg2 # - data: data-lv2 -# journal: /dev/sda +# journal: /dev/sda1 # data_vg: vg1 # - data: data-lv3 # journal: /dev/sdb1 # data_vg: vg2 +# - data: /dev/sda +# journal: /dev/sdb1 +# - data: /dev/sda1 +# journal: /dev/sdb1 # # Bluestore: Each dictionary must contain at least data. When defining wal or # db, it must have both the lv name and vg group (db and wal are not required). @@ -240,6 +244,8 @@ dummy: # db_vg: vg3 # - data: data-lv4 # data_vg: vg4 +# - data: /dev/sda +# - data: /dev/sdb1 #lvm_volumes: []