Merge pull request #891 from ceph/dmcrypt

ceph-osd: add dmcrypt scenario

group_vars/osds.sample

@@ -85,13 +85,7 @@ dummy:
 #journal_collocation: false
 
 
-# II. Second scenario: single journal device for N OSDs
+# II. N journal devices for N OSDs
-# Use 'true' to enable this scenario
-
-# deprecated, please use scenario III with a single raw_journal_device
-
-
-# III. Third scenario: N journal devices for N OSDs
 # Use 'true' to enable this scenario
 #
 # In the following example:
@@ -109,7 +103,7 @@ dummy:
 #  - /dev/sdc
 
 
-# IV. Fourth scenario: use directory instead of disk for OSDs
+# III. Use directory instead of disk for OSDs
 # Use 'true' to enable this scenario
 
 #osd_directory: false
@@ -118,11 +112,26 @@ dummy:
 #  - /var/lib/ceph/osd/mydir2
 
 
-# V. Fith scenario: this will partition disks for BlueStore
+# IV. This will partition disks for BlueStore
 # Use 'true' to enable this scenario
 #bluestore: false
 
 
+# V. Encrypt osd data and/or journal devices with dm-crypt.
+# Keys are stored into the monitors k/v store
+# Use 'true' to enable this scenario
+# Both journal and data are stored on the same dm-crypt encrypted device
+#dmcrypt_journal_colocation: false
+
+
+# VI. Encrypt osd data and/or journal devices with dm-crypt.
+# Keys are stored into the monitors k/v store
+# Use 'true' to enable this scenario
+# Journal and osd data are separated, each with their own dm-crypt device
+# You must use raw_journal_devices and set your journal devices
+#dmcrypt_dedicated_journal: false
+
+
 ##########
 # DOCKER #
 ##########

roles/ceph-common/tasks/checks/check_mandatory_vars.yml

@@ -71,6 +71,8 @@
     - not raw_multi_journal
     - not osd_directory
     - not bluestore
+    - not dmcrypt_journal_colocation
+    - not dmcrypt_dedicated_journal
 
 - name: verify only one osd scenario was chosen
   fail:
@@ -84,6 +86,14 @@
       or (raw_multi_journal and osd_directory)
       or (raw_multi_journal and bluestore)
       or (osd_directory and bluestore)
+      or (dmcrypt_journal_colocation and journal_collocation)
+      or (dmcrypt_journal_colocation and raw_multi_journal)
+      or (dmcrypt_journal_colocation and osd_directory)
+      or (dmcrypt_journal_colocation and bluestore)
+      or (dmcrypt_dedicated_journal and journal_collocation)
+      or (dmcrypt_dedicated_journal and raw_multi_journal)
+      or (dmcrypt_dedicated_journal and osd_directory)
+      or (dmcrypt_dedicated_journal and bluestore)
 
 - name: verify devices have been provided
   fail:

roles/ceph-osd/defaults/main.yml

@@ -77,13 +77,7 @@ osd_auto_discovery: false
 journal_collocation: false
 
 
-# II. Second scenario: single journal device for N OSDs
+# II. N journal devices for N OSDs
-# Use 'true' to enable this scenario
-
-# deprecated, please use scenario III with a single raw_journal_device
-
-
-# III. Third scenario: N journal devices for N OSDs
 # Use 'true' to enable this scenario
 #
 # In the following example:
@@ -101,7 +95,7 @@ raw_multi_journal: false
 #  - /dev/sdc
 
 
-# IV. Fourth scenario: use directory instead of disk for OSDs
+# III. Use directory instead of disk for OSDs
 # Use 'true' to enable this scenario
 
 osd_directory: false
@@ -110,11 +104,26 @@ osd_directory: false
 #  - /var/lib/ceph/osd/mydir2
 
 
-# V. Fith scenario: this will partition disks for BlueStore
+# IV. This will partition disks for BlueStore
 # Use 'true' to enable this scenario
 bluestore: false
 
 
+# V. Encrypt osd data and/or journal devices with dm-crypt.
+# Keys are stored into the monitors k/v store
+# Use 'true' to enable this scenario
+# Both journal and data are stored on the same dm-crypt encrypted device
+dmcrypt_journal_colocation: false
+
+
+# VI. Encrypt osd data and/or journal devices with dm-crypt.
+# Keys are stored into the monitors k/v store
+# Use 'true' to enable this scenario
+# Journal and osd data are separated, each with their own dm-crypt device
+# You must use raw_journal_devices and set your journal devices
+dmcrypt_dedicated_journal: false
+
+
 ##########
 # DOCKER #
 ##########

roles/ceph-osd/tasks/activate_osds.yml

@@ -25,13 +25,45 @@
     - not item.0.get("skipped")
     - item.0.get("rc", 0) != 0
     - not osd_auto_discovery
+    - raw_multi_journal
+
+- name: automatically activate osd disk(s) without partitions (dmcrypt)
+  command: ceph-disk activate --dmcrypt "/dev/{{ item.key }}"
+  ignore_errors: true
+  with_dict: ansible_devices
+  when:
+    - ansible_devices is defined
+    - item.value.removable == "0"
+    - item.value.partitions|count == 0
+    - osd_auto_discovery
+    - dmcrypt_journal_colocation
+
+- name: activate osd(s) when device is a disk (dmcrypt)
+  command: ceph-disk activate --dmcrypt {{ item.1 | regex_replace('^(\/dev\/cciss\/c[0-9]{1}d[0-9]{1})$', '\\1p') }}1
+  with_together:
+    - combined_ispartition_results.results
+    - devices
+  changed_when: false
+  failed_when: false
+  register: activate_osd_disk_dmcrypt
+  when:
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
+    - not osd_auto_discovery
+    - dmcrypt_dedicated_journal
+
+# NOTE (leseb): we must do this because of
+# https://github.com/ansible/ansible/issues/4297
+- name: combine ispartition results
+  set_fact:
+    combined_activate_osd_disk_results: "{{ activate_osd_disk if not dmcrypt_journal_colocation else activate_osd_disk_dmcrypt }}"
 
 - name: fail if ceph-disk cannot create an OSD
   fail:
     msg: "ceph-disk failed to create an OSD"
   when:
     " 'ceph-disk: Error: ceph osd create failed' in item.get('stderr', '') "
-  with_items: "{{activate_osd_disk.results}}"
+  with_items: "{{combined_activate_osd_disk_results.results}}"
 
 # NOTE (leseb): this task is for partitions because we don't explicitly use a partition.
 - name: activate osd(s) when device is a partition

roles/ceph-osd/tasks/main.yml

@@ -22,5 +22,15 @@
     - osd_objectstore == 'bluestore'
     - not osd_containerized_deployment
 
+- include: ./scenarios/dmcrypt-journal-colocation.yml
+  when:
+    - dmcrypt_journal_colocation
+    - not osd_containerized_deployment
+
+- include: ./scenarios/dmcrypt-dedicated-journal.yml
+  when:
+    - dmcrypt_dedicated_journal
+    - not osd_containerized_deployment
+
 - include: ./docker/main.yml
   when: osd_containerized_deployment

roles/ceph-osd/tasks/pre_requisite.yml

@@ -31,6 +31,12 @@
   when:
     cephx
 
+- name: copy ceph admin key when using dmcrypt
+  set_fact:
+    copy_admin_key: true
+  when:
+    - dmcrypt_journal_colocation or dmcrypt_dedicated_journal
+
 - name: copy osd bootstrap key
   copy:
     src: "{{ fetch_directory }}/{{ fsid }}{{ item.name }}"

roles/ceph-osd/tasks/scenarios/dmcrypt-dedicated-journal.yml

@@ -0,0 +1,29 @@
+---
+## SCENARIO 6: DMCRYPT N JOURNAL DEVICES FOR N OSDS
+
+- include: ../check_devices.yml
+
+# NOTE (leseb): the prepare process must be parallelized somehow...
+# if you have 64 disks with 4TB each, this will take a while
+# since Ansible will sequential process the loop
+
+# NOTE (alahouze): if the device is a partition, the parted command below has
+# failed, this is why we check if the device is a partition too.
+- name: prepare osd disk(s)
+  command: "ceph-disk prepare --dmcrypt --cluster {{ cluster }} {{ item.2 }} {{ item.3 }}"
+  with_together:
+    - combined_parted_results.results
+    - combined_ispartition_results.results
+    - devices
+    - raw_journal_devices
+  changed_when: false
+  when:
+    - not item.0.get("skipped")
+    - not item.1.get("skipped")
+    - item.0.get("rc", 0) != 0
+    - item.1.get("rc", 0) != 0
+    - raw_multi_journal
+    - not osd_auto_discovery
+    - dmcrypt_dedicated_journal
+
+- include: ../activate_osds.yml

roles/ceph-osd/tasks/scenarios/dmcrypt-journal-colocation.yml

@@ -0,0 +1,36 @@
+---
+## SCENARIO 5: DMCRYPT
+
+- include: ../check_devices.yml
+
+# NOTE (leseb): the prepare process must be parallelized somehow...
+# if you have 64 disks with 4TB each, this will take a while
+# since Ansible will sequential process the loop
+
+# NOTE (alahouze): if the device is a partition, the parted command below has
+# failed, this is why we check if the device is a partition too.
+- name: automatic prepare osd disk(s) without partitions (dmcrypt)
+  command: ceph-disk prepare --dmcrypt --cluster "{{ cluster }}" "/dev/{{ item.key }}"
+  with_dict: ansible_devices
+  when:
+    - ansible_devices is defined
+    - item.value.removable == "0"
+    - item.value.partitions|count == 0
+    - dmcrypt_journal_colocation
+    - osd_auto_discovery
+
+- name: manually prepare osd disk(s) (dmcrypt)
+  command: ceph-disk prepare --dmcrypt --cluster "{{ cluster }}" "{{ item.2 }}"
+  with_together:
+    - combined_parted_results.results
+    - combined_ispartition_results.results
+    - devices
+  when:
+    - not item.0.get("skipped")
+    - not item.1.get("skipped")
+    - item.0.get("rc", 0) != 0
+    - item.1.get("rc", 0) != 0
+    - dmcrypt_journal_colocation
+    - not osd_auto_discovery
+
+- include: ../activate_osds.yml