Merge pull request #1632 from ceph/bluestore-redundant-options

osd: remove redundant options to enable bluestore

group_vars/osds.yml.sample

@@ -103,8 +103,18 @@ dummy:
 # This will collocate both journal and data on the same disk
 # creating a partition at the beginning of the device
 # List devices under 'devices' variable above or choose 'osd_auto_discovery'
-
+#
-
+#
+# If osd_objectstore: bluestore is enabled both rocksdb DB and WAL will be stored
+# on the device. So the device will get 2 partitions:
+# - One for 'data', also called 'block'
+# - One for block, db, and wal data
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sda*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="9c43e346-dd6e-431f-92d8-cbed4ccb25f6" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="749c71c9-ed8f-4930-82a7-a48a3bcdb1c7"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="e6ca3e1d-4702-4569-abfa-e285de328e9d"
 #journal_collocation: false
 
 
@@ -130,12 +140,48 @@ dummy:
 # On a containerized scenario we only support A SINGLE journal
 # for all the OSDs on a given machine. If you don't, bad things will happen
 # This is a limitation we plan to fix at some point.
+#
+#
+# If osd_objectstore: bluestore is enabled, both rocksdb DB and WAL will be stored
+# on a dedicated device. So the following will happen:
+# - The devices listed in 'devices' will get 2 partitions, one for 'block' and one for 'data'.
+# 'data' is only 100MB big and do not store any of your data, it's just a bunch of Ceph metadata.
+# 'block' will store all your data.
+# - The devices in 'raw_journal_devices' will get 1 partition for RocksDB DB, called 'block.db'
+#  and one for RocksDB WAL, called 'block.wal'
+#
+# By default raw_journal_devices will represent block.db
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sd*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="c6821801-2f21-4980-add0-b7fc8bd424d5" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="f2cc6fa8-5b41-4428-8d3f-6187453464d0"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="ea454807-983a-4cf2-899e-b2680643bc1c"
+# /dev/sdb: PTTYPE="gpt"
+# /dev/sdb1: PARTLABEL="ceph block.db" PARTUUID="af5b2d74-4c08-42cf-be57-7248c739e217"
+# /dev/sdb2: PARTLABEL="ceph block.wal" PARTUUID="af3f8327-9aa9-4c2b-a497-cf0fe96d126a"
 #raw_journal_devices: []
 
 
 # IV. This will partition disks for BlueStore
-# Use 'true' to enable this scenario
+# To enable bluestore just set:
-#bluestore: false
+# osd_objectstore: bluestore
+#
+# If osd_objectstore: bluestore is enabled.
+# By default, if bluestore_wal_devices is empty, it will get the content of raw_journal_devices.
+# If set, then you will have a dedicated partition on a specific device (bluestore_wal_devices)
+# for block.wal
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sd*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="39241ae9-d119-4335-96b3-0898da8f45ce" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="961e7313-bdb7-49e7-9ae7-077d65c4c669"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="bff8e54e-b780-4ece-aa16-3b2f2b8eb699"
+# /dev/sdb: PTTYPE="gpt"
+# /dev/sdb1: PARTLABEL="ceph block.db" PARTUUID="0734f6b6-cc94-49e9-93de-ba7e1d5b79e3"
+# /dev/sdc: PTTYPE="gpt"
+# /dev/sdc1: PARTLABEL="ceph block.wal" PARTUUID="824b84ba-6777-4272-bbbd-bfe2a25cecf3"
+#bluestore_wal_devices: "{{ raw_journal_devices }}"
 
 
 # V. Encrypt osd data and/or journal devices with dm-crypt.
@@ -166,18 +212,26 @@ dummy:
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above
 #
 # Examples:
-# Journal collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+# Journal collocated on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_FILESTORE=1
-# Dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Dedicated journal on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
-# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1
+# Encrypted OSD on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
-# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Encrypted OSD with dedicated journal on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
+#
+# Bluestore OSD collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1
+# Bluestore OSD with dedicated db: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Bluestore OSD with dedicated db and wal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_BLUESTORE_BLOCK_WAL={{ bluestore_wal_devices[0] }}
+# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
+# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
+#
 #
 #ceph_osd_docker_devices: "{{ devices }}"
-#ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+#ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_BLUESTORE=1
 
 # ACTIVATE DEVICE
 # Examples:
-# Journal collocated or Dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+# Journal collocated or Dedicated journal on Filesore: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FILESTORE=1
-# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1
+# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
+# Bluestore OSD: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_BLUESTORE=1
 #
 #ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }}
 #ceph_osd_docker_run_script_path: "/usr/share" # script called by systemd to run the docker command

roles/ceph-osd/defaults/main.yml

@@ -95,8 +95,18 @@ osd_auto_discovery: false
 # This will collocate both journal and data on the same disk
 # creating a partition at the beginning of the device
 # List devices under 'devices' variable above or choose 'osd_auto_discovery'
-
+#
-
+#
+# If osd_objectstore: bluestore is enabled both rocksdb DB and WAL will be stored
+# on the device. So the device will get 2 partitions:
+# - One for 'data', also called 'block'
+# - One for block, db, and wal data
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sda*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="9c43e346-dd6e-431f-92d8-cbed4ccb25f6" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="749c71c9-ed8f-4930-82a7-a48a3bcdb1c7"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="e6ca3e1d-4702-4569-abfa-e285de328e9d"
 journal_collocation: false
 
 
@@ -122,12 +132,48 @@ raw_multi_journal: false
 # On a containerized scenario we only support A SINGLE journal
 # for all the OSDs on a given machine. If you don't, bad things will happen
 # This is a limitation we plan to fix at some point.
+#
+#
+# If osd_objectstore: bluestore is enabled, both rocksdb DB and WAL will be stored
+# on a dedicated device. So the following will happen:
+# - The devices listed in 'devices' will get 2 partitions, one for 'block' and one for 'data'.
+# 'data' is only 100MB big and do not store any of your data, it's just a bunch of Ceph metadata.
+# 'block' will store all your data.
+# - The devices in 'raw_journal_devices' will get 1 partition for RocksDB DB, called 'block.db'
+#  and one for RocksDB WAL, called 'block.wal'
+#
+# By default raw_journal_devices will represent block.db
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sd*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="c6821801-2f21-4980-add0-b7fc8bd424d5" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="f2cc6fa8-5b41-4428-8d3f-6187453464d0"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="ea454807-983a-4cf2-899e-b2680643bc1c"
+# /dev/sdb: PTTYPE="gpt"
+# /dev/sdb1: PARTLABEL="ceph block.db" PARTUUID="af5b2d74-4c08-42cf-be57-7248c739e217"
+# /dev/sdb2: PARTLABEL="ceph block.wal" PARTUUID="af3f8327-9aa9-4c2b-a497-cf0fe96d126a"
 raw_journal_devices: []
 
 
 # IV. This will partition disks for BlueStore
-# Use 'true' to enable this scenario
+# To enable bluestore just set:
-bluestore: false
+# osd_objectstore: bluestore
+#
+# If osd_objectstore: bluestore is enabled.
+# By default, if bluestore_wal_devices is empty, it will get the content of raw_journal_devices.
+# If set, then you will have a dedicated partition on a specific device (bluestore_wal_devices)
+# for block.wal
+#
+# Example of what you will get:
+# [root@ceph-osd0 ~]# blkid /dev/sd*
+# /dev/sda: PTTYPE="gpt"
+# /dev/sda1: UUID="39241ae9-d119-4335-96b3-0898da8f45ce" TYPE="xfs" PARTLABEL="ceph data" PARTUUID="961e7313-bdb7-49e7-9ae7-077d65c4c669"
+# /dev/sda2: PARTLABEL="ceph block" PARTUUID="bff8e54e-b780-4ece-aa16-3b2f2b8eb699"
+# /dev/sdb: PTTYPE="gpt"
+# /dev/sdb1: PARTLABEL="ceph block.db" PARTUUID="0734f6b6-cc94-49e9-93de-ba7e1d5b79e3"
+# /dev/sdc: PTTYPE="gpt"
+# /dev/sdc1: PARTLABEL="ceph block.wal" PARTUUID="824b84ba-6777-4272-bbbd-bfe2a25cecf3"
+bluestore_wal_devices: "{{ raw_journal_devices }}"
 
 
 # V. Encrypt osd data and/or journal devices with dm-crypt.
@@ -158,18 +204,26 @@ ceph_config_keys: [] # DON'T TOUCH ME
 # WARNING /!\ DMCRYPT scenario ONLY works with Docker version 1.12.5 and above
 #
 # Examples:
-# Journal collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+# Journal collocated on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_FILESTORE=1
-# Dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Dedicated journal on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
-# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1
+# Encrypted OSD on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
-# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Encrypted OSD with dedicated journal on Filestore: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
+#
+# Bluestore OSD collocated: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1
+# Bluestore OSD with dedicated db: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }}
+# Bluestore OSD with dedicated db and wal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_BLUESTORE=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_BLUESTORE_BLOCK_WAL={{ bluestore_wal_devices[0] }}
+# Encrypted OSD: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
+# Encrypted OSD with dedicated journal: ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_DMCRYPT=1 -e OSD_JOURNAL={{ raw_journal_devices[0] }} -e OSD_FILESTORE=1
+#
 #
 ceph_osd_docker_devices: "{{ devices }}"
-ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
+ceph_osd_docker_prepare_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1 -e OSD_BLUESTORE=1
 
 # ACTIVATE DEVICE
 # Examples:
-# Journal collocated or Dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }}
+# Journal collocated or Dedicated journal on Filesore: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FILESTORE=1
-# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1
+# Encrypted OSD or Encrypted OSD with dedicated journal: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_DMCRYPT=1 -e OSD_FILESTORE=1
+# Bluestore OSD: ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_BLUESTORE=1
 #
 ceph_osd_docker_extra_env: -e CLUSTER={{ cluster }} -e OSD_JOURNAL_SIZE={{ journal_size }}
 ceph_osd_docker_run_script_path: "/usr/share" # script called by systemd to run the docker command

roles/ceph-osd/tasks/check_mandatory_vars.yml

@@ -16,7 +16,7 @@
     msg: "WARNING: journal_size is configured to less than 5GB. This is not recommended and can lead to severe issues."
   when:
     - journal_size|int < 5120
-    - osd_objectstore != 'bluestore'
+    - osd_objectstore == 'filestore'
     - osd_group_name in group_names
 
 - name: make sure an osd scenario was chosen
@@ -28,7 +28,6 @@
     - not containerized_deployment
     - not journal_collocation
     - not raw_multi_journal
-    - not bluestore
     - not dmcrypt_journal_collocation
     - not dmcrypt_dedicated_journal
 
@@ -40,14 +39,10 @@
     - osd_group_name in group_names
     - not containerized_deployment
     - (journal_collocation and raw_multi_journal)
-      or (journal_collocation and bluestore)
-      or (raw_multi_journal and bluestore)
       or (dmcrypt_journal_collocation and journal_collocation)
       or (dmcrypt_journal_collocation and raw_multi_journal)
-      or (dmcrypt_journal_collocation and bluestore)
       or (dmcrypt_dedicated_journal and journal_collocation)
       or (dmcrypt_dedicated_journal and raw_multi_journal)
-      or (dmcrypt_dedicated_journal and bluestore)
       or (dmcrypt_dedicated_journal and dmcrypt_journal_collocation)
 
 - name: verify devices have been provided
@@ -71,3 +66,12 @@
     - raw_journal_devices|length == 0
       or devices|length == 0
 
+- name: check if bluestore is supported by the selected ceph version
+  fail:
+    msg: "bluestore is not supported by the selected Ceph version, use Luminous or above."
+  when:
+    - osd_group_name is defined
+    - osd_group_name in group_names
+    - not containerized_deployment
+    - osd_objectstore == 'bluestore'
+    - ceph_release_num.{{ ceph_release }} < ceph_release_num.luminous

roles/ceph-osd/tasks/main.yml

@@ -20,13 +20,6 @@
   # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
   static: False
 
-- include: ./scenarios/bluestore.yml
-  when:
-    - osd_objectstore == 'bluestore'
-    - not containerized_deployment
-  # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
-  static: False
-
 - include: ./scenarios/dmcrypt-journal-collocation.yml
   when:
     - dmcrypt_journal_collocation

roles/ceph-osd/tasks/scenarios/bluestore.yml

@@ -1,38 +0,0 @@
----
-## SCENARIO 4: BLUESTORE
-
-- include: ../check_devices.yml
-
-# NOTE (leseb): the prepare process must be parallelized somehow...
-# if you have 64 disks with 4TB each, this will take a while
-# since Ansible will sequential process the loop
-
-# NOTE (alahouze): if the device is a partition, the parted command below has
-# failed, this is why we check if the device is a partition too.
-- name: automatic prepare bluestore osd disk(s) without partitions
-  command: ceph-disk prepare --bluestore --cluster "{{ cluster }}" "/dev/{{ item.key }}"
-  register: prepared_osds
-  with_dict: "{{ ansible_devices }}"
-  when:
-    - ansible_devices is defined
-    - item.value.removable == "0"
-    - item.value.partitions|count == 0
-    - item.value.holders|count == 0
-    - bluestore
-    - osd_auto_discovery
-
-- name: manually prepare bluestore osd disk(s)
-  command: ceph-disk prepare --bluestore --cluster "{{ cluster }}" "{{ item.2 }}"
-  with_together:
-    - "{{ parted_results.results }}"
-    - "{{ ispartition_results.results }}"
-    - "{{ devices }}"
-  when:
-    - not item.0.get("skipped")
-    - not item.1.get("skipped")
-    - item.0.get("rc", 0) != 0
-    - item.1.get("rc", 0) != 0
-    - bluestore
-    - not osd_auto_discovery
-
-- include: ../activate_osds.yml

roles/ceph-osd/tasks/scenarios/journal_collocation.yml

@@ -19,6 +19,7 @@
     - item.value.partitions|count == 0
     - item.value.holders|count == 0
     - journal_collocation
+    - osd_objectstore == 'filestore'
     - osd_auto_discovery
 
 - name: manually prepare filestore osd disk(s) with collocated osd data and journal
@@ -33,6 +34,35 @@
     - item.0.get("rc", 0) != 0
     - item.1.get("rc", 0) != 0
     - journal_collocation
+    - osd_objectstore == 'filestore'
+    - not osd_auto_discovery
+
+- name: automatic prepare bluestore osd disk(s) without partitions
+  command: ceph-disk prepare --bluestore --cluster "{{ cluster }}" "/dev/{{ item.key }}"
+  register: prepared_osds
+  with_dict: "{{ ansible_devices }}"
+  when:
+    - ansible_devices is defined
+    - item.value.removable == "0"
+    - item.value.partitions|count == 0
+    - item.value.holders|count == 0
+    - journal_collocation
+    - osd_objectstore == 'bluestore'
+    - osd_auto_discovery
+
+- name: manually prepare bluestore osd disk(s)
+  command: ceph-disk prepare --bluestore --cluster "{{ cluster }}" "{{ item.2 }}"
+  with_together:
+    - "{{ parted_results.results }}"
+    - "{{ ispartition_results.results }}"
+    - "{{ devices }}"
+  when:
+    - not item.0.get("skipped")
+    - not item.1.get("skipped")
+    - item.0.get("rc", 0) != 0
+    - item.1.get("rc", 0) != 0
+    - journal_collocation
+    - osd_objectstore == 'bluestore'
     - not osd_auto_discovery
 
 - include: ../activate_osds.yml

roles/ceph-osd/tasks/scenarios/raw_multi_journal.yml

@@ -17,6 +17,21 @@
   when:
     - item.0.get("skipped") or item.0.get("rc", 0) != 0
     - raw_multi_journal
+    - osd_objectstore == 'filestore'
+    - not osd_auto_discovery
+
+- name: manually prepare bluestore osd disk(s) with a dedicated device for db and wal
+  command: "ceph-disk prepare --bluestore --cluster {{ cluster }} --block.db {{ item.1 }} --block.wal {{ item.2 }} {{ item.3 }}"
+  with_together:
+    - "{{ parted_results.results }}"
+    - "{{ raw_journal_devices }}"
+    - "{{ bluestore_wal_devices }}"
+    - "{{ devices }}"
+  when:
+    - not item.0.get("skipped")
+    - item.0.get("rc", 0) != 0
+    - raw_multi_journal
+    - osd_objectstore == 'bluestore'
     - not osd_auto_discovery
 
 - include: ../activate_osds.yml

tests/functional/centos/7/bs-cluster/Vagrantfile

@@ -0,0 +1 @@
+../../../../../Vagrantfile

tests/functional/centos/7/bs-cluster/group_vars/all

@@ -0,0 +1,23 @@
+---
+
+ceph_stable: True
+cluster: test
+monitor_interface: eth1
+public_network: "192.168.1.0/24"
+cluster_network: "192.168.2.0/24"
+journal_size: 100
+osd_objectstore: "bluestore"
+devices:
+  - '/dev/sda'
+raw_journal_devices:
+  - '/dev/sdb'
+raw_multi_journal: True
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }
+user_config: True
+openstack_config: True
+ceph_conf_overrides:
+  global:
+    osd_pool_default_pg_num: 8
+    osd_pool_default_size: 1

tests/functional/centos/7/bs-cluster/group_vars/osds

@@ -0,0 +1,4 @@
+---
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tests/functional/centos/7/bs-cluster/group_vars/rgws

@@ -0,0 +1,8 @@
+copy_admin_key: true
+create_pools:
+  foo:
+    pg_num: 17
+  bar:
+    pg_num: 19
+rgw_override_bucket_index_max_shards: 16
+rgw_bucket_default_quota_max_objects: 1638400

tests/functional/centos/7/bs-cluster/hosts

@@ -0,0 +1,19 @@
+[mons]
+mon0 monitor_address=192.168.1.10
+mon1 monitor_interface=eth1
+mon2 monitor_address=192.168.1.12
+
+[osds]
+osd0
+
+[mdss]
+mds0
+
+[rgws]
+rgw0
+
+[clients]
+client0
+
+[mgrs]
+mgr0

tests/functional/centos/7/bs-cluster/vagrant_variables.yml

@@ -0,0 +1,74 @@
+---
+
+# DEPLOY CONTAINERIZED DAEMONS
+docker: false
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 3
+osd_vms: 1
+mds_vms: 1
+rgw_vms: 1
+nfs_vms: 0
+rbd_mirror_vms: 0
+client_vms: 1
+iscsi_gw_vms: 0
+mgr_vms: 1
+
+# Deploy RESTAPI on each of the Monitors
+restapi: true
+
+# INSTALL SOURCE OF CEPH
+# valid values are 'stable' and 'dev'
+ceph_install_source: stable
+
+# SUBNETS TO USE FOR THE VMS
+public_subnet: 192.168.1
+cluster_subnet: 192.168.2
+
+# MEMORY
+# set 1024 for CentOS
+memory: 512
+
+# Ethernet interface name
+# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial
+eth: 'eth1'
+
+# Disks
+# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]"
+# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]"
+disks: "[ '/dev/sdb', '/dev/sdc' ]"
+
+# VAGRANT BOX
+# Ceph boxes are *strongly* suggested. They are under better control and will
+# not get updated frequently unless required for build systems. These are (for
+# now):
+#
+# * ceph/ubuntu-xenial
+#
+# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64
+# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet
+# libvirt CentOS: centos/7
+# parallels Ubuntu: parallels/ubuntu-14.04
+# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller'
+# For more boxes have a look at:
+#   - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q=
+#   - https://download.gluster.org/pub/gluster/purpleidea/vagrant/
+vagrant_box: centos/7
+#ssh_private_key_path: "~/.ssh/id_rsa"
+# The sync directory changes based on vagrant box
+# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
+#vagrant_sync_dir: /home/vagrant/sync
+#vagrant_sync_dir: /
+# Disables synced folder creation. Not needed for testing, will skip mounting
+# the vagrant directory on the remote box regardless of the provider.
+vagrant_disable_synced_folder: true
+# VAGRANT URL
+# This is a URL to download an image from an alternate location.  vagrant_box
+# above should be set to the filename of the image.
+# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box
+# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tests/functional/centos/7/bs-journal-collocation/Vagrantfile

@@ -0,0 +1 @@
+../../../../../Vagrantfile

tests/functional/centos/7/bs-journal-collocation/group_vars/all

@@ -0,0 +1,16 @@
+---
+
+ceph_stable: True
+cluster: test
+monitor_interface: eth1
+public_network: "192.168.3.0/24"
+cluster_network: "192.168.4.0/24"
+journal_size: 100
+devices:
+  - '/dev/sda'
+  - '/dev/sdb'
+journal_collocation: True
+osd_objectstore: "bluestore"
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tests/functional/centos/7/bs-journal-collocation/group_vars/osds

@@ -0,0 +1,4 @@
+---
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tests/functional/centos/7/bs-journal-collocation/hosts

@@ -0,0 +1,5 @@
+[mons]
+mon0 monitor_interface=eth1
+
+[osds]
+osd0

tests/functional/centos/7/bs-journal-collocation/vagrant_variables.yml

@@ -0,0 +1,74 @@
+---
+
+# DEPLOY CONTAINERIZED DAEMONS
+docker: false
+
+# DEFINE THE NUMBER OF VMS TO RUN
+mon_vms: 1
+osd_vms: 1
+mds_vms: 0
+rgw_vms: 0
+nfs_vms: 0
+rbd_mirror_vms: 0
+client_vms: 0
+iscsi_gw_vms: 0
+mgr_vms: 0
+
+# Deploy RESTAPI on each of the Monitors
+restapi: true
+
+# INSTALL SOURCE OF CEPH
+# valid values are 'stable' and 'dev'
+ceph_install_source: stable
+
+# SUBNETS TO USE FOR THE VMS
+public_subnet: 192.168.3
+cluster_subnet: 192.168.4
+
+# MEMORY
+# set 1024 for CentOS
+memory: 512
+
+# Ethernet interface name
+# use eth1 for libvirt and ubuntu precise, enp0s8 for CentOS and ubuntu xenial
+eth: 'eth1'
+
+# Disks
+# For libvirt use disks: "[ '/dev/vdb', '/dev/vdc' ]"
+# For CentOS7 use disks: "[ '/dev/sda', '/dev/sdb' ]"
+disks: "[ '/dev/sdb', '/dev/sdc' ]"
+
+# VAGRANT BOX
+# Ceph boxes are *strongly* suggested. They are under better control and will
+# not get updated frequently unless required for build systems. These are (for
+# now):
+#
+# * ceph/ubuntu-xenial
+#
+# Ubuntu: ceph/ubuntu-xenial bento/ubuntu-16.04 or ubuntu/trusty64 or ubuntu/wily64
+# CentOS: bento/centos-7.1 or puppetlabs/centos-7.0-64-puppet
+# libvirt CentOS: centos/7
+# parallels Ubuntu: parallels/ubuntu-14.04
+# Debian: deb/jessie-amd64 - be careful the storage controller is named 'SATA Controller'
+# For more boxes have a look at:
+#   - https://atlas.hashicorp.com/boxes/search?utf8=✓&sort=&provider=virtualbox&q=
+#   - https://download.gluster.org/pub/gluster/purpleidea/vagrant/
+vagrant_box: centos/7
+#ssh_private_key_path: "~/.ssh/id_rsa"
+# The sync directory changes based on vagrant box
+# Set to /home/vagrant/sync for Centos/7, /home/{ user }/vagrant for openstack and defaults to /vagrant
+#vagrant_sync_dir: /home/vagrant/sync
+#vagrant_sync_dir: /
+# Disables synced folder creation. Not needed for testing, will skip mounting
+# the vagrant directory on the remote box regardless of the provider.
+vagrant_disable_synced_folder: true
+# VAGRANT URL
+# This is a URL to download an image from an alternate location.  vagrant_box
+# above should be set to the filename of the image.
+# Fedora virtualbox: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+# Fedora libvirt: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-libvirt.box
+# vagrant_box_url: https://download.fedoraproject.org/pub/fedora/linux/releases/22/Cloud/x86_64/Images/Fedora-Cloud-Base-Vagrant-22-20150521.x86_64.vagrant-virtualbox.box
+
+os_tuning_params:
+  - { name: kernel.pid_max, value: 4194303 }
+  - { name: fs.file-max, value: 26234859 }

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = {jewel,kraken,rhcs}-{ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster}
+envlist = {jewel,kraken,rhcs}-{ansible2.2}-{xenial_cluster,journal_collocation,centos7_cluster,dmcrypt_journal,dmcrypt_journal_collocation,docker_cluster,purge_cluster,purge_dmcrypt,docker_dedicated_journal,docker_dmcrypt_journal_collocation,update_dmcrypt,update_cluster,cluster,purge_docker_cluster,update_docker_cluster,bluestore_journal_collocation,bluestore_cluster}
 skipsdist = True
 
 # extra commands for purging clusters
@@ -97,6 +97,9 @@ changedir=
   purge_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_dmcrypt: {toxinidir}/tests/functional/centos/7/dmcrypt-dedicated-journal
   update_cluster: {toxinidir}/tests/functional/centos/7/cluster
+  bluestore_journal_collocation: {toxinidir}/tests/functional/centos/7/bs-journal-collocation
+  bluestore_cluster: {toxinidir}/tests/functional/centos/7/bs-cluster
+
 commands=
   rhcs: ansible-playbook -vv -i "localhost," -c local {toxinidir}/tests/functional/rhcs_setup.yml --extra-vars "change_dir={changedir}" --tags "vagrant_setup"