diff --git a/library/ceph_volume.py b/library/ceph_volume.py index 2d281b1b8..0e965ecb2 100644 --- a/library/ceph_volume.py +++ b/library/ceph_volume.py @@ -38,7 +38,7 @@ options: description: - The action to take. Either creating OSDs or zapping devices. required: true - choices: ['create', 'zap', 'batch', 'list'] + choices: ['create', 'zap', 'batch', 'prepare', 'activate', 'list'] default: create data: description: @@ -65,7 +65,7 @@ options: required: false db_vg: description: - - If db is a lv, this must be the name of the volume group it belongs to. + - If db is a lv, this must be the name of the volume group it belongs to. # noqa E501 - Only applicable if objectstore is 'bluestore'. required: false wal: @@ -75,7 +75,7 @@ options: required: false wal_vg: description: - - If wal is a lv, this must be the name of the volume group it belongs to. + - If wal is a lv, this must be the name of the volume group it belongs to. # noqa E501 - Only applicable if objectstore is 'bluestore'. required: false crush_device_class: @@ -117,7 +117,12 @@ options: - Results will be returned in json format. - Only applicable if action is 'batch'. required: false - + containerized: + description: + - Wether or not this is a containerized cluster. The value is + assigned or not depending on how the playbook runs. + required: false + default: None author: - Andrew Schoen (@andrewschoen) @@ -130,23 +135,27 @@ EXAMPLES = ''' data: data-lv data_vg: data-vg journal: /dev/sdc1 + action: create - name: set up a bluestore osd with a raw device for data ceph_volume: objectstore: bluestore data: /dev/sdc + action: create -- name: set up a bluestore osd with an lv for data and partitions for block.wal and block.db + +- name: set up a bluestore osd with an lv for data and partitions for block.wal and block.db # noqa E501 ceph_volume: objectstore: bluestore data: data-lv data_vg: data-vg db: /dev/sdc1 wal: /dev/sdc2 + action: create ''' -from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.basic import AnsibleModule # noqa 4502 def get_data(data, data_vg): @@ -231,7 +240,8 @@ def batch(module): report = module.params['report'] if not batch_devices: - module.fail_json(msg='batch_devices must be provided if action is "batch"', changed=False, rc=1) + module.fail_json( + msg='batch_devices must be provided if action is "batch"', changed=False, rc=1) # noqa 4502 cmd = [ 'ceph-volume', @@ -326,7 +336,28 @@ def batch(module): module.exit_json(**result) -def create_osd(module): +def ceph_volume_cmd(subcommand, containerized, cluster=None): + cmd = ['ceph-volume'] + if cluster: + cmd.extend(["--cluster", cluster]) + cmd.append('lvm') + cmd.append(subcommand) + + if containerized: + cmd = containerized.split() + cmd + + return cmd + + +def activate_osd(module, containerized=None): + subcommand = "activate" + cmd = ceph_volume_cmd(subcommand) + cmd.append("--all") + + return True + + +def prepare_osd(module): cluster = module.params['cluster'] objectstore = module.params['objectstore'] data = module.params['data'] @@ -339,16 +370,12 @@ def create_osd(module): wal_vg = module.params.get('wal_vg', None) crush_device_class = module.params.get('crush_device_class', None) dmcrypt = module.params['dmcrypt'] + containerized = module.params.get('containerized', None) + subcommand = "prepare" - cmd = [ - 'ceph-volume', - '--cluster', - cluster, - 'lvm', - 'create', - '--%s' % objectstore, - '--data', - ] + cmd = ceph_volume_cmd(subcommand, containerized, cluster) + cmd.extend(["--%s" % objectstore]) + cmd.append("--data") data = get_data(data, data_vg) cmd.append(data) @@ -387,11 +414,17 @@ def create_osd(module): # check to see if osd already exists # FIXME: this does not work when data is a raw device - # support for 'lvm list' and raw devices was added with https://github.com/ceph/ceph/pull/20620 but + # support for 'lvm list' and raw devices + # was added with https://github.com/ceph/ceph/pull/20620 but # has not made it to a luminous release as of 12.2.4 - rc, out, err = module.run_command(["ceph-volume", "lvm", "list", data], encoding=None) + ceph_volume_list_cmd = ["ceph-volume", "lvm", "list", data] + if containerized: + ceph_volume_list_cmd = containerized.split() + ceph_volume_list_cmd + + rc, out, err = module.run_command(ceph_volume_list_cmd, encoding=None) if rc == 0: - result["stdout"] = "skipped, since {0} is already used for an osd".format(data) + result["stdout"] = "skipped, since {0} is already used for an osd".format( # noqa E501 + data) result['rc'] = 0 module.exit_json(**result) @@ -498,8 +531,10 @@ def zap_devices(module): def run_module(): module_args = dict( cluster=dict(type='str', required=False, default='ceph'), - objectstore=dict(type='str', required=False, choices=['bluestore', 'filestore'], default='bluestore'), - action=dict(type='str', required=False, choices=['create', 'zap', 'batch', 'list'], default='create'), + objectstore=dict(type='str', required=False, choices=[ + 'bluestore', 'filestore'], default='bluestore'), + action=dict(type='str', required=False, choices=[ + 'create', 'zap', 'batch', 'prepare', 'activate', 'list'], default='create'), # noqa 4502 data=dict(type='str', required=False), data_vg=dict(type='str', required=False), journal=dict(type='str', required=False), @@ -515,6 +550,7 @@ def run_module(): journal_size=dict(type='str', required=False, default="5120"), block_db_size=dict(type='str', required=False, default="-1"), report=dict(type='bool', required=False, default=False), + containerized=dict(type='str', required=False, default=False), ) module = AnsibleModule( @@ -525,7 +561,12 @@ def run_module(): action = module.params['action'] if action == "create": - create_osd(module) + prepare_osd(module) + activate_osd(module) + elif action == "prepare": + prepare_osd(module) + elif action == "activate": + activate_osd(module) elif action == "zap": zap_devices(module) elif action == "batch": @@ -533,7 +574,8 @@ def run_module(): elif action == "list": _list(module) - module.fail_json(msg='State must either be "present" or "absent".', changed=False, rc=1) + module.fail_json( + msg='State must either be "present" or "absent".', changed=False, rc=1) def main(): diff --git a/roles/ceph-osd/tasks/main.yml b/roles/ceph-osd/tasks/main.yml index d7ed8e83a..3b88fe19d 100644 --- a/roles/ceph-osd/tasks/main.yml +++ b/roles/ceph-osd/tasks/main.yml @@ -47,14 +47,16 @@ when: - osd_scenario == 'lvm' - lvm_volumes|length > 0 - - not containerized_deployment + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False - name: include_tasks scenarios/lvm-batch.yml include_tasks: scenarios/lvm-batch.yml when: - osd_scenario == 'lvm' - devices|length > 0 - - not containerized_deployment + # Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent) + static: False - name: include_tasks activate_osds.yml include_tasks: activate_osds.yml diff --git a/roles/ceph-osd/tasks/scenarios/lvm.yml b/roles/ceph-osd/tasks/scenarios/lvm.yml index ed53967fd..67ad9c341 100644 --- a/roles/ceph-osd/tasks/scenarios/lvm.yml +++ b/roles/ceph-osd/tasks/scenarios/lvm.yml @@ -1,4 +1,28 @@ --- +- name: set_fact docker_exec_prepare_cmd + set_fact: + docker_exec_prepare_cmd: "docker exec ceph-volume-prepare" + when: + - containerized_deployment + +- name: run a ceph-volume prepare container (sleep 3000) + command: > + docker run \ + --rm \ + --privileged=true \ + --net=host \ + -v /dev:/dev \ + -d \ + -v {{ ceph_conf_key_directory }}:{{ ceph_conf_key_directory }}:z \ + -v /var/lib/ceph/:/var/lib/ceph/:z \ + -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket \ + --name ceph-volume-prepare \ + --entrypoint=sleep \ + {{ ceph_docker_registry}}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ + 3000 + changed_when: false + when: + - containerized_deployment - name: "use ceph-volume to create {{ osd_objectstore }} osds" ceph_volume: @@ -14,6 +38,8 @@ wal_vg: "{{ item.wal_vg|default(omit) }}" crush_device_class: "{{ item.crush_device_class|default(omit) }}" dmcrypt: "{{ dmcrypt|default(omit) }}" + containerized: "{{ docker_exec_prepare_cmd | default(False) }}" + action: "{{ 'prepare' if containerized_deployment else 'create' }}" environment: CEPH_VOLUME_DEBUG: 1 - with_items: "{{ lvm_volumes }}" + with_items: "{{ lvm_volumes }}" \ No newline at end of file diff --git a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 index fc8f0a30b..ed3f47a3f 100644 --- a/roles/ceph-osd/templates/ceph-osd-run.sh.j2 +++ b/roles/ceph-osd/templates/ceph-osd-run.sh.j2 @@ -107,7 +107,12 @@ expose_partitions "$1" {% if (ceph_tcmalloc_max_total_thread_cache | int) > 0 and osd_objectstore == 'filestore' -%} -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ {% endif -%} + {% if osd_scenario == 'lvm' -%} + -v /run/lvm/lvmetad.socket:/run/lvm/lvmetad.socket \ + -e CEPH_DAEMON=OSD_CEPH_VOLUME_ACTIVATE \ + {% else -%} -e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \ + {% endif -%} {{ ceph_osd_docker_extra_env }} \ --name=ceph-osd-{{ ansible_hostname }}-${1} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}