Merge pull request #1956 from ceph/osd-container-id

Osd container
pull/1979/head
Sébastien Han 2017-10-03 18:52:24 +02:00 committed by GitHub
commit 0ce76113bf
46 changed files with 155 additions and 69 deletions

View File

@ -278,10 +278,21 @@
with_items: "{{ devices }}"
ignore_errors: true
- name: resolve device link
command: readlink -f {{ item }}
changed_when: false
with_items: "{{ devices }}"
register: purge_devices_links
- name: set_fact devices generate device list when osd_auto_discovery
set_fact:
devices: "{{ devices | default([]) + [ item.stdout ] }}"
with_items: "{{ purge_devices_links.results }}"
- name: remove ceph osd prepare container
docker:
image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
name: "ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
name: "ceph-osd-prepare-{{ ansible_hostname }}{{ item | regex_replace('/dev/', '') }}"
state: absent
with_items: "{{ devices }}"
ignore_errors: true
@ -289,7 +300,7 @@
- name: remove ceph osd container
docker:
image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
name: "ceph-osd-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
name: "ceph-osd-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }}"
state: absent
with_items: "{{ devices }}"
ignore_errors: true
@ -298,7 +309,7 @@
shell: |
docker run \
--privileged=true \
--name ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }} \
--name ceph-osd-zap-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }} \
-v /dev/:/dev/ \
-e OSD_DEVICE={{ item }} \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
@ -309,7 +320,7 @@
- name: wait until the zap containers die
shell: |
docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}-dev
docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}
register: zap_alive
failed_when: false
until: zap_alive.rc != 0
@ -319,7 +330,7 @@
- name: remove ceph osd zap disk container
docker:
image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
name: "ceph-osd-zap-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
name: "ceph-osd-zap-{{ ansible_hostname }}-{{ item | regex_replace('/dev/', '') }}"
state: absent
with_items:
- "{{ devices }}"

View File

@ -263,7 +263,7 @@
- name: check for a ceph socket in containerized deployment (osds)
shell: |
docker exec ceph-osd-"{{ ansible_hostname }}"-"{{ item | replace('/', '') }}" bash -c 'stat --printf=%n /var/run/ceph/*.asok'
docker exec ceph-osd-"{{ ansible_hostname }}"-"{{ item | replace('/dev/', '') }}" bash -c 'stat --printf=%n /var/run/ceph/*.asok'
changed_when: false
failed_when: false
always_run: true

View File

@ -124,3 +124,24 @@
when:
- rbd_client_directory_mode is not defined
or not rbd_client_directory_mode
- name: resolve device link(s)
command: readlink -f {{ item }}
changed_when: false
with_items: "{{ devices }}"
register: devices_prepare_canonicalize
when:
- inventory_hostname in groups.get(osd_group_name, [])
- name: set_fact build devices from resolved symlinks
set_fact:
devices: "{{ devices | default([]) + [ item.stdout ] }}"
with_items: "{{ devices_prepare_canonicalize.results }}"
when:
- inventory_hostname in groups.get(osd_group_name, [])
- name: set_fact build final devices list
set_fact:
devices: "{{ devices | reject('search','/dev/disk') | list | unique }}"
when:
- inventory_hostname in groups.get(osd_group_name, [])

View File

@ -36,4 +36,4 @@
- name: include generate_crt.yml
include: generate_crt.yml
when: generate_crt|bool|default(false)
when: generate_crt|default(false)|bool

View File

@ -10,6 +10,27 @@
# allow 2-digit partition numbers so fast SSDs can be shared by > 9 disks
# for SSD journals.
- name: resolve dedicated device link(s)
command: readlink -f {{ item }}
changed_when: false
with_items: "{{ dedicated_devices }}"
register: dedicated_devices_prepare_canonicalize
when:
- osd_scenario == 'non-collocated'
- name: set_fact build dedicated_devices from resolved symlinks
set_fact:
dedicated_devices: "{{ dedicated_devices | default([]) + [ item.stdout ] }}"
with_items: "{{ dedicated_devices_prepare_canonicalize.results }}"
when:
- osd_scenario == 'non-collocated'
- name: set_fact build final dedicated_devices list
set_fact:
dedicated_devices: "{{ dedicated_devices | reject('search','/dev/disk') | list | unique }}"
when:
- osd_scenario == 'non-collocated'
- name: include check_devices_static.yml
include: check_devices_static.yml
# Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
@ -33,10 +54,10 @@
- "{{ dedicated_devices|unique }}"
changed_when: false
when:
- not containerized_deployment
- osd_scenario == 'non-collocated'
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- osd_scenario == 'non-collocated'
- not containerized_deployment
- name: create gpt disk label of the journal device(s)
command: parted --script {{ item.1 }} mklabel gpt
@ -45,7 +66,7 @@
- "{{ dedicated_devices|unique }}"
changed_when: false
when:
- containerized_deployment
- osd_scenario == 'non-collocated'
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- osd_scenario == 'non-collocated'
- containerized_deployment

View File

@ -34,12 +34,3 @@
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- containerized_deployment
- name: check if a partition named 'ceph' exists
shell: "parted --script {{ item }} print | egrep -sq '^ 1.*ceph'"
with_items:
- "{{ devices }}"
changed_when: false
failed_when: false
always_run: true
register: parted_results

View File

@ -30,7 +30,7 @@
- name: systemd start osd container
systemd:
name: ceph-osd@{{ item | basename }}
name: ceph-osd@{{ item | regex_replace('/dev/', '') }}
state: started
enabled: yes
daemon_reload: yes

View File

@ -17,14 +17,25 @@
devices: "{{ devices | default([]) + [ item.key | regex_replace('^', '/dev/') ] }}"
with_dict: "{{ ansible_devices }}"
when:
- osd_auto_discovery
- ansible_devices is defined
- item.value.removable == "0"
- item.value.partitions|count == 0
- item.value.holders|count == 0
- osd_auto_discovery
- name: include check_devices.yml
include: check_devices.yml
when:
- not osd_auto_discovery
- name: check if a partition named 'ceph' exists
shell: "parted --script {{ item }} print | egrep -sq '^ 1.*ceph'"
with_items:
- "{{ devices }}"
changed_when: false
failed_when: false
always_run: true
register: parted_results
- name: include copy_configs.yml
include: copy_configs.yml
@ -42,6 +53,7 @@
- name: include scenarios/non-collocated.yml
include: scenarios/non-collocated.yml
when:
- not osd_auto_discovery
- osd_scenario == 'non-collocated'
# Hard code this so we will skip the entire file instead of individual tasks (Default isn't Consistent)
static: False

View File

@ -28,12 +28,6 @@
- /var/lib/ceph/bootstrap-osd/
- /var/lib/ceph/osd/
- name: set_fact copy_admin_key
set_fact:
copy_admin_key: true
when:
- osd_scenario == 'collocated'
- name: copy osd bootstrap key
copy:
src: "{{ fetch_directory }}/{{ fsid }}{{ item.name }}"

View File

@ -7,7 +7,7 @@
docker run --net=host \
--pid=host \
--privileged=true \
--name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
--name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /dev:/dev \
@ -22,17 +22,17 @@
- "{{ parted_results.results | default([]) }}"
- "{{ devices }}"
when:
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- not osd_auto_discovery
- containerized_deployment
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- name: automatic prepare ceph containerized osd disk collocated
shell: |
docker run --net=host \
--pid=host \
--privileged=true \
--name=ceph-osd-prepare-{{ ansible_hostname }}-devdev{{ item.key }} \
--name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.key }} \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /dev:/dev \
@ -45,12 +45,12 @@
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
with_dict: "{{ ansible_devices }}"
when:
- osd_auto_discovery
- containerized_deployment
- ansible_devices is defined
- item.value.removable == "0"
- item.value.partitions|count == 0
- item.value.holders|count == 0
- osd_auto_discovery
- containerized_deployment
- name: manually prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) with collocated osd data and journal
command: "ceph-disk prepare {{ ceph_disk_cli_options }} {{ item.1 }}"
@ -58,6 +58,6 @@
- "{{ parted_results.results | default([]) }}"
- "{{ devices }}"
when:
- not containerized_deployment
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- not containerized_deployment

View File

@ -7,7 +7,7 @@
docker run --net=host \
--pid=host \
--privileged=true \
--name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
--name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /dev:/dev \
@ -24,18 +24,17 @@
- "{{ devices }}"
- "{{ dedicated_devices }}"
when:
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- not osd_auto_discovery
- containerized_deployment
- osd_objectstore == 'filestore'
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- name: prepare ceph "{{ osd_objectstore }}" containerized osd disk(s) non-collocated with a dedicated device for db and wal
shell: |
docker run --net=host \
--pid=host \
--privileged=true \
--name=ceph-osd-prepare-{{ ansible_hostname }}-dev{{ item.1 | regex_replace('/', '') }} \
--name=ceph-osd-prepare-{{ ansible_hostname }}-{{ item.1 | regex_replace('/dev/', '') }} \
-v /etc/ceph:/etc/ceph \
-v /var/lib/ceph/:/var/lib/ceph/ \
-v /dev:/dev \
@ -54,11 +53,10 @@
- "{{ dedicated_devices }}"
- "{{ bluestore_wal_devices }}"
when:
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- not osd_auto_discovery
- containerized_deployment
- osd_objectstore == 'bluestore'
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- name: prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) non-collocated
command: "ceph-disk prepare {{ ceph_disk_cli_options }} {{ item.1 }} {{ item.2 }}"
@ -68,10 +66,10 @@
- "{{ dedicated_devices }}"
changed_when: false
when:
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- osd_objectstore == 'filestore'
- not containerized_deployment
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- name: manually prepare ceph "{{ osd_objectstore }}" non-containerized osd disk(s) with a dedicated device for db and wal
command: "ceph-disk prepare {{ ceph_disk_cli_options }} --block.db {{ item.1 }} --block.wal {{ item.2 }} {{ item.3 }}"
@ -79,9 +77,9 @@
- "{{ parted_results.results | default([]) }}"
- "{{ dedicated_devices }}"
- "{{ bluestore_wal_devices }}"
- "{{ devices }}"
- "{{ devices | unique }}"
when:
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0
- osd_objectstore == 'bluestore'
- not containerized_deployment
- not item.0.get("skipped")
- item.0.get("rc", 0) != 0

View File

@ -64,5 +64,5 @@ expose_partitions "$1"
-e OSD_DEVICE=/dev/${1} \
-e CEPH_DAEMON=OSD_CEPH_DISK_ACTIVATE \
{{ ceph_osd_docker_extra_env }} \
--name=ceph-osd-{{ ansible_hostname }}-dev${1} \
--name=ceph-osd-{{ ansible_hostname }}-${1} \
{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}

View File

@ -5,10 +5,10 @@ After=docker.service
[Service]
EnvironmentFile=-/etc/environment
ExecStartPre=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-dev%i
ExecStartPre=-/usr/bin/docker rm -f ceph-osd-{{ ansible_hostname }}-dev%i
ExecStartPre=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-%i
ExecStartPre=-/usr/bin/docker rm -f ceph-osd-{{ ansible_hostname }}-%i
ExecStart={{ ceph_osd_docker_run_script_path }}/ceph-osd-run.sh %i
ExecStop=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-dev%i
ExecStop=-/usr/bin/docker stop ceph-osd-{{ ansible_hostname }}-%i
Restart=always
RestartSec=10s
TimeoutStartSec=120

View File

@ -16,7 +16,7 @@ def node(host, request):
# tox will pass in this environment variable. we need to do it this way
# because testinfra does not collect and provide ansible config passed in
# from using --extra-vars
ceph_stable_release = os.environ.get("CEPH_STABLE_RELEASE", "kraken")
ceph_stable_release = os.environ.get("CEPH_STABLE_RELEASE", "luminous")
node_type = ansible_vars["group_names"][0]
docker = ansible_vars.get("docker")
osd_auto_discovery = ansible_vars.get("osd_auto_discovery")
@ -71,7 +71,11 @@ def node(host, request):
osd_ids = cmd.stdout.rstrip("\n").split("\n")
osds = osd_ids
if docker:
osds = [device.split("/")[-1] for device in ansible_vars.get("devices", [])]
osds = []
for device in ansible_vars.get("devices", []):
real_dev = host.run("sudo readlink -f %s" % device)
real_dev_split = real_dev.stdout.split("/")[-1]
osds.append(real_dev_split)
data = dict(
address=address,

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -10,7 +10,7 @@ cluster_network: "192.168.2.0/24"
journal_size: 100
osd_objectstore: "bluestore"
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
dedicated_devices:
- '/dev/sdc'

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -11,7 +11,7 @@ osd_scenario: non-collocated
dmcrypt: true
osd_objectstore: "bluestore"
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
dedicated_devices:
- '/dev/sdc'

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -11,7 +11,7 @@ osd_scenario: collocated
dmcrypt: true
osd_objectstore: bluestore
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
os_tuning_params:
- { name: kernel.pid_max, value: 4194303 }

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -11,7 +11,7 @@ osd_scenario: collocated
dmcrypt: true
osd_objectstore: bluestore
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
os_tuning_params:
- { name: kernel.pid_max, value: 4194303 }

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -16,7 +16,7 @@ ceph_rgw_civetweb_port: 8080
osd_scenario: non-collocated
osd_objectstore: bluestore
devices:
- /dev/sda
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- /dev/sdb
dedicated_devices:
- /dev/sdc

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -9,7 +9,7 @@ public_network: "192.168.3.0/24"
cluster_network: "192.168.4.0/24"
journal_size: 100
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
osd_scenario: collocated
osd_objectstore: "bluestore"

View File

@ -0,0 +1,7 @@
{
"ceph_conf_overrides": {
"global": {
"osd_pool_default_pg_num": 12
}
}
}

View File

@ -9,7 +9,7 @@ journal_size: 100
radosgw_interface: eth1
osd_objectstore: filestore
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
dedicated_devices:
- '/dev/sdc'

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -11,7 +11,7 @@ osd_scenario: non-collocated
dmcrypt: true
osd_objectstore: filestore
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
dedicated_devices:
- '/dev/sdc'

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -11,7 +11,7 @@ osd_scenario: collocated
osd_objectstore: filestore
dmcrypt: true
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
os_tuning_params:
- { name: kernel.pid_max, value: 4194303 }

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -17,6 +17,6 @@ osd_objectstore: filestore
dmcrypt: true
ceph_rgw_civetweb_port: 8080
devices:
- /dev/sda
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- /dev/sdb
ceph_osd_docker_prepare_env: -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -16,7 +16,7 @@ ceph_rgw_civetweb_port: 8080
osd_objectstore: filestore
osd_scenario: non-collocated
devices:
- /dev/sda
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- /dev/sdb
dedicated_devices:
- /dev/sdc

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -17,7 +17,7 @@ ceph_rgw_civetweb_port: 8080
osd_objectstore: filestore
ceph_osd_docker_prepare_env: -e OSD_JOURNAL_SIZE={{ journal_size }} -e OSD_FORCE_ZAP=1
devices:
- /dev/sda
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- /dev/sdb
ceph_osd_docker_run_script_path: /var/tmp
rgw_override_bucket_index_max_shards: 16

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -10,7 +10,7 @@ radosgw_interface: eth1
journal_size: 100
osd_objectstore: filestore
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
osd_scenario: collocated
os_tuning_params:

View File

@ -0,0 +1 @@
../cluster/ceph-override.json

View File

@ -7,4 +7,4 @@ class TestOSD(object):
if osd_auto_discovery:
node["vars"]["devices"] = ["/dev/sda", "/dev/sdb", "/dev/sdc"] # Hardcoded since we can't retrieve the devices list generated during playbook run
for device in node["vars"]["devices"]:
assert host.check_output("sudo blkid -s PARTLABEL -o value %s2" % device) in ["ceph journal", "ceph block"]
assert host.check_output("sudo blkid -s PARTLABEL -o value $(readlink -f %s)2" % device) in ["ceph journal", "ceph block"]

View File

@ -9,7 +9,7 @@ monitor_interface: eth1
radosgw_interface: eth1
journal_size: 100
devices:
- '/dev/sda'
- '/dev/disk/by-path/pci-0000:00:01.1-ata-1.0'
- '/dev/sdb'
dedicated_devices:
- '/dev/sdc'

12
tox.ini
View File

@ -228,6 +228,18 @@ commands=
# retest to ensure cluster came back up correctly after rebooting
testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory={changedir}/hosts {toxinidir}/tests/functional/tests
# handlers/idempotency test
ansible-playbook -vv -i {changedir}/hosts {toxinidir}/{env:PLAYBOOK:site.yml.sample} \
--extra-vars "\
fetch_directory={env:FETCH_DIRECTORY:{changedir}/fetch} \
ceph_stable_release={env:CEPH_STABLE_RELEASE:luminous} \
ceph_docker_registry={env:CEPH_DOCKER_REGISTRY:docker.io} \
ceph_docker_image={env:CEPH_DOCKER_IMAGE:ceph/daemon} \
ceph_docker_image_tag={env:CEPH_DOCKER_IMAGE_TAG:latest} \
ceph_dev_branch={env:CEPH_DEV_BRANCH:master} \
ceph_dev_sha1={env:CEPH_DEV_SHA1:latest} " \
--extra-vars @ceph-override.json
purge_cluster: {[purge]commands}
purge_lvm_osds: {[purge-lvm]commands}
purge_dmcrypt: {[purge]commands}