mirror of https://github.com/ceph/ceph-ansible.git
Merge pull request #1788 from ceph/improve-switch
switch-from-non-containerized-to-containerized: simplifypull/1798/head
Sébastien Han
GitHub
commit
f2499ff5ac
|
|
@ -39,7 +39,6 @@
|
|||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: install docker and dependencies for the docker module
|
||||
package:
|
||||
name: "{{ item }}"
|
||||
|
|
@ -85,6 +84,9 @@
|
|||
- name: switching from non-containerized to containerized ceph mon
|
||||
|
||||
vars:
|
||||
health_mon_check_retries: 5
|
||||
health_mon_check_delay: 15
|
||||
containerized_deployment: true
|
||||
mon_group_name: mons
|
||||
restapi_group_name: restapis
|
||||
|
||||
|
|
@ -94,45 +96,25 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
pre_tasks:
|
||||
- name: select a running monitor
|
||||
set_fact: mon_host={{ item }}
|
||||
with_items: "{{ groups[mon_group_name] }}"
|
||||
when: item != inventory_hostname
|
||||
|
||||
- name: get current ceph fsid
|
||||
command: ceph fsid
|
||||
register: ceph_fsid
|
||||
changed_when: false
|
||||
delegate_to: "{{ mon_host }}"
|
||||
|
||||
- name: stop ceph mon bare metal service
|
||||
- name: stop non-containerized ceph mon
|
||||
service:
|
||||
name: "ceph-mon@{{ ansible_hostname }}"
|
||||
state: stopped
|
||||
enabled: no
|
||||
|
||||
# NOTE(leseb): should we also create systemd files
|
||||
# instead of running raw docker commands?
|
||||
# It is probably more elegant but will require a template file...
|
||||
# which would make this single file playbook more complex
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -143,30 +125,40 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: start ceph mon container image
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: "ceph-mon-{{ ansible_hostname }}"
|
||||
net: "host"
|
||||
state: "running"
|
||||
privileged: "{{ mon_docker_privileged|default(false) }}"
|
||||
env: "MON_IP={{ hostvars[inventory_hostname]['ansible_' + monitor_interface]['ipv4']['address'] }},CEPH_DAEMON=MON,CEPH_PUBLIC_NETWORK={{ public_network }},CEPH_FSID={{ ceph_fsid.stdout }},{{ ceph_mon_docker_extra_env|default('') }}"
|
||||
volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
|
||||
|
||||
- name: waiting for the monitor to join the quorum...
|
||||
shell: |
|
||||
ceph -s --cluster {{ cluster|default('ceph') }} | grep monmap | sed 's/.*quorum//' | egrep -sq {{ ansible_hostname }}
|
||||
register: result
|
||||
until: result.rc == 0
|
||||
retries: 5
|
||||
delay: 10
|
||||
- name: check for existing old leveldb file extension (ldb)
|
||||
shell: stat /var/lib/ceph/mon/*/store.db/*.ldb
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: ldb_files
|
||||
|
||||
- name: rename leveldb extension from ldb to sst
|
||||
shell: rename -v .ldb .sst /var/lib/ceph/mon/*/store.db/*.ldb
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when: ldb_files.rc == 0
|
||||
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-mon
|
||||
|
||||
post_tasks:
|
||||
- name: waiting for the containerized monitor to join the quorum...
|
||||
shell: |
|
||||
docker exec ceph-mon-{{ hostvars[mon_host]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["quorum_names"])'
|
||||
register: result
|
||||
until: "{{ ansible_hostname in result.stdout }}"
|
||||
retries: "{{ health_mon_check_retries }}"
|
||||
delay: "{{ health_mon_check_delay }}"
|
||||
delegate_to: "{{ mon_host }}"
|
||||
|
||||
|
||||
- name: switching from non-containerized to containerized ceph osd
|
||||
|
||||
vars:
|
||||
health_osd_check_retries: 5
|
||||
health_osd_check_delay: 15
|
||||
containerized_deployment: true
|
||||
osd_group_name: osds
|
||||
|
||||
hosts:
|
||||
|
|
@ -175,12 +167,12 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: collect osd ids
|
||||
pre_tasks:
|
||||
- name: collect running osds
|
||||
shell: |
|
||||
if [ -d /var/lib/ceph/osd ] ; then ls /var/lib/ceph/osd | sed 's/.*-//' ; fi
|
||||
register: osd_ids
|
||||
systemctl list-units | grep "loaded active" | grep -Eo 'ceph-osd@[0-9]{1,2}.service'
|
||||
register: running_osds
|
||||
changed_when: false
|
||||
|
||||
- name: collect osd devices
|
||||
|
|
@ -189,28 +181,20 @@
|
|||
register: ceph_osd_docker_devices
|
||||
changed_when: false
|
||||
|
||||
- name: stop ceph osd service
|
||||
- name: stop non-containerized ceph osd(s)
|
||||
service:
|
||||
name: "ceph-osd@{{ item }}"
|
||||
name: "{{ item }}"
|
||||
state: stopped
|
||||
enabled: no
|
||||
with_items: "{{ osd_ids.stdout_lines }}"
|
||||
with_items: "{{ running_osds.stdout_lines }}"
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -221,45 +205,52 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: check for existing old leveldb file extension (ldb)
|
||||
shell: stat /var/lib/ceph/osd/*/current/omap/*.ldb
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
register: ldb_files
|
||||
|
||||
- name: rename leveldb extension from ldb to sst
|
||||
shell: rename -v .ldb .sst /var/lib/ceph/osd/*/current/omap/*.ldb
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when:
|
||||
- ldb_files.rc == 0
|
||||
|
||||
- name: check if containerized osds are already running
|
||||
shell: |
|
||||
docker ps | grep -sq {{ item | regex_replace('/', '') }}
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
with_items: "{{ ceph_osd_docker_devices.stdout_lines }}"
|
||||
register: osd_running
|
||||
|
||||
- name: unmount all the osd directories
|
||||
mount:
|
||||
name: "/var/lib/ceph/osd/{{ cluster }}-{{ item.0 }}"
|
||||
state: unmounted
|
||||
src: "{{ item.1 }}"
|
||||
fstype: xfs
|
||||
command: umount "{{ item.0 }}"1
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
with_together:
|
||||
- "{{ osd_ids.stdout_lines }}"
|
||||
- "{{ ceph_osd_docker_devices.stdout_lines }}"
|
||||
- "{{ osd_running.results }}"
|
||||
when: item.2.rc != 0
|
||||
when:
|
||||
- item.1.get("rc", 0) != 0
|
||||
|
||||
- name: start ceph osd container image(s)
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: "ceph-osd-{{ ansible_hostname }}-dev{{ item | regex_replace('/', '') }}"
|
||||
net: host
|
||||
pid: host
|
||||
state: started
|
||||
privileged: yes
|
||||
env: "OSD_DEVICE={{ item }},{{ ceph_osd_docker_extra_env }}"
|
||||
volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro,/dev:/dev,/run:/run"
|
||||
with_items: "{{ ceph_osd_docker_devices.stdout_lines }}"
|
||||
- set_fact: ceph_osd_docker_devices={{ ceph_osd_docker_devices.stdout_lines }}
|
||||
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-osd
|
||||
|
||||
post_tasks:
|
||||
- name: waiting for clean pgs...
|
||||
shell: |
|
||||
test "[""$(ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')""]" = "$(ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if i["state_name"] == "active+clean"]')"
|
||||
test "[""$(docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')""]" = "$(docker exec ceph-mon-{{ hostvars[groups[mon_group_name][0]]['ansible_hostname'] }} ceph --cluster {{ cluster }} -s -f json | python -c 'import sys, json; print [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if i["state_name"] == "active+clean"]')"
|
||||
register: result
|
||||
until: result.rc == 0
|
||||
retries: 10
|
||||
delay: 10
|
||||
changed_when: false
|
||||
retries: "{{ health_osd_check_retries }}"
|
||||
delay: "{{ health_osd_check_delay }}"
|
||||
delegate_to: "{{ groups[mon_group_name][0] }}"
|
||||
|
||||
|
||||
|
|
@ -271,9 +262,8 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: stop ceph mds service
|
||||
pre_tasks:
|
||||
- name: stop non-containerized ceph mds(s)
|
||||
service:
|
||||
name: "ceph-mds@{{ ansible_hostname }}"
|
||||
state: stopped
|
||||
|
|
@ -281,19 +271,11 @@
|
|||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -304,14 +286,10 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: start ceph metadata container image
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: ceph-mds-{{ ansible_hostname }}
|
||||
net: host
|
||||
state: running
|
||||
env: "CEPH_DAEMON=MDS,CEPHFS_CREATE=1,{{ ceph_mds_docker_extra_env }}"
|
||||
volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-mds
|
||||
|
||||
|
||||
- name: switching from non-containerized to containerized ceph rgw
|
||||
|
|
@ -322,9 +300,8 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: stop ceph rgw service
|
||||
pre_tasks:
|
||||
- name: stop non-containerized ceph rgw(s)
|
||||
service:
|
||||
name: "ceph-rgw@{{ ansible_hostname }}"
|
||||
state: stopped
|
||||
|
|
@ -332,19 +309,11 @@
|
|||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -355,15 +324,10 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: start ceph rados gateway container image
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: ceph-rgw-{{ ansible_hostname }}
|
||||
expose: "{{ ceph_rgw_civetweb_port }}"
|
||||
ports: "{{ ceph_rgw_civetweb_port }}:{{ ceph_rgw_civetweb_port }}"
|
||||
state: running
|
||||
env: "CEPH_DAEMON=RGW,{{ ceph_rgw_docker_extra_env }}"
|
||||
volumes: "/var/lib/ceph:/var/lib/ceph,/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-rgw
|
||||
|
||||
|
||||
- name: switching from non-containerized to containerized ceph rbd-mirror
|
||||
|
|
@ -374,9 +338,8 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: stop ceph rbd mirror service
|
||||
pre_tasks:
|
||||
- name: stop non-containerized ceph rbd mirror(s)
|
||||
service:
|
||||
name: "ceph-rbd-mirror@{{ ansible_hostname }}"
|
||||
state: stopped
|
||||
|
|
@ -384,19 +347,11 @@
|
|||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -407,13 +362,10 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: start ceph rbd mirror container image
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: "ceph-rbd-mirror-{{ ansible_hostname }}"
|
||||
net: host
|
||||
state: running
|
||||
volumes: "/etc/ceph:/etc/ceph,/etc/localtime:/etc/localtime:ro"
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-rbd-mirror
|
||||
|
||||
|
||||
- name: switching from non-containerized to containerized ceph nfs
|
||||
|
|
@ -424,9 +376,8 @@
|
|||
serial: 1
|
||||
become: true
|
||||
|
||||
tasks:
|
||||
|
||||
- name: stop ceph nfs service
|
||||
pre_tasks:
|
||||
- name: stop non-containerized ceph nfs(s)
|
||||
service:
|
||||
name: "ceph-nfs@{{ ansible_hostname }}"
|
||||
state: stopped
|
||||
|
|
@ -434,19 +385,11 @@
|
|||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | match("latest")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 64045
|
||||
when: ceph_docker_image_tag | search("ubuntu")
|
||||
when: ceph_docker_image_tag | match("latest") or ceph_docker_image_tag | search("ubuntu")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph")
|
||||
|
||||
- set_fact:
|
||||
ceph_uid: 167
|
||||
when: ceph_docker_image_tag | search("fedora")
|
||||
when: ceph_docker_image_tag | search("centos") or ceph_docker_image | search("rhceph") or ceph_docker_image_tag | search("fedora")
|
||||
|
||||
- name: set proper ownership on ceph directories
|
||||
file:
|
||||
|
|
@ -457,13 +400,7 @@
|
|||
- /var/lib/ceph
|
||||
- /etc/ceph
|
||||
|
||||
- name: start ceph nfs container image
|
||||
docker:
|
||||
image: "{{ ceph_docker_registry|default('docker.io') }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
|
||||
name: "ceph-nfs-{{ ansible_hostname }}"
|
||||
net: "host"
|
||||
state: "running"
|
||||
privileged: true
|
||||
ports: "{{ ceph_nfs_port }}:{{ ceph_nfs_port }},111:111"
|
||||
env: "CEPH_DAEMON=NFS,CEPH_PUBLIC_NETWORK={{ ceph_nfs_docker_subnet }},{{ ceph_nfs_docker_extra_env }}"
|
||||
volumes: "/etc/ceph:/etc/ceph,/etc/ganesha:/etc/ganesha,/etc/localtime:/etc/localtime:ro"
|
||||
roles:
|
||||
- ceph-defaults
|
||||
- ceph-docker-common
|
||||
- ceph-nfs
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ check_pgs() {
|
|||
|
||||
wait_for_socket_in_docker() {
|
||||
if ! docker exec "$1" timeout 10 bash -c "while [ ! -e /var/run/ceph/*.asok ]; do sleep 1 ; done"; then
|
||||
log "Timed out while trying to look for a Ceph OSD socket."
|
||||
log "Abort mission!"
|
||||
echo "Timed out while trying to look for a Ceph OSD socket."
|
||||
echo "Abort mission!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
|
@ -52,7 +52,7 @@ get_docker_osd_id() {
|
|||
|
||||
# For containerized deployments, the unit file looks like: ceph-osd@sda.service
|
||||
# For non-containerized deployments, the unit file looks like: ceph-osd@0.service
|
||||
for unit in $(systemctl list-units | grep -oE "ceph-osd@([0-9]{1,2}|[a-z]+).service"); do
|
||||
for unit in $(systemctl list-units | grep "loaded active" | grep -oE "ceph-osd@([0-9]{1,2}|[a-z]+).service"); do
|
||||
# First, restart daemon(s)
|
||||
systemctl restart "${unit}"
|
||||
# We need to wait because it may take some time for the socket to actually exists
|
||||
|
|
@ -66,7 +66,7 @@ for unit in $(systemctl list-units | grep -oE "ceph-osd@([0-9]{1,2}|[a-z]+).serv
|
|||
{% else %}
|
||||
osd_id=$(echo ${unit#ceph-osd@} | grep -oE '[0-9]{1,2}')
|
||||
{% endif %}
|
||||
SOCKET=/var/run/ceph/test-osd.${osd_id}.asok
|
||||
SOCKET=/var/run/ceph/{{ cluster }}-osd.${osd_id}.asok
|
||||
while [ $COUNT -ne 0 ]; do
|
||||
$docker_exec test -S "$SOCKET" && check_pgs && continue 2
|
||||
sleep 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue