ceph-ansible/infrastructure-playbooks/purge-docker-cluster.yml

---
# This playbook purges a containerized Ceph cluster
# It removes: packages, containers, configuration files and ALL THE DATA

- name: confirm whether user really meant to purge the cluster

  hosts:
    - localhost

  gather_facts: false

  vars_prompt:
    - name: ireallymeanit
      prompt: >
        Are you sure you want to purge the cluster?
        Note that if with_pkg is not set docker packages
        and more will be uninstalled from non-atomic hosts.
        Do you want to continue?        
      default: 'no'
      private: no

  tasks:
  - name: exit playbook, if user did not mean to purge cluster
    fail:
      msg: >
        "Exiting purge-docker-cluster playbook, cluster was NOT purged.
         To purge the cluster, either say 'yes' on the prompt or
         or use `-e ireallymeanit=yes` on the command line when
         invoking the playbook"        
    when: ireallymeanit != 'yes'

  - name: set ceph_docker_registry value if not set
    set_fact:
      ceph_docker_registry: "docker.io"
    when: ceph_docker_registry is not defined

- name: purge ceph mds cluster

  hosts:
    - "{{ mds_group_name|default('mdss') }}"

  become: true

  tasks:

  - name: disable ceph mds service
    service:
      name: "ceph-mds@{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: remove ceph mds container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "ceph-mds-{{ ansible_hostname }}"
      state: absent
    ignore_errors: true

  - name: remove ceph mds service
    file:
      path: /etc/systemd/system/ceph-mds@.service
      state: absent

  - name: remove ceph mds image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true

- name: purge ceph iscsigws cluster

  hosts:
    - "{{ iscsi_gw_group_name|default('iscsigws') }}"
    - iscsi-gws
  become: true
  tasks:

  - name: disable ceph iscsigw services
    service:
      name: "{{ item }}"
      state: stopped
      enabled: no
    ignore_errors: true
    with_items:
      - rbd-target-api
      - rbd-target-gw
      - tcmu-runner

  - name: remove ceph iscsigw containers
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "{{ item }}"
      state: absent
    ignore_errors: true
    with_items:
      - rbd-target-api
      - rbd-target-gw
      - tcmu-runner

  - name: remove ceph iscsigw systemd unit files
    file:
      path: /etc/systemd/system/{{ item }}.service
      state: absent
    ignore_errors: true
    with_items:
      - rbd-target-api
      - rbd-target-gw
      - tcmu-runner

  - name: remove ceph iscsigw image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true

- name: purge ceph mgr cluster

  hosts:
    - "{{ mgr_group_name|default('mgrs') }}"
  become: true
  tasks:

  - name: disable ceph mgr service
    service:
      name: "ceph-mgr@{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: remove ceph mgr container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "ceph-mgr-{{ ansible_hostname }}"
      state: absent
    ignore_errors: true

  - name: remove ceph mgr service
    file:
      path: /etc/systemd/system/ceph-mgr@.service
      state: absent

  - name: remove ceph mgr image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true

- name: purge ceph rgw cluster

  hosts:
    - "{{ rgw_group_name|default('rgws') }}"

  become: true

  tasks:

# For backward compatibility
  - name: disable ceph rgw service (old unit name, for backward compatibility)
    service:
      name: "ceph-rgw@{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: disable ceph rgw service (new unit name)
    service:
      name: "ceph-radosgw@rgw.{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: remove ceph rgw container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "ceph-rgw-{{ ansible_hostname }}"
      state: absent
    ignore_errors: true

  - name: remove ceph rgw service
    file:
      path: "{{ item }}"
      state: absent
    with_items:
# For backward compatibility
      - /etc/systemd/system/ceph-rgw@.service
      - /etc/systemd/system/ceph-radosgw@.service

  - name: remove ceph rgw image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true


- name: purge ceph rbd-mirror cluster

  hosts:
    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"

  become: true

  tasks:

  - name: disable ceph rbd-mirror service
    service:
      name: "ceph-rbd-mirror@rbd-mirror.{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: remove ceph rbd-mirror container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "ceph-rbd-mirror-{{ ansible_hostname }}"
      state: absent
    ignore_errors: true

  - name: remove ceph rbd-mirror service
    file:
      path: /etc/systemd/system/ceph-rbd-mirror@.service
      state: absent

  - name: remove ceph rbd-mirror image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img


- name: purge ceph nfs cluster

  hosts:
    - "{{ nfs_group_name|default('nfss') }}"

  become: true

  tasks:

  - name: disable ceph nfs service
    service:
      name: "ceph-nfs@{{ ansible_hostname }}"
      state: stopped
      enabled: no
    ignore_errors: true

  - name: remove ceph nfs container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "ceph-nfs-{{ ansible_hostname }}"
      state: absent
    ignore_errors: true

  - name: remove ceph nfs service
    file:
      path: /etc/systemd/system/ceph-nfs@.service
      state: absent

  - name: remove ceph nfs directories for "{{ ansible_hostname }}"
    file:
      path: "{{ item }}"
      state: absent
    with_items:
      - /etc/ganesha
      - /var/lib/nfs/ganesha
      - /var/run/ganesha

  - name: remove ceph nfs image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img


- name: purge ceph osd cluster

  hosts:
    - "{{ osd_group_name | default('osds') }}"

  gather_facts: true
  become: true

  tasks:

  - import_role:
      name: ceph-defaults
      private: false

  - name: gather monitors facts
    setup:
    delegate_to: "{{ item }}"
    delegate_facts: True
    with_items: "{{ groups.get(mon_group_name | default('mons')) }}"

  - import_role:
      name: ceph-facts
      private: false

  - name: get all the running osds
    shell: |
      systemctl list-units | grep 'loaded[[:space:]]\+active' | grep -oE "ceph-osd@([0-9]{1,2}|[a-z]+).service"      
    register: osd_units
    ignore_errors: true

  - name: disable ceph osd service
    service:
      name: "{{ item }}"
      state: stopped
      enabled: no
    with_items: "{{ osd_units.stdout_lines }}"

  - name: remove osd mountpoint tree
    file:
      path: /var/lib/ceph/osd/
      state: absent
    register: remove_osd_mountpoints
    ignore_errors: true

  - name: for ceph-disk based deployment
    block:
      - name: get prepare container
        command: "docker ps -a -q --filter='name=ceph-osd-prepare'"
        register: prepare_containers
        ignore_errors: true

      - name: remove ceph osd prepare container
        command: "docker rm -f {{ item }}"
        with_items: "{{ prepare_containers.stdout_lines }}"
        ignore_errors: true

      # NOTE(leseb): hope someone will find a more elegant way one day...
      - name: see if encrypted partitions are present
        shell: |
          blkid -t TYPE=crypto_LUKS -s PARTLABEL -s PARTUUID | grep "ceph.*." | grep -o PARTUUID.* | cut -d '"' -f 2          
        register: encrypted_ceph_partuuid

      - name: get ceph data partitions
        command: |
          blkid -o device -t PARTLABEL="ceph data"          
        failed_when: false
        register: ceph_data_partition_to_erase_path

      - name: get ceph lockbox partitions
        command: |
          blkid -o device -t PARTLABEL="ceph lockbox"          
        failed_when: false
        register: ceph_lockbox_partition_to_erase_path

      - name: get ceph block partitions
        command: |
          blkid -o device -t PARTLABEL="ceph block"          
        failed_when: false
        register: ceph_block_partition_to_erase_path

      - name: get ceph journal partitions
        command: |
          blkid -o device -t PARTLABEL="ceph journal"          
        failed_when: false
        register: ceph_journal_partition_to_erase_path

      - name: get ceph db partitions
        command: |
          blkid -o device -t PARTLABEL="ceph block.db"          
        failed_when: false
        register: ceph_db_partition_to_erase_path

      - name: get ceph wal partitions
        command: |
          blkid -o device -t PARTLABEL="ceph block.wal"          
        failed_when: false
        register: ceph_wal_partition_to_erase_path

      - name: set_fact combined_devices_list
        set_fact:
          combined_devices_list: "{{ ceph_data_partition_to_erase_path.get('stdout_lines', []) +
                                    ceph_lockbox_partition_to_erase_path.get('stdout_lines', []) +
                                    ceph_block_partition_to_erase_path.get('stdout_lines', []) +
                                    ceph_journal_partition_to_erase_path.get('stdout_lines', []) +
                                    ceph_db_partition_to_erase_path.get('stdout_lines', []) +
                                    ceph_wal_partition_to_erase_path.get('stdout_lines', []) }}"

      - name: resolve parent device
        command: lsblk --nodeps -no pkname "{{ item }}"
        register: tmp_resolved_parent_device
        with_items:
          - "{{ combined_devices_list }}"

      - name: set_fact resolved_parent_device
        set_fact:
          resolved_parent_device: "{{ tmp_resolved_parent_device.results | map(attribute='stdout') | list | unique }}"

      - name: zap ceph osd disks
        shell: |
          docker run --rm \
          --privileged=true \
          --name ceph-osd-zap-{{ ansible_hostname }}-{{ item }} \
          -v /dev/:/dev/ \
          -e OSD_DEVICE=/dev/{{ item }} \
          {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \
          zap_device          
        with_items:
          - "{{ resolved_parent_device }}"

      - name: wait until the zap containers die
        shell: |
          docker ps | grep -sq ceph-osd-zap-{{ ansible_hostname }}          
        register: zap_alive
        failed_when: false
        until: zap_alive.rc != 0
        retries: 5
        delay: 10

      - name: remove ceph osd zap disk container
        docker_container:
          image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
          name: "ceph-osd-zap-{{ ansible_hostname }}-{{ item }}"
          state: absent
        with_items:
          - "{{ resolved_parent_device }}"

      - name: remove ceph osd service
        file:
          path: /etc/systemd/system/ceph-osd@.service
          state: absent
    when:
      - osd_scenario != "lvm"

  - name: for ceph-volume based deployments
    block:
      - name: zap and destroy osds created by ceph-volume with lvm_volumes
        ceph_volume:
          data: "{{ item.data }}"
          data_vg: "{{ item.data_vg|default(omit) }}"
          journal: "{{ item.journal|default(omit) }}"
          journal_vg: "{{ item.journal_vg|default(omit) }}"
          db: "{{ item.db|default(omit) }}"
          db_vg: "{{ item.db_vg|default(omit) }}"
          wal: "{{ item.wal|default(omit) }}"
          wal_vg: "{{ item.wal_vg|default(omit) }}"
          action: "zap"
        environment:
          CEPH_VOLUME_DEBUG: 1
          CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
          CEPH_CONTAINER_BINARY: "{{ container_binary }}"
        with_items: "{{ lvm_volumes }}"

      - name: zap and destroy osds created by ceph-volume with devices
        ceph_volume:
          data: "{{ item }}"
          action: "zap"
        environment:
          CEPH_VOLUME_DEBUG: 1
          CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
          CEPH_CONTAINER_BINARY: "{{ container_binary }}"
        with_items: "{{ devices | default([]) }}"
    when:
      - osd_scenario == "lvm"

  - name: remove ceph osd image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true

  - name: include vars from group_vars/osds.yml
    include_vars:
      file: "{{ item }}"
    with_first_found:
      - files:
        - "{{ playbook_dir }}/group_vars/osds"
        - "{{ playbook_dir }}/group_vars/osds.yml"
        skip: true

  - name: find all osd_disk_prepare logs
    find:
      paths: "{{ ceph_osd_docker_run_script_path | default('/usr/share') }}"
      pattern: "ceph-osd-prepare-*.log"
    register: osd_disk_prepare_logs

  - name: ensure all osd_disk_prepare logs are removed
    file:
      path: "{{ item.path }}"
      state: absent
    with_items:
      - "{{ osd_disk_prepare_logs.files }}"

- name: purge ceph mon cluster

  hosts:
    - "{{ mon_group_name|default('mons') }}"

  become: true

  tasks:

  # since mgr are now collocated with mons by default
  - name: disable ceph mon and mgr service
    service:
      name: "{{ item }}"
      state: stopped
      enabled: no
    ignore_errors: true
    with_items:
      - "ceph-mgr@{{ ansible_hostname }}"
      - "ceph-mon@{{ ansible_hostname }}"

  - name: remove ceph mon and mgr container
    docker_container:
      image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
      name: "{{ item }}"
      state: absent
    ignore_errors: true
    with_items:
      - "ceph-mon-{{ ansible_hostname }}"
      - "ceph-mgr-{{ ansible_hostname }}"

  - name: remove ceph mon and mgr service
    file:
      path: "/etc/systemd/system/ceph-{{ item }}@.service"
      state: absent
    with_items:
      - mon
      - mgr

  - name: remove ceph mon and mgr image
    docker_image:
      state: absent
      repository: "{{ ceph_docker_registry }}"
      name: "{{ ceph_docker_image }}"
      tag: "{{ ceph_docker_image_tag }}"
      force: yes
    tags:
      remove_img
    ignore_errors: true

  - name: remove ceph aliases
    file:
      path: /etc/profile.d/ceph-aliases.sh
      state: absent

- name: check container hosts

  hosts:
    - "{{ mon_group_name|default('mons') }}"
    - "{{ osd_group_name|default('osds') }}"
    - "{{ mds_group_name|default('mdss') }}"
    - "{{ rgw_group_name|default('rgws') }}"
    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
    - "{{ nfs_group_name|default('nfss') }}"
    - "{{ mgr_group_name|default('mgrs') }}"

  gather_facts: true
  become: true

  tasks:

  - import_role:
      name: ceph-defaults
      private: false

  - import_role:
      name: ceph-facts
      private: false

  - name: show container list on all the nodes (should be empty)
    command: >
      {{ container_binary }} ps --filter='name=ceph' -a -q      
    register: containers_list
    changed_when: false

  - name: show container images on all the nodes (should be empty if tags was passed remove_img)
    command: >
      {{ container_binary }} images      
    register: images_list
    changed_when: false

  - name: fail if container are still present
    fail:
      msg: "It looks like container are still present."
    when: containers_list.stdout_lines|length > 0

- name: remove installed packages

  hosts:
    - "{{ mon_group_name|default('mons') }}"
    - "{{ osd_group_name|default('osds') }}"
    - "{{ mds_group_name|default('mdss') }}"
    - "{{ rgw_group_name|default('rgws') }}"
    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
    - "{{ nfs_group_name|default('nfss') }}"
    - "{{ mgr_group_name|default('mgrs') }}"

  become: true

  tags:
    with_pkg

  tasks:
  - name: check if it is Atomic host
    stat: path=/run/ostree-booted
    register: stat_ostree

  - name: set fact for using Atomic host
    set_fact:
      is_atomic: "{{ stat_ostree.stat.exists }}"

  - name: stop docker service
    service:
      name: docker
      state: stopped
      enabled: no
    when: not is_atomic
    ignore_errors: true

  - name: debian based systems tasks
    block:
      - name: remove docker-py on debian
        pip:
          name: docker-py
          state: absent

      - name: remove six on debian
        pip:
          name: six
          state: absent

      - name: remove pip and docker on debian
        apt:
          name: ['python-pip', 'docker-engine']
          state: absent
          update_cache: yes
          autoremove: yes
    when: ansible_distribution == 'Debian'

  - name: remove pip and docker on ubuntu
    apt:
      name: ['python-pip', 'docker', 'docker.io']
      state: absent
      update_cache: yes
      autoremove: yes
    when: ansible_distribution == 'Ubuntu'

  - name: red hat based systems tasks
    block:
      - name: yum related tasks on red hat
        block:
          - name: remove packages on redhat
            yum:
              name: ['epel-release', 'python-pip', 'docker-engine', 'docker']
              state: absent

          - name: remove package dependencies on redhat
            command: yum -y autoremove
            args:
              warn: no

          - name: remove package dependencies on redhat again
            command: yum -y autoremove
            args:
              warn: no
        when:
          ansible_pkg_mgr == "yum"

      - name: dnf related tasks on red hat
        block:
          - name: remove pip and docker on redhat
            dnf:
              name: ['python-pip', 'docker-engine', 'docker']
              state: absent

          - name: remove package dependencies on redhat
            command: dnf -y autoremove
            args:
              warn: no

          - name: remove package dependencies on redhat again
            command: dnf -y autoremove
            args:
              warn: no
        when:
          ansible_pkg_mgr == "dnf"
    when:
      ansible_os_family == 'RedHat' and
      not is_atomic

- name: purge ceph directories

  hosts:
    - "{{ mon_group_name|default('mons') }}"
    - "{{ osd_group_name|default('osds') }}"
    - "{{ mds_group_name|default('mdss') }}"
    - "{{ rgw_group_name|default('rgws') }}"
    - "{{ rbdmirror_group_name|default('rbdmirrors') }}"
    - "{{ nfs_group_name|default('nfss') }}"
    - "{{ mgr_group_name|default('mgrs') }}"

  gather_facts: false # Already gathered previously

  become: true

  tasks:
  - name: purge ceph directories for "{{ ansible_hostname }}"
    file:
      path: "{{ item }}"
      state: absent
    with_items:
      - /etc/ceph
      - /var/log/ceph

  - name: remove data
    shell: rm -rf /var/lib/ceph/*

- name: purge fetch directory

  hosts:
    - localhost

  gather_facts: false

  tasks:

  - name: set fetch_directory value if not set
    set_fact:
      fetch_directory: "fetch/"
    when: fetch_directory is not defined

  - name: purge fetch directory for localhost
    file:
      path: "{{ fetch_directory }}/"
      state: absent