kubeasz/tools/11.deletcd.yml

# WARNNING:  this playbook will clean the etcd {{ ETCD_TO_DEL }} 
# USAGE:     easzctl del-etcd 1.1.1.1

- hosts: localhost 
  vars_prompt:
  - name: "ETCD_TO_DEL"
    prompt: "which etcd node is about to be deleted?(e.g 192.168.1.1)"
    private: no
    confirm: yes
  tasks:
  # step0: run prechecks
  - fail: msg="{{ ETCD_TO_DEL }} is NOT a member of etcd cluster!"
    when: "ETCD_TO_DEL not in groups['etcd']" 

  - fail: msg="you CAN NOT delete the last member of etcd cluster!"
    when: "groups['etcd']|length < 2" 

  - block:
    # step1: find a healthy member in the etcd cluster
    - name: set NODE_IPS of the etcd cluster
      set_fact: NODE_IPS="{% for host in groups['etcd'] %}{{ host }} {% endfor %}"

    - name: get etcd cluster status
      shell: 'for ip in {{ NODE_IPS }};do \
                ETCDCTL_API=3 {{ base_dir }}/bin/etcdctl \
                --endpoints=https://"$ip":2379 \
                --cacert={{ base_dir }}/.cluster/ssl/ca.pem \
                --cert={{ base_dir }}/.cluster/ssl/admin.pem \
                --key={{ base_dir }}/.cluster/ssl/admin-key.pem \
                endpoint health; \
              done'
      register: ETCD_CLUSTER_STATUS
      ignore_errors: true

    - debug: var="ETCD_CLUSTER_STATUS.stdout"

    - debug: var="ETCD_CLUSTER_STATUS.stderr"

    - name: get a running ectd node
      shell: 'echo -e "{{ ETCD_CLUSTER_STATUS.stdout }}" \
               "{{ ETCD_CLUSTER_STATUS.stderr }}" \
               |grep "is healthy"|sed -n "1p"|cut -d: -f2|cut -d/ -f3'
      register: RUNNING_NODE

    - debug: var="RUNNING_NODE.stdout"

    # step2: remove jobs run on the healthy member if needed
    - name: get ID of etcd node to delete
      shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member list \
              |grep {{ ETCD_TO_DEL }}:2380|cut -d',' -f1"
      register: ETCD_ID
      delegate_to: "{{ RUNNING_NODE.stdout }}"
  
    - name: get NAME of etcd node to delete
      shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member list \
              |grep {{ ETCD_TO_DEL }}:2380|cut -d' ' -f3|cut -d',' -f1"
      register: ETCD_NAME
      delegate_to: "{{ RUNNING_NODE.stdout }}"
  
    - name: delete a etcd member
      shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member remove {{ ETCD_ID.stdout }}"
      delegate_to: "{{ RUNNING_NODE.stdout }}"
      when: "ETCD_ID.stdout != ''"
  
    - name: clean etcd {{ ETCD_TO_DEL }} if possible
      shell: "ansible-playbook {{ base_dir }}/roles/clean/clean_node.yml \
                -e NODE_TO_CLEAN={{ ETCD_TO_DEL }} \
                -e DEL_ETCD=yes >> /tmp/ansible-`date +'%Y%m%d%H%M%S'`.log 2>&1 \
              || echo 'data not cleaned on {{ ETCD_TO_DEL }}'"
      register: CLEAN_STATUS

    - debug: var="CLEAN_STATUS.stdout"

    # lineinfile is inadequate to delete lines between some specific line range
    - name: remove the etcd's node entry in hosts
      shell: 'sed -i "/^\[etcd/,/^\[kube-master/ {/^{{ ETCD_TO_DEL }}[^0-9]/d}" {{ base_dir }}/hosts'
      args:
        warn: false

    - name: reconfig and restart the etcd cluster
      shell: "ansible-playbook {{ base_dir }}/02.etcd.yml >> /tmp/ansible-`date +'%Y%m%d%H%M%S'`.log 2>&1"
    when: "groups['etcd']|length > 1 and ETCD_TO_DEL in groups['etcd']"