Handling docker clean up during docker upgrade and docker config changes. (#3321)

* Added changes to clean up orphan containers and reload docker & kubelet directories.

* Added new files for cleaning up orphans and docker & kubelet directories

* Added new lines at the end of these files

* removed the trailing whitespaces from main.yml and clean-up.yml

* Updated as per the review comments

* Updated as per the review comments

* Removed service_facts and package_facts because they are not supported in ansible 2.4.0

* Corrected yaml syntax errors

* Removed the use of json_query filter and utilized selectattr

* Removed trailing spaces

* Changed the default value of docker_clean_up to false

* Added Changes to only include cleanup-docker-orphans.sh

* Reverted back changes done inside handler.

* Removed trailing spaces and made default value of docker_orphan_clean_up as true

* Reverted the default value of docker_orphan_clean_up as false

* Made the docker clean up as drop in

* Made the docker clean up as drop in

* Reverted the value of boolean docker_orphan_clean_up to false

roles/container-engine/docker/defaults/main.yml

@@ -44,3 +44,6 @@ dockerproject_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'
 dockerproject_apt_repo_base_url: 'https://apt.dockerproject.org/repo'
 dockerproject_apt_repo_gpgkey: 'https://apt.dockerproject.org/gpg'
 docker_bin_dir: "/usr/bin"
+
+# flag to enable/disable docker cleanup
+docker_orphan_clean_up: false

roles/container-engine/docker/files/cleanup-docker-orphans.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+list_descendants ()
+{
+  local children=$(ps -o pid= --ppid "$1")
+  for pid in $children
+  do
+    list_descendants "$pid"
+  done
+  [[ -n "$children" ]] && echo "$children"
+}
+
+count_shim_processes=$(pgrep -f ^docker-containerd-shim | wc -l)
+live_restore=$(docker info --format {{.LiveRestoreEnabled}} 2>/dev/null)
+
+if [ ${count_shim_processes} -gt 0 ] && [ -n "${live_restore}" -a "${live_restore}" == "true" ]; then
+        # Find all container pids from shims
+        orphans=$(pgrep -P $(pgrep -d ',' -f ^docker-containerd-shim) |\
+        # Filter out valid docker pids, leaving the orphans
+        egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}'))
+
+        if [[ -n "$orphans" ]]
+        then
+                # Get shim pids of orphans
+                orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans))
+
+                # Find all orphaned container PIDs
+                orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done)
+
+                # Recursively kill all child PIDs of orphan shims
+                echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)"
+                #kill -9 $orphan_container_pids || true
+
+        else
+                echo "No orphaned containers found"
+        fi
+else
+        echo "Either live-restore is turned off or the node doesn't have any shim processes."
+fi

roles/container-engine/docker/tasks/systemd.yml

@@ -38,4 +38,18 @@
   notify: restart docker
   when: dns_mode != 'none' and resolvconf_mode == 'docker_dns'
 
+- name: Copy docker orphan clean up script to the node
+  copy:
+    src: cleanup-docker-orphans.sh
+    dest: "{{ bin_dir }}/cleanup-docker-orphans.sh"
+    mode: 0755
+  when: docker_orphan_clean_up | bool
+
+- name: Write docker orphan clean up systemd drop-in
+  template:
+    src: docker-orphan-cleanup.conf.j2
+    dest: "/etc/systemd/system/docker.service.d/docker-orphan-cleanup.conf"
+  notify: restart docker
+  when: docker_orphan_clean_up | bool
+
 - meta: flush_handlers

roles/container-engine/docker/templates/docker-orphan-cleanup.conf.j2

@@ -0,0 +1,2 @@
+[Service]
+ExecStop=-{{ bin_dir }}/cleanup-docker-orphans.sh