From a8f2af0503598993bbdd5562680899c3677c5336 Mon Sep 17 00:00:00 2001
From: Brad Beam <brad.beam@b-rad.info>
Date: Tue, 13 Dec 2016 09:36:17 -0600
Subject: [PATCH 1/3] Adding initial rkt support

---
 cluster.yml                      |  1 +
 inventory/group_vars/all.yml     |  4 ++++
 roles/download/defaults/main.yml |  2 +-
 roles/rkt/defaults/main.yml      |  6 +++++
 roles/rkt/tasks/install.yml      | 41 ++++++++++++++++++++++++++++++++
 roles/rkt/tasks/main.yml         |  4 ++++
 roles/rkt/vars/debian.yml        |  2 ++
 roles/rkt/vars/fedora.yml        |  2 ++
 roles/rkt/vars/redhat.yml        |  2 ++
 9 files changed, 63 insertions(+), 1 deletion(-)
 create mode 100644 roles/rkt/defaults/main.yml
 create mode 100644 roles/rkt/tasks/install.yml
 create mode 100644 roles/rkt/tasks/main.yml
 create mode 100644 roles/rkt/vars/debian.yml
 create mode 100644 roles/rkt/vars/fedora.yml
 create mode 100644 roles/rkt/vars/redhat.yml

diff --git a/cluster.yml b/cluster.yml
index e10c0d2c9..ce169a161 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -28,6 +28,7 @@
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
+    - { role: rkt, tags: rkt }
 
 - hosts: etcd:!k8s-cluster
   any_errors_fatal: true
diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
index 839da100a..a3598e9db 100644
--- a/inventory/group_vars/all.yml
+++ b/inventory/group_vars/all.yml
@@ -197,3 +197,7 @@ k8s_image_pull_policy: IfNotPresent
 # default packages to install within the cluster
 kpm_packages: []
 #  - name: kube-system/grafana
+
+rkt_version: 1.21.0
+etcd_deployment_type: rkt
+kubelet_deployment_type: docker
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index df9bd75c7..8c1225527 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -121,7 +121,7 @@ downloads:
     unarchive: true
     owner: "etcd"
     mode: "0755"
-    container: "{{ etcd_deployment_type == 'docker' }}"
+    container: "{{ etcd_deployment_type }} in [ 'docker', 'rkt' ]"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
   hyperkube:
diff --git a/roles/rkt/defaults/main.yml b/roles/rkt/defaults/main.yml
new file mode 100644
index 000000000..6794429a5
--- /dev/null
+++ b/roles/rkt/defaults/main.yml
@@ -0,0 +1,6 @@
+---
+
+rkt_version: 1.12.0
+rkt_pkg_version: "{{ rkt_version }}-1"
+rkt_download_src: https://github.com/coreos/rkt
+rkt_download_url: "{{ rkt_download_src }}/releases/download/v{{ rkt_version }}"
diff --git a/roles/rkt/tasks/install.yml b/roles/rkt/tasks/install.yml
new file mode 100644
index 000000000..0317c7a7e
--- /dev/null
+++ b/roles/rkt/tasks/install.yml
@@ -0,0 +1,41 @@
+---
+- name: gather os specific variables for rkt
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}.yml"
+      - "{{ ansible_os_family|lower }}.yml"
+      - defaults.yml
+      paths:
+      - ../vars
+      skip: true
+  tags: facts
+
+#- name: download rkt pkg
+#  get_url:
+#    url: "{{ rkt_download_url }}/{{rkt_pkg_name}}"
+#    dest: "{{ local_release_dir }}" 
+#  when: (ansible_os_family != "CoreOS") and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
+
+- name: install rkt pkg on ubuntu
+  apt:
+    deb: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
+    state: present
+  register: rkt_task_result
+  until: rkt_task_result|success
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: ansible_os_family == "Debian" and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
+
+- name: "install rkt pkg on centos"
+  yum:
+    pkg: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
+    state: present
+  register: rkt_task_result
+  until: rkt_task_result|success
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  when: ansible_os_family == "RedHat" and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
diff --git a/roles/rkt/tasks/main.yml b/roles/rkt/tasks/main.yml
new file mode 100644
index 000000000..9d87123bb
--- /dev/null
+++ b/roles/rkt/tasks/main.yml
@@ -0,0 +1,4 @@
+---
+
+- name: Install rkt
+  include: install.yml
diff --git a/roles/rkt/vars/debian.yml b/roles/rkt/vars/debian.yml
new file mode 100644
index 000000000..9cfffe5c9
--- /dev/null
+++ b/roles/rkt/vars/debian.yml
@@ -0,0 +1,2 @@
+---
+rkt_pkg_name: "rkt_{{ rkt_pkg_version }}_amd64.deb"
diff --git a/roles/rkt/vars/fedora.yml b/roles/rkt/vars/fedora.yml
new file mode 100644
index 000000000..13149e8fb
--- /dev/null
+++ b/roles/rkt/vars/fedora.yml
@@ -0,0 +1,2 @@
+---
+rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"
diff --git a/roles/rkt/vars/redhat.yml b/roles/rkt/vars/redhat.yml
new file mode 100644
index 000000000..13149e8fb
--- /dev/null
+++ b/roles/rkt/vars/redhat.yml
@@ -0,0 +1,2 @@
+---
+rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"

From 8dc19374cc2b4e7aa14cbd64fa88a46128a7b665 Mon Sep 17 00:00:00 2001
From: Brad Beam <brad.beam@b-rad.info>
Date: Tue, 13 Dec 2016 11:20:22 -0600
Subject: [PATCH 2/3] Allowing etcd to run via rkt

---
 cluster.yml                                   |  2 +-
 inventory/group_vars/all.yml                  |  2 +-
 .../tasks/{install.yml => install_docker.yml} | 13 +--------
 roles/etcd/tasks/install_host.yml             |  9 ++++++
 roles/etcd/tasks/install_rkt.yml              | 26 +++++++++++++++++
 roles/etcd/tasks/main.yml                     |  2 +-
 roles/etcd/templates/etcd-rkt.service.j2      | 29 +++++++++++++++++++
 roles/rkt/tasks/install.yml                   | 12 ++------
 8 files changed, 71 insertions(+), 24 deletions(-)
 rename roles/etcd/tasks/{install.yml => install_docker.yml} (74%)
 create mode 100644 roles/etcd/tasks/install_host.yml
 create mode 100644 roles/etcd/tasks/install_rkt.yml
 create mode 100644 roles/etcd/templates/etcd-rkt.service.j2

diff --git a/cluster.yml b/cluster.yml
index ce169a161..553261063 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -28,7 +28,7 @@
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: rkt, tags: rkt }
+    - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
 
 - hosts: etcd:!k8s-cluster
   any_errors_fatal: true
diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
index a3598e9db..04e20922e 100644
--- a/inventory/group_vars/all.yml
+++ b/inventory/group_vars/all.yml
@@ -199,5 +199,5 @@ kpm_packages: []
 #  - name: kube-system/grafana
 
 rkt_version: 1.21.0
-etcd_deployment_type: rkt
+etcd_deployment_type: docker
 kubelet_deployment_type: docker
diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install_docker.yml
similarity index 74%
rename from roles/etcd/tasks/install.yml
rename to roles/etcd/tasks/install_docker.yml
index 0ed3f4154..f87caeb4c 100644
--- a/roles/etcd/tasks/install.yml
+++ b/roles/etcd/tasks/install_docker.yml
@@ -1,17 +1,6 @@
 ---
-- name: Install | Copy etcd binary from downloaddir
-  command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
-  when: etcd_deployment_type == "host"
-  register: etcd_copy
-  changed_when: false
-
-- name: Install | Copy etcdctl binary from downloaddir
-  command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
-  when: etcd_deployment_type == "host"
-  changed_when: false
-
 #Plan A: no docker-py deps
-- name: Install | Copy etcdctl binary from container
+- name: Install | Copy etcdctl binary from docker container
   command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
            {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
            {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl &&
diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml
new file mode 100644
index 000000000..6f588a2f0
--- /dev/null
+++ b/roles/etcd/tasks/install_host.yml
@@ -0,0 +1,9 @@
+---
+- name: Install | Copy etcd binary from downloaddir
+  command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
+  register: etcd_copy
+  changed_when: false
+
+- name: Install | Copy etcdctl binary from downloaddir
+  command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
+  changed_when: false
diff --git a/roles/etcd/tasks/install_rkt.yml b/roles/etcd/tasks/install_rkt.yml
new file mode 100644
index 000000000..85f875383
--- /dev/null
+++ b/roles/etcd/tasks/install_rkt.yml
@@ -0,0 +1,26 @@
+---
+- name: Trust etcd container
+  command: >-
+    /usr/bin/rkt trust
+    --skip-fingerprint-review
+    --root
+    https://quay.io/aci-signing-key
+  register: etcd_rkt_trust_result
+  until: etcd_rkt_trust_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+
+- name: Install | Copy etcdctl binary from rkt container
+  command: >-
+    /usr/bin/rkt run
+    --volume=bin-dir,kind=host,source={{ bin_dir}},readOnly=false
+    --mount=volume=bin-dir,target=/host/bin
+    {{ etcd_image_repo }}:{{ etcd_image_tag }}
+    --name=etcdctl-binarycopy
+    --exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl
+  register: etcd_task_result
+  until: etcd_task_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index d6320619c..cdd634517 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -5,7 +5,7 @@
   tags: [etcd-secrets, facts]
 - include: gen_certs.yml
   tags: etcd-secrets
-- include: install.yml
+- include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade
 - include: set_cluster_health.yml
diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2
new file mode 100644
index 000000000..002f6dbe6
--- /dev/null
+++ b/roles/etcd/templates/etcd-rkt.service.j2
@@ -0,0 +1,29 @@
+[Unit]
+Description=etcd rkt wrapper
+Documentation=https://github.com/coreos/etcd
+Wants=network.target
+
+[Service]
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=0
+LimitNOFILE=40000
+
+ExecStart={{ rkt_bin_dir | default("/usr/bin") }}/rkt run \
+--uuid-file-save=/var/run/etcd.uuid \
+--volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
+--mount=volume=etc-ssl-certs,target=/etc/ssl/certs \
+--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \
+--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \
+--volume=var-lib-etcd,kind=host,source=/var/lib/etcd,readOnly=false \
+--mount=volume=var-lib-etcd,target=/var/lib/etcd \
+--set-env-file=/etc/etcd.env \
+--stage1-from-dir=stage1-fly.aci \
+{{ etcd_image_repo }}:{{ etcd_image_tag }} \
+--name={{ etcd_member_name | default("etcd") }}
+
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/etcd.uuid
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/rkt/tasks/install.yml b/roles/rkt/tasks/install.yml
index 0317c7a7e..41823bf54 100644
--- a/roles/rkt/tasks/install.yml
+++ b/roles/rkt/tasks/install.yml
@@ -14,12 +14,6 @@
       skip: true
   tags: facts
 
-#- name: download rkt pkg
-#  get_url:
-#    url: "{{ rkt_download_url }}/{{rkt_pkg_name}}"
-#    dest: "{{ local_release_dir }}" 
-#  when: (ansible_os_family != "CoreOS") and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
-
 - name: install rkt pkg on ubuntu
   apt:
     deb: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
@@ -28,9 +22,9 @@
   until: rkt_task_result|success
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  when: ansible_os_family == "Debian" and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
+  when: ansible_os_family == "Debian"
 
-- name: "install rkt pkg on centos"
+- name: install rkt pkg on centos
   yum:
     pkg: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
     state: present
@@ -38,4 +32,4 @@
   until: rkt_task_result|success
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  when: ansible_os_family == "RedHat" and "rkt" in [ etcd_deployment_type, kubelet_deployment_type ]
+  when: ansible_os_family == "RedHat"

From 4b6f29d5e10e6951ba51199cd8f511a2483832e0 Mon Sep 17 00:00:00 2001
From: Brad Beam <brad.beam@b-rad.info>
Date: Tue, 20 Dec 2016 17:00:46 -0600
Subject: [PATCH 3/3] Adding kubelet in rkt

---
 .gitlab-ci.yml                                | 24 ++++++++
 roles/download/defaults/main.yml              |  4 +-
 roles/etcd/templates/etcd-rkt.service.j2      |  2 +-
 roles/kubernetes/node/tasks/install.yml       | 25 +++++++-
 ...t.service.j2 => kubelet.docker.service.j2} |  0
 .../node/templates/kubelet.rkt.service.j2     | 58 +++++++++++++++++++
 6 files changed, 109 insertions(+), 4 deletions(-)
 rename roles/kubernetes/node/templates/{kubelet.service.j2 => kubelet.docker.service.j2} (100%)
 create mode 100644 roles/kubernetes/node/templates/kubelet.rkt.service.j2

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 72d83d584..feb84e4b4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -49,6 +49,8 @@ before_script:
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   BOOTSTRAP_OS: none
   LOG_LEVEL: "-vv"
+  ETCD_DEPLOYMENT: "docker"
+  KUBELET_DEPLOYMENT: "docker"
 
 .gce: &gce
   <<: *job
@@ -102,6 +104,8 @@ before_script:
       -e download_localhost=true
       -e deploy_netchecker=true
       -e local_release_dir=${PWD}/downloads
+      -e etcd_deployment_type=${ETCD_DEPLOYMENT}
+      -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       cluster.yml
 
 
@@ -202,6 +206,15 @@ before_script:
   CLUSTER_MODE: ha
   BOOTSTRAP_OS: coreos
 
+.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: flannel
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separated
+  ETCD_DEPLOYMENT: rkt
+  KUBELET_DEPLOYMENT: rkt
+
 # Builds for PRs only (auto) and triggers (auto)
 coreos-calico-sep:
   stage: deploy-gce-part1
@@ -405,6 +418,17 @@ coreos-alpha-weave-ha:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+ubuntu-rkt-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_rkt_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 syntax-check:
   <<: *job
   stage: unit-tests
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 8c1225527..77933bfb6 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -115,13 +115,13 @@ downloads:
     version: "{{etcd_version}}"
     dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
     sha256: >-
-      {%- if etcd_deployment_type == 'docker' -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
+      {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
     source_url: "{{ etcd_download_url }}"
     url: "{{ etcd_download_url }}"
     unarchive: true
     owner: "etcd"
     mode: "0755"
-    container: "{{ etcd_deployment_type }} in [ 'docker', 'rkt' ]"
+    container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
   hyperkube:
diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2
index 002f6dbe6..eb26bc473 100644
--- a/roles/etcd/templates/etcd-rkt.service.j2
+++ b/roles/etcd/templates/etcd-rkt.service.j2
@@ -9,7 +9,7 @@ RestartSec=10s
 TimeoutStartSec=0
 LimitNOFILE=40000
 
-ExecStart={{ rkt_bin_dir | default("/usr/bin") }}/rkt run \
+ExecStart=/usr/bin/rkt run \
 --uuid-file-save=/var/run/etcd.uuid \
 --volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
 --mount=volume=etc-ssl-certs,target=/etc/ssl/certs \
diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml
index c9061725a..bfe4a8cc8 100644
--- a/roles/kubernetes/node/tasks/install.yml
+++ b/roles/kubernetes/node/tasks/install.yml
@@ -1,8 +1,31 @@
 ---
+- name: Trust kubelet container
+  command: >-
+    /usr/bin/rkt trust
+    --skip-fingerprint-review
+    --root
+    {{ item }}
+  register: kubelet_rkt_trust_result
+  until: kubelet_rkt_trust_result.rc == 0
+  with_items:
+    - "https://quay.io/aci-signing-key"
+    - "https://coreos.com/dist/pubkeys/aci-pubkeys.gpg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+  when: kubelet_deployment_type == "rkt"
+
+- name: create kubelet working directory
+  file:
+    state: directory
+    path: /var/lib/kubelet
+  when: kubelet_deployment_type == "rkt"
+
 - name: install | Write kubelet systemd init file
-  template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
   notify: restart kubelet
 
 - name: install | Install kubelet launch script
   template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
   notify: restart kubelet
+  when: kubelet_deployment_type == "docker"
diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.docker.service.j2
similarity index 100%
rename from roles/kubernetes/node/templates/kubelet.service.j2
rename to roles/kubernetes/node/templates/kubelet.docker.service.j2
diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
new file mode 100644
index 000000000..89a9f01dc
--- /dev/null
+++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
@@ -0,0 +1,58 @@
+[Unit]
+Description=Kubernetes Kubelet Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
+After=calico-node.service
+Wants=network.target calico-node.service
+{% else %}
+Wants=network.target
+{% endif %}
+
+[Service]
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=0
+LimitNOFILE=40000
+
+ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid
+ExecStartPre=-/bin/mkdir -p /var/lib/kubelet
+
+EnvironmentFile={{kube_config_dir}}/kubelet.env
+# stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts
+ExecStart=/usr/bin/rkt run \
+        --volume var-log,kind=host,source=/var/log \
+        --volume dns,kind=host,source=/etc/resolv.conf \
+        --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \
+        --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
+        --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
+        --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
+	--volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \
+        --volume run,kind=host,source=/run,readOnly=false \
+        --mount volume=var-log,target=/var/log \
+        --mount volume=dns,target=/etc/resolv.conf \
+        --mount volume=etc-kubernetes,target={{ kube_config_dir }} \
+        --mount volume=etc-ssl-certs,target=/etc/ssl/certs \
+        --mount volume=usr-share-certs,target=/usr/share/ca-certificates \
+        --mount volume=var-lib-docker,target=/var/lib/docker \
+        --mount volume=var-lib-kubelet,target=/var/lib/kubelet \
+        --mount volume=run,target=/run \
+        --stage1-from-dir=stage1-fly.aci \
+        {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \
+        --uuid-file-save=/var/run/kubelet.uuid \
+        --debug --exec=/kubelet -- \
+                $KUBE_LOGTOSTDERR \
+                $KUBE_LOG_LEVEL \
+                $KUBELET_API_SERVER \
+                $KUBELET_ADDRESS \
+                $KUBELET_PORT \
+                $KUBELET_HOSTNAME \
+                $KUBE_ALLOW_PRIV \
+                $KUBELET_ARGS \
+                $DOCKER_SOCKET \
+                $KUBELET_REGISTER_NODE \
+                $KUBELET_NETWORK_PLUGIN
+
+ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet.uuid
+
+[Install]
+WantedBy=multi-user.target