Use find module for checking for certificates

Also generate certs only when absent on master (rather than
when absent on target node)

roles/etcd/tasks/check_certs.yml

@@ -1,18 +1,11 @@
 ---
 - name: "Check_certs | check if all certs have already been generated on first master"
-  stat:
+  find:
-    path: "{{ etcd_cert_dir }}/{{ item }}"
+    paths: "{{ etcd_cert_dir }}"
-    get_md5: no
+    patterns: "ca.pem,node*.pem"
   delegate_to: "{{groups['etcd'][0]}}"
   register: etcdcert_master
   run_once: true
-  with_items: >-
-       ['ca.pem',
-       {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique %}
-       {% for host in all_etcd_hosts %}
-       'node-{{ host }}-key.pem'
-       {% if not loop.last %}{{','}}{% endif %}
-       {% endfor %}]
 
 - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false"
   set_fact:
@@ -20,34 +13,56 @@
     gen_certs: false
     etcd_secret_changed: false
 
-- name: "Check_certs | Set 'gen_certs' to true"
+- name: "Check certs | check if a cert already exists on node"
-  set_fact:
-    gen_certs: true
-  when: "not {{item.stat.exists}}"
-  run_once: true
-  with_items: "{{etcdcert_master.results}}"
-
-- name: "Check certs | check if a cert already exists"
   stat:
     path: "{{ etcd_cert_dir }}/{{ item }}"
-  register: etcdcert
+  register: etcdcert_node
   with_items:
     - ca.pem
     - node-{{ inventory_hostname }}-key.pem
 
+
+- name: "Check_certs | Set 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: "not '{{ item }}' in etcdcert_master.files|map(attribute='path') | list"
+  run_once: true
+  with_items: >-
+       ['{{etcd_cert_dir}}/ca.pem',
+       {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %}
+       {% for host in all_etcd_hosts %}
+       '{{etcd_cert_dir}}/node-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
+
+
+- name: "Check_certs | Set 'gen_node_certs' to true"
+  set_fact:
+    gen_node_certs: |-
+      {
+      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%}
+      {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
+      {% for host in all_etcd_hosts -%}
+        {% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if host_cert in existing_certs -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique %}
+      {% if gen_node_certs[inventory_hostname] or 
-      {% for host in all_etcd_hosts %}
+        (not etcdcert_node.results[0].stat.exists|default(False)) or
-        {% if host == inventory_hostname %}
+          (not etcdcert_node.results[1].stat.exists|default(False)) or
-          {% if (not etcdcert.results[0].stat.exists|default(False)) or
+            (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
-              (not etcdcert.results[1].stat.exists|default(False)) or
+              {%- set _ = certs.update({'sync': True}) -%}
-              (etcdcert.results[1].stat.checksum|default('') != etcdcert_master.results[loop.index].stat.checksum|default('')) -%}
+      {% endif %}
-            {%- set _ = certs.update({'sync': True}) -%}
-          {% endif %}
-        {% endif %}
-      {%- endfor -%}
       {{ certs.sync }}
+

roles/etcd/tasks/gen_certs_script.yml

@@ -43,15 +43,15 @@
   when: gen_certs|default(false)
 
 - name: Gen_certs | run cert generation script
-  command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
   environment:
     - MASTERS: "{% for m in groups['etcd'] %}
-                  {% if hostvars[m].sync_certs|default(false) %}
+                  {% if gen_node_certs[m] %}
                     {{ m }}
                   {% endif %}
                 {% endfor %}"
     - HOSTS: "{% for h in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
-                {% if hostvars[h].sync_certs|default(false) %}
+                {% if gen_node_certs[h] %}
                     {{ h }}
                 {% endif %}
               {% endfor %}"
@@ -107,14 +107,38 @@
         sync_certs|default(false) and inventory_hostname not in groups['etcd']
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Copy certs on masters
+#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k 
-  shell: "base64 -d <<< '{{etcd_master_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}"
+#char limit when using shell command                                            
-  args:
+                                                                                
-    executable: /bin/bash
+#FIXME(mattymo): Use tempfile module in ansible 2.3                             
-  no_log: true
+- name: Gen_certs | Prepare tempfile for unpacking certs                        
-  changed_when: false
+  shell: mktemp /tmp/certsXXXXX.tar.gz                                          
+  register: cert_tempfile                                                       
+                                                                                
+- name: Gen_certs | Write master certs to tempfile                              
+  copy:                                                                         
+    content: "{{etcd_master_cert_data.stdout}}"                                      
+    dest: "{{cert_tempfile.stdout}}"                                            
+    owner: root                                                                 
+    mode: "0600"                                                                
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
-        inventory_hostname != groups['etcd'][0]
+        inventory_hostname != groups['etcd'][0]                          
+                                                                                
+- name: Gen_certs | Unpack certs on masters                                     
+  shell: "base64 -d < {{ cert_tempfile.stdout }} | tar xz -C {{ etcd_cert_dir }}"
+  no_log: true                                                                  
+  changed_when: false                                                           
+  check_mode: no                                                                
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]                          
+  notify: set secret_changed                                                    
+                                                                                
+- name: Gen_certs | Cleanup tempfile                                            
+  file:                                                                         
+    path: "{{cert_tempfile.stdout}}"                                            
+    state: absent                                                               
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]  
 
 - name: Gen_certs | Copy certs on nodes
   shell: "base64 -d <<< '{{etcd_node_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}"
@@ -163,4 +187,3 @@
 - name: Gen_certs | update ca-certificates (RedHat)
   command: update-ca-trust extract
   when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
-

roles/kubernetes/secrets/tasks/check-certs.yml

@@ -1,16 +1,11 @@
 ---
 - name: "Check_certs | check if the certs have already been generated on first master"
-  stat:
+  find:
-    path: "{{ kube_cert_dir }}/{{ item }}"
+    paths: "{{ kube_cert_dir }}"
+    patterns: "*.pem"
   delegate_to: "{{groups['kube-master'][0]}}"
   register: kubecert_master
   run_once: true
-  with_items: >-
-       ['ca.pem',
-       {% for host in groups['k8s-cluster'] %}
-       'node-{{ host }}-key.pem'
-       {% if not loop.last %}{{','}}{% endif %}
-       {% endfor %}]
 
 - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs', and 'secret_changed'  to false"
   set_fact:
@@ -18,33 +13,53 @@
     gen_certs: false
     secret_changed: false
 
-- name: "Check_certs | Set 'gen_certs' to true"
+- name: "Check certs | check if a cert already exists on node"
-  set_fact:
-    gen_certs: true
-  when: "not {{ item.stat.exists }}"
-  run_once: true
-  with_items: "{{ kubecert_master.results }}"
-
-- name: "Check certs | check if a cert already exists"
   stat:
     path: "{{ kube_cert_dir }}/{{ item }}"
-  register: kubecert
+  register: kubecert_node
   with_items:
     - ca.pem
     - node-{{ inventory_hostname }}-key.pem
 
+- name: "Check_certs | Set 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: "not item in kubecert_master.files|map(attribute='path') | list"
+  run_once: true
+  with_items: >-
+       ['{{ kube_cert_dir }}/ca.pem',
+       {% for host in groups['k8s-cluster'] %}
+       '{{ kube_cert_dir }}/node-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
+
+- name: "Check_certs | Set 'gen_node_certs' to true"
+  set_fact:
+    gen_node_certs: |-
+      {
+      {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
+      {% for host in groups['k8s-cluster'] -%}
+        {% set host_cert = "%s/node-%s-key.pem"|format(kube_cert_dir, host) %}
+        {% if host_cert in existing_certs -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
+
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {%- for host in groups['k8s-cluster'] %}
+      {% if gen_node_certs[inventory_hostname] or    
-        {% if host == inventory_hostname %}
+        (not kubecert_node.results[0].stat.exists|default(False)) or
-          {% if (not kubecert.results[0].stat.exists|default(False)) or
+          (not kubecert_node.results[1].stat.exists|default(False)) or
-              (not kubecert.results[1].stat.exists|default(False)) or
+            (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
-              (kubecert.results[1].stat.checksum|default('') != kubecert_master.results[loop.index].stat.checksum|default('')) -%}
+              {%- set _ = certs.update({'sync': True}) -%}
-            {%- set _ = certs.update({'sync': True}) -%}
+      {% endif %}
-          {% endif %}
-        {% endif %}
-      {%- endfor -%}
       {{ certs.sync }}
+

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -40,12 +40,12 @@
   command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}"
   environment:
     - MASTERS: "{% for m in groups['kube-master'] %}
-                  {% if hostvars[m].sync_certs|default(true) %}
+                  {% if gen_node_certs[m]|default(false) %}
                     {{ m }}
                   {% endif %}
                 {% endfor %}"
     - HOSTS: "{% for h in groups['k8s-cluster'] %}
-                {% if hostvars[h].sync_certs|default(true) %}
+                {% if gen_node_certs[h]|default(true) %}
                     {{ h }}
                 {% endif %}
               {% endfor %}"