Add etcd TLS support

roles/download/defaults/main.yml

@@ -10,7 +10,7 @@ kube_version: v1.4.3
 etcd_version: v3.0.6
 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: v0.22.0
+calico_version: v1.0.0-beta
 calico_cni_version: v1.4.2
 weave_version: v1.6.1
 flannel_version: v0.6.2
@@ -39,7 +39,8 @@ flannel_server_helper_image_tag: "{{ flannel_server_helper_version }}"
 flannel_image_repo: "quay.io/coreos/flannel"
 flannel_image_tag: "{{ flannel_version }}"
 calicoctl_image_repo: "calico/ctl"
-calicoctl_image_tag: "{{ calico_version }}"
+# TODO(mattymo): v1.0.0-beta has different syntax. Needs work to upgrade
+calicoctl_image_tag: "v0.22.0"
 calico_node_image_repo: "calico/node"
 calico_node_image_tag: "{{ calico_version }}"
 hyperkube_image_repo: "quay.io/coreos/hyperkube"

roles/etcd/defaults/main.yml

@@ -1,2 +1,8 @@
 ---
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
+
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+etcd_cert_group: root
+
+etcd_script_dir: "{{ bin_dir }}/etcd-scripts"

roles/etcd/files/make-ssl-etcd.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/ssl/etcd"
+fi
+
+tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+
+# ETCD member
+openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/

roles/etcd/handlers/main.yml

@@ -11,7 +11,7 @@
   when: ansible_service_mgr == "systemd"
 
 - name: wait for etcd up
-  uri: url="http://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" 
+  uri: url="https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" validate_certs=no
   register: result
   until: result.status is defined and result.status == 200
   retries: 10
@@ -22,3 +22,8 @@
     name: etcd
     state: restarted
   when: is_etcd_master
+
+- name: set etcd_secret_changed
+  set_fact:
+    etcd_secret_changed: true
+

roles/etcd/tasks/check_certs.yml

@@ -0,0 +1,36 @@
+---
+- name: "Check_certs | check if the certs have already been generated on first master"
+  stat:
+    path: "{{ etcd_cert_dir }}/ca.pem"
+  delegate_to: "{{groups['etcd'][0]}}"
+  register: etcdcert_master
+  run_once: true
+
+- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
+  set_fact:
+    sync_certs: false
+    gen_certs: false
+
+- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: not etcdcert_master.stat.exists
+  run_once: true
+
+- name: "Check certs | check if a cert already exists"
+  stat:
+    path: "{{ etcd_cert_dir }}/ca.pem"
+  register: etcdcert
+
+- name: "Check_certs | Set 'sync_certs' to true"
+  set_fact:
+    sync_certs: true
+  when: >-
+      {%- set certs = {'sync': False} -%}
+      {%- for server in play_hosts
+         if (not hostvars[server].etcdcert.stat.exists|default(False)) or
+         (hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
+         {%- set _ = certs.update({'sync': True}) -%}
+      {%- endfor -%}
+      {{ certs.sync }}
+  run_once: true

roles/etcd/tasks/gen_certs.yml

@@ -0,0 +1,111 @@
+---
+
+- name: Gen_certs | create etcd script dir
+  file:
+    path: "{{ etcd_script_dir }}"
+    state: directory
+    owner: root
+  when: inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | create etcd cert dir
+  file:
+    path={{ etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=root
+    recurse=yes
+
+- name: Gen_certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ etcd_config_dir }}/openssl.conf"
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+
+- name: Gen_certs | copy certs generation script
+  copy:
+    src: "make-ssl-etcd.sh"
+    dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh"
+    mode: 0700
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+
+- name: Gen_certs | run cert generation script
+  command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  run_once: yes
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: gen_certs|default(false)
+  notify: set etcd_secret_changed
+
+- set_fact:
+    master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
+    node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+
+- name: Gen_certs | Gather etcd master certs
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
+  register: etcd_master_cert_data
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
+  when: sync_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Gather etcd node certs
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+  register: etcd_node_cert_data
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
+  when: sync_certs|default(false)
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Copy certs on masters
+  shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+  changed_when: false
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]
+
+- name: Gen_certs | Copy certs on nodes
+  shell: "echo '{{etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+  changed_when: false
+  when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and
+        inventory_hostname not in groups['etcd']
+
+- name: Gen_certs | check certificate permissions
+  file:
+    path={{ etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=kube
+    recurse=yes
+
+- name: Gen_certs | set permissions on keys
+  shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
+  when: inventory_hostname in groups['etcd']
+  changed_when: false
+
+- name: Gen_certs | target ca-certificates directory
+  set_fact:
+    ca_cert_dir: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors
+      {%- elif ansible_os_family == "CoreOS" -%}
+      /etc/ssl/certs
+      {%- endif %}
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_dir }}/etcd-ca.crt"
+    remote_src: true
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: Gen_certs | update ca-certificatesa (RedHat)
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/tasks/main.yml

@@ -1,5 +1,7 @@
 ---
 - include: pre_upgrade.yml
+- include: check_certs.yml
+- include: gen_certs.yml
 - include: install.yml
   when: is_etcd_master
 - include: set_cluster_health.yml

roles/etcd/tasks/pre_upgrade.yml

@@ -0,0 +1,34 @@
+- name: "Pre-upgrade | check for etcd-proxy unit file"
+  stat:
+    path: /etc/systemd/system/etcd-proxy.service
+  register: kube_apiserver_service_file
+
+- name: "Pre-upgrade | check for etcd-proxy init script"
+  stat:
+    path: /etc/init.d/etcd-proxy
+  register: kube_apiserver_init_script
+
+- name: "Pre-upgrade | stop etcd-proxy if service defined"
+  service:
+    name: etcd-proxy
+    state: stopped
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+
+- name: "Pre-upgrade | remove etcd-proxy service definition"
+  file:
+    path: "{{ item }}"
+    state: absent
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+  with_items:
+    - /etc/systemd/system/etcd-proxy.service
+    - /etc/init.d/etcd-proxy
+
+- name: "Pre-upgrade | find etcd-proxy container"
+  command: docker ps -aq --filter "name=etcd-proxy*"
+  register: etcd_proxy_container
+  ignore_errors: true
+
+- name: "Pre-upgrade | remove etcd-proxy if it exists"
+  command: "docker rm -f {{item}}"
+  with_items: "{{etcd_proxy_container.stdout_lines}}"
+

roles/etcd/templates/deb-etcd-docker.initd.j2

@@ -19,8 +19,9 @@ DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
 DAEMON_EXEC=`basename $DAEMON`
 DAEMON_ARGS="run --restart=always --env-file=/etc/etcd.env \
 --net=host \
--v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
 -v /var/lib/etcd:/var/lib/etcd:rw \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
 --name={{ etcd_member_name | default("etcd") }} \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \
 {% if etcd_after_v3 %}

roles/etcd/templates/etcd-docker.service.j2

@@ -11,7 +11,8 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
 {# TODO(mattymo): Allow docker IP binding and disable in envfile
    -p 2380:2380 -p 2379:2379 #}
 --net=host \
--v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
 -v /var/lib/etcd:/var/lib/etcd:rw \
 --name={{ etcd_member_name | default("etcd") }} \
 {{ etcd_image_repo }}:{{ etcd_image_tag }} \

roles/etcd/templates/etcd.j2

@@ -3,10 +3,19 @@ ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
 ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
 ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
 
-ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379,http://127.0.0.1:2379
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
 ETCD_ELECTION_TIMEOUT=10000
 ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
-ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
 ETCD_NAME={{ etcd_member_name }}
 ETCD_PROXY=off
 ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH=true

roles/etcd/templates/openssl.conf.j2

@@ -0,0 +1,39 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = @alt_names
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+authorityKeyIdentifier=keyid:always,issuer
+
+[alt_names]
+DNS.1 = localhost
+{% for host in groups['etcd'] %}
+DNS.{{ 1 + loop.index }} = {{ host }}
+{% endfor %}
+{% if loadbalancer_apiserver is defined  and apiserver_loadbalancer_domain_name is defined %}
+{% set idx =  groups['etcd'] | length | int + 1 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['etcd'] %}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx =  groups['etcd'] | length | int * 2 + 1 %}
+IP.{{ idx }} = 127.0.0.1

roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2

@@ -27,6 +27,12 @@ spec:
           env:
             - name: ETCD_ENDPOINTS
               value: "{{ etcd_access_endpoint }}"
+            - name: ETCD_CA_CERT_FILE
+              value: "{{ etcd_cert_dir }}/ca.pem"
+            - name: ETCD_CERT_FILE
+              value: "{{ etcd_cert_dir }}/node.pem"
+            - name: ETCD_KEY_FILE
+              value: "{{ etcd_cert_dir }}/node-key.pem"
             # Location of the Kubernetes API - this shouldn't need to be
             # changed so long as it is used in conjunction with
             # CONFIGURE_ETC_HOSTS="true".

roles/kubernetes/master/defaults/main.yml

@@ -28,3 +28,9 @@ kube_apiserver_insecure_bind_address: 127.0.0.1
 
 # Logging directory (sysvinit systems)
 kube_log_dir: "/var/log/kubernetes"
+
+# ETCD cert dir for connecting apiserver to etcd
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+
+

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -14,6 +14,9 @@ spec:
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
     - --etcd-servers={{ etcd_access_endpoint }}
     - --etcd-quorum-read=true
+    - --etcd-cafile={{ etcd_cert_dir }}/ca.pem
+    - --etcd-certfile={{ etcd_cert_dir }}/node.pem
+    - --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
     - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
     - --apiserver-count={{ kube_apiserver_count }}
     - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
@@ -50,6 +53,9 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
+    - mountPath: {{ etcd_cert_dir }}
+      name: etcd-certs
+      readOnly: true
     - mountPath: /var/log/
       name: logfile
   volumes:
@@ -59,7 +65,9 @@ spec:
   - hostPath:
       path: /etc/ssl/certs/
     name: ssl-certs-host
+  - hostPath:
+      path: {{ etcd_cert_dir }}
+    name: etcd-certs
   - hostPath:
       path: /var/log/
     name: logfile
-

roles/kubernetes/node/templates/cni-calico.conf.j2

@@ -2,6 +2,9 @@
   "name": "calico-k8s-network",
   "type": "calico",
   "etcd_endpoints": "{{ etcd_access_endpoint }}",
+  "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
+  "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
+  "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
   "log_level": "info",
   "ipam": {
     "type": "calico-ipam"

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -23,14 +23,14 @@
 
 - set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
 - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
-- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
+- set_fact: etcd_peer_url="https://{{ etcd_access_address }}:2380"
-- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
+- set_fact: etcd_client_url="https://{{ etcd_access_address }}:2379"
 - set_fact: etcd_authority="127.0.0.1:2379"
-- set_fact: etcd_endpoint="http://{{ etcd_authority }}"
+- set_fact: etcd_endpoint="https://{{ etcd_authority }}"
 - set_fact:
     etcd_access_addresses: |-
       {% for item in groups['etcd'] -%}
-        http://{{ item }}:2379{% if not loop.last %},{% endif %}
+        https://{{ item }}:2379{% if not loop.last %},{% endif %}
       {%- endfor %}
 - set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}"
 - set_fact:
@@ -41,7 +41,7 @@
 - set_fact:
     etcd_peer_addresses: |-
       {% for item in groups['etcd'] -%}
-        {{ "etcd"+loop.index|string }}=http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
+        {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
       {%- endfor %}
 - set_fact:
     is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"

roles/network_plugin/calico/defaults/main.yml

@@ -8,3 +8,6 @@ ipip: false
 # Set to true if you want your calico cni binaries to overwrite the
 # ones from hyperkube while leaving other cni plugins intact.
 overwrite_hyperkube_cni: true
+
+calico_cert_dir: /etc/calico/certs
+etcd_cert_dir: /etc/ssl/etcd/ssl

roles/network_plugin/calico/tasks/main.yml

@@ -12,6 +12,24 @@
 
 - meta: flush_handlers
 
+- name: Calico | Create calico certs directory
+  file:
+    dest: "{{ calico_cert_dir }}"
+    state: directory
+    mode: 0750
+    owner: root
+    group: root
+
+- name: Calico | Link etcd certificates for calico-node
+  file:
+    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    dest: "{{ calico_cert_dir }}/{{ item.d }}"
+    state: hard
+  with_items:
+    - {s: "ca.pem", d: "ca_cert.crt"}
+    - {s: "node.pem", d: "cert.crt"}
+    - {s: "node-key.pem", d: "key.pem"}
+
 - name: Calico | Install calicoctl container script
   template:
     src: calicoctl-container.j2
@@ -41,7 +59,7 @@
   when: "{{ overwrite_hyperkube_cni|bool }}"
 
 - name: Calico | wait for etcd
-  uri: url=http://localhost:2379/health
+  uri: url=https://localhost:2379/health validate_certs=no
   register: result
   until: result.status == 200 or result.status == 401
   retries: 10
@@ -87,7 +105,11 @@
 
 - name: Calico | Get calico configuration from etcd
   command: |-
-    curl http://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
+    curl \
+      --cacert {{ etcd_cert_dir }}/ca.pem \
+      --cert {{ etcd_cert_dir}}/admin.pem \
+      --key {{ etcd_cert_dir }}/admin-key.pem \
+      https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
   register: calico_pools_raw
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true

roles/network_plugin/calico/templates/calicoctl-container.j2

@@ -2,8 +2,12 @@
 /usr/bin/docker run --privileged --rm \
 --net=host --pid=host \
 -e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
+-e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
+-e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
+-e ETCD_KEY_FILE=/etc/calico/certs/key.pem \
 -v /usr/bin/docker:/usr/bin/docker \
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v /var/run/calico:/var/run/calico \
+-v /etc/calico/certs:/etc/calico/certs:ro \
 {{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
 $@

roles/network_plugin/calico/templates/network-environment.j2

@@ -7,3 +7,6 @@ KUBERNETES_MASTER={{ kube_apiserver_endpoint }}
 
 # IP and port of etcd instance used by Calico
 ETCD_ENDPOINTS={{ etcd_access_endpoint }}
+ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt
+ETCD_CERT_FILE=/etc/calico/certs/cert.crt
+ETCD_KEY_FILE=/etc/calico/certs/key.pem

roles/network_plugin/flannel/tasks/main.yml

@@ -1,9 +1,11 @@
 ---
-- name: Flannel | Write flannel configuration
+- name: Flannel | Set Flannel etcd configuration
-  template:
+  command: |-
-    src: network.json
+    {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
-    dest: /etc/flannel-network.json
+    set /{{ cluster_name }}/network/config \
-    backup: yes
+    '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
 
 - name: Flannel | Create flannel pod manifest
   template:

roles/network_plugin/flannel/templates/flannel-pod.yml

@@ -12,26 +12,16 @@
       - name: "subnetenv"
         hostPath:
           path: "/run/flannel"
-      - name: "networkconfig"
+      - name: "etcd-certs"
         hostPath:
-          path: "/etc/flannel-network.json"
+          path: "{{ etcd_cert_dir }}"
     containers:
-      - name: "flannel-server-helper"
-        image: "{{ flannel_server_helper_image_repo }}:{{ flannel_server_helper_image_tag }}"
-        args:
-          - "--network-config=/etc/flannel-network.json"
-          - "--etcd-prefix=/{{ cluster_name }}/network"
-          - "--etcd-endpoints={{ etcd_access_endpoint }}"
-        volumeMounts:
-          - name: "networkconfig"
-            mountPath: "/etc/flannel-network.json"
-        imagePullPolicy: "Always"
       - name: "flannel-container"
         image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
         command:
           - "/bin/sh"
           - "-c"
-          - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
+          - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
         ports:
           - hostPort: 10253
             containerPort: 10253
@@ -41,6 +31,8 @@
         volumeMounts:
           - name: "subnetenv"
             mountPath: "/run/flannel"
+          - name: "etcd-certs"
+            mountPath: "{{ etcd_cert_dir }}"
         securityContext:
           privileged: true
     hostNetwork: true

roles/network_plugin/flannel/templates/network.json

@@ -1 +0,0 @@
-{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }

roles/uploads/defaults/main.yml

@@ -5,7 +5,7 @@ local_release_dir: /tmp
 kube_version: v1.4.3
 
 etcd_version: v3.0.6
-calico_version: v0.22.0
+calico_version: v0.23.0
 calico_cni_version: v1.4.2
 weave_version: v1.6.1