From a1fff30bd9bbab8ce1937738599158fb7a98eb6f Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 17 Oct 2019 17:02:38 +0300 Subject: [PATCH] Generate TLS certs for calico typha (#5258) * Generate TLS certs for calico typha Change-Id: I3883f49c124c52d0fc5b900ca2b44e4e2ed0d707 * Add group vars note Change-Id: I63550dfef616e884efdbd42010a90b2c04c5eb69 --- .../group_vars/k8s-cluster/k8s-net-calico.yml | 3 + roles/download/defaults/main.yml | 3 +- roles/network_plugin/calico/defaults/main.yml | 4 + .../calico/files/make-ssl-typha.sh | 89 +++++++++++++++++++ .../network_plugin/calico/files/openssl.conf | 20 +++++ roles/network_plugin/calico/tasks/install.yml | 6 ++ .../calico/tasks/typha_certs.yml | 49 ++++++++++ .../calico/templates/calico-config.yml.j2 | 2 +- .../calico/templates/calico-node.yml.j2 | 37 +++++++- .../calico/templates/calico-typha.yml.j2 | 33 +++++++ tests/files/gce_centos7-calico-ha.yml | 5 +- tests/files/packet_centos7-calico-ha.yml | 3 + 12 files changed, 249 insertions(+), 5 deletions(-) create mode 100644 roles/network_plugin/calico/files/make-ssl-typha.sh create mode 100644 roles/network_plugin/calico/files/openssl.conf create mode 100644 roles/network_plugin/calico/tasks/typha_certs.yml diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml index a7c78f6e5..3935681c2 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml @@ -34,5 +34,8 @@ # Use typha (only with kdd) # typha_enabled: false +# Generate TLS certs for secure typha<->calico-node communication +# typha_secure: false + # Number of typha replicas # typha_replicas: 1 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index aaa56a9fb..b80b94523 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -71,6 +71,7 @@ calico_cni_version: "v3.7.3" calico_policy_version: "v3.7.3" calico_rr_version: "v0.6.1" calico_typha_version: "v3.7.3" +typha_enabled: false flannel_version: "v0.11.0" flannel_cni_version: "v0.3.0" @@ -525,7 +526,7 @@ downloads: - calico-rr calico_typha: - enabled: "{{ typha_enabled == 'calico' }}" + enabled: "{{ typha_enabled }}" container: true repo: "{{ calico_typha_image_repo }}" tag: "{{ calico_typha_image_tag }}" diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 43f1c1e16..61462ba94 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -74,4 +74,8 @@ typha_enabled: false # Number of typha replicas typha_replicas: 1 +# Generate certifcates for typha<->calico-node communication +typha_secure: false + + calico_feature_control: {} diff --git a/roles/network_plugin/calico/files/make-ssl-typha.sh b/roles/network_plugin/calico/files/make-ssl-typha.sh new file mode 100644 index 000000000..67567e9ff --- /dev/null +++ b/roles/network_plugin/calico/files/make-ssl-typha.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Author: Smana smainklh@gmail.com +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail +usage() +{ + cat << EOF +Create self signed certificates + +Usage : $(basename $0) -f [-d ] + -h | --help : Show this message + -f | --config : Openssl configuration file + -d | --ssldir : Directory where the certificates will be installed + -c | --cadir : Directory where the existing CA is located + + ex : + $(basename $0) -f openssl.conf -d /srv/ssl +EOF +} + +# Options parsing +while (($#)); do + case "$1" in + -h | --help) usage; exit 0;; + -f | --config) CONFIG=${2}; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; + -c | --cadir) CADIR="${2}"; shift 2;; + *) + usage + echo "ERROR : Unknown option" + exit 3 + ;; + esac +done + +if [ -z ${CONFIG} ]; then + echo "ERROR: the openssl configuration file is missing. option -f" + exit 1 +fi +if [ -z ${SSLDIR} ]; then + SSLDIR="/etc/calico/certs" +fi + +tmpdir=$(mktemp -d /tmp/calico_typha_certs.XXXXXX) +trap 'rm -rf "${tmpdir}"' EXIT +cd "${tmpdir}" + +mkdir -p "${SSLDIR} ${CADIR}" + +# Root CA +if [ -e "$CADIR/ca.key" ]; then + # Reuse existing CA + cp $CADIR/{ca.crt,ca.key} . +else + openssl genrsa -out ca.key 2048 > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca.key -days 3650 -out ca.crt -subj "/CN=calico-typha-ca" > /dev/null 2>&1 +fi + +# Typha server +openssl genrsa -out typha-server.key 2048 > /dev/null 2>&1 +openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1 +openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + +# Typha client +openssl genrsa -out typha-client.key 2048 > /dev/null 2>&1 +openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1 +openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + +# Install certs +if [ -e "$CADIR/ca.key" ]; then + # No pass existing CA + rm -f ca.crt ca.key +fi + +mv {*.crt,*.key} ${SSLDIR}/ diff --git a/roles/network_plugin/calico/files/openssl.conf b/roles/network_plugin/calico/files/openssl.conf new file mode 100644 index 000000000..b1cf7bf8f --- /dev/null +++ b/roles/network_plugin/calico/files/openssl.conf @@ -0,0 +1,20 @@ +req_extensions = v3_req +distinguished_name = req_distinguished_name + +[req_distinguished_name] + +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment + +[ ssl_client ] +extendedKeyUsage = clientAuth, serverAuth +basicConstraints = CA:FALSE +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid,issuer + +[ v3_ca ] +basicConstraints = CA:TRUE +keyUsage = cRLSign, digitalSignature, keyCertSign +subjectKeyIdentifier=hash +authorityKeyIdentifier=keyid:always,issuer diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index ed883ac84..91af85941 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -57,6 +57,12 @@ - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} when: calico_datastore == "etcd" +- name: Calico | Generate typha certs + include_tasks: typha_certs.yml + when: + - typha_secure + - inventory_hostname == groups['kube-master'][0] + - name: Calico | Install calicoctl wrapper script template: src: "calicoctl.{{ calico_datastore }}.sh.j2" diff --git a/roles/network_plugin/calico/tasks/typha_certs.yml b/roles/network_plugin/calico/tasks/typha_certs.yml new file mode 100644 index 000000000..c2647a1cb --- /dev/null +++ b/roles/network_plugin/calico/tasks/typha_certs.yml @@ -0,0 +1,49 @@ +--- +- name: Calico | Check if typha-server exists + command: "{{ bin_dir }}/kubectl -n kube-system get secret typha-server" + register: typha_server_secret + changed_when: false + failed_when: false + +- name: Calico | Ensure calico certs dir + file: + path: /etc/calico/certs + state: directory + when: typha_server_secret.rc != 0 + +- name: Calico | Copy ssl script for typha certs + copy: + src: make-ssl-typha.sh + dest: "{{ bin_dir }}/make-ssl-typha.sh" + mode: 0755 + when: typha_server_secret.rc != 0 + +- name: Calico | Copy ssl config for typha certs + copy: + src: openssl.conf + dest: /etc/calico/certs/openssl.conf + mode: 0644 + when: typha_server_secret.rc != 0 + +- name: Calico | Generate typha certs + command: >- + {{ bin_dir }}/make-ssl-typha.sh + -f /etc/calico/certs/openssl.conf + -c {{ kube_cert_dir }} + -d /etc/calico/certs + when: typha_server_secret.rc != 0 + +- name: Calico | Create typha tls secrets + command: >- + {{ bin_dir }}/kubectl -n kube-system + create secret tls {{ item.name }} + --cert {{ item.cert }} + --key {{ item.key }} + with_items: + - name: typha-server + cert: /etc/calico/certs/typha-server.crt + key: /etc/calico/certs/typha-server.key + - name: typha-client + cert: /etc/calico/certs/typha-client.crt + key: /etc/calico/certs/typha-client.key + when: typha_server_secret.rc != 0 diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index 550e7b7e1..39b69d132 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -9,7 +9,7 @@ data: etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" -{% elif calico_datastore == "kdd" and typha_enabled == "true" %} +{% elif calico_datastore == "kdd" and typha_enabled %} # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is # essential. diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index ab64450c2..62d4f7bd3 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -155,14 +155,24 @@ spec: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" - {% if typha_enabled == "true" %} +{% if typha_enabled %} # Typha support: controlled by the ConfigMap. - name: FELIX_TYPHAK8SSERVICENAME valueFrom: configMapKeyRef: name: calico-config key: typha_service_name - {% endif %} +{% if typha_secure %} + - name: FELIX_TYPHACN + value: typha-server + - name: FELIX_TYPHACAFILE + value: /etc/typha-ca/ca.crt + - name: FELIX_TYPHACERTFILE + value: /etc/typha-client/typha-client.crt + - name: FELIX_TYPHAKEYFILE + value: /etc/typha-client/typha-client.key +{% endif %} +{% endif %} # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" @@ -296,6 +306,16 @@ spec: - name: xtables-lock mountPath: /run/xtables.lock readOnly: false +{% if typha_secure %} + - name: typha-client + mountPath: /etc/typha-client + readOnly: true + - name: typha-cacert + subPath: ca.crt + mountPath: /etc/typha-ca/ca.crt + readOnly: true +{% endif %} + volumes: # Used by calico/node. - name: lib-modules @@ -332,6 +352,19 @@ spec: - name: host-local-net-dir hostPath: path: /var/lib/cni/networks +{% endif %} +{% if typha_enabled and typha_secure %} + - name: typha-client + secret: + secretName: typha-client + items: + - key: tls.crt + path: typha-client.crt + - key: tls.key + path: typha-client.key + - name: typha-cacert + hostPath: + path: "/etc/kubernetes/ssl/" {% endif %} updateStrategy: rollingUpdate: diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2 index 4844a99fd..3efd9fe50 100644 --- a/roles/network_plugin/calico/templates/calico-typha.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2 @@ -62,6 +62,7 @@ spec: # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node + priorityClassName: system-cluster-critical containers: - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }} name: calico-typha @@ -86,6 +87,24 @@ spec: value: "kubernetes" - name: TYPHA_HEALTHENABLED value: "true" +{% if typha_secure %} + - name: TYPHA_CAFILE + value: /etc/ca/ca.crt + - name: TYPHA_CLIENTCN + value: typha-client + - name: TYPHA_SERVERCERTFILE + value: /etc/typha/server_certificate.pem + - name: TYPHA_SERVERKEYFILE + value: /etc/typha/server_key.pem + volumeMounts: + - mountPath: /etc/typha + name: typha-server + readOnly: true + - mountPath: /etc/ca/ca.crt + subPath: ca.crt + name: cacert + readOnly: true +{% endif %} # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, # this opens a port on the host, which may need to be secured. #- name: TYPHA_PROMETHEUSMETRICSENABLED @@ -127,6 +146,20 @@ spec: host: localhost {% endif %} periodSeconds: 10 +{% if typha_secure %} + volumes: + - name: typha-server + secret: + secretName: typha-server + items: + - key: tls.crt + path: server_certificate.pem + - key: tls.key + path: server_key.pem + - name: cacert + hostPath: + path: "{{ kube_cert_dir }}" +{% endif %} --- diff --git a/tests/files/gce_centos7-calico-ha.yml b/tests/files/gce_centos7-calico-ha.yml index 58584c7c6..741897d27 100644 --- a/tests/files/gce_centos7-calico-ha.yml +++ b/tests/files/gce_centos7-calico-ha.yml @@ -10,5 +10,8 @@ kube_network_plugin: calico download_localhost: true download_run_once: true deploy_netchecker: true -dns_min_replicas: 1 cloud_provider: gce +dns_min_replicas: 1 +typha_enabled: true +calico_backend: kdd +typha_secure: true diff --git a/tests/files/packet_centos7-calico-ha.yml b/tests/files/packet_centos7-calico-ha.yml index cdc51d8ff..b51556176 100644 --- a/tests/files/packet_centos7-calico-ha.yml +++ b/tests/files/packet_centos7-calico-ha.yml @@ -9,3 +9,6 @@ download_localhost: true download_run_once: true deploy_netchecker: true dns_min_replicas: 1 +typha_enabled: true +calico_backend: kdd +typha_secure: true