Rotate kubelet server certificate. (#6453)

* Rotate kubelet server certificate.

* CI test kubelet server cert rotation

* Approve kubelet serving certificates in tests.

docs/vars.md

@@ -128,6 +128,11 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
   to match Docker configuration.
 * *kubelet_rotate_certificates* - Auto rotate the kubelet client certificates by requesting new certificates
   from the kube-apiserver when the certificate expiration approaches.
+* *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates
+  from the kube-apiserver when the certificate expiration approaches.
+  **Note** that server certificates are **not** approved automatically. Approve them manually
+  (`kubectl get csr`, `kubectl certificate approve`) or implement custom approving controller like
+  [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp).
 * *node_labels* - Labels applied to nodes via kubelet --node-labels parameter.
   For example, labels can be set in the inventory as variables or more widely in group_vars.
   *node_labels* can be defined either as a dict or a comma-separated labels string:

roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2

@@ -198,7 +198,10 @@ apiServer:
 {% endif %}
 {% if event_ttl_duration is defined %}
     event-ttl: {{ event_ttl_duration }}
-{%endif%}
+{% endif %}
+{% if kubelet_rotate_server_certificates %}
+    kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt
+{% endif %}
 {% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
   extraVolumes:
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -34,6 +34,9 @@ clusterDomain: {{ dns_domain }}
 {% if kubelet_rotate_certificates|bool %}
 rotateCertificates: true
 {% endif %}
+{% if kubelet_rotate_server_certificates|bool %}
+serverTLSBootstrap: true
+{% endif %}
 {# DNS settings for kubelet #}
 {% if enable_nodelocaldns %}
 {% set kubelet_cluster_dns = [nodelocaldns_ip] %}

roles/kubespray-defaults/defaults/main.yaml

@@ -394,6 +394,8 @@ kubelet_authorization_mode_webhook: true
 # kubelet uses certificates for authenticating to the Kubernetes API
 # Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration
 kubelet_rotate_certificates: true
+# kubelet can also request a new server certificate from the Kubernetes API
+kubelet_rotate_server_certificates: false
 
 ## List of key=value pairs that describe feature gates for
 ## the k8s cluster.

tests/files/packet_centos7-flannel-containerd-addons-ha.yml

@@ -25,6 +25,7 @@ metrics_server_kubelet_insecure_tls: true
 kube_token_auth: true
 kube_basic_auth: true
 enable_nodelocaldns: false
+kubelet_rotate_server_certificates: true
 
 kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration
 kube_oidc_client_id: kubespray-example

tests/testcases/030_check-network.yml

@@ -15,6 +15,30 @@
       bin_dir: "/usr/local/bin"
     when: not ansible_os_family in ["Flatcar Container Linux by Kinvolk"]
 
+  - name: Approve kubelet serving certificates
+    block:
+
+    - name: Get certificate signing requests
+      command: "{{ bin_dir }}/kubectl get csr -o name"
+      register: get_csr
+      changed_when: false
+
+    - name: Check there are csrs
+      assert:
+        that: get_csr.stdout_lines | length > 0
+        fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
+
+    - name: Approve certificates
+      command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}"
+      register: certificate_approve
+      when: get_csr.stdout_lines | length > 0
+      changed_when: certificate_approve.stdout
+
+    - debug:
+        msg: "{{ certificate_approve.stdout.split('\n') }}"
+
+    when: kubelet_rotate_server_certificates | default(false)
+
   - name: Create test namespace  # noqa 301 305
     shell: "{{ bin_dir }}/kubectl create namespace test"