From 07657aecf47524f7a93ce6af7988babde272b3ed Mon Sep 17 00:00:00 2001 From: Michael Beatty Date: Thu, 1 Mar 2018 10:41:19 -0600 Subject: [PATCH 01/13] add support for azure vnetResourceGroup --- inventory/sample/group_vars/all.yml | 1 + roles/kubernetes/preinstall/tasks/azure-credential-check.yml | 5 +++++ roles/kubernetes/preinstall/templates/azure-cloud-config.j2 | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index c107b049f..282943a8d 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -76,6 +76,7 @@ bin_dir: /usr/local/bin #azure_subnet_name: #azure_security_group_name: #azure_vnet_name: +#azure_vnet_resource_group: #azure_route_table_name: ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) diff --git a/roles/kubernetes/preinstall/tasks/azure-credential-check.yml b/roles/kubernetes/preinstall/tasks/azure-credential-check.yml index fa2d82fd2..68cbaa160 100644 --- a/roles/kubernetes/preinstall/tasks/azure-credential-check.yml +++ b/roles/kubernetes/preinstall/tasks/azure-credential-check.yml @@ -44,6 +44,11 @@ msg: "azure_vnet_name is missing" when: azure_vnet_name is not defined or azure_vnet_name == "" +- name: check azure_vnet_resource_group value + fail: + msg: "azure_vnet_resource_group is missing" + when: azure_vnet_resource_group is not defined or azure_vnet_resource_group == "" + - name: check azure_route_table_name value fail: msg: "azure_route_table_name is missing" diff --git a/roles/kubernetes/preinstall/templates/azure-cloud-config.j2 b/roles/kubernetes/preinstall/templates/azure-cloud-config.j2 index 139a06cc1..d33c044b2 100644 --- a/roles/kubernetes/preinstall/templates/azure-cloud-config.j2 +++ b/roles/kubernetes/preinstall/templates/azure-cloud-config.j2 @@ -8,5 +8,6 @@ "subnetName": "{{ azure_subnet_name }}", "securityGroupName": "{{ azure_security_group_name }}", "vnetName": "{{ azure_vnet_name }}", + "vnetResourceGroup": "{{ azure_vnet_resource_group }}", "routeTableName": "{{ azure_route_table_name }}" -} \ No newline at end of file +} From 50e3ccfa2baed51522740ed3aa97e7f86f38bdae Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Mon, 12 Mar 2018 12:46:14 -0400 Subject: [PATCH 02/13] uses new kube_memory_reserved/kube_cpu_reserved variables in kubelt --- roles/kubernetes/node/templates/kubelet-container.j2 | 4 ++-- roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 4e8d4c371..22671b2c3 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -5,8 +5,8 @@ --privileged \ --name=kubelet \ --restart=on-failure:5 \ - --memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \ - --cpu-shares={{ kubelet_cpu_limit|regex_replace('m', '') }} \ + --memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \ + --cpu-shares={{ kube_cpu_reserved|regex_replace('m', '') }} \ -v /dev:/dev:rw \ -v /etc/cni:/etc/cni:ro \ -v /opt/cni:/opt/cni:ro \ diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index c8cf40e7b..f67c72bf8 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -29,7 +29,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --cadvisor-port={{ kube_cadvisor_port }} \ {# end kubeadm specific settings #} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ ---kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \ +--kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ From 6abe78ff461fc5006458f2478fb3cb8fb4a6dae9 Mon Sep 17 00:00:00 2001 From: Cyril Jouve Date: Mon, 12 Mar 2018 19:59:22 +0100 Subject: [PATCH 03/13] use archive instead of command --- scripts/collect-info.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/collect-info.yaml b/scripts/collect-info.yaml index 1a0e2307b..14daf9d19 100644 --- a/scripts/collect-info.yaml +++ b/scripts/collect-info.yaml @@ -114,7 +114,12 @@ with_items: "{{logs}}" - name: Pack results and logs - local_action: raw GZIP=-9 tar --remove-files -cvzf {{dir|default(".")}}/logs.tar.gz -C /tmp collect-info + archive: + path: "/tmp/collect-info" + dest: "{{ dir|default('.') }}/logs.tar.gz" + remove: true + delegate_to: localhost + become: false run_once: true - name: Clean up collected command outputs From d264da8f080c4917e0f6646d81e96e5bbaf71338 Mon Sep 17 00:00:00 2001 From: "rong.zhang" Date: Tue, 13 Mar 2018 14:28:49 +0800 Subject: [PATCH 04/13] Fix yamllint roles error for #2188 commit --- roles/network_plugin/weave/tasks/main.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml index dc0a032af..a8dfa0586 100644 --- a/roles/network_plugin/weave/tasks/main.yml +++ b/roles/network_plugin/weave/tasks/main.yml @@ -2,12 +2,11 @@ - import_tasks: seed.yml when: weave_mode_seed - - name: template weavenet conflist template: - src: weavenet.conflist.j2 - dest: /etc/cni/net.d/00-weave.conflist - owner: kube + src: weavenet.conflist.j2 + dest: /etc/cni/net.d/00-weave.conflist + owner: kube - name: Weave | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" From 2e0b33f75420bd7f82468ace18ed531cfe49ce8a Mon Sep 17 00:00:00 2001 From: "rong.zhang" Date: Tue, 13 Mar 2018 14:05:03 +0800 Subject: [PATCH 05/13] Add remove node to getting-started doc --- docs/getting-started.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/getting-started.md b/docs/getting-started.md index 961d1a9cf..26141050a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -51,6 +51,18 @@ ansible-playbook -i inventory/mycluster/hosts.ini scale.yml -b -v \ --private-key=~/.ssh/private_key ``` +Remove nodes +------------ + +You may want to remove **worker** nodes to your existing cluster. This can be done by re-running the `remove-node.yml` playbook. First, all nodes will be drained, then stop some kubernetes services and delete some certificates, and finally execute the kubectl command to delete these nodes. This can be combined with the add node function, This is generally helpful when doing something like autoscaling your clusters. Of course if a node is not working, you can remove the node and install it again. + +- Add worker nodes to the list under kube-node if you want to delete them (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)). +- Run the ansible-playbook command, substituting `remove-node.yml`: +``` +ansible-playbook -i inventory/mycluster/hosts.ini remove-node.yml -b -v \ + --private-key=~/.ssh/private_key +``` + Connecting to Kubernetes ------------------------ By default, Kubespray configures kube-master hosts with insecure access to From 39d247a2384339ed8969b9bd78634080583fded2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Tue, 13 Mar 2018 10:31:15 +0100 Subject: [PATCH 06/13] Add support to kubeadm too Explicitly defines the --kubelet-preferred-address-types parameter #2418 Fixes #2453 --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7add..e4657a601 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -37,6 +37,7 @@ apiServerExtraArgs: admission-control: {{ kube_apiserver_admission_control | join(',') }} apiserver-count: "{{ kube_apiserver_count }}" service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} From f3788525ffcae3229bbd22881271f44fb7cba1aa Mon Sep 17 00:00:00 2001 From: Dann Bohn Date: Tue, 13 Mar 2018 06:15:48 -0400 Subject: [PATCH 07/13] fixes yamllint for docker defaults, and weave network plugin --- roles/docker/defaults/main.yml | 8 ++++---- roles/network_plugin/weave/tasks/main.yml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index df7b97ab4..aa10371f5 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -21,16 +21,16 @@ docker_dns_servers_strict: yes docker_container_storage_setup: false -#CentOS/RedHat docker-ce repo +# CentOS/RedHat docker-ce repo docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/7/$basearch/stable' docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' -#Ubuntu docker-ce repo +# Ubuntu docker-ce repo docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' -#Debian docker-ce repo +# Debian docker-ce repo docker_debian_repo_base_url: "https://download.docker.com/linux/debian" docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' -#dockerproject repo +# dockerproject repo dockerproject_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/7' dockerproject_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg' dockerproject_apt_repo_base_url: 'https://apt.dockerproject.org/repo' diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml index dc0a032af..43cb81a1c 100644 --- a/roles/network_plugin/weave/tasks/main.yml +++ b/roles/network_plugin/weave/tasks/main.yml @@ -5,9 +5,9 @@ - name: template weavenet conflist template: - src: weavenet.conflist.j2 - dest: /etc/cni/net.d/00-weave.conflist - owner: kube + src: weavenet.conflist.j2 + dest: /etc/cni/net.d/00-weave.conflist + owner: kube - name: Weave | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" From d1e6632e6ae02c6e9c2246cc3aff892c1c8d2b81 Mon Sep 17 00:00:00 2001 From: zhengchuan hu Date: Wed, 14 Mar 2018 17:18:55 +0800 Subject: [PATCH 08/13] Fix err in kubelet.kubeadm.env.j2 1. 404 link url 2. kubelet_authentication_token_webhook is not work 3. kube_reserved variable set twice --- roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index f67c72bf8..5be20d533 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -1,4 +1,4 @@ -### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/10-kubeadm.conf +### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/ ### All upstream values should be present in this file # logging to stderr means we get it in the systemd journal @@ -23,13 +23,14 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if kubelet_authentication_token_webhook %} --authentication-token-webhook \ {% endif %} +{% if kubelet_authorization_mode_webhook %} --authorization-mode=Webhook \ +{% endif %} --client-ca-file={{ kube_cert_dir }}/ca.crt \ --pod-manifest-path={{ kube_manifest_dir }} \ --cadvisor-port={{ kube_cadvisor_port }} \ {# end kubeadm specific settings #} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ ---kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ From 788e41a315b36c86d60e68fa71f3493bb286c895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Wed, 14 Mar 2018 19:23:43 +0100 Subject: [PATCH 09/13] Make sure output from extra args is strings Setting the following: ``` kube_kubeadm_controller_extra_args: address: 0.0.0.0 terminated-pod-gc-threshold: "100" ``` Results in `terminated-pod-gc-threshold: 100` in the kubeadm config file. But it has to be a string to work. --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7add..a8ffbbb17 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -59,7 +59,7 @@ apiServerExtraArgs: {% endif %} allow-privileged: "true" {% for key in kube_kubeadm_apiserver_extra_args %} - {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key] }} + {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key]|string }} {% endfor %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} @@ -69,12 +69,12 @@ controllerManagerExtraArgs: feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} {% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: {{ kube_kubeadm_controller_extra_args[key] }} + {{ key }}: {{ kube_kubeadm_controller_extra_args[key]|string }} {% endfor %} {% if kube_kubeadm_scheduler_extra_args|length > 0 %} schedulerExtraArgs: {% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key] }} + {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key]|string }} {% endfor %} {% endif %} apiServerCertSANs: From d843e3d562080681d4da7fe218211c8b829d37e6 Mon Sep 17 00:00:00 2001 From: Oleg Vyukov Date: Thu, 15 Mar 2018 22:18:18 +0300 Subject: [PATCH 10/13] Fix indent Custom ConfigMap ingress-nginx (#2447) --- .../ingress_nginx/templates/ingress-nginx-cm.yml.j2 | 2 +- .../templates/ingress-nginx-tcp-servicecs-cm.yml.j2 | 2 +- .../templates/ingress-nginx-udp-servicecs-cm.yml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 index 79b9e17e7..7e47e81b1 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap | to_nice_yaml }} + {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 index 5fb875940..0a87e91b7 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap_tcp_services | to_nice_yaml }} + {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 index bcb004bc9..d943e5718 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap_udp_services | to_nice_yaml }} + {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }} From 3d6fd491795adb8a38493afe6c2968a46051d5ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Thu, 15 Mar 2018 20:20:05 +0100 Subject: [PATCH 11/13] Added option for encrypting secrets to etcd v.2 (#2428) * Added option for encrypting secrets to etcd * Fix keylength to 32 * Forgot the default * Rename secrets.yaml to secrets_encryption.yaml * Fix static path for secrets file to use ansible variable * Rename secrets.yaml.j2 to secrets_encryption.yaml.j2 * Base64 encode the token * Fixed merge error * Changed path to credentials dir * Update path to secrets file which is now readable inside the apiserver container. Set better file permissions * Add encryption option to k8s-cluster.yml --- inventory/sample/group_vars/k8s-cluster.yml | 5 ++++- roles/kubernetes/master/defaults/main.yml | 5 +++++ roles/kubernetes/master/tasks/encrypt-at-rest.yml | 10 ++++++++++ roles/kubernetes/master/tasks/main.yml | 3 +++ .../master/templates/kubeadm-config.yaml.j2 | 3 +++ .../templates/manifests/kube-apiserver.manifest.j2 | 3 +++ .../master/templates/secrets_encryption.yaml.j2 | 11 +++++++++++ 7 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 roles/kubernetes/master/tasks/encrypt-at-rest.yml create mode 100644 roles/kubernetes/master/templates/secrets_encryption.yaml.j2 diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 128e8cc99..8f69afc25 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -111,7 +111,10 @@ kube_apiserver_insecure_port: 8080 # (http) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables -kube_proxy_mode: iptables +kube_proxy_mode: iptables + +## Encrypting Secret Data at Rest (experimental) +kube_encrypt_secret_data: false # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 59e528822..a1b506d4e 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -92,3 +92,8 @@ kube_kubeadm_scheduler_extra_args: {} ## Variable for influencing kube-scheduler behaviour volume_cross_zone_attachment: false + +## Encrypting Secret Data at Rest +kube_encrypt_secret_data: false +kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token length=32 chars=ascii_letters,digits') }}" +kube_encryption_algorithm: "aescbc" # Must be either: aescbc, secretbox or aesgcm diff --git a/roles/kubernetes/master/tasks/encrypt-at-rest.yml b/roles/kubernetes/master/tasks/encrypt-at-rest.yml new file mode 100644 index 000000000..2e569b08b --- /dev/null +++ b/roles/kubernetes/master/tasks/encrypt-at-rest.yml @@ -0,0 +1,10 @@ +--- +- name: Write secrets for encrypting secret data at rest + template: + src: secrets_encryption.yaml.j2 + dest: "{{ kube_config_dir }}/ssl/secrets_encryption.yaml" + owner: root + group: "{{ kube_cert_group }}" + mode: 0640 + tags: + - kube-apiserver diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 04ad307fd..daa10fd79 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -12,6 +12,9 @@ - import_tasks: users-file.yml when: kube_basic_auth|default(true) +- import_tasks: encrypt-at-rest.yml + when: kube_encrypt_secret_data + - name: Compare host kubectl with hyperkube container command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl" register: kubectl_task_compare_result diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7add..cd266ed3d 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -52,6 +52,9 @@ apiServerExtraArgs: {% if kube_oidc_groups_claim is defined %} oidc-groups-claim: {{ kube_oidc_groups_claim }} {% endif %} +{% endif %} +{% if kube_encrypt_secret_data %} + experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml {% endif %} storage-backend: {{ kube_apiserver_storage_backend }} {% if kube_api_runtime_config is defined %} diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 0dbe93cab..c1685410d 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -103,6 +103,9 @@ spec: {% if authorization_modes %} - --authorization-mode={{ authorization_modes|join(',') }} {% endif %} +{% if kube_encrypt_secret_data %} + - --experimental-encryption-provider-config={{ kube_config_dir }}/ssl/secrets_encryption.yaml +{% endif %} {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 new file mode 100644 index 000000000..84c6a4ea8 --- /dev/null +++ b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 @@ -0,0 +1,11 @@ +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - {{ kube_encryption_algorithm }}: + keys: + - name: key + secret: {{ kube_encrypt_token | b64encode }} + - identity: {} From 40c0f3756bbabbbf8b9f05eaf3e86bce600a7e11 Mon Sep 17 00:00:00 2001 From: woopstar Date: Thu, 15 Mar 2018 20:27:19 +0100 Subject: [PATCH 12/13] Encapsulate item instead of casting to string --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index a8ffbbb17..bbe329b5f 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -59,7 +59,7 @@ apiServerExtraArgs: {% endif %} allow-privileged: "true" {% for key in kube_kubeadm_apiserver_extra_args %} - {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key]|string }} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} @@ -69,12 +69,12 @@ controllerManagerExtraArgs: feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} {% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: {{ kube_kubeadm_controller_extra_args[key]|string }} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} {% if kube_kubeadm_scheduler_extra_args|length > 0 %} schedulerExtraArgs: {% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key]|string }} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" {% endfor %} {% endif %} apiServerCertSANs: From 1a35948ff67369d856b08d8ccb747374330ab82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Thu, 15 Mar 2018 20:33:57 +0100 Subject: [PATCH 13/13] Enable encrypting the secrets Enable the CI test to check the encryption of secrets --- tests/files/gce_centos7-flannel-addons.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 0e4346f67..8ac8a901b 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -15,3 +15,4 @@ etcd_deployment_type: host deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce +kube_encrypt_secret_data: true