From c6fcbf6ee0bef57e4c880fe86a48cda131155ae4 Mon Sep 17 00:00:00 2001 From: Nicolas Goudry Date: Wed, 3 Apr 2024 08:54:12 +0200 Subject: [PATCH] Remove access to cluster from anonymous users (#11016) * feat: add user facing variable with default * feat: remove rolebinding to anonymous users after init and upgrade * feat: use file discovery for secondary control plane nodes * feat: use file discovery for nodes * fix: do not fail if rolebinding does not exist * docs: add warning about kube_api_anonymous_auth * style: improve readability of delegate_to parameter * refactor: rename discovery kubeconfig file * test: enable new variable in hardening and upgrade test cases * docs: add option to config parameters * test: multiple instances and upgrade --- docs/vars.md | 5 +++++ .../group_vars/k8s_cluster/k8s-cluster.yml | 3 +++ .../control-plane/defaults/main/main.yml | 3 +++ .../control-plane/tasks/kubeadm-secondary.yml | 20 +++++++++++++++++++ .../control-plane/tasks/kubeadm-setup.yml | 6 +++++- .../control-plane/tasks/kubeadm-upgrade.yml | 4 ++++ .../kubeadm-controlplane.v1beta3.yaml.j2 | 5 +++++ roles/kubernetes/kubeadm/defaults/main.yml | 3 +++ roles/kubernetes/kubeadm/tasks/main.yml | 18 +++++++++++++++++ .../templates/kubeadm-client.conf.v1beta3.j2 | 5 +++++ .../kubespray-defaults/defaults/main/main.yml | 5 +++++ .../files/packet_debian11-calico-upgrade.yml | 3 +++ ...t_ubuntu20-calico-all-in-one-hardening.yml | 3 +++ .../packet_ubuntu20-calico-etcd-kubeadm.yml | 3 +++ 14 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/vars.md b/docs/vars.md index 1c36e6734..0e83b1831 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -281,6 +281,11 @@ node_taints: * `audit_webhook_batch_max_wait`: 1s * *kubectl_alias* - Bash alias of kubectl to interact with Kubernetes cluster much easier. +* *remove_anonymous_access* - When set to `true`, removes the `kubeadm:bootstrap-signer-clusterinfo` rolebinding created by kubeadm. + By default, kubeadm creates a rolebinding in the `kube-public` namespace which grants permissions to anonymous users. This rolebinding allows kubeadm to discover and validate cluster information during the join phase. + In a nutshell, this option removes the rolebinding after the init phase of the first control plane node and then configures kubeadm to use file discovery for the join phase of other nodes. + This option does not remove the anonymous authentication feature of the API server. + ### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index d1a640fe4..c25d495be 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -371,3 +371,6 @@ kubeadm_patches: enabled: false source_dir: "{{ inventory_dir }}/patches" dest_dir: "{{ kube_config_dir }}/patches" + +# Set to true to remove the role binding to anonymous users created by kubeadm +remove_anonymous_access: false diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index fd7047767..df92c419b 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -240,3 +240,6 @@ kubeadm_upgrade_auto_cert_renewal: true kube_apiserver_tracing: false kube_apiserver_tracing_endpoint: 0.0.0.0:4317 kube_apiserver_tracing_sampling_rate_per_million: 100 + +# Enable kubeadm file discovery if anonymous access has been removed +kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml index f3fd207c4..e10ef7fab 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml @@ -63,6 +63,26 @@ - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - not kube_external_ca_mode +- name: Get kubeconfig for join discovery process + command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" + register: kubeconfig_file_discovery + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + when: + - kubeadm_use_file_discovery + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists + +- name: Copy discovery kubeconfig + copy: + dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" + content: "{{ kubeconfig_file_discovery.stdout }}" + owner: "root" + mode: 0644 + when: + - inventory_hostname != first_kube_control_plane + - kubeadm_use_file_discovery + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists + - name: Joining control plane node to the cluster. command: >- {{ bin_dir }}/kubeadm join diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 1f4ff20a3..ceaafa06c 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -221,12 +221,16 @@ {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }} changed_when: false when: - - inventory_hostname == first_kube_control_plane + - inventory_hostname == first_kube_control_plane - kubeadm_token is defined - kubeadm_refresh_token tags: - kubeadm_token +- name: Remove binding to anonymous user + command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" + when: inventory_hostname == first_kube_control_plane and remove_anonymous_access + - name: Create kubeadm token for joining nodes with 24h expiration (default) command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create" changed_when: false diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 12ab0b934..7638a8968 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -53,6 +53,10 @@ PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" notify: Master | restart kubelet +- name: Kubeadm | Remove binding to anonymous user + command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" + when: remove_anonymous_access + - name: Kubeadm | clean kubectl cache to refresh api types file: path: "{{ item }}" diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 index c950d00b3..cd19b5c2e 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 @@ -1,6 +1,10 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: JoinConfiguration discovery: +{% if kubeadm_use_file_discovery %} + file: + kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml +{% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} @@ -9,6 +13,7 @@ discovery: {% endif %} token: {{ kubeadm_token }} unsafeSkipCAVerification: true +{% endif %} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} controlPlane: diff --git a/roles/kubernetes/kubeadm/defaults/main.yml b/roles/kubernetes/kubeadm/defaults/main.yml index 61b132e61..5047de509 100644 --- a/roles/kubernetes/kubeadm/defaults/main.yml +++ b/roles/kubernetes/kubeadm/defaults/main.yml @@ -4,6 +4,9 @@ discovery_timeout: 60s kubeadm_join_timeout: 120s +# Enable kubeadm file discovery if anonymous access has been removed +kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" + # If non-empty, will use this string as identification instead of the actual hostname kube_override_hostname: >- {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 4a65dbbc9..e8b5dceb6 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -57,6 +57,24 @@ set_fact: kubeadmConfig_api_version: v1beta3 +- name: Get kubeconfig for join discovery process + command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" + register: kubeconfig_file_discovery + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + when: kubeadm_use_file_discovery + +- name: Copy discovery kubeconfig + copy: + dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" + content: "{{ kubeconfig_file_discovery.stdout }}" + owner: "root" + mode: 0644 + when: + - not is_kube_master + - not kubelet_conf.stat.exists + - kubeadm_use_file_discovery + - name: Create kubeadm client config template: src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2" diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 index 5104ecfb9..3b3bc57de 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 @@ -2,6 +2,10 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: JoinConfiguration discovery: +{% if kubeadm_use_file_discovery %} + file: + kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml +{% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} @@ -14,6 +18,7 @@ discovery: - sha256:{{ kubeadm_ca_hash.stdout }} {% else %} unsafeSkipCAVerification: true +{% endif %} {% endif %} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index d2dbccc22..ed71d8a06 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -6,6 +6,8 @@ ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='s # selinux state preinstall_selinux_state: permissive +# Setting this value to false will fail +# For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001 kube_api_anonymous_auth: true # Default value, but will be set to true automatically if detected @@ -50,6 +52,9 @@ kubeadm_join_phases_skip_default: [] kubeadm_join_phases_skip: >- {{ kubeadm_join_phases_skip_default }} +# Set to true to remove the role binding to anonymous users created by kubeadm +remove_anonymous_access: false + # A string slice of values which specify the addresses to use for NodePorts. # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). # The default empty string slice ([]) means to use all local addresses. diff --git a/tests/files/packet_debian11-calico-upgrade.yml b/tests/files/packet_debian11-calico-upgrade.yml index 1b05714e4..94aba7b92 100644 --- a/tests/files/packet_debian11-calico-upgrade.yml +++ b/tests/files/packet_debian11-calico-upgrade.yml @@ -11,3 +11,6 @@ calico_network_backend: bird # Needed to bypass deprecation check ignore_assert_errors: true + +# Remove anonymous access to cluster +remove_anonymous_access: true diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml index 55cbd5063..c494810cf 100644 --- a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml @@ -104,3 +104,6 @@ kube_cert_group: root # kube-system namespace is exempted by default kube_pod_security_use_default: true kube_pod_security_default_enforce: restricted + +# Remove anonymous access to cluster +remove_anonymous_access: true diff --git a/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml b/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml index 99f736544..ba9d7b34b 100644 --- a/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml +++ b/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml @@ -9,3 +9,6 @@ etcd_deployment_type: kubeadm # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: False + +# Remove anonymous access to cluster +remove_anonymous_access: true