From 26d7380c2e0ec19a0c1cd7ec5ba296b03832505a Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 10:01:45 +0200 Subject: [PATCH 1/5] Sync manifests from non-kubeadm to kubeadm deploy --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 10 ++++++++++ .../master/templates/kubeadm-config.v1alpha2.yaml.j2 | 10 ++++++++++ .../templates/manifests/kube-apiserver.manifest.j2 | 8 ++++---- .../manifests/kube-controller-manager.manifest.j2 | 2 +- .../templates/manifests/kube-scheduler.manifest.j2 | 2 +- roles/kubespray-defaults/defaults/main.yaml | 3 +++ 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index 416d23e83..adc71cd09 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -68,9 +68,17 @@ apiServerExtraArgs: {% endif %} service-node-port-range: {{ kube_apiserver_node_port_range }} kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + repair-malformed-updates: "false" +{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} oidc-issuer-url: {{ kube_oidc_url }} oidc-client-id: {{ kube_oidc_client_id }} @@ -102,6 +110,7 @@ controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -115,6 +124,7 @@ controllerManagerExtraVolumes: {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} schedulerExtraArgs: + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 4c729e9ac..411a7ac85 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -60,9 +60,17 @@ apiServerExtraArgs: {% endif %} service-node-port-range: {{ kube_apiserver_node_port_range }} kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + repair-malformed-updates: "false" +{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} oidc-issuer-url: {{ kube_oidc_url }} oidc-client-id: {{ kube_oidc_client_id }} @@ -101,6 +109,7 @@ controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -126,6 +135,7 @@ apiServerExtraVolumes: {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} schedulerExtraArgs: + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 928b16c75..e1023d088 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -33,7 +33,7 @@ spec: - --audit-log-maxage={{ audit_log_maxage }} - --audit-log-maxbackup={{ audit_log_maxbackups }} - --audit-log-maxsize={{ audit_log_maxsize }} - - --audit-policy-file={{ audit_policy_file }} + - --audit-policy-file={{ audit_policy_file }} {% endif %} - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - --etcd-servers={{ etcd_access_addresses }} @@ -58,16 +58,16 @@ spec: - --admission-control={{ kube_apiserver_admission_control | join(',') }} {% else %} {% if kube_apiserver_enable_admission_plugins|length > 0 %} - - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} + - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} {% endif %} {% if kube_apiserver_disable_admission_plugins|length > 0 %} - - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} + - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} {% endif %} {% endif %} - --service-cluster-ip-range={{ kube_service_addresses }} - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - - --profiling=false + - --profiling={{ kube_profiling }} - --repair-malformed-updates=false - --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem - --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 0557c4498..848a65a59 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -37,7 +37,7 @@ spec: - --node-monitor-grace-period={{ kube_controller_node_monitor_grace_period }} - --node-monitor-period={{ kube_controller_node_monitor_period }} - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }} - - --profiling=false + - --profiling={{ kube_profiling }} - --terminated-pod-gc-threshold=12500 - --v={{ kube_log_level }} {% if rbac_enabled %} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index fee223eec..813731fa2 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -32,7 +32,7 @@ spec: - --use-legacy-policy-config - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml {% endif %} - - --profiling=false + - --profiling={{ kube_profiling }} - --v={{ kube_log_level }} {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index b41134323..a81eb85f0 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -147,6 +147,9 @@ dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kube # Aggregator kube_api_aggregator_routing: false +# Profiling +kube_profiling: false + # Container for runtime container_manager: docker From 8d1c0c469c3d6b31a6b32bbae1eeb31e0514e885 Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 10:58:46 +0200 Subject: [PATCH 2/5] Added missing enable-aggregator-routing option --- .../kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 | 1 + .../kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index adc71cd09..2a69f8b4e 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -70,6 +70,7 @@ apiServerExtraArgs: kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" profiling: "{{ kube_profiling }}" repair-malformed-updates: "false" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 411a7ac85..9482ed083 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -62,6 +62,7 @@ apiServerExtraArgs: kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" profiling: "{{ kube_profiling }}" repair-malformed-updates: "false" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} From 8e37841a2ebbcc3c46cc845291636923d675fd48 Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:01:30 +0200 Subject: [PATCH 3/5] Add audit support to v1alpha1 of Kubeadm --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index 2a69f8b4e..d8d0a0af8 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -101,6 +101,13 @@ apiServerExtraArgs: runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} allow-privileged: "true" +{% if kubernetes_audit %} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" + audit-policy-file: {{ audit_policy_file }} +{% endif %} {% for key in kube_kubeadm_apiserver_extra_args %} {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} From 1c999b2a615be8cc04f37f7c35ad1d88f0f9781c Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:24:19 +0200 Subject: [PATCH 4/5] Move kube_kubeadm_controller_extra_args to controllerManagerExtraArgs section. It was placed in controllerManagerExtraVolumes --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 6 +++--- .../master/templates/kubeadm-config.v1alpha2.yaml.j2 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index d8d0a0af8..52f9c88f3 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -122,15 +122,15 @@ controllerManagerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} controllerManagerExtraVolumes: - name: openstackcacert hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 9482ed083..f7d7e9bc2 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -114,6 +114,9 @@ controllerManagerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} controllerManagerExtraVolumes: - name: openstackcacert @@ -132,9 +135,6 @@ apiServerExtraVolumes: writable: true {% endif %} {% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} From 940d2fdbb1763fdad97a435b14bcb72f7e980cdf Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:54:34 +0200 Subject: [PATCH 5/5] Add missing enforce-node-allocatable to kubelet for kubeadm deployments --- roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 0424efdf9..72b1e4884 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -26,6 +26,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if kubelet_authorization_mode_webhook %} --authorization-mode=Webhook \ {% endif %} +--enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \ --client-ca-file={{ kube_cert_dir }}/ca.crt \ --pod-manifest-path={{ kube_manifest_dir }} \ --cadvisor-port={{ kube_cadvisor_port }} \