From e9c891324859899718963c7ddd6acded38996df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Necatican=20Y=C4=B1ld=C4=B1r=C4=B1m?= Date: Tue, 22 Feb 2022 19:53:16 +0300 Subject: [PATCH] Add kubeadm option to etcd_deployment_type to replace the etcd_kubeadm_enabled variable (#8317) * Add kubeadm option to etcd_deployment_type to replace the etcd_kubeadm_enabled variable Signed-off-by: necatican * Add etcd kubeadm deployment documentation Signed-off-by: necatican * Refactor warning for the deprecated 'etcd_kubeadm_enabled' variable Signed-off-by: necatican --- cluster.yml | 4 +-- docs/cri-o.md | 8 +---- docs/etcd.md | 16 +++++++++ docs/upgrades.md | 12 +++++++ inventory/sample/group_vars/all/all.yml | 6 ---- inventory/sample/group_vars/all/etcd.yml | 16 +++++++++ inventory/sample/group_vars/etcd.yml | 11 ------ .../download/templates/kubeadm-images.yaml.j2 | 2 +- roles/etcdctl/tasks/main.yml | 2 +- .../control-plane/defaults/main/main.yml | 3 -- .../control-plane/tasks/kubeadm-etcd.yml | 2 +- .../tasks/kubeadm-fix-apiserver.yml | 2 +- .../control-plane/tasks/kubeadm-upgrade.yml | 4 +-- roles/kubernetes/control-plane/tasks/main.yml | 2 +- .../templates/kubeadm-config.v1beta2.yaml.j2 | 4 +-- roles/kubernetes/kubeadm/defaults/main.yml | 3 -- roles/kubernetes/kubeadm/tasks/main.yml | 2 +- .../preinstall/tasks/0020-verify-settings.yml | 36 ++++++++++++++----- .../preinstall/tasks/0040-set_facts.yml | 2 +- roles/kubespray-defaults/defaults/main.yaml | 5 +-- roles/kubespray-defaults/tasks/main.yaml | 9 +++++ scale.yml | 2 +- tests/files/packet_ubuntu16-flannel-ha.yml | 2 +- upgrade-cluster.yml | 4 +-- 24 files changed, 99 insertions(+), 60 deletions(-) create mode 100644 inventory/sample/group_vars/all/etcd.yml diff --git a/cluster.yml b/cluster.yml index 35c6fdbea..e13575e9c 100644 --- a/cluster.yml +++ b/cluster.yml @@ -46,7 +46,7 @@ vars: etcd_cluster_setup: true etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" - when: not etcd_kubeadm_enabled| default(false) + when: etcd_deployment_type != "kubeadm" - hosts: k8s_cluster gather_facts: False @@ -59,7 +59,7 @@ vars: etcd_cluster_setup: false etcd_events_cluster_setup: false - when: not etcd_kubeadm_enabled| default(false) + when: etcd_deployment_type != "kubeadm" - hosts: k8s_cluster gather_facts: False diff --git a/docs/cri-o.md b/docs/cri-o.md index 61a795f1d..43be723e1 100644 --- a/docs/cri-o.md +++ b/docs/cri-o.md @@ -13,7 +13,7 @@ _To use the CRI-O container runtime set the following variables:_ ```yaml download_container: false skip_downloads: false -etcd_kubeadm_enabled: true +etcd_deployment_type: host # optionally kubeadm ``` ## k8s_cluster/k8s_cluster.yml @@ -22,12 +22,6 @@ etcd_kubeadm_enabled: true container_manager: crio ``` -## etcd.yml - -```yaml -etcd_deployment_type: host # optionally and mutually exclusive with etcd_kubeadm_enabled -``` - ## all/crio.yml Enable docker hub registry mirrors diff --git a/docs/etcd.md b/docs/etcd.md index 2d42ffb10..17aa291f5 100644 --- a/docs/etcd.md +++ b/docs/etcd.md @@ -1,5 +1,21 @@ # etcd +## Deployment Types + +It is possible to deploy etcd with three methods. To change the default deployment method (host), use the `etcd_deployment_type` variable. Possible values are `host`, `kubeadm`, and `docker`. + +### Host + +Host deployment is the default method. Using this method will result in etcd installed as a systemd service. + +### Docker + +Installs docker in etcd group members and runs etcd on docker containers. Only usable when `container_manager` is set to `docker`. + +### Kubeadm + +This deployment method is experimental and is only available for new deployments. This deploys etcd as a static pod in master hosts. + ## Metrics To expose metrics on a separate HTTP port, define it in the inventory with: diff --git a/docs/upgrades.md b/docs/upgrades.md index 09a11d258..f494112b4 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -308,6 +308,18 @@ caprica Ready master,node 7h40m v1.14.1 ``` +## Upgrading to v2.19 + +`etcd_kubeadm_enabled` is being deprecated at v2.19. The same functionality is achievable by setting `etcd_deployment_type` to `kubeadm`. +Deploying etcd using kubeadm is experimental and is only available for either new or deployments where `etcd_kubeadm_enabled` was set to `true` while deploying the cluster. + +From 2.19 and onward `etcd_deployment_type` variable will be placed in `group_vars/all/etcd.yml` instead of `group_vars/etcd.yml`, due to scope issues. +The placement of the variable is only important for `etcd_deployment_type: kubeadm` right now. However, since this might change in future updates, it is recommended to move the variable. + +Upgrading is straightforward; no changes are required if `etcd_kubeadm_enabled` was not set to `true` when deploying. + +If you have a cluster where `etcd` was deployed using `kubeadm`, you will need to remove `etcd_kubeadm_enabled` the variable. Then move `etcd_deployment_type` variable from `group_vars/etcd.yml` to `group_vars/all/etcd.yml` due to scope issues and set `etcd_deployment_type` to `kubeadm`. + ## Upgrade order As mentioned above, components are upgraded in the order in which they were diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index b4b8212e0..d3c27ac35 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -1,10 +1,4 @@ --- -## Directory where etcd data stored -etcd_data_dir: /var/lib/etcd - -## Experimental kubeadm etcd deployment mode. Available only for new deployment -etcd_kubeadm_enabled: false - ## Directory where the binaries will be installed bin_dir: /usr/local/bin diff --git a/inventory/sample/group_vars/all/etcd.yml b/inventory/sample/group_vars/all/etcd.yml new file mode 100644 index 000000000..7206a06b5 --- /dev/null +++ b/inventory/sample/group_vars/all/etcd.yml @@ -0,0 +1,16 @@ +--- +## Directory where etcd data stored +etcd_data_dir: /var/lib/etcd + +## Container runtime +## docker for docker, crio for cri-o and containerd for containerd. +## Additionally you can set this to kubeadm if you want to install etcd using kubeadm +## Kubeadm etcd deployment is experimental and only available for new deployments +## If this is not set, container manager will be inherited from the Kubespray defaults +## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want. +## Also this makes possible to use different container manager for etcd nodes. +# container_manager: containerd + +## Settings for etcd deployment type +# Set this to docker if you are using container_manager: docker +etcd_deployment_type: host \ No newline at end of file diff --git a/inventory/sample/group_vars/etcd.yml b/inventory/sample/group_vars/etcd.yml index e7070a08a..cc09942fc 100644 --- a/inventory/sample/group_vars/etcd.yml +++ b/inventory/sample/group_vars/etcd.yml @@ -17,14 +17,3 @@ ### ETCD: disable peer client cert authentication. # This affects ETCD_PEER_CLIENT_CERT_AUTH variable # etcd_peer_client_auth: true - -## Container runtime -## docker for docker, crio for cri-o and containerd for containerd. -## If this is not set, container manager will be inherited from the Kubespray defaults -## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want. -## Also this makes possible to use different container manager for etcd nodes. -# container_manager: containerd - -## Settings for etcd deployment type -# Set this to docker if you are using container_manager: docker -etcd_deployment_type: host diff --git a/roles/download/templates/kubeadm-images.yaml.j2 b/roles/download/templates/kubeadm-images.yaml.j2 index 7cca0e058..565533087 100644 --- a/roles/download/templates/kubeadm-images.yaml.j2 +++ b/roles/download/templates/kubeadm-images.yaml.j2 @@ -8,7 +8,7 @@ kind: ClusterConfiguration imageRepository: {{ kube_image_repo }} kubernetesVersion: {{ kube_version }} etcd: -{% if etcd_kubeadm_enabled %} +{% if etcd_deployment_type == "kubeadm" %} local: imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" imageTag: "{{ etcd_image_tag }}" diff --git a/roles/etcdctl/tasks/main.yml b/roles/etcdctl/tasks/main.yml index a37603ade..fca078c4e 100644 --- a/roles/etcdctl/tasks/main.yml +++ b/roles/etcdctl/tasks/main.yml @@ -4,7 +4,7 @@ - name: Check unintentional include of this role assert: - that: etcd_kubeadm_enabled + that: etcd_deployment_type == "kubeadm" - name: Check if etcdctl exist stat: diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index 38b3c1e3e..65ba43a46 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -2,9 +2,6 @@ # disable upgrade cluster upgrade_cluster_setup: false -# Experimental kubeadm etcd deployment mode. Available only for new deployment -etcd_kubeadm_enabled: false - # change to 0.0.0.0 to enable insecure access from anywhere (not recommended) kube_apiserver_insecure_bind_address: 127.0.0.1 diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml b/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml index ebba677cc..8c0c47bb7 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml @@ -15,4 +15,4 @@ - name: Ensure etcdctl script is installed import_role: name: etcdctl - when: etcd_kubeadm_enabled + when: etcd_deployment_type == "kubeadm" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml index 038a193c9..8f2f38e26 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml @@ -21,4 +21,4 @@ dest: "{{ kube_config_dir }}/manifests/kube-apiserver.yaml" regexp: '^ - --etcd-servers=' line: ' - --etcd-servers={{ etcd_access_addresses }}' - when: not etcd_kubeadm_enabled | default(false) + when: etcd_deployment_type != "kubeadm" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 769ff3107..8459362cd 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -18,7 +18,7 @@ --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all --allow-experimental-upgrades - --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }} + --etcd-upgrade={{ etcd_deployment_type == "kubeadm" | bool | lower }} --force register: kubeadm_upgrade # Retry is because upload config sometimes fails @@ -39,7 +39,7 @@ --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all --allow-experimental-upgrades - --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }} + --etcd-upgrade={{ etcd_deployment_type == "kubeadm" | bool | lower }} --force register: kubeadm_upgrade when: inventory_hostname != first_kube_control_plane diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml index 7fa3d1b8d..bd8029af2 100644 --- a/roles/kubernetes/control-plane/tasks/main.yml +++ b/roles/kubernetes/control-plane/tasks/main.yml @@ -69,7 +69,7 @@ - name: Include kubeadm etcd extra tasks include_tasks: kubeadm-etcd.yml - when: etcd_kubeadm_enabled + when: etcd_deployment_type == "kubeadm" - name: Include kubeadm secondary server apiserver fixes include_tasks: kubeadm-fix-apiserver.yml diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 index ba14a9586..c329fcabe 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 @@ -33,7 +33,7 @@ apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration clusterName: {{ cluster_name }} etcd: -{% if not etcd_kubeadm_enabled %} +{% if etcd_deployment_type != "kubeadm" %} external: endpoints: {% for endpoint in etcd_access_addresses.split(',') %} @@ -42,7 +42,7 @@ etcd: caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} -{% elif etcd_kubeadm_enabled %} +{% elif etcd_deployment_type == "kubeadm" %} local: imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" imageTag: "{{ etcd_image_tag }}" diff --git a/roles/kubernetes/kubeadm/defaults/main.yml b/roles/kubernetes/kubeadm/defaults/main.yml index b6ff3fc7f..0449b8ae7 100644 --- a/roles/kubernetes/kubeadm/defaults/main.yml +++ b/roles/kubernetes/kubeadm/defaults/main.yml @@ -10,6 +10,3 @@ kube_override_hostname: >- {%- else -%} {{ inventory_hostname }} {%- endif -%} - -# Experimental kubeadm etcd deployment mode. Available only for new deployment -etcd_kubeadm_enabled: false diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index c8dac29b6..600e0dc7b 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -153,7 +153,7 @@ - name: Extract etcd certs from control plane if using etcd kubeadm mode include_tasks: kubeadm_etcd_node.yml when: - - etcd_kubeadm_enabled + - etcd_deployment_type == "kubeadm" - inventory_hostname not in groups['kube_control_plane'] - kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool - kube_network_plugin != "calico" or calico_datastore == "etcd" diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 1d68f7e0b..cf008e3f0 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -15,7 +15,7 @@ run_once: true when: - not ignore_assert_errors - - not etcd_kubeadm_enabled + - etcd_deployment_type != "kubeadm" - name: Stop if non systemd OS type assert: @@ -277,23 +277,41 @@ when: resolvconf_mode is defined run_once: true -- name: Stop if etcd deployment type is not host or docker +- name: Stop if etcd deployment type is not host, docker or kubeadm assert: - that: etcd_deployment_type in ['host', 'docker'] - msg: "The etcd deployment type, 'etcd_deployment_type', must be host or docker" + that: etcd_deployment_type in ['host', 'docker', 'kubeadm'] + msg: "The etcd deployment type, 'etcd_deployment_type', must be host, docker or kubeadm" when: - inventory_hostname in groups.get('etcd',[]) - - not etcd_kubeadm_enabled -- name: Stop if etcd deployment type is not host when container_manager != docker +- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker assert: - that: etcd_deployment_type == 'host' - msg: "The etcd deployment type, 'etcd_deployment_type', must be host when container_manager is not docker" + that: etcd_deployment_type in ['host', 'kubeadm'] + msg: "The etcd deployment type, 'etcd_deployment_type', must be host or kubeadm when container_manager is not docker" when: - inventory_hostname in groups.get('etcd',[]) - - not etcd_kubeadm_enabled - container_manager != 'docker' +# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled` +- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker and etcd_kubeadm_enabled is not defined + block: + - name: Warn the user if they are still using `etcd_kubeadm_enabled` + debug: + msg: > + "WARNING! => `etcd_kubeadm_enabled` is deprecated and will be removed in a future release. + You can set `etcd_deployment_type` to `kubeadm` instead of setting `etcd_kubeadm_enabled` to `true`." + changed_when: true + + - name: Stop if `etcd_kubeadm_enabled` is defined and `etcd_deployment_type` is not `kubadm` or `host` + assert: + that: etcd_deployment_type == 'kubeadm' + msg: > + It is not possible to use `etcd_kubeadm_enabled` when `etcd_deployment_type` is set to {{ etcd_deployment_type }}. + Unset the `etcd_kubeadm_enabled` variable and set `etcd_deployment_type` to desired deployment type (`host`, `kubeadm`, `docker`) instead." + when: etcd_kubeadm_enabled + run_once: yes + when: etcd_kubeadm_enabled is defined + - name: Stop if download_localhost is enabled but download_run_once is not assert: that: download_run_once diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml index 1cfd47777..fce7c485c 100644 --- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml @@ -205,7 +205,7 @@ kube_etcd_cert_file: "apiserver-etcd-client.crt" kube_etcd_key_file: "apiserver-etcd-client.key" when: - - etcd_kubeadm_enabled + - etcd_deployment_type == "kubeadm" - name: check /usr readonly stat: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index f609beba7..7cf1bd7b1 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -258,7 +258,7 @@ kubelet_shutdown_grace_period: 60s kubelet_shutdown_grace_period_critical_pods: 20s # Whether to deploy the container engine -deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type != 'host' }}" +deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type == 'docker' }}" # Container for runtime container_manager: containerd @@ -344,9 +344,6 @@ docker_registry_mirrors: [] ## Empty by default so no plugins will be installed. docker_plugins: [] -# Experimental kubeadm etcd deployment mode. Available only for new deployment -etcd_kubeadm_enabled: false - # Containerd options - thse are relevant when container_manager == 'containerd' containerd_use_systemd_cgroup: true diff --git a/roles/kubespray-defaults/tasks/main.yaml b/roles/kubespray-defaults/tasks/main.yaml index fe268e953..648a4af6e 100644 --- a/roles/kubespray-defaults/tasks/main.yaml +++ b/roles/kubespray-defaults/tasks/main.yaml @@ -22,3 +22,12 @@ - no_proxy is not defined tags: - always + +# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled` +- name: Set `etcd_deployment_type` to "kubeadm" if `etcd_kubeadm_enabled` is true + set_fact: + etcd_deployment_type: kubeadm + when: + - etcd_kubeadm_enabled is defined and etcd_kubeadm_enabled + tags: + - always diff --git a/scale.yml b/scale.yml index 97185d96e..de1d9b5fe 100644 --- a/scale.yml +++ b/scale.yml @@ -55,7 +55,7 @@ - { role: kubernetes/preinstall, tags: preinstall } - { role: container-engine, tags: "container-engine", when: deploy_container_engine } - { role: download, tags: download, when: "not skip_downloads" } - - { role: etcd, tags: etcd, etcd_cluster_setup: false, when: "not etcd_kubeadm_enabled|default(false)" } + - { role: etcd, tags: etcd, etcd_cluster_setup: false, when: "etcd_deployment_type != 'kubeadm'" } - name: Target only workers to get kubelet installed and checking in on any new nodes(node) hosts: kube_node diff --git a/tests/files/packet_ubuntu16-flannel-ha.yml b/tests/files/packet_ubuntu16-flannel-ha.yml index 6b683343f..8df48e35a 100644 --- a/tests/files/packet_ubuntu16-flannel-ha.yml +++ b/tests/files/packet_ubuntu16-flannel-ha.yml @@ -5,6 +5,6 @@ mode: ha # Kubespray settings kube_network_plugin: flannel -etcd_kubeadm_enabled: true +etcd_deployment_type: kubeadm kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085 skip_non_kubeadm_warning: true diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 8455082bd..010d64271 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -70,7 +70,7 @@ vars: etcd_cluster_setup: true etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" - when: not etcd_kubeadm_enabled | default(false) + when: etcd_deployment_type != "kubeadm" - hosts: k8s_cluster gather_facts: False @@ -83,7 +83,7 @@ vars: etcd_cluster_setup: false etcd_events_cluster_setup: false - when: not etcd_kubeadm_enabled | default(false) + when: etcd_deployment_type != "kubeadm" - name: Handle upgrades to master components first to maintain backwards compat. gather_facts: False