From 804e9a09c04b8956e1fbf731e6039c022ef55df0 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 6 Mar 2017 13:22:13 +0300 Subject: [PATCH] Migrate k8s data to etcd3 api store Default backend is now etcd3 (was etcd2). The migration process consists of the following steps: * check if migration is necessary * stop etcd on first etcd server * run migration script * start etcd on first etcd server * stop kube-apiserver until configuration is updated * update kube-apiserver * purge old etcdv2 data --- roles/kubernetes/master/defaults/main.yml | 4 +- roles/kubernetes/master/tasks/main.yml | 4 ++ .../kubernetes/master/tasks/post-upgrade.yml | 6 +++ roles/kubernetes/master/tasks/pre-upgrade.yml | 53 +++++++++++++++++-- 4 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 roles/kubernetes/master/tasks/post-upgrade.yml diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 527b168b9..659dcc847 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -13,6 +13,9 @@ kube_apiserver_node_port_range: "30000-32767" etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +# ETCD backend for k8s data +kube_apiserver_storage_backend: etcd3 + # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m @@ -29,7 +32,6 @@ kube_apiserver_memory_limit: 2000M kube_apiserver_cpu_limit: 800m kube_apiserver_memory_requests: 256M kube_apiserver_cpu_requests: 300m -kube_apiserver_storage_backend: etcd2 ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 67a64d4a6..baf3b5c7c 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -70,3 +70,7 @@ dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest" notify: Master | wait for kube-scheduler tags: kube-scheduler + +- include: post-upgrade.yml + tags: k8s-post-upgrade + diff --git a/roles/kubernetes/master/tasks/post-upgrade.yml b/roles/kubernetes/master/tasks/post-upgrade.yml new file mode 100644 index 000000000..07fc57b96 --- /dev/null +++ b/roles/kubernetes/master/tasks/post-upgrade.yml @@ -0,0 +1,6 @@ +--- +- name: "Post-upgrade | etcd3 upgrade | purge etcd2 k8s data" + command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} rm -r /registry" + environment: + ETCDCTL_API: 2 + when: kube_apiserver_storage_backend == "etcd3" and needs_etcd_migration|bool|default(false) diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 1bb0c0344..244c8b13e 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -32,19 +32,64 @@ stat: path: /etc/kubernetes/manifests/kube-apiserver.manifest register: kube_apiserver_manifest - when: secret_changed|default(false) or etcd_secret_changed|default(false) -- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed" +- name: "Pre-upgrade | etcd3 upgrade | see if old config exists" + command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions" + environment: + ETCDCTL_API: 2 + register: old_data_exists + delegate_to: "{{groups['kube-master'][0]}}" + when: kube_apiserver_storage_backend == "etcd3" + failed_when: false + +- name: "Pre-upgrade | etcd3 upgrade | see if data was already migrated" + command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} get --limit=1 --prefix=true /registry/minions" + environment: + ETCDCTL_API: 3 + register: data_migrated + delegate_to: "{{groups['etcd'][0]}}" + when: kube_apiserver_storage_backend == "etcd3" + failed_when: false + +- name: "Pre-upgrade | etcd3 upgrade | set needs_etcd_migration" + set_fact: + needs_etcd_migration: "{{ kube_apiserver_storage_backend == 'etcd3' and data_migrated.stdout_lines|length == 0 and old_data_exists.rc == 0 }}" + +- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if necessary" replace: dest: /etc/kubernetes/manifests/kube-apiserver.manifest regexp: '(\s+)image:\s+.*?$' replace: '\1image: kill.apiserver.using.fake.image.in:manifest' register: kube_apiserver_manifest_replaced - when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + when: (secret_changed|default(false) or etcd_secret_changed|default(false) or needs_etcd_migration|bool) and kube_apiserver_manifest.stat.exists - name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod" pause: seconds: 20 - when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + when: kube_apiserver_manifest_replaced.changed tags: kube-apiserver +- name: "Pre-upgrade | etcd3 upgrade | stop etcd" + service: + name: etcd + state: stopped + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + when: needs_etcd_migration|bool + +- name: "Pre-upgrade | etcd3 upgrade | migrate data" + command: "{{ bin_dir }}/etcdctl migrate --data-dir=\"{{ etcd_data_dir }}\" --wal-dir=\"{{ etcd_data_dir }}/member/wal\"" + environment: + ETCDCTL_API: 3 + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + register: etcd_migrated + when: needs_etcd_migration|bool + +- name: "Pre-upgrade | etcd3 upgrade | start etcd" + service: + name: etcd + state: started + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + when: needs_etcd_migration|bool