Upgrade etcd to 3.4.3 (#5998)
parent
a7ec0ed587
commit
bf8c8976dd
|
@ -116,7 +116,7 @@ Note: Upstart/SysV init based OS types are not supported.
|
|||
|
||||
- Core
|
||||
- [kubernetes](https://github.com/kubernetes/kubernetes) v1.18.5
|
||||
- [etcd](https://github.com/coreos/etcd) v3.3.12
|
||||
- [etcd](https://github.com/coreos/etcd) v3.4.3
|
||||
- [docker](https://www.docker.com/) v19.03 (see note)
|
||||
- [containerd](https://containerd.io/) v1.2.13
|
||||
- [cri-o](http://cri-o.io/) v1.17 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
|
||||
|
|
|
@ -51,7 +51,7 @@ image_arch: "{{host_architecture | default('amd64')}}"
|
|||
# Versions
|
||||
kube_version: v1.18.5
|
||||
kubeadm_version: "{{ kube_version }}"
|
||||
etcd_version: v3.3.12
|
||||
etcd_version: v3.4.3
|
||||
|
||||
# gcr and kubernetes image repo define
|
||||
gcr_image_repo: "gcr.io"
|
||||
|
@ -376,8 +376,8 @@ etcd_binary_checksums:
|
|||
# Etcd does not have arm32 builds at the moment, having some dummy value is
|
||||
# required to avoid "no attribute" error
|
||||
arm: 0
|
||||
arm64: 170b848ac1a071fe7d495d404a868a2c0090750b2944f8a260ef1c6125b2b4f4
|
||||
amd64: dc5d82df095dae0a2970e4d870b6929590689dd707ae3d33e7b86da0f7f211b6
|
||||
arm64: 01bd849ad99693600bd59db8d0e66ac64aac1e3801900665c31bd393972e3554
|
||||
amd64: 6c642b723a86941b99753dff6c00b26d3b033209b15ee33325dc8e7f4cd68f07
|
||||
cni_binary_checksums:
|
||||
arm: 28e61b5847265135dc1ca397bf94322ecce4acab5c79cc7d360ca3f6a655bdb7
|
||||
arm64: 43fbf750c5eccb10accffeeb092693c32b236fb25d919cf058c91a677822c999
|
||||
|
|
|
@ -49,7 +49,7 @@
|
|||
snapshot save {{ etcd_backup_directory }}/snapshot.db
|
||||
environment:
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}"
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
- name: Configure | Check if etcd cluster is healthy
|
||||
shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
|
||||
shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
|
||||
register: etcd_cluster_is_healthy
|
||||
failed_when: false
|
||||
changed_when: false
|
||||
|
@ -10,14 +10,14 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Check if etcd-events cluster is healthy
|
||||
shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
|
||||
shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
|
||||
register: etcd_events_cluster_is_healthy
|
||||
failed_when: false
|
||||
changed_when: false
|
||||
|
@ -27,11 +27,11 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- include_tasks: refresh_config.yml
|
||||
when: is_etcd_master
|
||||
|
@ -74,12 +74,11 @@
|
|||
when: is_etcd_master and etcd_events_cluster_setup
|
||||
|
||||
- name: Configure | Wait for etcd cluster to be healthy
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
|
||||
shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
|
||||
register: etcd_cluster_is_healthy
|
||||
until: etcd_cluster_is_healthy.rc == 0
|
||||
retries: "{{ etcd_retries }}"
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
ignore_errors: false
|
||||
changed_when: false
|
||||
check_mode: no
|
||||
run_once: yes
|
||||
|
@ -89,19 +88,18 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Wait for etcd-events cluster to be healthy
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
|
||||
shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
|
||||
register: etcd_events_cluster_is_healthy
|
||||
until: etcd_events_cluster_is_healthy.rc == 0
|
||||
retries: "{{ etcd_retries }}"
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
ignore_errors: false
|
||||
changed_when: false
|
||||
check_mode: no
|
||||
run_once: yes
|
||||
|
@ -111,14 +109,14 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Check if member is in etcd cluster
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
|
||||
register: etcd_member_in_cluster
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
|
@ -127,14 +125,14 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Check if member is in etcd-events cluster
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
|
||||
register: etcd_events_member_in_cluster
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
|
@ -143,11 +141,11 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Join member(s) to etcd cluster one at a time
|
||||
include_tasks: join_etcd_member.yml
|
||||
|
|
|
@ -1,15 +1,16 @@
|
|||
---
|
||||
- name: Join Member | Add member to etcd-events cluster
|
||||
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}"
|
||||
register: member_add_result
|
||||
until: member_add_result.rc == 0
|
||||
retries: "{{ etcd_retries }}"
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
|
||||
- include_tasks: refresh_config.yml
|
||||
vars:
|
||||
|
@ -24,17 +25,18 @@
|
|||
{%- endfor -%}
|
||||
|
||||
- name: Join Member | Ensure member is in etcd-events cluster
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_events_access_address }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_events_access_address }}"
|
||||
register: etcd_events_member_in_cluster
|
||||
changed_when: false
|
||||
check_mode: no
|
||||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
|
||||
- name: Configure | Ensure etcd-events is running
|
||||
service:
|
||||
|
|
|
@ -1,16 +1,16 @@
|
|||
---
|
||||
- name: Join Member | Add member to etcd cluster
|
||||
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_peer_url }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}"
|
||||
register: member_add_result
|
||||
until: member_add_result.rc == 0
|
||||
retries: "{{ etcd_retries }}"
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- include_tasks: refresh_config.yml
|
||||
vars:
|
||||
|
@ -25,18 +25,18 @@
|
|||
{%- endfor -%}
|
||||
|
||||
- name: Join Member | Ensure member is in etcd cluster
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
|
||||
register: etcd_member_in_cluster
|
||||
changed_when: false
|
||||
check_mode: no
|
||||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
|
||||
- name: Configure | Ensure etcd is running
|
||||
service:
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
ETCD_DATA_DIR={{ etcd_events_data_dir }}
|
||||
ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
|
||||
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
|
||||
ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
|
||||
ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
|
||||
|
||||
ETCD_METRICS={{ etcd_metrics }}
|
||||
ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
ETCD_DATA_DIR={{ etcd_data_dir }}
|
||||
ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
|
||||
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
|
||||
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
|
||||
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
|
||||
|
||||
ETCD_METRICS={{ etcd_metrics }}
|
||||
{% if etcd_metrics_port is defined %}
|
||||
|
@ -26,6 +26,8 @@ ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
|
|||
{% if etcd_log_package_levels is defined %}
|
||||
ETCD_LOG_PACKAGE_LEVELS={{ etcd_log_package_levels }}
|
||||
{% endif %}
|
||||
# Flannel need etcd v2 API
|
||||
ETCD_ENABLE_V2=true
|
||||
|
||||
# TLS settings
|
||||
ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
|
||||
|
@ -48,6 +50,6 @@ ETCD_UNSUPPORTED_ARCH={{host_architecture}}
|
|||
|
||||
# CLI settings
|
||||
ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
|
||||
ETCDCTL_CA_FILE={{ etcd_cert_dir }}/ca.pem
|
||||
ETCDCTL_KEY_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
|
||||
ETCDCTL_CERT_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem
|
||||
ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem
|
||||
ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
|
||||
ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem
|
||||
|
|
|
@ -26,10 +26,10 @@
|
|||
- {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
|
||||
- {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
|
||||
|
||||
# Flannel need etcd v2 API
|
||||
- name: Canal | Set Flannel etcd configuration
|
||||
command: |-
|
||||
{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
|
||||
set /{{ cluster_name }}/network/config \
|
||||
{{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \
|
||||
'{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
|
||||
register: output
|
||||
retries: 4
|
||||
|
@ -39,8 +39,11 @@
|
|||
changed_when: false
|
||||
run_once: true
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
|
||||
- name: Canal | Create canal node manifests
|
||||
template:
|
||||
|
|
|
@ -20,9 +20,10 @@
|
|||
when:
|
||||
- groups['broken_etcd']
|
||||
|
||||
# When there is an error, everything is printed in stderr_lines, even "is healthy" messages.
|
||||
- name: Set has_quorum fact
|
||||
set_fact:
|
||||
has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}"
|
||||
has_quorum: "{{ etcd_endpoint_health.stderr_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}"
|
||||
|
||||
- include_tasks: recover_lost_quorum.yml
|
||||
when:
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
---
|
||||
- name: Save etcd snapshot
|
||||
shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot save /tmp/snapshot.db"
|
||||
shell: "{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db"
|
||||
environment:
|
||||
- ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
- ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
- ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
- ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}"
|
||||
- ETCDCTL_API: 3
|
||||
when: etcd_snapshot is not defined
|
||||
|
||||
|
@ -22,8 +26,12 @@
|
|||
state: absent
|
||||
|
||||
- name: Restore etcd snapshot
|
||||
shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}"
|
||||
shell: "{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}"
|
||||
environment:
|
||||
- ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
- ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
- ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
- ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
- ETCDCTL_API: 3
|
||||
|
||||
- name: Remove etcd snapshot
|
||||
|
|
|
@ -2,4 +2,4 @@
|
|||
- name: Delete node
|
||||
command: "{{ bin_dir }}/kubectl delete node {{ kube_override_hostname|default(inventory_hostname) }}"
|
||||
delegate_to: "{{ groups['kube-master']|first }}"
|
||||
ignore_errors: yes
|
||||
ignore_errors: yes
|
|
@ -18,7 +18,7 @@
|
|||
- inventory_hostname in groups['etcd']
|
||||
|
||||
- name: Lookup etcd member id
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep {{ node_ip }} | cut -d: -f1"
|
||||
shell: "{{ bin_dir }}/etcdctl member list | grep {{ node_ip }} | cut -d: -f1"
|
||||
register: etcd_member_id
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
|
@ -26,32 +26,27 @@
|
|||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
delegate_to: "{{ groups['etcd']|first }}"
|
||||
when:
|
||||
- inventory_hostname in groups['etcd']
|
||||
when: inventory_hostname in groups['etcd']
|
||||
|
||||
- name: Remove etcd member from cluster
|
||||
shell: "{{ bin_dir }}/etcdctl --no-sync member remove {{ etcd_member_id.stdout }}"
|
||||
shell: "{{ bin_dir }}/etcdctl member remove {{ etcd_member_id.stdout }}"
|
||||
register: etcd_member_in_cluster
|
||||
ignore_errors: false
|
||||
retries: 6
|
||||
delay: 5
|
||||
until: etcd_member_in_cluster.rc == 0
|
||||
changed_when: false
|
||||
check_mode: no
|
||||
tags:
|
||||
- facts
|
||||
environment:
|
||||
ETCDCTL_API: 2
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem"
|
||||
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
||||
delegate_to: "{{ groups['etcd']|first }}"
|
||||
when:
|
||||
- inventory_hostname in groups['etcd']
|
||||
|
|
|
@ -30,7 +30,7 @@
|
|||
- name: errors_info
|
||||
cmd: journalctl -p err --no-pager
|
||||
- name: etcd_info
|
||||
cmd: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | default('http://127.0.0.1:2379') }} cluster-health"
|
||||
cmd: "{{ bin_dir }}/etcdctl endpoint --cluster health"
|
||||
- name: calico_info
|
||||
cmd: "{{ bin_dir }}/calicoctl node status"
|
||||
when: '{{ kube_network_plugin == "calico" }}'
|
||||
|
@ -97,8 +97,11 @@
|
|||
- /var/log/dmesg
|
||||
|
||||
environment:
|
||||
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_API: 3
|
||||
ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
||||
ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
||||
ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
|
||||
ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
|
||||
|
||||
tasks:
|
||||
- name: set etcd_access_addresses
|
||||
|
|
Loading…
Reference in New Issue