Etcd cluster setup makeover

The current way to setup the etc cluster is messy and buggy.

- It checks for cluster is healthy before the cluster is even created.
- The unit files are started on handlers, not in the task, so you mess with "flush handlers".
- The join_member.yml is not used.
- etcd events cluster is not configured for kubeadm
- remove duplicate runs between running the role on etcd nodes and k8s nodes
pull/2577/head
woopstar 2018-04-01 18:58:08 +02:00 committed by Andreas Kruger
parent 92fc2df214
commit 86e3506ae6
15 changed files with 135 additions and 172 deletions

View File

@ -51,13 +51,13 @@
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
- { role: kubespray-defaults}
- { role: etcd, tags: etcd, etcd_cluster_setup: true }
- { role: etcd, tags: etcd }
- hosts: k8s-cluster:calico-rr
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
- { role: kubespray-defaults}
- { role: etcd, tags: etcd, etcd_cluster_setup: false }
- { role: etcd, tags: etcd }
- hosts: etcd:k8s-cluster:vault:calico-rr
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"

View File

@ -1,6 +1,7 @@
---
# Set to false to only do certificate management
etcd_cluster_setup: true
etcd_events_cluster_setup: false
etcd_backup_prefix: "/var/backups"
etcd_data_dir: "/var/lib/etcd"

View File

@ -10,7 +10,7 @@
- name: restart etcd-events
command: /bin/true
notify:
- etcd-events | reload systemd
- etcd | reload systemd
- reload etcd-events
- wait for etcd-events up
@ -19,9 +19,6 @@
- name: etcd | reload systemd
command: systemctl daemon-reload
- name: etcd-events | reload systemd
command: systemctl daemon-reload
- name: reload etcd
service:
name: etcd

View File

@ -1,11 +1,104 @@
---
- name: Configure | Check if etcd cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master and etcd_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if etcd-events cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master and etcd_events_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- include_tasks: refresh_config.yml
when: is_etcd_master
- name: Configure | Copy etcd.service systemd file
template:
src: "etcd-{{ etcd_deployment_type }}.service.j2"
dest: /etc/systemd/system/etcd.service
backup: yes
when: is_etcd_master and etcd_cluster_setup
- name: Configure | Copy etcd-events.service systemd file
template:
src: "etcd-events-{{ etcd_deployment_type }}.service.j2"
dest: /etc/systemd/system/etcd-events.service
backup: yes
when: is_etcd_master and etcd_events_cluster_setup
- name: Configure | reload systemd
command: systemctl daemon-reload
when: is_etcd_master
- name: Configure | Ensure etcd is running
service:
name: etcd
state: started
enabled: yes
when: is_etcd_master and etcd_cluster_setup
- name: Configure | Ensure etcd-events is running
service:
name: etcd-events
state: started
enabled: yes
when: is_etcd_master and etcd_events_cluster_setup
- name: Configure | Check if etcd cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
until: etcd_cluster_is_healthy.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
ignore_errors: false
changed_when: false
check_mode: no
when: is_etcd_master and etcd_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if etcd-events cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
until: etcd_events_cluster_is_healthy.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
ignore_errors: false
changed_when: false
check_mode: no
when: is_etcd_master and etcd_events_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if member is in etcd cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master
when: is_etcd_master and etcd_cluster_setup
tags:
- facts
environment:
@ -25,44 +118,16 @@
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- name: Configure | Copy etcd.service systemd file
template:
src: "etcd-{{ etcd_deployment_type }}.service.j2"
dest: /etc/systemd/system/etcd.service
backup: yes
when: is_etcd_master
notify: restart etcd
- name: Configure | Copy etcd-events.service systemd file
template:
src: "etcd-events-host.service.j2"
dest: /etc/systemd/system/etcd-events.service
backup: yes
when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup
notify: restart etcd-events
- name: Configure | Copy etcd-events.service systemd file
template:
src: "etcd-events-docker.service.j2"
dest: /etc/systemd/system/etcd-events.service
backup: yes
when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup
notify: restart etcd-events
- name: Configure | Join member(s) to etcd cluster one at a time
include_tasks: join_etcd_member.yml
vars:
target_node: "{{ item }}"
loop_control:
pause: 10
with_items: "{{ groups['etcd'] }}"
when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
- name: Configure | Join member(s) to etcd-events cluster one at a time
include_tasks: join_etcd-evetns_member.yml
include_tasks: join_etcd-events_member.yml
vars:
target_node: "{{ item }}"
loop_control:
pause: 10
with_items: "{{ groups['etcd'] }}"
when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0

View File

@ -15,6 +15,7 @@
owner: root
mode: 0700
run_once: yes
when: inventory_hostname == groups['etcd'][0]
delegate_to: "{{groups['etcd'][0]}}"
- name: "Gen_certs | create etcd cert dir (on {{groups['etcd'][0]}})"
@ -26,6 +27,7 @@
recurse: yes
mode: 0700
run_once: yes
when: inventory_hostname == groups['etcd'][0]
delegate_to: "{{groups['etcd'][0]}}"
- name: Gen_certs | write openssl config
@ -34,7 +36,9 @@
dest: "{{ etcd_config_dir }}/openssl.conf"
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
when:
- gen_certs|default(false)
- inventory_hostname == groups['etcd'][0]
- name: Gen_certs | copy certs generation script
copy:
@ -43,8 +47,9 @@
mode: 0700
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
when:
- gen_certs|default(false)
- inventory_hostname == groups['etcd'][0]
- name: Gen_certs | run cert generation script
command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
@ -61,7 +66,9 @@
{% endfor %}"
run_once: yes
delegate_to: "{{groups['etcd'][0]}}"
when: gen_certs|default(false)
when:
- gen_certs|default(false)
- inventory_hostname == groups['etcd'][0]
notify: set etcd_secret_changed
- set_fact:
@ -160,5 +167,5 @@
group: "{{ etcd_cert_group }}"
state: directory
owner: kube
mode: "u=rwX,g-rwx,o-rwx"
mode: "640"
recurse: yes

View File

@ -9,22 +9,22 @@
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
when: etcd_cluster_setup
- name: Install etcd launch script
template:
src: etcd.j2
dest: "{{ bin_dir }}/etcd"
owner: 'root'
mode: 0755
mode: 0750
backup: yes
notify: restart etcd
when: etcd_cluster_setup
- name: Install etcd-events launch script
template:
src: etcd-events.j2
dest: "{{ bin_dir }}/etcd-events"
owner: 'root'
mode: 0755
mode: 0750
backup: yes
when: etcd_events_cluster_setup
notify: restart etcd-events

View File

@ -10,3 +10,4 @@
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
when: etcd_cluster_setup

View File

@ -11,6 +11,7 @@
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
environment: "{{proxy_env}}"
when: etcd_cluster_setup
- name: Install | Copy etcdctl binary from rkt container
command: >-
@ -26,3 +27,4 @@
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
environment: "{{proxy_env}}"
when: etcd_cluster_setup

View File

@ -1,5 +1,5 @@
---
- name: Join Member | Add member to cluster
- name: Join Member | Add member to etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
@ -23,17 +23,6 @@
{%- endfor -%}
when: target_node == inventory_hostname
- name: Join Member | reload systemd
command: systemctl daemon-reload
when: target_node == inventory_hostname
- name: Join Member | Ensure etcd-events is running
service:
name: etcd-events
state: started
enabled: yes
when: target_node == inventory_hostname
- name: Join Member | Ensure member is in etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
register: etcd_events_member_in_cluster

View File

@ -1,5 +1,5 @@
---
- name: Join Member | Add member to cluster
- name: Join Member | Add member to etcd cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
@ -23,18 +23,7 @@
{%- endfor -%}
when: target_node == inventory_hostname
- name: Join Member | reload systemd
command: systemctl daemon-reload
when: target_node == inventory_hostname
- name: Join Member | Ensure etcd is running
service:
name: etcd
state: started
enabled: yes
when: target_node == inventory_hostname
- name: Join Member | Ensure member is in cluster
- name: Join Member | Ensure member is in etcd cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
changed_when: false

View File

@ -1,47 +0,0 @@
---
- name: Join Member | Add member to cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- include_tasks: refresh_config.yml
vars:
etcd_peer_addresses: >-
{% for host in groups['etcd'] -%}
{%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
{{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
{%- endif -%}
{%- if loop.last -%}
{{ etcd_member_name }}={{ etcd_peer_url }}
{%- endif -%}
{%- endfor -%}
when: target_node == inventory_hostname
- name: Join Member | reload systemd
command: systemctl daemon-reload
when: target_node == inventory_hostname
- name: Join Member | Ensure etcd is running
service:
name: etcd
state: started
enabled: yes
when: target_node == inventory_hostname
- name: Join Member | Ensure member is in cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
changed_when: false
check_mode: no
tags:
- facts
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"

View File

@ -6,6 +6,7 @@
- facts
- include_tasks: "gen_certs_{{ cert_management }}.yml"
when:
tags:
- etcd-secrets
@ -29,47 +30,28 @@
tags:
- upgrade
- include_tasks: set_cluster_health.yml
when: is_etcd_master and etcd_cluster_setup
- include_tasks: configure.yml
when: is_etcd_master and etcd_cluster_setup
when: is_etcd_master
- include_tasks: refresh_config.yml
when: is_etcd_master and etcd_cluster_setup
when: is_etcd_master
- name: Restart etcd if certs changed
command: /bin/true
notify: restart etcd
when: is_etcd_master and etcd_secret_changed|default(false)
- name: Restart etcd-events if certs changed
command: /bin/true
notify: restart etcd
when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
# reload-systemd
- meta: flush_handlers
- name: Ensure etcd is running
service:
name: etcd
state: started
state: restarted
enabled: yes
when: is_etcd_master and etcd_cluster_setup
when: is_etcd_master and etcd_cluster_setup and etcd_secret_changed|default(false)
- name: Ensure etcd-events is running
- name: Restart etcd-events if certs changed
service:
name: etcd-events
state: started
state: restarted
enabled: yes
when: is_etcd_master and etcd_events_cluster_setup
when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
# After etcd cluster is assembled, make sure that
# initial state of the cluster is in `existing`
# state insted of `new`.
- include_tasks: set_cluster_health.yml
when: is_etcd_master and etcd_cluster_setup
- include_tasks: refresh_config.yml
when: is_etcd_master and etcd_cluster_setup
when: is_etcd_master

View File

@ -4,7 +4,7 @@
src: etcd.env.j2
dest: /etc/etcd.env
notify: restart etcd
when: is_etcd_master
when: is_etcd_master and etcd_cluster_setup
- name: Refresh config | Create etcd-events config file
template:

View File

@ -1,26 +0,0 @@
---
- name: Configure | Check if etcd cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if etcd-events cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master and etcd_events_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"

View File

@ -38,6 +38,9 @@ apiServerExtraArgs:
apiserver-count: "{{ kube_apiserver_count }}"
{% if kube_version | version_compare('v1.9', '>=') %}
endpoint-reconciler-type: lease
{% endif %}
{% if etcd_events_cluster_setup %}
etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
{% endif %}
service-node-port-range: {{ kube_apiserver_node_port_range }}
kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"