ha etcd with calico

cluster.yml

@@ -4,13 +4,9 @@
   roles:
     - { role: download, tags: download }
 
-# etcd must be running on master(s) before going on
-- hosts: etcd
-  roles:
-    - { role: etcd, tags: etcd }
-
 - hosts: k8s-cluster
   roles:
+    - { role: etcd, tags: etcd }
     - { role: docker, tags: docker }
     - { role: dnsmasq, tags: dnsmasq }
     - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }

roles/etcd/handlers/main.yml

@@ -1,15 +1,14 @@
 ---
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart etcd2
-
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart etcd2
-  service: name=etcd2 state=restarted
+- name: restart reloaded-etcd2
+  service:
+    name: etcd2
+    state: restarted
 
-- name: Save iptables rules
-  command: service iptables save
+- name: restart etcd2
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart reloaded-etcd2

roles/etcd/tasks/configure.yml

@@ -1,11 +1,18 @@
 ---
+- name: Copy etcd2.service systemd file
+  template:
+    src: systemd-etcd2.service.j2
+    dest: /lib/systemd/system/etcd2.service
+    backup: yes
+  notify:
+    - restart etcd2
+
 - name: Create etcd2 environment vars dir
   file: path=/etc/systemd/system/etcd2.service.d state=directory
 
 - name: Write etcd2 config file
-  template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2-cluster.conf backup=yes
+  template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2.conf backup=yes
   notify:
-    - reload systemd
     - restart etcd2
 
 - name: Ensure etcd2 is running

roles/etcd/tasks/install.yml

@@ -11,15 +11,7 @@
   with_items:
     - etcdctl
     - etcd
-  notify:
-    - restart daemons
+  notify: restart etcd2
 
 - name: Create etcd2 binary symlink
   file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link
-
-- name: Copy etcd2.service systemd file
-  template:
-    src: systemd-etcd2.service.j2
-    dest: /lib/systemd/system/etcd2.service
-    backup: yes
-  notify: restart daemons

roles/etcd/templates/etcd2.j2

@@ -1,16 +1,21 @@
 # etcd2.0
+[Service]
+{% if inventory_hostname in groups['etcd'] %}
 {% set etcd = {} %}
-{% for srv in groups['kube-master'] %}
+{% for srv in groups['etcd'] %}
 {% if inventory_hostname == srv %}
 {% set _dummy = etcd.update({'name':"master"+loop.index|string}) %}
 {% endif %}
 {% endfor %}
-[Service]
 Environment="ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
 Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
-Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['kube-master'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['etcd'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
 Environment="ETCD_INITIAL_CLUSTER_STATE=new"
 Environment="ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd"
 Environment="ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"
 Environment="ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
 Environment="ETCD_NAME={{ etcd.name }}"
+{% else  %}
+Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['etcd'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+Environment="ETCD_LISTEN_CLIENT_URLS=http://127.0.0.1:23799"
+{% endif %}

roles/etcd/templates/systemd-etcd2.service.j2

@@ -6,7 +6,11 @@ Conflicts=etcd.service
 User=etcd
 Environment=ETCD_DATA_DIR=/var/lib/etcd2
 Environment=ETCD_NAME=%m
+{% if inventory_hostname in groups['etcd'] %}
 ExecStart={{ bin_dir }}/etcd2
+{% else %}
+ExecStart={{ bin_dir }}/etcd2 -proxy on
+{% endif %}
 Restart=always
 RestartSec=10s
 LimitNOFILE=40000

roles/network_plugin/tasks/calico.yml

@@ -12,7 +12,7 @@
 - name: Calico | Configure calico-node desired pool
   shell: calicoctl pool add {{ kube_pods_subnet }}
   environment:
-     ETCD_AUTHORITY: "{{ loadbalancer_address | default(groups['kube-master'][0]) }}:2379"
+     ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379"
   run_once: true
 
 - name: Calico | Write calico-node systemd init file

roles/network_plugin/templates/network-environment.j2

@@ -4,20 +4,22 @@
 CALICO_IPAM=true
 DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
 
-{% if inventory_hostname in groups['kube-node'] %}
 # The kubernetes master IP
 KUBERNETES_MASTER={{ groups['kube-master'][0] }}
 
 # Location of etcd cluster used by Calico.  By default, this uses the etcd
 # instance running on the Kubernetes Master
-ETCD_AUTHORITY={{ loadbalancer_address | default(groups['kube-master'][0]) }}:2379
+{% if inventory_hostname in groups['etcd'] %}
+ETCD_AUTHORITY="127.0.0.1:2379"
+{% else %}
+ETCD_AUTHORITY="127.0.0.1:23799"
+{% endif %}
 
 # The kubernetes-apiserver location - used by the calico plugin
 KUBE_API_ROOT=http://{{ groups['kube-master'][0] }}:{{kube_apiserver_insecure_port}}/api/v1/
 
 # Location of the calicoctl binary - used by the calico plugin
 CALICOCTL_PATH="{{ bin_dir }}/calicoctl"
-{% endif %}
 {% else %}
 FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network"
 {% endif %}