Split contiv etcd and etcd-proxy into two daemonsets

Putting contiv etcd and etcd-proxy into the same daemonset and manage
the difference by a env file is not good for scaling (adding nodes).
This commit split them into two daemonsets so that when adding nodes,
k8s could automatically starting a etcd-proxy on new nodes without need
to run related play that putting env file.

roles/network_plugin/contiv/defaults/main.yml

@@ -6,6 +6,8 @@ contiv_etcd_data_dir: "/var/lib/etcd/contiv-data"
 contiv_netmaster_port: 9999
 contiv_cni_version: 0.1.0
 
+contiv_etcd_image_repo: "{{ etcd_image_repo }}"
+contiv_etcd_image_tag: "{{ etcd_image_tag }}"
 contiv_etcd_listen_ip: "{{ ip | default(ansible_default_ipv4['address']) }}"
 contiv_etcd_listen_port: 6666
 contiv_etcd_peer_port: 6667
@@ -14,6 +16,14 @@ contiv_etcd_peer_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_peer_po
 contiv_etcd_listen_urls:
   - http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }}
   - http://127.0.0.1:{{ contiv_etcd_listen_port }}
+contiv_etcd_endpoints: |-
+  {% for host in groups['kube-master'] -%}
+    contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
+  {%- endfor %}
+contiv_etcd_name: |-
+  {% for host in groups['kube-master'] %}
+  {% if host == inventory_hostname -%}contiv_etcd{{ loop.index }}{%- endif %}
+  {% endfor %}
 
 # Parameters for Contiv api-proxy
 contiv_enable_api_proxy: true

roles/network_plugin/contiv/tasks/main.yml

@@ -21,6 +21,7 @@
   template:
     src: contiv-etcd.env.j2
     dest: "{{ contiv_etcd_conf_dir }}/contiv-etcd.env"
+  when: inventory_hostname in groups['kube-master']
 
 - set_fact:
     contiv_config_dir: "{{ contiv_config_dir }}"
@@ -38,6 +39,7 @@
       - {name: contiv-netplugin, file: contiv-netplugin-clusterrole.yml, type: clusterrole}
       - {name: contiv-netplugin, file: contiv-netplugin-serviceaccount.yml, type: serviceaccount}
       - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset}
+      - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset}
       - {name: contiv-netplugin, file: contiv-netplugin.yml, type: daemonset}
       - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset}
 

roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2

@@ -0,0 +1,31 @@
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: contiv-etcd-proxy
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: contiv-etcd-proxy
+spec:
+  selector:
+    matchLabels:
+      k8s-app: contiv-etcd-proxy
+  template:
+    metadata:
+      labels:
+        k8s-app: contiv-etcd-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - name: contiv-etcd-proxy
+          image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}
+          env:
+            - name: ETCD_LISTEN_CLIENT_URLS
+              value: http://127.0.0.1:{{ contiv_etcd_listen_port }}
+            - name: ETCD_PROXY
+              value: "on"
+            - name: ETCD_INITIAL_CLUSTER
+              value: {{ contiv_etcd_endpoints }}

roles/network_plugin/contiv/templates/contiv-etcd.env.j2

@@ -1,22 +1,8 @@
 # contiv etcd config
-{% if inventory_hostname in groups['kube-master'] %}
 export ETCD_DATA_DIR=/var/lib/etcd/contiv-data
 export ETCD_ADVERTISE_CLIENT_URLS={{ contiv_etcd_ad_urls }}
 export ETCD_INITIAL_ADVERTISE_PEER_URLS={{ contiv_etcd_peer_urls }}
 export ETCD_LISTEN_PEER_URLS={{ contiv_etcd_peer_urls }}
 export ETCD_LISTEN_CLIENT_URLS={{ contiv_etcd_listen_urls | join(",") }}
-export ETCD_NAME=
-{%- for host in groups['kube-master'] -%}
-{%- if host == inventory_hostname -%}
-contiv_etcd{{ loop.index }}
-{%- endif %}
-{%- endfor %}
-
-{% else %}
-export ETCD_LISTEN_CLIENT_URLS=http://127.0.0.1:{{ contiv_etcd_listen_port }}
-export ETCD_PROXY=on
-{% endif %}
-export ETCD_INITIAL_CLUSTER=
-{%- for host in groups['kube-master'] -%}
-contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }},
-{%- endfor -%}
+export ETCD_NAME={{ contiv_etcd_name }}
+export ETCD_INITIAL_CLUSTER={{ contiv_etcd_endpoints }}

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -19,26 +19,26 @@ spec:
     spec:
       hostNetwork: true
       hostPID: true
+      nodeSelector:
+        node-role.kubernetes.io/master: "true"
       tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
       containers:
         - name: contiv-etcd
-          image: {{ etcd_image_repo }}:{{ etcd_image_tag }}
+          image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}
           command: ["sh","-c"]
           args:
             - '. {{ contiv_etcd_conf_dir }}/contiv-etcd.env && /usr/local/bin/etcd'
           volumeMounts:
-            - name: etc-contiv-etcd
+            - name: contiv-etcd-conf-dir
               mountPath: {{ contiv_etcd_conf_dir }}
-            - name: var-lib-etcd-contiv-data
+            - name: contiv-etcd-data-dir
               mountPath: {{ contiv_etcd_data_dir }}
-          securityContext:
-            privileged: true
       volumes:
-        - name: etc-contiv-etcd
-          hostPath:
-            path: {{ contiv_etcd_conf_dir }}
-        - name: var-lib-etcd-contiv-data
+        - name: contiv-etcd-data-dir
           hostPath:
             path: {{ contiv_etcd_data_dir }}
+        - name: contiv-etcd-conf-dir
+          hostPath:
+            path: {{ contiv_etcd_conf_dir }}