优化添加etcd脚本，并集成到easzctl

19.addetcd.yml

@@ -1,54 +1,22 @@
 # add new-etcd node, one at a time
-- hosts:
+- hosts: "{{ NODE_TO_ADD }}"
-  - new-etcd
   tasks:
   - name: add a new etcd member
-    shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member add {{ NODE_NAME }} --peer-urls=https://{{ inventory_hostname }}:2380"
+    shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member add {{ NODE_NAME }} --peer-urls=https://{{ NODE_TO_ADD }}:2380"
-    delegate_to: "{{ groups.etcd[0] }}"
+    delegate_to: "{{ groups.etcd[1] }}"
-    when: "inventory_hostname == groups['new-etcd'][0]"
 
 # start the new-etcd node
-- hosts:
+- hosts: "{{ NODE_TO_ADD }}"
-  - new-etcd
+  vars:
+    CLUSTER_STATE: existing
   roles:
-  - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes' and inventory_hostname == groups['new-etcd'][0]" } 
+  - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" }
-  - { role: prepare, when: "inventory_hostname == groups['new-etcd'][0]" }
+  - prepare
-  - { role: new-etcd, when: "inventory_hostname == groups['new-etcd'][0]" }
-
-# restart the original etcd cluster with the new configuration 
-- hosts:
   - etcd
+
+# restart the original etcd cluster with the new configuration
+- hosts: etcd
+  vars:
+    CLUSTER_STATE: existing
   roles:
-  - { role: new-etcd, when: "groups['new-etcd']|length > 0" }
+  - etcd
-
-# modify the ansible hosts file
-- hosts:
-  - new-etcd
-  tasks:
-  - name: tag new-etcd's node FINISHED=yes
-    lineinfile:
-      dest: "{{ base_dir }}/hosts"
-      state: present
-      regexp: '{{ NODE_NAME }}'
-      line: "{{ inventory_hostname }} NODE_NAME={{ NODE_NAME }} FINISHED=yes"
-    connection: local
-    when: "inventory_hostname == groups['new-etcd'][0]"
-
-  - name: cp new-etcd's node to etcd group
-    lineinfile:
-      dest: "{{ base_dir }}/hosts"
-      state: present
-      insertafter: '^\[etcd'
-      firstmatch: yes
-      line: "{{ inventory_hostname }} NODE_NAME={{ NODE_NAME }}"
-    connection: local
-    when: "inventory_hostname == groups['new-etcd'][0]"
-
-- hosts: deploy
-  tasks:
-  - name: rm new-etcd's node
-    lineinfile:
-      dest: "{{ base_dir }}/hosts"
-      state: absent
-      regexp: 'FINISHED=yes'
-    connection: local

roles/etcd/clean-etcd.yml

@@ -1,7 +1,6 @@
-# to clean 'etcd' nodes
+# WARNNING: clean 'etcd' nodes service & data
 - hosts:
   - etcd
-  - new-etcd
   tasks:
   - name: stop and disable etcd service
     service:

roles/etcd/defaults/main.yml

@@ -1,3 +1,6 @@
 # etcd 集群间通信的IP和端口, 根据etcd组成员自动生成
 TMP_NODES: "{% for h in groups['etcd'] %}{{ hostvars[h]['NODE_NAME'] }}=https://{{ h }}:2380,{% endfor %}"
 ETCD_NODES: "{{ TMP_NODES.rstrip(',') }}"
+
+# etcd 集群初始状态 new/existing
+CLUSTER_STATE: "new"

roles/etcd/templates/etcd.service.j2

@@ -22,7 +22,7 @@ ExecStart={{ bin_dir }}/etcd \
   --advertise-client-urls=https://{{ inventory_hostname }}:2379 \
   --initial-cluster-token=etcd-cluster-0 \
   --initial-cluster={{ ETCD_NODES }} \
-  --initial-cluster-state=new \
+  --initial-cluster-state={{ CLUSTER_STATE }} \
   --data-dir=/var/lib/etcd
 Restart=on-failure
 RestartSec=5

roles/lb/tasks/main.yml

@@ -2,9 +2,11 @@
 - name: 注册变量 LB_IF_TMP
   shell: "ip a|grep '{{ inventory_hostname }}/'|awk '{print $NF}'"
   register: LB_IF_TMP
+  tags: restart_lb
 
 - name: 设置变量 LB_IF
   set_fact: LB_IF="{{ LB_IF_TMP.stdout }}"
+  tags: restart_lb
 
 - name: apt更新缓存刷新
   apt: update_cache=yes cache_valid_time=72000

roles/new-etcd/defaults/main.yml

@@ -1,4 +0,0 @@
-# etcd 集群间通信的IP和端口, 根据etcd组成员自动生成
-# 新增 etcd 节点，一次只能增加一个
-TMP_NODES: "{% for h in groups['etcd'] %}{{ hostvars[h]['NODE_NAME'] }}=https://{{ h }}:2380,{% endfor %}{% if groups['new-etcd']|length > 0 %}{{ hostvars[groups['new-etcd'][0]]['NODE_NAME'] }}=https://{{ groups['new-etcd'][0] }}:2380,{% endif %}"
-ETCD_NODES: "{{ TMP_NODES.rstrip(',') }}"

roles/new-etcd/tasks/main.yml

@@ -1,62 +0,0 @@
-- name: prepare some dirs
-  file: name={{ item }} state=directory
-  with_items:
-  - "{{ bin_dir }}"
-  - "{{ ca_dir }}"
-  - "/etc/etcd/ssl"    # etcd 证书目录
-  - "/var/lib/etcd"    # etcd 工作目录
-
-- name: 下载etcd二进制文件
-  copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
-  with_items:
-  - etcd
-  - etcdctl
-  tags: upgrade_etcd
-
-- name: 分发证书相关
-  synchronize: src={{ ca_dir }}/{{ item }} dest={{ ca_dir }}/{{ item }}
-  with_items:
-  - ca.pem
-  - ca-key.pem
-  - ca.csr
-  - ca-config.json
-  delegate_to: "{{ groups.deploy[0] }}"
-
-# 注册变量p，根据p的stat信息判断是否已经生成过etcd证书，如果没有，下一步生成证书
-# 如果已经有etcd证书，为了保证整个安装的幂等性，跳过证书生成的步骤
-- name: 读取etcd证书stat信息
-  stat: path="/etc/etcd/ssl/etcd.pem"
-  register: p
-
-- name: 创建etcd证书请求
-  template: src=etcd-csr.json.j2 dest=/etc/etcd/ssl/etcd-csr.json
-  when: p.stat.isreg is not defined 
-
-- name: 创建 etcd证书和私钥
-  when: p.stat.isreg is not defined 
-  shell: "cd /etc/etcd/ssl && {{ bin_dir }}/cfssl gencert \
-        -ca={{ ca_dir }}/ca.pem \
-        -ca-key={{ ca_dir }}/ca-key.pem \
-        -config={{ ca_dir }}/ca-config.json \
-        -profile=kubernetes etcd-csr.json | {{ bin_dir }}/cfssljson -bare etcd"
-
-- name: 创建etcd的systemd unit文件
-  template: src=etcd.service.j2 dest=/etc/systemd/system/etcd.service
-  tags: upgrade_etcd
-
-- name: 开机启用etcd服务
-  shell: systemctl enable etcd
-  ignore_errors: true
-
-- name: 开启etcd服务
-  shell: systemctl daemon-reload && systemctl restart etcd
-  ignore_errors: true
-  tags: upgrade_etcd
-
-- name: 以轮询的方式等待服务同步完成
-  shell: "systemctl status etcd.service|grep Active"
-  register: etcd_status
-  until: '"running" in etcd_status.stdout'
-  retries: 8
-  delay: 8
-  tags: upgrade_etcd

roles/new-etcd/templates/etcd-csr.json.j2

@@ -1,20 +0,0 @@
-{
-  "CN": "etcd",
-  "hosts": [
-    "127.0.0.1",
-    "{{ inventory_hostname }}"
-  ],
-  "key": {
-    "algo": "rsa",
-    "size": 2048
-  },
-  "names": [
-    {
-      "C": "CN",
-      "ST": "HangZhou",
-      "L": "XS",
-      "O": "k8s",
-      "OU": "System"
-    }
-  ]
-}

roles/new-etcd/templates/etcd.service.j2

@@ -1,32 +0,0 @@
-[Unit]
-Description=Etcd Server
-After=network.target
-After=network-online.target
-Wants=network-online.target
-Documentation=https://github.com/coreos
-
-[Service]
-Type=notify
-WorkingDirectory=/var/lib/etcd/
-ExecStart={{ bin_dir }}/etcd \
-  --name={{ NODE_NAME }} \
-  --cert-file=/etc/etcd/ssl/etcd.pem \
-  --key-file=/etc/etcd/ssl/etcd-key.pem \
-  --peer-cert-file=/etc/etcd/ssl/etcd.pem \
-  --peer-key-file=/etc/etcd/ssl/etcd-key.pem \
-  --trusted-ca-file={{ ca_dir }}/ca.pem \
-  --peer-trusted-ca-file={{ ca_dir }}/ca.pem \
-  --initial-advertise-peer-urls=https://{{ inventory_hostname }}:2380 \
-  --listen-peer-urls=https://{{ inventory_hostname }}:2380 \
-  --listen-client-urls=https://{{ inventory_hostname }}:2379,http://127.0.0.1:2379 \
-  --advertise-client-urls=https://{{ inventory_hostname }}:2379 \
-  --initial-cluster-token=etcd-cluster-0 \
-  --initial-cluster={{ ETCD_NODES }} \
-  --initial-cluster-state=existing \
-  --data-dir=/var/lib/etcd
-Restart=on-failure
-RestartSec=5
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

tools/easzctl

@@ -29,17 +29,53 @@ function process_cmd() {
 
 function add-node() {
     # check new node's address regexp
-    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "Invalid ip address!"; exit 2; }
+    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; }
 
     # check if the new node already exsited
-    sed -n '/^\[kube-master/,/^\[harbor/p' $BASEPATH/hosts|grep "^$1" && { echo "$1 already existed!"; exit 2; }
+    sed -n '/^\[kube-master/,/^\[harbor/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: node $1 already existed!"; exit 2; }
 
-    # add a node in 'kube-node' group of ansible hosts
+    # add a node into 'kube-node' group
     sed -i "/\[kube-node/a $1 NEW_NODE=yes" $BASEPATH/hosts
 
-    # check if playbook success
+    # check if playbook runs successfully
     ansible-playbook $BASEPATH/20.addnode.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_NODE=yes/d" $BASEPATH/hosts; exit 2; }
 }
+
+function add-master() {
+    # check new master's address regexp
+    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; }
+
+    # check if k8s with DPLOY_MODE='multi-master'
+    grep '^DEPLOY_MODE=multi-master' $BASEPATH/hosts || { echo "ERROR: only k8s with DPLOY_MODE='multi-master' can have master node added!"; exit 2; }
+
+    # check if the new master already exsited
+    sed -n '/^\[kube-master/,/^\[kube-node/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: master $1 already existed!"; exit 2; }
+
+    # add a node into 'kube-master' group
+    sed -i "/\[kube-master/a $1 NEW_MASTER=yes" $BASEPATH/hosts
+
+    # check if playbook runs successfully
+    ansible-playbook $BASEPATH/21.addmaster.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_MASTER=yes/d" $BASEPATH/hosts; exit 2; }
+}
+
+function add-etcd() {
+    # check new node's address regexp
+    [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; }
+
+    # check if the new node already exsited
+    sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: node $1 already existed!"; exit 2; }
+
+    # input an unique NODE_NAME of the node in etcd cluster
+    echo "give an unique name(string) for the new node: "
+    read NAME
+    sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "$NAME" && { echo "ERROR: name [$NAME] already existed!"; exit 2; }
+
+    # add a node into 'kube-node' group
+    sed -i "/\[etcd/a $1 NODE_NAME=$NAME" $BASEPATH/hosts
+
+    # check if playbook runs successfully
+    ansible-playbook $BASEPATH/19.addetcd.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NODE_NAME=$NAME/d" $BASEPATH/hosts; exit 2; }
+}
 ###############################################################
 
 BASEPATH=/etc/ansible
@@ -49,13 +85,17 @@ BASEPATH=/etc/ansible
 case "$1" in
 
 (add-node)
-        ACTION="+---\033[33maction add a k8s work node\033[0m---+"
+        ACTION="+---\033[33mAction: add a k8s work node\033[0m---+"
         CMD="add-node $2" 
         ;;
 (add-master)
-        ACTION="+---\033[33maction add a k8s master node\033[0m---+"
+        ACTION="+---\033[33mAction: add a k8s master node\033[0m---+"
         CMD="add-master $2" 
         ;;
+(add-etcd)
+        ACTION="+---\033[33mAction: add a etcd node\033[0m---+"
+        CMD="add-etcd $2" 
+        ;;
 (*)
         usage
         exit 0