diff --git a/19.addetcd.yml b/19.addetcd.yml index 2b4c598..9aca35f 100644 --- a/19.addetcd.yml +++ b/19.addetcd.yml @@ -1,54 +1,22 @@ # add new-etcd node, one at a time -- hosts: - - new-etcd +- hosts: "{{ NODE_TO_ADD }}" tasks: - name: add a new etcd member - shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member add {{ NODE_NAME }} --peer-urls=https://{{ inventory_hostname }}:2380" - delegate_to: "{{ groups.etcd[0] }}" - when: "inventory_hostname == groups['new-etcd'][0]" + shell: "ETCDCTL_API=3 {{ bin_dir }}/etcdctl member add {{ NODE_NAME }} --peer-urls=https://{{ NODE_TO_ADD }}:2380" + delegate_to: "{{ groups.etcd[1] }}" # start the new-etcd node -- hosts: - - new-etcd +- hosts: "{{ NODE_TO_ADD }}" + vars: + CLUSTER_STATE: existing roles: - - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes' and inventory_hostname == groups['new-etcd'][0]" } - - { role: prepare, when: "inventory_hostname == groups['new-etcd'][0]" } - - { role: new-etcd, when: "inventory_hostname == groups['new-etcd'][0]" } - -# restart the original etcd cluster with the new configuration -- hosts: + - { role: chrony, when: "hostvars[groups.deploy[0]]['NTP_ENABLED'] == 'yes'" } + - prepare - etcd + +# restart the original etcd cluster with the new configuration +- hosts: etcd + vars: + CLUSTER_STATE: existing roles: - - { role: new-etcd, when: "groups['new-etcd']|length > 0" } - -# modify the ansible hosts file -- hosts: - - new-etcd - tasks: - - name: tag new-etcd's node FINISHED=yes - lineinfile: - dest: "{{ base_dir }}/hosts" - state: present - regexp: '{{ NODE_NAME }}' - line: "{{ inventory_hostname }} NODE_NAME={{ NODE_NAME }} FINISHED=yes" - connection: local - when: "inventory_hostname == groups['new-etcd'][0]" - - - name: cp new-etcd's node to etcd group - lineinfile: - dest: "{{ base_dir }}/hosts" - state: present - insertafter: '^\[etcd' - firstmatch: yes - line: "{{ inventory_hostname }} NODE_NAME={{ NODE_NAME }}" - connection: local - when: "inventory_hostname == groups['new-etcd'][0]" - -- hosts: deploy - tasks: - - name: rm new-etcd's node - lineinfile: - dest: "{{ base_dir }}/hosts" - state: absent - regexp: 'FINISHED=yes' - connection: local + - etcd diff --git a/roles/new-etcd/clean-etcd.yml b/roles/etcd/clean-etcd.yml similarity index 88% rename from roles/new-etcd/clean-etcd.yml rename to roles/etcd/clean-etcd.yml index 7c841f9..f0b79f9 100644 --- a/roles/new-etcd/clean-etcd.yml +++ b/roles/etcd/clean-etcd.yml @@ -1,7 +1,6 @@ -# to clean 'etcd' nodes +# WARNNING: clean 'etcd' nodes service & data - hosts: - etcd - - new-etcd tasks: - name: stop and disable etcd service service: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 65980df..d6a8d3c 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -1,3 +1,6 @@ # etcd 集群间通信的IP和端口, 根据etcd组成员自动生成 TMP_NODES: "{% for h in groups['etcd'] %}{{ hostvars[h]['NODE_NAME'] }}=https://{{ h }}:2380,{% endfor %}" ETCD_NODES: "{{ TMP_NODES.rstrip(',') }}" + +# etcd 集群初始状态 new/existing +CLUSTER_STATE: "new" diff --git a/roles/etcd/templates/etcd.service.j2 b/roles/etcd/templates/etcd.service.j2 index a79e456..46d3bcc 100644 --- a/roles/etcd/templates/etcd.service.j2 +++ b/roles/etcd/templates/etcd.service.j2 @@ -22,7 +22,7 @@ ExecStart={{ bin_dir }}/etcd \ --advertise-client-urls=https://{{ inventory_hostname }}:2379 \ --initial-cluster-token=etcd-cluster-0 \ --initial-cluster={{ ETCD_NODES }} \ - --initial-cluster-state=new \ + --initial-cluster-state={{ CLUSTER_STATE }} \ --data-dir=/var/lib/etcd Restart=on-failure RestartSec=5 diff --git a/roles/lb/tasks/main.yml b/roles/lb/tasks/main.yml index b585ff7..659e9b2 100644 --- a/roles/lb/tasks/main.yml +++ b/roles/lb/tasks/main.yml @@ -2,9 +2,11 @@ - name: 注册变量 LB_IF_TMP shell: "ip a|grep '{{ inventory_hostname }}/'|awk '{print $NF}'" register: LB_IF_TMP + tags: restart_lb - name: 设置变量 LB_IF set_fact: LB_IF="{{ LB_IF_TMP.stdout }}" + tags: restart_lb - name: apt更新缓存刷新 apt: update_cache=yes cache_valid_time=72000 diff --git a/roles/new-etcd/defaults/main.yml b/roles/new-etcd/defaults/main.yml deleted file mode 100644 index d2e1352..0000000 --- a/roles/new-etcd/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -# etcd 集群间通信的IP和端口, 根据etcd组成员自动生成 -# 新增 etcd 节点,一次只能增加一个 -TMP_NODES: "{% for h in groups['etcd'] %}{{ hostvars[h]['NODE_NAME'] }}=https://{{ h }}:2380,{% endfor %}{% if groups['new-etcd']|length > 0 %}{{ hostvars[groups['new-etcd'][0]]['NODE_NAME'] }}=https://{{ groups['new-etcd'][0] }}:2380,{% endif %}" -ETCD_NODES: "{{ TMP_NODES.rstrip(',') }}" diff --git a/roles/new-etcd/tasks/main.yml b/roles/new-etcd/tasks/main.yml deleted file mode 100644 index 7cafffe..0000000 --- a/roles/new-etcd/tasks/main.yml +++ /dev/null @@ -1,62 +0,0 @@ -- name: prepare some dirs - file: name={{ item }} state=directory - with_items: - - "{{ bin_dir }}" - - "{{ ca_dir }}" - - "/etc/etcd/ssl" # etcd 证书目录 - - "/var/lib/etcd" # etcd 工作目录 - -- name: 下载etcd二进制文件 - copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755 - with_items: - - etcd - - etcdctl - tags: upgrade_etcd - -- name: 分发证书相关 - synchronize: src={{ ca_dir }}/{{ item }} dest={{ ca_dir }}/{{ item }} - with_items: - - ca.pem - - ca-key.pem - - ca.csr - - ca-config.json - delegate_to: "{{ groups.deploy[0] }}" - -# 注册变量p,根据p的stat信息判断是否已经生成过etcd证书,如果没有,下一步生成证书 -# 如果已经有etcd证书,为了保证整个安装的幂等性,跳过证书生成的步骤 -- name: 读取etcd证书stat信息 - stat: path="/etc/etcd/ssl/etcd.pem" - register: p - -- name: 创建etcd证书请求 - template: src=etcd-csr.json.j2 dest=/etc/etcd/ssl/etcd-csr.json - when: p.stat.isreg is not defined - -- name: 创建 etcd证书和私钥 - when: p.stat.isreg is not defined - shell: "cd /etc/etcd/ssl && {{ bin_dir }}/cfssl gencert \ - -ca={{ ca_dir }}/ca.pem \ - -ca-key={{ ca_dir }}/ca-key.pem \ - -config={{ ca_dir }}/ca-config.json \ - -profile=kubernetes etcd-csr.json | {{ bin_dir }}/cfssljson -bare etcd" - -- name: 创建etcd的systemd unit文件 - template: src=etcd.service.j2 dest=/etc/systemd/system/etcd.service - tags: upgrade_etcd - -- name: 开机启用etcd服务 - shell: systemctl enable etcd - ignore_errors: true - -- name: 开启etcd服务 - shell: systemctl daemon-reload && systemctl restart etcd - ignore_errors: true - tags: upgrade_etcd - -- name: 以轮询的方式等待服务同步完成 - shell: "systemctl status etcd.service|grep Active" - register: etcd_status - until: '"running" in etcd_status.stdout' - retries: 8 - delay: 8 - tags: upgrade_etcd diff --git a/roles/new-etcd/templates/etcd-csr.json.j2 b/roles/new-etcd/templates/etcd-csr.json.j2 deleted file mode 100644 index 674a334..0000000 --- a/roles/new-etcd/templates/etcd-csr.json.j2 +++ /dev/null @@ -1,20 +0,0 @@ -{ - "CN": "etcd", - "hosts": [ - "127.0.0.1", - "{{ inventory_hostname }}" - ], - "key": { - "algo": "rsa", - "size": 2048 - }, - "names": [ - { - "C": "CN", - "ST": "HangZhou", - "L": "XS", - "O": "k8s", - "OU": "System" - } - ] -} diff --git a/roles/new-etcd/templates/etcd.service.j2 b/roles/new-etcd/templates/etcd.service.j2 deleted file mode 100644 index a38b16b..0000000 --- a/roles/new-etcd/templates/etcd.service.j2 +++ /dev/null @@ -1,32 +0,0 @@ -[Unit] -Description=Etcd Server -After=network.target -After=network-online.target -Wants=network-online.target -Documentation=https://github.com/coreos - -[Service] -Type=notify -WorkingDirectory=/var/lib/etcd/ -ExecStart={{ bin_dir }}/etcd \ - --name={{ NODE_NAME }} \ - --cert-file=/etc/etcd/ssl/etcd.pem \ - --key-file=/etc/etcd/ssl/etcd-key.pem \ - --peer-cert-file=/etc/etcd/ssl/etcd.pem \ - --peer-key-file=/etc/etcd/ssl/etcd-key.pem \ - --trusted-ca-file={{ ca_dir }}/ca.pem \ - --peer-trusted-ca-file={{ ca_dir }}/ca.pem \ - --initial-advertise-peer-urls=https://{{ inventory_hostname }}:2380 \ - --listen-peer-urls=https://{{ inventory_hostname }}:2380 \ - --listen-client-urls=https://{{ inventory_hostname }}:2379,http://127.0.0.1:2379 \ - --advertise-client-urls=https://{{ inventory_hostname }}:2379 \ - --initial-cluster-token=etcd-cluster-0 \ - --initial-cluster={{ ETCD_NODES }} \ - --initial-cluster-state=existing \ - --data-dir=/var/lib/etcd -Restart=on-failure -RestartSec=5 -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target diff --git a/tools/easzctl b/tools/easzctl index e30d240..93d96df 100755 --- a/tools/easzctl +++ b/tools/easzctl @@ -29,17 +29,53 @@ function process_cmd() { function add-node() { # check new node's address regexp - [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "Invalid ip address!"; exit 2; } + [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; } # check if the new node already exsited - sed -n '/^\[kube-master/,/^\[harbor/p' $BASEPATH/hosts|grep "^$1" && { echo "$1 already existed!"; exit 2; } + sed -n '/^\[kube-master/,/^\[harbor/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: node $1 already existed!"; exit 2; } - # add a node in 'kube-node' group of ansible hosts + # add a node into 'kube-node' group sed -i "/\[kube-node/a $1 NEW_NODE=yes" $BASEPATH/hosts - # check if playbook success + # check if playbook runs successfully ansible-playbook $BASEPATH/20.addnode.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_NODE=yes/d" $BASEPATH/hosts; exit 2; } } + +function add-master() { + # check new master's address regexp + [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; } + + # check if k8s with DPLOY_MODE='multi-master' + grep '^DEPLOY_MODE=multi-master' $BASEPATH/hosts || { echo "ERROR: only k8s with DPLOY_MODE='multi-master' can have master node added!"; exit 2; } + + # check if the new master already exsited + sed -n '/^\[kube-master/,/^\[kube-node/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: master $1 already existed!"; exit 2; } + + # add a node into 'kube-master' group + sed -i "/\[kube-master/a $1 NEW_MASTER=yes" $BASEPATH/hosts + + # check if playbook runs successfully + ansible-playbook $BASEPATH/21.addmaster.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NEW_MASTER=yes/d" $BASEPATH/hosts; exit 2; } +} + +function add-etcd() { + # check new node's address regexp + [[ $1 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { echo "ERROR: Invalid ip address!"; exit 2; } + + # check if the new node already exsited + sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "^$1" && { echo "ERROR: node $1 already existed!"; exit 2; } + + # input an unique NODE_NAME of the node in etcd cluster + echo "give an unique name(string) for the new node: " + read NAME + sed -n '/^\[etcd/,/^\[kube-master/p' $BASEPATH/hosts|grep "$NAME" && { echo "ERROR: name [$NAME] already existed!"; exit 2; } + + # add a node into 'kube-node' group + sed -i "/\[etcd/a $1 NODE_NAME=$NAME" $BASEPATH/hosts + + # check if playbook runs successfully + ansible-playbook $BASEPATH/19.addetcd.yml -e NODE_TO_ADD=$1 || { sed -i "/$1 NODE_NAME=$NAME/d" $BASEPATH/hosts; exit 2; } +} ############################################################### BASEPATH=/etc/ansible @@ -49,13 +85,17 @@ BASEPATH=/etc/ansible case "$1" in (add-node) - ACTION="+---\033[33maction add a k8s work node\033[0m---+" + ACTION="+---\033[33mAction: add a k8s work node\033[0m---+" CMD="add-node $2" ;; (add-master) - ACTION="+---\033[33maction add a k8s master node\033[0m---+" + ACTION="+---\033[33mAction: add a k8s master node\033[0m---+" CMD="add-master $2" ;; +(add-etcd) + ACTION="+---\033[33mAction: add a etcd node\033[0m---+" + CMD="add-etcd $2" + ;; (*) usage exit 0