feat:[kube-lb] a l4 nginx proxy for apiservers

ezctl

@@ -270,14 +270,14 @@ function list() {
     which md5sum > /dev/null 2>&1 || { logger error "md5sum not found"; return 1; }
 
     CLUSTERS=$(cd clusters && echo -- *)
-    CFG_MD5=$(md5sum -t ~/.kube/config |cut -d' ' -f1)
+    CFG_MD5=$(sed '/server/d' ~/.kube/config|md5sum|cut -d' ' -f1)
     cd "$BASE"
 
     logger info "list of managed clusters:"
     i=1; for c in $CLUSTERS;
     do
         if [[ -f "clusters/$c/kubectl.kubeconfig" ]];then
-            c_md5=$(md5sum -t "clusters/$c/kubectl.kubeconfig" |cut -d' ' -f1)
+            c_md5=$(sed '/server/d' "clusters/$c/kubectl.kubeconfig"|md5sum|cut -d' ' -f1)
             if [[ "$c_md5" = "$CFG_MD5" ]];then
                 echo -e "==> cluster $i:\t$c (\033[32mcurrent\033[0m)"
             else
@@ -327,8 +327,8 @@ function add-master() {
     logger info "start to add a master node:$2 into cluster:$1"
     ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/23.addmaster.yml" -e "NODE_TO_ADD=$2" -e "@clusters/$1/config.yml"
 
-    logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
-    ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml" 
+    logger info "reconfigure and restart 'kube-lb' service"
+    ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml" 
 }
 
 function add-etcd() {
@@ -371,7 +371,7 @@ function del-etcd() {
 
 function del-node() {
     # check node's address regexp
-    [[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger "Invalid ip add:$2"; return 2; }
+    [[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger error "Invalid ip add:$2"; return 2; }
 
     # check if the deleting node exsited
     sed -n '/^\[kube_master/,/^\[harbor/p' "$BASE/clusters/$1/hosts"|grep -E "^$2$|^$2 " || { logger error "node $2 not existed in $BASE/clusters/$1/hosts"; return 2; }
@@ -393,8 +393,8 @@ function del-master() {
     logger info "reconfig kubeconfig in ansible manage node"
     ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/roles/deploy/deploy.yml" -t create_kctl_cfg -e "@clusters/$1/config.yml"
 
-    logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
-    ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml" 
+    logger info "reconfigure and restart 'kube-lb' service"
+    ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml" 
 }
 
 

ezdown

@@ -16,7 +16,7 @@ set -o errexit
 DOCKER_VER=20.10.5
 KUBEASZ_VER=3.0.1
 K8S_BIN_VER=v1.20.5
-EXT_BIN_VER=0.8.1
+EXT_BIN_VER=0.9.0
 SYS_PKG_VER=0.3.3
 HARBOR_VER=v2.1.3
 REGISTRY_MIRROR=CN

playbooks/04.kube-master.yml

@@ -1,6 +1,7 @@
 # to set up 'kube_master' nodes
 - hosts: kube_master
   roles:
+  - kube-lb
   - kube-master
   - kube-node
   tasks:

playbooks/05.kube-node.yml

@@ -1,4 +1,5 @@
 # to set up 'kube_node' nodes
 - hosts: kube_node
   roles:
-  - { role: kube-node, when: "inventory_hostname not in groups['kube_master']" } 
+  - { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
+  - { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }

playbooks/22.addnode.yml

@@ -7,6 +7,7 @@
   - prepare
   - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
   - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
+  - kube-lb
   - kube-node
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
   - { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }

playbooks/23.addmaster.yml

@@ -7,6 +7,7 @@
   - prepare
   - { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
   - { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
+  - kube-lb
   - kube-master
   - kube-node
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }

playbooks/90.setup.yml

@@ -38,6 +38,7 @@
 # to set up 'kube_master' nodes
 - hosts: kube_master
   roles:
+  - kube-lb
   - kube-master
   - kube-node
   tasks:
@@ -53,6 +54,7 @@
 # to set up 'kube_node' nodes
 - hosts: kube_node
   roles:
+  - { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
   - { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }
 
 # to install network plugin, only one can be choosen

playbooks/91.start.yml

@@ -3,6 +3,13 @@
   - name: starting etcd cluster
     service: name=etcd state=started enabled=yes
 
+- hosts:
+  - kube_master
+  - kube_node
+  tasks:
+  - name: starting kube-lb
+    service: name=kube-lb state=started enabled=yes
+
 - hosts: kube_master
   tasks:
   - name: starting kube_master services
@@ -24,12 +31,6 @@
     service: name=containerd state=started enabled=yes
     when: "CONTAINER_RUNTIME == 'containerd'"
 
-  - name: starting haproxy on kube_node
-    service: name=haproxy state=started enabled=yes
-    when: 
-    - "inventory_hostname not in groups['kube_master']"
-    - "groups['kube_master']|length > 1"
-
   - name: starting kube_node services
     service: name={{ item }} state=started enabled=yes
     with_items:

playbooks/92.stop.yml

@@ -24,15 +24,10 @@
   - kube_master
   - kube_node
   tasks:
-  - name: stopping haproxy on kube_node
-    service: name=haproxy state=stopped enabled=no
-    when: 
-    - "inventory_hostname not in groups['kube_master']"
-    - "groups['kube_master']|length > 1"
-
   - name: stopping kube_node services
     service: name={{ item }} state=stopped enabled=no
     with_items:
+    - kube-lb
     - kubelet
     - kube-proxy
 

roles/clean/tasks/clean_node.yml

@@ -3,6 +3,7 @@
   - name: stop and disable kube_node service
     service: name={{ item }} state=stopped enabled=no
     with_items:
+    - kube-lb
     - kubelet
     - kube-proxy
     ignore_errors: true
@@ -18,9 +19,11 @@
     with_items:
     - "/var/lib/kubelet/"
     - "/var/lib/kube-proxy/"
+    - "/etc/systemd/system/kube-lb.service"
     - "/etc/systemd/system/kubelet.service"
     - "/etc/systemd/system/kube-proxy.service"
     - "/opt/kube/kube-system/"
+    - "/etc/kube-lb/"
     - "/etc/kubernetes/"
     - "/root/.kube/config"
 

roles/ex-lb/tasks/main.yml

@@ -1,8 +1,3 @@
-# ex_lb 节点成员不能同时是 kube_node 节点，因为它们都需要安装 haproxy
-- name: fail info1
-  fail: msg="an 'ex_lb' node CAN NOT be a 'kube_node' node at the same time"
-  when: "inventory_hostname in groups['kube_node']"
-
 # 自动设置LB节点变量'LB_IF'
 - name: 注册变量 LB_IF_TMP
   shell: "ip a|grep '{{ inventory_hostname }}/'|awk '{print $NF}'"

roles/kube-lb/clean-kube-lb.yml

@@ -0,0 +1,16 @@
+- hosts:
+  - kube-master
+  - kube-node
+  tasks:
+  - name: stop and disable kube-lb service
+    service:
+      name: kube-lb
+      state: stopped
+      enabled: no
+    ignore_errors: true
+
+  - name: remove files and dirs
+    file: name={{ item }} state=absent
+    with_items:
+    - "/etc/kube-lb"
+    - "/etc/systemd/system/kube-lb.service"

roles/kube-lb/tasks/main.yml

@@ -0,0 +1,34 @@
+- name: prepare some dirs
+  file: name={{ item }} state=directory
+  with_items:
+  - "/etc/kube-lb/sbin"
+  - "/etc/kube-lb/logs"
+  - "/etc/kube-lb/conf"
+
+- name: 下载二进制文件kube-lb(nginx)
+  copy: src={{ base_dir }}/bin/nginx dest=/etc/kube-lb/sbin/kube-lb mode=0755
+
+- name: 创建kube-lb的配置文件
+  template: src=kube-lb.conf.j2 dest=/etc/kube-lb/conf/kube-lb.conf
+  tags: restart_kube-lb
+
+- name: 创建kube-lb的systemd unit文件
+  template: src=kube-lb.service.j2 dest=/etc/systemd/system/kube-lb.service
+  tags: restart_kube-lb
+
+- name: 开机启用kube-lb服务
+  shell: systemctl enable kube-lb
+  ignore_errors: true
+
+- name: 开启kube-lb服务
+  shell: systemctl daemon-reload && systemctl restart kube-lb
+  ignore_errors: true
+  tags: restart_kube-lb
+
+- name: 以轮询的方式等待kube-lb服务启动
+  shell: "systemctl status kube-lb.service|grep Active"
+  register: svc_status
+  until: '"running" in svc_status.stdout'
+  retries: 3
+  delay: 3
+  tags: restart_kube-lb

roles/kube-lb/templates/kube-lb.conf.j2

@@ -0,0 +1,22 @@
+user root;
+worker_processes 1;
+
+error_log  /etc/kube-lb/logs/error.log warn;
+
+events {
+    worker_connections  3000;
+}
+
+stream {
+    upstream backend {
+{% for host in groups['kube_master'] %}
+        server {{ host }}:6443    max_fails=2 fail_timeout=3s;
+{% endfor %}
+    }
+
+    server {
+        listen 127.0.0.1:6443;
+        proxy_connect_timeout 1s;
+        proxy_pass backend;
+    }
+}

roles/kube-lb/templates/kube-lb.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=l4 nginx proxy for kube-apiservers
+After=network.target
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=forking
+ExecStartPre=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -t
+ExecStart=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb
+ExecReload=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -s reload
+PrivateTmp=true
+Restart=always
+RestartSec=15
+StartLimitInterval=0
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

roles/kube-master/tasks/main.yml

@@ -57,7 +57,7 @@
   lineinfile:
     dest: "{{ item }}"
     regexp: "^    server"
-    line: "    server: https://{{ inventory_hostname }}:6443"
+    line: "    server: https://127.0.0.1:6443"
   with_items:
   - "/root/.kube/config"
   - "/etc/kubernetes/kube-controller-manager.kubeconfig"

roles/kube-node/tasks/main.yml

@@ -16,10 +16,6 @@
   - loopback
   tags: upgrade_k8s
 
-# 每个 node 节点运行 haproxy 连接到多个 apiserver
-- import_tasks: node_lb.yml
-  when: "inventory_hostname not in groups['kube_master']"
-
 - name: 替换 kubeconfig 的 apiserver 地址
   lineinfile:
     dest: /root/.kube/config

roles/kube-node/tasks/node_lb.yml

@@ -1,66 +0,0 @@
-# kube_node 节点成员不能同时是 ex_lb 节点，因为它们都需要安装 haproxy
-- name: fail info1
-  fail: msg="an 'kube_node' node CAN NOT be a 'ex_lb' node at the same time"
-  when: "inventory_hostname in groups['ex_lb']"
-  tags: restart_lb
-
-- name: 安装 haproxy
-  package: name=haproxy state=present
-  when: 'INSTALL_SOURCE != "offline"'
-
-# 离线安装 haproxy
-- import_tasks: offline.yml
-  when: 'INSTALL_SOURCE == "offline"'
-
-- name: 创建haproxy配置目录
-  file: name=/etc/haproxy state=directory
-
-- name: 修改centos的haproxy.service
-  template: src=haproxy.service.j2 dest=/usr/lib/systemd/system/haproxy.service
-  when:
-  - 'ansible_distribution in ["CentOS","RedHat","Amazon","Aliyun"]'
-  tags: restart_lb
-
-- name: 配置 haproxy
-  template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
-  tags: restart_lb
-
-- name: daemon-reload for haproxy.service
-  shell: systemctl daemon-reload
-  tags: restart_lb
-
-- name: 开机启用haproxy服务
-  shell: systemctl enable haproxy
-  ignore_errors: true
-
-- name: 停止haproxy服务
-  shell: systemctl stop haproxy
-  tags: restart_lb
-
-# 仅 master 节点数大于1时才启动haproxy
-- name: 开启haproxy服务
-  shell: systemctl start haproxy
-  when: "groups['kube_master']|length > 1"
-  tags: restart_lb
-
-# master 节点从1个增加到2个时候，需要修改如下配置
-# master 节点从2个减少到1个时候，也需要修改
-- block:
-  - name: 替换 kubeconfig 的 apiserver 地址
-    lineinfile:
-      dest: "{{ item }}"
-      regexp: "^    server"
-      line: "    server: {{ KUBE_APISERVER }}"
-    with_items:
-    - "/root/.kube/config"
-    - "/etc/kubernetes/kubelet.kubeconfig"
-    - "/etc/kubernetes/kube-proxy.kubeconfig"
-  
-  - name: restart kube_node service
-    service: name={{ item }} state=restarted
-    with_items:
-    - kubelet
-    - kube-proxy
-    ignore_errors: true
-  when: "MASTER_CHG == 'yes' and groups['kube_master']|length < 3"
-  tags: restart_lb

roles/kube-node/tasks/offline.yml

@@ -1,65 +0,0 @@
-# 离线安装 haproxy
-- name: 准备离线安装包目录
-  file: name=/opt/kube/packages/haproxy state=directory
-
-- block:
-  - name: 分发 haproxy_xenial 离线包
-    copy:
-      src: "{{ base_dir }}/down/packages/haproxy_xenial.tar.gz"
-      dest: "/opt/kube/packages/haproxy/haproxy_xenial.tar.gz"
-
-  - name: 安装 haproxy_xenial 离线包
-    shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_xenial.tar.gz && \
-           dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
-  when: ansible_distribution_release == "xenial"
-  ignore_errors: true
-
-- block:
-  - name: 分发 haproxy_bionic 离线包
-    copy:
-      src: "{{ base_dir }}/down/packages/haproxy_bionic.tar.gz"
-      dest: "/opt/kube/packages/haproxy/haproxy_bionic.tar.gz"
-
-  - name: 安装 haproxy_bionic 离线包
-    shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_bionic.tar.gz && \
-           dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
-  when: ansible_distribution_release == "bionic"
-  ignore_errors: true
-
-- block:
-  - name: 分发 haproxy_centos7 离线包
-    copy:
-      src: "{{ base_dir }}/down/packages/haproxy_centos7.tar.gz"
-      dest: "/opt/kube/packages/haproxy/haproxy_centos7.tar.gz"
-
-  - name: 安装 haproxy_centos7 离线包
-    shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_centos7.tar.gz && \
-           rpm -Uvh --force --nodeps *.rpm > /tmp/install_haproxy.log 2>&1'
-  when:
-  - 'ansible_distribution == "CentOS"'
-  - 'ansible_distribution_major_version == "7"'
-  ignore_errors: true
-
-- block:
-  - name: 分发 haproxy_stretch 离线包
-    copy:
-      src: "{{ base_dir }}/down/packages/haproxy_stretch.tar.gz"
-      dest: "/opt/kube/packages/haproxy/haproxy_stretch.tar.gz"
-
-  - name: 安装 haproxy_stretch 离线包
-    shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_stretch.tar.gz && \
-           dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
-  when: ansible_distribution_release == "stretch"
-  ignore_errors: true
-
-- block:
-  - name: 分发 haproxy_buster 离线包
-    copy:
-      src: "{{ base_dir }}/down/packages/haproxy_buster.tar.gz"
-      dest: "/opt/kube/packages/haproxy/haproxy_buster.tar.gz"
-
-  - name: 安装 haproxy_buster 离线包
-    shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_buster.tar.gz && \
-           dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
-  when: ansible_distribution_release == "buster"
-  ignore_errors: true

roles/kube-node/vars/main.yml

@@ -1,16 +1,5 @@
-# 设置 APISERVER 地址
-KUBE_APISERVER: "{%- if inventory_hostname in groups['kube_master'] -%} \
-                     https://{{ inventory_hostname }}:6443 \
-                 {%- else -%} \
-                     {%- if groups['kube_master']|length > 1 -%} \
-                         https://127.0.0.1:6443 \
-                     {%- else -%} \
-                         https://{{ groups['kube_master'][0] }}:6443 \
-                     {%- endif -%} \
-                 {%- endif -%}"
+# 设置 APISERVER 地址，使用kube-lb负载均衡监听地址
+KUBE_APISERVER: "https://127.0.0.1:6443"
 
 # node local dns cache 离线镜像
 dnscache_offline: "k8s-dns-node-cache_{{ dnsNodeCacheVer }}.tar"
-
-# 增加/删除 master 节点时，node 节点需要重新配置 haproxy
-MASTER_CHG: "no"