feat:[kube-lb] a l4 nginx proxy for apiservers

pull/999/head
gjmzj 2021-04-01 19:18:59 +08:00
parent 10a900f8c3
commit e13fe39eda
20 changed files with 120 additions and 175 deletions

14
ezctl
View File

@ -270,14 +270,14 @@ function list() {
which md5sum > /dev/null 2>&1 || { logger error "md5sum not found"; return 1; }
CLUSTERS=$(cd clusters && echo -- *)
CFG_MD5=$(md5sum -t ~/.kube/config |cut -d' ' -f1)
CFG_MD5=$(sed '/server/d' ~/.kube/config|md5sum|cut -d' ' -f1)
cd "$BASE"
logger info "list of managed clusters:"
i=1; for c in $CLUSTERS;
do
if [[ -f "clusters/$c/kubectl.kubeconfig" ]];then
c_md5=$(md5sum -t "clusters/$c/kubectl.kubeconfig" |cut -d' ' -f1)
c_md5=$(sed '/server/d' "clusters/$c/kubectl.kubeconfig"|md5sum|cut -d' ' -f1)
if [[ "$c_md5" = "$CFG_MD5" ]];then
echo -e "==> cluster $i:\t$c (\033[32mcurrent\033[0m)"
else
@ -327,8 +327,8 @@ function add-master() {
logger info "start to add a master node:$2 into cluster:$1"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/23.addmaster.yml" -e "NODE_TO_ADD=$2" -e "@clusters/$1/config.yml"
logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml"
logger info "reconfigure and restart 'kube-lb' service"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml"
}
function add-etcd() {
@ -371,7 +371,7 @@ function del-etcd() {
function del-node() {
# check node's address regexp
[[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger "Invalid ip add:$2"; return 2; }
[[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger error "Invalid ip add:$2"; return 2; }
# check if the deleting node exsited
sed -n '/^\[kube_master/,/^\[harbor/p' "$BASE/clusters/$1/hosts"|grep -E "^$2$|^$2 " || { logger error "node $2 not existed in $BASE/clusters/$1/hosts"; return 2; }
@ -393,8 +393,8 @@ function del-master() {
logger info "reconfig kubeconfig in ansible manage node"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/roles/deploy/deploy.yml" -t create_kctl_cfg -e "@clusters/$1/config.yml"
logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml"
logger info "reconfigure and restart 'kube-lb' service"
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml"
}

2
ezdown
View File

@ -16,7 +16,7 @@ set -o errexit
DOCKER_VER=20.10.5
KUBEASZ_VER=3.0.1
K8S_BIN_VER=v1.20.5
EXT_BIN_VER=0.8.1
EXT_BIN_VER=0.9.0
SYS_PKG_VER=0.3.3
HARBOR_VER=v2.1.3
REGISTRY_MIRROR=CN

View File

@ -1,6 +1,7 @@
# to set up 'kube_master' nodes
- hosts: kube_master
roles:
- kube-lb
- kube-master
- kube-node
tasks:

View File

@ -1,4 +1,5 @@
# to set up 'kube_node' nodes
- hosts: kube_node
roles:
- { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }
- { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
- { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }

View File

@ -7,6 +7,7 @@
- prepare
- { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
- { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
- kube-lb
- kube-node
- { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
- { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }

View File

@ -7,6 +7,7 @@
- prepare
- { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
- { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
- kube-lb
- kube-master
- kube-node
- { role: calico, when: "CLUSTER_NETWORK == 'calico'" }

View File

@ -38,6 +38,7 @@
# to set up 'kube_master' nodes
- hosts: kube_master
roles:
- kube-lb
- kube-master
- kube-node
tasks:
@ -53,6 +54,7 @@
# to set up 'kube_node' nodes
- hosts: kube_node
roles:
- { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
- { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }
# to install network plugin, only one can be choosen

View File

@ -3,6 +3,13 @@
- name: starting etcd cluster
service: name=etcd state=started enabled=yes
- hosts:
- kube_master
- kube_node
tasks:
- name: starting kube-lb
service: name=kube-lb state=started enabled=yes
- hosts: kube_master
tasks:
- name: starting kube_master services
@ -24,12 +31,6 @@
service: name=containerd state=started enabled=yes
when: "CONTAINER_RUNTIME == 'containerd'"
- name: starting haproxy on kube_node
service: name=haproxy state=started enabled=yes
when:
- "inventory_hostname not in groups['kube_master']"
- "groups['kube_master']|length > 1"
- name: starting kube_node services
service: name={{ item }} state=started enabled=yes
with_items:

View File

@ -24,15 +24,10 @@
- kube_master
- kube_node
tasks:
- name: stopping haproxy on kube_node
service: name=haproxy state=stopped enabled=no
when:
- "inventory_hostname not in groups['kube_master']"
- "groups['kube_master']|length > 1"
- name: stopping kube_node services
service: name={{ item }} state=stopped enabled=no
with_items:
- kube-lb
- kubelet
- kube-proxy

View File

@ -3,6 +3,7 @@
- name: stop and disable kube_node service
service: name={{ item }} state=stopped enabled=no
with_items:
- kube-lb
- kubelet
- kube-proxy
ignore_errors: true
@ -18,9 +19,11 @@
with_items:
- "/var/lib/kubelet/"
- "/var/lib/kube-proxy/"
- "/etc/systemd/system/kube-lb.service"
- "/etc/systemd/system/kubelet.service"
- "/etc/systemd/system/kube-proxy.service"
- "/opt/kube/kube-system/"
- "/etc/kube-lb/"
- "/etc/kubernetes/"
- "/root/.kube/config"

View File

@ -1,8 +1,3 @@
# ex_lb 节点成员不能同时是 kube_node 节点,因为它们都需要安装 haproxy
- name: fail info1
fail: msg="an 'ex_lb' node CAN NOT be a 'kube_node' node at the same time"
when: "inventory_hostname in groups['kube_node']"
# 自动设置LB节点变量'LB_IF'
- name: 注册变量 LB_IF_TMP
shell: "ip a|grep '{{ inventory_hostname }}/'|awk '{print $NF}'"

View File

@ -0,0 +1,16 @@
- hosts:
- kube-master
- kube-node
tasks:
- name: stop and disable kube-lb service
service:
name: kube-lb
state: stopped
enabled: no
ignore_errors: true
- name: remove files and dirs
file: name={{ item }} state=absent
with_items:
- "/etc/kube-lb"
- "/etc/systemd/system/kube-lb.service"

View File

@ -0,0 +1,34 @@
- name: prepare some dirs
file: name={{ item }} state=directory
with_items:
- "/etc/kube-lb/sbin"
- "/etc/kube-lb/logs"
- "/etc/kube-lb/conf"
- name: 下载二进制文件kube-lb(nginx)
copy: src={{ base_dir }}/bin/nginx dest=/etc/kube-lb/sbin/kube-lb mode=0755
- name: 创建kube-lb的配置文件
template: src=kube-lb.conf.j2 dest=/etc/kube-lb/conf/kube-lb.conf
tags: restart_kube-lb
- name: 创建kube-lb的systemd unit文件
template: src=kube-lb.service.j2 dest=/etc/systemd/system/kube-lb.service
tags: restart_kube-lb
- name: 开机启用kube-lb服务
shell: systemctl enable kube-lb
ignore_errors: true
- name: 开启kube-lb服务
shell: systemctl daemon-reload && systemctl restart kube-lb
ignore_errors: true
tags: restart_kube-lb
- name: 以轮询的方式等待kube-lb服务启动
shell: "systemctl status kube-lb.service|grep Active"
register: svc_status
until: '"running" in svc_status.stdout'
retries: 3
delay: 3
tags: restart_kube-lb

View File

@ -0,0 +1,22 @@
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
{% for host in groups['kube_master'] %}
server {{ host }}:6443 max_fails=2 fail_timeout=3s;
{% endfor %}
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}

View File

@ -0,0 +1,19 @@
[Unit]
Description=l4 nginx proxy for kube-apiservers
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=forking
ExecStartPre=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -t
ExecStart=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb
ExecReload=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -s reload
PrivateTmp=true
Restart=always
RestartSec=15
StartLimitInterval=0
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

View File

@ -57,7 +57,7 @@
lineinfile:
dest: "{{ item }}"
regexp: "^ server"
line: " server: https://{{ inventory_hostname }}:6443"
line: " server: https://127.0.0.1:6443"
with_items:
- "/root/.kube/config"
- "/etc/kubernetes/kube-controller-manager.kubeconfig"

View File

@ -16,10 +16,6 @@
- loopback
tags: upgrade_k8s
# 每个 node 节点运行 haproxy 连接到多个 apiserver
- import_tasks: node_lb.yml
when: "inventory_hostname not in groups['kube_master']"
- name: 替换 kubeconfig 的 apiserver 地址
lineinfile:
dest: /root/.kube/config

View File

@ -1,66 +0,0 @@
# kube_node 节点成员不能同时是 ex_lb 节点,因为它们都需要安装 haproxy
- name: fail info1
fail: msg="an 'kube_node' node CAN NOT be a 'ex_lb' node at the same time"
when: "inventory_hostname in groups['ex_lb']"
tags: restart_lb
- name: 安装 haproxy
package: name=haproxy state=present
when: 'INSTALL_SOURCE != "offline"'
# 离线安装 haproxy
- import_tasks: offline.yml
when: 'INSTALL_SOURCE == "offline"'
- name: 创建haproxy配置目录
file: name=/etc/haproxy state=directory
- name: 修改centos的haproxy.service
template: src=haproxy.service.j2 dest=/usr/lib/systemd/system/haproxy.service
when:
- 'ansible_distribution in ["CentOS","RedHat","Amazon","Aliyun"]'
tags: restart_lb
- name: 配置 haproxy
template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
tags: restart_lb
- name: daemon-reload for haproxy.service
shell: systemctl daemon-reload
tags: restart_lb
- name: 开机启用haproxy服务
shell: systemctl enable haproxy
ignore_errors: true
- name: 停止haproxy服务
shell: systemctl stop haproxy
tags: restart_lb
# 仅 master 节点数大于1时才启动haproxy
- name: 开启haproxy服务
shell: systemctl start haproxy
when: "groups['kube_master']|length > 1"
tags: restart_lb
# master 节点从1个增加到2个时候需要修改如下配置
# master 节点从2个减少到1个时候也需要修改
- block:
- name: 替换 kubeconfig 的 apiserver 地址
lineinfile:
dest: "{{ item }}"
regexp: "^ server"
line: " server: {{ KUBE_APISERVER }}"
with_items:
- "/root/.kube/config"
- "/etc/kubernetes/kubelet.kubeconfig"
- "/etc/kubernetes/kube-proxy.kubeconfig"
- name: restart kube_node service
service: name={{ item }} state=restarted
with_items:
- kubelet
- kube-proxy
ignore_errors: true
when: "MASTER_CHG == 'yes' and groups['kube_master']|length < 3"
tags: restart_lb

View File

@ -1,65 +0,0 @@
# 离线安装 haproxy
- name: 准备离线安装包目录
file: name=/opt/kube/packages/haproxy state=directory
- block:
- name: 分发 haproxy_xenial 离线包
copy:
src: "{{ base_dir }}/down/packages/haproxy_xenial.tar.gz"
dest: "/opt/kube/packages/haproxy/haproxy_xenial.tar.gz"
- name: 安装 haproxy_xenial 离线包
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_xenial.tar.gz && \
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
when: ansible_distribution_release == "xenial"
ignore_errors: true
- block:
- name: 分发 haproxy_bionic 离线包
copy:
src: "{{ base_dir }}/down/packages/haproxy_bionic.tar.gz"
dest: "/opt/kube/packages/haproxy/haproxy_bionic.tar.gz"
- name: 安装 haproxy_bionic 离线包
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_bionic.tar.gz && \
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
when: ansible_distribution_release == "bionic"
ignore_errors: true
- block:
- name: 分发 haproxy_centos7 离线包
copy:
src: "{{ base_dir }}/down/packages/haproxy_centos7.tar.gz"
dest: "/opt/kube/packages/haproxy/haproxy_centos7.tar.gz"
- name: 安装 haproxy_centos7 离线包
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_centos7.tar.gz && \
rpm -Uvh --force --nodeps *.rpm > /tmp/install_haproxy.log 2>&1'
when:
- 'ansible_distribution == "CentOS"'
- 'ansible_distribution_major_version == "7"'
ignore_errors: true
- block:
- name: 分发 haproxy_stretch 离线包
copy:
src: "{{ base_dir }}/down/packages/haproxy_stretch.tar.gz"
dest: "/opt/kube/packages/haproxy/haproxy_stretch.tar.gz"
- name: 安装 haproxy_stretch 离线包
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_stretch.tar.gz && \
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
when: ansible_distribution_release == "stretch"
ignore_errors: true
- block:
- name: 分发 haproxy_buster 离线包
copy:
src: "{{ base_dir }}/down/packages/haproxy_buster.tar.gz"
dest: "/opt/kube/packages/haproxy/haproxy_buster.tar.gz"
- name: 安装 haproxy_buster 离线包
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_buster.tar.gz && \
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
when: ansible_distribution_release == "buster"
ignore_errors: true

View File

@ -1,16 +1,5 @@
# 设置 APISERVER 地址
KUBE_APISERVER: "{%- if inventory_hostname in groups['kube_master'] -%} \
https://{{ inventory_hostname }}:6443 \
{%- else -%} \
{%- if groups['kube_master']|length > 1 -%} \
https://127.0.0.1:6443 \
{%- else -%} \
https://{{ groups['kube_master'][0] }}:6443 \
{%- endif -%} \
{%- endif -%}"
# 设置 APISERVER 地址使用kube-lb负载均衡监听地址
KUBE_APISERVER: "https://127.0.0.1:6443"
# node local dns cache 离线镜像
dnscache_offline: "k8s-dns-node-cache_{{ dnsNodeCacheVer }}.tar"
# 增加/删除 master 节点时node 节点需要重新配置 haproxy
MASTER_CHG: "no"