mirror of https://github.com/easzlab/kubeasz.git
feat:[kube-lb] a l4 nginx proxy for apiservers
parent
10a900f8c3
commit
e13fe39eda
14
ezctl
14
ezctl
|
@ -270,14 +270,14 @@ function list() {
|
|||
which md5sum > /dev/null 2>&1 || { logger error "md5sum not found"; return 1; }
|
||||
|
||||
CLUSTERS=$(cd clusters && echo -- *)
|
||||
CFG_MD5=$(md5sum -t ~/.kube/config |cut -d' ' -f1)
|
||||
CFG_MD5=$(sed '/server/d' ~/.kube/config|md5sum|cut -d' ' -f1)
|
||||
cd "$BASE"
|
||||
|
||||
logger info "list of managed clusters:"
|
||||
i=1; for c in $CLUSTERS;
|
||||
do
|
||||
if [[ -f "clusters/$c/kubectl.kubeconfig" ]];then
|
||||
c_md5=$(md5sum -t "clusters/$c/kubectl.kubeconfig" |cut -d' ' -f1)
|
||||
c_md5=$(sed '/server/d' "clusters/$c/kubectl.kubeconfig"|md5sum|cut -d' ' -f1)
|
||||
if [[ "$c_md5" = "$CFG_MD5" ]];then
|
||||
echo -e "==> cluster $i:\t$c (\033[32mcurrent\033[0m)"
|
||||
else
|
||||
|
@ -327,8 +327,8 @@ function add-master() {
|
|||
logger info "start to add a master node:$2 into cluster:$1"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/23.addmaster.yml" -e "NODE_TO_ADD=$2" -e "@clusters/$1/config.yml"
|
||||
|
||||
logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml"
|
||||
logger info "reconfigure and restart 'kube-lb' service"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml"
|
||||
}
|
||||
|
||||
function add-etcd() {
|
||||
|
@ -371,7 +371,7 @@ function del-etcd() {
|
|||
|
||||
function del-node() {
|
||||
# check node's address regexp
|
||||
[[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger "Invalid ip add:$2"; return 2; }
|
||||
[[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger error "Invalid ip add:$2"; return 2; }
|
||||
|
||||
# check if the deleting node exsited
|
||||
sed -n '/^\[kube_master/,/^\[harbor/p' "$BASE/clusters/$1/hosts"|grep -E "^$2$|^$2 " || { logger error "node $2 not existed in $BASE/clusters/$1/hosts"; return 2; }
|
||||
|
@ -393,8 +393,8 @@ function del-master() {
|
|||
logger info "reconfig kubeconfig in ansible manage node"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/roles/deploy/deploy.yml" -t create_kctl_cfg -e "@clusters/$1/config.yml"
|
||||
|
||||
logger info "reconfigure and restart the haproxy service on 'kube_node' nodes"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/05.kube-node.yml" -t restart_lb -e MASTER_CHG=yes -e "@clusters/$1/config.yml"
|
||||
logger info "reconfigure and restart 'kube-lb' service"
|
||||
ansible-playbook -i "$BASE/clusters/$1/hosts" "$BASE/playbooks/90.setup.yml" -t restart_kube-lb -e "@clusters/$1/config.yml"
|
||||
}
|
||||
|
||||
|
||||
|
|
2
ezdown
2
ezdown
|
@ -16,7 +16,7 @@ set -o errexit
|
|||
DOCKER_VER=20.10.5
|
||||
KUBEASZ_VER=3.0.1
|
||||
K8S_BIN_VER=v1.20.5
|
||||
EXT_BIN_VER=0.8.1
|
||||
EXT_BIN_VER=0.9.0
|
||||
SYS_PKG_VER=0.3.3
|
||||
HARBOR_VER=v2.1.3
|
||||
REGISTRY_MIRROR=CN
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
# to set up 'kube_master' nodes
|
||||
- hosts: kube_master
|
||||
roles:
|
||||
- kube-lb
|
||||
- kube-master
|
||||
- kube-node
|
||||
tasks:
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
# to set up 'kube_node' nodes
|
||||
- hosts: kube_node
|
||||
roles:
|
||||
- { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
|
||||
- { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
- prepare
|
||||
- { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
|
||||
- { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
|
||||
- kube-lb
|
||||
- kube-node
|
||||
- { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
|
||||
- { role: cilium, when: "CLUSTER_NETWORK == 'cilium'" }
|
||||
|
|
|
@ -7,6 +7,7 @@
|
|||
- prepare
|
||||
- { role: docker, when: "CONTAINER_RUNTIME == 'docker'" }
|
||||
- { role: containerd, when: "CONTAINER_RUNTIME == 'containerd'" }
|
||||
- kube-lb
|
||||
- kube-master
|
||||
- kube-node
|
||||
- { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
|
||||
|
|
|
@ -38,6 +38,7 @@
|
|||
# to set up 'kube_master' nodes
|
||||
- hosts: kube_master
|
||||
roles:
|
||||
- kube-lb
|
||||
- kube-master
|
||||
- kube-node
|
||||
tasks:
|
||||
|
@ -53,6 +54,7 @@
|
|||
# to set up 'kube_node' nodes
|
||||
- hosts: kube_node
|
||||
roles:
|
||||
- { role: kube-lb, when: "inventory_hostname not in groups['kube_master']" }
|
||||
- { role: kube-node, when: "inventory_hostname not in groups['kube_master']" }
|
||||
|
||||
# to install network plugin, only one can be choosen
|
||||
|
|
|
@ -3,6 +3,13 @@
|
|||
- name: starting etcd cluster
|
||||
service: name=etcd state=started enabled=yes
|
||||
|
||||
- hosts:
|
||||
- kube_master
|
||||
- kube_node
|
||||
tasks:
|
||||
- name: starting kube-lb
|
||||
service: name=kube-lb state=started enabled=yes
|
||||
|
||||
- hosts: kube_master
|
||||
tasks:
|
||||
- name: starting kube_master services
|
||||
|
@ -24,12 +31,6 @@
|
|||
service: name=containerd state=started enabled=yes
|
||||
when: "CONTAINER_RUNTIME == 'containerd'"
|
||||
|
||||
- name: starting haproxy on kube_node
|
||||
service: name=haproxy state=started enabled=yes
|
||||
when:
|
||||
- "inventory_hostname not in groups['kube_master']"
|
||||
- "groups['kube_master']|length > 1"
|
||||
|
||||
- name: starting kube_node services
|
||||
service: name={{ item }} state=started enabled=yes
|
||||
with_items:
|
||||
|
|
|
@ -24,15 +24,10 @@
|
|||
- kube_master
|
||||
- kube_node
|
||||
tasks:
|
||||
- name: stopping haproxy on kube_node
|
||||
service: name=haproxy state=stopped enabled=no
|
||||
when:
|
||||
- "inventory_hostname not in groups['kube_master']"
|
||||
- "groups['kube_master']|length > 1"
|
||||
|
||||
- name: stopping kube_node services
|
||||
service: name={{ item }} state=stopped enabled=no
|
||||
with_items:
|
||||
- kube-lb
|
||||
- kubelet
|
||||
- kube-proxy
|
||||
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
- name: stop and disable kube_node service
|
||||
service: name={{ item }} state=stopped enabled=no
|
||||
with_items:
|
||||
- kube-lb
|
||||
- kubelet
|
||||
- kube-proxy
|
||||
ignore_errors: true
|
||||
|
@ -18,9 +19,11 @@
|
|||
with_items:
|
||||
- "/var/lib/kubelet/"
|
||||
- "/var/lib/kube-proxy/"
|
||||
- "/etc/systemd/system/kube-lb.service"
|
||||
- "/etc/systemd/system/kubelet.service"
|
||||
- "/etc/systemd/system/kube-proxy.service"
|
||||
- "/opt/kube/kube-system/"
|
||||
- "/etc/kube-lb/"
|
||||
- "/etc/kubernetes/"
|
||||
- "/root/.kube/config"
|
||||
|
||||
|
|
|
@ -1,8 +1,3 @@
|
|||
# ex_lb 节点成员不能同时是 kube_node 节点,因为它们都需要安装 haproxy
|
||||
- name: fail info1
|
||||
fail: msg="an 'ex_lb' node CAN NOT be a 'kube_node' node at the same time"
|
||||
when: "inventory_hostname in groups['kube_node']"
|
||||
|
||||
# 自动设置LB节点变量'LB_IF'
|
||||
- name: 注册变量 LB_IF_TMP
|
||||
shell: "ip a|grep '{{ inventory_hostname }}/'|awk '{print $NF}'"
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
- hosts:
|
||||
- kube-master
|
||||
- kube-node
|
||||
tasks:
|
||||
- name: stop and disable kube-lb service
|
||||
service:
|
||||
name: kube-lb
|
||||
state: stopped
|
||||
enabled: no
|
||||
ignore_errors: true
|
||||
|
||||
- name: remove files and dirs
|
||||
file: name={{ item }} state=absent
|
||||
with_items:
|
||||
- "/etc/kube-lb"
|
||||
- "/etc/systemd/system/kube-lb.service"
|
|
@ -0,0 +1,34 @@
|
|||
- name: prepare some dirs
|
||||
file: name={{ item }} state=directory
|
||||
with_items:
|
||||
- "/etc/kube-lb/sbin"
|
||||
- "/etc/kube-lb/logs"
|
||||
- "/etc/kube-lb/conf"
|
||||
|
||||
- name: 下载二进制文件kube-lb(nginx)
|
||||
copy: src={{ base_dir }}/bin/nginx dest=/etc/kube-lb/sbin/kube-lb mode=0755
|
||||
|
||||
- name: 创建kube-lb的配置文件
|
||||
template: src=kube-lb.conf.j2 dest=/etc/kube-lb/conf/kube-lb.conf
|
||||
tags: restart_kube-lb
|
||||
|
||||
- name: 创建kube-lb的systemd unit文件
|
||||
template: src=kube-lb.service.j2 dest=/etc/systemd/system/kube-lb.service
|
||||
tags: restart_kube-lb
|
||||
|
||||
- name: 开机启用kube-lb服务
|
||||
shell: systemctl enable kube-lb
|
||||
ignore_errors: true
|
||||
|
||||
- name: 开启kube-lb服务
|
||||
shell: systemctl daemon-reload && systemctl restart kube-lb
|
||||
ignore_errors: true
|
||||
tags: restart_kube-lb
|
||||
|
||||
- name: 以轮询的方式等待kube-lb服务启动
|
||||
shell: "systemctl status kube-lb.service|grep Active"
|
||||
register: svc_status
|
||||
until: '"running" in svc_status.stdout'
|
||||
retries: 3
|
||||
delay: 3
|
||||
tags: restart_kube-lb
|
|
@ -0,0 +1,22 @@
|
|||
user root;
|
||||
worker_processes 1;
|
||||
|
||||
error_log /etc/kube-lb/logs/error.log warn;
|
||||
|
||||
events {
|
||||
worker_connections 3000;
|
||||
}
|
||||
|
||||
stream {
|
||||
upstream backend {
|
||||
{% for host in groups['kube_master'] %}
|
||||
server {{ host }}:6443 max_fails=2 fail_timeout=3s;
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 127.0.0.1:6443;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_pass backend;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
[Unit]
|
||||
Description=l4 nginx proxy for kube-apiservers
|
||||
After=network.target
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
ExecStartPre=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -t
|
||||
ExecStart=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb
|
||||
ExecReload=/etc/kube-lb/sbin/kube-lb -c /etc/kube-lb/conf/kube-lb.conf -p /etc/kube-lb -s reload
|
||||
PrivateTmp=true
|
||||
Restart=always
|
||||
RestartSec=15
|
||||
StartLimitInterval=0
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -57,7 +57,7 @@
|
|||
lineinfile:
|
||||
dest: "{{ item }}"
|
||||
regexp: "^ server"
|
||||
line: " server: https://{{ inventory_hostname }}:6443"
|
||||
line: " server: https://127.0.0.1:6443"
|
||||
with_items:
|
||||
- "/root/.kube/config"
|
||||
- "/etc/kubernetes/kube-controller-manager.kubeconfig"
|
||||
|
|
|
@ -16,10 +16,6 @@
|
|||
- loopback
|
||||
tags: upgrade_k8s
|
||||
|
||||
# 每个 node 节点运行 haproxy 连接到多个 apiserver
|
||||
- import_tasks: node_lb.yml
|
||||
when: "inventory_hostname not in groups['kube_master']"
|
||||
|
||||
- name: 替换 kubeconfig 的 apiserver 地址
|
||||
lineinfile:
|
||||
dest: /root/.kube/config
|
||||
|
|
|
@ -1,66 +0,0 @@
|
|||
# kube_node 节点成员不能同时是 ex_lb 节点,因为它们都需要安装 haproxy
|
||||
- name: fail info1
|
||||
fail: msg="an 'kube_node' node CAN NOT be a 'ex_lb' node at the same time"
|
||||
when: "inventory_hostname in groups['ex_lb']"
|
||||
tags: restart_lb
|
||||
|
||||
- name: 安装 haproxy
|
||||
package: name=haproxy state=present
|
||||
when: 'INSTALL_SOURCE != "offline"'
|
||||
|
||||
# 离线安装 haproxy
|
||||
- import_tasks: offline.yml
|
||||
when: 'INSTALL_SOURCE == "offline"'
|
||||
|
||||
- name: 创建haproxy配置目录
|
||||
file: name=/etc/haproxy state=directory
|
||||
|
||||
- name: 修改centos的haproxy.service
|
||||
template: src=haproxy.service.j2 dest=/usr/lib/systemd/system/haproxy.service
|
||||
when:
|
||||
- 'ansible_distribution in ["CentOS","RedHat","Amazon","Aliyun"]'
|
||||
tags: restart_lb
|
||||
|
||||
- name: 配置 haproxy
|
||||
template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
|
||||
tags: restart_lb
|
||||
|
||||
- name: daemon-reload for haproxy.service
|
||||
shell: systemctl daemon-reload
|
||||
tags: restart_lb
|
||||
|
||||
- name: 开机启用haproxy服务
|
||||
shell: systemctl enable haproxy
|
||||
ignore_errors: true
|
||||
|
||||
- name: 停止haproxy服务
|
||||
shell: systemctl stop haproxy
|
||||
tags: restart_lb
|
||||
|
||||
# 仅 master 节点数大于1时才启动haproxy
|
||||
- name: 开启haproxy服务
|
||||
shell: systemctl start haproxy
|
||||
when: "groups['kube_master']|length > 1"
|
||||
tags: restart_lb
|
||||
|
||||
# master 节点从1个增加到2个时候,需要修改如下配置
|
||||
# master 节点从2个减少到1个时候,也需要修改
|
||||
- block:
|
||||
- name: 替换 kubeconfig 的 apiserver 地址
|
||||
lineinfile:
|
||||
dest: "{{ item }}"
|
||||
regexp: "^ server"
|
||||
line: " server: {{ KUBE_APISERVER }}"
|
||||
with_items:
|
||||
- "/root/.kube/config"
|
||||
- "/etc/kubernetes/kubelet.kubeconfig"
|
||||
- "/etc/kubernetes/kube-proxy.kubeconfig"
|
||||
|
||||
- name: restart kube_node service
|
||||
service: name={{ item }} state=restarted
|
||||
with_items:
|
||||
- kubelet
|
||||
- kube-proxy
|
||||
ignore_errors: true
|
||||
when: "MASTER_CHG == 'yes' and groups['kube_master']|length < 3"
|
||||
tags: restart_lb
|
|
@ -1,65 +0,0 @@
|
|||
# 离线安装 haproxy
|
||||
- name: 准备离线安装包目录
|
||||
file: name=/opt/kube/packages/haproxy state=directory
|
||||
|
||||
- block:
|
||||
- name: 分发 haproxy_xenial 离线包
|
||||
copy:
|
||||
src: "{{ base_dir }}/down/packages/haproxy_xenial.tar.gz"
|
||||
dest: "/opt/kube/packages/haproxy/haproxy_xenial.tar.gz"
|
||||
|
||||
- name: 安装 haproxy_xenial 离线包
|
||||
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_xenial.tar.gz && \
|
||||
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
|
||||
when: ansible_distribution_release == "xenial"
|
||||
ignore_errors: true
|
||||
|
||||
- block:
|
||||
- name: 分发 haproxy_bionic 离线包
|
||||
copy:
|
||||
src: "{{ base_dir }}/down/packages/haproxy_bionic.tar.gz"
|
||||
dest: "/opt/kube/packages/haproxy/haproxy_bionic.tar.gz"
|
||||
|
||||
- name: 安装 haproxy_bionic 离线包
|
||||
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_bionic.tar.gz && \
|
||||
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
|
||||
when: ansible_distribution_release == "bionic"
|
||||
ignore_errors: true
|
||||
|
||||
- block:
|
||||
- name: 分发 haproxy_centos7 离线包
|
||||
copy:
|
||||
src: "{{ base_dir }}/down/packages/haproxy_centos7.tar.gz"
|
||||
dest: "/opt/kube/packages/haproxy/haproxy_centos7.tar.gz"
|
||||
|
||||
- name: 安装 haproxy_centos7 离线包
|
||||
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_centos7.tar.gz && \
|
||||
rpm -Uvh --force --nodeps *.rpm > /tmp/install_haproxy.log 2>&1'
|
||||
when:
|
||||
- 'ansible_distribution == "CentOS"'
|
||||
- 'ansible_distribution_major_version == "7"'
|
||||
ignore_errors: true
|
||||
|
||||
- block:
|
||||
- name: 分发 haproxy_stretch 离线包
|
||||
copy:
|
||||
src: "{{ base_dir }}/down/packages/haproxy_stretch.tar.gz"
|
||||
dest: "/opt/kube/packages/haproxy/haproxy_stretch.tar.gz"
|
||||
|
||||
- name: 安装 haproxy_stretch 离线包
|
||||
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_stretch.tar.gz && \
|
||||
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
|
||||
when: ansible_distribution_release == "stretch"
|
||||
ignore_errors: true
|
||||
|
||||
- block:
|
||||
- name: 分发 haproxy_buster 离线包
|
||||
copy:
|
||||
src: "{{ base_dir }}/down/packages/haproxy_buster.tar.gz"
|
||||
dest: "/opt/kube/packages/haproxy/haproxy_buster.tar.gz"
|
||||
|
||||
- name: 安装 haproxy_buster 离线包
|
||||
shell: 'cd /opt/kube/packages/haproxy && tar zxf haproxy_buster.tar.gz && \
|
||||
dpkg -i *.deb > /tmp/install_haproxy.log 2>&1'
|
||||
when: ansible_distribution_release == "buster"
|
||||
ignore_errors: true
|
|
@ -1,16 +1,5 @@
|
|||
# 设置 APISERVER 地址
|
||||
KUBE_APISERVER: "{%- if inventory_hostname in groups['kube_master'] -%} \
|
||||
https://{{ inventory_hostname }}:6443 \
|
||||
{%- else -%} \
|
||||
{%- if groups['kube_master']|length > 1 -%} \
|
||||
https://127.0.0.1:6443 \
|
||||
{%- else -%} \
|
||||
https://{{ groups['kube_master'][0] }}:6443 \
|
||||
{%- endif -%} \
|
||||
{%- endif -%}"
|
||||
# 设置 APISERVER 地址,使用kube-lb负载均衡监听地址
|
||||
KUBE_APISERVER: "https://127.0.0.1:6443"
|
||||
|
||||
# node local dns cache 离线镜像
|
||||
dnscache_offline: "k8s-dns-node-cache_{{ dnsNodeCacheVer }}.tar"
|
||||
|
||||
# 增加/删除 master 节点时,node 节点需要重新配置 haproxy
|
||||
MASTER_CHG: "no"
|
||||
|
|
Loading…
Reference in New Issue