重构kube-master/kube-node流程

04.kube-master.yml

@@ -6,11 +6,9 @@
   tasks:
   - name: Making master nodes SchedulingDisabled
     shell: "{{ bin_dir }}/kubectl cordon {{ inventory_hostname }} "
-    delegate_to: "{{ groups.deploy[0] }}"
-    when: DEPLOY_MODE != "allinone"
+    when: "inventory_hostname not in groups['kube-node']"
     ignore_errors: true
 
   - name: Setting master role name 
     shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=master --overwrite"
     ignore_errors: true
-    delegate_to: "{{ groups.deploy[0] }}"

05.kube-node.yml

@@ -1,4 +1,4 @@
 # to set up 'kube-node' nodes
 - hosts: kube-node
   roles:
-  - kube-node
+  - { role: kube-node, when: "inventory_hostname not in groups['kube-master']" } 

roles/deploy/tasks/main.yml

@@ -110,3 +110,11 @@
     regexp: 'kubeasz'
     line: 'export PATH={{ base_dir }}/bin/:$PATH # generated by kubeasz'
   ignore_errors: true
+
+- name: ansible 控制端添加 kubectl 自动补全
+  lineinfile:
+    dest: ~/.bashrc
+    state: present
+    regexp: 'kubectl completion'
+    line: 'source <(kubectl completion bash)'
+  ignore_errors: true

roles/kube-master/tasks/main.yml

@@ -6,15 +6,6 @@
   - kube-scheduler
   tags: upgrade_k8s
 
-- name: 分发证书相关
-  copy: src={{ base_dir }}/.cluster/ssl/{{ item }} dest={{ ca_dir }}/{{ item }}
-  with_items:
-  - admin.pem
-  - admin-key.pem
-  - ca.pem
-  - ca-key.pem
-  - ca-config.json
-
 - name: 创建 kubernetes 证书签名请求
   template: src=kubernetes-csr.json.j2 dest={{ ca_dir }}/kubernetes-csr.json
   tags: change_cert
@@ -88,9 +79,6 @@
 	systemctl restart kube-controller-manager && systemctl restart kube-scheduler"
   tags: upgrade_k8s, restart_master
 
-- name: 分发 kubeconfig 给 master 节点
-  copy: src=/root/.kube/config dest=/root/.kube/config
-
 - name: 替换 kubeconfig 的 apiserver 地址
   lineinfile:
     dest: /root/.kube/config

roles/kube-node/defaults/main.yml

@@ -17,3 +17,10 @@ KUBE_RESERVED: "cpu=200m,memory=500Mi,ephemeral-storage=1Gi"
 
 # 配置kubelet的hard eviction条件
 HARD_EVICTION: "memory.available<200Mi,nodefs.available<10%"
+
+# node 请求 apiserver 负载均衡算法，常见如下：
+# "roundrobin": 基于服务器权重的轮询
+# "leastconn": 基于服务器最小连接数
+# "source": 基于请求源IP地址
+# "uri": 基于请求的URI
+BALANCE_ALG: "roundrobin"

roles/kube-node/tasks/main.yml

@@ -1,3 +1,14 @@
+# 每个 node 节点运行 haproxy 连接到多个 apiserver
+- import_tasks: node_lb.yml
+  when: "inventory_hostname not in groups['kube-master']"
+
+- name: 替换 kubeconfig 的 apiserver 地址
+  lineinfile:
+    dest: /root/.kube/config
+    regexp: "^    server"
+    line: "    server: https://127.0.0.1:6443"
+  when: "inventory_hostname not in groups['kube-master']"
+
 # 创建kubelet,kube-proxy工作目录和cni配置目录
 - name: 创建kube-node 相关目录
   file: name={{ item }} state=directory
@@ -5,7 +16,6 @@
   - /var/lib/kubelet
   - /var/lib/kube-proxy
   - /etc/cni/net.d
-  - /root/.kube
 
 - name: 下载 kubelet,kube-proxy 二进制和基础 cni plugins
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
@@ -18,36 +28,8 @@
   - loopback
   tags: upgrade_k8s
 
-- name: 分发 kubeconfig配置文件
-  synchronize: src=/root/.kube/config dest=/root/.kube/config
-  delegate_to: "{{ groups.deploy[0] }}"
-
-- name: 添加 kubectl 命令自动补全
-  lineinfile:
-    dest: ~/.bashrc
-    state: present
-    regexp: 'kubectl completion'
-    line: 'source <(kubectl completion bash)'
-
-- name: ansible 控制端添加 kubectl 自动补全
-  lineinfile:
-    dest: ~/.bashrc
-    state: present
-    regexp: 'kubectl completion'
-    line: 'source <(kubectl completion bash)'
-  connection: local
-  run_once: true
-  ignore_errors: true
-
-- name: 分发证书相关
-  synchronize: src={{ ca_dir }}/{{ item }} dest={{ ca_dir }}/{{ item }}
-  with_items:
-  - ca.pem
-  - ca-key.pem
-  - ca-config.json
-  delegate_to: "{{ groups.deploy[0] }}"
-
 ##----------kubelet 配置部分--------------
+
 - name: 准备kubelet 证书签名请求
   template: src=kubelet-csr.json.j2 dest={{ ca_dir }}/kubelet-csr.json
 
@@ -63,28 +45,34 @@
   shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
         --certificate-authority={{ ca_dir }}/ca.pem \
         --embed-certs=true \
-        --server={{ KUBE_APISERVER }} \
-	--kubeconfig=kubelet.kubeconfig"
+        --server=https://127.0.0.1:6443 \
+	--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
+  when: "inventory_hostname not in groups['kube-master']"
+
+- name: 设置集群参数
+  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
+        --certificate-authority={{ ca_dir }}/ca.pem \
+        --embed-certs=true \
+        --server=https://{{ inventory_hostname }}:6443 \
+	--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
+  when: "inventory_hostname in groups['kube-master']"
 
 - name: 设置客户端认证参数
   shell: "{{ bin_dir }}/kubectl config set-credentials system:node:{{ inventory_hostname }} \
         --client-certificate={{ ca_dir }}/kubelet.pem \
         --embed-certs=true \
         --client-key={{ ca_dir }}/kubelet-key.pem \
-	--kubeconfig=kubelet.kubeconfig"
+	--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
 
 - name: 设置上下文参数
   shell: "{{ bin_dir }}/kubectl config set-context default \
         --cluster=kubernetes \
 	--user=system:node:{{ inventory_hostname }} \
-	--kubeconfig=kubelet.kubeconfig"
+	--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
 
 - name: 选择默认上下文
   shell: "{{ bin_dir }}/kubectl config use-context default \
-	--kubeconfig=kubelet.kubeconfig"
-
-- name: 移动 kubelet.kubeconfig
-  shell: "mv /root/kubelet.kubeconfig /etc/kubernetes/"
+	--kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
 
 - name: 准备 cni配置文件
   template: src=cni-default.conf.j2 dest=/etc/cni/net.d/10-default.conf
@@ -102,9 +90,20 @@
   tags: upgrade_k8s, restart_node
 
 ##-------kube-proxy部分----------------
-- name: 安装kube-proxy.kubeconfig配置文件
-  synchronize: src=/etc/kubernetes/kube-proxy.kubeconfig dest=/etc/kubernetes/kube-proxy.kubeconfig
-  delegate_to: "{{ groups.deploy[0] }}"
+
+- name: 替换 kube-proxy.kubeconfig 的 apiserver 地址
+  lineinfile:
+    dest: /etc/kubernetes/kube-proxy.kubeconfig
+    regexp: "^    server"
+    line: "    server: https://127.0.0.1:6443"
+  when: "inventory_hostname not in groups['kube-master']"
+
+- name: 替换 kube-proxy.kubeconfig 的 apiserver 地址
+  lineinfile:
+    dest: /etc/kubernetes/kube-proxy.kubeconfig
+    regexp: "^    server"
+    line: "    server: https://{{ inventory_hostname }}:6443"
+  when: "inventory_hostname in groups['kube-master']"
 
 - name: 创建kube-proxy 服务文件
   tags: reload-kube-proxy, upgrade_k8s, restart_node
@@ -118,7 +117,7 @@
   shell: systemctl daemon-reload && systemctl restart kube-proxy
   tags: reload-kube-proxy, upgrade_k8s, restart_node
 
-# 批准 node 节点，首先轮询等待kubelet启动完成
+# 轮询等待kubelet启动完成
 - name: 轮询等待kubelet启动
   shell: "systemctl status kubelet.service|grep Active"
   register: kubelet_status
@@ -126,23 +125,9 @@
   retries: 8
   delay: 2
 
-#- name: 获取csr 请求信息
-#  shell: "sleep 3 && {{ bin_dir }}/kubectl get csr"
-#  delegate_to: "{{ groups.deploy[0] }}"
-#  register: csr_info
-#  run_once: true
-
-#- name: approve-kubelet-csr
-#  shell: "{{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| \
-#        xargs {{ bin_dir }}/kubectl certificate approve"
-#  when: '"Pending" in csr_info.stdout'
-#  delegate_to: "{{ groups.deploy[0] }}"
-#  run_once: true
-
 - name: 轮询等待node达到Ready状态
   shell: "{{ bin_dir }}/kubectl get node {{ inventory_hostname }}|awk 'NR>1{print $2}'"
   register: node_status
-  delegate_to: "{{ groups.deploy[0] }}"
   until: node_status.stdout == "Ready" or node_status.stdout == "Ready,SchedulingDisabled"
   retries: 8 
   delay: 8
@@ -151,4 +136,3 @@
 - name: 设置node节点role
   shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=node --overwrite"
   ignore_errors: true
-  delegate_to: "{{ groups.deploy[0] }}"

roles/kube-node/tasks/node_lb.yml

@@ -0,0 +1,26 @@
+- name: 安装 haproxy
+  package: name=haproxy state=present
+
+- name: 创建haproxy配置目录
+  file: name=/etc/haproxy state=directory
+
+- name: 修改centos的haproxy.service
+  template: src=haproxy.service.j2 dest=/usr/lib/systemd/system/haproxy.service
+  when: 'ansible_distribution in ["CentOS","RedHat","Amazon"]' 
+  tags: restart_lb
+
+- name: 配置 haproxy
+  template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
+  tags: restart_lb
+
+- name: daemon-reload for haproxy.service
+  shell: systemctl daemon-reload
+  tags: restart_lb
+
+- name: 开机启用haproxy服务
+  shell: systemctl enable haproxy
+  ignore_errors: true
+
+- name: 重启haproxy服务
+  shell: systemctl restart haproxy
+  tags: restart_lb

roles/kube-node/templates/haproxy.cfg.j2

@@ -0,0 +1,26 @@
+global
+        log /dev/log    local1 warning
+        chroot /var/lib/haproxy
+        stats socket /run/haproxy/admin.sock mode 660 level admin
+        stats timeout 30s
+        user haproxy
+        group haproxy
+        daemon
+        nbproc 1
+
+defaults
+        log     global
+        timeout connect 5s
+        timeout client  10m
+        timeout server  10m
+
+listen kube-master
+        bind 127.0.0.1:6443
+        mode tcp
+        option tcplog
+        option dontlognull
+        option dontlog-normal
+        balance {{ BALANCE_ALG }} 
+{% for host in groups['kube-master'] %}
+        server {{ host }} {{ host }}:6443 check inter 10s fall 2 rise 2 weight 1
+{% endfor %}

roles/kube-node/templates/haproxy.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=HAProxy Load Balancer
+After=syslog.target network.target
+
+[Service]
+EnvironmentFile=/etc/sysconfig/haproxy
+ExecStartPre=/usr/bin/mkdir -p /run/haproxy
+ExecStart=/usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid $OPTIONS
+ExecReload=/bin/kill -USR2 $MAINPID
+KillMode=mixed
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

roles/prepare/tasks/main.yml

@@ -28,3 +28,27 @@
     state: present
     regexp: 'kubeasz'
     line: 'export PATH={{ bin_dir }}:$PATH # generated by kubeasz'
+
+- block:
+    - name: 分发证书相关
+      copy: src={{ base_dir }}/.cluster/ssl/{{ item }} dest={{ ca_dir }}/{{ item }}
+      with_items:
+      - admin.pem
+      - admin-key.pem
+      - ca.pem
+      - ca-key.pem
+      - ca-config.json
+
+    - name: 添加 kubectl 命令自动补全
+      lineinfile:
+        dest: ~/.bashrc
+        state: present
+        regexp: 'kubectl completion'
+        line: 'source <(kubectl completion bash)'
+
+    - name: 分发 kubeconfig配置文件
+      copy: src=/root/.kube/config dest=/root/.kube/config
+
+    - name: 分发 kube-proxy.kubeconfig配置文件
+      copy: src={{ base_dir }}/.cluster/kube-proxy.kubeconfig dest=/etc/kubernetes/kube-proxy.kubeconfig
+  when: "inventory_hostname in groups['kube-master'] or inventory_hostname in groups['kube-node']"