优化安装流程，修复多主模式dashboard访问bug

.gitignore

@@ -5,4 +5,3 @@ bin/*
 hosts
 *.crt
 *.pem
-roles/prepare/files/ca*

03.kubectl.yml

@@ -1,6 +0,0 @@
-- hosts: 
-  - kube-master
-  - kube-node
-  - deploy
-  roles:
-  - kubectl

04.docker.yml

@@ -1,4 +1,5 @@
 - hosts:
+  - kube-master
   - kube-node
   roles:
   - docker

05.kube-master.yml

@@ -1,3 +1,11 @@
 - hosts: kube-master
   roles:
   - kube-master
+  - kube-node
+  # 禁止业务 pod调度到 master节点
+  tasks:
+  - name: 禁止业务 pod调度到 master节点
+    shell: "{{ bin_dir }}/kubectl cordon {{ NODE_IP }} "
+    when: DEPLOY_MODE != "allinone"
+    ignore_errors: true
+

07.calico.yml

@@ -1,4 +0,0 @@
-- hosts:
-  - kube-node
-  roles:
-  - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }

07.flannel.yml

@@ -1,4 +0,0 @@
-- hosts:
-  - kube-node
-  roles:
-  - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" } 

07.network.yml

@@ -0,0 +1,7 @@
+# 集群网络插件部署，只能选择一种安装
+- hosts:
+  - kube-master
+  - kube-node
+  roles:
+  - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
+  - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }

11.harbor.yml

@@ -5,6 +5,7 @@
   - harbor
 
 - hosts: 
+  - kube-master
   - kube-node
   - new-node
   tasks:

20.addnode.yml

@@ -1,15 +1,8 @@
 - hosts: new-node
   roles:
   - prepare
-  - kubectl
   - docker
   - kube-node
   # 根据hosts中配置，以下两种网络只会安装一种
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }
   - { role: flannel, when: "CLUSTER_NETWORK == 'flannel'" }
-
-- hosts: deploy
-  tasks:
-  - name: 批准新增node节点
-    shell: "sleep 15 && {{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs {{ bin_dir }}/kubectl certificate approve"
-    ignore_errors: true

21.addmaster.yml

@@ -1,14 +1,13 @@
-# 集群节点的公共配置任务
-- hosts:
-  - kube-master
-  roles:
-  - prepare
-
-# [可选]多master部署时的负载均衡配置
+# 重新配置启动 haproxy
 - hosts: lb
-  roles:
-  - lb
+  tasks:
+  - name: 配置 haproxy
+    template: src=haproxy.cfg.j2 dest=/etc/haproxy/haproxy.cfg
+  - name: 重启haproxy服务
+    shell: systemctl enable haproxy && systemctl restart haproxy
 
 - hosts: kube-master
   roles:
+  - prepare
   - kube-master
+  - kube-node

90.setup.yml

@@ -24,32 +24,32 @@
   roles:
   - etcd
 
-# kubectl 客户端配置
-- hosts: 
-  - kube-master
-  - kube-node
-  - deploy
-  roles:
-  - kubectl
-
 # docker服务安装
 - hosts:
+  - kube-master
   - kube-node
   roles:
   - docker
 
-# master 节点部署
 - hosts: kube-master
   roles:
   - kube-master
+  - kube-node
+  # 禁止业务 pod调度到 master节点
+  tasks:
+  - name: 禁止业务 pod调度到 master节点
+    shell: "{{ bin_dir }}/kubectl cordon {{ NODE_IP }} "
+    when: DEPLOY_MODE != "allinone"
+    ignore_errors: true
 
 # node 节点部署
 - hosts: kube-node
   roles:
-  - kube-node
+  - { role: kube-node, when: "DEPLOY_MODE != 'allinone'" }
 
 # 集群网络插件部署，只能选择一种安装
 - hosts:
+  - kube-master
   - kube-node
   roles:
   - { role: calico, when: "CLUSTER_NETWORK == 'calico'" }

99.clean.yml

@@ -3,6 +3,7 @@
 
 # 清理 kube-node 相关服务
 - hosts:
+  - kube-master
   - kube-node
   - new-node
   tasks:
@@ -21,7 +22,7 @@
     - "/var/lib/kube-proxy/"
     - "/etc/systemd/system/kubelet.service"
     - "/etc/systemd/system/kube-proxy.service"
-   # - "/root/local/bin/"
+    - "/root/local/kube-system/"
 
 # 清理 kube-master 相关
 - hosts: kube-master
@@ -40,6 +41,7 @@
 
 # 清理集群docker服务、网络相关
 - hosts:
+  - kube-master
   - kube-node
   - new-node
   tasks:
@@ -117,7 +119,6 @@
     with_items:
     - "/etc/haproxy"
     - "/etc/keepalived"
-    ignore_errors: true
 
 - hosts:
   - kube-master

example/hosts.allinone.example

@@ -22,6 +22,9 @@
 
 [all:vars]
 # ---------集群主要参数---------------
+#集群部署模式：allinone, single-master, multi-master
+DEPLOY_MODE=allinone
+
 #集群 MASTER IP
 MASTER_IP="192.168.1.1"
 

example/hosts.m-masters.example

@@ -39,6 +39,9 @@ MASTER_PORT="8443"		# 设置 api-server VIP地址的服务端口
 
 [all:vars]
 # ---------集群主要参数---------------
+#集群部署模式：allinone, single-master, multi-master
+DEPLOY_MODE=multi-master
+
 #集群 MASTER IP，一般为VIP地址
 MASTER_IP="192.168.1.10"
 KUBE_APISERVER="https://192.168.1.10:8443"

example/hosts.s-master.example

@@ -26,6 +26,9 @@
 
 [all:vars]
 # ---------集群主要参数---------------
+#集群部署模式：allinone, single-master, multi-master
+DEPLOY_MODE=single-master
+
 #集群 MASTER IP
 MASTER_IP="192.168.1.1"
 

manifests/heapster/influxdb.yaml

@@ -38,7 +38,6 @@ metadata:
   name: monitoring-influxdb
   namespace: kube-system
 spec:
-  type: NodePort
   ports:
   - port: 8086
     targetPort: 8086

roles/calico/tasks/main.yml

@@ -28,13 +28,13 @@
   template: src=calico-rbac.yaml.j2 dest=/root/local/kube-system/calico/calico-rbac.yaml
 
 - name: 获取所有已经创建的POD信息
-  command: "kubectl get pod --all-namespaces"
+  command: "{{ bin_dir }}/kubectl get pod --all-namespaces"
   register: pod_info
   run_once: true
 
 # 只需单节点执行一次
 - name: 运行 calico网络
-  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/calico/ && sleep 15"
+  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/calico/ && sleep 5"
   run_once: true
   when: '"calico" not in pod_info.stdout'
 

roles/deploy/tasks/main.yml

@@ -3,32 +3,118 @@
   with_items:
   - "{{ bin_dir }}"
   - "{{ ca_dir }}"
-  - "{{ base_dir }}/roles/prepare/files/"
+  - "/etc/kubernetes"
 
-- name: 下载证书工具 CFSSL
+- name: 下载证书工具 CFSSL和 kubectl
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
   with_items:
   - cfssl
   - cfssl-certinfo
   - cfssljson
+  - kubectl
+
+# 注册变量result，根据result结果判断是否已经生成过ca证书
+# result|failed 说明没有生成过证书，下一步生成证书
+# result|succeeded 说明已经有ca证书，为了保证整个安装的幂等性，跳过证书生成的步骤
+- name: 注册变量result
+  command: "ls {{ ca_dir }}/ca.pem" 
+  register: result
+  ignore_errors: True
 
 - name: 准备CA配置文件
   template: src=ca-config.json.j2 dest={{ ca_dir }}/ca-config.json
+  when: result|failed
 
 - name: 准备CA签名请求
   template: src=ca-csr.json.j2 dest={{ ca_dir }}/ca-csr.json
+  when: result|failed
 
 - name: 生成 CA 证书和私钥
+  when: result|failed
   shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert -initca ca-csr.json | {{ bin_dir }}/cfssljson -bare ca" 
 
-# 为了保证整个安装的幂等性，如果已经生成过CA证书，就使用已经存在的CA；删除/roles/prepare/files/ca* 可以使用新CA 证书
-- name: 准备分发 CA证书
-  copy: src={{ ca_dir }}/{{ item }} dest={{ base_dir }}/roles/prepare/files/{{ item }} force=no
-  with_items:
-  - ca.pem
-  - ca-key.pem
-  - ca.csr
-  - ca-config.json
+# 创建kubectl kubeconfig文件: /root/.kube/config
+- name: 准备kubectl使用的admin 证书签名请求
+  template: src=admin-csr.json.j2 dest={{ ca_dir }}/admin-csr.json
+
+- name: 创建 admin证书与私钥
+  shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \
+        -ca={{ ca_dir }}/ca.pem \
+        -ca-key={{ ca_dir }}/ca-key.pem \
+        -config={{ ca_dir }}/ca-config.json \
+        -profile=kubernetes admin-csr.json | {{ bin_dir }}/cfssljson -bare admin"
+
+- name: 设置集群参数
+  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
+        --certificate-authority={{ ca_dir }}/ca.pem \
+        --embed-certs=true \
+        --server={{ KUBE_APISERVER }}"
+- name: 设置客户端认证参数
+  shell: "{{ bin_dir }}/kubectl config set-credentials admin \
+        --client-certificate={{ ca_dir }}/admin.pem \
+        --embed-certs=true \
+        --client-key={{ ca_dir }}/admin-key.pem"
+- name: 设置上下文参数
+  shell: "{{ bin_dir }}/kubectl config set-context kubernetes \
+        --cluster=kubernetes --user=admin"
+- name: 选择默认上下文
+  shell: "{{ bin_dir }}/kubectl config use-context kubernetes"
+
+#创建bootstrap.kubeconfig配置文件: /root/bootstrap.kubeconfig
+- name: 设置集群参数
+  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
+        --certificate-authority={{ ca_dir }}/ca.pem \
+        --embed-certs=true \
+        --server={{ KUBE_APISERVER }} \
+        --kubeconfig=bootstrap.kubeconfig"
+- name: 设置客户端认证参数
+  shell: "{{ bin_dir }}/kubectl config set-credentials kubelet-bootstrap \
+        --token={{ BOOTSTRAP_TOKEN }} \
+        --kubeconfig=bootstrap.kubeconfig"
+- name: 设置上下文参数
+  shell: "{{ bin_dir }}/kubectl config set-context default \
+        --cluster=kubernetes \
+        --user=kubelet-bootstrap \
+        --kubeconfig=bootstrap.kubeconfig"
+- name: 选择默认上下文
+  shell: "{{ bin_dir }}/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig"
+
+- name: 移动 bootstrap.kubeconfig
+  shell: "mv /root/bootstrap.kubeconfig /etc/kubernetes/"
+
+#创建kube-proxy.kubeconfig配置文件: /root/kube-proxy.kubeconfig
+- name: 准备kube-proxy 证书签名请求
+  template: src=kube-proxy-csr.json.j2 dest={{ ca_dir }}/kube-proxy-csr.json
+
+- name: 创建 kube-proxy证书与私钥
+  shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \
+        -ca={{ ca_dir }}/ca.pem \
+        -ca-key={{ ca_dir }}/ca-key.pem \
+        -config={{ ca_dir }}/ca-config.json \
+        -profile=kubernetes kube-proxy-csr.json | {{ bin_dir }}/cfssljson -bare kube-proxy"
+
+- name: 设置集群参数
+  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
+        --certificate-authority={{ ca_dir }}/ca.pem \
+        --embed-certs=true \
+        --server={{ KUBE_APISERVER }} \
+        --kubeconfig=kube-proxy.kubeconfig"
+- name: 设置客户端认证参数
+  shell: "{{ bin_dir }}/kubectl config set-credentials kube-proxy \
+        --client-certificate={{ ca_dir }}/kube-proxy.pem \
+        --client-key={{ ca_dir }}/kube-proxy-key.pem \
+        --embed-certs=true \
+        --kubeconfig=kube-proxy.kubeconfig"
+- name: 设置上下文参数
+  shell: "{{ bin_dir }}/kubectl config set-context default \
+        --cluster=kubernetes \
+        --user=kube-proxy \
+        --kubeconfig=kube-proxy.kubeconfig"
+- name: 选择默认上下文
+  shell: "{{ bin_dir }}/kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig"
+
+- name: 移动 kube-proxy.kubeconfig
+  shell: "mv /root/kube-proxy.kubeconfig /etc/kubernetes/"
 
 # kubedns.yaml文件中部分参数根据hosts文件设置而定，因此需要用template模块替换参数
 - name: 准备 kubedns的部署文件 kubedns.yaml

roles/flannel/tasks/main.yml

@@ -17,13 +17,13 @@
   template: src=kube-flannel.yaml.j2 dest=/root/local/kube-system/flannel/kube-flannel.yaml
 
 - name: 获取所有已经创建的POD信息
-  command: "kubectl get pod --all-namespaces"
+  command: "{{ bin_dir }}/kubectl get pod --all-namespaces"
   register: pod_info
   run_once: true
 
 # 只需单节点执行一次
 - name: 运行 flannel网络
-  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/flannel/ && sleep 15"
+  shell: "{{ bin_dir }}/kubectl create -f /root/local/kube-system/flannel/ && sleep 5"
   run_once: true
   when: '"flannel" not in pod_info.stdout'
 

roles/kube-master/tasks/main.yml

@@ -4,9 +4,6 @@
   - kube-apiserver
   - kube-controller-manager
   - kube-scheduler
-  - kubectl
-  - kube-proxy
-  - kubelet
 
 # 注册变量result，根据result结果判断是否已经生成过 kubernetes证书
 # result|failed 说明没有生成过证书，下一步生成证书
@@ -43,29 +40,15 @@
 - name: 创建kube-scheduler的systemd unit文件
   template: src=kube-scheduler.service.j2 dest=/etc/systemd/system/kube-scheduler.service
 
-- name: daemon-reload
-  shell: systemctl daemon-reload
+- name: enable master 服务
+  shell: systemctl enable kube-apiserver kube-controller-manager kube-scheduler
 
-- name: enable-kube-apiserver
-  shell: systemctl enable kube-apiserver
-
-- name: enable-kube-controller-manager
-  shell: systemctl enable kube-controller-manager
-
-- name: enable-kube-scheduler
-  shell: systemctl enable kube-scheduler
-
-- name: start-kube-apiserver
-  shell: systemctl restart kube-apiserver
-
-- name: start-kube-controller-manager
-  shell: systemctl restart kube-controller-manager
-
-- name: start-kube-scheduler
-  shell: systemctl restart kube-scheduler
+- name: 启动 master 服务
+  shell: "systemctl daemon-reload && systemctl restart kube-apiserver && \
+	systemctl restart kube-controller-manager && systemctl restart kube-scheduler"
 
 - name: 以轮询的方式等待master服务启动完成
-  command: "kubectl get node"
+  command: "{{ bin_dir }}/kubectl get node"
   register: result
   until:    result.rc == 0
   retries:  5

roles/kube-node/tasks/main.yml

@@ -15,41 +15,22 @@
   - host-local
   - loopback
 
-- name: get clusterrolebinding info
-  command: "kubectl get clusterrolebinding --all-namespaces"
-  register: clusterrolebinding_info
-  run_once: true
-  
 ##----------kubelet 配置部分--------------
 # kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要绑定该角色
 # 只需单节点执行一次
+- name: get clusterrolebinding info
+  shell: "{{ bin_dir }}/kubectl get clusterrolebinding --all-namespaces"
+  register: clusterrolebinding_info
+  run_once: true
+  
 - name: kubelet-bootstrap-setting
   shell: "{{ bin_dir }}/kubectl create clusterrolebinding kubelet-bootstrap \
         --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap"
   run_once: True
   when: '"kubelet-bootstrap" not in clusterrolebinding_info.stdout'
 
-#创建bootstrap.kubeconfig配置文件
-- name: 设置集群参数
-  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
-        --certificate-authority={{ ca_dir }}/ca.pem \
-        --embed-certs=true \
-        --server={{ KUBE_APISERVER }} \
-        --kubeconfig=bootstrap.kubeconfig"
-- name: 设置客户端认证参数
-  shell: "{{ bin_dir }}/kubectl config set-credentials kubelet-bootstrap \
-        --token={{ BOOTSTRAP_TOKEN }} \
-        --kubeconfig=bootstrap.kubeconfig"
-- name: 设置上下文参数
-  shell: "{{ bin_dir }}/kubectl config set-context default \
-	--cluster=kubernetes \
-	--user=kubelet-bootstrap \
-	--kubeconfig=bootstrap.kubeconfig"
-- name: 选择默认上下文
-  shell: "{{ bin_dir }}/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig"
-
 - name: 安装bootstrap.kubeconfig配置文件
-  shell: "mv $HOME/bootstrap.kubeconfig /etc/kubernetes/bootstrap.kubeconfig"
+  copy: src=/etc/kubernetes/bootstrap.kubeconfig dest=/etc/kubernetes/bootstrap.kubeconfig  
 
 - name: 准备 cni配置文件
   template: src=cni-default.conf.j2 dest=/etc/cni/net.d/10-default.conf
@@ -62,45 +43,9 @@
   shell: systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet
   tags: kubelet 
 
-- name: approve-kubelet-csr
-  shell: "{{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs {{ bin_dir }}/kubectl certificate approve"
-  run_once: true
-  ignore_errors: true
-
 ##-------kube-proxy部分----------------
-- name: 准备kube-proxy 证书签名请求
-  template: src=kube-proxy-csr.json.j2 dest={{ ca_dir }}/kube-proxy-csr.json
-
-- name: 创建 kube-proxy证书与私钥
-  shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \
-        -ca={{ ca_dir }}/ca.pem \
-        -ca-key={{ ca_dir }}/ca-key.pem \
-        -config={{ ca_dir }}/ca-config.json \
-        -profile=kubernetes kube-proxy-csr.json | {{ bin_dir }}/cfssljson -bare kube-proxy"
-
-#创建kube-proxy.kubeconfig配置文件
-- name: 设置集群参数
-  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
-        --certificate-authority={{ ca_dir }}/ca.pem \
-        --embed-certs=true \
-        --server={{ KUBE_APISERVER }} \
-        --kubeconfig=kube-proxy.kubeconfig"
-- name: 设置客户端认证参数
-  shell: "{{ bin_dir }}/kubectl config set-credentials kube-proxy \
-	--client-certificate={{ ca_dir }}/kube-proxy.pem \
-	--client-key={{ ca_dir }}/kube-proxy-key.pem \
-        --embed-certs=true \
-        --kubeconfig=kube-proxy.kubeconfig"
-- name: 设置上下文参数
-  shell: "{{ bin_dir }}/kubectl config set-context default \
-        --cluster=kubernetes \
-        --user=kube-proxy \
-        --kubeconfig=kube-proxy.kubeconfig"
-- name: 选择默认上下文
-  shell: "{{ bin_dir }}/kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig"
-
 - name: 安装kube-proxy.kubeconfig配置文件
-  shell: "mv $HOME/kube-proxy.kubeconfig /etc/kubernetes/kube-proxy.kubeconfig"
+  copy: src=/etc/kubernetes/kube-proxy.kubeconfig dest=/etc/kubernetes/kube-proxy.kubeconfig
 
 - name: 创建kube-proxy 服务文件
   tags: reload-kube-proxy
@@ -110,3 +55,9 @@
   tags: reload-kube-proxy
   shell: systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy
 
+# 批准 node 节点
+- name: approve-kubelet-csr
+  shell: "sleep 10 && {{ bin_dir }}/kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| \
+	xargs {{ bin_dir }}/kubectl certificate approve"
+  run_once: true
+  ignore_errors: true

roles/kubectl/tasks/main.yml

@@ -1,29 +0,0 @@
-- name: 下载kubectl二进制
-  copy: src={{ base_dir }}/bin/kubectl dest={{ bin_dir }}/kubectl mode=0755
-
-- name: 准备kubectl使用的admin 证书签名请求
-  template: src=admin-csr.json.j2 dest={{ ca_dir }}/admin-csr.json
-
-- name: 创建 admin证书与私钥
-  shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \
-        -ca={{ ca_dir }}/ca.pem \
-        -ca-key={{ ca_dir }}/ca-key.pem \
-        -config={{ ca_dir }}/ca-config.json \
-        -profile=kubernetes admin-csr.json | {{ bin_dir }}/cfssljson -bare admin"
-
-# 创建kubectl kubeconfig 文件
-- name: 设置集群参数
-  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
-	--certificate-authority={{ ca_dir }}/ca.pem \
-	--embed-certs=true \
-	--server={{ KUBE_APISERVER }}"
-- name: 设置客户端认证参数
-  shell: "{{ bin_dir }}/kubectl config set-credentials admin \
-	--client-certificate={{ ca_dir }}/admin.pem \
-	--embed-certs=true \
-	--client-key={{ ca_dir }}/admin-key.pem"
-- name: 设置上下文参数
-  shell: "{{ bin_dir }}/kubectl config set-context kubernetes \
-	--cluster=kubernetes --user=admin"
-- name: 选择默认上下文
-  shell: "{{ bin_dir }}/kubectl config use-context kubernetes"

roles/prepare/tasks/main.yml

@@ -6,22 +6,23 @@
   - /root/.kube
   - /etc/docker
 
-#- name: 集群hosts文件更新
-#  copy: src=hosts.j2 dest=/etc/hosts
-
 - name: 写入环境变量$PATH 
   shell: "sed -i '/export PATH=/d' /etc/profile && \
 	echo export PATH={{ bin_dir }}:$PATH >> /etc/profile"
 
-- name: 下载证书工具 CFSSL
+- name: 下载证书工具 CFSSL和 kubectl
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
   with_items:
   - cfssl
   - cfssl-certinfo
   - cfssljson
+  - kubectl
+
+- name: 安装kubeconfig配置文件
+  copy: src=/root/.kube/config dest=/root/.kube/config
 
 - name: 分发CA 证书
-  copy: src={{ item }} dest={{ ca_dir }}/{{ item }} mode=0644
+  copy: src={{ ca_dir }}/{{ item }} dest={{ ca_dir }}/{{ item }} mode=0644
   with_items:
   - ca.pem
   - ca-key.pem