remove kubectl admin kubeconfig to improve security

roles/calico/tasks/calico-rr.yml

@@ -10,14 +10,19 @@
     - name: 显示rr节点
       debug: var="NODE_IPS"
 
-    - name: 配置routeReflectorClusterID 和 node label
+    - name: 配置routeReflectorClusterID
       shell: 'for ip in {{ NODE_IPS }};do \
                 node_name=$({{ bin_dir }}/calicoctl get node -owide|grep " $ip/"|cut -d" " -f1) && \
-                {{ bin_dir }}/kubectl label node "$ip" route-reflector=true --overwrite && \
                 {{ bin_dir }}/calicoctl patch node "$node_name" \
-                -p "{\"spec\": {\"bgp\": {\"routeReflectorClusterID\": \"244.0.0.1\"}}}"; \
+                  -p "{\"spec\": {\"bgp\": {\"routeReflectorClusterID\": \"244.0.0.1\"}}}"; \
              done'
 
+    - name: node label
+      shell: 'for ip in {{ NODE_IPS }};do \
+                {{ base_dir }}/bin/kubectl label node "$ip" route-reflector=true --overwrite;
+             done'
+      connection: local
+
     - name: 配置 calico bgp yaml文件
       template: src={{ item }}.j2 dest=/etc/calico/{{ item }}
       with_items:

roles/calico/tasks/main.yml

@@ -1,20 +1,39 @@
+- block:
+    - name: 创建calico 证书请求
+      template: src=calico-csr.json.j2 dest={{ cluster_dir }}/ssl/calico-csr.json
+
+    - name: 创建 calico证书和私钥
+      shell: "cd {{ cluster_dir }}/ssl && {{ base_dir }}/bin/cfssl gencert \
+            -ca=ca.pem \
+            -ca-key=ca-key.pem \
+            -config=ca-config.json \
+            -profile=kubernetes calico-csr.json|{{ base_dir }}/bin/cfssljson -bare calico"
+
+    - name: get calico-etcd-secrets info
+      shell: "{{ base_dir }}/bin/kubectl get secrets -n kube-system"
+      register: secrets_info
+
+    - name: 创建 calico-etcd-secrets
+      shell: "cd {{ cluster_dir }}/ssl && \
+            {{ base_dir }}/bin/kubectl create secret generic -n kube-system calico-etcd-secrets \
+            --from-file=etcd-ca=ca.pem \
+            --from-file=etcd-key=calico-key.pem \
+            --from-file=etcd-cert=calico.pem"
+      when: '"calico-etcd-secrets" not in secrets_info.stdout'
+
+    - name: 配置 calico DaemonSet yaml文件
+      template: src=calico-{{ calico_ver_main }}.yaml.j2 dest={{ cluster_dir }}/yml/calico.yaml
+
+    - name: 运行 calico网络
+      shell: "{{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/calico.yaml"
+  run_once: true
+  connection: local
+
 - name: 在节点创建相关目录
   file: name={{ item }} state=directory
   with_items:
   - /etc/calico/ssl
 
-- name: 创建calico 证书请求
-  template: src=calico-csr.json.j2 dest={{ cluster_dir }}/ssl/calico-csr.json
-  connection: local
-
-- name: 创建 calico证书和私钥
-  shell: "cd {{ cluster_dir }}/ssl && {{ base_dir }}/bin/cfssl gencert \
-        -ca=ca.pem \
-        -ca-key=ca-key.pem \
-        -config=ca-config.json \
-        -profile=kubernetes calico-csr.json|{{ base_dir }}/bin/cfssljson -bare calico"
-  connection: local
-
 - name: 分发calico证书相关
   copy: src={{ cluster_dir }}/ssl/{{ item }} dest=/etc/calico/ssl/{{ item }}
   with_items:
@@ -22,30 +41,6 @@
   - calico.pem
   - calico-key.pem
 
-- name: get calico-etcd-secrets info
-  shell: "{{ bin_dir }}/kubectl get secrets -n kube-system"
-  register: secrets_info
-  run_once: true
-
-- name: 创建 calico-etcd-secrets
-  shell: "cd /etc/calico/ssl && \
-        {{ bin_dir }}/kubectl create secret generic -n kube-system calico-etcd-secrets \
-        --from-file=etcd-ca=ca.pem \
-        --from-file=etcd-key=calico-key.pem \
-        --from-file=etcd-cert=calico.pem"
-  when: '"calico-etcd-secrets" not in secrets_info.stdout'
-  run_once: true
-
-- name: 配置 calico DaemonSet yaml文件
-  template: src=calico-{{ calico_ver_main }}.yaml.j2 dest={{ cluster_dir }}/yml/calico.yaml
-  run_once: true
-  connection: local
-
-- name: 运行 calico网络
-  shell: "{{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/calico.yaml"
-  run_once: true
-  connection: local
-
 - name: 删除默认cni配置
   file: path=/etc/cni/net.d/10-default.conf state=absent
 
@@ -59,13 +54,14 @@
 - name: 准备 calicoctl配置文件
   template: src=calicoctl.cfg.j2 dest=/etc/calico/calicoctl.cfg
 
-- name: 轮询等待calico-node 运行，视下载镜像速度而定
+- name: 轮询等待calico-node 运行
-  shell: "{{ bin_dir }}/kubectl get pod -n kube-system -o wide|grep 'calico-node'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  shell: "{{ base_dir }}/bin/kubectl get pod -n kube-system -o wide|grep 'calico-node'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
   register: pod_status
   until: pod_status.stdout == "Running"
   retries: 15
   delay: 15
   ignore_errors: true
+  connection: local
 
 - import_tasks: calico-rr.yml
   when: 'CALICO_RR_ENABLED|bool'

roles/calico/templates/calico-v3.19.yaml.j2

@@ -52,7 +52,7 @@ data:
               "type": "k8s"
           },
           "kubernetes": {
-              "kubeconfig": "/root/.kube/config"
+              "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
           }
         },
         {

roles/calico/templates/calico-v3.23.yaml.j2

@@ -55,7 +55,7 @@ data:
               "type": "k8s"
           },
           "kubernetes": {
-              "kubeconfig": "/root/.kube/config"
+              "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
           }
         },
         {

roles/cilium/tasks/main.yml

@@ -28,13 +28,14 @@
   file: path=/etc/cni/net.d/10-default.conf state=absent
 
 # 等待网络插件部署成功，视下载镜像速度而定
-- name: 轮询等待cilium-node 运行，视下载镜像速度而定
+- name: 轮询等待cilium-node 运行
-  shell: "{{ bin_dir }}/kubectl get pod -n kube-system -owide -lk8s-app=cilium|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  shell: "{{ base_dir }}/bin/kubectl get pod -n kube-system -owide -lk8s-app=cilium|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
   register: pod_status
   until: pod_status.stdout == "Running"
   retries: 15
   delay: 8
   ignore_errors: true
+  connection: local
 
 # hubble-relay 可能需要重启一下
 - name: 重启hubble-relay pod

roles/deploy/tasks/main.yml

@@ -27,7 +27,7 @@
   shell: "cd {{ cluster_dir }}/ssl && \
 	 {{ base_dir }}/bin/cfssl gencert -initca ca-csr.json | {{ base_dir }}/bin/cfssljson -bare ca" 
 
-#----------- 创建配置文件: /root/.kube/config
+#----------- 创建配置文件: kubectl.kubeconfig
 - import_tasks: create-kubectl-kubeconfig.yml
   tags: create_kctl_cfg
 

roles/flannel/tasks/main.yml

@@ -20,9 +20,10 @@
   file: path=/etc/cni/net.d/10-default.conf state=absent
 
 - name: 轮询等待flannel 运行，视下载镜像速度而定
-  shell: "{{ bin_dir }}/kubectl get pod -n kube-system -o wide|grep 'flannel'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  shell: "{{ base_dir }}/bin/kubectl get pod -n kube-system -o wide|grep 'flannel'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
   register: pod_status
   until: pod_status.stdout == "Running"
   retries: 15
   delay: 8
   ignore_errors: true
+  connection: local

roles/kube-master/tasks/main.yml

@@ -7,6 +7,15 @@
   - kubectl
   tags: upgrade_k8s
 
+- name: 分发 kubeconfig配置文件
+  copy: src={{ cluster_dir }}/kubectl.kubeconfig dest=/root/.kube/config mode=0400
+
+- name: 分发controller/scheduler kubeconfig配置文件
+  copy: src={{ cluster_dir }}/{{ item }} dest=/etc/kubernetes/{{ item }}
+  with_items:
+  - kube-controller-manager.kubeconfig
+  - kube-scheduler.kubeconfig
+
 - name: 注册变量 KUBERNETES_SVC_IP
   shell: echo {{ SERVICE_CIDR }}|cut -d/ -f1|awk -F. '{print $1"."$2"."$3"."$4+1}'
   register: KUBERNETES_SVC_IP

roles/kube-node/tasks/create-kubelet-kubeconfig.yml

@@ -1,13 +1,37 @@
-- name: 准备kubelet 证书签名请求
+- block:
-  template: src=kubelet-csr.json.j2 dest={{ cluster_dir }}/ssl/{{ inventory_hostname }}-kubelet-csr.json
+    - name: 准备kubelet 证书签名请求
-  connection: local
+      template: src=kubelet-csr.json.j2 dest={{ cluster_dir }}/ssl/{{ inventory_hostname }}-kubelet-csr.json
 
-- name: 创建 kubelet 证书与私钥
+    - name: 创建 kubelet 证书与私钥
-  shell: "cd {{ cluster_dir }}/ssl && {{ base_dir }}/bin/cfssl gencert \
+      shell: "cd {{ cluster_dir }}/ssl && {{ base_dir }}/bin/cfssl gencert \
-        -ca=ca.pem \
+            -ca=ca.pem \
-        -ca-key=ca-key.pem \
+            -ca-key=ca-key.pem \
-        -config=ca-config.json \
+            -config=ca-config.json \
-        -profile=kubernetes {{ inventory_hostname }}-kubelet-csr.json | {{ base_dir }}/bin/cfssljson -bare {{ inventory_hostname }}-kubelet"
+            -profile=kubernetes {{ inventory_hostname }}-kubelet-csr.json | {{ base_dir }}/bin/cfssljson -bare {{ inventory_hostname }}-kubelet"
+
+    - name: 设置集群参数
+      shell: "{{ base_dir }}/bin/kubectl config set-cluster kubernetes \
+            --certificate-authority={{ cluster_dir }}/ssl/ca.pem \
+            --embed-certs=true \
+            --server={{ KUBE_APISERVER }} \
+            --kubeconfig={{ cluster_dir }}/{{ inventory_hostname }}-kubelet.kubeconfig"
+
+    - name: 设置客户端认证参数
+      shell: "{{ base_dir }}/bin/kubectl config set-credentials system:node:{{ inventory_hostname }} \
+            --client-certificate={{ cluster_dir }}/ssl/{{ inventory_hostname }}-kubelet.pem \
+            --embed-certs=true \
+            --client-key={{ cluster_dir }}/ssl/{{ inventory_hostname }}-kubelet-key.pem \
+            --kubeconfig={{ cluster_dir }}/{{ inventory_hostname }}-kubelet.kubeconfig"
+
+    - name: 设置上下文参数
+      shell: "{{ base_dir }}/bin/kubectl config set-context default \
+            --cluster=kubernetes \
+            --user=system:node:{{ inventory_hostname }} \
+            --kubeconfig={{ cluster_dir }}/{{ inventory_hostname }}-kubelet.kubeconfig"
+
+    - name: 选择默认上下文
+      shell: "{{ base_dir }}/bin/kubectl config use-context default \
+            --kubeconfig={{ cluster_dir }}/{{ inventory_hostname }}-kubelet.kubeconfig"
   connection: local
 
 - name: 分发ca 证书
@@ -19,27 +43,5 @@
   - kubelet.pem
   - kubelet-key.pem
 
-# 创建kubelet.kubeconfig
+- name: 分发kubeconfig
-- name: 设置集群参数
+  copy: src={{ cluster_dir }}/{{ inventory_hostname }}-kubelet.kubeconfig dest=/etc/kubernetes/kubelet.kubeconfig
-  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
-        --certificate-authority={{ ca_dir }}/ca.pem \
-        --embed-certs=true \
-        --server={{ KUBE_APISERVER }} \
-        --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
-
-- name: 设置客户端认证参数
-  shell: "{{ bin_dir }}/kubectl config set-credentials system:node:{{ inventory_hostname }} \
-        --client-certificate={{ ca_dir }}/kubelet.pem \
-        --embed-certs=true \
-        --client-key={{ ca_dir }}/kubelet-key.pem \
-        --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
-
-- name: 设置上下文参数
-  shell: "{{ bin_dir }}/kubectl config set-context default \
-        --cluster=kubernetes \
-        --user=system:node:{{ inventory_hostname }} \
-        --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"
-
-- name: 选择默认上下文
-  shell: "{{ bin_dir }}/kubectl config use-context default \
-        --kubeconfig=/etc/kubernetes/kubelet.kubeconfig"

roles/kube-node/tasks/main.yml

@@ -3,7 +3,6 @@
   with_items:
   - /var/lib/kubelet
   - /var/lib/kube-proxy
-  - /etc/cni/net.d
 
 - name: 下载 kubelet,kube-proxy 二进制和基础 cni plugins
   copy: src={{ base_dir }}/bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
@@ -16,12 +15,6 @@
   - loopback
   tags: upgrade_k8s
 
-- name: 替换 kubeconfig 的 apiserver 地址
-  lineinfile:
-    dest: /root/.kube/config
-    regexp: "^    server"
-    line: "    server: {{ KUBE_APISERVER }}"
-
 ##----------kubelet 配置部分--------------
 # 创建 kubelet 相关证书及 kubelet.kubeconfig
 - import_tasks: create-kubelet-kubeconfig.yml
@@ -53,6 +46,9 @@
   tags: upgrade_k8s, restart_node
 
 ##-------kube-proxy部分----------------
+- name: 分发 kube-proxy.kubeconfig配置文件
+  copy: src={{ cluster_dir }}/kube-proxy.kubeconfig dest=/etc/kubernetes/kube-proxy.kubeconfig
+
 - name: 替换 kube-proxy.kubeconfig 的 apiserver 地址
   lineinfile:
     dest: /etc/kubernetes/kube-proxy.kubeconfig
@@ -94,13 +90,15 @@
   tags: reload-kube-proxy, upgrade_k8s, restart_node
 
 - name: 轮询等待node达到Ready状态
-  shell: "{{ bin_dir }}/kubectl get node {{ inventory_hostname }}|awk 'NR>1{print $2}'"
+  shell: "{{ base_dir }}/bin/kubectl get node {{ inventory_hostname }}|awk 'NR>1{print $2}'"
   register: node_status
   until: node_status.stdout == "Ready" or node_status.stdout == "Ready,SchedulingDisabled"
   retries: 8 
   delay: 8
   tags: upgrade_k8s, restart_node
+  connection: local
 
 - name: 设置node节点role
-  shell: "{{ bin_dir }}/kubectl label node {{ inventory_hostname }} kubernetes.io/role=node --overwrite"
+  shell: "{{ base_dir }}/bin/kubectl label node {{ inventory_hostname }} kubernetes.io/role=node --overwrite"
   ignore_errors: true
+  connection: local

roles/kube-ovn/tasks/main.yml

@@ -1,45 +1,39 @@
-- name: 创建相关目录
+- block:
-  file: name={{ item }} state=directory
+    - name: 注册变量 ovn_default_gateway
-  with_items:
+      shell: echo {{ CLUSTER_CIDR }}|cut -d/ -f1|awk -F. '{print $1"."$2"."$3"."$4+1}'
-  - /etc/cni/net.d
+      register: ovn_default_gateway
-  - /opt/kube/kube-ovn
 
-- name: 注册变量 ovn_default_gateway
+    - name: 设置变量 kube_ovn_default_gateway
-  shell: echo {{ CLUSTER_CIDR }}|cut -d/ -f1|awk -F. '{print $1"."$2"."$3"."$4+1}'
+      set_fact: kube_ovn_default_gateway={{ ovn_default_gateway.stdout }}
-  register: ovn_default_gateway
 
-- name: 设置变量 kube_ovn_default_gateway
+    - name: 创建配置文件
-  set_fact: kube_ovn_default_gateway={{ ovn_default_gateway.stdout }}
+      template: src={{ item }}.j2 dest={{ cluster_dir }}/yml/{{ item }}
+      with_items:
+      - crd.yaml
+      - kube-ovn.yaml
+      - ovn.yaml
 
-- name: 配置 crd.yaml 文件
+    - name: 运行 kube-ovn网络
-  template: src=crd.yaml.j2 dest=/opt/kube/kube-ovn/crd.yaml
+      shell: "{{ base_dir }}/bin/kubectl label node {{ OVN_DB_NODE }} kube-ovn/role=master --overwrite && \
-
+        {{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/crd.yaml && sleep 5 && \
-- name: 配置 kube-ovn.yaml 文件
+        {{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/ovn.yaml && sleep 5 && \
-  template: src=kube-ovn.yaml.j2 dest=/opt/kube/kube-ovn/kube-ovn.yaml
+        {{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/kube-ovn.yaml"
-
-- name: 配置 ovn.yaml 文件
-  template: src=ovn.yaml.j2 dest=/opt/kube/kube-ovn/ovn.yaml
-
-- name: 配置 kubectl plugin
-  template: src=kubectl-ko.j2 dest=/usr/local/bin/kubectl-ko mode=0755
-
-# 只需单节点执行一次
-- name: 运行 kube-ovn网络
-  shell: "{{ bin_dir }}/kubectl label node {{ OVN_DB_NODE }} kube-ovn/role=master --overwrite && \
-    {{ bin_dir }}/kubectl apply -f /opt/kube/kube-ovn/crd.yaml && sleep 5 && \
-	{{ bin_dir }}/kubectl apply -f /opt/kube/kube-ovn/ovn.yaml && sleep 5 && \
-	{{ bin_dir }}/kubectl apply -f /opt/kube/kube-ovn/kube-ovn.yaml"
   run_once: true
+  connection: local
 
 # 删除原有cni配置
 - name: 删除默认cni配置
   file: path=/etc/cni/net.d/10-default.conf state=absent
 
+- name: 配置 kubectl plugin
+  template: src=kubectl-ko.j2 dest=/usr/local/bin/kubectl-ko mode=0755
+
 # 等待网络插件部署成功，视下载镜像速度而定
 - name: 轮询等待kube-ovn 运行，视下载镜像速度而定
-  shell: "{{ bin_dir }}/kubectl get pod -n kube-ovn -o wide|grep 'kube-ovn-cni'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  shell: "{{ base_dir }}/bin/kubectl get pod -n kube-ovn -o wide|grep 'kube-ovn-cni'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
   register: pod_status
   until: pod_status.stdout == "Running"
   retries: 15
   delay: 8
   ignore_errors: true
+  connection: local

roles/kube-router/tasks/main.yml

@@ -23,9 +23,10 @@
 
 # 等待网络插件部署成功，视下载镜像速度而定
 - name: 轮询等待kube-router 运行，视下载镜像速度而定
-  shell: "{{ bin_dir }}/kubectl get pod -n kube-system -o wide|grep 'kube-router'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
+  shell: "{{ base_dir }}/bin/kubectl get pod -n kube-system -o wide|grep 'kube-router'|grep ' {{ inventory_hostname }} '|awk '{print $3}'"
   register: pod_status
   until: pod_status.stdout == "Running"
   retries: 15
   delay: 8
   ignore_errors: true
+  connection: local

roles/prepare/tasks/main.yml

@@ -59,18 +59,3 @@
     state: present
     regexp: 'easzlab.io.local'
     line: "{{ ansible_env.SSH_CLIENT.split(' ')[0] }}    easzlab.io.local"
-
-- block:
-    - name: 分发 kubeconfig配置文件
-      copy: src={{ cluster_dir }}/kubectl.kubeconfig dest=/root/.kube/config mode=0400
-
-    - name: 分发 kube-proxy.kubeconfig配置文件
-      copy: src={{ cluster_dir }}/kube-proxy.kubeconfig dest=/etc/kubernetes/kube-proxy.kubeconfig
-
-    - name: 分发controller/scheduler kubeconfig配置文件
-      copy: src={{ cluster_dir }}/{{ item }} dest=/etc/kubernetes/{{ item }}
-      with_items:
-      - kube-controller-manager.kubeconfig
-      - kube-scheduler.kubeconfig
-      when: "inventory_hostname in groups['kube_master']"
-  when: "inventory_hostname in groups['kube_master'] or inventory_hostname in groups['kube_node']"