取消 Node节点 Bootstrap机制

example/hosts.allinone.example

@@ -33,9 +33,6 @@ K8S_VER="v1.10"
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
-#TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
-BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
-
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.m-masters.example

@@ -47,9 +47,6 @@ K8S_VER="v1.10"
 MASTER_IP="192.168.1.10"
 KUBE_APISERVER="https://{{ MASTER_IP }}:8443"
 
-#TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
-BOOTSTRAP_TOKEN="c30302226d4b810e08731702d3890f50"
-
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

example/hosts.s-master.example

@@ -34,9 +34,6 @@ K8S_VER="v1.11"
 MASTER_IP="{{ groups['kube-master'][0] }}"
 KUBE_APISERVER="https://{{ MASTER_IP }}:6443"
 
-#TLS Bootstrapping 使用的 Token，使用 head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 生成
-BOOTSTRAP_TOKEN="d18f94b5fa585c7123f56803d925d2e7"
-
 # 集群网络插件，目前支持calico, flannel, kube-router, cilium
 CLUSTER_NETWORK="flannel"
 

roles/deploy/tasks/main.yml

@@ -61,28 +61,6 @@
 - name: 选择默认上下文
   shell: "{{ bin_dir }}/kubectl config use-context kubernetes"
 
-#-------------创建bootstrap.kubeconfig配置文件: /root/bootstrap.kubeconfig
-- name: 设置集群参数
-  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
-        --certificate-authority={{ ca_dir }}/ca.pem \
-        --embed-certs=true \
-        --server={{ KUBE_APISERVER }} \
-        --kubeconfig=bootstrap.kubeconfig"
-- name: 设置客户端认证参数
-  shell: "{{ bin_dir }}/kubectl config set-credentials kubelet-bootstrap \
-        --token={{ BOOTSTRAP_TOKEN }} \
-        --kubeconfig=bootstrap.kubeconfig"
-- name: 设置上下文参数
-  shell: "{{ bin_dir }}/kubectl config set-context default \
-        --cluster=kubernetes \
-        --user=kubelet-bootstrap \
-        --kubeconfig=bootstrap.kubeconfig"
-- name: 选择默认上下文
-  shell: "{{ bin_dir }}/kubectl config use-context default --kubeconfig=bootstrap.kubeconfig"
-
-- name: 移动 bootstrap.kubeconfig
-  shell: "mv /root/bootstrap.kubeconfig /etc/kubernetes/"
-
 #------------创建kube-proxy.kubeconfig配置文件: /root/kube-proxy.kubeconfig
 - name: 准备kube-proxy 证书签名请求
   template: src=kube-proxy-csr.json.j2 dest={{ ca_dir }}/kube-proxy-csr.json

roles/kube-master/tasks/main.yml

@@ -29,9 +29,6 @@
         -profile=kubernetes aggregator-proxy-csr.json | {{ bin_dir }}/cfssljson -bare aggregator-proxy"
   tags: upgrade_k8s
 
-- name: 创建 token.csv
-  template: src=token.csv.j2 dest={{ ca_dir }}/token.csv
-
 - name: 创建 basic-auth.csv
   template: src=basic-auth.csv.j2 dest={{ ca_dir }}/basic-auth.csv
 

roles/kube-master/templates/kube-apiserver-v1.8.service.j2

@@ -14,8 +14,6 @@ ExecStart={{ bin_dir }}/kube-apiserver \
   --kubelet-client-key={{ ca_dir }}/kubernetes-key.pem \
   --anonymous-auth=false \
   --basic-auth-file={{ ca_dir }}/basic-auth.csv \
-  --enable-bootstrap-token-auth \
-  --token-auth-file={{ ca_dir }}/token.csv \
   --service-cluster-ip-range={{ SERVICE_CIDR }} \
   --service-node-port-range={{ NODE_PORT_RANGE }} \
   --tls-cert-file={{ ca_dir }}/kubernetes.pem \

roles/kube-master/templates/kube-apiserver.service.j2

@@ -14,8 +14,6 @@ ExecStart={{ bin_dir }}/kube-apiserver \
   --kubelet-client-key={{ ca_dir }}/kubernetes-key.pem \
   --anonymous-auth=false \
   --basic-auth-file={{ ca_dir }}/basic-auth.csv \
-  --enable-bootstrap-token-auth \
-  --token-auth-file={{ ca_dir }}/token.csv \
   --service-cluster-ip-range={{ SERVICE_CIDR }} \
   --service-node-port-range={{ NODE_PORT_RANGE }} \
   --tls-cert-file={{ ca_dir }}/kubernetes.pem \

roles/kube-master/templates/token.csv.j2

@@ -1 +0,0 @@
-{{ BOOTSTRAP_TOKEN }},kubelet-bootstrap,10001,"system:kubelet-bootstrap"

roles/kube-node/defaults/main.yml

@@ -3,3 +3,6 @@ PROXY_MODE: "iptables"
 
 # Kubelet 根目录
 KUBELET_ROOT_DIR: "/var/lib/kubelet"
+
+# node节点最大pod 数
+MAX_PODS: 110

roles/kube-node/tasks/main.yml

@@ -17,22 +17,43 @@
   tags: upgrade_k8s
 
 ##----------kubelet 配置部分--------------
-# kubelet 启动时向 kube-apiserver 发送 TLS bootstrapping 请求，需要绑定该角色
+- name: 准备kubelet 证书签名请求
-# 只需单节点执行一次
+  template: src=kubelet-csr.json.j2 dest={{ ca_dir }}/kubelet-csr.json
-- name: get clusterrolebinding info
-  shell: "{{ bin_dir }}/kubectl get clusterrolebinding --all-namespaces"
-  register: clusterrolebinding_info
-  run_once: true
-  
-- name: kubelet-bootstrap-setting
-  shell: "{{ bin_dir }}/kubectl create clusterrolebinding kubelet-bootstrap \
-        --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap"
-  run_once: True
-  when: '"kubelet-bootstrap" not in clusterrolebinding_info.stdout'
 
-- name: 安装bootstrap.kubeconfig配置文件
+- name: 创建 kubelet 证书与私钥
-  synchronize: src=/etc/kubernetes/bootstrap.kubeconfig dest=/etc/kubernetes/bootstrap.kubeconfig
+  shell: "cd {{ ca_dir }} && {{ bin_dir }}/cfssl gencert \
-  delegate_to: "{{ groups.deploy[0] }}"
+        -ca={{ ca_dir }}/ca.pem \
+        -ca-key={{ ca_dir }}/ca-key.pem \
+        -config={{ ca_dir }}/ca-config.json \
+        -profile=kubernetes kubelet-csr.json | {{ bin_dir }}/cfssljson -bare kubelet"
+
+# 创建kubelet.kubeconfig 
+- name: 设置集群参数
+  shell: "{{ bin_dir }}/kubectl config set-cluster kubernetes \
+        --certificate-authority={{ ca_dir }}/ca.pem \
+        --embed-certs=true \
+        --server={{ KUBE_APISERVER }} \
+	--kubeconfig=kubelet.kubeconfig"
+
+- name: 设置客户端认证参数
+  shell: "{{ bin_dir }}/kubectl config set-credentials system:node:{{ inventory_hostname }} \
+        --client-certificate={{ ca_dir }}/kubelet.pem \
+        --embed-certs=true \
+        --client-key={{ ca_dir }}/kubelet-key.pem \
+	--kubeconfig=kubelet.kubeconfig"
+
+- name: 设置上下文参数
+  shell: "{{ bin_dir }}/kubectl config set-context default \
+        --cluster=kubernetes \
+	--user=system:node:{{ inventory_hostname }} \
+	--kubeconfig=kubelet.kubeconfig"
+
+- name: 选择默认上下文
+  shell: "{{ bin_dir }}/kubectl config use-context default \
+	--kubeconfig=kubelet.kubeconfig"
+
+- name: 移动 kubelet.kubeconfig
+  shell: "mv /root/kubelet.kubeconfig /etc/kubernetes/"
 
 - name: 准备 cni配置文件
   template: src=cni-default.conf.j2 dest=/etc/cni/net.d/10-default.conf

roles/kube-node/templates/kubelet-csr.json.j2

@@ -0,0 +1,20 @@
+{
+  "CN": "system:node:{{ inventory_hostname }}",
+  "hosts": [
+    "127.0.0.1",
+    "{{ inventory_hostname }}"
+  ],
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "C": "CN",
+      "ST": "HangZhou",
+      "L": "XS",
+      "O": "system:nodes",
+      "OU": "System"
+    }
+  ]
+}

roles/kube-node/templates/kubelet.service.j2

@@ -9,23 +9,24 @@ WorkingDirectory=/var/lib/kubelet
 #--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest
 ExecStart={{ bin_dir }}/kubelet \
   --address={{ inventory_hostname }} \
-  --hostname-override={{ inventory_hostname }} \
+  --allow-privileged=true \
-  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.1 \
+  --anonymous-auth=false \
-  --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \
-  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
-  --cert-dir={{ ca_dir }} \
   --client-ca-file={{ ca_dir }}/ca.pem \
-  --network-plugin=cni \
-  --cni-conf-dir=/etc/cni/net.d \
-  --cni-bin-dir={{ bin_dir }} \
   --cluster-dns={{ CLUSTER_DNS_SVC_IP }} \
   --cluster-domain={{ CLUSTER_DNS_DOMAIN }} \
-  --hairpin-mode hairpin-veth \
+  --cni-bin-dir={{ bin_dir }} \
-  --allow-privileged=true \
+  --cni-conf-dir=/etc/cni/net.d \
   --fail-swap-on=false \
-  --anonymous-auth=false \
+  --hairpin-mode hairpin-veth \
-  --logtostderr=true \
+  --hostname-override={{ inventory_hostname }} \
+  --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
+  --max-pods={{ MAX_PODS }} \
+  --network-plugin=cni \
+  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.1 \
+  --register-node=true \
   --root-dir={{ KUBELET_ROOT_DIR }} \
+  --tls-cert-file={{ ca_dir }}/kubelet.pem \
+  --tls-private-key-file={{ ca_dir }}/kubelet-key.pem \
   --v=2
 #kubelet cAdvisor 默认在所有接口监听 4194 端口的请求, 以下iptables限制内网访问
 ExecStartPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT