add support for k8s v1.16

manifests/dashboard/kubernetes-dashboard.yaml

@@ -90,7 +90,7 @@ subjects:
 # ------------------- Dashboard Deployment ------------------- #
 
 kind: Deployment
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 metadata:
   labels:
     k8s-app: kubernetes-dashboard

manifests/heapster/heapster-only/heapster.yaml

@@ -20,7 +20,7 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 ---
 
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: heapster

manifests/ingress/nginx-ingress/nginx-ingress.yaml

@@ -45,7 +45,7 @@ metadata:
     app.kubernetes.io/part-of: ingress-nginx
 
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: nginx-ingress-clusterrole
@@ -101,7 +101,7 @@ rules:
       - update
 
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: nginx-ingress-role
@@ -146,7 +146,7 @@ rules:
       - get
 
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: nginx-ingress-role-nisa-binding
@@ -164,7 +164,7 @@ subjects:
     namespace: ingress-nginx
 
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: nginx-ingress-clusterrole-nisa-binding
@@ -182,7 +182,7 @@ subjects:
 
 ---
 
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1 
 kind: Deployment
 metadata:
   name: nginx-ingress-controller

manifests/ingress/test-hello.ing.yaml

@@ -1,5 +1,5 @@
 # kubectl run test-hello --image=nginx --expose --port=80
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1 
 kind: Ingress
 metadata:
   name: test-hello

manifests/ingress/traefik/tls/hello-tls.ing.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1 
 kind: Ingress
 metadata:
   name: hello-tls-ingress

manifests/ingress/traefik/tls/k8s-dashboard.ing.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1 
 kind: Ingress
 metadata:
   name:  kubernetes-dashboard

manifests/ingress/traefik/tls/traefik-controller.yaml

@@ -25,7 +25,7 @@ data:
             KeyFile = "/ssl/tls.key"
 ---
 kind: Deployment
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 metadata:
   name: traefik-ingress-controller
   namespace: kube-system

manifests/ingress/traefik/traefik-ingress.yaml

@@ -44,7 +44,7 @@ metadata:
   namespace: kube-system
 ---
 kind: Deployment
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 metadata:
   name: traefik-ingress-controller
   namespace: kube-system

manifests/ingress/traefik/traefik-ui.ing.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
   name: traefik-web-ui

manifests/ingress/whoami.ing.yaml

@@ -1,5 +1,5 @@
 # kubectl run whoami --image=emilevauge/whoami --port=80 --expose
-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1beta1 
 kind: Ingress
 metadata:
   name: test-whoami

manifests/metrics-server/auth-delegator.yaml

@@ -1,8 +1,10 @@
----
+apiVersion: rbac.authorization.k8s.io/v1
-apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
 metadata:
   name: metrics-server:system:auth-delegator
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

manifests/metrics-server/auth-reader.yaml

@@ -1,9 +1,11 @@
----
+apiVersion: rbac.authorization.k8s.io/v1
-apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: RoleBinding
 metadata:
   name: metrics-server-auth-reader
   namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

manifests/metrics-server/metrics-apiservice.yaml

@@ -1,8 +1,10 @@
----
+apiVersion: apiregistration.k8s.io/v1
-apiVersion: apiregistration.k8s.io/v1beta1
 kind: APIService
 metadata:
   name: v1beta1.metrics.k8s.io
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   service:
     name: metrics-server

manifests/metrics-server/metrics-server-service.yaml

@@ -1,16 +1,16 @@
----
 apiVersion: v1
 kind: Service
 metadata:
   name: metrics-server
   namespace: kube-system
   labels:
-    kubernetes.io/name: "Metrics-server"
+    addonmanager.kubernetes.io/mode: Reconcile
     kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Metrics-server"
 spec:
   selector:
     k8s-app: metrics-server
   ports:
   - port: 443
     protocol: TCP
-    targetPort: 443
+    targetPort: https

manifests/metrics-server/resource-reader.yaml

@@ -1,15 +1,16 @@
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: system:metrics-server
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 rules:
 - apiGroups:
   - ""
   resources:
   - pods
   - nodes
-  - nodes/stats
   - namespaces
   verbs:
   - get
@@ -22,12 +23,16 @@ rules:
   verbs:
   - get
   - list
+  - update
   - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: system:metrics-server
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole

roles/cluster-addon/templates/kubedns.yaml.j2

@@ -41,7 +41,7 @@ spec:
     protocol: TCP
 
 ---
-apiVersion: apps/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-dns

roles/flannel/templates/kube-flannel.yaml.j2

@@ -1,9 +1,60 @@
 ---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: psp.flannel.unprivileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
+    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
+spec:
+  privileged: false
+  volumes:
+    - configMap
+    - secret
+    - emptyDir
+    - hostPath
+  allowedHostPaths:
+    - pathPrefix: "/etc/cni/net.d"
+    - pathPrefix: "/etc/kube-flannel"
+    - pathPrefix: "/run/flannel"
+  readOnlyRootFilesystem: false
+  # Users and groups
+  runAsUser:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  fsGroup:
+    rule: RunAsAny
+  # Privilege Escalation
+  allowPrivilegeEscalation: false
+  defaultAllowPrivilegeEscalation: false
+  # Capabilities
+  allowedCapabilities: ['NET_ADMIN']
+  defaultAddCapabilities: []
+  requiredDropCapabilities: []
+  # Host namespaces
+  hostPID: false
+  hostIPC: false
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  # SELinux
+  seLinux:
+    # SELinux is unsed in CaaSP
+    rule: 'RunAsAny'
+---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 rules:
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['psp.flannel.unprivileged']
   - apiGroups:
       - ""
     resources:
@@ -25,7 +76,7 @@ rules:
       - patch
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 roleRef:
@@ -55,6 +106,7 @@ data:
   cni-conf.json: |
     {
       "name": "cbr0",
+      "cniVersion": "0.3.1",
       "plugins": [
         {
           "type": "flannel",
@@ -82,7 +134,7 @@ data:
       }
     }
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-flannel-ds-amd64
@@ -91,15 +143,29 @@ metadata:
     tier: node
     app: flannel
 spec:
+  selector:
+    matchLabels:
+      app: flannel
   template:
     metadata:
       labels:
         tier: node
         app: flannel
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: beta.kubernetes.io/os
+                    operator: In
+                    values:
+                      - linux
+                  - key: beta.kubernetes.io/arch
+                    operator: In
+                    values:
+                      - amd64
       hostNetwork: true
-      nodeSelector:
-        beta.kubernetes.io/arch: amd64
       tolerations:
       - operator: Exists
         effect: NoSchedule
@@ -134,7 +200,9 @@ spec:
             cpu: "100m"
             memory: "50Mi"
         securityContext:
-          privileged: true
+          privileged: false
+          capabilities:
+             add: ["NET_ADMIN"]
         env:
         - name: POD_NAME
           valueFrom:
@@ -146,13 +214,13 @@ spec:
               fieldPath: metadata.namespace
         volumeMounts:
         - name: run
-          mountPath: /run
+          mountPath: /run/flannel
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       volumes:
         - name: run
           hostPath:
-            path: /run
+            path: /run/flannel
         - name: cni
           hostPath:
             path: /etc/cni/net.d

roles/kube-master/tasks/main.yml

@@ -57,17 +57,6 @@
   - kube-scheduler.service
   tags: restart_master, upgrade_k8s
 
-# 为兼容v1.8版本，配置不同 kube-apiserver的systemd unit文件
-- name: 获取 k8s 版本信息
-  shell: "{{ bin_dir }}/kube-apiserver --version"
-  register: k8s_ver
-  tags: restart_master, upgrade_k8s
-
-- name: 创建kube-apiserver v1.8的systemd unit文件
-  template: src=kube-apiserver-v1.8.service.j2 dest=/etc/systemd/system/kube-apiserver.service
-  tags: restart_master, upgrade_k8s
-  when: "'v1.8' in k8s_ver.stdout"
-
 - name: enable master 服务
   shell: systemctl enable kube-apiserver kube-controller-manager kube-scheduler
   ignore_errors: true

roles/kube-master/templates/kube-apiserver-v1.8.service.j2

@@ -1,54 +0,0 @@
-[Unit]
-Description=Kubernetes API Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-After=network.target
-
-[Service]
-ExecStart={{ bin_dir }}/kube-apiserver \
-  --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \
-  --advertise-address={{ inventory_hostname }} \
-  --bind-address={{ inventory_hostname }} \
-  --insecure-bind-address=127.0.0.1 \
-  --authorization-mode=Node,RBAC \
-  --kubelet-https=true \
-  --kubelet-client-certificate={{ ca_dir }}/admin.pem \
-  --kubelet-client-key={{ ca_dir }}/admin-key.pem \
-  --anonymous-auth=false \
-{% if BASIC_AUTH_ENABLE == "yes" %}
-  --basic-auth-file={{ ca_dir }}/basic-auth.csv \
-{% endif %}
-  --service-cluster-ip-range={{ SERVICE_CIDR }} \
-  --service-node-port-range={{ NODE_PORT_RANGE }} \
-  --tls-cert-file={{ ca_dir }}/kubernetes.pem \
-  --tls-private-key-file={{ ca_dir }}/kubernetes-key.pem \
-  --client-ca-file={{ ca_dir }}/ca.pem \
-  --service-account-key-file={{ ca_dir }}/ca-key.pem \
-  --etcd-cafile={{ ca_dir }}/ca.pem \
-  --etcd-certfile={{ ca_dir }}/kubernetes.pem \
-  --etcd-keyfile={{ ca_dir }}/kubernetes-key.pem \
-  --etcd-servers={{ ETCD_ENDPOINTS }} \
-  --enable-swagger-ui=true \
-  --apiserver-count={% if DEPLOY_MODE == "multi-master" %}{{ groups['kube-master']|length }}{% else %}1{% endif %} \
-  --allow-privileged=true \
-  --audit-log-maxage=30 \
-  --audit-log-maxbackup=3 \
-  --audit-log-maxsize=100 \
-  --audit-log-path=/var/lib/audit.log \
-  --event-ttl=1h \
-  --requestheader-client-ca-file={{ ca_dir }}/ca.pem \
-  --requestheader-allowed-names= \
-  --requestheader-extra-headers-prefix=X-Remote-Extra- \
-  --requestheader-group-headers=X-Remote-Group \
-  --requestheader-username-headers=X-Remote-User \
-  --proxy-client-cert-file={{ ca_dir }}/aggregator-proxy.pem \
-  --proxy-client-key-file={{ ca_dir }}/aggregator-proxy-key.pem \
-  --enable-aggregator-routing=true \
-  --runtime-config=batch/v2alpha1=true \
-  --v=2
-Restart=always
-RestartSec=5
-Type=notify
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kube-node/templates/cni-default.conf.j2

@@ -1,5 +1,6 @@
 {
 	"name": "mynet",
+   "cniVersion": "0.3.1",
 	"type": "bridge",
 	"bridge": "mynet0",
 	"isDefaultGateway": true,