From a3e6896a43195fa49141a4f00d1bdf0a70ce20f4 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Mon, 4 Sep 2017 11:29:40 +0300
Subject: [PATCH] Add RBAC support for canal (#1604)

Refactored how rbac_enabled is set
Added RBAC to ubuntu-canal-ha CI job
Added rbac for calico policy controller
---
 .gitlab-ci.yml                                | 11 +--
 .../netchecker-server-deployment.yml.j2       |  4 +-
 .../network_plugin/canal/tasks/main.yml       | 25 ++----
 .../calico/defaults/main.yml                  |  5 ++
 .../policy_controller/calico/tasks/main.yml   | 49 +++++++++---
 .../templates/calico-policy-controller.yml.j2 |  9 ++-
 .../calico/templates/calico-policy-cr.yml.j2  | 17 ++++
 .../calico/templates/calico-policy-crb.yml.j2 | 13 +++
 .../calico/templates/calico-policy-sa.yml.j2  |  8 ++
 roles/network_plugin/canal/defaults/main.yml  |  5 ++
 roles/network_plugin/canal/tasks/main.yml     | 24 +++---
 ...nal-config.yml.j2 => canal-config.yaml.j2} |  0
 .../canal/templates/canal-cr-calico.yml.j2    | 80 +++++++++++++++++++
 .../canal/templates/canal-cr-flannel.yml.j2   | 26 ++++++
 .../canal/templates/canal-crb-calico.yml.j2   | 14 ++++
 .../canal/templates/canal-crb-flannel.yml.j2  | 14 ++++
 .../canal/templates/canal-node-sa.yml.j2      |  9 +++
 .../{canal-node.yml.j2 => canal-node.yaml.j2} |  7 ++
 18 files changed, 274 insertions(+), 46 deletions(-)
 create mode 100644 roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2
 create mode 100644 roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2
 create mode 100644 roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2
 rename roles/network_plugin/canal/templates/{canal-config.yml.j2 => canal-config.yaml.j2} (100%)
 create mode 100644 roles/network_plugin/canal/templates/canal-cr-calico.yml.j2
 create mode 100644 roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
 create mode 100644 roles/network_plugin/canal/templates/canal-crb-calico.yml.j2
 create mode 100644 roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2
 create mode 100644 roles/network_plugin/canal/templates/canal-node-sa.yml.j2
 rename roles/network_plugin/canal/templates/{canal-node.yml.j2 => canal-node.yaml.j2} (97%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6a456f9df..17851b19c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -269,9 +269,10 @@ before_script:
   ##User-data to simply turn off coreos upgrades
   STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   CLOUD_IMAGE: ubuntu-1604-xenial
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
@@ -445,24 +446,24 @@ ubuntu-weave-sep-triggers:
   only: ['triggers']
 
 # More builds for PRs/merges (manual) and triggers (auto)
-ubuntu-canal-ha:
+ubuntu-canal-ha-rbac:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-ubuntu-canal-ha-triggers:
+ubuntu-canal-ha-rbac-triggers:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: on_success
   only: ['triggers']
 
diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2
index c3dbf3cb5..6e2738e6f 100644
--- a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2
@@ -25,12 +25,14 @@ spec:
               memory: {{ netchecker_server_memory_requests }}
           ports:
             - containerPort: 8081
-              hostPort: 8081
           args:
             - "-v=5"
             - "-logtostderr"
             - "-kubeproxyinit"
             - "-endpoint=0.0.0.0:8081"
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
 {% if rbac_enabled %}
       serviceAccountName: netchecker-server
 {% endif %}
diff --git a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml
index 72956dac9..6f3bb4d85 100644
--- a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml
+++ b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml
@@ -1,20 +1,11 @@
 ---
-- name: Create canal ConfigMap
-  run_once: true
+- name: Canal | Start Resources
   kube:
-    name: "canal-config"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-config.yaml"
-    resource: "configmap"
-    namespace: "{{system_namespace}}"
-
-- name: Start flannel and calico-node
-  run_once: true
-  kube:
-    name: "canal-node"
-    kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-node.yaml"
-    resource: "ds"
-    namespace: "{{system_namespace}}"
-    state: "{{ item | ternary('latest','present') }}"
-  with_items: "{{ canal_node_manifest.changed }}"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ canal_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]
diff --git a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml
index 93d12c901..0e66359cc 100644
--- a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml
+++ b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml
@@ -8,3 +8,8 @@ calico_policy_controller_memory_requests: 64M
 # SSL
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding
diff --git a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
index de102f31d..79bb535b7 100644
--- a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
+++ b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
@@ -1,22 +1,49 @@
 ---
-- set_fact:
+- name: Set cert dir
+  set_fact:
     calico_cert_dir: "{{ canal_cert_dir }}"
   when: kube_network_plugin == 'canal'
   tags: [facts, canal]
 
-- name: Write calico-policy-controller yaml
+- name: Get calico-policy-controller version if running
+  shell: "{{ bin_dir }}/kubectl -n {{ system_namespace }} get rs calico-policy-controller -o=jsonpath='{$.spec.template.spec.containers[:1].image}' | cut -d':' -f2"
+  register: existing_calico_policy_version
+  run_once: true
+  failed_when: false
+
+# FIXME(mattymo): This should not be necessary
+- name: Delete calico-policy-controller if an old one is installed
+  kube:
+    name: calico-policy-controller
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: rs
+    namespace: "{{ system_namespace }}"
+    state: absent
+  run_once: true
+  when:
+    - not "NotFound" in existing_calico_policy_version.stderr
+    - existing_calico_policy_version.stdout | version_compare('v0.7.0', '<')
+
+- name: Create calico-policy-controller manifests
   template:
-    src: calico-policy-controller.yml.j2
-    dest: "{{kube_config_dir}}/calico-policy-controller.yml"
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: canal
+    src: "{{item.file}}.j2"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: calico-policy-controller, file: calico-policy-controller.yml, type: rs}
+    - {name: calico-policy-controller, file: calico-policy-sa.yml, type: sa}
+    - {name: calico-policy-controller, file: calico-policy-cr.yml, type: clusterrole}
+    - {name: calico-policy-controller, file: calico-policy-crb.yml, type: clusterrolebinding}
+  register: calico_policy_manifests
+  when:
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Start of Calico policy controller
   kube:
-    name: "calico-policy-controller"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/calico-policy-controller.yml"
-    namespace: "{{system_namespace}}"
-    resource: "rs"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ calico_policy_manifests.results }}"
   when: inventory_hostname == groups['kube-master'][0]
-  tags: canal
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2
index 4722cbc53..ca1711463 100644
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2
@@ -15,15 +15,18 @@ spec:
   template:
     metadata:
       name: calico-policy-controller
-      namespace: {{system_namespace}}
+      namespace: {{ system_namespace }}
       labels:
         kubernetes.io/cluster-service: "true"
         k8s-app: calico-policy
     spec:
       hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: calico-policy-controller
+{% endif %}
       tolerations:
-       - effect: NoSchedule
-         operator: Exists
+        - effect: NoSchedule
+          operator: Exists
       containers:
         - name: calico-policy-controller
           image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2
new file mode 100644
index 000000000..aac341ca6
--- /dev/null
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2
@@ -0,0 +1,17 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups:
+    - ""
+    - extensions
+    resources:
+      - pods
+      - namespaces
+      - networkpolicies
+    verbs:
+      - watch
+      - list
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2
new file mode 100644
index 000000000..d5c192018
--- /dev/null
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2
@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-policy-controller
+subjects:
+- kind: ServiceAccount
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2
new file mode 100644
index 000000000..c6bc07fbb
--- /dev/null
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2
@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"
diff --git a/roles/network_plugin/canal/defaults/main.yml b/roles/network_plugin/canal/defaults/main.yml
index 38696b87a..bf74653c7 100644
--- a/roles/network_plugin/canal/defaults/main.yml
+++ b/roles/network_plugin/canal/defaults/main.yml
@@ -31,3 +31,8 @@ calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
 calicoctl_cpu_requests: 25m
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding
diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml
index ea67e20cd..2cc1a8ffe 100644
--- a/roles/network_plugin/canal/tasks/main.yml
+++ b/roles/network_plugin/canal/tasks/main.yml
@@ -32,16 +32,22 @@
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true
 
-- name: Canal | Write canal configmap
+- name: Canal | Create canal node manifests
   template:
-    src: canal-config.yml.j2
-    dest: "{{kube_config_dir}}/canal-config.yaml"
-
-- name: Canal | Write canal node configuration
-  template:
-    src: canal-node.yml.j2
-    dest: "{{kube_config_dir}}/canal-node.yaml"
-  register: canal_node_manifest
+    src: "{{item.file}}.j2"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: canal-config, file: canal-config.yaml, type: cm}
+    - {name: canal-node, file: canal-node.yaml, type: ds}
+    - {name: canal, file: canal-node-sa.yml, type: sa}
+    - {name: calico, file: canal-cr-calico.yml, type: clusterrole}
+    - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
+    - {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding}
+    - {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding}
+  register: canal_manifests
+  when:
+    - inventory_hostname in groups['kube-master']
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Canal | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"
diff --git a/roles/network_plugin/canal/templates/canal-config.yml.j2 b/roles/network_plugin/canal/templates/canal-config.yaml.j2
similarity index 100%
rename from roles/network_plugin/canal/templates/canal-config.yml.j2
rename to roles/network_plugin/canal/templates/canal-config.yaml.j2
diff --git a/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2
new file mode 100644
index 000000000..e3b048c64
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2
@@ -0,0 +1,80 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - thirdpartyresources
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalbgppeers
+    verbs:
+      - get
+      - list
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalconfigs
+      - globalbgpconfigs
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - ippools
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["alpha.projectcalico.org"]
+    resources:
+      - systemnetworkpolicies
+    verbs:
+      - get
+      - list
diff --git a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
new file mode 100644
index 000000000..0be8e938c
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
@@ -0,0 +1,26 @@
+---
+# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
diff --git a/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2
new file mode 100644
index 000000000..e1c1f5050
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2
@@ -0,0 +1,14 @@
+---
+# Bind the calico ClusterRole to the canal ServiceAccount.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: canal-calico
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}
diff --git a/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2
new file mode 100644
index 000000000..3b00017b1
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2
@@ -0,0 +1,14 @@
+---
+# Bind the flannel ClusterRole to the canal ServiceAccount.
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: canal-flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}
diff --git a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2
new file mode 100644
index 000000000..d5b9a6e97
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: canal
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+
diff --git a/roles/network_plugin/canal/templates/canal-node.yml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2
similarity index 97%
rename from roles/network_plugin/canal/templates/canal-node.yml.j2
rename to roles/network_plugin/canal/templates/canal-node.yaml.j2
index cd9312832..972b02d5f 100644
--- a/roles/network_plugin/canal/templates/canal-node.yml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@@ -19,6 +19,9 @@ spec:
         k8s-app: canal-node
     spec:
       hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: canal
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists
@@ -169,6 +172,10 @@ spec:
                 configMapKeyRef:
                   name: canal-config
                   key: etcd_keyfile
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
           securityContext:
             privileged: true
           volumeMounts: