Add RBAC support for canal (#1604)

Refactored how rbac_enabled is set
Added RBAC to ubuntu-canal-ha CI job
Added rbac for calico policy controller

.gitlab-ci.yml

@@ -269,9 +269,10 @@ before_script:
   ##User-data to simply turn off coreos upgrades
   STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   CLOUD_IMAGE: ubuntu-1604-xenial
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
@@ -445,24 +446,24 @@ ubuntu-weave-sep-triggers:
   only: ['triggers']
 
 # More builds for PRs/merges (manual) and triggers (auto)
-ubuntu-canal-ha:
+ubuntu-canal-ha-rbac:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-ubuntu-canal-ha-triggers:
+ubuntu-canal-ha-rbac-triggers:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: on_success
   only: ['triggers']
 

roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -25,12 +25,14 @@ spec:
               memory: {{ netchecker_server_memory_requests }}
           ports:
             - containerPort: 8081
-              hostPort: 8081
           args:
             - "-v=5"
             - "-logtostderr"
             - "-kubeproxyinit"
             - "-endpoint=0.0.0.0:8081"
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
 {% if rbac_enabled %}
       serviceAccountName: netchecker-server
 {% endif %}

roles/kubernetes-apps/network_plugin/canal/tasks/main.yml

@@ -1,20 +1,11 @@
 ---
-- name: Create canal ConfigMap
-  run_once: true
+- name: Canal | Start Resources
   kube:
-    name: "canal-config"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-config.yaml"
-    resource: "configmap"
-    namespace: "{{system_namespace}}"
-
-- name: Start flannel and calico-node
-  run_once: true
-  kube:
-    name: "canal-node"
-    kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-node.yaml"
-    resource: "ds"
-    namespace: "{{system_namespace}}"
-    state: "{{ item | ternary('latest','present') }}"
-  with_items: "{{ canal_node_manifest.changed }}"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ canal_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/policy_controller/calico/defaults/main.yml

@@ -8,3 +8,8 @@ calico_policy_controller_memory_requests: 64M
 # SSL
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding

roles/kubernetes-apps/policy_controller/calico/tasks/main.yml

@@ -1,22 +1,49 @@
 ---
-- set_fact:
+- name: Set cert dir
+  set_fact:
     calico_cert_dir: "{{ canal_cert_dir }}"
   when: kube_network_plugin == 'canal'
   tags: [facts, canal]
 
-- name: Write calico-policy-controller yaml
+- name: Get calico-policy-controller version if running
+  shell: "{{ bin_dir }}/kubectl -n {{ system_namespace }} get rs calico-policy-controller -o=jsonpath='{$.spec.template.spec.containers[:1].image}' | cut -d':' -f2"
+  register: existing_calico_policy_version
+  run_once: true
+  failed_when: false
+
+# FIXME(mattymo): This should not be necessary
+- name: Delete calico-policy-controller if an old one is installed
+  kube:
+    name: calico-policy-controller
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: rs
+    namespace: "{{ system_namespace }}"
+    state: absent
+  run_once: true
+  when:
+    - not "NotFound" in existing_calico_policy_version.stderr
+    - existing_calico_policy_version.stdout | version_compare('v0.7.0', '<')
+
+- name: Create calico-policy-controller manifests
   template:
-    src: calico-policy-controller.yml.j2
-    dest: "{{kube_config_dir}}/calico-policy-controller.yml"
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: canal
+    src: "{{item.file}}.j2"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: calico-policy-controller, file: calico-policy-controller.yml, type: rs}
+    - {name: calico-policy-controller, file: calico-policy-sa.yml, type: sa}
+    - {name: calico-policy-controller, file: calico-policy-cr.yml, type: clusterrole}
+    - {name: calico-policy-controller, file: calico-policy-crb.yml, type: clusterrolebinding}
+  register: calico_policy_manifests
+  when:
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Start of Calico policy controller
   kube:
-    name: "calico-policy-controller"
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/calico-policy-controller.yml"
-    namespace: "{{system_namespace}}"
-    resource: "rs"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ calico_policy_manifests.results }}"
   when: inventory_hostname == groups['kube-master'][0]
-  tags: canal

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2

@@ -15,12 +15,15 @@ spec:
   template:
     metadata:
       name: calico-policy-controller
-      namespace: {{system_namespace}}
+      namespace: {{ system_namespace }}
       labels:
         kubernetes.io/cluster-service: "true"
         k8s-app: calico-policy
     spec:
       hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: calico-policy-controller
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-cr.yml.j2

@@ -0,0 +1,17 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups:
+    - ""
+    - extensions
+    resources:
+      - pods
+      - namespaces
+      - networkpolicies
+    verbs:
+      - watch
+      - list

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-crb.yml.j2

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-policy-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-policy-controller
+subjects:
+- kind: ServiceAccount
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-sa.yml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-policy-controller
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/network_plugin/canal/defaults/main.yml

@@ -31,3 +31,8 @@ calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
 calicoctl_cpu_requests: 25m
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding

roles/network_plugin/canal/tasks/main.yml

@@ -32,16 +32,22 @@
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true
 
-- name: Canal | Write canal configmap
+- name: Canal | Create canal node manifests
   template:
-    src: canal-config.yml.j2
-    dest: "{{kube_config_dir}}/canal-config.yaml"
-
-- name: Canal | Write canal node configuration
-  template:
-    src: canal-node.yml.j2
-    dest: "{{kube_config_dir}}/canal-node.yaml"
-  register: canal_node_manifest
+    src: "{{item.file}}.j2"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: canal-config, file: canal-config.yaml, type: cm}
+    - {name: canal-node, file: canal-node.yaml, type: ds}
+    - {name: canal, file: canal-node-sa.yml, type: sa}
+    - {name: calico, file: canal-cr-calico.yml, type: clusterrole}
+    - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
+    - {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding}
+    - {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding}
+  register: canal_manifests
+  when:
+    - inventory_hostname in groups['kube-master']
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Canal | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"

roles/network_plugin/canal/templates/canal-cr-calico.yml.j2

@@ -0,0 +1,80 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - thirdpartyresources
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalbgppeers
+    verbs:
+      - get
+      - list
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalconfigs
+      - globalbgpconfigs
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - ippools
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["alpha.projectcalico.org"]
+    resources:
+      - systemnetworkpolicies
+    verbs:
+      - get
+      - list

roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2

@@ -0,0 +1,26 @@
+---
+# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch

roles/network_plugin/canal/templates/canal-crb-calico.yml.j2

@@ -0,0 +1,14 @@
+---
+# Bind the calico ClusterRole to the canal ServiceAccount.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: canal-calico
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}

roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2

@@ -0,0 +1,14 @@
+---
+# Bind the flannel ClusterRole to the canal ServiceAccount.
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: canal-flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}

roles/network_plugin/canal/templates/canal-node-sa.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: canal
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -19,6 +19,9 @@ spec:
         k8s-app: canal-node
     spec:
       hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: canal
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists
@@ -169,6 +172,10 @@ spec:
                 configMapKeyRef:
                   name: canal-config
                   key: etcd_keyfile
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
           securityContext:
             privileged: true
           volumeMounts: