Move calico-policy-controller into separate role

By default Calico CNI does not create any network access policies
or profiles if 'policy' is enabled in CNI config. And without any
policies/profiles network access to/from PODs is blocked.

K8s related policies are created by calico-policy-controller in
such case. So we need to start it as soon as possible, before any
real workloads.

This patch also fixes kube-api port in calico-policy-controller
yaml template.

Closes #1132

cluster.yml

@@ -70,6 +70,7 @@
     - { role: kargo-defaults}
     - { role: kubernetes/master, tags: master }
     - { role: kubernetes-apps/network_plugin, tags: network }
+    - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - hosts: calico-rr
   any_errors_fatal: true

inventory/group_vars/k8s-cluster.yml

@@ -80,6 +80,9 @@ kube_users:
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
 
+# Enable kubernetes network policies
+enable_network_policy: false
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -19,12 +19,6 @@ kubednsmasq_image_tag: "{{ kubednsmasq_version }}"
 exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64"
 exechealthz_image_tag: "{{ exechealthz_version }}"
 
-# Limits for calico apps
-calico_policy_controller_cpu_limit: 100m
-calico_policy_controller_memory_limit: 256M
-calico_policy_controller_cpu_requests: 30m
-calico_policy_controller_memory_requests: 64M
-
 # Netchecker
 deploy_netchecker: false
 netchecker_port: 31081
@@ -45,5 +39,4 @@ netchecker_server_memory_requests: 64M
 
 # SSL
 etcd_cert_dir: "/etc/ssl/etcd/ssl"
-calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -32,11 +32,6 @@
   when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
   tags: dnsmasq
 
-- include: tasks/calico-policy-controller.yml
-  when: ( enable_network_policy is defined and enable_network_policy == True ) or
-    ( kube_network_plugin == 'canal' )
-  tags: [network, canal]
-
 - name: Kubernetes Apps | Netchecker
   include: tasks/netchecker.yml
   when: deploy_netchecker

roles/kubernetes-apps/meta/main.yml

@@ -1,9 +1,4 @@
 dependencies:
-  - role: download
-    file: "{{ downloads.calico_policy }}"
-    when: ( enable_network_policy is defined and enable_network_policy == True ) or
-      ( kube_network_plugin == 'canal' )
-    tags: [download, network, canal]
   - role: download
     file: "{{ downloads.netcheck_server }}"
     when: deploy_netchecker

roles/kubernetes-apps/policy_controller/calico/defaults/main.yml

@@ -0,0 +1,9 @@
+# Limits for calico apps
+calico_policy_controller_cpu_limit: 100m
+calico_policy_controller_memory_limit: 256M
+calico_policy_controller_cpu_requests: 30m
+calico_policy_controller_memory_requests: 64M
+
+# SSL
+calico_cert_dir: "/etc/calico/certs"
+canal_cert_dir: "/etc/canal/certs"

roles/kubernetes-apps/policy_controller/calico/tasks/main.yml

@@ -1,14 +1,14 @@
----
 - set_fact:
     calico_cert_dir: "{{ canal_cert_dir }}"
   when: kube_network_plugin == 'canal'
-  tags: facts
+  tags: [facts, canal]
 
 - name: Write calico-policy-controller yaml
   template:
     src: calico-policy-controller.yml.j2
     dest: "{{kube_config_dir}}/calico-policy-controller.yml"
   when: inventory_hostname == groups['kube-master'][0]
+  tags: canal
 
 - name: Start of Calico policy controller
   kube:
@@ -18,3 +18,4 @@
     namespace: "{{system_namespace}}"
     resource: "rs"
   when: inventory_hostname == groups['kube-master'][0]
+  tags: canal

roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2

@@ -45,7 +45,7 @@ spec:
             # changed so long as it is used in conjunction with
             # CONFIGURE_ETC_HOSTS="true".
             - name: K8S_API
-              value: "https://kubernetes.default:{{ kube_apiserver_port }}"
+              value: "https://kubernetes.default"
             # Configure /etc/hosts within the container to resolve
             # the kubernetes.default Service to the correct clusterIP
             # using the environment provided by the kubelet.

roles/kubernetes-apps/policy_controller/meta/main.yml

@@ -0,0 +1,14 @@
+---
+dependencies:
+  - role: download
+    file: "{{ downloads.calico_policy }}"
+    when: enable_network_policy and
+      kube_network_plugin in ['calico', 'canal']
+    tags: [download, canal, policy-controller]
+  - role: policy_controller/calico
+    when: kube_network_plugin == 'calico' and
+      enable_network_policy
+    tags: policy-controller
+  - role: policy_controller/calico
+    when: kube_network_plugin == 'canal'
+    tags: policy-controller

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -64,7 +64,7 @@ spec:
     - --runtime-config={{ conf }}
 {%   endfor %}
 {% endif %}
-{% if enable_network_policy is defined and enable_network_policy == True %}
+{% if enable_network_policy %}
     - --runtime-config=extensions/v1beta1/networkpolicies=true
 {% endif %}
     - --v={{ kube_log_level }}

roles/network_plugin/calico/templates/cni-calico.conf.j2

@@ -12,7 +12,7 @@
   "ipam": {
     "type": "calico-ipam"
   },
-{% if enable_network_policy is defined and enable_network_policy == True %}
+{% if enable_network_policy %}
   "policy": {
     "type": "k8s"
   },