[calico] add calico apiserver (#8690)

* [calico] add calico apiserver

* fix yamllint

* remove addext argument

* Configure API server with the CA bundle

* add check kdd

inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml

@@ -112,3 +112,6 @@
 # Under certain situations liveness and readiness probes may need tunning
 # calico_node_livenessprobe_timeout: 10
 # calico_node_readinessprobe_timeout: 10
+
+# Calico apiserver (only with kdd)
+# calico_apiserver_enabled: false

roles/download/defaults/main.yml

@@ -101,7 +101,9 @@ calico_cni_version: "{{ calico_version }}"
 calico_flexvol_version: "{{ calico_version }}"
 calico_policy_version: "{{ calico_version }}"
 calico_typha_version: "{{ calico_version }}"
+calico_apiserver_version: "{{ calico_version }}"
 typha_enabled: false
+calico_apiserver_enabled: false
 
 flannel_version: "v0.15.1"
 flannel_cni_version: "v1.0.0"
@@ -838,6 +840,8 @@ calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"
 calico_typha_image_tag: "{{ calico_typha_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
+calico_apiserver_image_repo: "{{ quay_image_repo }}/calico/apiserver"
+calico_apiserver_image_tag: "{{ calico_apiserver_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 pod_infra_image_repo: "{{ kube_image_repo }}/pause"
 pod_infra_image_tag: "{{ pod_infra_version }}"
 install_socat_image_repo: "{{ docker_image_repo }}/xueshanf/install-socat"
@@ -1313,6 +1317,15 @@ downloads:
     groups:
     - k8s_cluster
 
+  calico_apiserver:
+    enabled: "{{ calico_apiserver_enabled }}"
+    container: true
+    repo: "{{ calico_apiserver_image_repo }}"
+    tag: "{{ calico_apiserver_image_tag }}"
+    sha256: "{{ calico_apiserver_digest_checksum|default(None) }}"
+    groups:
+    - k8s_cluster
+
   calico_crds:
     file: true
     enabled: "{{ kube_network_plugin == 'calico' and calico_datastore == 'kdd' }}"

roles/kubespray-defaults/defaults/main.yaml

@@ -621,6 +621,8 @@ etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 
 typha_enabled: false
 
+calico_apiserver_enabled: false
+
 _host_architecture_groups:
   x86_64: amd64
   aarch64: arm64

roles/network_plugin/calico/defaults/main.yml

@@ -131,3 +131,6 @@ calico_allow_ip_forwarding: false
 
 # Calico IPAM strictaffinity
 calico_ipam_strictaffinity: false
+
+# Calico apiserver (only with kdd)
+calico_apiserver_enabled: false

roles/network_plugin/calico/tasks/calico_apiserver_certs.yml

@@ -0,0 +1,60 @@
+---
+- name: Calico | Check if calico apiserver exists
+  command: "{{ kubectl }} -n calico-apiserver get secret calico-apiserver-certs"
+  register: calico_apiserver_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Create ns manifests
+  template:
+    src: "calico-apiserver-ns.yml.j2"
+    dest: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    mode: 0644
+
+- name: Calico | Apply ns manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/calico-apiserver-ns.yml"
+    state: "latest"
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+    mode: 0755
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl script for apiserver certs
+  template:
+    src: make-ssl-calico.sh.j2
+    dest: "{{ bin_dir }}/make-ssl-apiserver.sh"
+    mode: 0755
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Copy ssl config for apiserver certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: 0644
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Generate apiserver certs
+  command: >-
+    {{ bin_dir }}/make-ssl-apiserver.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+    -s apiserver
+  when: calico_apiserver_secret.rc != 0
+
+- name: Calico | Create calico apiserver generic secrets
+  command: >-
+    {{ kubectl }} -n calico-apiserver
+    create secret generic {{ item.name }}
+    --from-file={{ item.cert }}
+    --from-file={{ item.key }}
+  with_items:
+    - name: calico-apiserver-certs
+      cert: /etc/calico/certs/apiserver.crt
+      key: /etc/calico/certs/apiserver.key
+  when: calico_apiserver_secret.rc != 0

roles/network_plugin/calico/tasks/check.yml

@@ -59,3 +59,15 @@
     msg: "Your inventory doesn't match the current cluster configuration"
   when:
     - calico_pool_conf is defined
+
+- name: "Check kdd calico_datastore if calico_apiserver_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+  when:
+    - calico_apiserver_enabled
+
+- name: "Check kdd calico_datastore if typha_enabled"
+  assert:
+    that: calico_datastore == "kdd"
+  when:
+    - typha_enabled

roles/network_plugin/calico/tasks/install.yml

@@ -53,6 +53,12 @@
     - typha_secure
     - inventory_hostname == groups['kube_control_plane'][0]
 
+- name: Calico | Generate apiserver certs
+  include_tasks: calico_apiserver_certs.yml
+  when:
+    - calico_apiserver_enabled
+    - inventory_hostname == groups['kube_control_plane'][0]
+
 - name: Calico | Install calicoctl wrapper script
   template:
     src: "calicoctl.{{ calico_datastore }}.sh.j2"
@@ -362,7 +368,34 @@
   register: calico_node_typha_manifest
   when:
     - inventory_hostname in groups['kube_control_plane']
-    - typha_enabled and calico_datastore == "kdd"
+    - typha_enabled
+
+- name: Calico | get calico apiserver caBundle
+  command: "{{ bin_dir }}/kubectl get secret -n calico-apiserver calico-apiserver-certs -o jsonpath='{.data.apiserver\\.crt}'"
+  changed_when: false
+  register: calico_apiserver_cabundle
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | set calico apiserver caBundle fact
+  set_fact:
+    calico_apiserver_cabundle: "{{ calico_apiserver_cabundle.stdout }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calico_apiserver_enabled
+
+- name: Calico | Create calico manifests for apiserver
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    mode: 0644
+  with_items:
+    - {name: calico, file: calico-apiserver.yml, type: calico-apiserver}
+  register: calico_apiserver_manifest
+  when:
+    - inventory_hostname in groups['kube_control_plane']
+    - calico_apiserver_enabled
 
 - name: Start Calico resources
   kube:
@@ -381,6 +414,22 @@
   loop_control:
     label: "{{ item.item.file }}"
 
+- name: Start Calico apiserver resources
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "calico-apiserver"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ calico_apiserver_manifest.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+
 - name: Wait for calico kubeconfig to be created
   wait_for:
     path: /etc/cni/net.d/calico-kubeconfig

roles/network_plugin/calico/tasks/typha_certs.yml

@@ -13,8 +13,8 @@
   when: typha_server_secret.rc != 0
 
 - name: Calico | Copy ssl script for typha certs
-  copy:
+  template:
-    src: make-ssl-typha.sh
+    src: make-ssl-calico.sh.j2
     dest: "{{ bin_dir }}/make-ssl-typha.sh"
     mode: 0755
   when: typha_server_secret.rc != 0
@@ -32,6 +32,7 @@
     -f /etc/calico/certs/openssl.conf
     -c {{ kube_cert_dir }}
     -d /etc/calico/certs
+    -s typha
   when: typha_server_secret.rc != 0
 
 - name: Calico | Create typha tls secrets

roles/network_plugin/calico/templates/calico-apiserver-ns.yml.j2

@@ -0,0 +1,10 @@
+# This is a tech-preview manifest which installs the Calico API server. Note that this manifest is liable to change
+# or be removed in future releases without further warning.
+#
+# Namespace and namespace-scoped resources.
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    name: calico-apiserver
+  name: calico-apiserver

roles/network_plugin/calico/templates/calico-apiserver.yml.j2

@@ -0,0 +1,320 @@
+# Policy to ensure the API server isn't cut off. Can be modified, but ensure 
+# that the main API server is always able to reach the Calico API server.
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: allow-apiserver
+  namespace: calico-apiserver
+spec:
+  podSelector:
+    matchLabels:
+      apiserver: "true"
+  ingress:
+  - ports:
+    - protocol: TCP
+      port: 5443
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: calico-api
+  namespace: calico-apiserver
+spec:
+  ports:
+  - name: apiserver
+    port: 443
+    protocol: TCP
+    targetPort: 5443
+  selector:
+    apiserver: "true"
+  type: ClusterIP
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    apiserver: "true"
+    k8s-app: calico-apiserver
+  name: calico-apiserver
+  namespace: calico-apiserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      apiserver: "true"
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        apiserver: "true"
+        k8s-app: calico-apiserver
+      name: calico-apiserver
+      namespace: calico-apiserver
+    spec:
+      containers:
+      - args:
+        - --secure-port=5443
+        - -v=5
+        env:
+        - name: DATASTORE_TYPE
+          value: kubernetes
+        image: {{ calico_apiserver_image_repo }}:{{ calico_apiserver_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        livenessProbe:
+          httpGet:
+            path: /version
+            port: 5443
+            scheme: HTTPS
+          initialDelaySeconds: 90
+          periodSeconds: 10
+        name: calico-apiserver
+        readinessProbe:
+          exec:
+            command:
+            - /code/filecheck
+          failureThreshold: 5
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        securityContext:
+          privileged: false
+          runAsUser: 0
+        volumeMounts:
+        - mountPath: /code/apiserver.local.config/certificates
+          name: calico-apiserver-certs
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: Always
+      serviceAccount: calico-apiserver
+      serviceAccountName: calico-apiserver
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+      volumes:
+      - name: calico-apiserver-certs
+        secret:
+          secretName: calico-apiserver-certs
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-apiserver 
+  namespace: calico-apiserver 
+
+---
+
+# Cluster-scoped resources below here.
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v3.projectcalico.org
+spec:
+  group: projectcalico.org
+  groupPriorityMinimum: 1500
+  caBundle: {{ calico_apiserver_cabundle }}
+  service:
+    name: calico-api
+    namespace: calico-apiserver
+    port: 443
+  version: v3
+  versionPriority: 200
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-crds
+rules:
+- apiGroups:
+  - extensions
+  - networking.k8s.io
+  - ""
+  resources:
+  - networkpolicies
+  - nodes
+  - namespaces
+  - pods
+  - serviceaccounts
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - crd.projectcalico.org
+  resources:
+  - globalnetworkpolicies
+  - networkpolicies
+  - clusterinformations
+  - hostendpoints
+  - globalnetworksets
+  - networksets
+  - bgpconfigurations
+  - bgppeers
+  - felixconfigurations
+  - kubecontrollersconfigurations
+  - ippools
+  - ipreservations
+  - ipamblocks
+  - blockaffinities
+  - caliconodestatuses
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - delete
+- apiGroups:
+  - policy
+  resourceNames:
+  - calico-apiserver
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-extension-apiserver-auth-access
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - extension-apiserver-authentication
+  resources:
+  - configmaps
+  verbs:
+  - list
+  - watch
+  - get
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  - clusterrolebindings
+  - roles
+  - rolebindings
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-webhook-reader
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-access-crds
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-crds
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-delegate-auth
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-apiserver-webhook-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-webhook-reader
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-extension-apiserver-auth-access
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-extension-apiserver-auth-access
+subjects:
+- kind: ServiceAccount
+  name: calico-apiserver
+  namespace: calico-apiserver
+
+---
+
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+  name: calico-apiserver
+spec:
+  allowPrivilegeEscalation: false
+  fsGroup:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  hostPorts:
+  - max: 65535
+    min: 0
+  requiredDropCapabilities:
+  - ALL
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    ranges:
+    - max: 65535
+      min: 1
+    rule: MustRunAs
+  volumes:
+  - secret

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -43,6 +43,7 @@ spec:
         # upgraded to use calico-ipam.
         - name: upgrade-ipam
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
           envFrom:
             - configMapRef:
@@ -71,6 +72,7 @@ spec:
         # and CNI network config file on each node.
         - name: install-cni
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           command: ["/opt/cni/bin/install"]
           envFrom:
           - configMapRef:
@@ -108,6 +110,7 @@ spec:
         # to communicate with Felix over the Policy Sync API.
         - name: flexvol-driver
           image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           volumeMounts:
             - name: flexvol-driver-host
               mountPath: /host/driver
@@ -119,6 +122,7 @@ spec:
         # host.
         - name: calico-node
           image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           envFrom:
             - configMapRef:
                 # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.

roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -70,6 +70,7 @@ spec:
         fsGroup: 65534
       containers:
       - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
         name: calico-typha
         ports:
         - containerPort: 5473

roles/network_plugin/calico/templates/make-ssl-calico.sh.j2

@@ -26,6 +26,7 @@ Usage : $(basename $0) -f <config> [-d <ssldir>]
       -f | --config       : Openssl configuration file
       -d | --ssldir       : Directory where the certificates will be installed
       -c | --cadir        : Directory where the existing CA is located
+      -s | --service      : Service for the ca
 
                ex :
                $(basename $0) -f openssl.conf -d /srv/ssl
@@ -39,6 +40,7 @@ while (($#)); do
         -f | --config) CONFIG=${2}; shift 2;;
         -d | --ssldir) SSLDIR="${2}"; shift 2;;
         -c | --cadir) CADIR="${2}"; shift 2;;
+        -s | --service) SERVICE="${2}"; shift 2;;
         *)
             usage
             echo "ERROR : Unknown option"
@@ -55,7 +57,7 @@ if [ -z ${SSLDIR} ]; then
     SSLDIR="/etc/calico/certs"
 fi
 
-tmpdir=$(mktemp -d /tmp/calico_typha_certs.XXXXXX)
+tmpdir=$(mktemp -d /tmp/calico_${SERVICE}_certs.XXXXXX)
 trap 'rm -rf "${tmpdir}"' EXIT
 cd "${tmpdir}"
 
@@ -66,19 +68,30 @@ if [ -e "$CADIR/ca.key" ]; then
   # Reuse existing CA
   cp $CADIR/{ca.crt,ca.key} .
 else
-  openssl genrsa -out ca.key 2048 > /dev/null 2>&1
+  openssl genrsa -out ca.key {{certificates_key_size}} > /dev/null 2>&1
-  openssl req -x509 -new -nodes -key ca.key -days 3650 -out ca.crt -subj "/CN=calico-typha-ca" > /dev/null 2>&1
+  openssl req -x509 -new -nodes -key ca.key -days {{certificates_duration}}  -out ca.crt -subj "/CN=calico-${SERVICE}-ca" > /dev/null 2>&1
 fi
 
-# Typha server
+if [ $SERVICE == "typha" ]; then
-openssl genrsa -out typha-server.key 2048 > /dev/null 2>&1
+    # Typha server
-openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1
+    openssl genrsa -out typha-server.key {{certificates_key_size}} > /dev/null 2>&1
-openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+    openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days {{certificates_duration}}  -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
 
-# Typha client
+    # Typha client
-openssl genrsa -out typha-client.key 2048 > /dev/null 2>&1
+    openssl genrsa -out typha-client.key {{certificates_key_size}} > /dev/null 2>&1
-openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1
+    openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1
-openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days {{certificates_duration}}  -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+elif [ $SERVICE == "apiserver" ]; then
+    # calico-apiserver
+    openssl genrsa -out apiserver.key {{certificates_key_size}} > /dev/null 2>&1
+    openssl req -new -key apiserver.key -out apiserver.csr -subj "/CN=calico-apiserver" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in apiserver.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out apiserver.crt -days {{certificates_duration}}  -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+else
+    echo "ERROR: the openssl configuration file is missing. option -s"
+    exit 1
+fi
 
 # Install certs
 if [ -e "$CADIR/ca.key" ]; then

tests/files/packet_centos7-calico-ha-once-localhost.yml

@@ -7,6 +7,7 @@ mode: ha
 download_localhost: true
 download_run_once: true
 typha_enabled: true
+calico_apiserver_enabled: true
 calico_backend: kdd
 typha_secure: true
 disable_ipv6_dns: true

tests/files/packet_centos7-calico-ha.yml

@@ -7,6 +7,7 @@ mode: ha
 download_localhost: false
 download_run_once: true
 typha_enabled: true
+calico_apiserver_enabled: true
 calico_backend: kdd
 typha_secure: true
 auto_renew_certificates: true