Feat: Add external OCI cloud controller manager (#11378)

* Feat: add external OCI cloud controller manager template & variable

Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com>

* Feat: add external OCI cloud controller manager workflow

Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com>

* Feat: migrate external OCI CCM config check from OCI cloud provider

Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com>

* cloud_controller: oracle: simpler asserts

Make the asserts check for Oracle Cloud Infrastructure external cloud
controller more compact, and hence readable.
Allows to put them back in the main tasks for less back and forth when
reading the code.

---------

Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com>
Co-authored-by: Max Gautier <mg@max.gautier.name>

inventory/sample/group_vars/all/oci.yml

@@ -1,3 +1,30 @@
+## When External Oracle Cloud Infrastructure is used, set these variables
+## External OCI Cloud Controller Manager
+## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml
+# external_oracle_auth_region: ""
+# external_oracle_auth_tenancy: ""
+# external_oracle_auth_user: ""
+# external_oracle_auth_key: ""
+# external_oracle_auth_passphrase: ""
+# external_oracle_auth_fingerprint: ""
+# external_oracle_auth_use_instance_principals: false
+
+# external_oracle_compartment: ""
+# external_oracle_vcn: ""
+# external_oracle_load_balancer_subnet1: ""
+# external_oracle_load_balancer_subnet2: ""
+# external_oracle_load_balancer_security_list_management_mode: All
+# external_oracle_load_balancer_security_lists: {}
+
+# external_oracle_ratelimiter_qps_read: 20.0
+# external_oracle_ratelimiter_bucket_read: 5
+# external_oracle_ratelimiter_qps_write: 20.0
+# external_oracle_ratelimiter_bucket_write: 5
+
+# external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci
+# external_oracle_cloud_controller_image_tag: "v1.29.0"
+
+
 ## When Oracle Cloud Infrastructure is used, set these variables
 # oci_private_key:
 # oci_region_id:

roles/kubernetes-apps/external_cloud_controller/meta/main.yml

@@ -40,3 +40,13 @@ dependencies:
     tags:
       - external-cloud-controller
       - external-huaweicloud
+  - role: kubernetes-apps/external_cloud_controller/oci
+    when:
+      - cloud_provider is defined
+      - cloud_provider == "external"
+      - external_cloud_provider is defined
+      - external_cloud_provider == "oci"
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - external-cloud-controller
+      - external-oci

roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml

@@ -0,0 +1,25 @@
+---
+## External Oracle Cloud Controller Manager
+## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml
+external_oracle_auth_region: ""
+external_oracle_auth_tenancy: ""
+external_oracle_auth_user: ""
+external_oracle_auth_key: ""
+external_oracle_auth_passphrase: ""
+external_oracle_auth_fingerprint: ""
+external_oracle_auth_use_instance_principals: false
+
+external_oracle_compartment: ""
+external_oracle_vcn: ""
+external_oracle_load_balancer_subnet1: ""
+external_oracle_load_balancer_subnet2: ""
+external_oracle_load_balancer_security_list_management_mode: All
+external_oracle_load_balancer_security_lists: {}
+
+external_oracle_ratelimiter_qps_read: 20.0
+external_oracle_ratelimiter_bucket_read: 5
+external_oracle_ratelimiter_qps_write: 20.0
+external_oracle_ratelimiter_bucket_write: 5
+
+external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci
+external_oracle_cloud_controller_image_tag: "v1.29.0"

roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml

@@ -0,0 +1,53 @@
+---
+- name: "External OCI Cloud Controller Manager | Check credentials"
+  ansible.builtin.assert:
+    that:
+      - external_oracle_auth_key | length > 0
+      - external_oracle_auth_region | length > 0
+      - external_oracle_auth_tenancy | length > 0
+      - external_oracle_auth_user | length > 0
+      - external_oracle_auth_fingerprint | length > 0
+  when: not external_oracle_auth_use_instance_principals
+
+- name: "External OCI Cloud Controller Manager | Check settings"
+  ansible.builtin.assert:
+    that:
+      - external_oracle_compartment | length > 0
+      - external_oracle_vcn | length > 0
+      - external_oracle_load_balancer_subnet1 | length > 0
+      - external_oracle_load_balancer_subnet2 | length > 0
+      - external_oracle_load_balancer_security_list_management_mode in ["All", "Frontend", "None"]
+
+- name: External OCI Cloud Controller Manager | Get base64 cloud-config
+  set_fact:
+    external_oracle_cloud_config_secret: "{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-oci
+
+- name: External OCI Cloud Controller Manager | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    group: "{{ kube_cert_group }}"
+    mode: "0640"
+  with_items:
+    - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml}
+    - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml}
+    - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml}
+  register: external_oracle_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-oci
+
+- name: External OCI Cloud Controller Manager | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ external_oracle_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: external-oci

roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2

@@ -0,0 +1,10 @@
+# This YAML file contains secret objects,
+# which are necessary to run external oci cloud controller.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+data:
+  cloud-provider.yaml: {{ external_oracle_cloud_config_secret }}

roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2

@@ -0,0 +1,45 @@
+{% if external_oracle_auth_use_instance_principals %}
+useInstancePrincipals: true
+{% endif %}
+
+auth:
+{% if external_oracle_auth_use_instance_principals %}
+  useInstancePrincipals: true
+{% else %}
+  useInstancePrincipals: false
+  region: {{ external_oracle_auth_region }}
+  tenancy: {{ external_oracle_auth_tenancy }}
+  user: {{ external_oracle_auth_user }}
+  key: |
+    {{ external_oracle_auth_key }}
+  {% if external_oracle_auth_passphrase is defined %}
+  # Omit if there is not a password for the key
+  passphrase: {{ external_oracle_auth_passphrase }}
+  {% endif %}
+  fingerprint: {{ external_oracle_auth_fingerprint }}
+{% endif %}
+
+compartment: {{ external_oracle_compartment }}
+
+vcn: {{ external_oracle_vcn }}
+
+loadBalancer:
+  subnet1: {{ external_oracle_load_balancer_subnet1 }}
+  subnet2: {{ external_oracle_load_balancer_subnet2 }}
+
+  securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }}
+
+{% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %}
+  # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off.
+  securityLists:
+  {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %}
+    {{ subnet_ocid }}: {{ list_ocid }}
+  {% endfor %}
+{% endif %}
+
+# Optional rate limit controls for accessing OCI API
+rateLimiter:
+  rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }}
+  rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }}
+  rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }}
+  rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }}

roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2

@@ -0,0 +1,166 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  labels:
+    kubernetes.io/cluster-service: "true"
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - watch
+  - patch
+  - get
+
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - patch
+  - get
+  - update
+
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+  resourceNames:
+    - "extension-apiserver-authentication"
+  verbs:
+    - get
+
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - patch
+  - update
+
+# For leader election
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+- apiGroups:
+    - "coordination.k8s.io"
+  resources:
+    - leases
+  verbs:
+    - get
+    - create
+    - update
+    - delete
+    - patch
+    - watch
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - update
+
+- apiGroups:
+    - ""
+  resources:
+    - configmaps
+  resourceNames:
+    - "extension-apiserver-authentication"
+  verbs:
+    - get
+    - list
+    - watch
+
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - list
+  - get
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+
+# For the PVL
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - list
+  - watch
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: oci-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system

roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2

@@ -0,0 +1,59 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: oci-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      component: oci-cloud-controller-manager
+      tier: control-plane
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: oci-cloud-controller-manager
+        tier: control-plane
+    spec:
+      serviceAccountName: cloud-controller-manager
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+        - name: cfg
+          secret:
+            secretName: oci-cloud-controller-manager
+        - name: kubernetes
+          hostPath:
+            path: /etc/kubernetes
+      containers:
+        - name: oci-cloud-controller-manager
+          image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }}
+          command: ["/usr/local/bin/oci-cloud-controller-manager"]
+          args:
+            - --cloud-config=/etc/oci/cloud-provider.yaml
+            - --cloud-provider=oci
+            - --leader-elect-resource-lock=leases
+            - --concurrent-service-syncs=3
+            - --v=2
+          volumeMounts:
+            - name: cfg
+              mountPath: /etc/oci
+              readOnly: true
+            - name: kubernetes
+              mountPath: /etc/kubernetes
+              readOnly: true