From 20df44521d146ef524974ff1de94aa3bd13c63bf Mon Sep 17 00:00:00 2001 From: ChengHao Yang <17496418+tico88612@users.noreply.github.com> Date: Thu, 31 Oct 2024 20:53:26 +0800 Subject: [PATCH] Feat: Add external OCI cloud controller manager (#11378) * Feat: add external OCI cloud controller manager template & variable Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * Feat: add external OCI cloud controller manager workflow Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * Feat: migrate external OCI CCM config check from OCI cloud provider Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> * cloud_controller: oracle: simpler asserts Make the asserts check for Oracle Cloud Infrastructure external cloud controller more compact, and hence readable. Allows to put them back in the main tasks for less back and forth when reading the code. --------- Signed-off-by: tico88612 <17496418+tico88612@users.noreply.github.com> Co-authored-by: Max Gautier --- inventory/sample/group_vars/all/oci.yml | 27 +++ .../external_cloud_controller/meta/main.yml | 10 ++ .../oci/defaults/main.yml | 25 +++ .../oci/tasks/main.yml | 53 ++++++ .../external-oci-cloud-config-secret.yml.j2 | 10 ++ .../external-oci-cloud-config.yml.j2 | 45 +++++ ...l-oci-cloud-controller-manager-rbac.yml.j2 | 166 ++++++++++++++++++ ...ternal-oci-cloud-controller-manager.yml.j2 | 59 +++++++ 8 files changed, 395 insertions(+) create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 diff --git a/inventory/sample/group_vars/all/oci.yml b/inventory/sample/group_vars/all/oci.yml index 541d0e6c9..38871f874 100644 --- a/inventory/sample/group_vars/all/oci.yml +++ b/inventory/sample/group_vars/all/oci.yml @@ -1,3 +1,30 @@ +## When External Oracle Cloud Infrastructure is used, set these variables +## External OCI Cloud Controller Manager +## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml +# external_oracle_auth_region: "" +# external_oracle_auth_tenancy: "" +# external_oracle_auth_user: "" +# external_oracle_auth_key: "" +# external_oracle_auth_passphrase: "" +# external_oracle_auth_fingerprint: "" +# external_oracle_auth_use_instance_principals: false + +# external_oracle_compartment: "" +# external_oracle_vcn: "" +# external_oracle_load_balancer_subnet1: "" +# external_oracle_load_balancer_subnet2: "" +# external_oracle_load_balancer_security_list_management_mode: All +# external_oracle_load_balancer_security_lists: {} + +# external_oracle_ratelimiter_qps_read: 20.0 +# external_oracle_ratelimiter_bucket_read: 5 +# external_oracle_ratelimiter_qps_write: 20.0 +# external_oracle_ratelimiter_bucket_write: 5 + +# external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci +# external_oracle_cloud_controller_image_tag: "v1.29.0" + + ## When Oracle Cloud Infrastructure is used, set these variables # oci_private_key: # oci_region_id: diff --git a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml index b1fc4ad69..468fae8eb 100644 --- a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml @@ -40,3 +40,13 @@ dependencies: tags: - external-cloud-controller - external-huaweicloud + - role: kubernetes-apps/external_cloud_controller/oci + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "oci" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-oci diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml new file mode 100644 index 000000000..a38327390 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml @@ -0,0 +1,25 @@ +--- +## External Oracle Cloud Controller Manager +## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml +external_oracle_auth_region: "" +external_oracle_auth_tenancy: "" +external_oracle_auth_user: "" +external_oracle_auth_key: "" +external_oracle_auth_passphrase: "" +external_oracle_auth_fingerprint: "" +external_oracle_auth_use_instance_principals: false + +external_oracle_compartment: "" +external_oracle_vcn: "" +external_oracle_load_balancer_subnet1: "" +external_oracle_load_balancer_subnet2: "" +external_oracle_load_balancer_security_list_management_mode: All +external_oracle_load_balancer_security_lists: {} + +external_oracle_ratelimiter_qps_read: 20.0 +external_oracle_ratelimiter_bucket_read: 5 +external_oracle_ratelimiter_qps_write: 20.0 +external_oracle_ratelimiter_bucket_write: 5 + +external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci +external_oracle_cloud_controller_image_tag: "v1.29.0" diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml b/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml new file mode 100644 index 000000000..da80e9d16 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: "External OCI Cloud Controller Manager | Check credentials" + ansible.builtin.assert: + that: + - external_oracle_auth_key | length > 0 + - external_oracle_auth_region | length > 0 + - external_oracle_auth_tenancy | length > 0 + - external_oracle_auth_user | length > 0 + - external_oracle_auth_fingerprint | length > 0 + when: not external_oracle_auth_use_instance_principals + +- name: "External OCI Cloud Controller Manager | Check settings" + ansible.builtin.assert: + that: + - external_oracle_compartment | length > 0 + - external_oracle_vcn | length > 0 + - external_oracle_load_balancer_subnet1 | length > 0 + - external_oracle_load_balancer_subnet2 | length > 0 + - external_oracle_load_balancer_security_list_management_mode in ["All", "Frontend", "None"] + +- name: External OCI Cloud Controller Manager | Get base64 cloud-config + set_fact: + external_oracle_cloud_config_secret: "{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-oci + +- name: External OCI Cloud Controller Manager | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: "0640" + with_items: + - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml} + - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml} + - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml} + register: external_oracle_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-oci + +- name: External OCI Cloud Controller Manager | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_oracle_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-oci diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 new file mode 100644 index 000000000..baa20b69e --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run external oci cloud controller. + +kind: Secret +apiVersion: v1 +metadata: + name: oci-cloud-controller-manager + namespace: kube-system +data: + cloud-provider.yaml: {{ external_oracle_cloud_config_secret }} diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 new file mode 100644 index 000000000..5af04b242 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 @@ -0,0 +1,45 @@ +{% if external_oracle_auth_use_instance_principals %} +useInstancePrincipals: true +{% endif %} + +auth: +{% if external_oracle_auth_use_instance_principals %} + useInstancePrincipals: true +{% else %} + useInstancePrincipals: false + region: {{ external_oracle_auth_region }} + tenancy: {{ external_oracle_auth_tenancy }} + user: {{ external_oracle_auth_user }} + key: | + {{ external_oracle_auth_key }} + {% if external_oracle_auth_passphrase is defined %} + # Omit if there is not a password for the key + passphrase: {{ external_oracle_auth_passphrase }} + {% endif %} + fingerprint: {{ external_oracle_auth_fingerprint }} +{% endif %} + +compartment: {{ external_oracle_compartment }} + +vcn: {{ external_oracle_vcn }} + +loadBalancer: + subnet1: {{ external_oracle_load_balancer_subnet1 }} + subnet2: {{ external_oracle_load_balancer_subnet2 }} + + securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }} + +{% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %} + # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off. + securityLists: + {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %} + {{ subnet_ocid }}: {{ list_ocid }} + {% endfor %} +{% endif %} + +# Optional rate limit controls for accessing OCI API +rateLimiter: + rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }} + rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }} + rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }} + rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }} diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 new file mode 100644 index 000000000..e7ab8cab0 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 @@ -0,0 +1,166 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + +- apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + - get + +- apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + +# For leader election +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - create + - update + - delete + - patch + - watch + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + - list + - watch + +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - list + - get + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + +# For the PVL +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: oci-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 new file mode 100644 index 000000000..8d14ca427 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager +spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }} + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=leases + - --concurrent-service-syncs=3 + - --v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true