diff --git a/inventory/sample/group_vars/all/oci.yml b/inventory/sample/group_vars/all/oci.yml index 541d0e6c9..38871f874 100644 --- a/inventory/sample/group_vars/all/oci.yml +++ b/inventory/sample/group_vars/all/oci.yml @@ -1,3 +1,30 @@ +## When External Oracle Cloud Infrastructure is used, set these variables +## External OCI Cloud Controller Manager +## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml +# external_oracle_auth_region: "" +# external_oracle_auth_tenancy: "" +# external_oracle_auth_user: "" +# external_oracle_auth_key: "" +# external_oracle_auth_passphrase: "" +# external_oracle_auth_fingerprint: "" +# external_oracle_auth_use_instance_principals: false + +# external_oracle_compartment: "" +# external_oracle_vcn: "" +# external_oracle_load_balancer_subnet1: "" +# external_oracle_load_balancer_subnet2: "" +# external_oracle_load_balancer_security_list_management_mode: All +# external_oracle_load_balancer_security_lists: {} + +# external_oracle_ratelimiter_qps_read: 20.0 +# external_oracle_ratelimiter_bucket_read: 5 +# external_oracle_ratelimiter_qps_write: 20.0 +# external_oracle_ratelimiter_bucket_write: 5 + +# external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci +# external_oracle_cloud_controller_image_tag: "v1.29.0" + + ## When Oracle Cloud Infrastructure is used, set these variables # oci_private_key: # oci_region_id: diff --git a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml index b1fc4ad69..468fae8eb 100644 --- a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml @@ -40,3 +40,13 @@ dependencies: tags: - external-cloud-controller - external-huaweicloud + - role: kubernetes-apps/external_cloud_controller/oci + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "oci" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-oci diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml new file mode 100644 index 000000000..a38327390 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/defaults/main.yml @@ -0,0 +1,25 @@ +--- +## External Oracle Cloud Controller Manager +## https://github.com/oracle/oci-cloud-controller-manager/blob/v1.29.0/manifests/provider-config-example.yaml +external_oracle_auth_region: "" +external_oracle_auth_tenancy: "" +external_oracle_auth_user: "" +external_oracle_auth_key: "" +external_oracle_auth_passphrase: "" +external_oracle_auth_fingerprint: "" +external_oracle_auth_use_instance_principals: false + +external_oracle_compartment: "" +external_oracle_vcn: "" +external_oracle_load_balancer_subnet1: "" +external_oracle_load_balancer_subnet2: "" +external_oracle_load_balancer_security_list_management_mode: All +external_oracle_load_balancer_security_lists: {} + +external_oracle_ratelimiter_qps_read: 20.0 +external_oracle_ratelimiter_bucket_read: 5 +external_oracle_ratelimiter_qps_write: 20.0 +external_oracle_ratelimiter_bucket_write: 5 + +external_oracle_cloud_controller_image_repo: ghcr.io/oracle/cloud-provider-oci +external_oracle_cloud_controller_image_tag: "v1.29.0" diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml b/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml new file mode 100644 index 000000000..da80e9d16 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/tasks/main.yml @@ -0,0 +1,53 @@ +--- +- name: "External OCI Cloud Controller Manager | Check credentials" + ansible.builtin.assert: + that: + - external_oracle_auth_key | length > 0 + - external_oracle_auth_region | length > 0 + - external_oracle_auth_tenancy | length > 0 + - external_oracle_auth_user | length > 0 + - external_oracle_auth_fingerprint | length > 0 + when: not external_oracle_auth_use_instance_principals + +- name: "External OCI Cloud Controller Manager | Check settings" + ansible.builtin.assert: + that: + - external_oracle_compartment | length > 0 + - external_oracle_vcn | length > 0 + - external_oracle_load_balancer_subnet1 | length > 0 + - external_oracle_load_balancer_subnet2 | length > 0 + - external_oracle_load_balancer_security_list_management_mode in ["All", "Frontend", "None"] + +- name: External OCI Cloud Controller Manager | Get base64 cloud-config + set_fact: + external_oracle_cloud_config_secret: "{{ lookup('template', 'external-oci-cloud-config.j2') | b64encode }}" + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-oci + +- name: External OCI Cloud Controller Manager | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: "0640" + with_items: + - {name: external-oci-cloud-config-secret, file: external-oci-cloud-config-secret.yml} + - {name: external-oci-cloud-controller-manager-rbac, file: external-oci-cloud-controller-manager-rbac.yml} + - {name: external-oci-cloud-controller-manager, file: external-oci-cloud-controller-manager.yml} + register: external_oracle_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-oci + +- name: External OCI Cloud Controller Manager | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_oracle_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-oci diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 new file mode 100644 index 000000000..baa20b69e --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run external oci cloud controller. + +kind: Secret +apiVersion: v1 +metadata: + name: oci-cloud-controller-manager + namespace: kube-system +data: + cloud-provider.yaml: {{ external_oracle_cloud_config_secret }} diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 new file mode 100644 index 000000000..5af04b242 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-config.yml.j2 @@ -0,0 +1,45 @@ +{% if external_oracle_auth_use_instance_principals %} +useInstancePrincipals: true +{% endif %} + +auth: +{% if external_oracle_auth_use_instance_principals %} + useInstancePrincipals: true +{% else %} + useInstancePrincipals: false + region: {{ external_oracle_auth_region }} + tenancy: {{ external_oracle_auth_tenancy }} + user: {{ external_oracle_auth_user }} + key: | + {{ external_oracle_auth_key }} + {% if external_oracle_auth_passphrase is defined %} + # Omit if there is not a password for the key + passphrase: {{ external_oracle_auth_passphrase }} + {% endif %} + fingerprint: {{ external_oracle_auth_fingerprint }} +{% endif %} + +compartment: {{ external_oracle_compartment }} + +vcn: {{ external_oracle_vcn }} + +loadBalancer: + subnet1: {{ external_oracle_load_balancer_subnet1 }} + subnet2: {{ external_oracle_load_balancer_subnet2 }} + + securityListManagementMode: {{ external_oracle_load_balancer_security_list_management_mode }} + +{% if external_oracle_security_lists is defined and external_oracle_security_lists | length > 0 %} + # Optional specification of which security lists to modify per subnet. This does not apply if security list management is off. + securityLists: + {% for subnet_ocid, list_ocid in external_oracle_load_balancer_security_lists.items() %} + {{ subnet_ocid }}: {{ list_ocid }} + {% endfor %} +{% endif %} + +# Optional rate limit controls for accessing OCI API +rateLimiter: + rateLimitQPSRead: {{ external_oracle_ratelimiter_qps_read }} + rateLimitBucketRead: {{ external_oracle_ratelimiter_bucket_read }} + rateLimitQPSWrite: {{ external_oracle_ratelimiter_qps_write }} + rateLimitBucketWrite: {{ external_oracle_ratelimiter_bucket_write }} diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 new file mode 100644 index 000000000..e7ab8cab0 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager-rbac.yml.j2 @@ -0,0 +1,166 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager + labels: + kubernetes.io/cluster-service: "true" +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + +- apiGroups: + - "" + resources: + - services + verbs: + - list + - watch + - patch + - get + +- apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - get + - update + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - patch + - update + +# For leader election +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + +- apiGroups: + - "" + resources: + - endpoints + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - list + - watch + - update + +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - create + - update + - delete + - patch + - watch + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "cloud-controller-manager" + verbs: + - get + - update + +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "extension-apiserver-authentication" + verbs: + - get + - list + - watch + +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - list + - get + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + +# For the PVL +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - watch + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: oci-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system diff --git a/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 new file mode 100644 index 000000000..8d14ca427 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/oci/templates/external-oci-cloud-controller-manager.yml.j2 @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: oci-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: oci-cloud-controller-manager +spec: + selector: + matchLabels: + component: oci-cloud-controller-manager + tier: control-plane + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + component: oci-cloud-controller-manager + tier: control-plane + spec: + serviceAccountName: cloud-controller-manager + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: cfg + secret: + secretName: oci-cloud-controller-manager + - name: kubernetes + hostPath: + path: /etc/kubernetes + containers: + - name: oci-cloud-controller-manager + image: {{ external_oracle_cloud_controller_image_repo }}:{{ external_oracle_cloud_controller_image_tag }} + command: ["/usr/local/bin/oci-cloud-controller-manager"] + args: + - --cloud-config=/etc/oci/cloud-provider.yaml + - --cloud-provider=oci + - --leader-elect-resource-lock=leases + - --concurrent-service-syncs=3 + - --v=2 + volumeMounts: + - name: cfg + mountPath: /etc/oci + readOnly: true + - name: kubernetes + mountPath: /etc/kubernetes + readOnly: true