From ddc19f43bad3acff139da31b433714686dd198af Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Tue, 27 Nov 2018 21:03:03 +0800 Subject: [PATCH] Add cloud provider config to kubeadm deployments (#3766) --- inventory/sample/group_vars/all/all.yml | 3 +- .../templates/kubeadm-config.v1alpha1.yaml.j2 | 26 ++++++-- .../templates/kubeadm-config.v1alpha2.yaml.j2 | 35 ++++++++-- .../templates/kubeadm-config.v1alpha3.yaml.j2 | 64 +++++++++++++------ .../node/templates/kubelet.kubeadm.env.j2 | 4 +- 5 files changed, 101 insertions(+), 31 deletions(-) diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 24d74ed60..6e835ca90 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -42,9 +42,10 @@ bin_dir: /usr/local/bin ## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external' ## When openstack is used make sure to source in the openstack credentials ## like you would do when using nova-client before starting the playbook. +## Note: The 'external' cloud provider is not supported. +## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager #cloud_provider: - ## kubeadm deployment mode kubeadm_enabled: true diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index f1b05bb59..8adc777fd 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -22,8 +22,11 @@ networking: podSubnet: {{ kube_pods_subnet }} podNetworkCidr: "{{ kube_network_node_prefix }}" kubernetesVersion: {{ kube_version }} -{% if cloud_provider is defined and cloud_provider not in ["gce", "oci"] %} -cloudProvider: {{ cloud_provider }} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} +cloudProvider: {{cloud_provider}} +cloudConfig: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} +cloudConfig: {{ kube_config_dir }}/cloud_config {% endif %} {% if kube_proxy_mode == 'ipvs' %} kubeProxy: @@ -40,7 +43,7 @@ kubeProxy: {% if kube_proxy_nodeport_addresses %} nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}] {% endif %} - resourceContainer: "" +resourceContainer: "" authorizationModes: {% for mode in authorization_modes %} - {{ mode }} @@ -111,6 +114,9 @@ apiServerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} + configure-cloud-routes: "true" +{% endif %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} @@ -123,12 +129,19 @@ controllerManagerExtraArgs: {% for key in kube_kubeadm_controller_extra_args %} {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} -{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} controllerManagerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} - name: openstackcacert hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% endif %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} @@ -141,6 +154,11 @@ schedulerExtraArgs: {% endif %} {% if kube_basic_auth|default(true) or kube_token_auth|default(true) %} apiServerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} {% if kube_basic_auth|default(true) %} - name: basic-auth-config hostPath: {{ kube_users_dir }} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index c0fc65714..53e1703ce 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -23,9 +23,6 @@ networking: podSubnet: {{ kube_pods_subnet }} podNetworkCidr: "{{ kube_network_node_prefix }}" kubernetesVersion: {{ kube_version }} -{% if cloud_provider is defined and cloud_provider != "gce" %} -cloudProvider: {{ cloud_provider }} -{% endif %} kubeProxy: config: mode: {{ kube_proxy_mode }} @@ -109,6 +106,15 @@ apiServerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} + configure-cloud-routes: "true" +{% endif %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} @@ -121,12 +127,25 @@ controllerManagerExtraArgs: {% for key in kube_kubeadm_controller_extra_args %} {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} -{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} controllerManagerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} - name: openstackcacert hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% endif %} {% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) %} apiServerExtraVolumes: {% if kube_basic_auth|default(true) %} @@ -151,11 +170,19 @@ apiServerExtraVolumes: {% endif %} {% endif %} {% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% if volume_cross_zone_attachment %} + policy-config-file: {{ kube_config_dir }}/kube-scheduler-policy.yaml +{% endif %} {% if kube_kubeadm_scheduler_extra_args|length > 0 %} {% for key in kube_kubeadm_scheduler_extra_args %} {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 index d4d20d2ed..4376426c8 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -43,6 +43,13 @@ controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.po {% else %} controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }} {% endif %} +apiServerCertSANs: +{% for san in apiserver_sans.split(' ') | unique %} + - {{ san }} +{% endfor %} +certificatesDir: {{ kube_config_dir }}/ssl +imageRepository: {{ kube_image_repo }} +unifiedControlPlaneImage: "" apiServerExtraArgs: authorization-mode: {{ authorization_modes | join(',') }} bind-address: {{ kube_apiserver_bind_address }} @@ -109,6 +116,12 @@ apiServerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} @@ -116,14 +129,28 @@ controllerManagerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} -{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} -controllerManagerExtraVolumes: -- name: openstackcacert - hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" - mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config {% endif %} -{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) %} +schedulerExtraArgs: +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} +{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) %} apiServerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} {% if kube_basic_auth|default(true) %} - name: basic-auth-config hostPath: {{ kube_users_dir }} @@ -149,22 +176,19 @@ apiServerExtraVolumes: {% for key in kube_kubeadm_controller_extra_args %} {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} -schedulerExtraArgs: -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +controllerManagerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} +- name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config {% endif %} -{% if kube_kubeadm_scheduler_extra_args|length > 0 %} -{% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" -{% endfor %} {% endif %} -apiServerCertSANs: -{% for san in apiserver_sans.split(' ') | unique %} - - {{ san }} -{% endfor %} -certificatesDir: {{ kube_config_dir }}/ssl -imageRepository: {{ kube_image_repo }} -unifiedControlPlaneImage: "" --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 38f58831b..e8e5e7ed5 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -106,8 +106,8 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben KUBE_ALLOW_PRIV="--allow-privileged=true" {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" -{% elif cloud_provider is defined and cloud_provider in ["oci", "external"] %} -KUBELET_CLOUDPROVIDER="--cloud-provider=external" +{% elif cloud_provider is defined and cloud_provider in ["external"] %} +KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config" {% else %} KUBELET_CLOUDPROVIDER="" {% endif %}