From 0536125f75651c7cf6baf998c21063be5c0a4256 Mon Sep 17 00:00:00 2001 From: sangwook <123514+sangwook@users.noreply.github.com> Date: Fri, 28 Sep 2018 14:15:47 +0900 Subject: [PATCH] Better fix for openstack cinder zone issue using ignore-volume-az option (#2980) * Better fix for openstack cinder zone issue[1][2] using ignore-volume-az option[3]. [1]: https://github.com/kubernetes-incubator/kubespray/pull/2155 [2]: https://github.com/kubernetes-incubator/kubespray/pull/2346 [3]: https://github.com/kubernetes/kubernetes/pull/53523 * Remove kube-scheduler-policy.yaml --- inventory/sample/group_vars/all/openstack.yml | 1 + roles/kubernetes/master/defaults/main.yml | 3 --- .../master/tasks/static-pod-setup.yml | 8 ------ .../templates/kube-scheduler-policy.yaml.j2 | 27 ------------------- .../manifests/kube-scheduler.manifest.j2 | 14 ---------- .../node/templates/openstack-cloud-config.j2 | 5 +++- roles/kubespray-defaults/defaults/main.yaml | 6 +---- 7 files changed, 6 insertions(+), 58 deletions(-) delete mode 100644 roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 diff --git a/inventory/sample/group_vars/all/openstack.yml b/inventory/sample/group_vars/all/openstack.yml index 6347d0522..ae5dae2cf 100644 --- a/inventory/sample/group_vars/all/openstack.yml +++ b/inventory/sample/group_vars/all/openstack.yml @@ -1,5 +1,6 @@ ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) #openstack_blockstorage_version: "v1/v2/auto (default)" +#openstack_blockstorage_ignore_volume_az: yes ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. #openstack_lbaas_enabled: True #openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index cf0ca459b..49a09e212 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -129,9 +129,6 @@ kube_kubeadm_apiserver_extra_args: {} kube_kubeadm_controller_extra_args: {} kube_kubeadm_scheduler_extra_args: {} -## Variable for influencing kube-scheduler behaviour -volume_cross_zone_attachment: false - ## Encrypting Secret Data at Rest kube_encrypt_secret_data: false kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}" diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml index 4b5638289..33b28e637 100644 --- a/roles/kubernetes/master/tasks/static-pod-setup.yml +++ b/roles/kubernetes/master/tasks/static-pod-setup.yml @@ -26,14 +26,6 @@ - meta: flush_handlers -- name: Write kube-scheduler policy file - template: - src: kube-scheduler-policy.yaml.j2 - dest: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" - notify: Master | Restart kube-scheduler - tags: - - kube-scheduler - - name: Write kube-scheduler kubeconfig template: src: kube-scheduler-kubeconfig.yaml.j2 diff --git a/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 b/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 deleted file mode 100644 index 5a13d7a1e..000000000 --- a/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 +++ /dev/null @@ -1,27 +0,0 @@ -{ -"kind" : "Policy", -"apiVersion" : "v1", -"predicates" : [ - {"name" : "MaxEBSVolumeCount"}, - {"name" : "MaxGCEPDVolumeCount"}, - {"name" : "MaxAzureDiskVolumeCount"}, - {"name" : "MatchInterPodAffinity"}, - {"name" : "NoDiskConflict"}, - {"name" : "GeneralPredicates"}, - {"name" : "CheckNodeMemoryPressure"}, - {"name" : "CheckNodeDiskPressure"}, - {"name" : "CheckNodeCondition"}, - {"name" : "PodToleratesNodeTaints"}, - {"name" : "CheckVolumeBinding"} - ], -"priorities" : [ - {"name" : "SelectorSpreadPriority", "weight" : 1}, - {"name" : "InterPodAffinityPriority", "weight" : 1}, - {"name" : "LeastRequestedPriority", "weight" : 1}, - {"name" : "BalancedResourceAllocation", "weight" : 1}, - {"name" : "NodePreferAvoidPodsPriority", "weight" : 1}, - {"name" : "NodeAffinityPriority", "weight" : 1}, - {"name" : "TaintTolerationPriority", "weight" : 1} - ], -"hardPodAffinitySymmetricWeight" : 10 -} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index b42ad7cfb..b1178f420 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -31,10 +31,6 @@ spec: - scheduler - --leader-elect=true - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml -{% if volume_cross_zone_attachment %} - - --use-legacy-policy-config - - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml -{% endif %} - --profiling={{ kube_profiling }} - --v={{ kube_log_level }} {% if kube_feature_gates %} @@ -69,11 +65,6 @@ spec: - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" name: kubeconfig readOnly: true -{% if volume_cross_zone_attachment %} - - mountPath: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" - name: kube-scheduler-policy - readOnly: true -{% endif %} volumes: - name: ssl-certs-host hostPath: @@ -89,8 +80,3 @@ spec: - name: kubeconfig hostPath: path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" -{% if volume_cross_zone_attachment %} - - name: kube-scheduler-policy - hostPath: - path: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" -{% endif %} diff --git a/roles/kubernetes/node/templates/openstack-cloud-config.j2 b/roles/kubernetes/node/templates/openstack-cloud-config.j2 index cf9c74226..81877c895 100644 --- a/roles/kubernetes/node/templates/openstack-cloud-config.j2 +++ b/roles/kubernetes/node/templates/openstack-cloud-config.j2 @@ -20,10 +20,13 @@ domain-id ="{{ openstack_domain_id }}" ca-file="{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} -{% if openstack_blockstorage_version is defined %} [BlockStorage] +{% if openstack_blockstorage_version is defined %} bs-version={{ openstack_blockstorage_version }} {% endif %} +{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %} +ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }} +{% endif %} {% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %} [LoadBalancer] diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 2f22545c9..359bad349 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -249,6 +249,7 @@ cert_manager_enabled: false ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461) # openstack_blockstorage_version: "v1/v2/auto (default)" +openstack_blockstorage_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}" ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables. openstack_lbaas_enabled: false # openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP" @@ -293,11 +294,6 @@ vault_secrets_dir: "{{ vault_base_dir }}/secrets" local_volume_provisioner_base_dir: /mnt/disks local_volume_provisioner_mount_dir: /mnt/disks -## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. -## See https://github.com/kubernetes-incubator/kubespray/issues/2141 -## Set this variable to true to get rid of this issue -volume_cross_zone_attachment: false - # weave's network password for encryption # if null then no network encryption # you can use --extra-vars to pass the password in command line