From c87bb2f239b3f309542accd822b14c988956fd82 Mon Sep 17 00:00:00 2001 From: Virgil Chereches Date: Fri, 12 Jan 2018 07:07:02 +0000 Subject: [PATCH 1/4] Fix for Issue #2141 --- roles/kubernetes/master/tasks/static-pod-setup.yml | 9 ++++++++- .../templates/manifests/kube-scheduler.manifest.j2 | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml index cd6c09194..151f7ad54 100644 --- a/roles/kubernetes/master/tasks/static-pod-setup.yml +++ b/roles/kubernetes/master/tasks/static-pod-setup.yml @@ -9,6 +9,13 @@ - meta: flush_handlers +- name: Write kube-scheduler policy file + template: + src: kube-scheduler-policy.yaml.j2 + dest: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" + tags: + - kube-scheduler + - name: Write kube-scheduler kubeconfig template: src: kube-scheduler-kubeconfig.yaml.j2 @@ -39,4 +46,4 @@ tags: - kube-controller-manager -- meta: flush_handlers \ No newline at end of file +- meta: flush_handlers diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index d50c10ed7..fd1d24547 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -28,6 +28,9 @@ spec: - scheduler - --leader-elect=true - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml +{% if cloud_provider == 'openstack' %} + - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml +{% endif %} - --profiling=false - --v={{ kube_log_level }} {% if kube_feature_gates %} @@ -62,6 +65,11 @@ spec: - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" name: kubeconfig readOnly: true +{% if cloud_provider == 'openstack' %} + - mountPath: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" + name: kube-scheduler-policy + readOnly: true +{% endif %} volumes: - name: ssl-certs-host hostPath: @@ -77,3 +85,8 @@ spec: - name: kubeconfig hostPath: path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" +{% if cloud_provider == 'openstack' %} + - name: kube-scheduler-policy + hostPath: + path: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" +{% endif %} From 8c45c88d15153d4f5cabfa0ace08ff23b1ca108b Mon Sep 17 00:00:00 2001 From: Virgil Chereches Date: Fri, 12 Jan 2018 07:15:35 +0000 Subject: [PATCH 2/4] Fix for Issue #2141 - added policy file --- .../templates/kube-scheduler-policy.yaml.j2 | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 diff --git a/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 b/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 new file mode 100644 index 000000000..6616adc6f --- /dev/null +++ b/roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2 @@ -0,0 +1,18 @@ +{ +"kind" : "Policy", +"apiVersion" : "v1", +"predicates" : [ + {"name" : "PodFitsHostPorts"}, + {"name" : "PodFitsResources"}, + {"name" : "NoDiskConflict"}, + {"name" : "MatchNodeSelector"}, + {"name" : "HostName"} + ], +"priorities" : [ + {"name" : "LeastRequestedPriority", "weight" : 1}, + {"name" : "BalancedResourceAllocation", "weight" : 1}, + {"name" : "ServiceSpreadingPriority", "weight" : 1}, + {"name" : "EqualPriority", "weight" : 1} + ], +"hardPodAffinitySymmetricWeight" : 10 +} From 3125f93b3f8a843dd9726e727a46a6a306fc348c Mon Sep 17 00:00:00 2001 From: Virgil Chereches Date: Thu, 18 Jan 2018 10:55:23 +0000 Subject: [PATCH 3/4] Added disable_volume_zone_conflict variable --- inventory/group_vars/k8s-cluster.yml | 5 +++++ roles/kubernetes/master/defaults/main.yml | 3 +++ .../master/templates/manifests/kube-scheduler.manifest.j2 | 6 +++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index 800d9dbb9..b925f5d7a 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -179,3 +179,8 @@ local_volumes_enabled: false ## Supplementary addresses that can be added in kubernetes ssl keys. ## That can be usefull for example to setup a keepalived virtual IP # supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3] + +## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. +## See https://github.com/kubernetes-incubator/kubespray/issues/2141 +## Set this variable to true to get rid of this issue +disable_volume_zone_conflict: false diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index fb19290cd..4615591a7 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -80,3 +80,6 @@ scheduler_custom_flags: [] # kubeadm settings # Value of 0 means it never expires kubeadm_token_ttl: 0 + +## Variable for influencing kube-scheduler behaviour +disable_volume_zone_conflict: false diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index fd1d24547..7b7c2b804 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -28,7 +28,7 @@ spec: - scheduler - --leader-elect=true - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml -{% if cloud_provider == 'openstack' %} +{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml {% endif %} - --profiling=false @@ -65,7 +65,7 @@ spec: - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" name: kubeconfig readOnly: true -{% if cloud_provider == 'openstack' %} +{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} - mountPath: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" name: kube-scheduler-policy readOnly: true @@ -85,7 +85,7 @@ spec: - name: kubeconfig hostPath: path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" -{% if cloud_provider == 'openstack' %} +{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} - name: kube-scheduler-policy hostPath: path: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" From a4d142368bbda46f33c86258bcffd3baec26fdbc Mon Sep 17 00:00:00 2001 From: Virgil Chereches Date: Tue, 23 Jan 2018 13:14:00 +0000 Subject: [PATCH 4/4] Renamed variable from disable_volume_zone_conflict to volume_cross_zone_attachment and removed cloud provider condition; fix identation --- inventory/group_vars/k8s-cluster.yml | 2 +- roles/kubernetes/master/defaults/main.yml | 2 +- roles/kubernetes/master/tasks/static-pod-setup.yml | 4 ++-- .../master/templates/manifests/kube-scheduler.manifest.j2 | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index b925f5d7a..c50041db2 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -183,4 +183,4 @@ local_volumes_enabled: false ## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler. ## See https://github.com/kubernetes-incubator/kubespray/issues/2141 ## Set this variable to true to get rid of this issue -disable_volume_zone_conflict: false +volume_cross_zone_attachment: false diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 4615591a7..da1a1cced 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -82,4 +82,4 @@ scheduler_custom_flags: [] kubeadm_token_ttl: 0 ## Variable for influencing kube-scheduler behaviour -disable_volume_zone_conflict: false +volume_cross_zone_attachment: false diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml index 151f7ad54..e8308798f 100644 --- a/roles/kubernetes/master/tasks/static-pod-setup.yml +++ b/roles/kubernetes/master/tasks/static-pod-setup.yml @@ -11,8 +11,8 @@ - name: Write kube-scheduler policy file template: - src: kube-scheduler-policy.yaml.j2 - dest: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" + src: kube-scheduler-policy.yaml.j2 + dest: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" tags: - kube-scheduler diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 7b7c2b804..b13fc7fa3 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -28,7 +28,7 @@ spec: - scheduler - --leader-elect=true - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml -{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} +{% if volume_cross_zone_attachment %} - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml {% endif %} - --profiling=false @@ -65,7 +65,7 @@ spec: - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" name: kubeconfig readOnly: true -{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} +{% if volume_cross_zone_attachment %} - mountPath: "{{ kube_config_dir }}/kube-scheduler-policy.yaml" name: kube-scheduler-policy readOnly: true @@ -85,7 +85,7 @@ spec: - name: kubeconfig hostPath: path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml" -{% if cloud_provider == 'openstack' and disable_volume_zone_conflict %} +{% if volume_cross_zone_attachment %} - name: kube-scheduler-policy hostPath: path: "{{ kube_config_dir }}/kube-scheduler-policy.yaml"