From 1bb1ba22749ffe3a21a8107df99dfa6713e6e1c6 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Tue, 25 Dec 2018 16:32:20 +0800 Subject: [PATCH] Fix GPU node Scheduling --- .../node/templates/kubelet.kubeadm.env.j2 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 963a87b43..be0b34b17 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -88,6 +88,12 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set dummy = role_node_labels.append("node-role.kubernetes.io/node=''") %} {% endif %} +{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} +{% if inventory_hostname in nvidia_gpu_nodes %} +{% set dummy = role_node_labels.append('nvidia.com/gpu=true') %} +{% endif %} +{% endif %} + {% set inventory_node_labels = [] %} {% if node_labels is defined and node_labels is mapping %} {% for labelname, labelvalue in node_labels.items() %} @@ -96,6 +102,16 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% set all_node_labels = role_node_labels + inventory_node_labels %} +{# Kubelet node taints for gpu #} +{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} +{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %} +{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} +{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %} +{% set node_taints = [] %} +{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} +{% endif %} +{% endif %} + KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"