diff --git a/.gitignore b/.gitignore index 1be0694..79a19c7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ hosts *.crt *.pem roles/prepare/files/ca* +.idea diff --git a/21.gpunode.yml b/21.gpunode.yml new file mode 100644 index 0000000..8a3d0bb --- /dev/null +++ b/21.gpunode.yml @@ -0,0 +1,3 @@ +- hosts: gpu-node + roles: + - gpu diff --git a/docs/08-配置GPU-node节点.md b/docs/08-配置GPU-node节点.md new file mode 100644 index 0000000..10e215b --- /dev/null +++ b/docs/08-配置GPU-node节点.md @@ -0,0 +1,46 @@ +## 08-配置GPU-node节点.md + +推荐阅读[官方GPU节点配置文档](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/) + +### 1. 允许k8s系统使用Device Plugins + +实现方式：通过修改kube-apiserver, kubelet, kube-proxy配置文件模板 + +因为GPU node是kube node的子集，正常执行`90.setup.yml`搭建k8s即可实现此目标 + +### 2. 配置GPU节点 +在[官方驱动网页](http://www.nvidia.com/Download/index.aspx?lang=en-uk)下载对应操作系统与显卡的驱动， +改名为nvidia-diag-driver-local-repo.deb， +并放入到invertory file中{{ base_dir }}路线下的 bin 文件夹中 + +执行命令：`ansible-playbook 21.gpunode.yml` + +相当于完成以下任务 + +#### 1). GPU 节点安装 nvidia driver +建议在node上只安装驱动，而不安装CUDA包，所有CUDA包都放到镜像中去，否则容易出现版本不匹配的问题。 + + + +#### 2). GPU 节点安装 nvidia docker 2.0 +这一部分可能失败，原因可能是系统所用docker版本与nvidia docker 2.0依赖的docker版本不一致， +具体参考[文档](https://github.com/NVIDIA/nvidia-docker/wiki/Frequently-Asked-Questions) + + +可以通过命令`apt-cache madison nvidia-docker2 nvidia-container-runtime`查询可以安装的nvidia docker 2.0 +的 docker 版本 + +可以通过命令`dpkg -l | grep docker`来查看实际安装docker 版本 + +nvidia docker 安装失败，可以[手动更新docker](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#upgrade-docker-ce), +再重新运行安装playbook + + +#### 3). GPU 节点配置 nvidia-container-runtime 为 docker default runtime + +通过修改`/etc/docker/daemon.json`实现 + +### 3. 配置Nvidia device plugin +执行命令 `kubectl create -f manifests/gpu-device-plugin/v1.9/nvidia-device-plugin.yml` + +可以通过执行`kubectl describe nodes | grep nvidia.com/gpu` 来查看GPU节点配置是否成功 \ No newline at end of file diff --git a/example/hosts.allinone.example b/example/hosts.allinone.example index b9aa8e4..6ada1f5 100644 --- a/example/hosts.allinone.example +++ b/example/hosts.allinone.example @@ -14,6 +14,11 @@ [kube-node] 192.168.1.1 NODE_ID=node1 NODE_IP="192.168.1.1" +#gpu-node 是 kube-node的子集 +[gpu-node] +192.168.1.1 NODE_IP="192.168.1.1" + + [kube-cluster:children] kube-node kube-master diff --git a/example/hosts.m-masters.example b/example/hosts.m-masters.example index 92e33c0..b9e763b 100644 --- a/example/hosts.m-masters.example +++ b/example/hosts.m-masters.example @@ -30,6 +30,12 @@ MASTER_PORT="8443" # api-server 服务端口 192.168.1.3 NODE_ID=node2 NODE_IP="192.168.1.3" 192.168.1.4 NODE_ID=node3 NODE_IP="192.168.1.4" +#gpu-node 是 kube-node的子集 +[gpu-node] +192.168.1.2 NODE_IP="192.168.1.2" +192.168.1.3 NODE_IP="192.168.1.3" + + [kube-cluster:children] kube-node kube-master diff --git a/example/hosts.s-master.example b/example/hosts.s-master.example index 2bb0b75..24ed476 100644 --- a/example/hosts.s-master.example +++ b/example/hosts.s-master.example @@ -18,6 +18,12 @@ 192.168.1.2 NODE_ID=node2 NODE_IP="192.168.1.2" 192.168.1.3 NODE_ID=node3 NODE_IP="192.168.1.3" +#gpu-node 是 kube-node的子集 +[gpu-node] +192.168.1.2 NODE_IP="192.168.1.2" +192.168.1.3 NODE_IP="192.168.1.3" + + [kube-cluster:children] kube-node kube-master diff --git a/manifests/gpu-device-plugin/v1.8/nvidia-device-plugin.yml b/manifests/gpu-device-plugin/v1.8/nvidia-device-plugin.yml new file mode 100644 index 0000000..f0901fe --- /dev/null +++ b/manifests/gpu-device-plugin/v1.8/nvidia-device-plugin.yml @@ -0,0 +1,24 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset +spec: + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + containers: + - image: nvidia/k8s-device-plugin:1.8 + name: nvidia-device-plugin-ctr + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins \ No newline at end of file diff --git a/manifests/gpu-device-plugin/v1.9/nvidia-device-plugin.yml b/manifests/gpu-device-plugin/v1.9/nvidia-device-plugin.yml new file mode 100644 index 0000000..7db7bda --- /dev/null +++ b/manifests/gpu-device-plugin/v1.9/nvidia-device-plugin.yml @@ -0,0 +1,25 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + containers: + - image: nvidia/k8s-device-plugin:1.9 + name: nvidia-device-plugin-ctr + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins \ No newline at end of file diff --git a/roles/gpu/files/daemon.json b/roles/gpu/files/daemon.json new file mode 100644 index 0000000..aab8bd6 --- /dev/null +++ b/roles/gpu/files/daemon.json @@ -0,0 +1,12 @@ +{ + "registry-mirrors": ["https://registry.docker-cn.com"], + "max-concurrent-downloads": 6, + "default-runtime": "nvidia", + "runtimes": { + "nvidia": { + "path": "/usr/bin/nvidia-container-runtime", + "runtimeArgs": [] + } + } +} + diff --git a/roles/gpu/tasks/main.yml b/roles/gpu/tasks/main.yml new file mode 100644 index 0000000..2f9ce85 --- /dev/null +++ b/roles/gpu/tasks/main.yml @@ -0,0 +1,30 @@ +- name: 下载nvidia driver安装包 + copy: src={{ base_dir }}/bin/nvidia-diag-driver-local-repo.deb dest=/tmp/nvidia-diag-driver-local-repo.deb + when: ansible_distribution == "Ubuntu" + +- name: 安装nvidia dirver驱动 + apt: + deb: /tmp/nvidia-diag-driver-local-repo.deb + when: ansible_distribution == "Ubuntu" + +- name: Add the package repositories 1 + shell: curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add - + when: ansible_distribution == "Ubuntu" + +- name: Add the package repositories 2 + shell: curl -s -L https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64/nvidia-docker.list | tee /etc/apt/sources.list.d/nvidia-docker.list + when: ansible_distribution == "Ubuntu" + +- name: Update apt-get and Install nvidia docker 2 + apt: + name: nvidia-docker2 + update_cache: yes + when: ansible_distribution == "Ubuntu" + +- name: 配置nvidia docker runtime + copy: src=daemon.json dest=/etc/docker/daemon.json + +- name: restart docker + service: + name: docker + state: restarted diff --git a/roles/kube-master/templates/kube-apiserver.service.j2 b/roles/kube-master/templates/kube-apiserver.service.j2 index 5fc4e07..e382fb7 100644 --- a/roles/kube-master/templates/kube-apiserver.service.j2 +++ b/roles/kube-master/templates/kube-apiserver.service.j2 @@ -31,6 +31,7 @@ ExecStart={{ bin_dir }}/kube-apiserver \ --audit-log-maxsize=100 \ --audit-log-path=/var/lib/audit.log \ --event-ttl=1h \ + --feature-gates="DevicePlugins=true" \ --v=2 Restart=on-failure RestartSec=5 diff --git a/roles/kube-node/templates/kube-proxy.service.j2 b/roles/kube-node/templates/kube-proxy.service.j2 index 6a680a3..e680a16 100644 --- a/roles/kube-node/templates/kube-proxy.service.j2 +++ b/roles/kube-node/templates/kube-proxy.service.j2 @@ -12,6 +12,7 @@ ExecStart={{ bin_dir }}/kube-proxy \ --hostname-override={{ NODE_IP }} \ --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \ --logtostderr=true \ + --feature-gates="DevicePlugins=true" \ --v=2 Restart=on-failure RestartSec=5 diff --git a/roles/kube-node/templates/kubelet.service.j2 b/roles/kube-node/templates/kubelet.service.j2 index b534c69..a4cc49a 100644 --- a/roles/kube-node/templates/kubelet.service.j2 +++ b/roles/kube-node/templates/kubelet.service.j2 @@ -23,6 +23,7 @@ ExecStart={{ bin_dir }}/kubelet \ --allow-privileged=true \ --fail-swap-on=false \ --logtostderr=true \ + --feature-gates="DevicePlugins=true" \ --v=2 #kubelet cAdvisor 默认在所有接口监听 4194 端口的请求, 以下iptables限制内网访问 ExecStartPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT