From 94839ffccb69b568b96ea35e08267a26a9434804 Mon Sep 17 00:00:00 2001 From: gjmzj Date: Mon, 15 May 2023 12:10:32 +0800 Subject: [PATCH] =?UTF-8?q?fix=20kube-ovn=20v1.11.5=20=E5=AE=89=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- example/config.yml | 3 - roles/kube-ovn/tasks/main.yml | 21 +- roles/kube-ovn/templates/coredns.yaml.j2 | 216 ++++++++++++++++++ .../templates/nodelocaldns-iptables.yaml.j2 | 212 +++++++++++++++++ .../templates/nodelocaldns-ipvs.yaml.j2 | 212 +++++++++++++++++ roles/kube-ovn/vars/main.yml | 4 +- 6 files changed, 661 insertions(+), 7 deletions(-) create mode 100644 roles/kube-ovn/templates/coredns.yaml.j2 create mode 100644 roles/kube-ovn/templates/nodelocaldns-iptables.yaml.j2 create mode 100644 roles/kube-ovn/templates/nodelocaldns-ipvs.yaml.j2 diff --git a/example/config.yml b/example/config.yml index 859c921..a988b39 100644 --- a/example/config.yml +++ b/example/config.yml @@ -148,9 +148,6 @@ cilium_hubble_enabled: false cilium_hubble_ui_enabled: false # ------------------------------------------- kube-ovn -# [kube-ovn]选择 OVN DB and OVN Control Plane 节点，默认为第一个master节点 -OVN_DB_NODE: "{{ groups['kube_master'][0] }}" - # [kube-ovn]离线镜像tar包 kube_ovn_ver: "__kube_ovn__" diff --git a/roles/kube-ovn/tasks/main.yml b/roles/kube-ovn/tasks/main.yml index b1724e9..13f1f4f 100644 --- a/roles/kube-ovn/tasks/main.yml +++ b/roles/kube-ovn/tasks/main.yml @@ -3,8 +3,23 @@ file: path=/etc/cni/net.d/10-default.conf state=absent - block: - - name: 准备安装文件install - template: src=install.sh.j2 dest={{ cluster_dir }}/yml/install.sh + - name: 准备安装相关文件 + template: src={{ item }}.j2 dest={{ cluster_dir }}/yml/{{ item }} + with_items: + - "install.sh" + - "coredns.yaml" + + - name: 准备dnscache的部署文件 + template: src=nodelocaldns-ipvs.yaml.j2 dest={{ cluster_dir }}/yml/nodelocaldns.yaml + when: "PROXY_MODE == 'ipvs'" + + - name: 准备dnscache的部署文件 + template: src=nodelocaldns-iptables.yaml.j2 dest={{ cluster_dir }}/yml/nodelocaldns.yaml + when: "PROXY_MODE == 'iptables'" + + - name: 创建coredns,dnscache部署 + shell: "{{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/coredns.yaml && \ + {{ base_dir }}/bin/kubectl apply -f {{ cluster_dir }}/yml/nodelocaldns.yaml" - name: 安装kube-ovn网络 shell: 'export PATH="{{ base_dir }}/bin/:$PATH"; cd {{ cluster_dir }}/yml/ && \ @@ -15,7 +30,7 @@ # 等待网络插件部署成功 - name: 轮询等待kube-ovn 运行，视下载镜像速度而定 - shell: "{{ base_dir }}/bin/kubectl get pod -n kube-ovn -o wide|grep 'kube-ovn-cni'|grep ' {{ K8S_NODENAME }} '|awk '{print $3}'" + shell: "{{ base_dir }}/bin/kubectl get pod -n kube-system -o wide|grep 'kube-ovn-cni'|grep ' {{ K8S_NODENAME }} '|awk '{print $3}'" register: pod_status until: pod_status.stdout == "Running" retries: 15 diff --git a/roles/kube-ovn/templates/coredns.yaml.j2 b/roles/kube-ovn/templates/coredns.yaml.j2 new file mode 100644 index 0000000..df8a747 --- /dev/null +++ b/roles/kube-ovn/templates/coredns.yaml.j2 @@ -0,0 +1,216 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes {{ CLUSTER_DNS_DOMAIN }} in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + reload + loadbalance + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + priorityClassName: system-cluster-critical + serviceAccountName: coredns + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["kube-dns"] + topologyKey: kubernetes.io/hostname + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + nodeSelector: + kubernetes.io/os: linux + containers: + - name: coredns + image: easzlab.io.local:5000/coredns/coredns:{{ corednsVer }} + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 300Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: {{ CLUSTER_DNS_SVC_IP }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP diff --git a/roles/kube-ovn/templates/nodelocaldns-iptables.yaml.j2 b/roles/kube-ovn/templates/nodelocaldns-iptables.yaml.j2 new file mode 100644 index 0000000..ff60cc1 --- /dev/null +++ b/roles/kube-ovn/templates/nodelocaldns-iptables.yaml.j2 @@ -0,0 +1,212 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ CLUSTER_DNS_DOMAIN }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ LOCAL_DNS_CACHE }} {{ CLUSTER_DNS_SVC_IP }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + health {{ LOCAL_DNS_CACHE }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} {{ CLUSTER_DNS_SVC_IP }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} {{ CLUSTER_DNS_SVC_IP }} + forward . __PILLAR__CLUSTER__DNS__ { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} {{ CLUSTER_DNS_SVC_IP }} + forward . __PILLAR__UPSTREAM__SERVERS__ + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + #image: k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0 + image: easzlab.io.local:5000/easzlab/k8s-dns-node-cache:{{ dnsNodeCacheVer }} + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ LOCAL_DNS_CACHE }},{{ CLUSTER_DNS_SVC_IP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + capabilities: + add: + - NET_ADMIN + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ LOCAL_DNS_CACHE }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns diff --git a/roles/kube-ovn/templates/nodelocaldns-ipvs.yaml.j2 b/roles/kube-ovn/templates/nodelocaldns-ipvs.yaml.j2 new file mode 100644 index 0000000..837b16f --- /dev/null +++ b/roles/kube-ovn/templates/nodelocaldns-ipvs.yaml.j2 @@ -0,0 +1,212 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + {{ CLUSTER_DNS_DOMAIN }}:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ LOCAL_DNS_CACHE }} + forward . {{ CLUSTER_DNS_SVC_IP }} { + force_tcp + } + prometheus :9253 + health {{ LOCAL_DNS_CACHE }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} + forward . {{ CLUSTER_DNS_SVC_IP }} { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} + forward . {{ CLUSTER_DNS_SVC_IP }} { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ LOCAL_DNS_CACHE }} + forward . __PILLAR__CLUSTER__DNS__ + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + #image: k8s.gcr.io/dns/k8s-dns-node-cache:1.16.0 + image: easzlab.io.local:5000/easzlab/k8s-dns-node-cache:{{ dnsNodeCacheVer }} + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ LOCAL_DNS_CACHE }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + capabilities: + add: + - NET_ADMIN + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ LOCAL_DNS_CACHE }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns diff --git a/roles/kube-ovn/vars/main.yml b/roles/kube-ovn/vars/main.yml index 35a9c0c..94ad1da 100644 --- a/roles/kube-ovn/vars/main.yml +++ b/roles/kube-ovn/vars/main.yml @@ -1,3 +1,5 @@ # CLUSTER_CIDR_GW 作为 POD_GATEWAY，选取CLUSTER_CIDR 网段中的第一个地址 - CLUSTER_CIDR_GW: "{{ CLUSTER_CIDR.split('.')[0] }}.{{ CLUSTER_CIDR.split('.')[1] }}.{{ CLUSTER_CIDR.split('.')[2] }}.{{ CLUSTER_CIDR.split('.')[3]|int + 1 }}" + +# coredns 服务地址，根据SERVICE_CIDR 设置，默认选择网段第二个地址 +CLUSTER_DNS_SVC_IP: "{{ SERVICE_CIDR.split('.')[0] }}.{{ SERVICE_CIDR.split('.')[1] }}.{{ SERVICE_CIDR.split('.')[2] }}.{{ SERVICE_CIDR.split('.')[3]|int + 2 }}"