feat: update kube-ovn to 0.8.0

roles/kube-ovn/defaults/main.yml

@@ -4,7 +4,7 @@ OVN_DB_NODE: "{{ groups['kube-master'][0] }}"
 kube_ovn_default_cidr: "{{ CLUSTER_CIDR }}"
 kube_ovn_default_gateway: "{{ CLUSTER_CIDR | ipaddr('net') | ipaddr(1) | ipaddr('address') }}"
 kube_ovn_node_switch_cidr: 100.64.0.0/16
-kube_ovn_enable_mirror: false
+kube_ovn_enable_mirror: true
 
 # 离线镜像tar包
-kube_ovn_offline: "kube_ovn_0.6.0.tar"
+kube_ovn_offline: "kube_ovn_0.8.0.tar"

roles/kube-ovn/tasks/main.yml

@@ -14,6 +14,9 @@
 - name: 配置 ovn.yaml 文件
   template: src=ovn.yaml.j2 dest=/opt/kube/kube-ovn/ovn.yaml
 
+- name: 配置 kubectl plugin
+  template: src=kubectl-ko.j2 dest=/usr/local/bin/kubectl-ko mode=0755
+
 # 【可选】推送离线镜像，可以忽略执行错误
 - block:
     - name: 检查是否已下载离线kube_ovn镜像

roles/kube-ovn/templates/crd.yaml.j2

@@ -12,6 +12,19 @@ spec:
     kind: IP
     shortNames:
       - ip
+  additionalPrinterColumns:
+    - name: IP
+      type: string
+      JSONPath: .spec.ipAddress
+    - name: Mac
+      type: string
+      JSONPath: .spec.macAddress
+    - name: Node
+      type: string
+      JSONPath: .spec.nodeName
+    - name: Subnet
+      type: string
+      JSONPath: .spec.subnet
 ---
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
@@ -27,6 +40,8 @@ spec:
     kind: Subnet
     shortNames:
       - subnet
+  subresources:
+    status: {}
   additionalPrinterColumns:
     - name: Protocol
       type: string
@@ -40,11 +55,23 @@ spec:
     - name: NAT
       type: boolean
       JSONPath: .spec.natOutgoing
+    - name: Default
+      type: boolean
+      JSONPath: .spec.default
+    - name: GatewayType
+      type: string
+      JSONPath: .spec.gatewayType
+    - name: Used
+      type: integer
+      JSONPath: .status.usingIPs
+    - name: Available
+      type: integer
+      JSONPath: .status.availableIPs
   validation:
     openAPIV3Schema:
       properties:
         spec:
-          required: ["cidrBlock","gateway"]
+          required: ["cidrBlock"]
           properties:
             cidrBlock:
               type: "string"

roles/kube-ovn/templates/kube-ovn.yaml.j2

@@ -38,7 +38,7 @@ spec:
       hostNetwork: true
       containers:
         - name: kube-ovn-controller
-          image: "index.alauda.cn/alaudak8s/kube-ovn-controller:v0.6.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-controller:v0.8.0"
           imagePullPolicy: IfNotPresent
           command:
           - /kube-ovn/start-controller.sh
@@ -112,7 +112,7 @@ spec:
       hostPID: true
       initContainers:
       - name: install-cni
-        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.6.0"
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.8.0"
         imagePullPolicy: IfNotPresent
         command: ["/kube-ovn/install-cni.sh"]
         volumeMounts:
@@ -122,7 +122,7 @@ spec:
             name: cni-bin
       containers:
       - name: cni-server
-        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.6.0"
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.8.0"
         imagePullPolicy: IfNotPresent
         command:
           - sh
@@ -177,3 +177,106 @@ spec:
         - name: cni-bin
           hostPath:
             path: {{ bin_dir }}
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+spec:
+  selector:
+    matchLabels:
+      app: kube-ovn-pinger
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-pinger
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: ovn
+      hostPID: true
+      containers:
+        - name: pinger
+          image: "index.alauda.cn/alaudak8s/kube-ovn-pinger:v0.8.0"
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            runAsUser: 0
+            privileged: false
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: host-modules
+              readOnly: true
+            - mountPath: /run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /sys
+              name: host-sys
+              readOnly: true
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /var/log/openvswitch
+              name: host-log
+          resources:
+            requests:
+              cpu: 100m
+              memory: 300Mi
+            limits:
+              cpu: 200m
+              memory: 400Mi
+      nodeSelector:
+        beta.kubernetes.io/os: "linux"
+      volumes:
+        - name: host-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-log
+          hostPath:
+            path: /var/log/openvswitch
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-pinger
+spec:
+  selector:
+    app: kube-ovn-pinger
+  ports:
+    - port: 8080
+      name: http

roles/kube-ovn/templates/kubectl-ko.j2

@@ -0,0 +1,182 @@
+#!/bin/bash
+set -euo pipefail
+
+KUBE_OVN_NS=kube-ovn
+CENTRAL_POD=
+
+showHelp(){
+  echo "kubectl ko {subcommand} [option...]"
+  echo "Available Subcommands:"
+  echo "  nbctl [ovn-nbctl options ...]    invoke ovn-nbctl"
+  echo "  sbctl [ovn-sbctl options ...]    invoke ovn-sbctl"
+  echo "  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic"
+  echo "  trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]    trace ovn microflow of specific packet"
+  echo "  diagnose {all|node} [nodename]    diagnose connectivity of all nodes or a specific node"
+}
+
+tcpdump(){
+  namespacedPod="$1"; shift
+  namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
+  podName=$(echo "$namespacedPod" | cut -d "/" -f2)
+  if [ "$podName" = "$namespacedPod" ]; then
+    nodeName=$(kubectl get pod "$podName" -o jsonpath={.spec.nodeName})
+    mac=$(kubectl get pod "$podName" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
+    hostNetwork=$(kubectl get pod "$podName" -o jsonpath={.spec.hostNetwork})
+  else
+    nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
+    mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
+    hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
+  fi
+
+  if [ -z "$nodeName" ]; then
+    echo "Pod $namespacedPod not exists on any node"
+    exit 1
+  fi
+
+  if [ -z "$mac" ] && [ "$hostNetwork" != "true" ]; then
+     echo "pod mac address not ready"
+     exit 1
+  fi
+
+  ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide| grep kube-ovn-cni| grep " $nodeName " | awk '{print $1}')
+  if [ -z "$ovnCni" ]; then
+    echo "kube-ovn-cni not exist on node $nodeName"
+    exit 1
+  fi
+
+  if [ "$hostNetwork" = "true" ]; then
+    set -x
+    kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@"
+  else
+    nicName=$(kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface mac_in_use="${mac//:/\\:}" | tr -d '\r')
+    if [ -z "$nicName" ]; then
+      echo "nic doesn't exist on node $nodeName"
+      exit 1
+    fi
+    set -x
+    kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn -i "$nicName" "$@"
+  fi
+}
+
+trace(){
+  namespacedPod="$1"
+  namespace=$(echo "$1" | cut -d "/" -f1)
+  podName=$(echo "$1" | cut -d "/" -f2)
+  if [ "$podName" = "$1" ]; then
+    echo "namespace is required"
+    exit 1
+  fi
+
+  podIP=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address})
+  mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
+  ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
+  hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
+
+  if [ "$hostNetwork" = "true" ]; then
+    echo "Can not trace host network pod"
+    exit 1
+  fi
+
+  if [ -z "$ls" ]; then
+    echo "pod address not ready"
+    exit 1
+  fi
+
+  gwMac=$(kubectl exec -it $CENTRAL_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r')
+
+  if [ -z "$gwMac" ]; then
+    echo "get gw mac failed"
+    exit 1
+  fi
+
+  dst="$2"
+  if [ -z "$dst" ]; then
+    echo "need a target ip address"
+    exit 1
+  fi
+
+  type="$3"
+
+  case $type in
+    icmp)
+      set -x
+      kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
+      ;;
+    tcp|udp)
+      set -x
+      kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
+      ;;
+    *)
+      echo "type $type not supported"
+      echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]"
+      ;;
+  esac
+}
+
+diagnose(){
+  type="$1"
+  case $type in
+    all)
+      pingers=$(kubectl get pod -n $KUBE_OVN_NS | grep kube-ovn-pinger | awk '{print $1}')
+      for pinger in $pingers
+      do
+        nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName})
+        echo "### start to diagnose node $nodeName"
+        kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
+        echo "### finish diagnose node $nodeName"
+        echo ""
+      done
+      ;;
+    node)
+      node="$2"
+      pinger=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep kube-ovn-pinger | grep " $node " | awk '{print $1}')
+      echo "### start to diagnose node $node"
+      kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
+      echo "### finish diagnose node $node"
+      echo ""
+      ;;
+    *)
+      echo "type $type not supported"
+      echo "kubectl ko diagnose {all|node} [nodename]"
+      ;;
+    esac
+}
+
+getOvnCentralPod(){
+    centralPod=$(kubectl get pod -n $KUBE_OVN_NS | grep ovn-central | head -n 1 | awk '{print $1}')
+    if [ -z "$centralPod" ]; then
+      echo "ovn-central not exists"
+      exit 1
+    fi
+    CENTRAL_POD=$centralPod
+}
+
+if [ $# -lt 1 ]; then
+  showHelp
+  exit 0
+else
+  subcommand="$1"; shift
+fi
+
+getOvnCentralPod
+
+case $subcommand in
+  nbctl)
+    kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@"
+    ;;
+  sbctl)
+    kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@"
+    ;;
+  tcpdump)
+    tcpdump "$@"
+    ;;
+  trace)
+    trace "$@"
+    ;;
+  diagnose)
+    diagnose "$@"
+    ;;
+  *)
+    showHelp
+    ;;
+esac

roles/kube-ovn/templates/ovn.yaml.j2

@@ -29,6 +29,7 @@ rules:
       - "kubeovn.io"
     resources:
       - subnets
+      - subnets/status
       - ips
     verbs:
       - "*"
@@ -49,10 +50,13 @@ rules:
   - apiGroups:
       - ""
       - networking.k8s.io
+      - apps
     resources:
       - networkpolicies
       - services
       - endpoints
+      - statefulsets
+      - daemonsets
     verbs:
       - get
       - list
@@ -154,7 +158,7 @@ spec:
       hostNetwork: true
       containers:
         - name: ovn-central
-          image: "index.alauda.cn/alaudak8s/kube-ovn-db:v0.6.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-db:v0.8.0"
           imagePullPolicy: IfNotPresent
           env:
             - name: POD_IP
@@ -241,7 +245,7 @@ spec:
       hostPID: true
       containers:
         - name: openvswitch
-          image: "index.alauda.cn/alaudak8s/kube-ovn-node:v0.6.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-node:v0.8.0"
           imagePullPolicy: IfNotPresent
           securityContext:
             runAsUser: 0