mirror of https://github.com/easzlab/kubeasz.git
289 lines
9.6 KiB
Django/Jinja
289 lines
9.6 KiB
Django/Jinja
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
KUBE_OVN_NS=kube-system
|
|
OVN_NB_POD=
|
|
OVN_SB_POD=
|
|
|
|
showHelp(){
|
|
echo "kubectl ko {subcommand} [option...]"
|
|
echo "Available Subcommands:"
|
|
echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl"
|
|
echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl"
|
|
echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node"
|
|
echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic"
|
|
echo " trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet"
|
|
echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node"
|
|
}
|
|
|
|
tcpdump(){
|
|
namespacedPod="$1"; shift
|
|
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
|
|
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
|
|
if [ "$podName" = "$namespacedPod" ]; then
|
|
nodeName=$(kubectl get pod "$podName" -o jsonpath={.spec.nodeName})
|
|
mac=$(kubectl get pod "$podName" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
|
|
hostNetwork=$(kubectl get pod "$podName" -o jsonpath={.spec.hostNetwork})
|
|
else
|
|
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
|
|
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
|
|
fi
|
|
|
|
if [ -z "$nodeName" ]; then
|
|
echo "Pod $namespacedPod not exists on any node"
|
|
exit 1
|
|
fi
|
|
|
|
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide| grep kube-ovn-cni| grep " $nodeName " | awk '{print $1}')
|
|
if [ -z "$ovnCni" ]; then
|
|
echo "kube-ovn-cni not exist on node $nodeName"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$hostNetwork" = "true" ]; then
|
|
set -x
|
|
kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@"
|
|
else
|
|
nicName=$(kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
|
|
if [ -z "$nicName" ]; then
|
|
echo "nic doesn't exist on node $nodeName"
|
|
exit 1
|
|
fi
|
|
set -x
|
|
kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn -i "$nicName" "$@"
|
|
fi
|
|
}
|
|
|
|
trace(){
|
|
namespacedPod="$1"
|
|
namespace=$(echo "$1" | cut -d "/" -f1)
|
|
podName=$(echo "$1" | cut -d "/" -f2)
|
|
if [ "$podName" = "$1" ]; then
|
|
echo "namespace is required"
|
|
exit 1
|
|
fi
|
|
|
|
podIP=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address})
|
|
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
|
|
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
|
|
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
|
|
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
|
|
|
|
if [ "$hostNetwork" = "true" ]; then
|
|
echo "Can not trace host network pod"
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$ls" ]; then
|
|
echo "pod address not ready"
|
|
exit 1
|
|
fi
|
|
|
|
gwMac=$(kubectl exec -it $OVN_NB_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r')
|
|
|
|
if [ -z "$gwMac" ]; then
|
|
echo "get gw mac failed"
|
|
exit 1
|
|
fi
|
|
|
|
dst="$2"
|
|
if [ -z "$dst" ]; then
|
|
echo "need a target ip address"
|
|
exit 1
|
|
fi
|
|
|
|
type="$3"
|
|
|
|
case $type in
|
|
icmp)
|
|
set -x
|
|
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
|
|
;;
|
|
tcp|udp)
|
|
set -x
|
|
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
|
|
;;
|
|
*)
|
|
echo "type $type not supported"
|
|
echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
set +x
|
|
echo "--------"
|
|
echo "Start OVS Tracing"
|
|
echo ""
|
|
echo ""
|
|
|
|
ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
|
|
if [ -z "$ovsPod" ]; then
|
|
echo "ovs pod doesn't exist on node $nodeName"
|
|
exit 1
|
|
fi
|
|
|
|
inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
|
|
case $type in
|
|
icmp)
|
|
set -x
|
|
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac"
|
|
;;
|
|
tcp|udp)
|
|
set -x
|
|
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4"
|
|
;;
|
|
*)
|
|
echo "type $type not supported"
|
|
echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]"
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
vsctl(){
|
|
nodeName="$1"; shift
|
|
kubectl get no "$nodeName" > /dev/null
|
|
ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
|
|
if [ -z "$ovsPod" ]; then
|
|
echo "ovs pod doesn't exist on node $nodeName"
|
|
exit 1
|
|
fi
|
|
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl "$@"
|
|
}
|
|
|
|
diagnose(){
|
|
kubectl get crd subnets.kubeovn.io
|
|
kubectl get crd ips.kubeovn.io
|
|
kubectl get svc kube-dns -n kube-system
|
|
kubectl get svc kubernetes -n default
|
|
|
|
kubectl get no -o wide
|
|
kubectl ko nbctl show
|
|
kubectl ko sbctl show
|
|
|
|
checkDaemonSet kube-proxy
|
|
checkDeployment ovn-central
|
|
checkDeployment kube-ovn-controller
|
|
checkDaemonSet kube-ovn-cni
|
|
checkDaemonSet ovs-ovn
|
|
checkDeployment coredns
|
|
type="$1"
|
|
case $type in
|
|
all)
|
|
echo "### kube-ovn-controller recent log"
|
|
set +e
|
|
kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d)
|
|
set -e
|
|
echo ""
|
|
pingers=$(kubectl get pod -n $KUBE_OVN_NS | grep kube-ovn-pinger | awk '{print $1}')
|
|
for pinger in $pingers
|
|
do
|
|
nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName})
|
|
echo "### start to diagnose node $nodeName"
|
|
echo "#### ovn-controller log:"
|
|
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log
|
|
echo ""
|
|
echo "#### ovs-vsctl show results:"
|
|
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- ovs-vsctl show
|
|
echo ""
|
|
echo "#### pinger diagnose results:"
|
|
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
|
|
echo "### finish diagnose node $nodeName"
|
|
echo ""
|
|
done
|
|
;;
|
|
node)
|
|
nodeName="$2"
|
|
kubectl get no "$nodeName" > /dev/null
|
|
pinger=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep kube-ovn-pinger | grep " $nodeName " | awk '{print $1}')
|
|
echo "### start to diagnose node nodeName"
|
|
echo "#### ovn-controller log:"
|
|
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log
|
|
echo ""
|
|
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
|
|
echo "### finish diagnose node nodeName"
|
|
echo ""
|
|
;;
|
|
*)
|
|
echo "type $type not supported"
|
|
echo "kubectl ko diagnose {all|node} [nodename]"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
getOvnCentralPod(){
|
|
NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
|
|
if [ -z "$NB_POD" ]; then
|
|
echo "nb leader not exists"
|
|
exit 1
|
|
fi
|
|
OVN_NB_POD=$NB_POD
|
|
SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
|
|
if [ -z "$SB_POD" ]; then
|
|
echo "nb leader not exists"
|
|
exit 1
|
|
fi
|
|
OVN_SB_POD=$SB_POD
|
|
}
|
|
|
|
checkDaemonSet(){
|
|
name="$1"
|
|
currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled})
|
|
desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled})
|
|
available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable})
|
|
ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady})
|
|
if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then
|
|
echo "ds $name ready"
|
|
else
|
|
echo "Error ds $name not ready"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
checkDeployment(){
|
|
name="$1"
|
|
ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas})
|
|
updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas})
|
|
desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas})
|
|
available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas})
|
|
if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then
|
|
echo "deployment $name ready"
|
|
else
|
|
echo "Error deployment $name not ready"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
if [ $# -lt 1 ]; then
|
|
showHelp
|
|
exit 0
|
|
else
|
|
subcommand="$1"; shift
|
|
fi
|
|
|
|
getOvnCentralPod
|
|
|
|
case $subcommand in
|
|
nbctl)
|
|
kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@"
|
|
;;
|
|
sbctl)
|
|
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@"
|
|
;;
|
|
vsctl)
|
|
vsctl "$@"
|
|
;;
|
|
tcpdump)
|
|
tcpdump "$@"
|
|
;;
|
|
trace)
|
|
trace "$@"
|
|
;;
|
|
diagnose)
|
|
diagnose "$@"
|
|
;;
|
|
*)
|
|
showHelp
|
|
;;
|
|
esac
|