easzlab
/
kubeasz
mirror of https://github.com/easzlab/kubeasz.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: upgrade kube-ovn to 1.5.3 #958

pull/992/head
gjmzj 2021-01-18 10:21:48 +08:00
parent 88bf729126
commit a3d754ccf8
5 changed files with 645 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
roles/clean/tasks/clean_node.yml
View File

@ -125,6 +125,10 @@
- "/etc/origin/openvswitch/" - "/etc/origin/openvswitch/"
- "/etc/openvswitch/" - "/etc/openvswitch/"
- "/var/log/openvswitch/" - "/var/log/openvswitch/"
- "/var/run/ovn/"
- "/etc/origin/ovn/"
- "/etc/ovn/"
- "/var/log/ovn/"
# - name: cleanup networks1 # - name: cleanup networks1
# shell: "ip link del tunl0; \ # shell: "ip link del tunl0; \

245
roles/kube-ovn/templates/crd.yaml.j2
View File

@ -1,10 +1,59 @@
apiVersion: apiextensions.k8s.io/v1beta1 apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition kind: CustomResourceDefinition
metadata: metadata:
name: ips.kubeovn.io name: ips.kubeovn.io
spec: spec:
group: kubeovn.io group: kubeovn.io
version: v1 versions:
- name: v1
served: true
storage: true
additionalPrinterColumns:
- name: IP
type: string
jsonPath: .spec.ipAddress
- name: Mac
type: string
jsonPath: .spec.macAddress
- name: Node
type: string
jsonPath: .spec.nodeName
- name: Subnet
type: string
jsonPath: .spec.subnet
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
podName:
type: string
namespace:
type: string
subnet:
type: string
attachSubnets:
type: array
items:
type: string
nodeName:
type: string
ipAddress:
type: string
attachIps:
type: array
items:
type: string
macAddress:
type: string
attachMacs:
type: array
items:
type: string
containerID:
type: string
scope: Cluster scope: Cluster
names: names:
plural: ips plural: ips
@ -12,27 +61,116 @@ spec:
kind: IP kind: IP
shortNames: shortNames:
- ip - ip
additionalPrinterColumns:
- name: IP
type: string
JSONPath: .spec.ipAddress
- name: Mac
type: string
JSONPath: .spec.macAddress
- name: Node
type: string
JSONPath: .spec.nodeName
- name: Subnet
type: string
JSONPath: .spec.subnet
--- ---
apiVersion: apiextensions.k8s.io/v1beta1 apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition kind: CustomResourceDefinition
metadata: metadata:
name: subnets.kubeovn.io name: subnets.kubeovn.io
spec: spec:
group: kubeovn.io group: kubeovn.io
version: v1 versions:
- name: v1
served: true
storage: true
subresources:
status: {}
additionalPrinterColumns:
- name: Provider
type: string
jsonPath: .spec.provider
- name: Protocol
type: string
jsonPath: .spec.protocol
- name: CIDR
type: string
jsonPath: .spec.cidrBlock
- name: Private
type: boolean
jsonPath: .spec.private
- name: NAT
type: boolean
jsonPath: .spec.natOutgoing
- name: Default
type: boolean
jsonPath: .spec.default
- name: GatewayType
type: string
jsonPath: .spec.gatewayType
- name: Used
type: number
jsonPath: .status.usingIPs
- name: Available
type: number
jsonPath: .status.availableIPs
schema:
openAPIV3Schema:
type: object
properties:
status:
type: object
properties:
availableIPs:
type: number
usingIPs:
type: number
activateGateway:
type: string
conditions:
type: array
items:
type: object
properties:
type:
type: string
status:
type: string
reason:
type: string
message:
type: string
lastUpdateTime:
type: string
lastTransitionTime:
type: string
spec:
type: object
properties:
default:
type: boolean
protocol:
type: string
cidrBlock:
type: string
namespaces:
type: array
items:
type: string
gateway:
type: string
provider:
type: string
excludeIps:
type: array
items:
type: string
gatewayType:
type: string
allowSubnets:
type: array
items:
type: string
gatewayNode:
type: string
natOutgoing:
type: boolean
private:
type: boolean
vlan:
type: string
underlayGateway:
type: boolean
disableInterConnection:
type: boolean
scope: Cluster scope: Cluster
names: names:
plural: subnets plural: subnets
@ -40,40 +178,47 @@ spec:
kind: Subnet kind: Subnet
shortNames: shortNames:
- subnet - subnet
subresources:
status: {} ---
additionalPrinterColumns: apiVersion: apiextensions.k8s.io/v1
- name: Protocol kind: CustomResourceDefinition
type: string metadata:
JSONPath: .spec.protocol name: vlans.kubeovn.io
- name: CIDR spec:
type: string group: kubeovn.io
JSONPath: .spec.cidrBlock versions:
- name: Private - name: v1
type: boolean served: true
JSONPath: .spec.private storage: true
- name: NAT schema:
type: boolean
JSONPath: .spec.natOutgoing
- name: Default
type: boolean
JSONPath: .spec.default
- name: GatewayType
type: string
JSONPath: .spec.gatewayType
- name: Used
type: integer
JSONPath: .status.usingIPs
- name: Available
type: integer
JSONPath: .status.availableIPs
validation:
openAPIV3Schema: openAPIV3Schema:
type: object
properties: properties:
spec: spec:
required: ["cidrBlock"] type: object
properties: properties:
cidrBlock: vlanId:
type: "string" type: integer
gateway: providerInterfaceName:
type: "string" type: string
logicalInterfaceName:
type: string
subnet:
type: string
additionalPrinterColumns:
- name: VlanID
type: string
jsonPath: .spec.vlanId
- name: ProviderInterfaceName
type: string
jsonPath: .spec.providerInterfaceName
- name: Subnet
type: string
jsonPath: .spec.subnet
scope: Cluster
names:
plural: vlans
singular: vlan
kind: Vlan
shortNames:
- vlan

153
roles/kube-ovn/templates/kube-ovn.yaml.j2
View File

@ -3,7 +3,7 @@ kind: Deployment
apiVersion: apps/v1 apiVersion: apps/v1
metadata: metadata:
name: kube-ovn-controller name: kube-ovn-controller
namespace: kube-ovn namespace: kube-system
annotations: annotations:
kubernetes.io/description: | kubernetes.io/description: |
kube-ovn controller kube-ovn controller
@ -26,7 +26,6 @@ spec:
spec: spec:
tolerations: tolerations:
- operator: Exists - operator: Exists
effect: NoSchedule
affinity: affinity:
podAntiAffinity: podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution: requiredDuringSchedulingIgnoredDuringExecution:
@ -34,11 +33,12 @@ spec:
matchLabels: matchLabels:
app: kube-ovn-controller app: kube-ovn-controller
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
priorityClassName: system-cluster-critical
serviceAccountName: ovn serviceAccountName: ovn
hostNetwork: true hostNetwork: true
containers: containers:
- name: kube-ovn-controller - name: kube-ovn-controller
image: "index.alauda.cn/alaudak8s/kube-ovn-controller:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: command:
- /kube-ovn/start-controller.sh - /kube-ovn/start-controller.sh
@ -46,7 +46,12 @@ spec:
- --default-cidr={{ kube_ovn_default_cidr }} - --default-cidr={{ kube_ovn_default_cidr }}
- --default-gateway={{ kube_ovn_default_gateway }} - --default-gateway={{ kube_ovn_default_gateway }}
- --node-switch-cidr={{ kube_ovn_node_switch_cidr }} - --node-switch-cidr={{ kube_ovn_node_switch_cidr }}
- --network-type=geneve
- --default-interface-name=
- --default-vlan-id=100
env: env:
- name: ENABLE_SSL
value: "false"
- name: POD_NAME - name: POD_NAME
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -59,35 +64,46 @@ spec:
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: spec.nodeName fieldPath: spec.nodeName
volumeMounts:
- mountPath: /var/run/tls
name: kube-ovn-tls
readinessProbe: readinessProbe:
exec: exec:
command: command:
- nc - sh
- -z - /kube-ovn/kube-ovn-controller-healthcheck.sh
- -w3
- 127.0.0.1
- "10660"
periodSeconds: 3 periodSeconds: 3
timeoutSeconds: 45
livenessProbe: livenessProbe:
exec: exec:
command: command:
- nc - sh
- -z - /kube-ovn/kube-ovn-controller-healthcheck.sh
- -w3 initialDelaySeconds: 300
- 127.0.0.1
- "10660"
initialDelaySeconds: 30
periodSeconds: 7 periodSeconds: 7
failureThreshold: 5 failureThreshold: 5
timeoutSeconds: 45
resources:
requests:
cpu: 200m
memory: 200Mi
limits:
cpu: 1000m
memory: 1Gi
nodeSelector: nodeSelector:
beta.kubernetes.io/os: "linux" kubernetes.io/os: "linux"
volumes:
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls
--- ---
kind: DaemonSet kind: DaemonSet
apiVersion: apps/v1 apiVersion: apps/v1
metadata: metadata:
name: kube-ovn-cni name: kube-ovn-cni
namespace: kube-ovn namespace: kube-system
annotations: annotations:
kubernetes.io/description: | kubernetes.io/description: |
This daemon set launches the kube-ovn cni daemon. This daemon set launches the kube-ovn cni daemon.
@ -95,8 +111,6 @@ spec:
selector: selector:
matchLabels: matchLabels:
app: kube-ovn-cni app: kube-ovn-cni
updateStrategy:
type: OnDelete
template: template:
metadata: metadata:
labels: labels:
@ -106,26 +120,24 @@ spec:
spec: spec:
tolerations: tolerations:
- operator: Exists - operator: Exists
effect: NoSchedule priorityClassName: system-cluster-critical
serviceAccountName: ovn serviceAccountName: ovn
hostNetwork: true hostNetwork: true
hostPID: true hostPID: true
initContainers: initContainers:
- name: install-cni - name: install-cni
image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: ["/kube-ovn/install-cni.sh"] command: ["/kube-ovn/install-cni.sh"]
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
privileged: true privileged: true
volumeMounts: volumeMounts:
- mountPath: /etc/cni/net.d
name: cni-conf
- mountPath: /opt/cni/bin - mountPath: /opt/cni/bin
name: cni-bin name: cni-bin
containers: containers:
- name: cni-server - name: cni-server
image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: command:
- sh - sh
@ -133,10 +145,16 @@ spec:
args: args:
- --enable-mirror={{ kube_ovn_enable_mirror|string|lower }} - --enable-mirror={{ kube_ovn_enable_mirror|string|lower }}
- --service-cluster-ip-range={{ SERVICE_CIDR }} - --service-cluster-ip-range={{ SERVICE_CIDR }}
- --encap-checksum=true
- --iface=
- --network-type=geneve
- --default-interface-name=
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
privileged: true privileged: true
env: env:
- name: ENABLE_SSL
value: "false"
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -146,8 +164,15 @@ spec:
fieldRef: fieldRef:
fieldPath: spec.nodeName fieldPath: spec.nodeName
volumeMounts: volumeMounts:
- mountPath: /etc/cni/net.d
name: cni-conf
- mountPath: /run/openvswitch - mountPath: /run/openvswitch
name: host-run-ovs name: host-run-ovs
- mountPath: /run/ovn
name: host-run-ovn
- mountPath: /var/run/netns
name: host-ns
mountPropagation: HostToContainer
readinessProbe: readinessProbe:
exec: exec:
command: command:
@ -168,25 +193,41 @@ spec:
initialDelaySeconds: 30 initialDelaySeconds: 30
periodSeconds: 7 periodSeconds: 7
failureThreshold: 5 failureThreshold: 5
resources:
requests:
cpu: 200m
memory: 200Mi
limits:
cpu: 1000m
memory: 1Gi
nodeSelector: nodeSelector:
beta.kubernetes.io/os: "linux" kubernetes.io/os: "linux"
volumes: volumes:
- name: host-run-ovs - name: host-run-ovs
hostPath: hostPath:
path: /run/openvswitch path: /run/openvswitch
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: cni-conf - name: cni-conf
hostPath: hostPath:
path: /etc/cni/net.d path: /etc/cni/net.d
- name: cni-bin - name: cni-bin
hostPath: hostPath:
path: {{ bin_dir }} path: {{ bin_dir }}
- name: host-ns
hostPath:
path: /var/run/netns
--- ---
kind: DaemonSet kind: DaemonSet
apiVersion: apps/v1 apiVersion: apps/v1
metadata: metadata:
name: kube-ovn-pinger name: kube-ovn-pinger
namespace: kube-ovn namespace: kube-system
annotations:
kubernetes.io/description: |
This daemon set launches the openvswitch daemon.
spec: spec:
selector: selector:
matchLabels: matchLabels:
@ -202,17 +243,19 @@ spec:
spec: spec:
tolerations: tolerations:
- operator: Exists - operator: Exists
effect: NoSchedule
serviceAccountName: ovn serviceAccountName: ovn
hostPID: true hostPID: true
containers: containers:
- name: pinger - name: pinger
image: "index.alauda.cn/alaudak8s/kube-ovn-pinger:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114", "--external-dns=alauda.cn"]
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
privileged: false privileged: false
env: env:
- name: ENABLE_SSL
value: "false"
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:
fieldRef: fieldRef:
@ -237,22 +280,28 @@ spec:
name: host-run-ovs name: host-run-ovs
- mountPath: /var/run/openvswitch - mountPath: /var/run/openvswitch
name: host-run-ovs name: host-run-ovs
- mountPath: /var/run/ovn
name: host-run-ovn
- mountPath: /sys - mountPath: /sys
name: host-sys name: host-sys
readOnly: true readOnly: true
- mountPath: /etc/openvswitch - mountPath: /etc/openvswitch
name: host-config-openvswitch name: host-config-openvswitch
- mountPath: /var/log/openvswitch - mountPath: /var/log/openvswitch
name: host-log name: host-log-ovs
- mountPath: /var/log/ovn
name: host-log-ovn
- mountPath: /var/run/tls
name: kube-ovn-tls
resources: resources:
requests: requests:
cpu: 100m cpu: 100m
memory: 300Mi memory: 200Mi
limits: limits:
cpu: 200m cpu: 200m
memory: 400Mi memory: 400Mi
nodeSelector: nodeSelector:
beta.kubernetes.io/os: "linux" kubernetes.io/os: "linux"
volumes: volumes:
- name: host-modules - name: host-modules
hostPath: hostPath:
@ -260,21 +309,31 @@ spec:
- name: host-run-ovs - name: host-run-ovs
hostPath: hostPath:
path: /run/openvswitch path: /run/openvswitch
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: host-sys - name: host-sys
hostPath: hostPath:
path: /sys path: /sys
- name: host-config-openvswitch - name: host-config-openvswitch
hostPath: hostPath:
path: /etc/origin/openvswitch path: /etc/origin/openvswitch
- name: host-log - name: host-log-ovs
hostPath: hostPath:
path: /var/log/openvswitch path: /var/log/openvswitch
- name: host-log-ovn
hostPath:
path: /var/log/ovn
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls
--- ---
kind: Service kind: Service
apiVersion: v1 apiVersion: v1
metadata: metadata:
name: kube-ovn-pinger name: kube-ovn-pinger
namespace: kube-ovn namespace: kube-system
labels: labels:
app: kube-ovn-pinger app: kube-ovn-pinger
spec: spec:
@ -282,4 +341,32 @@ spec:
app: kube-ovn-pinger app: kube-ovn-pinger
ports: ports:
- port: 8080 - port: 8080
name: http name: metrics
---
kind: Service
apiVersion: v1
metadata:
name: kube-ovn-controller
namespace: kube-system
labels:
app: kube-ovn-controller
spec:
selector:
app: kube-ovn-controller
ports:
- port: 10660
name: metrics
---
kind: Service
apiVersion: v1
metadata:
name: kube-ovn-cni
namespace: kube-system
labels:
app: kube-ovn-cni
spec:
selector:
app: kube-ovn-cni
ports:
- port: 10665
name: metrics

150
roles/kube-ovn/templates/kubectl-ko.j2
View File

@ -1,14 +1,16 @@
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
KUBE_OVN_NS=kube-ovn KUBE_OVN_NS=kube-system
CENTRAL_POD= OVN_NB_POD=
OVN_SB_POD=
showHelp(){ showHelp(){
echo "kubectl ko {subcommand} [option...]" echo "kubectl ko {subcommand} [option...]"
echo "Available Subcommands:" echo "Available Subcommands:"
echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl"
echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl"
echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node"
echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic" echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic"
echo " trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet" echo " trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet"
echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node" echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node"
@ -24,7 +26,6 @@ tcpdump(){
hostNetwork=$(kubectl get pod "$podName" -o jsonpath={.spec.hostNetwork}) hostNetwork=$(kubectl get pod "$podName" -o jsonpath={.spec.hostNetwork})
else else
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
fi fi
@ -33,11 +34,6 @@ tcpdump(){
exit 1 exit 1
fi fi
if [ -z "$mac" ] && [ "$hostNetwork" != "true" ]; then
echo "pod mac address not ready"
exit 1
fi
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide| grep kube-ovn-cni| grep " $nodeName " | awk '{print $1}') ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide| grep kube-ovn-cni| grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then if [ -z "$ovnCni" ]; then
echo "kube-ovn-cni not exist on node $nodeName" echo "kube-ovn-cni not exist on node $nodeName"
@ -48,7 +44,7 @@ tcpdump(){
set -x set -x
kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@" kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@"
else else
nicName=$(kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface mac_in_use="${mac//:/\\:}" | tr -d '\r') nicName=$(kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName" echo "nic doesn't exist on node $nodeName"
exit 1 exit 1
@ -71,6 +67,7 @@ trace(){
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
if [ "$hostNetwork" = "true" ]; then if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod" echo "Can not trace host network pod"
@ -82,7 +79,7 @@ trace(){
exit 1 exit 1
fi fi
gwMac=$(kubectl exec -it $CENTRAL_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r') gwMac=$(kubectl exec -it $OVN_NB_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r')
if [ -z "$gwMac" ]; then if [ -z "$gwMac" ]; then
echo "get gw mac failed" echo "get gw mac failed"
@ -100,39 +97,111 @@ trace(){
case $type in case $type in
icmp) icmp)
set -x set -x
kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst" kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst"
;; ;;
tcp|udp) tcp|udp)
set -x set -x
kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4" kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4"
;; ;;
*) *)
echo "type $type not supported" echo "type $type not supported"
echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]" echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]"
exit 1
;;
esac
set +x
echo "--------"
echo "Start OVS Tracing"
echo ""
echo ""
ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
if [ -z "$ovsPod" ]; then
echo "ovs pod doesn't exist on node $nodeName"
exit 1
fi
inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace")
case $type in
icmp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac"
;;
tcp|udp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4"
;;
*)
echo "type $type not supported"
echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]"
exit 1
;; ;;
esac esac
} }
vsctl(){
nodeName="$1"; shift
kubectl get no "$nodeName" > /dev/null
ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}')
if [ -z "$ovsPod" ]; then
echo "ovs pod doesn't exist on node $nodeName"
exit 1
fi
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl "$@"
}
diagnose(){ diagnose(){
kubectl get crd subnets.kubeovn.io
kubectl get crd ips.kubeovn.io
kubectl get svc kube-dns -n kube-system
kubectl get svc kubernetes -n default
kubectl get no -o wide
kubectl ko nbctl show
kubectl ko sbctl show
checkDaemonSet kube-proxy
checkDeployment ovn-central
checkDeployment kube-ovn-controller
checkDaemonSet kube-ovn-cni
checkDaemonSet ovs-ovn
checkDeployment coredns
type="$1" type="$1"
case $type in case $type in
all) all)
echo "### kube-ovn-controller recent log"
set +e
kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d)
set -e
echo ""
pingers=$(kubectl get pod -n $KUBE_OVN_NS | grep kube-ovn-pinger | awk '{print $1}') pingers=$(kubectl get pod -n $KUBE_OVN_NS | grep kube-ovn-pinger | awk '{print $1}')
for pinger in $pingers for pinger in $pingers
do do
nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName}) nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName})
echo "### start to diagnose node $nodeName" echo "### start to diagnose node $nodeName"
echo "#### ovn-controller log:"
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log
echo ""
echo "#### ovs-vsctl show results:"
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- ovs-vsctl show
echo ""
echo "#### pinger diagnose results:"
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
echo "### finish diagnose node $nodeName" echo "### finish diagnose node $nodeName"
echo "" echo ""
done done
;; ;;
node) node)
node="$2" nodeName="$2"
pinger=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep kube-ovn-pinger | grep " $node " | awk '{print $1}') kubectl get no "$nodeName" > /dev/null
echo "### start to diagnose node $node" pinger=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep kube-ovn-pinger | grep " $nodeName " | awk '{print $1}')
echo "### start to diagnose node nodeName"
echo "#### ovn-controller log:"
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log
echo ""
kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job
echo "### finish diagnose node $node" echo "### finish diagnose node nodeName"
echo "" echo ""
;; ;;
*) *)
@ -143,12 +212,46 @@ diagnose(){
} }
getOvnCentralPod(){ getOvnCentralPod(){
centralPod=$(kubectl get pod -n $KUBE_OVN_NS | grep ovn-central | head -n 1 | awk '{print $1}') NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
if [ -z "$centralPod" ]; then if [ -z "$NB_POD" ]; then
echo "ovn-central not exists" echo "nb leader not exists"
exit 1
fi
OVN_NB_POD=$NB_POD
SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}')
if [ -z "$SB_POD" ]; then
echo "nb leader not exists"
exit 1
fi
OVN_SB_POD=$SB_POD
}
checkDaemonSet(){
name="$1"
currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled})
desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled})
available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable})
ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady})
if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then
echo "ds $name ready"
else
echo "Error ds $name not ready"
exit 1
fi
}
checkDeployment(){
name="$1"
ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas})
updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas})
desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas})
available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas})
if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then
echo "deployment $name ready"
else
echo "Error deployment $name not ready"
exit 1 exit 1
fi fi
CENTRAL_POD=$centralPod
} }
if [ $# -lt 1 ]; then if [ $# -lt 1 ]; then
@ -162,10 +265,13 @@ getOvnCentralPod
case $subcommand in case $subcommand in
nbctl) nbctl)
kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@" kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@"
;; ;;
sbctl) sbctl)
kubectl exec "$CENTRAL_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@" kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@"
;;
vsctl)
vsctl "$@"
;; ;;
tcpdump) tcpdump)
tcpdump "$@" tcpdump "$@"

171
roles/kube-ovn/templates/ovn.yaml.j2
View File

@ -1,22 +1,44 @@
apiVersion: v1 apiVersion: policy/v1beta1
kind: Namespace kind: PodSecurityPolicy
metadata: metadata:
name: kube-ovn name: kube-ovn
annotations:
seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
spec:
privileged: true
allowPrivilegeEscalation: true
allowedCapabilities:
- '*'
volumes:
- '*'
hostNetwork: true
hostPorts:
- min: 0
max: 65535
hostIPC: true
hostPID: true
runAsUser:
rule: 'RunAsAny'
seLinux:
rule: 'RunAsAny'
supplementalGroups:
rule: 'RunAsAny'
fsGroup:
rule: 'RunAsAny'
--- ---
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:
name: ovn-config name: ovn-config
namespace: kube-ovn namespace: kube-system
--- ---
apiVersion: v1 apiVersion: v1
kind: ServiceAccount kind: ServiceAccount
metadata: metadata:
name: ovn name: ovn
namespace: kube-ovn namespace: kube-system
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole kind: ClusterRole
@ -25,12 +47,19 @@ metadata:
rbac.authorization.k8s.io/system-only: "true" rbac.authorization.k8s.io/system-only: "true"
name: system:ovn name: system:ovn
rules: rules:
- apiGroups: ['policy']
resources: ['podsecuritypolicies']
verbs: ['use']
resourceNames:
- kube-ovn
- apiGroups: - apiGroups:
- "kubeovn.io" - "kubeovn.io"
resources: resources:
- subnets - subnets
- subnets/status - subnets/status
- ips - ips
- vlans
- networks
verbs: verbs:
- "*" - "*"
- apiGroups: - apiGroups:
@ -51,12 +80,14 @@ rules:
- "" - ""
- networking.k8s.io - networking.k8s.io
- apps - apps
- extensions
resources: resources:
- networkpolicies - networkpolicies
- services - services
- endpoints - endpoints
- statefulsets - statefulsets
- daemonsets - daemonsets
- deployments
verbs: verbs:
- get - get
- list - list
@ -69,7 +100,6 @@ rules:
- create - create
- patch - patch
- update - update
--- ---
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding kind: ClusterRoleBinding
@ -82,14 +112,13 @@ roleRef:
subjects: subjects:
- kind: ServiceAccount - kind: ServiceAccount
name: ovn name: ovn
namespace: kube-ovn namespace: kube-system
--- ---
kind: Service kind: Service
apiVersion: v1 apiVersion: v1
metadata: metadata:
name: ovn-nb name: ovn-nb
namespace: kube-ovn namespace: kube-system
spec: spec:
ports: ports:
- name: ovn-nb - name: ovn-nb
@ -99,14 +128,14 @@ spec:
type: ClusterIP type: ClusterIP
selector: selector:
app: ovn-central app: ovn-central
ovn-nb-leader: "true"
sessionAffinity: None sessionAffinity: None
--- ---
kind: Service kind: Service
apiVersion: v1 apiVersion: v1
metadata: metadata:
name: ovn-sb name: ovn-sb
namespace: kube-ovn namespace: kube-system
spec: spec:
ports: ports:
- name: ovn-sb - name: ovn-sb
@ -116,14 +145,14 @@ spec:
type: ClusterIP type: ClusterIP
selector: selector:
app: ovn-central app: ovn-central
ovn-sb-leader: "true"
sessionAffinity: None sessionAffinity: None
--- ---
kind: Deployment kind: Deployment
apiVersion: apps/v1 apiVersion: apps/v1
metadata: metadata:
name: ovn-central name: ovn-central
namespace: kube-ovn namespace: kube-system
annotations: annotations:
kubernetes.io/description: | kubernetes.io/description: |
OVN components: northd, nb and sb. OVN components: northd, nb and sb.
@ -146,7 +175,6 @@ spec:
spec: spec:
tolerations: tolerations:
- operator: Exists - operator: Exists
effect: NoSchedule
affinity: affinity:
podAntiAffinity: podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution: requiredDuringSchedulingIgnoredDuringExecution:
@ -154,73 +182,108 @@ spec:
matchLabels: matchLabels:
app: ovn-central app: ovn-central
topologyKey: kubernetes.io/hostname topologyKey: kubernetes.io/hostname
priorityClassName: system-cluster-critical
serviceAccountName: ovn serviceAccountName: ovn
hostNetwork: true hostNetwork: true
containers: containers:
- name: ovn-central - name: ovn-central
image: "index.alauda.cn/alaudak8s/kube-ovn-db:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: ["/kube-ovn/start-db.sh"]
securityContext:
capabilities:
add: ["SYS_NICE"]
env: env:
- name: ENABLE_SSL
value: "false"
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: status.podIP fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
resources: resources:
requests: requests:
cpu: 200m cpu: 500m
memory: 300Mi memory: 200Mi
limits: limits:
cpu: 400m cpu: 3
memory: 800Mi memory: 3Gi
volumeMounts: volumeMounts:
- mountPath: /run/openvswitch
name: host-run-ovs
- mountPath: /var/run/openvswitch - mountPath: /var/run/openvswitch
name: host-run-ovs name: host-run-ovs
- mountPath: /var/run/ovn
name: host-run-ovn
- mountPath: /sys - mountPath: /sys
name: host-sys name: host-sys
readOnly: true readOnly: true
- mountPath: /etc/openvswitch - mountPath: /etc/openvswitch
name: host-config-openvswitch name: host-config-openvswitch
- mountPath: /etc/ovn
name: host-config-ovn
- mountPath: /var/log/openvswitch - mountPath: /var/log/openvswitch
name: host-log name: host-log-ovs
- mountPath: /var/log/ovn
name: host-log-ovn
- mountPath: /var/run/tls
name: kube-ovn-tls
readinessProbe: readinessProbe:
exec: exec:
command: command:
- sh - sh
- /root/ovn-is-leader.sh - /kube-ovn/ovn-is-leader.sh
periodSeconds: 3 periodSeconds: 3
timeoutSeconds: 45
livenessProbe: livenessProbe:
exec: exec:
command: command:
- sh - sh
- /root/ovn-healthcheck.sh - /kube-ovn/ovn-healthcheck.sh
initialDelaySeconds: 30 initialDelaySeconds: 30
periodSeconds: 7 periodSeconds: 7
failureThreshold: 5 failureThreshold: 5
timeoutSeconds: 45
nodeSelector: nodeSelector:
beta.kubernetes.io/os: "linux" kubernetes.io/os: "linux"
kube-ovn/role: "master" kube-ovn/role: "master"
volumes: volumes:
- name: host-run-ovs - name: host-run-ovs
hostPath: hostPath:
path: /run/openvswitch path: /run/openvswitch
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: host-sys - name: host-sys
hostPath: hostPath:
path: /sys path: /sys
- name: host-config-openvswitch - name: host-config-openvswitch
hostPath: hostPath:
path: /etc/origin/openvswitch path: /etc/origin/openvswitch
- name: host-log - name: host-config-ovn
hostPath:
path: /etc/origin/ovn
- name: host-log-ovs
hostPath: hostPath:
path: /var/log/openvswitch path: /var/log/openvswitch
- name: host-log-ovn
hostPath:
path: /var/log/ovn
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls
--- ---
kind: DaemonSet kind: DaemonSet
apiVersion: apps/v1 apiVersion: apps/v1
metadata: metadata:
name: ovs-ovn name: ovs-ovn
namespace: kube-ovn namespace: kube-system
annotations: annotations:
kubernetes.io/description: | kubernetes.io/description: |
This daemon set launches the openvswitch daemon. This daemon set launches the openvswitch daemon.
@ -239,60 +302,77 @@ spec:
spec: spec:
tolerations: tolerations:
- operator: Exists - operator: Exists
effect: NoSchedule priorityClassName: system-cluster-critical
serviceAccountName: ovn serviceAccountName: ovn
hostNetwork: true hostNetwork: true
hostPID: true hostPID: true
containers: containers:
- name: openvswitch - name: openvswitch
image: "index.alauda.cn/alaudak8s/kube-ovn-node:v0.9.1" image: "kubeovn/kube-ovn:v1.5.3"
imagePullPolicy: IfNotPresent imagePullPolicy: IfNotPresent
command: ["/kube-ovn/start-ovs.sh"]
securityContext: securityContext:
runAsUser: 0 runAsUser: 0
privileged: true privileged: true
env: env:
- name: ENABLE_SSL
value: "false"
- name: POD_IP - name: POD_IP
valueFrom: valueFrom:
fieldRef: fieldRef:
fieldPath: status.podIP fieldPath: status.podIP
- name: HW_OFFLOAD
value: "false"
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts: volumeMounts:
- mountPath: /lib/modules - mountPath: /lib/modules
name: host-modules name: host-modules
readOnly: true readOnly: true
- mountPath: /run/openvswitch
name: host-run-ovs
- mountPath: /var/run/openvswitch - mountPath: /var/run/openvswitch
name: host-run-ovs name: host-run-ovs
- mountPath: /var/run/ovn
name: host-run-ovn
- mountPath: /sys - mountPath: /sys
name: host-sys name: host-sys
readOnly: true readOnly: true
- mountPath: /etc/openvswitch - mountPath: /etc/openvswitch
name: host-config-openvswitch name: host-config-openvswitch
- mountPath: /etc/ovn
name: host-config-ovn
- mountPath: /var/log/openvswitch - mountPath: /var/log/openvswitch
name: host-log name: host-log-ovs
- mountPath: /var/log/ovn
name: host-log-ovn
- mountPath: /var/run/tls
name: kube-ovn-tls
readinessProbe: readinessProbe:
exec: exec:
command: command:
- sh - sh
- /root/ovs-healthcheck.sh - /kube-ovn/ovs-healthcheck.sh
periodSeconds: 5 periodSeconds: 5
timeoutSeconds: 45
livenessProbe: livenessProbe:
exec: exec:
command: command:
- sh - sh
- /root/ovs-healthcheck.sh - /kube-ovn/ovs-healthcheck.sh
initialDelaySeconds: 10 initialDelaySeconds: 10
periodSeconds: 5 periodSeconds: 5
failureThreshold: 5 failureThreshold: 5
timeoutSeconds: 45
resources: resources:
requests: requests:
cpu: 200m cpu: 200m
memory: 300Mi memory: 200Mi
limits: limits:
cpu: 1000m cpu: 1000m
memory: 800Mi memory: 800Mi
nodeSelector: nodeSelector:
beta.kubernetes.io/os: "linux" kubernetes.io/os: "linux"
volumes: volumes:
- name: host-modules - name: host-modules
hostPath: hostPath:
@ -300,12 +380,25 @@ spec:
- name: host-run-ovs - name: host-run-ovs
hostPath: hostPath:
path: /run/openvswitch path: /run/openvswitch
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: host-sys - name: host-sys
hostPath: hostPath:
path: /sys path: /sys
- name: host-config-openvswitch - name: host-config-openvswitch
hostPath: hostPath:
path: /etc/origin/openvswitch path: /etc/origin/openvswitch
- name: host-log - name: host-config-ovn
hostPath:
path: /etc/origin/ovn
- name: host-log-ovs
hostPath: hostPath:
path: /var/log/openvswitch path: /var/log/openvswitch
- name: host-log-ovn
hostPath:
path: /var/log/ovn
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls