diff --git a/ezdown b/ezdown index 80d1b7d..6524b53 100755 --- a/ezdown +++ b/ezdown @@ -33,14 +33,11 @@ pauseVer=3.9 # images not downloaded by default(only download with '-X') ciliumVer=1.13.2 flannelVer=v0.21.4 +kubeRouterVer=v0.3.1 +kubeOvnVer=v1.11.5 nfsProvisionerVer=v4.0.2 promChartVer=45.23.0 -# images not downloaded -kubeRouterVer=v0.3.1 -kubeOvnVer=v1.5.3 - - function usage() { echo -e "\033[33mUsage:\033[0m ezdown [options] [args]" cat <> /tmp/install-kube-ovn-`date +"%Y%m%d%H%M%S"`.log 2>&1' + run_once: true + ignore_errors: true + connection: local + +# 等待网络插件部署成功 - name: 轮询等待kube-ovn 运行，视下载镜像速度而定 shell: "{{ base_dir }}/bin/kubectl get pod -n kube-ovn -o wide|grep 'kube-ovn-cni'|grep ' {{ K8S_NODENAME }} '|awk '{print $3}'" register: pod_status diff --git a/roles/kube-ovn/templates/crd.yaml.j2 b/roles/kube-ovn/templates/crd.yaml.j2 deleted file mode 100644 index 9d43022..0000000 --- a/roles/kube-ovn/templates/crd.yaml.j2 +++ /dev/null @@ -1,224 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: ips.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - additionalPrinterColumns: - - name: IP - type: string - jsonPath: .spec.ipAddress - - name: Mac - type: string - jsonPath: .spec.macAddress - - name: Node - type: string - jsonPath: .spec.nodeName - - name: Subnet - type: string - jsonPath: .spec.subnet - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - podName: - type: string - namespace: - type: string - subnet: - type: string - attachSubnets: - type: array - items: - type: string - nodeName: - type: string - ipAddress: - type: string - attachIps: - type: array - items: - type: string - macAddress: - type: string - attachMacs: - type: array - items: - type: string - containerID: - type: string - scope: Cluster - names: - plural: ips - singular: ip - kind: IP - shortNames: - - ip ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: subnets.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - subresources: - status: {} - additionalPrinterColumns: - - name: Provider - type: string - jsonPath: .spec.provider - - name: Protocol - type: string - jsonPath: .spec.protocol - - name: CIDR - type: string - jsonPath: .spec.cidrBlock - - name: Private - type: boolean - jsonPath: .spec.private - - name: NAT - type: boolean - jsonPath: .spec.natOutgoing - - name: Default - type: boolean - jsonPath: .spec.default - - name: GatewayType - type: string - jsonPath: .spec.gatewayType - - name: Used - type: number - jsonPath: .status.usingIPs - - name: Available - type: number - jsonPath: .status.availableIPs - schema: - openAPIV3Schema: - type: object - properties: - status: - type: object - properties: - availableIPs: - type: number - usingIPs: - type: number - activateGateway: - type: string - conditions: - type: array - items: - type: object - properties: - type: - type: string - status: - type: string - reason: - type: string - message: - type: string - lastUpdateTime: - type: string - lastTransitionTime: - type: string - spec: - type: object - properties: - default: - type: boolean - protocol: - type: string - cidrBlock: - type: string - namespaces: - type: array - items: - type: string - gateway: - type: string - provider: - type: string - excludeIps: - type: array - items: - type: string - gatewayType: - type: string - allowSubnets: - type: array - items: - type: string - gatewayNode: - type: string - natOutgoing: - type: boolean - private: - type: boolean - vlan: - type: string - underlayGateway: - type: boolean - disableInterConnection: - type: boolean - scope: Cluster - names: - plural: subnets - singular: subnet - kind: Subnet - shortNames: - - subnet - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vlans.kubeovn.io -spec: - group: kubeovn.io - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - vlanId: - type: integer - providerInterfaceName: - type: string - logicalInterfaceName: - type: string - subnet: - type: string - additionalPrinterColumns: - - name: VlanID - type: string - jsonPath: .spec.vlanId - - name: ProviderInterfaceName - type: string - jsonPath: .spec.providerInterfaceName - - name: Subnet - type: string - jsonPath: .spec.subnet - scope: Cluster - names: - plural: vlans - singular: vlan - kind: Vlan - shortNames: - - vlan diff --git a/roles/kube-ovn/templates/install.sh.j2 b/roles/kube-ovn/templates/install.sh.j2 new file mode 100644 index 0000000..da3e8d0 --- /dev/null +++ b/roles/kube-ovn/templates/install.sh.j2 @@ -0,0 +1,3657 @@ +#!/usr/bin/env bash +set -euo pipefail + +IPV6=${IPV6:-false} +DUAL_STACK=${DUAL_STACK:-false} +ENABLE_SSL=${ENABLE_SSL:-false} +ENABLE_VLAN=${ENABLE_VLAN:-false} +CHECK_GATEWAY=${CHECK_GATEWAY:-true} +LOGICAL_GATEWAY=${LOGICAL_GATEWAY:-false} +U2O_INTERCONNECTION=${U2O_INTERCONNECTION:-false} +ENABLE_MIRROR=${ENABLE_MIRROR:-false} +VLAN_NIC=${VLAN_NIC:-} +HW_OFFLOAD=${HW_OFFLOAD:-false} +ENABLE_LB=${ENABLE_LB:-false} +ENABLE_NP=${ENABLE_NP:-false} +ENABLE_EIP_SNAT=${ENABLE_EIP_SNAT:-false} +LS_DNAT_MOD_DL_DST=${LS_DNAT_MOD_DL_DST:-true} +ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} +CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} +ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} +ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} + +# exchange link names of OVS bridge and the provider nic +# in the default provider-network +EXCHANGE_LINK_NAME=${EXCHANGE_LINK_NAME:-false} +# The nic to support container network can be a nic name or a group of regex +# separated by comma, if empty will use the nic that the default route use +IFACE=${IFACE:-} +# Specifies the name of the dpdk tunnel iface. +# Note that the dpdk tunnel iface and tunnel ip cidr should be diffierent with Kubernetes api cidr,otherwise the route will be a problem. +DPDK_TUNNEL_IFACE=${DPDK_TUNNEL_IFACE:-br-phy} +ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-true} + +CNI_CONF_DIR="/etc/cni/net.d" +CNI_BIN_DIR="/opt/cni/bin" + +REGISTRY="easzlab.io.local:5000/kubeovn" +VERSION="v1.11.5" +IMAGE_PULL_POLICY="IfNotPresent" +POD_CIDR="{{ CLUSTER_CIDR }}" # Do NOT overlap with NODE/SVC/JOIN CIDR +POD_GATEWAY="{{ CLUSTER_CIDR_GW }}" +SVC_CIDR="{{ SERVICE_CIDR }}" # Do NOT overlap with NODE/POD/JOIN CIDR +JOIN_CIDR="100.64.0.0/16" # Do NOT overlap with NODE/POD/SVC CIDR +PINGER_EXTERNAL_ADDRESS="114.114.114.114" # Pinger check external ip probe +PINGER_EXTERNAL_DOMAIN="alauda.cn" # Pinger check external domain probe +SVC_YAML_IPFAMILYPOLICY="" +if [ "$IPV6" = "true" ]; then + POD_CIDR="fd00:10:16::/64" # Do NOT overlap with NODE/SVC/JOIN CIDR + POD_GATEWAY="fd00:10:16::1" + SVC_CIDR="fd00:10:96::/112" # Do NOT overlap with NODE/POD/JOIN CIDR + JOIN_CIDR="fd00:100:64::/64" # Do NOT overlap with NODE/POD/SVC CIDR + PINGER_EXTERNAL_ADDRESS="2400:3200::1" + PINGER_EXTERNAL_DOMAIN="google.com" +fi +if [ "$DUAL_STACK" = "true" ]; then + POD_CIDR="10.16.0.0/16,fd00:10:16::/64" # Do NOT overlap with NODE/SVC/JOIN CIDR + POD_GATEWAY="10.16.0.1,fd00:10:16::1" + SVC_CIDR="10.96.0.0/12,fd00:10:96::/112" # Do NOT overlap with NODE/POD/JOIN CIDR + JOIN_CIDR="100.64.0.0/16,fd00:100:64::/64" # Do NOT overlap with NODE/POD/SVC CIDR + PINGER_EXTERNAL_ADDRESS="114.114.114.114,2400:3200::1" + PINGER_EXTERNAL_DOMAIN="google.com" + SVC_YAML_IPFAMILYPOLICY="ipFamilyPolicy: PreferDualStack" +fi + +EXCLUDE_IPS="" # EXCLUDE_IPS for default subnet +LABEL="kubernetes.io/role=master" # The node label to deploy OVN DB +DEPRECATED_LABEL="node-role.kubernetes.io/master" # The node label to deploy OVN DB in earlier versions +NETWORK_TYPE="geneve" # geneve or vlan +TUNNEL_TYPE="geneve" # geneve, vxlan or stt. ATTENTION: some networkpolicy cannot take effect when using vxlan and stt need custom compile ovs kernel module +POD_NIC_TYPE="veth-pair" # veth-pair or internal-port +POD_DEFAULT_FIP_TYPE="" # iptables, pod can set iptables fip automatically by enable fip annotation + +# VLAN Config only take effect when NETWORK_TYPE is vlan +PROVIDER_NAME="provider" +VLAN_INTERFACE_NAME="" +VLAN_NAME="ovn-vlan" +VLAN_ID="100" + +if [ "$ENABLE_VLAN" = "true" ]; then + NETWORK_TYPE="vlan" + if [ "$VLAN_NIC" != "" ]; then + VLAN_INTERFACE_NAME="$VLAN_NIC" + fi +fi + +# hybrid dpdk +HYBRID_DPDK="false" + +# DPDK +DPDK="false" +DPDK_SUPPORTED_VERSIONS=("19.11") +DPDK_VERSION="" +DPDK_CPU="1000m" # Default CPU configuration for if --dpdk-cpu flag is not included +DPDK_MEMORY="2Gi" # Default Memory configuration for it --dpdk-memory flag is not included + +# performance +MODULES="kube_ovn_fastpath.ko" +RPMS="openvswitch-kmod" +GC_INTERVAL=360 +INSPECT_INTERVAL=20 + +display_help() { + echo "Usage: $0 [option...]" + echo + echo " -h, --help Print Help (this message) and exit" + echo " --with-hybrid-dpdk Install Kube-OVN with nodes which run ovs-dpdk or ovs-kernel" + echo " --with-dpdk= Install Kube-OVN with OVS-DPDK instead of kernel OVS" + echo " --dpdk-cpu=m Configure DPDK to use a specific amount of CPU" + echo " --dpdk-memory=Gi Configure DPDK to use a specific amount of memory" + echo + exit 0 +} + +if [ -n "${1-}" ] +then + set +u + while :; do + case $1 in + -h|--help) + display_help + ;; + --with-hybrid-dpdk) + HYBRID_DPDK="true" + ;; + --with-dpdk=*) + DPDK=true + DPDK_VERSION="${1#*=}" + if [[ ! "${DPDK_SUPPORTED_VERSIONS[@]}" = "${DPDK_VERSION}" ]] || [[ -z "${DPDK_VERSION}" ]]; then + echo "Unsupported DPDK version: ${DPDK_VERSION}" + echo "Supported DPDK versions: ${DPDK_SUPPORTED_VERSIONS[*]}" + exit 1 + fi + ;; + --dpdk-cpu=*) + DPDK_CPU="${1#*=}" + if [[ $DPDK_CPU =~ ^[0-9]+(m)$ ]] + then + echo "CPU $DPDK_CPU" + else + echo "$DPDK_CPU is not valid, please use the format --dpdk-cpu=m" + exit 1 + fi + ;; + --dpdk-memory=*) + DPDK_MEMORY="${1#*=}" + if [[ $DPDK_MEMORY =~ ^[0-9]+(Gi)$ ]] + then + echo "MEMORY $DPDK_MEMORY" + else + echo "$DPDK_MEMORY is not valid, please use the format --dpdk-memory=Gi" + exit 1 + fi + ;; + -?*) + echo "Unknown argument $1" + exit 1 + ;; + *) break + esac + shift + done + set -u +fi + +echo "-------------------------------" +echo "Kube-OVN Version: $VERSION" +echo "Default Network Mode: $NETWORK_TYPE" +if [[ $NETWORK_TYPE = "vlan" ]];then + echo "Default Vlan Nic: $VLAN_INTERFACE_NAME" + echo "Default Vlan ID: $VLAN_ID" +fi +echo "Default Subnet CIDR: $POD_CIDR" +echo "Join Subnet CIDR: $JOIN_CIDR" +echo "Enable SVC LB: $ENABLE_LB" +echo "Enable Networkpolicy: $ENABLE_NP" +echo "Enable EIP and SNAT: $ENABLE_EIP_SNAT" +echo "Enable Mirror: $ENABLE_MIRROR" +echo "-------------------------------" + +if [[ $ENABLE_SSL = "true" ]];then + echo "[Step 0/6] Generate SSL key and cert" + exist=$(kubectl get secret -n kube-system kube-ovn-tls --ignore-not-found) + if [[ $exist == "" ]];then + docker run --rm -v "$PWD":/etc/ovn $REGISTRY/kube-ovn:$VERSION bash generate-ssl.sh + kubectl create secret generic -n kube-system kube-ovn-tls --from-file=cacert=cacert.pem --from-file=cert=ovn-cert.pem --from-file=key=ovn-privkey.pem + rm -rf cacert.pem ovn-cert.pem ovn-privkey.pem ovn-req.pem + fi + echo "-------------------------------" + echo "" +fi + +echo "[Step 1/6] Label kube-ovn-master node and label datapath type" +count=$(kubectl get no -l$LABEL --no-headers | wc -l) +node_label="$LABEL" +if [ $count -eq 0 ]; then + count=$(kubectl get no -l$DEPRECATED_LABEL --no-headers | wc -l) + node_label="$DEPRECATED_LABEL" + if [ $count -eq 0 ]; then + echo "ERROR: No node with label $LABEL or $DEPRECATED_LABEL found" + exit 1 + fi +fi +kubectl label no -l$node_label kube-ovn/role=master --overwrite + +if [ "$DPDK" = "true" -o "$HYBRID_DPDK" = "true" ]; then + kubectl label no -lovn.kubernetes.io/ovs_dp_type!=userspace ovn.kubernetes.io/ovs_dp_type=kernel --overwrite +fi + +echo "-------------------------------" +echo "" + +echo "[Step 2/6] Install OVN components" +addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',') +count=$(kubectl get no -lkube-ovn/role=master --no-headers | wc -l) +echo "Install OVN DB in $addresses" + +cat < kube-ovn-crd.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-dnses.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-dnses + singular: vpc-dns + shortNames: + - vpc-dns + kind: VpcDns + listKind: VpcDnsList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.active + name: Active + type: boolean + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + vpc: + type: string + subnet: + type: string + status: + type: object + properties: + active: + type: boolean + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: switch-lb-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: switch-lb-rules + singular: switch-lb-rule + shortNames: + - slr + kind: SwitchLBRule + listKind: SwitchLBRuleList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vip + name: vip + type: string + - jsonPath: .status.ports + name: port(s) + type: string + - jsonPath: .status.service + name: service + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + namespace: + type: string + vip: + type: string + sessionAffinity: + type: string + ports: + items: + properties: + name: + type: string + port: + type: integer + minimum: 1 + maximum: 65535 + protocol: + type: string + targetPort: + type: integer + minimum: 1 + maximum: 65535 + type: object + type: array + selector: + items: + type: string + type: array + status: + type: object + properties: + ports: + type: string + service: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-nat-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-nat-gateways + singular: vpc-nat-gateway + shortNames: + - vpc-nat-gw + kind: VpcNatGateway + listKind: VpcNatGatewayList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + - jsonPath: .spec.lanIp + name: LanIP + type: string + name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + lanIp: + type: string + subnet: + type: string + vpc: + type: string + selector: + type: array + items: + type: string + tolerations: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + value: + type: string + effect: + type: string + tolerationSeconds: + type: integer +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-eips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-eips + singular: iptables-eip + shortNames: + - eip + kind: IptablesEIP + listKind: IptablesEIPList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.ip + name: IP + type: string + - jsonPath: .spec.macAddress + name: Mac + type: string + - jsonPath: .status.nat + name: Nat + type: string + - jsonPath: .spec.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + ip: + type: string + nat: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + v4ip: + type: string + v6ip: + type: string + macAddress: + type: string + natGwDp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-fip-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-fip-rules + singular: iptables-fip-rule + shortNames: + - fip + kind: IptablesFIPRule + listKind: IptablesFIPRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalIp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-dnat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-dnat-rules + singular: iptables-dnat-rule + shortNames: + - dnat + kind: IptablesDnatRule + listKind: IptablesDnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .spec.protocol + name: Protocol + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .spec.externalPort + name: ExternalPort + type: string + - jsonPath: .spec.internalPort + name: InternalPort + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + protocol: + type: string + internalIp: + type: string + internalPort: + type: string + externalPort: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + externalPort: + type: string + protocol: + type: string + internalIp: + type: string + internalPort: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-snat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-snat-rules + singular: iptables-snat-rule + shortNames: + - snat + kind: IptablesSnatRule + listKind: IptablesSnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: EIP + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalCIDR + name: InternalCIDR + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalCIDR: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-eips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-eips + singular: ovn-eip + shortNames: + - oeip + kind: OvnEip + listKind: OvnEipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.v4ip + name: IP + type: string + - jsonPath: .spec.macAddress + name: Mac + type: string + - jsonPath: .spec.type + name: Type + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + v4Ip: + type: string + macAddress: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + externalSubnet: + type: string + type: + type: string + v4ip: + type: string + macAddress: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-fips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-fips + singular: ovn-fip + shortNames: + - ofip + kind: OvnFip + listKind: OvnFipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string + - jsonPath: .status.v4Eip + name: V4Eip + type: string + - jsonPath: .status.v4Ip + name: V4Ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4Eip: + type: string + v4Ip: + type: string + macAddress: + type: string + vpc: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + ovnEip: + type: string + ipName: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-snat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-snat-rules + singular: ovn-snat-rule + shortNames: + - osnat + kind: OvnSnatRule + listKind: OvnSnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string + - jsonPath: .status.v4Eip + name: V4Eip + type: string + - jsonPath: .status.v4ipCidr + name: V4Ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4Eip: + type: string + v4ipCidr: + type: string + vpc: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + ovnEip: + type: string + vpcSubnet: + type: string + ipName: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.kubeovn.io +spec: + group: kubeovn.io + versions: + - additionalPrinterColumns: + - jsonPath: .status.enableExternal + name: EnableExternal + type: boolean + - jsonPath: .status.standby + name: Standby + type: boolean + - jsonPath: .status.subnets + name: Subnets + type: string + - jsonPath: .spec.namespaces + name: Namespaces + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + enableExternal: + type: boolean + namespaces: + items: + type: string + type: array + staticRoutes: + items: + properties: + policy: + type: string + cidr: + type: string + nextHopIP: + type: string + type: object + type: array + policyRoutes: + items: + properties: + priority: + type: integer + action: + type: string + match: + type: string + nextHopIP: + type: string + type: object + type: array + vpcPeerings: + items: + properties: + remoteVpc: + type: string + localConnectIP: + type: string + type: object + type: array + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + type: string + lastUpdateTime: + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + default: + type: boolean + defaultLogicalSwitch: + type: string + router: + type: string + standby: + type: boolean + enableExternal: + type: boolean + subnets: + items: + type: string + type: array + vpcPeerings: + items: + type: string + type: array + tcpLoadBalancer: + type: string + tcpSessionLoadBalancer: + type: string + udpLoadBalancer: + type: string + udpSessionLoadBalancer: + type: string + sctpLoadBalancer: + type: string + sctpSessionLoadBalancer: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + names: + kind: Vpc + listKind: VpcList + plural: vpcs + shortNames: + - vpc + singular: vpc + scope: Cluster +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ips.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: V4IP + type: string + jsonPath: .spec.v4IpAddress + - name: V6IP + type: string + jsonPath: .spec.v6IpAddress + - name: Mac + type: string + jsonPath: .spec.macAddress + - name: Node + type: string + jsonPath: .spec.nodeName + - name: Subnet + type: string + jsonPath: .spec.subnet + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + podName: + type: string + namespace: + type: string + subnet: + type: string + attachSubnets: + type: array + items: + type: string + nodeName: + type: string + ipAddress: + type: string + v4IpAddress: + type: string + v6IpAddress: + type: string + attachIps: + type: array + items: + type: string + macAddress: + type: string + attachMacs: + type: array + items: + type: string + containerID: + type: string + podType: + type: string + scope: Cluster + names: + plural: ips + singular: ip + kind: IP + shortNames: + - ip +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vips + singular: vip + shortNames: + - vip + kind: Vip + listKind: VipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: V4IP + type: string + jsonPath: .status.v4ip + - name: PV4IP + type: string + jsonPath: .spec.parentV4ip + - name: Mac + type: string + jsonPath: .status.mac + - name: PMac + type: string + jsonPath: .spec.parentMac + - name: V6IP + type: string + jsonPath: .status.v6ip + - name: PV6IP + type: string + jsonPath: .spec.parentV6ip + - name: Subnet + type: string + jsonPath: .spec.subnet + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + mac: + type: string + pv4ip: + type: string + pv6ip: + type: string + pmac: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + namespace: + type: string + subnet: + type: string + attachSubnets: + type: array + items: + type: string + v4ip: + type: string + macAddress: + type: string + v6ip: + type: string + parentV4ip: + type: string + parentMac: + type: string + parentV6ip: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: subnets.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Provider + type: string + jsonPath: .spec.provider + - name: Vpc + type: string + jsonPath: .spec.vpc + - name: Protocol + type: string + jsonPath: .spec.protocol + - name: CIDR + type: string + jsonPath: .spec.cidrBlock + - name: Private + type: boolean + jsonPath: .spec.private + - name: NAT + type: boolean + jsonPath: .spec.natOutgoing + - name: Default + type: boolean + jsonPath: .spec.default + - name: GatewayType + type: string + jsonPath: .spec.gatewayType + - name: V4Used + type: number + jsonPath: .status.v4usingIPs + - name: V4Available + type: number + jsonPath: .status.v4availableIPs + - name: V6Used + type: number + jsonPath: .status.v6usingIPs + - name: V6Available + type: number + jsonPath: .status.v6availableIPs + - name: ExcludeIPs + type: string + jsonPath: .spec.excludeIps + - name: U2OInterconnectionIP + type: string + jsonPath: .status.u2oInterconnectionIP + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + v4availableIPs: + type: number + v4usingIPs: + type: number + v6availableIPs: + type: number + v6usingIPs: + type: number + activateGateway: + type: string + dhcpV4OptionsUUID: + type: string + dhcpV6OptionsUUID: + type: string + u2oInterconnectionIP: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + vpc: + type: string + default: + type: boolean + protocol: + type: string + enum: + - IPv4 + - IPv6 + - Dual + cidrBlock: + type: string + namespaces: + type: array + items: + type: string + gateway: + type: string + provider: + type: string + excludeIps: + type: array + items: + type: string + vips: + type: array + items: + type: string + gatewayType: + type: string + allowSubnets: + type: array + items: + type: string + gatewayNode: + type: string + natOutgoing: + type: boolean + u2oRouting: + type: boolean + externalEgressGateway: + type: string + policyRoutingPriority: + type: integer + minimum: 1 + maximum: 32765 + policyRoutingTableID: + type: integer + minimum: 1 + maximum: 2147483647 + not: + enum: + - 252 # compat + - 253 # default + - 254 # main + - 255 # local + private: + type: boolean + vlan: + type: string + logicalGateway: + type: boolean + disableGatewayCheck: + type: boolean + disableInterConnection: + type: boolean + enableDHCP: + type: boolean + dhcpV4Options: + type: string + dhcpV6Options: + type: string + enableIPv6RA: + type: boolean + ipv6RAConfigs: + type: string + acls: + type: array + items: + type: object + properties: + direction: + type: string + enum: + - from-lport + - to-lport + priority: + type: integer + minimum: 0 + maximum: 32767 + match: + type: string + action: + type: string + enum: + - allow-related + - allow-stateless + - allow + - drop + - reject + u2oInterconnection: + type: boolean + scope: Cluster + names: + plural: subnets + singular: subnet + kind: Subnet + shortNames: + - subnet +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vlans.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + id: + type: integer + minimum: 0 + maximum: 4095 + provider: + type: string + vlanId: + type: integer + description: Deprecated in favor of id + providerInterfaceName: + type: string + description: Deprecated in favor of provider + required: + - provider + status: + type: object + properties: + subnets: + type: array + items: + type: string + additionalPrinterColumns: + - name: ID + type: string + jsonPath: .spec.id + - name: Provider + type: string + jsonPath: .spec.provider + scope: Cluster + names: + plural: vlans + singular: vlan + kind: Vlan + shortNames: + - vlan +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: provider-networks.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + metadata: + type: object + properties: + name: + type: string + maxLength: 12 + not: + enum: + - int + - external + spec: + type: object + properties: + defaultInterface: + type: string + maxLength: 15 + pattern: '^[^/\s]+$' + customInterfaces: + type: array + items: + type: object + properties: + interface: + type: string + maxLength: 15 + pattern: '^[^/\s]+$' + nodes: + type: array + items: + type: string + exchangeLinkName: + type: boolean + excludeNodes: + type: array + items: + type: string + required: + - defaultInterface + status: + type: object + properties: + ready: + type: boolean + readyNodes: + type: array + items: + type: string + notReadyNodes: + type: array + items: + type: string + vlans: + type: array + items: + type: string + conditions: + type: array + items: + type: object + properties: + node: + type: string + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + additionalPrinterColumns: + - name: DefaultInterface + type: string + jsonPath: .spec.defaultInterface + - name: Ready + type: boolean + jsonPath: .status.ready + scope: Cluster + names: + plural: provider-networks + singular: provider-network + kind: ProviderNetwork + listKind: ProviderNetworkList +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: security-groups.kubeovn.io +spec: + group: kubeovn.io + names: + plural: security-groups + singular: security-group + shortNames: + - sg + kind: SecurityGroup + listKind: SecurityGroupList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + ingressRules: + type: array + items: + type: object + properties: + ipVersion: + type: string + protocol: + type: string + priority: + type: integer + remoteType: + type: string + remoteAddress: + type: string + remoteSecurityGroup: + type: string + portRangeMin: + type: integer + portRangeMax: + type: integer + policy: + type: string + egressRules: + type: array + items: + type: object + properties: + ipVersion: + type: string + protocol: + type: string + priority: + type: integer + remoteType: + type: string + remoteAddress: + type: string + remoteSecurityGroup: + type: string + portRangeMin: + type: integer + portRangeMax: + type: integer + policy: + type: string + allowSameGroupTraffic: + type: boolean + status: + type: object + properties: + portGroup: + type: string + allowSameGroupTraffic: + type: boolean + ingressMd5: + type: string + egressMd5: + type: string + ingressLastSyncSuccess: + type: boolean + egressLastSyncSuccess: + type: boolean + subresources: + status: {} + conversion: + strategy: None +EOF + +if $DPDK; then + cat < ovn.yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:ovn +rules: + - apiGroups: + - "kubeovn.io" + resources: + - vpcs + - vpcs/status + - vpc-nat-gateways + - subnets + - subnets/status + - ips + - vips + - vips/status + - vlans + - vlans/status + - provider-networks + - provider-networks/status + - security-groups + - security-groups/status + - iptables-eips + - iptables-fip-rules + - iptables-dnat-rules + - iptables-snat-rules + - iptables-eips/status + - iptables-fip-rules/status + - iptables-dnat-rules/status + - iptables-snat-rules/status + - ovn-eips + - ovn-fips + - ovn-snat-rules + - ovn-eips/status + - ovn-fips/status + - ovn-snat-rules/status + - switch-lb-rules + - switch-lb-rules/status + - vpc-dnses + - vpc-dnses/status + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - pods/exec + - namespaces + - nodes + - configmaps + verbs: + - create + - get + - list + - watch + - patch + - update + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - update + - apiGroups: + - "" + - networking.k8s.io + - apps + - extensions + resources: + - networkpolicies + - services + - services/status + - endpoints + - statefulsets + - daemonsets + - deployments + - deployments/scale + verbs: + - create + - delete + - update + - patch + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" + - apiGroups: + - "kubevirt.io" + resources: + - virtualmachines + - virtualmachineinstances + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn +roleRef: + name: system:ovn + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovn + namespace: kube-system + +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-nb + namespace: kube-system +spec: + ports: + - name: ovn-nb + protocol: TCP + port: 6641 + targetPort: 6641 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-nb-leader: "true" + sessionAffinity: None + +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-sb + namespace: kube-system +spec: + ports: + - name: ovn-sb + protocol: TCP + port: 6642 + targetPort: 6642 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-sb-leader: "true" + sessionAffinity: None + +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-northd + namespace: kube-system +spec: + ports: + - name: ovn-northd + protocol: TCP + port: 6643 + targetPort: 6643 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-northd-leader: "true" + sessionAffinity: None +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-central + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN components: northd, nb and sb. +spec: + replicas: $count + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-central + template: + metadata: + labels: + app: ovn-central + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-central + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-central + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: NODE_IPS + value: $addresses + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" + resources: + requests: + cpu: 300m + memory: 300Mi + limits: + cpu: 3 + memory: 4Gi + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + periodSeconds: 15 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovs-ovn + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: ovs + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: ovs + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + - name: openvswitch + image: "$REGISTRY/kube-ovn-dpdk:$DPDK_VERSION-$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ovs-dpdk.sh"] + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: $addresses + volumeMounts: + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /opt/ovs-config + name: host-config-ovs + - mountPath: /dev/hugepages + name: hugepage + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovs-dpdk-healthcheck.sh + periodSeconds: 5 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovs-dpdk-healthcheck.sh + initialDelaySeconds: 60 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 45 + resources: + requests: + cpu: $DPDK_CPU + memory: $DPDK_MEMORY + limits: + cpu: $DPDK_CPU + memory: $DPDK_MEMORY + hugepages-1Gi: 1Gi + nodeSelector: + kubernetes.io/os: "linux" + ovn.kubernetes.io/ovs_dp_type: "kernel" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-ns + hostPath: + path: /var/run/netns + - name: cni-conf + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: host-config-ovs + hostPath: + path: /opt/ovs-config + type: DirectoryOrCreate + - name: hugepage + emptyDir: + medium: HugePages + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF + +else + cat < ovn.yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovn + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:ovn +rules: + - apiGroups: + - "kubeovn.io" + resources: + - vpcs + - vpcs/status + - vpc-nat-gateways + - subnets + - subnets/status + - ips + - vips + - vips/status + - vlans + - vlans/status + - provider-networks + - provider-networks/status + - security-groups + - security-groups/status + - iptables-eips + - iptables-fip-rules + - iptables-dnat-rules + - iptables-snat-rules + - iptables-eips/status + - iptables-fip-rules/status + - iptables-dnat-rules/status + - iptables-snat-rules/status + - ovn-eips + - ovn-fips + - ovn-snat-rules + - ovn-eips/status + - ovn-fips/status + - ovn-snat-rules/status + - vpc-dnses + - vpc-dnses/status + - switch-lb-rules + - switch-lb-rules/status + verbs: + - "*" + - apiGroups: + - "" + resources: + - pods + - pods/exec + - namespaces + - nodes + - configmaps + verbs: + - create + - get + - list + - watch + - patch + - update + - apiGroups: + - "" + - networking.k8s.io + - apps + - extensions + resources: + - networkpolicies + - services + - services/status + - endpoints + - statefulsets + - daemonsets + - deployments + - deployments/scale + verbs: + - create + - delete + - update + - patch + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - update + - apiGroups: + - "kubevirt.io" + resources: + - virtualmachines + - virtualmachineinstances + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn +roleRef: + name: system:ovn + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovn + namespace: kube-system +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-nb + namespace: kube-system +spec: + ports: + - name: ovn-nb + protocol: TCP + port: 6641 + targetPort: 6641 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-nb-leader: "true" + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-sb + namespace: kube-system +spec: + ports: + - name: ovn-sb + protocol: TCP + port: 6642 + targetPort: 6642 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-sb-leader: "true" + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: ovn-northd + namespace: kube-system +spec: + ports: + - name: ovn-northd + protocol: TCP + port: 6643 + targetPort: 6643 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: ovn-central + ovn-northd-leader: "true" + sessionAffinity: None +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-central + namespace: kube-system + annotations: + kubernetes.io/description: | + OVN components: northd, nb and sb. +spec: + replicas: $count + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: ovn-central + template: + metadata: + labels: + app: ovn-central + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: ovn-central + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: ovn-central + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: NODE_IPS + value: $addresses + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" + resources: + requests: + cpu: 300m + memory: 200Mi + limits: + cpu: 3 + memory: 4Gi + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + periodSeconds: 15 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovn-healthcheck.sh + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovs-ovn + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: ovs + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: ovs + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + - name: openvswitch + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ovs.sh"] + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HW_OFFLOAD + value: "$HW_OFFLOAD" + - name: TUNNEL_TYPE + value: "$TUNNEL_TYPE" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: $addresses + volumeMounts: + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + - mountPath: /var/run/containerd + name: cruntime + readinessProbe: + exec: + command: + - bash + - -c + - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh + periodSeconds: 5 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovs-healthcheck.sh + initialDelaySeconds: 60 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 45 + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 1000m + memory: 1000Mi + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-ns + hostPath: + path: /var/run/netns + - name: cni-conf + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - hostPath: + path: /var/run/containerd + name: cruntime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF +fi + +kubectl apply -f kube-ovn-crd.yaml +kubectl apply -f ovn.yaml + +if $HYBRID_DPDK; then + +cat < ovn-dpdk.yaml +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovs-ovn-dpdk + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: ovs-dpdk + updateStrategy: + type: OnDelete + template: + metadata: + labels: + app: ovs-dpdk + component: network + type: infra + spec: + tolerations: + - operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + containers: + - name: openvswitch + image: "$REGISTRY/kube-ovn:${VERSION}-dpdk" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ovs-dpdk-v2.sh"] + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HW_OFFLOAD + value: "$HW_OFFLOAD" + - name: TUNNEL_TYPE + value: "$TUNNEL_TYPE" + - name: DPDK_TUNNEL_IFACE + value: "$DPDK_TUNNEL_IFACE" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: $addresses + volumeMounts: + - mountPath: /opt/ovs-config + name: host-config-ovs + - name: shareddir + mountPath: /var/lib/kubelet/pods + - name: hugepage + mountPath: /dev/hugepages + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /var/run/openvswitch + name: host-run-ovs + mountPropagation: HostToContainer + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - bash + - -c + - LOG_ROTATE=true /kube-ovn/ovs-healthcheck.sh + periodSeconds: 5 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - bash + - /kube-ovn/ovs-healthcheck.sh + initialDelaySeconds: 60 + periodSeconds: 5 + failureThreshold: 5 + timeoutSeconds: 45 + resources: + requests: + cpu: 200m + hugepages-2Mi: 1Gi + memory: 200Mi + limits: + cpu: 1000m + hugepages-2Mi: 1Gi + memory: 800Mi + nodeSelector: + kubernetes.io/os: "linux" + ovn.kubernetes.io/ovs_dp_type: "userspace" + volumes: + - name: host-config-ovs + hostPath: + path: /opt/ovs-config + type: DirectoryOrCreate + - name: shareddir + hostPath: + path: /var/lib/kubelet/pods + type: '' + - name: hugepage + emptyDir: + medium: HugePages + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: cni-conf + hostPath: + path: /etc/cni/net.d + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +EOF +kubectl apply -f ovn-dpdk.yaml +fi +kubectl rollout status deployment/ovn-central -n kube-system --timeout 300s +echo "-------------------------------" +echo "" + +echo "[Step 3/6] Install Kube-OVN" + +cat < kube-ovn.yaml +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-ovn-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + kube-ovn controller +spec: + replicas: $count + selector: + matchLabels: + app: kube-ovn-controller + strategy: + rollingUpdate: + maxSurge: 0% + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + labels: + app: kube-ovn-controller + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: kube-ovn-controller + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: kube-ovn-controller + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + args: + - /kube-ovn/start-controller.sh + - --default-cidr=$POD_CIDR + - --default-gateway=$POD_GATEWAY + - --default-gateway-check=$CHECK_GATEWAY + - --default-logical-gateway=$LOGICAL_GATEWAY + - --default-u2o-interconnection=$U2O_INTERCONNECTION + - --default-exclude-ips=$EXCLUDE_IPS + - --node-switch-cidr=$JOIN_CIDR + - --service-cluster-ip-range=$SVC_CIDR + - --network-type=$NETWORK_TYPE + - --default-interface-name=$VLAN_INTERFACE_NAME + - --default-exchange-link-name=$EXCHANGE_LINK_NAME + - --default-vlan-id=$VLAN_ID + - --ls-dnat-mod-dl-dst=$LS_DNAT_MOD_DL_DST + - --pod-nic-type=$POD_NIC_TYPE + - --enable-lb=$ENABLE_LB + - --enable-np=$ENABLE_NP + - --enable-eip-snat=$ENABLE_EIP_SNAT + - --enable-external-vpc=$ENABLE_EXTERNAL_VPC + - --logtostderr=false + - --alsologtostderr=true + - --gc-interval=$GC_INTERVAL + - --inspect-interval=$INSPECT_INTERVAL + - --log_file=/var/log/kube-ovn/kube-ovn-controller.log + - --log_file_max_size=0 + - --enable-lb-svc=$ENABLE_LB_SVC + - --keep-vm-ip=$ENABLE_KEEP_VM_IP + - --pod-default-fip-type=$POD_DEFAULT_FIP_TYPE + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_DB_IPS + value: $addresses + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" + volumeMounts: + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - /kube-ovn/kube-ovn-controller-healthcheck + periodSeconds: 3 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - /kube-ovn/kube-ovn-controller-healthcheck + initialDelaySeconds: 300 + periodSeconds: 7 + failureThreshold: 5 + timeoutSeconds: 45 + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 1000m + memory: 1Gi + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-ovn-cni + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the kube-ovn cni daemon. +spec: + selector: + matchLabels: + app: kube-ovn-cni + template: + metadata: + labels: + app: kube-ovn-cni + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: ovn + hostNetwork: true + hostPID: true + initContainers: + - name: install-cni + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/install-cni.sh"] + securityContext: + runAsUser: 0 + privileged: true + volumeMounts: + - mountPath: /opt/cni/bin + name: cni-bin + - mountPath: /usr/local/bin + name: local-bin + containers: + - name: cni-server + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: + - bash + - /kube-ovn/start-cniserver.sh + args: + - --enable-mirror=$ENABLE_MIRROR + - --encap-checksum=true + - --service-cluster-ip-range=$SVC_CIDR + - --iface=${IFACE} + - --dpdk-tunnel-iface=${DPDK_TUNNEL_IFACE} + - --network-type=$TUNNEL_TYPE + - --default-interface-name=$VLAN_INTERFACE_NAME + - --cni-conf-name=${CNI_CONFIG_PRIORITY}-kube-ovn.conflist + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-cni.log + - --log_file_max_size=0 + securityContext: + runAsUser: 0 + privileged: true + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MODULES + value: $MODULES + - name: RPMS + value: $RPMS + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" + - name: DBUS_SYSTEM_BUS_ADDRESS + value: "unix:path=/host/var/run/dbus/system_bus_socket" + volumeMounts: + - name: host-modules + mountPath: /lib/modules + readOnly: true + - name: shared-dir + mountPath: /var/lib/kubelet/pods + - mountPath: /etc/openvswitch + name: systemid + - mountPath: /etc/cni/net.d + name: cni-conf + - mountPath: /run/openvswitch + name: host-run-ovs + mountPropagation: Bidirectional + - mountPath: /run/ovn + name: host-run-ovn + - mountPath: /host/var/run/dbus + name: host-dbus + mountPropagation: HostToContainer + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /tmp + name: tmp + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1000m + memory: 1Gi + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: shared-dir + hostPath: + path: /var/lib/kubelet/pods + - name: systemid + hostPath: + path: /etc/origin/openvswitch + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: cni-conf + hostPath: + path: $CNI_CONF_DIR + - name: cni-bin + hostPath: + path: $CNI_BIN_DIR + - name: host-ns + hostPath: + path: /var/run/netns + - name: host-dbus + hostPath: + path: /var/run/dbus + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: tmp + hostPath: + path: /tmp + - name: local-bin + hostPath: + path: /usr/local/bin + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-ovn-pinger + namespace: kube-system + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: kube-ovn-pinger + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: kube-ovn-pinger + component: network + type: infra + spec: + priorityClassName: system-node-critical + serviceAccountName: ovn + hostPID: true + containers: + - name: pinger + image: "$REGISTRY/kube-ovn:$VERSION" + command: + - /kube-ovn/kube-ovn-pinger + args: + - --external-address=$PINGER_EXTERNAL_ADDRESS + - --external-dns=$PINGER_EXTERNAL_DOMAIN + - --logtostderr=false + - --alsologtostderr=true + - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log + - --log_file_max_size=0 + imagePullPolicy: $IMAGE_PULL_POLICY + securityContext: + runAsUser: 0 + privileged: false + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /run/openvswitch + name: host-run-ovs + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /var/log/kube-ovn + name: kube-ovn-log + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 200m + memory: 400Mi + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + annotations: + kubernetes.io/description: | + Metrics for OVN components: northd, nb and sb. +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: kube-ovn-monitor + template: + metadata: + labels: + app: kube-ovn-monitor + component: network + type: infra + spec: + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: kube-ovn-monitor + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + hostNetwork: true + containers: + - name: kube-ovn-monitor + image: "$REGISTRY/kube-ovn:$VERSION" + imagePullPolicy: $IMAGE_PULL_POLICY + command: ["/kube-ovn/start-ovn-monitor.sh"] + securityContext: + runAsUser: 0 + privileged: false + env: + - name: ENABLE_SSL + value: "$ENABLE_SSL" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IPS + valueFrom: + fieldRef: + fieldPath: status.podIPs + - name: ENABLE_BIND_LOCAL_IP + value: "$ENABLE_BIND_LOCAL_IP" + resources: + requests: + cpu: 200m + memory: 200Mi + limits: + cpu: 200m + memory: 200Mi + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - cat + - /var/run/ovn/ovn-controller.pid + periodSeconds: 10 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - cat + - /var/run/ovn/ovn-controller.pid + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + labels: + app: kube-ovn-monitor +spec: + ports: + - name: metrics + port: 10661 + type: ClusterIP + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-monitor + sessionAffinity: None +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-pinger + namespace: kube-system + labels: + app: kube-ovn-pinger +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-pinger + ports: + - port: 8080 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-controller + namespace: kube-system + labels: + app: kube-ovn-controller +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-controller + ports: + - port: 10660 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-cni + namespace: kube-system + labels: + app: kube-ovn-cni +spec: + ${SVC_YAML_IPFAMILYPOLICY} + selector: + app: kube-ovn-cni + ports: + - port: 10665 + name: metrics +EOF + +kubectl apply -f kube-ovn.yaml +kubectl rollout status deployment/kube-ovn-controller -n kube-system --timeout 300s +kubectl rollout status daemonset/kube-ovn-cni -n kube-system --timeout 300s +echo "-------------------------------" +echo "" + +echo "[Step 4/6] Delete pod that not in host network mode" +for ns in $(kubectl get ns --no-headers -o custom-columns=NAME:.metadata.name); do + for pod in $(kubectl get pod --no-headers -n "$ns" --field-selector spec.restartPolicy=Always -o custom-columns=NAME:.metadata.name,HOST:spec.hostNetwork | awk '{if ($2!="true") print $1}'); do + kubectl delete pod "$pod" -n "$ns" --ignore-not-found + done +done + +sleep 5 +kubectl rollout status daemonset/kube-ovn-pinger -n kube-system --timeout 300s +#kubectl rollout status deployment/coredns -n kube-system --timeout 600s +echo "-------------------------------" +echo "" + +echo "[Step 5/6] Add kubectl plugin PATH" + +if ! sh -c "echo \":$PATH:\" | grep -q \":/usr/local/bin:\""; then + echo "Tips:Please join the /usr/local/bin to your PATH. Temporarily, we do it for this execution." + export PATH=/usr/local/bin:$PATH + echo "-------------------------------" + echo "" +fi + +echo "[Step 6/6] Run network diagnose" +kubectl cp kube-system/$(kubectl -n kube-system get pods -o wide | grep cni | awk '{print $1}' | awk 'NR==1{print}'):/kube-ovn/kubectl-ko /usr/local/bin/kubectl-ko +chmod +x /usr/local/bin/kubectl-ko +kubectl ko diagnose all + +echo "-------------------------------" +echo " + ,,,, + ,::, + ,,::,,,, + ,,,,,::::::::::::,,,,, + ,,,::::::::::::::::::::::,,, + ,,::::::::::::::::::::::::::::,, + ,,::::::::::::::::::::::::::::::::,, + ,::::::::::::::::::::::::::::::::::::, + ,:::::::::::::,, ,,:::::,,,::::::::::, + ,,:::::::::::::, ,::, ,:::::::::, + ,:::::::::::::, :x, ,:: :, ,:::::::::, +,:::::::::::::::, ,,, ,::, ,, ,::::::::::, +,:::::::::::::::::,,,,,,:::::,,,,::::::::::::, ,:, ,:, ,xx, ,:::::, ,:, ,:: :::, ,x +,::::::::::::::::::::::::::::::::::::::::::::, :x: ,:xx: , :xx, :xxxxxxxxx, :xx, ,xx:,xxxx, :x +,::::::::::::::::::::::::::::::::::::::::::::, :xxxxx:, ,xx, :x: :xxx:x::, ::xxxx: :xx:, ,:xxx :xx, ,xx: ,xxxxx:, :x +,::::::::::::::::::::::::::::::::::::::::::::, :xxxxx, :xx, :x: :xxx,,:xx,:xx:,:xx, ,,,,,,,,,xxx, ,xx: :xx:xx: ,xxx,:xx::x +,::::::,,::::::::,,::::::::,,:::::::,,,::::::, :x:,xxx: ,xx, :xx :xx: ,xx,xxxxxx:, ,xxxxxxx:,xxx:, ,xxx, :xxx: ,xxx, :xxxx +,::::, ,::::, ,:::::, ,,::::, ,::::, :x: ,:xx,,:xx::xxxx,,xxx::xx: :xx::::x: ,,,,,, ,xxxxxxxxx, ,xx: ,xxx, :xxx +,::::, ,::::, ,::::, ,::::, ,::::, ,:, ,:, ,,::,,:, ,::::,, ,:::::, ,,:::::, ,, :x: ,:: +,::::, ,::::, ,::::, ,::::, ,::::, + ,,,,, ,::::, ,::::, ,::::, ,:::, ,,,,,,,,,,,,, + ,::::, ,::::, ,::::, ,:::, ,,,:::::::::::::::, + ,::::, ,::::, ,::::, ,::::, ,,,,:::::::::,,,,,,,:::, + ,::::, ,::::, ,::::, ,::::::::::::,,,,, + ,,,, ,::::, ,,,, ,,,::::,,,, + ,::::, + ,,::, +" +echo "Thanks for choosing Kube-OVN! +For more advanced features, please read https://kubeovn.github.io/docs/stable/en/ +If you have any question, please file an issue https://github.com/kubeovn/kube-ovn/issues/new/choose" + + + + + + + + + + + diff --git a/roles/kube-ovn/templates/kube-ovn.yaml.j2 b/roles/kube-ovn/templates/kube-ovn.yaml.j2 deleted file mode 100644 index b1f4720..0000000 --- a/roles/kube-ovn/templates/kube-ovn.yaml.j2 +++ /dev/null @@ -1,372 +0,0 @@ ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - annotations: - kubernetes.io/description: | - kube-ovn controller -spec: - replicas: 1 - selector: - matchLabels: - app: kube-ovn-controller - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 100% - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-controller - component: network - type: infra - spec: - tolerations: - - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: kube-ovn-controller - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: kube-ovn-controller - image: "kubeovn/kube-ovn:v1.5.3" - imagePullPolicy: IfNotPresent - command: - - /kube-ovn/start-controller.sh - args: - - --default-cidr={{ kube_ovn_default_cidr }} - - --default-gateway={{ kube_ovn_default_gateway }} - - --node-switch-cidr={{ kube_ovn_node_switch_cidr }} - - --network-type=geneve - - --default-interface-name= - - --default-vlan-id=100 - env: - - name: ENABLE_SSL - value: "false" - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KUBE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - sh - - /kube-ovn/kube-ovn-controller-healthcheck.sh - periodSeconds: 3 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - sh - - /kube-ovn/kube-ovn-controller-healthcheck.sh - initialDelaySeconds: 300 - periodSeconds: 7 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the kube-ovn cni daemon. -spec: - selector: - matchLabels: - app: kube-ovn-cni - template: - metadata: - labels: - app: kube-ovn-cni - component: network - type: infra - spec: - tolerations: - - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - initContainers: - - name: install-cni - image: "kubeovn/kube-ovn:v1.5.3" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/install-cni.sh"] - securityContext: - runAsUser: 0 - privileged: true - volumeMounts: - - mountPath: /opt/cni/bin - name: cni-bin - containers: - - name: cni-server - image: "kubeovn/kube-ovn:v1.5.3" - imagePullPolicy: IfNotPresent - command: - - sh - - /kube-ovn/start-cniserver.sh - args: - - --enable-mirror={{ kube_ovn_enable_mirror|string|lower }} - - --service-cluster-ip-range={{ SERVICE_CIDR }} - - --encap-checksum=true - - --iface= - - --network-type=geneve - - --default-interface-name= - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "false" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /etc/cni/net.d - name: cni-conf - - mountPath: /run/openvswitch - name: host-run-ovs - - mountPath: /run/ovn - name: host-run-ovn - - mountPath: /var/run/netns - name: host-ns - mountPropagation: HostToContainer - readinessProbe: - exec: - command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10665" - periodSeconds: 3 - livenessProbe: - exec: - command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10665" - initialDelaySeconds: 30 - periodSeconds: 7 - failureThreshold: 5 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 1Gi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: cni-conf - hostPath: - path: /etc/cni/net.d - - name: cni-bin - hostPath: - path: /opt/cni/bin - - name: host-ns - hostPath: - path: /var/run/netns - ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: kube-ovn-pinger - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - app: kube-ovn-pinger - component: network - type: infra - spec: - tolerations: - - operator: Exists - serviceAccountName: ovn - hostPID: true - containers: - - name: pinger - image: "kubeovn/kube-ovn:v1.5.3" - command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114", "--external-dns=alauda.cn"] - imagePullPolicy: IfNotPresent - securityContext: - runAsUser: 0 - privileged: false - env: - - name: ENABLE_SSL - value: "false" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /run/openvswitch - name: host-run-ovs - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/run/tls - name: kube-ovn-tls - resources: - requests: - cpu: 100m - memory: 200Mi - limits: - cpu: 200m - memory: 400Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-pinger - namespace: kube-system - labels: - app: kube-ovn-pinger -spec: - selector: - app: kube-ovn-pinger - ports: - - port: 8080 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-controller - namespace: kube-system - labels: - app: kube-ovn-controller -spec: - selector: - app: kube-ovn-controller - ports: - - port: 10660 - name: metrics ---- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-cni - namespace: kube-system - labels: - app: kube-ovn-cni -spec: - selector: - app: kube-ovn-cni - ports: - - port: 10665 - name: metrics diff --git a/roles/kube-ovn/templates/kubectl-ko.j2 b/roles/kube-ovn/templates/kubectl-ko.j2 deleted file mode 100644 index b4d999a..0000000 --- a/roles/kube-ovn/templates/kubectl-ko.j2 +++ /dev/null @@ -1,288 +0,0 @@ -#!/bin/bash -set -euo pipefail - -KUBE_OVN_NS=kube-system -OVN_NB_POD= -OVN_SB_POD= - -showHelp(){ - echo "kubectl ko {subcommand} [option...]" - echo "Available Subcommands:" - echo " nbctl [ovn-nbctl options ...] invoke ovn-nbctl" - echo " sbctl [ovn-sbctl options ...] invoke ovn-sbctl" - echo " vsctl {nodeName} [ovs-vsctl options ...] invoke ovs-vsctl on selected node" - echo " tcpdump {namespace/podname} [tcpdump options ...] capture pod traffic" - echo " trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port] trace ovn microflow of specific packet" - echo " diagnose {all|node} [nodename] diagnose connectivity of all nodes or a specific node" -} - -tcpdump(){ - namespacedPod="$1"; shift - namespace=$(echo "$namespacedPod" | cut -d "/" -f1) - podName=$(echo "$namespacedPod" | cut -d "/" -f2) - if [ "$podName" = "$namespacedPod" ]; then - nodeName=$(kubectl get pod "$podName" -o jsonpath={.spec.nodeName}) - mac=$(kubectl get pod "$podName" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) - hostNetwork=$(kubectl get pod "$podName" -o jsonpath={.spec.hostNetwork}) - else - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - fi - - if [ -z "$nodeName" ]; then - echo "Pod $namespacedPod not exists on any node" - exit 1 - fi - - ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide| grep kube-ovn-cni| grep " $nodeName " | awk '{print $1}') - if [ -z "$ovnCni" ]; then - echo "kube-ovn-cni not exist on node $nodeName" - exit 1 - fi - - if [ "$hostNetwork" = "true" ]; then - set -x - kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn "$@" - else - nicName=$(kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r') - if [ -z "$nicName" ]; then - echo "nic doesn't exist on node $nodeName" - exit 1 - fi - set -x - kubectl exec -it "$ovnCni" -n $KUBE_OVN_NS -- tcpdump -nn -i "$nicName" "$@" - fi -} - -trace(){ - namespacedPod="$1" - namespace=$(echo "$1" | cut -d "/" -f1) - podName=$(echo "$1" | cut -d "/" -f2) - if [ "$podName" = "$1" ]; then - echo "namespace is required" - exit 1 - fi - - podIP=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address}) - mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address}) - ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch}) - hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork}) - nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName}) - - if [ "$hostNetwork" = "true" ]; then - echo "Can not trace host network pod" - exit 1 - fi - - if [ -z "$ls" ]; then - echo "pod address not ready" - exit 1 - fi - - gwMac=$(kubectl exec -it $OVN_NB_POD -n $KUBE_OVN_NS -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name=ovn-cluster-"$ls" | tr -d '\r') - - if [ -z "$gwMac" ]; then - echo "get gw mac failed" - exit 1 - fi - - dst="$2" - if [ -z "$dst" ]; then - echo "need a target ip address" - exit 1 - fi - - type="$3" - - case $type in - icmp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst" - ;; - tcp|udp) - set -x - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip4.src == $podIP && eth.dst == $gwMac && ip4.dst == $dst && $type.src == 10000 && $type.dst == $4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac - - set +x - echo "--------" - echo "Start OVS Tracing" - echo "" - echo "" - - ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}') - if [ -z "$ovsPod" ]; then - echo "ovs pod doesn't exist on node $nodeName" - exit 1 - fi - - inPort=$(kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl --format=csv --data=bare --no-heading --columns=ofport find interface external_id:iface-id="$podName"."$namespace") - case $type in - icmp) - set -x - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort",icmp,nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac" - ;; - tcp|udp) - set -x - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int in_port="$inPort","$type",nw_src="$podIP",nw_dst="$dst",dl_src="$mac",dl_dst="$gwMac","$type"_src=1000,"$type"_dst="$4" - ;; - *) - echo "type $type not supported" - echo "kubectl ko trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]" - exit 1 - ;; - esac -} - -vsctl(){ - nodeName="$1"; shift - kubectl get no "$nodeName" > /dev/null - ovsPod=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep " $nodeName " | grep ovs-ovn | awk '{print $1}') - if [ -z "$ovsPod" ]; then - echo "ovs pod doesn't exist on node $nodeName" - exit 1 - fi - kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-vsctl "$@" -} - -diagnose(){ - kubectl get crd subnets.kubeovn.io - kubectl get crd ips.kubeovn.io - kubectl get svc kube-dns -n kube-system - kubectl get svc kubernetes -n default - - kubectl get no -o wide - kubectl ko nbctl show - kubectl ko sbctl show - - checkDaemonSet kube-proxy - checkDeployment ovn-central - checkDeployment kube-ovn-controller - checkDaemonSet kube-ovn-cni - checkDaemonSet ovs-ovn - checkDeployment coredns - type="$1" - case $type in - all) - echo "### kube-ovn-controller recent log" - set +e - kubectl logs -n $KUBE_OVN_NS -l app=kube-ovn-controller --tail=100 | grep E$(date +%m%d) - set -e - echo "" - pingers=$(kubectl get pod -n $KUBE_OVN_NS | grep kube-ovn-pinger | awk '{print $1}') - for pinger in $pingers - do - nodeName=$(kubectl get pod "$pinger" -n "$KUBE_OVN_NS" -o jsonpath={.spec.nodeName}) - echo "### start to diagnose node $nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - echo "#### ovs-vsctl show results:" - kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- ovs-vsctl show - echo "" - echo "#### pinger diagnose results:" - kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node $nodeName" - echo "" - done - ;; - node) - nodeName="$2" - kubectl get no "$nodeName" > /dev/null - pinger=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep kube-ovn-pinger | grep " $nodeName " | awk '{print $1}') - echo "### start to diagnose node nodeName" - echo "#### ovn-controller log:" - kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- tail /var/log/ovn/ovn-controller.log - echo "" - kubectl exec -n $KUBE_OVN_NS -it "$pinger" -- /kube-ovn/kube-ovn-pinger --mode=job - echo "### finish diagnose node nodeName" - echo "" - ;; - *) - echo "type $type not supported" - echo "kubectl ko diagnose {all|node} [nodename]" - ;; - esac -} - -getOvnCentralPod(){ - NB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-nb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$NB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_NB_POD=$NB_POD - SB_POD=$(kubectl get pod -n $KUBE_OVN_NS -l ovn-sb-leader=true | grep ovn-central | head -n 1 | awk '{print $1}') - if [ -z "$SB_POD" ]; then - echo "nb leader not exists" - exit 1 - fi - OVN_SB_POD=$SB_POD -} - -checkDaemonSet(){ - name="$1" - currentScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.currentNumberScheduled}) - desiredScheduled=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.desiredNumberScheduled}) - available=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberAvailable}) - ready=$(kubectl get ds -n $KUBE_OVN_NS "$name" -o jsonpath={.status.numberReady}) - if [ "$currentScheduled" = "$desiredScheduled" ] && [ "$desiredScheduled" = "$available" ] && [ "$available" = "$ready" ]; then - echo "ds $name ready" - else - echo "Error ds $name not ready" - exit 1 - fi -} - -checkDeployment(){ - name="$1" - ready=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.readyReplicas}) - updated=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.updatedReplicas}) - desire=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.replicas}) - available=$(kubectl get deployment -n $KUBE_OVN_NS "$name" -o jsonpath={.status.availableReplicas}) - if [ "$ready" = "$updated" ] && [ "$updated" = "$desire" ] && [ "$desire" = "$available" ]; then - echo "deployment $name ready" - else - echo "Error deployment $name not ready" - exit 1 - fi -} - -if [ $# -lt 1 ]; then - showHelp - exit 0 -else - subcommand="$1"; shift -fi - -getOvnCentralPod - -case $subcommand in - nbctl) - kubectl exec "$OVN_NB_POD" -n $KUBE_OVN_NS -- ovn-nbctl "$@" - ;; - sbctl) - kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -- ovn-sbctl "$@" - ;; - vsctl) - vsctl "$@" - ;; - tcpdump) - tcpdump "$@" - ;; - trace) - trace "$@" - ;; - diagnose) - diagnose "$@" - ;; - *) - showHelp - ;; -esac diff --git a/roles/kube-ovn/templates/ovn.yaml.j2 b/roles/kube-ovn/templates/ovn.yaml.j2 deleted file mode 100644 index a1ddcbb..0000000 --- a/roles/kube-ovn/templates/ovn.yaml.j2 +++ /dev/null @@ -1,404 +0,0 @@ -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: kube-ovn - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' -spec: - privileged: true - allowPrivilegeEscalation: true - allowedCapabilities: - - '*' - volumes: - - '*' - hostNetwork: true - hostPorts: - - min: 0 - max: 65535 - hostIPC: true - hostPID: true - runAsUser: - rule: 'RunAsAny' - seLinux: - rule: 'RunAsAny' - supplementalGroups: - rule: 'RunAsAny' - fsGroup: - rule: 'RunAsAny' - ---- - -apiVersion: v1 -kind: ConfigMap -metadata: - name: ovn-config - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovn - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - name: system:ovn -rules: - - apiGroups: ['policy'] - resources: ['podsecuritypolicies'] - verbs: ['use'] - resourceNames: - - kube-ovn - - apiGroups: - - "kubeovn.io" - resources: - - subnets - - subnets/status - - ips - - vlans - - networks - verbs: - - "*" - - apiGroups: - - "" - resources: - - pods - - namespaces - - nodes - - configmaps - verbs: - - create - - get - - list - - watch - - patch - - update - - apiGroups: - - "" - - networking.k8s.io - - apps - - extensions - resources: - - networkpolicies - - services - - endpoints - - statefulsets - - daemonsets - - deployments - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovn -roleRef: - name: system:ovn - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovn - namespace: kube-system ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-nb - namespace: kube-system -spec: - ports: - - name: ovn-nb - protocol: TCP - port: 6641 - targetPort: 6641 - type: ClusterIP - selector: - app: ovn-central - ovn-nb-leader: "true" - sessionAffinity: None ---- -kind: Service -apiVersion: v1 -metadata: - name: ovn-sb - namespace: kube-system -spec: - ports: - - name: ovn-sb - protocol: TCP - port: 6642 - targetPort: 6642 - type: ClusterIP - selector: - app: ovn-central - ovn-sb-leader: "true" - sessionAffinity: None ---- -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ovn-central - namespace: kube-system - annotations: - kubernetes.io/description: | - OVN components: northd, nb and sb. -spec: - replicas: 1 - strategy: - rollingUpdate: - maxSurge: 0% - maxUnavailable: 100% - type: RollingUpdate - selector: - matchLabels: - app: ovn-central - template: - metadata: - labels: - app: ovn-central - component: network - type: infra - spec: - tolerations: - - operator: Exists - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: ovn-central - topologyKey: kubernetes.io/hostname - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - containers: - - name: ovn-central - image: "kubeovn/kube-ovn:v1.5.3" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-db.sh"] - securityContext: - capabilities: - add: ["SYS_NICE"] - env: - - name: ENABLE_SSL - value: "false" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: 500m - memory: 200Mi - limits: - cpu: 3 - memory: 3Gi - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - sh - - /kube-ovn/ovn-is-leader.sh - periodSeconds: 3 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - sh - - /kube-ovn/ovn-healthcheck.sh - initialDelaySeconds: 30 - periodSeconds: 7 - failureThreshold: 5 - timeoutSeconds: 45 - nodeSelector: - kubernetes.io/os: "linux" - kube-ovn/role: "master" - volumes: - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ovs-ovn - namespace: kube-system - annotations: - kubernetes.io/description: | - This daemon set launches the openvswitch daemon. -spec: - selector: - matchLabels: - app: ovs - updateStrategy: - type: OnDelete - template: - metadata: - labels: - app: ovs - component: network - type: infra - spec: - tolerations: - - operator: Exists - priorityClassName: system-cluster-critical - serviceAccountName: ovn - hostNetwork: true - hostPID: true - containers: - - name: openvswitch - image: "kubeovn/kube-ovn:v1.5.3" - imagePullPolicy: IfNotPresent - command: ["/kube-ovn/start-ovs.sh"] - securityContext: - runAsUser: 0 - privileged: true - env: - - name: ENABLE_SSL - value: "false" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: HW_OFFLOAD - value: "false" - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - sh - - /kube-ovn/ovs-healthcheck.sh - periodSeconds: 5 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - sh - - /kube-ovn/ovs-healthcheck.sh - initialDelaySeconds: 10 - periodSeconds: 5 - failureThreshold: 5 - timeoutSeconds: 45 - resources: - requests: - cpu: 200m - memory: 200Mi - limits: - cpu: 1000m - memory: 800Mi - nodeSelector: - kubernetes.io/os: "linux" - volumes: - - name: host-modules - hostPath: - path: /lib/modules - - name: host-run-ovs - hostPath: - path: /run/openvswitch - - name: host-run-ovn - hostPath: - path: /run/ovn - - name: host-sys - hostPath: - path: /sys - - name: host-config-openvswitch - hostPath: - path: /etc/origin/openvswitch - - name: host-config-ovn - hostPath: - path: /etc/origin/ovn - - name: host-log-ovs - hostPath: - path: /var/log/openvswitch - - name: host-log-ovn - hostPath: - path: /var/log/ovn - - name: kube-ovn-tls - secret: - optional: true - secretName: kube-ovn-tls diff --git a/roles/kube-ovn/vars/main.yml b/roles/kube-ovn/vars/main.yml index 1105454..35a9c0c 100644 --- a/roles/kube-ovn/vars/main.yml +++ b/roles/kube-ovn/vars/main.yml @@ -1,3 +1,3 @@ -kube_ovn_default_cidr: "{{ CLUSTER_CIDR }}" -kube_ovn_node_switch_cidr: 100.64.0.0/16 -kube_ovn_enable_mirror: true +# CLUSTER_CIDR_GW 作为 POD_GATEWAY，选取CLUSTER_CIDR 网段中的第一个地址 + +CLUSTER_CIDR_GW: "{{ CLUSTER_CIDR.split('.')[0] }}.{{ CLUSTER_CIDR.split('.')[1] }}.{{ CLUSTER_CIDR.split('.')[2] }}.{{ CLUSTER_CIDR.split('.')[3]|int + 1 }}"