diff --git a/README.md b/README.md index 136778e72..fdefd7eb0 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Note: Upstart/SysV init based OS types are not supported. - [calico](https://github.com/projectcalico/calico) v3.27.3 - [cilium](https://github.com/cilium/cilium) v1.15.4 - [flannel](https://github.com/flannel-io/flannel) v0.22.0 - - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5 + - [kube-ovn](https://github.com/alauda/kube-ovn) v1.12.21 - [kube-router](https://github.com/cloudnativelabs/kube-router) v2.0.0 - [multus](https://github.com/k8snetworkplumbingwg/multus-cni) v3.8 - [weave](https://github.com/rajch/weave) v2.8.7 diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml index 7e6a576e5..b32467e34 100644 --- a/roles/kubespray-defaults/defaults/main/download.yml +++ b/roles/kubespray-defaults/defaults/main/download.yml @@ -120,7 +120,7 @@ cilium_version: "v1.15.4" cilium_cli_version: "v0.16.0" cilium_enable_hubble: false -kube_ovn_version: "v1.11.5" +kube_ovn_version: "v1.12.21" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v2.0.0" multus_version: "v3.8" @@ -275,6 +275,8 @@ cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy" cilium_hubble_envoy_image_tag: "v1.22.5" kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" kube_ovn_container_image_tag: "{{ kube_ovn_version }}" +kube_ovn_vpc_container_image_repo: "{{ docker_image_repo }}/kubeovn/vpc-nat-gateway" +kube_ovn_vpc_container_image_tag: "{{ kube_ovn_version }}" kube_ovn_dpdk_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-dpdk" kube_ovn_dpdk_container_image_tag: "{{ kube_ovn_dpdk_version }}" kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 8040cc77b..c531ffcbb 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -39,6 +39,10 @@ spec: type: string subnet: type: string + replicas: + type: integer + minimum: 1 + maximum: 3 status: type: object properties: @@ -129,6 +133,10 @@ spec: items: type: string type: array + endpoints: + items: + type: string + type: array status: type: object properties: @@ -165,23 +173,25 @@ spec: name: v1 served: true storage: true + subresources: + status: {} schema: openAPIV3Schema: type: object properties: - spec: + status: type: object properties: - lanIp: - type: string - subnet: - type: string - vpc: - type: string + externalSubnets: + items: + type: string + type: array selector: type: array items: type: string + qosPolicy: + type: string tolerations: type: array items: @@ -191,12 +201,600 @@ spec: type: string operator: type: string + enum: + - Equal + - Exists value: type: string effect: type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule tolerationSeconds: type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + spec: + type: object + properties: + lanIp: + type: string + subnet: + type: string + externalSubnets: + items: + type: string + type: array + vpc: + type: string + selector: + type: array + items: + type: string + qosPolicy: + type: string + tolerations: + type: array + items: + type: object + properties: + key: + type: string + operator: + type: string + enum: + - Equal + - Exists + value: + type: string + effect: + type: string + enum: + - NoExecute + - NoSchedule + - PreferNoSchedule + tolerationSeconds: + type: integer + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + x-kubernetes-patch-strategy: merge + x-kubernetes-patch-merge-key: key + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -249,6 +847,8 @@ spec: type: string redo: type: string + qosPolicy: + type: string conditions: type: array items: @@ -277,6 +877,10 @@ spec: type: string natGwDp: type: string + qosPolicy: + type: string + externalSubnet: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -334,6 +938,8 @@ spec: type: string redo: type: string + internalIp: + type: string conditions: type: array items: @@ -519,6 +1125,8 @@ spec: type: string redo: type: string + internalCIDR: + type: string conditions: type: array items: @@ -565,15 +1173,24 @@ spec: subresources: status: {} additionalPrinterColumns: - - jsonPath: .spec.v4ip - name: IP + - jsonPath: .status.v4Ip + name: V4IP type: string - - jsonPath: .spec.macAddress + - jsonPath: .status.v6Ip + name: V6IP + type: string + - jsonPath: .status.macAddress name: Mac type: string - - jsonPath: .spec.type + - jsonPath: .status.type name: Type type: string + - jsonPath: .status.nat + name: Nat + type: string + - jsonPath: .status.ready + name: Ready + type: boolean schema: openAPIV3Schema: type: object @@ -581,8 +1198,16 @@ spec: status: type: object properties: + type: + type: string + nat: + type: string + ready: + type: boolean v4Ip: type: string + v6Ip: + type: string macAddress: type: string conditions: @@ -609,7 +1234,9 @@ spec: type: string type: type: string - v4ip: + v4Ip: + type: string + v6Ip: type: string macAddress: type: string @@ -647,6 +1274,12 @@ spec: - jsonPath: .status.ready name: Ready type: boolean + - jsonPath: .spec.ipType + name: IpType + type: string + - jsonPath: .spec.ipName + name: IpName + type: string schema: openAPIV3Schema: type: object @@ -660,8 +1293,6 @@ spec: type: string v4Ip: type: string - macAddress: - type: string vpc: type: string conditions: @@ -686,8 +1317,14 @@ spec: properties: ovnEip: type: string + ipType: + type: string ipName: type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -716,8 +1353,8 @@ spec: - jsonPath: .status.v4Eip name: V4Eip type: string - - jsonPath: .status.v4ipCidr - name: V4Ip + - jsonPath: .status.v4IpCidr + name: V4IpCidr type: string - jsonPath: .status.ready name: Ready @@ -733,7 +1370,7 @@ spec: type: boolean v4Eip: type: string - v4ipCidr: + v4IpCidr: type: string vpc: type: string @@ -763,6 +1400,118 @@ spec: type: string ipName: type: string + vpc: + type: string + v4IpCidr: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ovn-dnat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: ovn-dnat-rules + singular: ovn-dnat-rule + shortNames: + - odnat + kind: OvnDnatRule + listKind: OvnDnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.vpc + name: Vpc + type: string + - jsonPath: .spec.ovnEip + name: Eip + type: string + - jsonPath: .status.protocol + name: Protocol + type: string + - jsonPath: .status.v4Eip + name: V4Eip + type: string + - jsonPath: .status.v4Ip + name: V4Ip + type: string + - jsonPath: .status.internalPort + name: InternalPort + type: string + - jsonPath: .status.externalPort + name: ExternalPort + type: string + - jsonPath: .spec.ipName + name: IpName + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4Eip: + type: string + v4Ip: + type: string + vpc: + type: string + externalPort: + type: string + internalPort: + type: string + protocol: + type: string + ipName: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + ovnEip: + type: string + ipType: + type: string + ipName: + type: string + externalPort: + type: string + internalPort: + type: string + protocol: + type: string + vpc: + type: string + v4Ip: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -775,12 +1524,18 @@ spec: - jsonPath: .status.enableExternal name: EnableExternal type: boolean + - jsonPath: .status.enableBfd + name: EnableBfd + type: boolean - jsonPath: .status.standby name: Standby type: boolean - jsonPath: .status.subnets name: Subnets type: string + - jsonPath: .status.extraExternalSubnets + name: ExtraExternalSubnets + type: string - jsonPath: .spec.namespaces name: Namespaces type: string @@ -792,10 +1547,16 @@ spec: properties: enableExternal: type: boolean + enableBfd: + type: boolean namespaces: items: type: string type: array + extraExternalSubnets: + items: + type: string + type: array staticRoutes: items: properties: @@ -805,6 +1566,12 @@ spec: type: string nextHopIP: type: string + ecmpMode: + type: string + bfdId: + type: string + routeTable: + type: string type: object type: array policyRoutes: @@ -859,10 +1626,16 @@ spec: type: boolean enableExternal: type: boolean + enableBfd: + type: boolean subnets: items: type: string type: array + extraExternalSubnets: + items: + type: string + type: array vpcPeerings: items: type: string @@ -989,27 +1762,24 @@ spec: - name: V4IP type: string jsonPath: .status.v4ip - - name: PV4IP + - name: V6IP type: string - jsonPath: .spec.parentV4ip + jsonPath: .status.v6ip - name: Mac type: string jsonPath: .status.mac - name: PMac type: string jsonPath: .spec.parentMac - - name: V6IP - type: string - jsonPath: .status.v6ip - - name: PV6IP - type: string - jsonPath: .spec.parentV6ip - name: Subnet type: string jsonPath: .spec.subnet - jsonPath: .status.ready name: Ready type: boolean + - jsonPath: .status.type + name: Type + type: string schema: openAPIV3Schema: type: object @@ -1017,6 +1787,8 @@ spec: status: type: object properties: + type: + type: string ready: type: boolean v4ip: @@ -1055,6 +1827,8 @@ spec: type: string subnet: type: string + type: + type: string attachSubnets: type: array items: @@ -1131,6 +1905,12 @@ spec: openAPIV3Schema: type: object properties: + metadata: + type: object + properties: + name: + type: string + pattern: ^[^0-9] status: type: object properties: @@ -1150,6 +1930,35 @@ spec: type: string u2oInterconnectionIP: type: string + u2oInterconnectionVPC: + type: string + v4usingIPrange: + type: string + v4availableIPrange: + type: string + v6usingIPrange: + type: string + v6availableIPrange: + type: string + natOutgoingPolicyRules: + type: array + items: + type: object + properties: + ruleID: + type: string + action: + type: string + enum: + - nat + - forward + match: + type: object + properties: + srcIPs: + type: string + dstIPs: + type: string conditions: type: array items: @@ -1208,8 +2017,6 @@ spec: type: string natOutgoing: type: boolean - u2oRouting: - type: boolean externalEgressGateway: type: string policyRoutingPriority: @@ -1226,6 +2033,10 @@ spec: - 253 # default - 254 # main - 255 # local + mtu: + type: integer + minimum: 68 + maximum: 65535 private: type: boolean vlan: @@ -1270,8 +2081,35 @@ spec: - allow - drop - reject + natOutgoingPolicyRules: + type: array + items: + type: object + properties: + action: + type: string + enum: + - nat + - forward + match: + type: object + properties: + srcIPs: + type: string + dstIPs: + type: string u2oInterconnection: type: boolean + u2oInterconnectionIP: + type: string + enableLb: + type: boolean + enableEcmp: + type: boolean + enableMulticastSnoop: + type: boolean + routeTable: + type: string scope: Cluster names: plural: subnets @@ -1282,6 +2120,113 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: ippools.kubeovn.io +spec: + group: kubeovn.io + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Subnet + type: string + jsonPath: .spec.subnet + - name: IPs + type: string + jsonPath: .spec.ips + - name: V4Used + type: number + jsonPath: .status.v4UsingIPs + - name: V4Available + type: number + jsonPath: .status.v4AvailableIPs + - name: V6Used + type: number + jsonPath: .status.v6UsingIPs + - name: V6Available + type: number + jsonPath: .status.v6AvailableIPs + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + subnet: + type: string + x-kubernetes-validations: + - rule: "self == oldSelf" + message: "This field is immutable." + namespaces: + type: array + x-kubernetes-list-type: set + items: + type: string + ips: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + anyOf: + - format: ipv4 + - format: ipv6 + - format: cidr + - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.\.(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$ + - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\.\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$ + required: + - subnet + - ips + status: + type: object + properties: + v4AvailableIPs: + type: number + v4UsingIPs: + type: number + v6AvailableIPs: + type: number + v6UsingIPs: + type: number + v4AvailableIPRange: + type: string + v4UsingIPRange: + type: string + v6AvailableIPRange: + type: string + v6UsingIPRange: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + scope: Cluster + names: + plural: ippools + singular: ippool + kind: IPPool + shortNames: + - ippool +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: vlans.kubeovn.io spec: @@ -1360,7 +2305,6 @@ spec: not: enum: - int - - external spec: type: object properties: @@ -1531,3 +2475,113 @@ spec: status: {} conversion: strategy: None +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: qos-policies.kubeovn.io +spec: + group: kubeovn.io + names: + plural: qos-policies + singular: qos-policy + shortNames: + - qos + kind: QoSPolicy + listKind: QoSPolicyList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.shared + name: Shared + type: string + - jsonPath: .spec.bindingType + name: BindingType + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + shared: + type: boolean + bindingType: + type: string + bandwidthLimitRules: + type: array + items: + type: object + properties: + name: + type: string + interface: + type: string + rateMax: + type: string + burstMax: + type: string + priority: + type: integer + direction: + type: string + matchType: + type: string + matchValue: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + shared: + type: boolean + bindingType: + type: string + bandwidthLimitRules: + type: array + items: + type: object + properties: + name: + type: string + interface: + type: string + rateMax: + type: string + burstMax: + type: string + priority: + type: integer + direction: + type: string + matchType: + type: string + matchValue: + type: string + required: + - name + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index 60344ea0c..f4acdedac 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -1,4 +1,190 @@ --- +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-vpc-nat-config + namespace: kube-system + annotations: + kubernetes.io/description: | + kube-ovn vpc-nat common config +data: + image: {{ kube_ovn_vpc_container_image_repo }}:{{ kube_ovn_vpc_container_image_tag }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: ovn-vpc-nat-gw-config + namespace: kube-system +data: + enable-vpc-nat-gw: "true" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-ovn-cni + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:kube-ovn-cni +rules: + - apiGroups: + - "kubeovn.io" + resources: + - subnets + - vlans + - provider-networks + verbs: + - get + - list + - watch + - apiGroups: + - "" + - "kubeovn.io" + resources: + - ovn-eips + - ovn-eips/status + - nodes + - pods + - vlans + verbs: + - get + - list + - patch + - watch + - apiGroups: + - "kubeovn.io" + resources: + - ips + verbs: + - get + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-ovn-cni +roleRef: + name: system:kube-ovn-cni + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: kube-ovn-cni + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-cni + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-cni + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-ovn-app + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:kube-ovn-app +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-ovn-app +roleRef: + name: system:kube-ovn-app + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: kube-ovn-app + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kube-ovn-app + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: kube-ovn-app + namespace: kube-system +--- kind: Deployment apiVersion: apps/v1 metadata: @@ -25,8 +211,20 @@ spec: type: infra spec: tolerations: - - operator: Exists + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: "ovn.kubernetes.io/ic-gw" + operator: NotIn + values: + - "true" + weight: 100 podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -67,7 +265,12 @@ spec: - --log_file_max_size=0 - --enable-lb-svc=false - --keep-vm-ip={{ kube_ovn_keep_vm_ip }} - - --pod-default-fip-type="" + securityContext: + runAsUser: 0 + privileged: false + capabilities: + add: + - NET_BIND_SERVICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" @@ -85,6 +288,10 @@ spec: fieldPath: spec.nodeName - name: OVN_DB_IPS value: "{{ kube_ovn_central_ips }}" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -96,18 +303,24 @@ spec: name: localtime - mountPath: /var/log/kube-ovn name: kube-ovn-log + - mountPath: /var/log/ovn + name: ovn-log - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 + - --tls=false periodSeconds: 3 timeoutSeconds: 45 livenessProbe: exec: command: - - /kube-ovn/kube-ovn-controller-healthcheck + - /kube-ovn/kube-ovn-healthcheck + - --port=10660 + - --tls=false initialDelaySeconds: 300 periodSeconds: 7 failureThreshold: 5 @@ -128,6 +341,9 @@ spec: - name: kube-ovn-log hostPath: path: /var/log/kube-ovn + - name: ovn-log + hostPath: + path: /var/log/ovn - name: kube-ovn-tls secret: optional: true @@ -161,7 +377,7 @@ spec: - key: CriticalAddonsOnly operator: Exists priorityClassName: system-node-critical - serviceAccountName: ovn + serviceAccountName: kube-ovn-cni hostNetwork: true hostPID: true initContainers: @@ -202,7 +418,13 @@ spec: - --log_file_max_size=0 securityContext: runAsUser: 0 - privileged: true + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + - NET_RAW + - SYS_ADMIN env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" @@ -231,14 +453,15 @@ spec: mountPath: /lib/modules readOnly: true - name: shared-dir - mountPath: /var/lib/kubelet/pods + mountPath: $KUBELET_DIR/pods - mountPath: /etc/openvswitch name: systemid + readOnly: true - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs - mountPropagation: Bidirectional + mountPropagation: HostToContainer - mountPath: /run/ovn name: host-run-ovn - mountPath: /host/var/run/dbus @@ -255,6 +478,7 @@ spec: name: host-log-ovn - mountPath: /etc/localtime name: localtime + readOnly: true - mountPath: /tmp name: tmp livenessProbe: @@ -262,17 +486,22 @@ spec: initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=false + timeoutSeconds: 5 readinessProbe: failureThreshold: 3 - initialDelaySeconds: 30 periodSeconds: 7 successThreshold: 1 - tcpSocket: - port: 10665 - timeoutSeconds: 3 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10665 + - --tls=false + timeoutSeconds: 5 resources: requests: cpu: {{ kube_ovn_cni_server_cpu_request }} @@ -389,28 +618,23 @@ spec: fieldRef: fieldPath: spec.nodeName volumeMounts: - - mountPath: /lib/modules - name: host-modules - readOnly: true - - mountPath: /run/openvswitch - name: host-run-ovs - mountPath: /var/run/openvswitch name: host-run-ovs - mountPath: /var/run/ovn name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /var/log/openvswitch name: host-log-ovs + readOnly: true - mountPath: /var/log/ovn name: host-log-ovn + readOnly: true - mountPath: /var/log/kube-ovn name: kube-ovn-log - mountPath: /etc/localtime name: localtime + readOnly: true - mountPath: /var/run/tls name: kube-ovn-tls resources: @@ -423,18 +647,12 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: - - name: host-modules - hostPath: - path: /lib/modules - name: host-run-ovs hostPath: path: /run/openvswitch - name: host-run-ovn hostPath: path: /run/ovn - - name: host-sys - hostPath: - path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch @@ -500,6 +718,12 @@ spec: image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/kube-ovn/start-ovn-monitor.sh"] + args: + - --secure-serving=false + - --log_file=/var/log/kube-ovn/kube-ovn-monitor.log + - --logtostderr=false + - --alsologtostderr=true + - --log_file_max_size=200 securityContext: runAsUser: 0 privileged: false @@ -510,6 +734,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: POD_IPS valueFrom: fieldRef: @@ -532,30 +760,38 @@ spec: name: host-config-openvswitch - mountPath: /etc/ovn name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn + readOnly: true - mountPath: /etc/localtime name: localtime + readOnly: true - mountPath: /var/run/tls name: kube-ovn-tls - readinessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-controller.pid - periodSeconds: 10 - timeoutSeconds: 45 + - mountPath: /var/log/kube-ovn + name: kube-ovn-log livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 exec: command: - - cat - - /var/run/ovn/ovn-controller.pid + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=false + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 45 + periodSeconds: 7 + successThreshold: 1 + exec: + command: + - /kube-ovn/kube-ovn-healthcheck + - --port=10661 + - --tls=false + timeoutSeconds: 5 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" @@ -585,6 +821,9 @@ spec: secret: optional: true secretName: kube-ovn-tls + - name: kube-ovn-log + hostPath: + path: /var/log/kube-ovn --- kind: Service apiVersion: v1 diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index d632f3b8f..453ac6072 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -1,6 +1,54 @@ --- apiVersion: v1 kind: ServiceAccount +metadata: + name: ovn-ovs + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.k8s.io/system-only: "true" + name: system:ovn-ovs +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - patch + - apiGroups: + - "" + resources: + - services + - endpoints + verbs: + - get + - apiGroups: + - apps + resources: + - controllerrevisions + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-ovs +roleRef: + name: system:ovn-ovs + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: ovn-ovs + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount metadata: name: ovn namespace: kube-system @@ -18,8 +66,11 @@ rules: - vpcs - vpcs/status - vpc-nat-gateways + - vpc-nat-gateways/status - subnets - subnets/status + - ippools + - ippools/status - ips - vips - vips/status @@ -43,59 +94,98 @@ rules: - ovn-eips/status - ovn-fips/status - ovn-snat-rules/status + - ovn-dnat-rules + - ovn-dnat-rules/status - switch-lb-rules - switch-lb-rules/status - vpc-dnses - vpc-dnses/status + - qos-policies + - qos-policies/status verbs: - "*" - apiGroups: - "" resources: - pods - - pods/exec - namespaces - - nodes - - configmaps verbs: - - create - get - list + - patch - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list - patch - update + - watch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create - apiGroups: - "k8s.cni.cncf.io" resources: - network-attachment-definitions verbs: - - create - - delete - get - - list - - update - apiGroups: - "" - networking.k8s.io - - apps - - extensions resources: - networkpolicies - - services - - services/status - - endpoints - - statefulsets - - daemonsets - - deployments - - deployments/scale + - configmaps verbs: - - create - - delete - - update - - patch - get - list - watch + - apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - apiGroups: + - "" + resources: + - services + - services/status + verbs: + - get + - list + - update + - create + - delete + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - update + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - deployments + - deployments/scale + verbs: + - get + - list + - create + - delete + - update - apiGroups: - "" resources: @@ -118,6 +208,18 @@ rules: verbs: - get - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -132,6 +234,20 @@ subjects: name: ovn namespace: kube-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: ovn + namespace: kube-system +--- kind: Service apiVersion: v1 metadata: @@ -218,7 +334,12 @@ spec: type: infra spec: tolerations: - - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -227,7 +348,7 @@ spec: app: ovn-central topologyKey: kubernetes.io/hostname priorityClassName: system-cluster-critical - serviceAccountName: ovn + serviceAccountName: ovn-ovs hostNetwork: true containers: - name: ovn-central @@ -236,7 +357,9 @@ spec: command: ["/kube-ovn/start-db.sh"] securityContext: capabilities: - add: ["SYS_NICE"] + add: + - NET_BIND_SERVICE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" @@ -260,6 +383,12 @@ spec: fieldPath: status.podIPs - name: ENABLE_BIND_LOCAL_IP value: "{{ kube_ovn_bind_local_ip_enabled }}" + - name: PROBE_INTERVAL + value: "180000" + - name: OVN_NORTHD_PROBE_INTERVAL + value: "5000" + - name: OVN_LEADER_PROBE_INTERVAL + value: "5" resources: requests: cpu: {{ kube_ovn_db_cpu_request }} @@ -349,7 +478,10 @@ spec: matchLabels: app: ovs updateStrategy: - type: OnDelete + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 template: metadata: labels: @@ -358,9 +490,14 @@ spec: type: infra spec: tolerations: - - operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + - key: CriticalAddonsOnly + operator: Exists priorityClassName: system-node-critical - serviceAccountName: ovn + serviceAccountName: ovn-ovs hostNetwork: true hostPID: true containers: @@ -371,7 +508,13 @@ spec: command: [{% if kube_ovn_dpdk_enabled %}"/kube-ovn/start-ovs-dpdk.sh"{% else %}"/kube-ovn/start-ovs.sh"{% endif %}] securityContext: runAsUser: 0 - privileged: true + privileged: false + capabilities: + add: + - NET_ADMIN + - NET_BIND_SERVICE + - SYS_MODULE + - SYS_NICE env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" @@ -379,6 +522,14 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace {% if not kube_ovn_dpdk_enabled %} - name: HW_OFFLOAD value: "{{ kube_ovn_hw_offload | string | lower }}" @@ -425,6 +576,9 @@ spec: name: localtime - mountPath: /var/run/tls name: kube-ovn-tls + - mountPath: /var/run/containerd + name: cruntime + readOnly: true readinessProbe: exec: command: @@ -511,6 +665,9 @@ spec: - name: localtime hostPath: path: /etc/localtime + - name: cruntime + hostPath: + path: /var/run/containerd - name: kube-ovn-tls secret: optional: true