From bc9e14a762823cbef9384e959ab2ca0fc1d650fc Mon Sep 17 00:00:00 2001 From: Louis Woods Date: Sun, 4 Nov 2018 01:07:38 -0800 Subject: [PATCH] Adds support for Multus (multiple interfaces) CNI plugin (#3166) * Adds support for Multus (multiple interfaces) CNI plugin Multus is a latin word for "Multi". As the name suggests, it acts as a Multi plugin in Kubernetes and provides multiple network interface support in a pod. Multus uses the concept of invoking delegates by grouping multiple plugins into delegates and invoking them in the sequential order of the CNI configuration file provided in json format. * Change CNI version (0.1.0->0.3.1) of Contiv to be compatible with Multus --- .gitlab-ci.yml | 15 ++++ README.md | 3 + Vagrantfile | 5 +- docs/multus.md | 73 +++++++++++++++++++ roles/download/defaults/main.yml | 12 +++ .../network_plugin/meta/main.yml | 5 ++ .../network_plugin/multus/tasks/main.yml | 11 +++ roles/kubespray-defaults/defaults/main.yaml | 1 + roles/network_plugin/contiv/defaults/main.yml | 2 +- roles/network_plugin/meta/main.yml | 5 ++ roles/network_plugin/multus/defaults/main.yml | 7 ++ .../multus/files/multus-clusterrole.yml | 16 ++++ .../files/multus-clusterrolebinding.yml | 13 ++++ .../multus/files/multus-crd.yml | 22 ++++++ .../multus/files/multus-serviceaccount.yml | 6 ++ roles/network_plugin/multus/tasks/main.yml | 19 +++++ .../multus/templates/multus-daemonset.yml.j2 | 54 ++++++++++++++ tests/files/gce_centos7-multus-calico.yml | 12 +++ tests/testcases/040_check-network-adv.yml | 65 +++++++++++++++++ 19 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 docs/multus.md create mode 100644 roles/kubernetes-apps/network_plugin/multus/tasks/main.yml create mode 100644 roles/network_plugin/multus/defaults/main.yml create mode 100644 roles/network_plugin/multus/files/multus-clusterrole.yml create mode 100644 roles/network_plugin/multus/files/multus-clusterrolebinding.yml create mode 100644 roles/network_plugin/multus/files/multus-crd.yml create mode 100644 roles/network_plugin/multus/files/multus-serviceaccount.yml create mode 100644 roles/network_plugin/multus/tasks/main.yml create mode 100644 roles/network_plugin/multus/templates/multus-daemonset.yml.j2 create mode 100644 tests/files/gce_centos7-multus-calico.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fc458239..de4285c90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -300,6 +300,10 @@ before_script: # stage: deploy-special MOVED_TO_GROUP_VARS: "true" +.centos7_multus_calico_variables: ¢os7_multus_calico_variables +# stage: deploy-part2 + MOVED_TO_GROUP_VARS: "true" + .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" @@ -638,6 +642,17 @@ gce_centos7-kube-router: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_centos7-multus-calico: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *centos7_multus_calico_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_opensuse-canal: stage: deploy-part2 <<: *job diff --git a/README.md b/README.md index 2c7175936..f9d197184 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Supported Components - [weave](https://github.com/weaveworks/weave) v2.4.1 - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.1 - Application + - [multus](https://github.com/intel/multus-cni) v3.1 - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 - [cert-manager](https://github.com/jetstack/cert-manager) v0.5.0 - [coredns](https://github.com/coredns/coredns) v1.2.5 @@ -176,6 +177,8 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. +- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. + The choice is defined with the variable `kube_network_plugin`. There is also an option to leverage built-in cloud provider networking instead. See also [Network checker](docs/netcheck.md). diff --git a/Vagrantfile b/Vagrantfile index 1c0d6d7b9..de612516f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,6 +35,8 @@ $forwarded_ports = {} $subnet = "172.17.8" $os = "ubuntu1804" $network_plugin = "flannel" +# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni +$multi_networking = false # The first three nodes are etcd servers $etcd_instances = $num_instances # The first two nodes are kube masters @@ -140,7 +142,8 @@ Vagrant.configure("2") do |config| "ip": ip, "local_release_dir" => $local_release_dir, "download_run_once": "False", - "kube_network_plugin": $network_plugin + "kube_network_plugin": $network_plugin, + "kube_network_plugin_multus": $multi_networking } config.vm.network :private_network, ip: ip diff --git a/docs/multus.md b/docs/multus.md new file mode 100644 index 000000000..2d46135f5 --- /dev/null +++ b/docs/multus.md @@ -0,0 +1,73 @@ +Multus +=========== + +Multus is a meta CNI plugin that provides multiple network interface support to +pods. For each interface, Multus delegates CNI calls to secondary CNI plugins +such as Calico, macvlan, etc. + +See [multus documentation](https://github.com/intel/multus-cni). + +## Multus installation + +Since Multus itself does not implement networking, it requires a master plugin, which is specified through the variable `kube_network_plugin`. To enable Multus an additional variable `kube_network_plugin_multus` must be set to `true`. For example, +``` +kube_network_plugin: calico +kube_network_plugin_multus: true +``` +will install Multus and Calico and configure Multus to use Calico as the primary network plugin. + +## Using Multus + +Once Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes. + +``` +cat < 0 }}" diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml index 5a3778937..82316357c 100644 --- a/roles/network_plugin/contiv/defaults/main.yml +++ b/roles/network_plugin/contiv/defaults/main.yml @@ -4,7 +4,7 @@ contiv_config_dir: "{{ kube_config_dir }}/contiv" contiv_etcd_conf_dir: "/etc/contiv/etcd" contiv_etcd_data_dir: "/var/lib/etcd/contiv-data" contiv_netmaster_port: 9999 -contiv_cni_version: 0.1.0 +contiv_cni_version: 0.3.1 # No need to download it by default, but must be defined contiv_etcd_image_repo: "{{ etcd_image_repo }}" diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml index a0fae7207..120642371 100644 --- a/roles/network_plugin/meta/main.yml +++ b/roles/network_plugin/meta/main.yml @@ -37,3 +37,8 @@ dependencies: - role: network_plugin/cloud when: kube_network_plugin == 'cloud' + + - role: network_plugin/multus + when: kube_network_plugin_multus + tags: + - multus diff --git a/roles/network_plugin/multus/defaults/main.yml b/roles/network_plugin/multus/defaults/main.yml new file mode 100644 index 000000000..2fb723103 --- /dev/null +++ b/roles/network_plugin/multus/defaults/main.yml @@ -0,0 +1,7 @@ +--- +multus_conf_file: "auto" +multus_cni_conf_dir_host: "/etc/cni/net.d" +multus_cni_bin_dir_host: "/opt/cni/bin" +multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}" +multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}" +multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}" diff --git a/roles/network_plugin/multus/files/multus-clusterrole.yml b/roles/network_plugin/multus/files/multus-clusterrole.yml new file mode 100644 index 000000000..337775be2 --- /dev/null +++ b/roles/network_plugin/multus/files/multus-clusterrole.yml @@ -0,0 +1,16 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: multus +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - '*' + verbs: + - '*' diff --git a/roles/network_plugin/multus/files/multus-clusterrolebinding.yml b/roles/network_plugin/multus/files/multus-clusterrolebinding.yml new file mode 100644 index 000000000..5980330eb --- /dev/null +++ b/roles/network_plugin/multus/files/multus-clusterrolebinding.yml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: multus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: +- kind: ServiceAccount + name: multus + namespace: kube-system diff --git a/roles/network_plugin/multus/files/multus-crd.yml b/roles/network_plugin/multus/files/multus-crd.yml new file mode 100644 index 000000000..eab4406e2 --- /dev/null +++ b/roles/network_plugin/multus/files/multus-crd.yml @@ -0,0 +1,22 @@ +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + version: v1 + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + type: string diff --git a/roles/network_plugin/multus/files/multus-serviceaccount.yml b/roles/network_plugin/multus/files/multus-serviceaccount.yml new file mode 100644 index 000000000..62423082c --- /dev/null +++ b/roles/network_plugin/multus/files/multus-serviceaccount.yml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system diff --git a/roles/network_plugin/multus/tasks/main.yml b/roles/network_plugin/multus/tasks/main.yml new file mode 100644 index 000000000..7f603973d --- /dev/null +++ b/roles/network_plugin/multus/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Multus | Copy manifest files + copy: + src: "{{ item.file }}" + dest: "{{ kube_config_dir }}" + with_items: + - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition} + - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount} + - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole} + - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding} + register: multus_manifest_1 + +- name: Multus | Copy manifest templates + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: multus-daemonset, file: multus-daemonset.yml, type: daemonset} + register: multus_manifest_2 diff --git a/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 b/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 new file mode 100644 index 000000000..11cf427d0 --- /dev/null +++ b/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 @@ -0,0 +1,54 @@ +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: kube-multus-ds-amd64 + namespace: kube-system + labels: + tier: node + app: multus +spec: + template: + metadata: + labels: + tier: node + app: multus + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: multus + containers: + - name: kube-multus + image: {{ multus_image_repo }}:{{ multus_image_tag }} + command: ["/entrypoint.sh"] + args: + - "--cni-conf-dir={{ multus_cni_conf_dir }}" + - "--cni-bin-dir={{ multus_cni_bin_dir }}" + - "--multus-conf-file={{ multus_conf_file }}" + - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}" + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + volumeMounts: + - name: cni + mountPath: {{ multus_cni_conf_dir }} + - name: cnibin + mountPath: {{ multus_cni_bin_dir }} + volumes: + - name: cni + hostPath: + path: {{ multus_cni_conf_dir_host }} + - name: cnibin + hostPath: + path: {{ multus_cni_bin_dir_host }} diff --git a/tests/files/gce_centos7-multus-calico.yml b/tests/files/gce_centos7-multus-calico.yml new file mode 100644 index 000000000..57615e285 --- /dev/null +++ b/tests/files/gce_centos7-multus-calico.yml @@ -0,0 +1,12 @@ +# Instance settings +cloud_image_family: centos-7 +cloud_region: us-central1-c +cloud_machine_type: "n1-standard-1" +mode: default + +# Deployment settings +kube_network_plugin_multus: true +kube_network_plugin: calico +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce diff --git a/tests/testcases/040_check-network-adv.yml b/tests/testcases/040_check-network-adv.yml index 709e8d8b4..819a7a485 100644 --- a/tests/testcases/040_check-network-adv.yml +++ b/tests/testcases/040_check-network-adv.yml @@ -80,3 +80,68 @@ run_once: true when: - agents.content == '{}' + + - name: Create macvlan network conf + # We cannot use only shell: below because Ansible will render the text + # with leading spaces, which means the shell will never find the string + # EOF at the beginning of a line. We can avoid Ansible's unhelpful + # heuristics by using the cmd parameter like this: + shell: + cmd: | + cat <