add macvlan cni to kubespray (#4901)

* add macvlan cni to kubespray

* macvlan: lint yaml files and fix sample config file

* macvlan: add OWNERS file

* add macvlan to README

* macvlan : CI first shoot

* macvlan : CI add full masquerade

* delegate retrive pod cidr to master only

* macvlan: add config for CI

* macvlan: add netchecker deployment

.gitlab-ci/packet.yml

@@ -71,6 +71,11 @@ packet_ubuntu18-cilium-sep:
   <<: *packet
   when: manual
 
+packet_debian9-macvlan-sep:
+  stage: deploy-part2
+  <<: *packet
+  when: on_success
+
 packet_debian9-calico-upgrade:
   stage: deploy-part2
   <<: *packet

README.md

@@ -178,6 +178,8 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use
     iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
     It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
 
+-   [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
+
 -   [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
 
 The choice is defined with the variable `kube_network_plugin`. There is also an

docs/macvlan.md

@@ -0,0 +1,48 @@
+Macvlan
+===============
+
+How to use it :
+-------------
+
+
+* Enable macvlan in `group_vars/k8s-cluster/k8s-cluster.yml`
+```
+...
+kube_network_plugin: macvlan
+...
+```
+
+
+* Adjust the `macvlan_interface` in `group_vars/k8s-cluster/k8s-net-macvlan.yml` or by host in the `host.yml` file: 
+```
+all:
+  hosts:
+    node1:
+      ip: 10.2.2.1
+      access_ip: 10.2.2.1
+      ansible_host: 10.2.2.1
+      macvlan_interface: ens5
+```
+
+
+
+Issue encountered :
+-------------
+
+- Service DNS
+
+reply from unexpected source:
+
+add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`
+
+
+- Disable nodelocaldns
+
+The nodelocal dns IP is not reacheable.
+
+Disable it in `sample/group_vars/k8s-cluster/k8s-cluster.yml`
+```
+enable_nodelocaldns: false
+```
+
+

inventory/sample/group_vars/k8s-cluster/k8s-net-macvlan.yml

@@ -0,0 +1,6 @@
+---
+# private interface, on a l2-network
+macvlan_interface: "eth1"
+
+# Enable nat in default gateway network interface
+enable_nat_default_gateway: true

roles/kubernetes/node/templates/kubelet.env.j2

@@ -114,7 +114,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 
 KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -62,7 +62,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 
 KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
 KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -21,7 +21,7 @@
 
 - name: Stop if unknown network plugin
   assert:
-    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router']
+    that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router', 'macvlan']
   when: kube_network_plugin is defined
   ignore_errors: "{{ ignore_assert_errors }}"
 

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -51,7 +51,7 @@
     - "/opt/cni/bin"
     - "/var/lib/calico"
   when:
-    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"]
+    - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router", "macvlan"]
     - inventory_hostname in groups['k8s-cluster']
   tags:
     - network

roles/network_plugin/macvlan/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - simon
+reviewers:
+  - simon

roles/network_plugin/macvlan/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+macvlan_interface: eth0
+enable_nat_default_gateway: true
+
+# sysctl_file_path to add sysctl conf to
+sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"

roles/network_plugin/macvlan/files/ifdown-local

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1"
+if [ -x $POSTDOWNNAME ]; then
+  exec $POSTDOWNNAME
+fi

roles/network_plugin/macvlan/files/ifdown-macvlan

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# initscripts-macvlan
+# Copyright (C) 2014 Lars Kellogg-Stedman
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /etc/init.d/functions
+
+cd /etc/sysconfig/network-scripts
+. ./network-functions
+
+[ -f ../network ] && . ../network
+
+CONFIG=${1}
+
+need_config ${CONFIG}
+
+source_config
+
+OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}"
+
+if [ ! -x ${OTHERSCRIPT} ]; then
+	OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth"
+fi
+
+${OTHERSCRIPT} ${CONFIG}
+
+ip link del ${DEVICE} type ${TYPE:-macvlan}
+

roles/network_plugin/macvlan/files/ifup-local

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1"
+if [ -x $POSTUPNAME ]; then
+  exec $POSTUPNAME
+fi

roles/network_plugin/macvlan/files/ifup-macvlan

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# initscripts-macvlan
+# Copyright (C) 2014 Lars Kellogg-Stedman
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /etc/init.d/functions
+
+cd /etc/sysconfig/network-scripts
+. ./network-functions
+
+[ -f ../network ] && . ../network
+
+CONFIG=${1}
+
+need_config ${CONFIG}
+
+source_config
+
+OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}"
+
+if [ ! -x ${OTHERSCRIPT} ]; then
+	OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth"
+fi
+
+ip link add \
+	link ${MACVLAN_PARENT} \
+	name ${DEVICE} \
+	type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private}
+
+${OTHERSCRIPT} ${CONFIG}
+

roles/network_plugin/macvlan/handlers/main.yml

@@ -0,0 +1,19 @@
+---
+- name: Macvlan | restart network
+  command: /bin/true
+  notify:
+    - Macvlan | reload network
+  when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+
+- name: Macvlan | reload network
+  service:
+    name: >-
+      {% if ansible_os_family == "RedHat" -%}
+      network
+      {%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%}
+      systemd-networkd
+      {%- elif ansible_os_family == "Debian" -%}
+      networking
+      {%- endif %}
+    state: restarted
+  when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and kube_network_plugin not in ['canal', 'calico']

roles/network_plugin/macvlan/tasks/main.yml

@@ -0,0 +1,115 @@
+---
+
+- name: Macvlan | Set cni directory permissions
+  file:
+    path: /opt/cni/bin
+    state: directory
+    owner: kube
+    recurse: true
+    mode: 0755
+
+- name: Macvlan | Copy cni plugins
+  unarchive:
+    src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+    dest: "/opt/cni/bin"
+    mode: 0755
+    remote_src: yes
+
+- name: Macvlan | Retreive Pod Cidr
+  command: "{{ bin_dir }}/kubectl get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'"
+  register: node_pod_cidr_cmd
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Macvlan | set node_pod_cidr
+  set_fact:
+    node_pod_cidr={{ node_pod_cidr_cmd.stdout }}
+
+- name: Macvlan | Retreive default gateway network interface
+  become: false
+  raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/'
+  register: node_default_gateway_interface_cmd
+
+- name: Macvlan | set node_default_gateway_interface
+  set_fact:
+    node_default_gateway_interface={{ node_default_gateway_interface_cmd.stdout | trim }}
+
+- name: Macvlan | Install network gateway interface on debian
+  template:
+    src: debian-network-macvlan.cfg.j2
+    dest: /etc/network/interfaces.d/60-mac0.cfg
+  notify: Macvlan | restart network
+  when: ansible_os_family in ["Debian"]
+
+- name: Macvlan | Install macvlan script on centos
+  copy:
+    src: "{{ item }}"
+    dest: /etc/sysconfig/network-scripts/
+    owner: root
+    group: root
+    mode: "0755"
+  with_fileglob:
+    - files/*
+  when: ansible_os_family in ["CentOS","RedHat"]
+
+- name: Macvlan | Install post-up script on centos
+  copy:
+    src: "files/ifup-local"
+    dest: /sbin/
+    owner: root
+    group: root
+    mode: "0755"
+  when: ansible_os_family in ["CentOS","RedHat"] and enable_nat_default_gateway
+
+- name: Macvlan | Install network gateway interface on centos
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "/etc/sysconfig/network-scripts/{{ item.dst }}"
+  with_items:
+    - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 }
+    - {src: centos-routes-macvlan.cfg, dst: route-mac0 }
+    - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 }
+  notify: Macvlan | restart network
+  when: ansible_os_family in ["CentOS","RedHat"]
+
+- name: Macvlan | Install service nat via gateway on coreos
+  template:
+    src: coreos-service-nat_ouside.j2
+    dest: /etc/systemd/system/enable_nat_ouside.service
+  when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway
+
+- name: Macvlan | Enable service nat via gateway on coreos
+  command: "{{ item }}"
+  with_items:
+    - systemctl daemon-reload
+    - systemctl enable enable_nat_ouside.service
+  when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway
+
+- name: Macvlan | Install network gateway interface on coreos
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "/etc/systemd/network/{{ item.dst }}"
+  with_items:
+    - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev }
+    - {src: coreos-interface-macvlan.cfg, dst: output.network }
+    - {src: coreos-network-macvlan.cfg, dst: macvlan.network }
+  notify: Macvlan | restart network
+  when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+
+- name: Macvlan | Install cni definition for Macvlan
+  template:
+    src: 10-macvlan.conf.j2
+    dest: /etc/cni/net.d/10-macvlan.conf
+
+- name: Macvlan | Install loopback definition for Macvlan
+  template:
+    src: 99-loopback.conf.j2
+    dest: /etc/cni/net.d/99-loopback.conf
+
+- name: Enable net.ipv4.conf.all.arp_notify in sysctl
+  sysctl:
+    name: net.ipv4.conf.all.arp_notify
+    value: 1
+    sysctl_set: yes
+    sysctl_file: "{{ sysctl_file_path }}"
+    state: present
+    reload: yes

roles/network_plugin/macvlan/templates/10-macvlan.conf.j2

@@ -0,0 +1,15 @@
+{
+  "cniVersion": "0.3.0",
+  "name": "mynet",
+  "type": "macvlan",
+  "master": "{{ macvlan_interface }}",
+  "hairpinMode": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "{{ node_pod_cidr }}",
+    "routes": [
+      { "dst": "0.0.0.0/0" }
+    ],
+    "gateway": "{{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
+  }
+}

roles/network_plugin/macvlan/templates/99-loopback.conf.j2

@@ -0,0 +1,5 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "lo",
+  "type": "loopback"
+}

roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2

@@ -0,0 +1,14 @@
+DEVICE=mac0
+DEVICETYPE=macvlan
+TYPE=macvlan
+BOOTPROTO=none
+ONBOOT=yes
+NM_CONTROLLED=no
+
+MACVLAN_PARENT=eth2
+MACVLAN_MODE=bridge
+
+IPADDR={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}
+NETMASK={{ node_pod_cidr|ipaddr('netmask') }}
+NETWORK={{ node_pod_cidr|ipaddr('network') }}
+

roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2

@@ -0,0 +1,4 @@
+{% if enable_nat_default_gateway %}
+iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}
+

roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2

@@ -0,0 +1,4 @@
+{% if enable_nat_default_gateway %}
+iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}
+

roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2

@@ -0,0 +1,7 @@
+{% for host in groups['kube-node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}

roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2

@@ -0,0 +1,6 @@
+[NetDev]
+Name=mac0
+Kind=macvlan
+
+[MACVLAN]
+Mode=bridge

roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2

@@ -0,0 +1,6 @@
+[Match]
+Name={{ macvlan_interface }}
+
+[Network]
+MACVLAN=mac0
+DHCP=yes

roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2

@@ -0,0 +1,18 @@
+[Match]
+Name=mac0
+
+[Network]
+Address={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}/{{ node_pod_cidr|ipaddr('prefix') }}
+
+{% for host in groups['kube-node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+[Route]
+Gateway={{ hostvars[host]['access_ip'] }}
+Destination={{ hostvars[host]['node_pod_cidr'] }}
+GatewayOnlink=yes
+
+{% endif %}
+{% endif %}
+{% endfor %}
+

roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2

@@ -0,0 +1,6 @@
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE"
+
+[Install]
+WantedBy=sys-subsystem-net-devices-mac0.device

roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2

@@ -0,0 +1,27 @@
+auto mac0
+iface mac0 inet static
+    address {{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}
+    network {{ node_pod_cidr|ipaddr('network') }}
+    netmask {{ node_pod_cidr|ipaddr('netmask') }}
+    broadcast {{ node_pod_cidr|ipaddr('broadcast') }}
+    pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge
+{% for host in groups['kube-node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+    post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}
+{% if enable_nat_default_gateway %}
+    post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+{% endif %}
+{% for host in groups['kube-node'] %}
+{% if hostvars[host]['access_ip'] is defined  %}
+{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr  %}
+    post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
+{% endif %}
+{% endif %}
+{% endfor %}
+    post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
+    post-down ip link delete mac0
+

roles/network_plugin/meta/main.yml

@@ -30,6 +30,11 @@ dependencies:
     tags:
       - cni
 
+  - role: network_plugin/macvlan
+    when: kube_network_plugin == 'macvlan'
+    tags:
+      - macvlan
+
   - role: network_plugin/contiv
     when: kube_network_plugin == 'contiv'
     tags:

tests/files/packet_debian9-macvlan-sep.yml

@@ -0,0 +1,12 @@
+---
+# Instance settings
+cloud_image: debian-9
+mode: default
+
+# Kubespray settings
+kube_network_plugin: macvlan
+deploy_netchecker: true
+enable_nodelocaldns: false
+dns_min_replicas: 1
+kube_proxy_masquerade_all: true
+macvlan_interface: "eth0"