From f599c2a691ebac0379b577baebc2578624327af4 Mon Sep 17 00:00:00 2001 From: Simon Lelievre Date: Fri, 28 Jun 2019 09:35:38 +0200 Subject: [PATCH] add macvlan cni to kubespray (#4901) * add macvlan cni to kubespray * macvlan: lint yaml files and fix sample config file * macvlan: add OWNERS file * add macvlan to README * macvlan : CI first shoot * macvlan : CI add full masquerade * delegate retrive pod cidr to master only * macvlan: add config for CI * macvlan: add netchecker deployment --- .gitlab-ci/packet.yml | 5 + README.md | 2 + docs/macvlan.md | 48 ++++++++ .../k8s-cluster/k8s-net-macvlan.yml | 6 + .../kubernetes/node/templates/kubelet.env.j2 | 2 +- .../node/templates/kubelet.env.v1beta1.j2 | 4 +- .../preinstall/tasks/0020-verify-settings.yml | 2 +- .../tasks/0050-create_directories.yml | 2 +- roles/network_plugin/macvlan/OWNERS | 6 + .../network_plugin/macvlan/defaults/main.yml | 6 + .../network_plugin/macvlan/files/ifdown-local | 6 + .../macvlan/files/ifdown-macvlan | 41 +++++++ roles/network_plugin/macvlan/files/ifup-local | 6 + .../network_plugin/macvlan/files/ifup-macvlan | 44 +++++++ .../network_plugin/macvlan/handlers/main.yml | 19 +++ roles/network_plugin/macvlan/tasks/main.yml | 115 ++++++++++++++++++ .../macvlan/templates/10-macvlan.conf.j2 | 15 +++ .../macvlan/templates/99-loopback.conf.j2 | 5 + .../templates/centos-network-macvlan.cfg.j2 | 14 +++ .../templates/centos-postdown-macvlan.cfg.j2 | 4 + .../templates/centos-postup-macvlan.cfg.j2 | 4 + .../templates/centos-routes-macvlan.cfg.j2 | 7 ++ .../templates/coreos-device-macvlan.cfg.j2 | 6 + .../templates/coreos-interface-macvlan.cfg.j2 | 6 + .../templates/coreos-network-macvlan.cfg.j2 | 18 +++ .../templates/coreos-service-nat_ouside.j2 | 6 + .../templates/debian-network-macvlan.cfg.j2 | 27 ++++ roles/network_plugin/meta/main.yml | 5 + tests/files/packet_debian9-macvlan-sep.yml | 12 ++ 29 files changed, 438 insertions(+), 5 deletions(-) create mode 100644 docs/macvlan.md create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-macvlan.yml create mode 100644 roles/network_plugin/macvlan/OWNERS create mode 100644 roles/network_plugin/macvlan/defaults/main.yml create mode 100644 roles/network_plugin/macvlan/files/ifdown-local create mode 100755 roles/network_plugin/macvlan/files/ifdown-macvlan create mode 100755 roles/network_plugin/macvlan/files/ifup-local create mode 100755 roles/network_plugin/macvlan/files/ifup-macvlan create mode 100644 roles/network_plugin/macvlan/handlers/main.yml create mode 100644 roles/network_plugin/macvlan/tasks/main.yml create mode 100644 roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 create mode 100644 roles/network_plugin/macvlan/templates/99-loopback.conf.j2 create mode 100644 roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 create mode 100644 roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 create mode 100644 roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 create mode 100644 tests/files/packet_debian9-macvlan-sep.yml diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index d139e76e9..9a7da6956 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -71,6 +71,11 @@ packet_ubuntu18-cilium-sep: <<: *packet when: manual +packet_debian9-macvlan-sep: + stage: deploy-part2 + <<: *packet + when: on_success + packet_debian9-calico-upgrade: stage: deploy-part2 <<: *packet diff --git a/README.md b/README.md index 6e051f411..dc6da09dc 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,8 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. +- [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network. + - [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. The choice is defined with the variable `kube_network_plugin`. There is also an diff --git a/docs/macvlan.md b/docs/macvlan.md new file mode 100644 index 000000000..66950aed3 --- /dev/null +++ b/docs/macvlan.md @@ -0,0 +1,48 @@ +Macvlan +=============== + +How to use it : +------------- + + +* Enable macvlan in `group_vars/k8s-cluster/k8s-cluster.yml` +``` +... +kube_network_plugin: macvlan +... +``` + + +* Adjust the `macvlan_interface` in `group_vars/k8s-cluster/k8s-net-macvlan.yml` or by host in the `host.yml` file: +``` +all: + hosts: + node1: + ip: 10.2.2.1 + access_ip: 10.2.2.1 + ansible_host: 10.2.2.1 + macvlan_interface: ens5 +``` + + + +Issue encountered : +------------- + +- Service DNS + +reply from unexpected source: + +add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml` + + +- Disable nodelocaldns + +The nodelocal dns IP is not reacheable. + +Disable it in `sample/group_vars/k8s-cluster/k8s-cluster.yml` +``` +enable_nodelocaldns: false +``` + + diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-macvlan.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-macvlan.yml new file mode 100644 index 000000000..d2534e72f --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-macvlan.yml @@ -0,0 +1,6 @@ +--- +# private interface, on a l2-network +macvlan_interface: "eth1" + +# Enable nat in default gateway network interface +enable_nat_default_gateway: true diff --git a/roles/kubernetes/node/templates/kubelet.env.j2 b/roles/kubernetes/node/templates/kubelet.env.j2 index a5fd64b1a..a02949786 100644 --- a/roles/kubernetes/node/templates/kubelet.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.j2 @@ -114,7 +114,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index e86ee4565..703c0dca0 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -62,7 +62,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" @@ -75,4 +75,4 @@ KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_d KUBELET_CLOUDPROVIDER="" {% endif %} -PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ No newline at end of file +PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 02112c7ed..8153bfc85 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -21,7 +21,7 @@ - name: Stop if unknown network plugin assert: - that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router'] + that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router', 'macvlan'] when: kube_network_plugin is defined ignore_errors: "{{ ignore_assert_errors }}" diff --git a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml index 91230c9de..4ba782bdc 100644 --- a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml +++ b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml @@ -51,7 +51,7 @@ - "/opt/cni/bin" - "/var/lib/calico" when: - - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"] + - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router", "macvlan"] - inventory_hostname in groups['k8s-cluster'] tags: - network diff --git a/roles/network_plugin/macvlan/OWNERS b/roles/network_plugin/macvlan/OWNERS new file mode 100644 index 000000000..c5dfbc7e9 --- /dev/null +++ b/roles/network_plugin/macvlan/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - simon +reviewers: + - simon diff --git a/roles/network_plugin/macvlan/defaults/main.yml b/roles/network_plugin/macvlan/defaults/main.yml new file mode 100644 index 000000000..70a8dd028 --- /dev/null +++ b/roles/network_plugin/macvlan/defaults/main.yml @@ -0,0 +1,6 @@ +--- +macvlan_interface: eth0 +enable_nat_default_gateway: true + +# sysctl_file_path to add sysctl conf to +sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf" diff --git a/roles/network_plugin/macvlan/files/ifdown-local b/roles/network_plugin/macvlan/files/ifdown-local new file mode 100644 index 000000000..003b8a1b4 --- /dev/null +++ b/roles/network_plugin/macvlan/files/ifdown-local @@ -0,0 +1,6 @@ +#!/bin/bash + +POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1" +if [ -x $POSTDOWNNAME ]; then + exec $POSTDOWNNAME +fi diff --git a/roles/network_plugin/macvlan/files/ifdown-macvlan b/roles/network_plugin/macvlan/files/ifdown-macvlan new file mode 100755 index 000000000..3e6f67b21 --- /dev/null +++ b/roles/network_plugin/macvlan/files/ifdown-macvlan @@ -0,0 +1,41 @@ +#!/bin/bash +# +# initscripts-macvlan +# Copyright (C) 2014 Lars Kellogg-Stedman +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /etc/init.d/functions + +cd /etc/sysconfig/network-scripts +. ./network-functions + +[ -f ../network ] && . ../network + +CONFIG=${1} + +need_config ${CONFIG} + +source_config + +OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}" + +if [ ! -x ${OTHERSCRIPT} ]; then + OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth" +fi + +${OTHERSCRIPT} ${CONFIG} + +ip link del ${DEVICE} type ${TYPE:-macvlan} + diff --git a/roles/network_plugin/macvlan/files/ifup-local b/roles/network_plugin/macvlan/files/ifup-local new file mode 100755 index 000000000..3b6891eb9 --- /dev/null +++ b/roles/network_plugin/macvlan/files/ifup-local @@ -0,0 +1,6 @@ +#!/bin/bash + +POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1" +if [ -x $POSTUPNAME ]; then + exec $POSTUPNAME +fi diff --git a/roles/network_plugin/macvlan/files/ifup-macvlan b/roles/network_plugin/macvlan/files/ifup-macvlan new file mode 100755 index 000000000..2cf33f614 --- /dev/null +++ b/roles/network_plugin/macvlan/files/ifup-macvlan @@ -0,0 +1,44 @@ +#!/bin/bash +# +# initscripts-macvlan +# Copyright (C) 2014 Lars Kellogg-Stedman +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /etc/init.d/functions + +cd /etc/sysconfig/network-scripts +. ./network-functions + +[ -f ../network ] && . ../network + +CONFIG=${1} + +need_config ${CONFIG} + +source_config + +OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}" + +if [ ! -x ${OTHERSCRIPT} ]; then + OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth" +fi + +ip link add \ + link ${MACVLAN_PARENT} \ + name ${DEVICE} \ + type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private} + +${OTHERSCRIPT} ${CONFIG} + diff --git a/roles/network_plugin/macvlan/handlers/main.yml b/roles/network_plugin/macvlan/handlers/main.yml new file mode 100644 index 000000000..9c14d19f6 --- /dev/null +++ b/roles/network_plugin/macvlan/handlers/main.yml @@ -0,0 +1,19 @@ +--- +- name: Macvlan | restart network + command: /bin/true + notify: + - Macvlan | reload network + when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + +- name: Macvlan | reload network + service: + name: >- + {% if ansible_os_family == "RedHat" -%} + network + {%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%} + systemd-networkd + {%- elif ansible_os_family == "Debian" -%} + networking + {%- endif %} + state: restarted + when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and kube_network_plugin not in ['canal', 'calico'] diff --git a/roles/network_plugin/macvlan/tasks/main.yml b/roles/network_plugin/macvlan/tasks/main.yml new file mode 100644 index 000000000..f2cb3a581 --- /dev/null +++ b/roles/network_plugin/macvlan/tasks/main.yml @@ -0,0 +1,115 @@ +--- + +- name: Macvlan | Set cni directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + +- name: Macvlan | Copy cni plugins + unarchive: + src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" + dest: "/opt/cni/bin" + mode: 0755 + remote_src: yes + +- name: Macvlan | Retreive Pod Cidr + command: "{{ bin_dir }}/kubectl get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'" + register: node_pod_cidr_cmd + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: Macvlan | set node_pod_cidr + set_fact: + node_pod_cidr={{ node_pod_cidr_cmd.stdout }} + +- name: Macvlan | Retreive default gateway network interface + become: false + raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/' + register: node_default_gateway_interface_cmd + +- name: Macvlan | set node_default_gateway_interface + set_fact: + node_default_gateway_interface={{ node_default_gateway_interface_cmd.stdout | trim }} + +- name: Macvlan | Install network gateway interface on debian + template: + src: debian-network-macvlan.cfg.j2 + dest: /etc/network/interfaces.d/60-mac0.cfg + notify: Macvlan | restart network + when: ansible_os_family in ["Debian"] + +- name: Macvlan | Install macvlan script on centos + copy: + src: "{{ item }}" + dest: /etc/sysconfig/network-scripts/ + owner: root + group: root + mode: "0755" + with_fileglob: + - files/* + when: ansible_os_family in ["CentOS","RedHat"] + +- name: Macvlan | Install post-up script on centos + copy: + src: "files/ifup-local" + dest: /sbin/ + owner: root + group: root + mode: "0755" + when: ansible_os_family in ["CentOS","RedHat"] and enable_nat_default_gateway + +- name: Macvlan | Install network gateway interface on centos + template: + src: "{{ item.src }}.j2" + dest: "/etc/sysconfig/network-scripts/{{ item.dst }}" + with_items: + - {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 } + - {src: centos-routes-macvlan.cfg, dst: route-mac0 } + - {src: centos-postup-macvlan.cfg, dst: post-up-mac0 } + notify: Macvlan | restart network + when: ansible_os_family in ["CentOS","RedHat"] + +- name: Macvlan | Install service nat via gateway on coreos + template: + src: coreos-service-nat_ouside.j2 + dest: /etc/systemd/system/enable_nat_ouside.service + when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway + +- name: Macvlan | Enable service nat via gateway on coreos + command: "{{ item }}" + with_items: + - systemctl daemon-reload + - systemctl enable enable_nat_ouside.service + when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway + +- name: Macvlan | Install network gateway interface on coreos + template: + src: "{{ item.src }}.j2" + dest: "/etc/systemd/network/{{ item.dst }}" + with_items: + - {src: coreos-device-macvlan.cfg, dst: macvlan.netdev } + - {src: coreos-interface-macvlan.cfg, dst: output.network } + - {src: coreos-network-macvlan.cfg, dst: macvlan.network } + notify: Macvlan | restart network + when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + +- name: Macvlan | Install cni definition for Macvlan + template: + src: 10-macvlan.conf.j2 + dest: /etc/cni/net.d/10-macvlan.conf + +- name: Macvlan | Install loopback definition for Macvlan + template: + src: 99-loopback.conf.j2 + dest: /etc/cni/net.d/99-loopback.conf + +- name: Enable net.ipv4.conf.all.arp_notify in sysctl + sysctl: + name: net.ipv4.conf.all.arp_notify + value: 1 + sysctl_set: yes + sysctl_file: "{{ sysctl_file_path }}" + state: present + reload: yes diff --git a/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 b/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 new file mode 100644 index 000000000..6a3874783 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/10-macvlan.conf.j2 @@ -0,0 +1,15 @@ +{ + "cniVersion": "0.3.0", + "name": "mynet", + "type": "macvlan", + "master": "{{ macvlan_interface }}", + "hairpinMode": true, + "ipam": { + "type": "host-local", + "subnet": "{{ node_pod_cidr }}", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "{{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" + } +} diff --git a/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 b/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 new file mode 100644 index 000000000..b41ab6584 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/99-loopback.conf.j2 @@ -0,0 +1,5 @@ +{ + "cniVersion": "0.2.0", + "name": "lo", + "type": "loopback" +} diff --git a/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 new file mode 100644 index 000000000..6bef22c84 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/centos-network-macvlan.cfg.j2 @@ -0,0 +1,14 @@ +DEVICE=mac0 +DEVICETYPE=macvlan +TYPE=macvlan +BOOTPROTO=none +ONBOOT=yes +NM_CONTROLLED=no + +MACVLAN_PARENT=eth2 +MACVLAN_MODE=bridge + +IPADDR={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }} +NETMASK={{ node_pod_cidr|ipaddr('netmask') }} +NETWORK={{ node_pod_cidr|ipaddr('network') }} + diff --git a/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 new file mode 100644 index 000000000..f3edd99b9 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/centos-postdown-macvlan.cfg.j2 @@ -0,0 +1,4 @@ +{% if enable_nat_default_gateway %} +iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} + diff --git a/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 new file mode 100644 index 000000000..35cd5b542 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/centos-postup-macvlan.cfg.j2 @@ -0,0 +1,4 @@ +{% if enable_nat_default_gateway %} +iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} + diff --git a/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 new file mode 100644 index 000000000..51b9ff51f --- /dev/null +++ b/roles/network_plugin/macvlan/templates/centos-routes-macvlan.cfg.j2 @@ -0,0 +1,7 @@ +{% for host in groups['kube-node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} +{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} diff --git a/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 new file mode 100644 index 000000000..2418dacfe --- /dev/null +++ b/roles/network_plugin/macvlan/templates/coreos-device-macvlan.cfg.j2 @@ -0,0 +1,6 @@ +[NetDev] +Name=mac0 +Kind=macvlan + +[MACVLAN] +Mode=bridge diff --git a/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 new file mode 100644 index 000000000..342f68081 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/coreos-interface-macvlan.cfg.j2 @@ -0,0 +1,6 @@ +[Match] +Name={{ macvlan_interface }} + +[Network] +MACVLAN=mac0 +DHCP=yes diff --git a/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 new file mode 100644 index 000000000..ea96cb404 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/coreos-network-macvlan.cfg.j2 @@ -0,0 +1,18 @@ +[Match] +Name=mac0 + +[Network] +Address={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}/{{ node_pod_cidr|ipaddr('prefix') }} + +{% for host in groups['kube-node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} +[Route] +Gateway={{ hostvars[host]['access_ip'] }} +Destination={{ hostvars[host]['node_pod_cidr'] }} +GatewayOnlink=yes + +{% endif %} +{% endif %} +{% endfor %} + diff --git a/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 b/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 new file mode 100644 index 000000000..5f00b00c4 --- /dev/null +++ b/roles/network_plugin/macvlan/templates/coreos-service-nat_ouside.j2 @@ -0,0 +1,6 @@ +[Service] +Type=oneshot +ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE" + +[Install] +WantedBy=sys-subsystem-net-devices-mac0.device diff --git a/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 b/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 new file mode 100644 index 000000000..0f2cbc15f --- /dev/null +++ b/roles/network_plugin/macvlan/templates/debian-network-macvlan.cfg.j2 @@ -0,0 +1,27 @@ +auto mac0 +iface mac0 inet static + address {{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }} + network {{ node_pod_cidr|ipaddr('network') }} + netmask {{ node_pod_cidr|ipaddr('netmask') }} + broadcast {{ node_pod_cidr|ipaddr('broadcast') }} + pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge +{% for host in groups['kube-node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} + post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} +{% if enable_nat_default_gateway %} + post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE +{% endif %} +{% for host in groups['kube-node'] %} +{% if hostvars[host]['access_ip'] is defined %} +{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %} + post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }} +{% endif %} +{% endif %} +{% endfor %} + post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE + post-down ip link delete mac0 + diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml index 34e026b8c..4b8850787 100644 --- a/roles/network_plugin/meta/main.yml +++ b/roles/network_plugin/meta/main.yml @@ -30,6 +30,11 @@ dependencies: tags: - cni + - role: network_plugin/macvlan + when: kube_network_plugin == 'macvlan' + tags: + - macvlan + - role: network_plugin/contiv when: kube_network_plugin == 'contiv' tags: diff --git a/tests/files/packet_debian9-macvlan-sep.yml b/tests/files/packet_debian9-macvlan-sep.yml new file mode 100644 index 000000000..01e4fc3fc --- /dev/null +++ b/tests/files/packet_debian9-macvlan-sep.yml @@ -0,0 +1,12 @@ +--- +# Instance settings +cloud_image: debian-9 +mode: default + +# Kubespray settings +kube_network_plugin: macvlan +deploy_netchecker: true +enable_nodelocaldns: false +dns_min_replicas: 1 +kube_proxy_masquerade_all: true +macvlan_interface: "eth0"