add macvlan cni to kubespray (#4901)

* add macvlan cni to kubespray

* macvlan: lint yaml files and fix sample config file

* macvlan: add OWNERS file

* add macvlan to README

* macvlan : CI first shoot

* macvlan : CI add full masquerade

* delegate retrive pod cidr to master only

* macvlan: add config for CI

* macvlan: add netchecker deployment
pull/4931/head
Simon Lelievre 2019-06-28 09:35:38 +02:00 committed by Kubernetes Prow Robot
parent bc7d1f36ea
commit f599c2a691
29 changed files with 438 additions and 5 deletions

View File

@ -71,6 +71,11 @@ packet_ubuntu18-cilium-sep:
<<: *packet <<: *packet
when: manual when: manual
packet_debian9-macvlan-sep:
stage: deploy-part2
<<: *packet
when: on_success
packet_debian9-calico-upgrade: packet_debian9-calico-upgrade:
stage: deploy-part2 stage: deploy-part2
<<: *packet <<: *packet

View File

@ -178,6 +178,8 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use
iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers).
It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs.
- [macvlan](docs/macvlan.md): Macvlan is a Linux network driver. Pods have their own unique Mac and Ip address, connected directly the physical (layer 2) network.
- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. - [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc.
The choice is defined with the variable `kube_network_plugin`. There is also an The choice is defined with the variable `kube_network_plugin`. There is also an

48
docs/macvlan.md 100644
View File

@ -0,0 +1,48 @@
Macvlan
===============
How to use it :
-------------
* Enable macvlan in `group_vars/k8s-cluster/k8s-cluster.yml`
```
...
kube_network_plugin: macvlan
...
```
* Adjust the `macvlan_interface` in `group_vars/k8s-cluster/k8s-net-macvlan.yml` or by host in the `host.yml` file:
```
all:
hosts:
node1:
ip: 10.2.2.1
access_ip: 10.2.2.1
ansible_host: 10.2.2.1
macvlan_interface: ens5
```
Issue encountered :
-------------
- Service DNS
reply from unexpected source:
add `kube_proxy_masquerade_all: true` in `group_vars/all/all.yml`
- Disable nodelocaldns
The nodelocal dns IP is not reacheable.
Disable it in `sample/group_vars/k8s-cluster/k8s-cluster.yml`
```
enable_nodelocaldns: false
```

View File

@ -0,0 +1,6 @@
---
# private interface, on a l2-network
macvlan_interface: "eth1"
# Enable nat in default gateway network interface
enable_nat_default_gateway: true

View File

@ -114,7 +114,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %} {% endif %}
KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %} {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"

View File

@ -62,7 +62,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %} {% endif %}
KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %} {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
@ -75,4 +75,4 @@ KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_d
KUBELET_CLOUDPROVIDER="" KUBELET_CLOUDPROVIDER=""
{% endif %} {% endif %}
PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

View File

@ -21,7 +21,7 @@
- name: Stop if unknown network plugin - name: Stop if unknown network plugin
assert: assert:
that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router'] that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'contiv', 'kube-router', 'macvlan']
when: kube_network_plugin is defined when: kube_network_plugin is defined
ignore_errors: "{{ ignore_assert_errors }}" ignore_errors: "{{ ignore_assert_errors }}"

View File

@ -51,7 +51,7 @@
- "/opt/cni/bin" - "/opt/cni/bin"
- "/var/lib/calico" - "/var/lib/calico"
when: when:
- kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router"] - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium", "kube-router", "macvlan"]
- inventory_hostname in groups['k8s-cluster'] - inventory_hostname in groups['k8s-cluster']
tags: tags:
- network - network

View File

@ -0,0 +1,6 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- simon
reviewers:
- simon

View File

@ -0,0 +1,6 @@
---
macvlan_interface: eth0
enable_nat_default_gateway: true
# sysctl_file_path to add sysctl conf to
sysctl_file_path: "/etc/sysctl.d/99-sysctl.conf"

View File

@ -0,0 +1,6 @@
#!/bin/bash
POSTDOWNNAME="/etc/sysconfig/network-scripts/post-down-$1"
if [ -x $POSTDOWNNAME ]; then
exec $POSTDOWNNAME
fi

View File

@ -0,0 +1,41 @@
#!/bin/bash
#
# initscripts-macvlan
# Copyright (C) 2014 Lars Kellogg-Stedman
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. /etc/init.d/functions
cd /etc/sysconfig/network-scripts
. ./network-functions
[ -f ../network ] && . ../network
CONFIG=${1}
need_config ${CONFIG}
source_config
OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-${REAL_DEVICETYPE}"
if [ ! -x ${OTHERSCRIPT} ]; then
OTHERSCRIPT="/etc/sysconfig/network-scripts/ifdown-eth"
fi
${OTHERSCRIPT} ${CONFIG}
ip link del ${DEVICE} type ${TYPE:-macvlan}

View File

@ -0,0 +1,6 @@
#!/bin/bash
POSTUPNAME="/etc/sysconfig/network-scripts/post-up-$1"
if [ -x $POSTUPNAME ]; then
exec $POSTUPNAME
fi

View File

@ -0,0 +1,44 @@
#!/bin/bash
#
# initscripts-macvlan
# Copyright (C) 2014 Lars Kellogg-Stedman
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
. /etc/init.d/functions
cd /etc/sysconfig/network-scripts
. ./network-functions
[ -f ../network ] && . ../network
CONFIG=${1}
need_config ${CONFIG}
source_config
OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-${REAL_DEVICETYPE}"
if [ ! -x ${OTHERSCRIPT} ]; then
OTHERSCRIPT="/etc/sysconfig/network-scripts/ifup-eth"
fi
ip link add \
link ${MACVLAN_PARENT} \
name ${DEVICE} \
type ${TYPE:-macvlan} mode ${MACVLAN_MODE:-private}
${OTHERSCRIPT} ${CONFIG}

View File

@ -0,0 +1,19 @@
---
- name: Macvlan | restart network
command: /bin/true
notify:
- Macvlan | reload network
when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
- name: Macvlan | reload network
service:
name: >-
{% if ansible_os_family == "RedHat" -%}
network
{%- elif ansible_distribution == "Ubuntu" and ansible_distribution_release == "bionic" -%}
systemd-networkd
{%- elif ansible_os_family == "Debian" -%}
networking
{%- endif %}
state: restarted
when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and kube_network_plugin not in ['canal', 'calico']

View File

@ -0,0 +1,115 @@
---
- name: Macvlan | Set cni directory permissions
file:
path: /opt/cni/bin
state: directory
owner: kube
recurse: true
mode: 0755
- name: Macvlan | Copy cni plugins
unarchive:
src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
dest: "/opt/cni/bin"
mode: 0755
remote_src: yes
- name: Macvlan | Retreive Pod Cidr
command: "{{ bin_dir }}/kubectl get nodes {{ kube_override_hostname | default(inventory_hostname) }} -o jsonpath='{.spec.podCIDR}'"
register: node_pod_cidr_cmd
delegate_to: "{{ groups['kube-master'][0] }}"
- name: Macvlan | set node_pod_cidr
set_fact:
node_pod_cidr={{ node_pod_cidr_cmd.stdout }}
- name: Macvlan | Retreive default gateway network interface
become: false
raw: ip -4 route list 0/0 | sed 's/.*dev \([[:alnum:]]*\).*/\1/'
register: node_default_gateway_interface_cmd
- name: Macvlan | set node_default_gateway_interface
set_fact:
node_default_gateway_interface={{ node_default_gateway_interface_cmd.stdout | trim }}
- name: Macvlan | Install network gateway interface on debian
template:
src: debian-network-macvlan.cfg.j2
dest: /etc/network/interfaces.d/60-mac0.cfg
notify: Macvlan | restart network
when: ansible_os_family in ["Debian"]
- name: Macvlan | Install macvlan script on centos
copy:
src: "{{ item }}"
dest: /etc/sysconfig/network-scripts/
owner: root
group: root
mode: "0755"
with_fileglob:
- files/*
when: ansible_os_family in ["CentOS","RedHat"]
- name: Macvlan | Install post-up script on centos
copy:
src: "files/ifup-local"
dest: /sbin/
owner: root
group: root
mode: "0755"
when: ansible_os_family in ["CentOS","RedHat"] and enable_nat_default_gateway
- name: Macvlan | Install network gateway interface on centos
template:
src: "{{ item.src }}.j2"
dest: "/etc/sysconfig/network-scripts/{{ item.dst }}"
with_items:
- {src: centos-network-macvlan.cfg, dst: ifcfg-mac0 }
- {src: centos-routes-macvlan.cfg, dst: route-mac0 }
- {src: centos-postup-macvlan.cfg, dst: post-up-mac0 }
notify: Macvlan | restart network
when: ansible_os_family in ["CentOS","RedHat"]
- name: Macvlan | Install service nat via gateway on coreos
template:
src: coreos-service-nat_ouside.j2
dest: /etc/systemd/system/enable_nat_ouside.service
when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway
- name: Macvlan | Enable service nat via gateway on coreos
command: "{{ item }}"
with_items:
- systemctl daemon-reload
- systemctl enable enable_nat_ouside.service
when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and enable_nat_default_gateway
- name: Macvlan | Install network gateway interface on coreos
template:
src: "{{ item.src }}.j2"
dest: "/etc/systemd/network/{{ item.dst }}"
with_items:
- {src: coreos-device-macvlan.cfg, dst: macvlan.netdev }
- {src: coreos-interface-macvlan.cfg, dst: output.network }
- {src: coreos-network-macvlan.cfg, dst: macvlan.network }
notify: Macvlan | restart network
when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
- name: Macvlan | Install cni definition for Macvlan
template:
src: 10-macvlan.conf.j2
dest: /etc/cni/net.d/10-macvlan.conf
- name: Macvlan | Install loopback definition for Macvlan
template:
src: 99-loopback.conf.j2
dest: /etc/cni/net.d/99-loopback.conf
- name: Enable net.ipv4.conf.all.arp_notify in sysctl
sysctl:
name: net.ipv4.conf.all.arp_notify
value: 1
sysctl_set: yes
sysctl_file: "{{ sysctl_file_path }}"
state: present
reload: yes

View File

@ -0,0 +1,15 @@
{
"cniVersion": "0.3.0",
"name": "mynet",
"type": "macvlan",
"master": "{{ macvlan_interface }}",
"hairpinMode": true,
"ipam": {
"type": "host-local",
"subnet": "{{ node_pod_cidr }}",
"routes": [
{ "dst": "0.0.0.0/0" }
],
"gateway": "{{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
}
}

View File

@ -0,0 +1,5 @@
{
"cniVersion": "0.2.0",
"name": "lo",
"type": "loopback"
}

View File

@ -0,0 +1,14 @@
DEVICE=mac0
DEVICETYPE=macvlan
TYPE=macvlan
BOOTPROTO=none
ONBOOT=yes
NM_CONTROLLED=no
MACVLAN_PARENT=eth2
MACVLAN_MODE=bridge
IPADDR={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}
NETMASK={{ node_pod_cidr|ipaddr('netmask') }}
NETWORK={{ node_pod_cidr|ipaddr('network') }}

View File

@ -0,0 +1,4 @@
{% if enable_nat_default_gateway %}
iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
{% endif %}

View File

@ -0,0 +1,4 @@
{% if enable_nat_default_gateway %}
iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
{% endif %}

View File

@ -0,0 +1,7 @@
{% for host in groups['kube-node'] %}
{% if hostvars[host]['access_ip'] is defined %}
{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}
{{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
{% endif %}
{% endif %}
{% endfor %}

View File

@ -0,0 +1,6 @@
[NetDev]
Name=mac0
Kind=macvlan
[MACVLAN]
Mode=bridge

View File

@ -0,0 +1,6 @@
[Match]
Name={{ macvlan_interface }}
[Network]
MACVLAN=mac0
DHCP=yes

View File

@ -0,0 +1,18 @@
[Match]
Name=mac0
[Network]
Address={{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}/{{ node_pod_cidr|ipaddr('prefix') }}
{% for host in groups['kube-node'] %}
{% if hostvars[host]['access_ip'] is defined %}
{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}
[Route]
Gateway={{ hostvars[host]['access_ip'] }}
Destination={{ hostvars[host]['node_pod_cidr'] }}
GatewayOnlink=yes
{% endif %}
{% endif %}
{% endfor %}

View File

@ -0,0 +1,6 @@
[Service]
Type=oneshot
ExecStart=/bin/bash -c "iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE"
[Install]
WantedBy=sys-subsystem-net-devices-mac0.device

View File

@ -0,0 +1,27 @@
auto mac0
iface mac0 inet static
address {{ node_pod_cidr|ipaddr('net')|ipaddr(1)|ipaddr('address') }}
network {{ node_pod_cidr|ipaddr('network') }}
netmask {{ node_pod_cidr|ipaddr('netmask') }}
broadcast {{ node_pod_cidr|ipaddr('broadcast') }}
pre-up ip link add link {{ macvlan_interface }} mac0 type macvlan mode bridge
{% for host in groups['kube-node'] %}
{% if hostvars[host]['access_ip'] is defined %}
{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}
post-up ip route add {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
{% endif %}
{% endif %}
{% endfor %}
{% if enable_nat_default_gateway %}
post-up iptables -t nat -I POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
{% endif %}
{% for host in groups['kube-node'] %}
{% if hostvars[host]['access_ip'] is defined %}
{% if hostvars[host]['node_pod_cidr'] != node_pod_cidr %}
post-down ip route del {{ hostvars[host]['node_pod_cidr'] }} via {{ hostvars[host]['access_ip'] }}
{% endif %}
{% endif %}
{% endfor %}
post-down iptables -t nat -D POSTROUTING -s {{ node_pod_cidr|ipaddr('net') }} -o {{ node_default_gateway_interface }} -j MASQUERADE
post-down ip link delete mac0

View File

@ -30,6 +30,11 @@ dependencies:
tags: tags:
- cni - cni
- role: network_plugin/macvlan
when: kube_network_plugin == 'macvlan'
tags:
- macvlan
- role: network_plugin/contiv - role: network_plugin/contiv
when: kube_network_plugin == 'contiv' when: kube_network_plugin == 'contiv'
tags: tags:

View File

@ -0,0 +1,12 @@
---
# Instance settings
cloud_image: debian-9
mode: default
# Kubespray settings
kube_network_plugin: macvlan
deploy_netchecker: true
enable_nodelocaldns: false
dns_min_replicas: 1
kube_proxy_masquerade_all: true
macvlan_interface: "eth0"