diff --git a/docs/flannel.md b/docs/flannel.md index 307eab56c..06351538d 100644 --- a/docs/flannel.md +++ b/docs/flannel.md @@ -23,13 +23,6 @@ ip a show dev flannel.1 valid_lft forever preferred_lft forever ``` -* Docker must be configured with a bridge ip in the flannel subnet. - -``` -ps aux | grep docker -root 20196 1.7 2.7 1260616 56840 ? Ssl 10:18 0:07 /usr/bin/docker daemon --bip=10.233.16.1/24 --mtu=1450 -``` - * Try to run a container and check its ip address ``` diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2 index 3f54c853d..c70f3d89f 100644 --- a/roles/docker/templates/docker-options.conf.j2 +++ b/roles/docker/templates/docker-options.conf.j2 @@ -1,3 +1,3 @@ [Service] Environment="DOCKER_OPTS={{ docker_options | default('') }} \ ---iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}" +--iptables=false" diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 9b78044e1..21599d986 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -27,7 +27,8 @@ calico_ctl_version: "v1.4.0" calico_cni_version: "v1.10.0" calico_policy_version: "v0.7.0" weave_version: 2.0.1 -flannel_version: v0.8.0 +flannel_version: "v0.8.0" +flannel_cni_version: "v0.2.0" pod_infra_version: 3.0 # Download URL's @@ -43,6 +44,8 @@ etcd_image_repo: "quay.io/coreos/etcd" etcd_image_tag: "{{ etcd_version }}" flannel_image_repo: "quay.io/coreos/flannel" flannel_image_tag: "{{ flannel_version }}" +flannel_cni_image_repo: "quay.io/coreos/flannel-cni" +flannel_cni_image_tag: "{{ flannel_cni_version }}" calicoctl_image_repo: "quay.io/calico/ctl" calicoctl_image_tag: "{{ calico_ctl_version }}" calico_node_image_repo: "quay.io/calico/node" @@ -138,6 +141,12 @@ downloads: tag: "{{ flannel_image_tag }}" sha256: "{{ flannel_digest_checksum|default(None) }}" enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}" + flannel_cni: + container: true + repo: "{{ flannel_cni_image_repo }}" + tag: "{{ flannel_cni_image_tag }}" + sha256: "{{ flannel_cni_digest_checksum|default(None) }}" + enabled: "{{ kube_network_plugin == 'flannel' }}" calicoctl: container: true repo: "{{ calicoctl_image_repo }}" diff --git a/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml b/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml new file mode 100644 index 000000000..cfe931375 --- /dev/null +++ b/roles/kubernetes-apps/network_plugin/flannel/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: "Flannel | Create ServiceAccount ClusterRole and ClusterRoleBinding" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-flannel-rbac.yml" + run_once: true + when: rbac_enabled and flannel_rbac_manifest.changed + +- name: Flannel | Start Resources + kube: + name: "kube-flannel" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/cni-flannel.yml" + resource: "ds" + namespace: "{{system_namespace}}" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ flannel_manifest.changed }}" + when: inventory_hostname == groups['kube-master'][0] + +- name: Flannel | Wait for flannel subnet.env file presence + wait_for: + path: /run/flannel/subnet.env + delay: 5 + timeout: 600 \ No newline at end of file diff --git a/roles/kubernetes-apps/network_plugin/meta/main.yml b/roles/kubernetes-apps/network_plugin/meta/main.yml index 4559d25c6..18c786c1d 100644 --- a/roles/kubernetes-apps/network_plugin/meta/main.yml +++ b/roles/kubernetes-apps/network_plugin/meta/main.yml @@ -3,6 +3,9 @@ dependencies: - role: kubernetes-apps/network_plugin/canal when: kube_network_plugin == 'canal' tags: canal + - role: kubernetes-apps/network_plugin/flannel + when: kube_network_plugin == 'flannel' + tags: flannel - role: kubernetes-apps/network_plugin/weave when: kube_network_plugin == 'weave' tags: weave diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 406994286..44a1c253c 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -45,9 +45,12 @@ spec: - --cloud-provider={{cloud_provider}} {% endif %} {% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} - - --allocate-node-cidrs=true - --configure-cloud-routes=true +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel"] %} + - --allocate-node-cidrs=true - --cluster-cidr={{ kube_pods_subnet }} + - --service-cluster-ip-range={{ kube_service_addresses }} {% endif %} {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index ce83dea48..14d7e0ddd 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -56,7 +56,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock" diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml index 360756f0a..08f4ac145 100644 --- a/roles/network_plugin/flannel/defaults/main.yml +++ b/roles/network_plugin/flannel/defaults/main.yml @@ -1,7 +1,8 @@ --- # Flannel public IP # The address that flannel should advertise as how to access the system -flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" +# Disabled until https://github.com/coreos/flannel/issues/712 is fixed +# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" ## interface that should be used for flannel operations ## This is actually an inventory node-level item @@ -17,5 +18,5 @@ flannel_cpu_limit: 300m flannel_memory_requests: 64M flannel_cpu_requests: 150m -flannel_cert_dir: /etc/flannel/certs -etcd_cert_dir: /etc/ssl/etcd/ssl +# Legacy directory, will be removed if found. +flannel_cert_dir: /etc/flannel/certs \ No newline at end of file diff --git a/roles/network_plugin/flannel/handlers/main.yml b/roles/network_plugin/flannel/handlers/main.yml index 3726c900e..00c5667b0 100644 --- a/roles/network_plugin/flannel/handlers/main.yml +++ b/roles/network_plugin/flannel/handlers/main.yml @@ -4,6 +4,10 @@ failed_when: false notify: Flannel | restart docker +- name: Flannel | delete flannel interface + command: ip link delete flannel.1 + failed_when: false + # special cases for atomic because it defaults to live-restore: true # So we disable live-restore to pickup the new flannel IP. After # we enable it, we have to restart docker again to pickup the new diff --git a/roles/network_plugin/flannel/meta/main.yml b/roles/network_plugin/flannel/meta/main.yml index 4a2caef72..791209357 100644 --- a/roles/network_plugin/flannel/meta/main.yml +++ b/roles/network_plugin/flannel/meta/main.yml @@ -3,3 +3,6 @@ dependencies: - role: download file: "{{ downloads.flannel }}" tags: download + - role: download + file: "{{ downloads.flannel_cni }}" + tags: download diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml index 573b51f19..037684fa4 100644 --- a/roles/network_plugin/flannel/tasks/main.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -1,83 +1,47 @@ --- -- name: Flannel | Set Flannel etcd configuration - command: |- - {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ - set /{{ cluster_name }}/network/config \ - '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }' - delegate_to: "{{groups['etcd'][0]}}" - run_once: true +- include: pre-upgrade.yml -- name: Flannel | Create flannel certs directory - file: - dest: "{{ flannel_cert_dir }}" - state: directory - mode: 0750 - owner: root - group: root +- name: Flannel | Verify if br_netfilter module exists + shell: "modinfo br_netfilter" + register: modinfo_br_netfilter + failed_when: modinfo_br_netfilter.rc not in [0, 1] + changed_when: false -- name: Flannel | Link etcd certificates for flanneld - file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" - dest: "{{ flannel_cert_dir }}/{{ item.d }}" - state: hard - force: yes +- name: Flannel | Enable br_netfilter module + modprobe: + name: br_netfilter + state: present + when: modinfo_br_netfilter.rc == 0 + +# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found if br_netfilter is not a module +- name: Flannel | Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + register: sysctl_bridge_nf_call_iptables + +- name: Flannel | Enable bridge-nf-call tables + sysctl: + name: "{{ item }}" + state: present + value: 1 + reload: yes + when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0 with_items: - - {s: "ca.pem", d: "ca_cert.crt"} - - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} - - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-arptables + - net.bridge.bridge-nf-call-ip6tables -- name: Flannel | Create flannel pod manifest +- name: Flannel | Create cni-flannel-rbac manifest template: - src: flannel-pod.yml - dest: "{{kube_manifest_dir}}/flannel-pod.manifest" - notify: Flannel | delete default docker bridge + src: cni-flannel-rbac.yml.j2 + dest: "{{ kube_config_dir }}/cni-flannel-rbac.yml" + register: flannel_rbac_manifest + when: inventory_hostname == groups['kube-master'][0] and rbac_enabled -- name: Flannel | Wait for flannel subnet.env file presence - wait_for: - path: /run/flannel/subnet.env - delay: 5 - timeout: 600 - -- name: Flannel | Get flannel_subnet from subnet.env - shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}' - register: flannel_subnet_output - changed_when: false - check_mode: no - -- set_fact: - flannel_subnet: "{{ flannel_subnet_output.stdout }}" - -- name: Flannel | Get flannel_mtu from subnet.env - shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_MTU" {print $2}' - register: flannel_mtu_output - changed_when: false - check_mode: no - -- set_fact: - flannel_mtu: "{{ flannel_mtu_output.stdout }}" - -- set_fact: - docker_options_file: >- - {%- if ansible_os_family == "Debian" -%}/etc/default/docker{%- elif ansible_os_family == "RedHat" -%}/etc/sysconfig/docker{%- endif -%} - tags: facts - -- set_fact: - docker_options_name: >- - {%- if ansible_os_family == "Debian" -%}DOCKER_OPTS{%- elif ansible_os_family == "RedHat" -%}other_args{%- endif -%} - tags: facts - -- set_fact: - docker_network_options: '"--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"' - -- name: Flannel | Ensure path for docker network systemd drop-in - file: - path: "/etc/systemd/system/docker.service.d" - state: directory - owner: root - -- name: Flannel | Create docker network systemd drop-in +- name: Flannel | Create cni-flannel manifest template: - src: flannel-options.conf.j2 - dest: "/etc/systemd/system/docker.service.d/flannel-options.conf" - notify: - - Flannel | restart docker + src: cni-flannel.yml.j2 + dest: "{{ kube_config_dir }}/cni-flannel.yml" + register: flannel_manifest + when: inventory_hostname == groups['kube-master'][0] \ No newline at end of file diff --git a/roles/network_plugin/flannel/tasks/pre-upgrade.yml b/roles/network_plugin/flannel/tasks/pre-upgrade.yml new file mode 100644 index 000000000..6b6fcd54f --- /dev/null +++ b/roles/network_plugin/flannel/tasks/pre-upgrade.yml @@ -0,0 +1,19 @@ +--- +- name: Flannel pre-upgrade | Purge legacy flannel systemd unit file + file: + path: "/etc/systemd/system/docker.service.d/flannel-options.conf" + state: absent + notify: + - Flannel | delete default docker bridge + +- name: Flannel pre-upgrade | Purge legacy Flannel static pod manifest + file: + path: "{{ kube_manifest_dir }}/flannel-pod.manifest" + state: absent + notify: + - Flannel | delete flannel interface + +- name: Flannel pre-upgrade | Remove Flannel's certificate directory not required by CNI + file: + dest: "{{ flannel_cert_dir }}" + state: absent \ No newline at end of file diff --git a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 new file mode 100644 index 000000000..aafe2a0f5 --- /dev/null +++ b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: "{{system_namespace}}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: flannel + namespace: "{{system_namespace}}" \ No newline at end of file diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 new file mode 100644 index 000000000..0012228d7 --- /dev/null +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -0,0 +1,125 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: "{{system_namespace}}" + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name":"cbr0", + "cniVersion":"0.3.1", + "plugins":[ + { + "type":"flannel", + "delegate":{ + "forceAddress":true, + "isDefaultGateway":true + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + } + } + ] + } + net-conf.json: | + { + "Network": "{{ kube_pods_subnet }}", + "Backend": { + "Type": "{{ flannel_backend_type }}" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel + namespace: "{{system_namespace}}" + labels: + tier: node + k8s-app: flannel +spec: + template: + metadata: + labels: + tier: node + k8s-app: flannel + spec: +{% if rbac_enabled %} + serviceAccountName: flannel +{% endif %} + containers: + - name: kube-flannel + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ flannel_cpu_limit }} + memory: {{ flannel_memory_limit }} + requests: + cpu: {{ flannel_cpu_requests }} + memory: {{ flannel_memory_requests }} + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %} ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: install-cni + image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: kube-flannel-cfg + key: cni-conf.json + - name: CNI_CONF_NAME + value: "10-flannel.conflist" + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + - name: host-cni-bin + mountPath: /host/opt/cni/bin/ + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg + - name: host-cni-bin + hostPath: + path: /opt/cni/bin + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate \ No newline at end of file diff --git a/roles/network_plugin/flannel/templates/flannel-options.conf.j2 b/roles/network_plugin/flannel/templates/flannel-options.conf.j2 deleted file mode 100644 index 9ee22b4bc..000000000 --- a/roles/network_plugin/flannel/templates/flannel-options.conf.j2 +++ /dev/null @@ -1,6 +0,0 @@ -[Service] -{% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %} -Environment="DOCKER_OPT_BIP=--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}" -{% else %} -Environment="DOCKER_NETWORK_OPTIONS=--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}" -{% endif %} diff --git a/roles/network_plugin/flannel/templates/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml deleted file mode 100644 index a6e075b8c..000000000 --- a/roles/network_plugin/flannel/templates/flannel-pod.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -kind: "Pod" -apiVersion: "v1" -metadata: - name: "flannel" - namespace: "{{system_namespace}}" - labels: - app: "flannel" - version: "v0.1" -spec: - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - name: "subnetenv" - hostPath: - path: "/run/flannel" - - name: "etcd-certs" - hostPath: - path: "{{ flannel_cert_dir }}" - containers: - - name: "flannel-container" - image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" - imagePullPolicy: {{ k8s_image_pull_policy }} - resources: - limits: - cpu: {{ flannel_cpu_limit }} - memory: {{ flannel_memory_limit }} - requests: - cpu: {{ flannel_cpu_requests }} - memory: {{ flannel_memory_requests }} - command: - - "/bin/sh" - - "-c" - - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ flannel_cert_dir }}/ca_cert.crt -etcd-certfile {{ flannel_cert_dir }}/cert.crt -etcd-keyfile {{ flannel_cert_dir }}/key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" - ports: - - hostPort: 10253 - containerPort: 10253 - volumeMounts: - - name: "subnetenv" - mountPath: "/run/flannel" - - name: "etcd-certs" - mountPath: "{{ flannel_cert_dir }}" - readOnly: true - securityContext: - privileged: true - hostNetwork: true