From f216302f9561de5a0e154df7dc835fcfa1057930 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 27 Oct 2015 15:42:46 +0100 Subject: [PATCH] Calico is not a network overlay --- README.md | 51 +++++++++++++++++-- cluster.yml | 2 +- environments/dev/group_vars/k8s-cluster.yml | 11 ++-- .../production/group_vars/k8s-cluster.yml | 13 +++-- roles/docker/tasks/configure.yml | 6 +-- roles/docker/tasks/install.yml | 2 +- roles/docker/templates/create_cbr.j2 | 4 +- roles/docker/templates/default-docker.j2 | 2 +- .../systemd-init/kube-proxy.service.j2 | 2 +- roles/kubernetes/node/templates/kubelet.j2 | 4 +- .../systemd-init/kube-proxy.service.j2 | 2 +- .../templates/systemd-init/kubelet.service.j2 | 2 +- .../handlers/main.yml | 0 .../tasks/calico.yml | 2 +- .../tasks/flannel.yml | 0 roles/network_plugin/tasks/main.yml | 13 +++++ .../templates/calico/calico-node.service.j2 | 0 .../templates/calico/network-environment.j2 | 0 .../templates/flannel/flannel-conf.json.j2 | 1 + .../templates/flannel/network-environment.j2 | 0 .../flannel/systemd-docker.service.j2 | 0 .../flannel/systemd-flannel.service.j2 | 0 roles/overlay_network/tasks/main.yml | 13 ----- .../templates/flannel/flannel-conf.json.j2 | 1 - 24 files changed, 86 insertions(+), 45 deletions(-) rename roles/{overlay_network => network_plugin}/handlers/main.yml (100%) rename roles/{overlay_network => network_plugin}/tasks/calico.yml (96%) rename roles/{overlay_network => network_plugin}/tasks/flannel.yml (100%) create mode 100644 roles/network_plugin/tasks/main.yml rename roles/{overlay_network => network_plugin}/templates/calico/calico-node.service.j2 (100%) rename roles/{overlay_network => network_plugin}/templates/calico/network-environment.j2 (100%) create mode 100644 roles/network_plugin/templates/flannel/flannel-conf.json.j2 rename roles/{overlay_network => network_plugin}/templates/flannel/network-environment.j2 (100%) rename roles/{overlay_network => network_plugin}/templates/flannel/systemd-docker.service.j2 (100%) rename roles/{overlay_network => network_plugin}/templates/flannel/systemd-flannel.service.j2 (100%) delete mode 100644 roles/overlay_network/tasks/main.yml delete mode 100644 roles/overlay_network/templates/flannel/flannel-conf.json.j2 diff --git a/README.md b/README.md index 5888390d1..dc52e3643 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ kubernetes-ansible ======== -Install and configure a kubernetes cluster including network overlay and optionnal addons. +Install and configure a kubernetes cluster including network plugin and optionnal addons. Based on [CiscoCloud](https://github.com/CiscoCloud/kubernetes-ansible) work. ### Requirements @@ -32,6 +32,49 @@ Please ensure that you have enough disk space there (about **1G**). ### Variables The main variables to change are located in the directory ```environments/[env_name]/group_vars/k8s-cluster.yml```. +### Inventory +Below is an example of an inventory. +Note : The bgp vars (local_as, peers) are not mandatory if the var "peer_with_router" is set to false +``` +[downloader] +10.99.0.26 + +[kube-master] +# NB : the br_addr must be in the {{ calico_pool }} subnet +# it will assign a /24 subnet per node +10.99.0.26 br_addr=10.99.64.1 + +[etcd] +10.99.0.26 + +[kube-node] +10.99.0.4 +10.99.0.5 +10.99.0.6 +10.99.0.36 +10.99.0.37 + +[itx2] +10.99.0.26 br_addr=10.99.16.1 +10.99.0.4 br_addr=10.99.65.1 local_as=xxxxxxxx +10.99.0.5 br_addr=10.99.66.1 local_as=xxxxxxxx +10.99.0.6 br_addr=10.99.69.1 local_as=xxxxxxxx + +[rmv] +10.99.0.36 br_addr=10.99.67.1 local_as=xxxxxxxx +10.99.0.37 br_addr=10.99.68.1 local_as=xxxxxxxx + +[k8s-cluster:children] +kube-node +kube-master + +[itx2:vars] +peers=[{"router_id": "10.99.0.2", "as": "65xxx"}, {"router_id": "10.99.0.3", "as": "65xxx"}] + +[rmv:vars] +peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "as": "65xxx"}] +``` + ### Playbook ``` --- @@ -44,7 +87,7 @@ The main variables to change are located in the directory ```environments/[env_n roles: - { role: etcd, tags: etcd } - { role: docker, tags: docker } - - { role: overlay_network, tags: ['calico', 'flannel', 'network'] } + - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } - { role: dnsmasq, tags: dnsmasq } - hosts: kube-master @@ -72,13 +115,13 @@ Kubernetes ------------------------- ### Network Overlay -You can choose between 2 network overlays. Only one must be chosen. +You can choose between 2 network plugins. Only one must be chosen. * **flannel**: gre/vxlan (layer 2) networking. ([official docs]('https://github.com/coreos/flannel')) * **calico**: bgp (layer 3) networking. ([official docs]('http://docs.projectcalico.org/en/0.13/')) -The choice is defined with the variable '**overlay_network_plugin**' +The choice is defined with the variable '**kube_network_plugin**' ### Expose a service There are several loadbalancing solutions. diff --git a/cluster.yml b/cluster.yml index 178459ba6..421f642de 100644 --- a/cluster.yml +++ b/cluster.yml @@ -8,7 +8,7 @@ roles: - { role: etcd, tags: etcd } - { role: docker, tags: docker } - - { role: overlay_network, tags: ['calico', 'flannel', 'network'] } + - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } - { role: dnsmasq, tags: dnsmasq } - hosts: kube-master diff --git a/environments/dev/group_vars/k8s-cluster.yml b/environments/dev/group_vars/k8s-cluster.yml index 61247c5a8..a88e0402a 100644 --- a/environments/dev/group_vars/k8s-cluster.yml +++ b/environments/dev/group_vars/k8s-cluster.yml @@ -11,7 +11,7 @@ kube_users: cluster_name: cluster.local # # set this variable to calico if needed. keep it empty if flannel is used -overlay_network_plugin: calico +kube_network_plugin: calico # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 @@ -19,17 +19,16 @@ kube_service_addresses: 10.233.0.0/18 # internal network. When used, it will assign IP # addresses from this range to individual pods. # This network must be unused in your network infrastructure! -overlay_network_subnet: 10.233.64.0/18 +kube_pods_subnet: 10.233.64.0/18 # internal network total size (optional). This is the prefix of the -# entire overlay network. So the entirety of 4.0.0.0/16 must be -# unused in your environment. -# overlay_network_prefix: 18 +# entire network. Must be unused in your environment. +# kube_network_prefix: 18 # internal network node size allocation (optional). This is the size allocated # to each node on your network. With these defaults you should have # room for 4096 nodes with 254 pods per node. -overlay_network_host_prefix: 24 +kube_network_node_prefix: 24 # With calico it is possible to distributed routes with border routers of the datacenter. peer_with_router: false diff --git a/environments/production/group_vars/k8s-cluster.yml b/environments/production/group_vars/k8s-cluster.yml index 95c89cead..222f7613c 100644 --- a/environments/production/group_vars/k8s-cluster.yml +++ b/environments/production/group_vars/k8s-cluster.yml @@ -9,9 +9,9 @@ # Kubernetes cluster name, also will be used as DNS domain # cluster_name: cluster.local - # + # set this variable to calico if needed. keep it empty if flannel is used -# overlay_network_plugin: calico +# kube_network_plugin: calico # Kubernetes internal network for services, unused block of space. # kube_service_addresses: 10.233.0.0/18 @@ -19,17 +19,16 @@ # internal network. When used, it will assign IP # addresses from this range to individual pods. # This network must be unused in your network infrastructure! -# overlay_network_subnet: 10.233.64.0/18 +# kube_pods_subnet: 10.233.64.0/18 # internal network total size (optional). This is the prefix of the -# entire overlay network. So the entirety of 4.0.0.0/16 must be -# unused in your environment. -# overlay_network_prefix: 18 +# entire network. Must be unused in your environment. +# kube_network_prefix: 18 # internal network node size allocation (optional). This is the size allocated # to each node on your network. With these defaults you should have # room for 4096 nodes with 254 pods per node. -# overlay_network_host_prefix: 24 +# kube_network_node_prefix: 24 # With calico it is possible to distributed routes with border routers of the datacenter. # peer_with_router: false diff --git a/roles/docker/tasks/configure.yml b/roles/docker/tasks/configure.yml index c095d5572..bf4e6df7e 100644 --- a/roles/docker/tasks/configure.yml +++ b/roles/docker/tasks/configure.yml @@ -1,11 +1,11 @@ --- - name: Write script for calico/docker bridge configuration template: src=create_cbr.j2 dest=/etc/network/if-up.d/create_cbr mode=u+x - when: overlay_network_plugin is defined and overlay_network_plugin == "calico" + when: kube_network_plugin is defined and kube_network_plugin == "calico" - name: Configure calico/docker bridge shell: /etc/network/if-up.d/create_cbr - when: overlay_network_plugin is defined and overlay_network_plugin == "calico" + when: kube_network_plugin is defined and kube_network_plugin == "calico" - name: Configure docker to use cbr0 bridge lineinfile: @@ -14,7 +14,7 @@ line='DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"' notify: - restart docker - when: overlay_network_plugin is defined and overlay_network_plugin == "calico" + when: kube_network_plugin is defined and kube_network_plugin == "calico" - name: enable docker service: diff --git a/roles/docker/tasks/install.yml b/roles/docker/tasks/install.yml index 4b6d15a46..9e551b504 100644 --- a/roles/docker/tasks/install.yml +++ b/roles/docker/tasks/install.yml @@ -13,7 +13,7 @@ with_items: - aufs-tools - cgroupfs-mount - - docker-engine=1.8.2-0~{{ ansible_distribution_release }} + - docker-engine=1.8.3-0~{{ ansible_distribution_release }} - name: Copy default docker configuration template: src=default-docker.j2 dest=/etc/default/docker diff --git a/roles/docker/templates/create_cbr.j2 b/roles/docker/templates/create_cbr.j2 index 86974aaa1..24bf172e6 100644 --- a/roles/docker/templates/create_cbr.j2 +++ b/roles/docker/templates/create_cbr.j2 @@ -9,6 +9,6 @@ fi # Configure calico bridge ip br_ips=$(ip addr list cbr0 |grep "inet " |cut -d' ' -f6) -if ! [[ "${br_ips}" =~ "{{ br_addr }}/{{ overlay_network_host_prefix }}" ]];then - ip a add {{ br_addr }}/{{ overlay_network_host_prefix }} dev cbr0 +if ! [[ "${br_ips}" =~ "{{ br_addr }}/{{ kube_network_node_prefix }}" ]];then + ip a add {{ br_addr }}/{{ kube_network_node_prefix }} dev cbr0 fi diff --git a/roles/docker/templates/default-docker.j2 b/roles/docker/templates/default-docker.j2 index bd71bd746..66e3cd2ee 100644 --- a/roles/docker/templates/default-docker.j2 +++ b/roles/docker/templates/default-docker.j2 @@ -4,7 +4,7 @@ #DOCKER="/usr/local/bin/docker" # Use DOCKER_OPTS to modify the daemon startup options. -{% if overlay_network_plugin is defined and overlay_network_plugin == "calico" %} +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false" {% endif %} diff --git a/roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2 b/roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2 index 55e3e6195..cc3fdc77c 100644 --- a/roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2 +++ b/roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2 @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes -{% if overlay_network_plugin|default('') %} +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} After=docker.service calico-node.service {% else %} After=docker.service diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index d464008fc..0a516b5cc 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -16,6 +16,6 @@ Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns {% else %} Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" {% endif %} -{% if overlay_network_plugin|default('') %} -Environment="KUBELET_NETWORK_PLUGIN=--network_plugin={{ overlay_network_plugin }}" +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} +Environment="KUBELET_NETWORK_PLUGIN=--network_plugin={{ kube_network_plugin }}" {% endif %} diff --git a/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 b/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 index 55e3e6195..cc3fdc77c 100644 --- a/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 +++ b/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes -{% if overlay_network_plugin|default('') %} +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} After=docker.service calico-node.service {% else %} After=docker.service diff --git a/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 b/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 index 8fb5fc89a..5b309c793 100644 --- a/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 @@ -1,7 +1,7 @@ [Unit] Description=Kubernetes Kubelet Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes -{% if overlay_network_plugin|default('') %} +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} After=docker.service calico-node.service {% else %} After=docker.service diff --git a/roles/overlay_network/handlers/main.yml b/roles/network_plugin/handlers/main.yml similarity index 100% rename from roles/overlay_network/handlers/main.yml rename to roles/network_plugin/handlers/main.yml diff --git a/roles/overlay_network/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml similarity index 96% rename from roles/overlay_network/tasks/calico.yml rename to roles/network_plugin/tasks/calico.yml index f09526fb0..e025e04a8 100644 --- a/roles/overlay_network/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -31,7 +31,7 @@ run_once: true - name: Calico | Configure calico-node desired pool - shell: calicoctl pool add {{ overlay_network_subnet }} + shell: calicoctl pool add {{ kube_pods_subnet }} environment: ETCD_AUTHORITY: "{{ groups['kube-master'][0] }}:4001" run_once: true diff --git a/roles/overlay_network/tasks/flannel.yml b/roles/network_plugin/tasks/flannel.yml similarity index 100% rename from roles/overlay_network/tasks/flannel.yml rename to roles/network_plugin/tasks/flannel.yml diff --git a/roles/network_plugin/tasks/main.yml b/roles/network_plugin/tasks/main.yml new file mode 100644 index 000000000..e64e58f12 --- /dev/null +++ b/roles/network_plugin/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: "Test if network plugin is defined" + fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)" + when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or + kube_network_plugin is not defined + +- include: flannel.yml + when: kube_network_plugin == "flannel" +- include: calico.yml + when: kube_network_plugin == "calico" + +- meta: flush_handlers + diff --git a/roles/overlay_network/templates/calico/calico-node.service.j2 b/roles/network_plugin/templates/calico/calico-node.service.j2 similarity index 100% rename from roles/overlay_network/templates/calico/calico-node.service.j2 rename to roles/network_plugin/templates/calico/calico-node.service.j2 diff --git a/roles/overlay_network/templates/calico/network-environment.j2 b/roles/network_plugin/templates/calico/network-environment.j2 similarity index 100% rename from roles/overlay_network/templates/calico/network-environment.j2 rename to roles/network_plugin/templates/calico/network-environment.j2 diff --git a/roles/network_plugin/templates/flannel/flannel-conf.json.j2 b/roles/network_plugin/templates/flannel/flannel-conf.json.j2 new file mode 100644 index 000000000..6c64556ee --- /dev/null +++ b/roles/network_plugin/templates/flannel/flannel-conf.json.j2 @@ -0,0 +1 @@ +{ "Network": "{{ kube_service_addresses }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "vxlan" } } diff --git a/roles/overlay_network/templates/flannel/network-environment.j2 b/roles/network_plugin/templates/flannel/network-environment.j2 similarity index 100% rename from roles/overlay_network/templates/flannel/network-environment.j2 rename to roles/network_plugin/templates/flannel/network-environment.j2 diff --git a/roles/overlay_network/templates/flannel/systemd-docker.service.j2 b/roles/network_plugin/templates/flannel/systemd-docker.service.j2 similarity index 100% rename from roles/overlay_network/templates/flannel/systemd-docker.service.j2 rename to roles/network_plugin/templates/flannel/systemd-docker.service.j2 diff --git a/roles/overlay_network/templates/flannel/systemd-flannel.service.j2 b/roles/network_plugin/templates/flannel/systemd-flannel.service.j2 similarity index 100% rename from roles/overlay_network/templates/flannel/systemd-flannel.service.j2 rename to roles/network_plugin/templates/flannel/systemd-flannel.service.j2 diff --git a/roles/overlay_network/tasks/main.yml b/roles/overlay_network/tasks/main.yml deleted file mode 100644 index 47a5d8b30..000000000 --- a/roles/overlay_network/tasks/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -- name: "Test if overlay network is defined" - fail: msg="ERROR, One overlay_network variable must be defined (Flannel or Calico)" - when: ( overlay_network_plugin is defined and overlay_network_plugin == "calico" and overlay_network_plugin == "flannel" ) or - overlay_network_plugin is not defined - -- include: flannel.yml - when: overlay_network_plugin == "flannel" -- include: calico.yml - when: overlay_network_plugin == "calico" - -- meta: flush_handlers - diff --git a/roles/overlay_network/templates/flannel/flannel-conf.json.j2 b/roles/overlay_network/templates/flannel/flannel-conf.json.j2 deleted file mode 100644 index e14c4a945..000000000 --- a/roles/overlay_network/templates/flannel/flannel-conf.json.j2 +++ /dev/null @@ -1 +0,0 @@ -{ "Network": "{{ kube_service_addresses }}", "SubnetLen": {{ overlay_network_host_prefix }}, "Backend": { "Type": "vxlan" } }