diff --git a/cluster.yml b/cluster.yml index bc0580331..3e8b78115 100644 --- a/cluster.yml +++ b/cluster.yml @@ -19,14 +19,14 @@ - { role: kubespray-defaults} - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} -- hosts: k8s-cluster:etcd:calico-rr +- hosts: k8s-cluster:etcd any_errors_fatal: "{{ any_errors_fatal | default(true) }}" gather_facts: false roles: - { role: kubespray-defaults} - { role: bootstrap-os, tags: bootstrap-os} -- hosts: k8s-cluster:etcd:calico-rr +- hosts: k8s-cluster:etcd any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} @@ -46,7 +46,7 @@ etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" when: not etcd_kubeadm_enabled| default(false) -- hosts: k8s-cluster:calico-rr +- hosts: k8s-cluster any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} @@ -79,6 +79,12 @@ - { role: kubernetes/kubeadm, tags: kubeadm} - { role: network_plugin, tags: network } +- hosts: calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + roles: + - { role: kubespray-defaults} + - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr']} + - hosts: kube-master[0] any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: @@ -95,12 +101,6 @@ - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } -- hosts: calico-rr - any_errors_fatal: "{{ any_errors_fatal | default(true) }}" - roles: - - { role: kubespray-defaults} - - { role: network_plugin/calico/rr, tags: network } - - hosts: kube-master any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: diff --git a/docs/calico.md b/docs/calico.md index 5eedef8ac..52586b18f 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -119,13 +119,13 @@ recommended here: You need to edit your inventory and add: -* `calico-rr` group with nodes in it. At the moment it's incompatible with - `kube-node` due to BGP port conflict with `calico-node` container. So you - should not have nodes in both `calico-rr` and `kube-node` groups. +* `calico-rr` group with nodes in it. `calico-rr` can be combined with + `kube-node` and/or `kube-master`. `calico-rr` group also must be a child + group of `k8s-cluster` group. * `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/)) -Here's an example of Kubespray inventory with route reflectors: +Here's an example of Kubespray inventory with standalone route reflectors: ``` [all] @@ -154,6 +154,7 @@ node5 [k8s-cluster:children] kube-node kube-master +calico-rr [calico-rr] rr0 diff --git a/inventory/local/hosts.ini b/inventory/local/hosts.ini index 425ad23ef..7834d27c0 100644 --- a/inventory/local/hosts.ini +++ b/inventory/local/hosts.ini @@ -12,3 +12,4 @@ node1 [k8s-cluster:children] kube-node kube-master +calico-rr diff --git a/inventory/sample/inventory.ini b/inventory/sample/inventory.ini index 8e32a3a75..e5a53b05a 100644 --- a/inventory/sample/inventory.ini +++ b/inventory/sample/inventory.ini @@ -28,6 +28,9 @@ # node5 # node6 +[calico-rr] + [k8s-cluster:children] kube-master kube-node +calico-rr diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 index 81efb98fc..0110c058c 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -17,3 +17,8 @@ discoveryTokenUnsafeSkipCAVerification: true nodeRegistration: name: {{ kube_override_hostname }} criSocket: {{ cri_socket }} +{% if 'calico-rr' in group_names and 'kube-node' not in group_names %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/calico-rr +{% endif %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 index 66583412b..75f27fad3 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -21,3 +21,8 @@ caCertPath: {{ kube_cert_dir }}/ca.crt nodeRegistration: name: {{ kube_override_hostname }} criSocket: {{ cri_socket }} +{% if 'calico-rr' in group_names and 'kube-node' not in group_names %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/calico-rr +{% endif %} diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 48938d06b..c52d0fa21 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -166,6 +166,26 @@ - inventory_hostname == groups['kube-master'][0] run_once: yes +- name: "Check that cluster_id is set if calico_rr enabled" + assert: + that: + - cluster_id is defined + msg: "A unique cluster_id is required if using calico_rr" + when: + - kube_network_plugin == 'calico' + - peer_with_calico_rr + - inventory_hostname == groups['kube-master'][0] + run_once: yes + +- name: "Check that calico_rr nodes are in k8s-cluster group" + assert: + that: + - '"k8s-cluster" in group_names' + msg: "calico-rr must be a child group of k8s-cluster group" + when: + - kube_network_plugin == 'calico' + - '"calico-rr" in group_names' + - name: "Check that kube_service_addresses is a network range" assert: that: diff --git a/roles/network_plugin/calico/rr/defaults/main.yml b/roles/network_plugin/calico/rr/defaults/main.yml index 4871f34a8..dedda197c 100644 --- a/roles/network_plugin/calico/rr/defaults/main.yml +++ b/roles/network_plugin/calico/rr/defaults/main.yml @@ -2,15 +2,4 @@ # Global as_num (/calico/bgp/v1/global/as_num) # should be the same as in calico role global_as_num: "64512" - -calico_cert_dir: /etc/calico/certs - -# Limits for apps -calico_rr_memory_limit: 1000M -calico_rr_cpu_limit: 300m -calico_rr_memory_requests: 128M -calico_rr_cpu_requests: 150m - -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem +calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" diff --git a/roles/network_plugin/calico/rr/handlers/main.yml b/roles/network_plugin/calico/rr/handlers/main.yml deleted file mode 100644 index 1436649e3..000000000 --- a/roles/network_plugin/calico/rr/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: restart calico-rr - command: /bin/true - notify: - - Calico-rr | reload systemd - - Calico-rr | reload calico-rr - -- name: Calico-rr | reload systemd - systemd: - daemon_reload: true - -- name: Calico-rr | reload calico-rr - service: - name: calico-rr - state: restarted diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 2b023a2aa..0c93a5506 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -1,82 +1,29 @@ --- -# Required from inventory: -# calico_rr_ip - which specific IP to use for RR, defaults to -# "ip" from inventory or "ansible_default_ipv4.address" +- name: Calico-rr | Pre-upgrade tasks + include_tasks: pre.yml -- name: Calico-rr | Set IP fact - set_fact: - rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}" +- name: Calico-rr | Fetch current node object + command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -oyaml" + register: calico_rr_node -- name: Calico-rr | Create calico certs directory - file: - dest: "{{ calico_cert_dir }}" - state: directory - mode: 0750 - owner: root - group: root - -- name: Calico-rr | Link etcd certificates for calico-node - file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" - dest: "{{ calico_cert_dir }}/{{ item.d }}" - state: hard - force: yes - with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} - -- name: Calico-rr | Create dir for logs - file: - path: /var/log/calico-rr - state: directory - mode: 0755 - owner: root - group: root - -- name: Calico-rr | Write calico-rr.env for systemd init file - template: - src: calico-rr.env.j2 - dest: /etc/calico/calico-rr.env - notify: restart calico-rr - -- name: Calico-rr | Write calico-rr systemd init file - template: - src: calico-rr-docker.service.j2 - dest: /etc/systemd/system/calico-rr.service - notify: restart calico-rr - when: - - container_manager in ['crio', 'docker', 'rkt'] - -- name: Calico-rr | Write calico-rr systemd init file - template: - src: calico-rr-containerd.service.j2 - dest: /etc/systemd/system/calico-rr.service - notify: restart calico-rr - when: - - container_manager == 'containerd' +# FIXME(mattymo): Use jsonpatch when ansible/ansible#52931 is merged +- name: Calico-rr | Set route reflector cluster ID + shell: >- + echo -e '{{ calico_rr_node.stdout }}' | + sed '/bgp:/a \ \ \ \ routeReflectorClusterID: {{ cluster_id }}' + register: calico_rr_node + when: '("routeReflectorClusterID: " + cluster_id|string) not in calico_rr_node.stdout_lines' - name: Calico-rr | Configure route reflector - command: |- - {{ bin_dir }}/etcdctl \ - --endpoints={{ etcd_access_addresses }} \ - put /calico/bgp/v1/rr_v4/{{ rr_ip }} \ - '{ - "ip": "{{ rr_ip }}", - "cluster_id": "{{ cluster_id }}" - }' - environment: - ETCDCTL_API: 3 - ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem" - ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem" + shell: |- + echo -e '{{ calico_rr_node.stdout }}' | + {{ bin_dir }}/calicoctl.sh replace -f- retries: 4 delay: "{{ retry_stagger | random + 3 }}" - delegate_to: "{{ groups['etcd'][0] }}" -- meta: flush_handlers - -- name: Calico-rr | Enable calico-rr - service: - name: calico-rr - state: started - enabled: yes +- name: Calico-rr | Set label for route reflector + command: >- + {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} + 'i-am-a-route-reflector=true' --overwrite + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" diff --git a/roles/network_plugin/calico/rr/tasks/pre.yml b/roles/network_plugin/calico/rr/tasks/pre.yml new file mode 100644 index 000000000..d8dbd8072 --- /dev/null +++ b/roles/network_plugin/calico/rr/tasks/pre.yml @@ -0,0 +1,15 @@ +--- +- name: Calico-rr | Disable calico-rr service if it exists + service: + name: calico-rr + state: stopped + enabled: no + failed_when: false + +- name: Calico-rr | Delete obsolete files + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/calico/calico-rr.env + - /etc/systemd/system/calico-rr.service diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 deleted file mode 100644 index db719afd3..000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 +++ /dev/null @@ -1,27 +0,0 @@ -[Unit] -Description=calico-rr -After=containerd.service -Requires=containerd.service - -[Service] -EnvironmentFile=/etc/calico/calico-rr.env -ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr -ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \ - --env IP=${IP} \ - --env IP6=${IP6} \ - --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ - --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \ - --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \ - --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \ - --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \ - --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \ - {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \ - calico-rr - -Restart=always -RestartSec=10s - -ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 deleted file mode 100644 index f6da04a4d..000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 +++ /dev/null @@ -1,28 +0,0 @@ -[Unit] -Description=calico-rr -After=docker.service -Requires=docker.service - -[Service] -EnvironmentFile=/etc/calico/calico-rr.env -ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr -ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ - --name=calico-rr \ - -e IP=${IP} \ - -e IP6=${IP6} \ - -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ - -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \ - -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \ - -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ - -v /var/log/calico-rr:/var/log/calico \ - -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ - --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ - {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} - -Restart=always -RestartSec=10s - -ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 deleted file mode 100644 index 1cdb2659c..000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 +++ /dev/null @@ -1,6 +0,0 @@ -ETCD_ENDPOINTS="{{ etcd_access_addresses }}" -ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt" -ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt" -ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem" -IP="{{ rr_ip }}" -IP6="" diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 7ee560095..c4b03822b 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -163,16 +163,16 @@ - name: Calico | Configure peering with router(s) at global scope shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "global-{{ item.router_id }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "global-{{ item.router_id }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "peerIP": "{{ item.router_id }}" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -181,6 +181,46 @@ - inventory_hostname == groups['kube-master'][0] - peer_with_router|default(false) +- name: Calico | Configure peering with route reflectors at global scope + shell: | + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "peer-to-rrs" + }, + "spec": { + "nodeSelector": "!has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico-rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube-master'][0] + - peer_with_calico_rr|default(false) + +- name: Calico | Configure route reflectors to peer with each other + shell: > + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "rr-mesh" + }, + "spec": { + "nodeSelector": "has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico-rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube-master'][0] + - peer_with_calico_rr|default(false) + - name: Calico | Create calico manifests template: src: "{{ item.file }}.j2" @@ -234,18 +274,18 @@ - name: Calico | Configure node asNumber for per node peering shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "Node", - "metadata": { - "name": "{{ inventory_hostname }}" - }, - "spec": { - "bgp": { - "asNumber": "{{ local_as }}" - }, - "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }} + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "Node", + "metadata": { + "name": "{{ inventory_hostname }}" + }, + "spec": { + "bgp": { + "asNumber": "{{ local_as }}" + }, + "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] + }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }} retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: @@ -256,17 +296,17 @@ - name: Calico | Configure peering with router(s) at node scope shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "{{ inventory_hostname }}-{{ item.router_id }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "node": "{{ inventory_hostname }}", - "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ inventory_hostname }}-{{ item.router_id }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "node": "{{ inventory_hostname }}", + "peerIP": "{{ item.router_id }}" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -274,25 +314,3 @@ when: - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] - -- name: Calico | Configure peering with route reflectors - shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}" - }, - "spec": { - "asNumber": "{{ local_as | default(global_as_num) }}", - "node": "{{ inventory_hostname }}", - "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ groups['calico-rr'] | default([]) }}" - when: - - peer_with_calico_rr|default(false) - - inventory_hostname in groups['k8s-cluster'] - - hostvars[item]['cluster_id'] == cluster_id diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 index a2f301def..82293e0cd 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -50,5 +50,8 @@ instance-1 [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] [fake_hosts] diff --git a/tests/templates/inventory-aws.j2 b/tests/templates/inventory-aws.j2 index ee89bb5a4..92f107f65 100644 --- a/tests/templates/inventory-aws.j2 +++ b/tests/templates/inventory-aws.j2 @@ -22,3 +22,6 @@ node2 [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] diff --git a/tests/templates/inventory-do.j2 b/tests/templates/inventory-do.j2 index 95b6f3027..83a749afc 100644 --- a/tests/templates/inventory-do.j2 +++ b/tests/templates/inventory-do.j2 @@ -43,6 +43,9 @@ {{droplets.results[0].droplet.name}} {% endif %} +[calico-rr] + [k8s-cluster:children] kube-node kube-master +calico-rr diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 896cc157d..503bb4091 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -66,6 +66,9 @@ [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] {% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %} [fake_hosts]