Update MetalLB deployment, wait for resource. (#9995)

* Update MetalLB deployment, wait for resource.

Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>

* yml to yaml, add basic test for metallb

Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>

---------

Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>

docs/metallb.md

@@ -26,15 +26,39 @@ By default only the MetalLB BGP speaker is allowed to run on control plane nodes
 ```yaml
 metallb_config:
   controller:
+    nodeselector:
+      kubernetes.io/os: linux
     tolerations:
-      - key: "node-role.kubernetes.io/master"
-        operator: "Equal"
-        value: ""
-        effect: "NoSchedule"
-      - key: "node-role.kubernetes.io/control-plane"
-        operator: "Equal"
-        value: ""
-        effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/master"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+```
+
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:
+
+```yaml
+metallb_config:
+  controller:
+    nodeselector:
+      kubernetes.io/os: linux
+    tolerations:
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
+  speaker:
+    nodeselector:
+      kubernetes.io/os: linux
+    tolerations:
+    - key: "node-role.kubernetes.io/control-plane"
+      operator: "Equal"
+      value: ""
+      effect: "NoSchedule"
 ```
 
 ## Pools
@@ -137,7 +161,6 @@ In this scenario you should disable the MetalLB speaker and configure the `calic
 
 ```yaml
 metallb_speaker_enabled: false
-metallb_avoid_buggy_ips: true
 metallb_config:
   address_pools:
     primary:
@@ -177,22 +200,22 @@ metallb_config:
       vpn-only: "1234:1"
       NO_ADVERTISE: "65535:65282"
     metallb_peers:
-        peer1:
-          peer_address: 10.6.0.1
-          peer_asn: 64512
-          my_asn: 4200000000
-          communities:
-            - vpn-only
-          address_pool:
-            - pool1
-        peer2:
-          peer_address: 10.10.0.1
-          peer_asn: 64513
-          my_asn: 4200000000
-          communities:
-            - NO_ADVERTISE
-          address_pool:
-            - pool2
+      peer1:
+        peer_address: 10.6.0.1
+        peer_asn: 64512
+        my_asn: 4200000000
+        communities:
+          - vpn-only
+        address_pool:
+          - pool1
+      peer2:
+        peer_address: 10.10.0.1
+        peer_asn: 64513
+        my_asn: 4200000000
+        communities:
+          - NO_ADVERTISE
+        address_pool:
+          - pool2
 calico_advertise_service_loadbalancer_ips:
   - 10.5.0.0/16
   - 10.6.0.0/16

roles/kubernetes-apps/metallb/defaults/main.yml

@@ -1,10 +1,8 @@
 ---
 metallb_enabled: false
 metallb_log_level: info
-metallb_protocol: "layer2"
 metallb_port: "7472"
 metallb_memberlist_port: "7946"
-metallb_peers: []
 metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_speaker_nodeselector:
   kubernetes.io/os: "linux"
@@ -18,6 +16,3 @@ metallb_speaker_tolerations:
     key: node-role.kubernetes.io/control-plane
     operator: Exists
 metallb_controller_tolerations: []
-metallb_pool_name: "loadbalanced"
-metallb_auto_assign: true
-metallb_avoid_buggy_ips: false

roles/kubernetes-apps/metallb/tasks/main.yml

@@ -5,13 +5,6 @@
   when:
     - "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"
 
-- name: Kubernetes Apps | Check BGP peers for MetalLB
-  fail:
-    msg: "metallb_peers is mandatory when metallb_protocol is bgp and metallb_speaker_enabled"
-  when:
-    - metallb_config.layer3 is defined and metallb_speaker_enabled
-    - metallb_config.metallb_peers is not defined or not metallb_config.metallb_peers
-
 - name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore
   fail:
     msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead"
@@ -36,46 +29,95 @@
 - name: Kubernetes Apps | Lay Down MetalLB
   become: true
   template:
-    src: "{{ item }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item }}"
+    src: "metallb.yaml.j2"
+    dest: "{{ kube_config_dir }}/metallb.yaml"
     mode: 0644
-  with_items: ["metallb.yml", "pools.yaml", "layer2.yaml", "layer3.yaml"]
-  register: "rendering"
+  register: metallb_rendering
   when:
-    - "inventory_hostname == groups['kube_control_plane'][0]"
-
-- name: Kubernetes Apps | Create MetalLB resources and replace existing
-  k8s:
-    definition: "{{ lookup('template', 'metallb.yaml') }}"
-
-- name: Kubernetes Apps | Wait for MetalLB controller to be running
-  k8s_info:
-    kind: Deployment
-    namespace: metallb-system
-    name: controller
-    wait: True
-    wait_sleep: 10
-    wait_timeout: 360
-    wait_condition:
-      status: "True"
-      type: Available
-  register: result
-  until: result is not failed
+    - inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Kubernetes Apps | Install and configure MetalLB
   kube:
     name: "MetalLB"
     kubectl: "{{ bin_dir }}/kubectl"
-    filename: "{{ kube_config_dir }}/{{ item.item }}"
-    state: "{{ item.changed | ternary('latest','present') }}"
+    filename: "{{ kube_config_dir }}/metallb.yaml"
+    state: "{{ metallb_rendering.changed | ternary('latest','present') }}"
+    wait: true
   become: true
-  with_items: "{{ rendering.results }}"
   when:
-    - "inventory_hostname == groups['kube_control_plane'][0]"
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Wait for MetalLB controller to be running
+  command: "{{ bin_dir }}/kubectl -n metallb-system wait --for=condition=ready pod -l app=metallb,component=controller"
+  become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: MetalLB | Address pools
+  block:
+    - name: MetalLB | Layout address pools template
+      ansible.builtin.template:
+        src: pools.yaml.j2
+        dest: "{{ kube_config_dir }}/pools.yaml"
+        mode: 0644
+      register: pools_rendering
+
+    - name: MetalLB | Create address pools configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/pools.yaml"
+        state: "{{ pools_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.address_pools is defined
+
+- name: MetalLB | Layer2
+  block:
+    - name: MetalLB | Layout layer2 template
+      ansible.builtin.template:
+        src: layer2.yaml.j2
+        dest: "{{ kube_config_dir }}/layer2.yaml"
+        mode: 0644
+      register: layer2_rendering
+
+    - name: MetalLB | Create layer2 configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/layer2.yaml"
+        state: "{{ layer2_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.layer2 is defined
+
+- name: MetalLB | Layer3
+  block:
+    - name: MetalLB | Layout layer3 template
+      ansible.builtin.template:
+        src: layer3.yaml.j2
+        dest: "{{ kube_config_dir }}/layer3.yaml"
+        mode: 0644
+      register: layer3_rendering
+
+    - name: MetalLB | Create layer3 configuration
+      kube:
+        name: "MetalLB"
+        kubectl: "{{ bin_dir }}/kubectl"
+        filename: "{{ kube_config_dir }}/layer3.yaml"
+        state: "{{ layer3_rendering.changed | ternary('latest','present') }}"
+      become: true
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - metallb_config.layer3 is defined
+
 
 - name: Kubernetes Apps | Delete MetalLB ConfigMap
-  k8s:
+  kube:
     name: config
-    kind: ConfigMap
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: ConfigMap
     namespace: metallb-system
     state: absent

roles/kubernetes-apps/metallb/templates/metallb.yaml.j2

@@ -1,3 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
+  name: metallb-system
+
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -1703,8 +1713,8 @@ spec:
   template:
     metadata:
       annotations:
-        prometheus.io/port: "{{ metallb_port }}"
-        prometheus.io/scrape: "true"
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
       labels:
         app: metallb
         component: controller
@@ -1719,7 +1729,7 @@ spec:
           value: memberlist
         - name: METALLB_DEPLOYMENT
           value: controller
-        image: {{ metallb_controller_image_repo }}:{{ metallb_version }}
+        image: "{{ metallb_controller_image_repo }}:{{ metallb_version }}"
         livenessProbe:
           failureThreshold: 3
           httpGet:
@@ -1755,14 +1765,15 @@ spec:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
           name: cert
           readOnly: true
-{% if metallb_config.controller.tolerations %}
+{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %}
       tolerations:
         {{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
-{% if metallb_controller_nodeselector %}
-      nodeSelector:
-        {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) }}
 {%- endif %}
+      nodeSelector:
+        {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}}
+        {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %}
+        {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}}
+        {%- endif %}
       securityContext:
         fsGroup: 65534
         runAsNonRoot: true
@@ -1793,8 +1804,8 @@ spec:
   template:
     metadata:
       annotations:
-        prometheus.io/port: "{{ metallb_port }}"
-        prometheus.io/scrape: "true"
+        prometheus.io/port: '{{ metallb_port }}'
+        prometheus.io/scrape: 'true'
       labels:
         app: metallb
         component: speaker
@@ -1823,7 +1834,7 @@ spec:
             secretKeyRef:
               key: secretkey
               name: memberlist
-        image: {{ metallb_speaker_image_repo }}:{{ metallb_version }}
+        image: "{{ metallb_speaker_image_repo }}:{{ metallb_version }}"
         livenessProbe:
           failureThreshold: 3
           httpGet:
@@ -1860,16 +1871,19 @@ spec:
             - ALL
           readOnlyRootFilesystem: true
       hostNetwork: true
-{% if metallb_speaker_nodeselector %}
       nodeSelector:
-        {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) }}
-{%- endif %}
+        {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}}
+        {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %}
+        {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}}
+        {%- endif %}
+
       serviceAccountName: speaker
       terminationGracePeriodSeconds: 2
-{% if metallb_speaker_tolerations %}
       tolerations:
-        {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
+        {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+        {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %}
+        {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+        {% endif %}
 {% endif %}
 
 ---
@@ -2004,7 +2018,7 @@ webhooks:
   clientConfig:
     service:
       name: webhook-service
-  namespace: metallb-system
+      namespace: metallb-system
       path: /validate-metallb-io-v1beta1-l2advertisement
   failurePolicy: Fail
   name: l2advertisementvalidationwebhook.metallb.io

tests/files/packet_centos7-flannel-addons-ha.yml

@@ -49,3 +49,26 @@ kube_vip_enabled: true
 kube_vip_arp_enabled: true
 kube_vip_controlplane_enabled: true
 kube_vip_address: 192.168.1.100
+
+# MetalLB
+metallb_enabled: true
+metallb_speaker_enabled: true
+metallb_config:
+  address_pools:
+    primary:
+      ip_range:
+        - 192.0.1.0-192.0.1.254
+      auto_assign: true
+    pool1:
+      ip_range:
+        - 192.0.2.1-192.0.2.1
+      auto_assign: false
+    pool2:
+      ip_range:
+        - 192.0.2.2-192.0.2.2
+      auto_assign: false
+
+  layer2:
+    - primary
+    - pool1
+    - pool2