Add metrics_server_resizer option (#8018) (#8031)

The addon-resizer container can reduce resource limits of cpu and
memory of metrics-server container in the pod, and that caused
OOMKilled.
In addition, the original metrics-server manifest doesn't contain
the addon-resizer container as [1].
So this adds metrics_server_resizer option to control the addon-resizer
container deployment and the default value is false to make it stable
for most environments.

This is a cherry-pick of 8d3961edbe

[1]: 527679e5e8/manifests/base/deployment.yaml

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -14,6 +14,7 @@ registry_enabled: false
 
 # Metrics Server deployment
 metrics_server_enabled: false
+# metrics_server_resizer: false
 # metrics_server_kubelet_insecure_tls: true
 # metrics_server_metric_resolution: 15s
 # metrics_server_kubelet_preferred_address_types: "InternalIP"

roles/kubernetes-apps/metrics_server/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+metrics_server_resizer: false
 metrics_server_kubelet_insecure_tls: true
 metrics_server_kubelet_preferred_address_types: "InternalIP"
 metrics_server_metric_resolution: 15s

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -81,6 +81,7 @@ spec:
           requests:
             cpu: {{ metrics_server_requests_cpu }}
             memory: {{ metrics_server_requests_memory }}
+{% if metrics_server_resizer %}
       - name: metrics-server-nanny
         image: {{ addon_resizer_image_repo }}:{{ addon_resizer_image_tag }}
         imagePullPolicy: {{ k8s_image_pull_policy }}
@@ -118,6 +119,7 @@ spec:
           # Specifies the smallest cluster (defined in number of nodes)
           # resources will be scaled to.
           - --minClusterSize={{ metrics_server_min_cluster_size }}
+{% endif %}
       volumes:
         - name: metrics-server-config-volume
           configMap: