Fix containerd config_path mirrors and remove nerdctl insecure_registry (#10196)

* Fix containerd_registries in config_path for mirrors and remove nerdctl global insecure_registry setting

* Make containerd hosts.toml mode 0640

* Add containerd_registries_mirrors and keep containerd_registries to pass packet_debian11-calico-upgrade

docs/ci-setup.md

@@ -80,10 +80,15 @@ docker_registry_mirrors:
 containerd_grpc_max_recv_message_size: 16777216
 containerd_grpc_max_send_message_size: 16777216
 
-containerd_registries:
-  "docker.io":
-    - "https://mirror.gcr.io"
-    - "https://registry-1.docker.io"
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
 
 containerd_max_container_log_line_size: -1
 

docs/containerd.md

@@ -24,15 +24,20 @@ etcd_deployment_type: host
 Example: define registry mirror for docker hub
 
 ```yaml
-containerd_registries:
-  "docker.io":
-    - "https://mirror.gcr.io"
-    - "https://registry-1.docker.io"
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
 ```
 
-`containerd_registries` is ignored for pulling images when `image_command_tool=nerdctl`
+`containerd_registries_mirrors` is ignored for pulling images when `image_command_tool=nerdctl`
 (the default for `container_manager=containerd`). Use `crictl` instead, it supports
-`containerd_registries` but lacks proper multi-arch support (see
+`containerd_registries_mirrors` but lacks proper multi-arch support (see
 [#8375](https://github.com/kubernetes-sigs/kubespray/issues/8375)):
 
 ```yaml
@@ -103,10 +108,22 @@ containerd_runc_runtime:
 Config insecure-registry access to self hosted registries.
 
 ```yaml
-containerd_insecure_registries:
-  "test.registry.io": "http://test.registry.io"
-  "172.19.16.11:5000": "http://172.19.16.11:5000"
-  "repo:5000": "http://repo:5000"
+containerd_registries_mirrors:
+  - prefix: test.registry.io
+    mirrors:
+      - host: http://test.registry.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: true
+  - prefix: 172.19.16.11:5000
+    mirrors:
+      - host: http://172.19.16.11:5000
+        capabilities: ["pull", "resolve"]
+        skip_verify: true
+  - prefix: repo:5000
+    mirrors:
+      - host: http://repo:5000
+        capabilities: ["pull", "resolve"]
+        skip_verify: true
 ```
 
 [containerd]: https://containerd.io/

docs/offline-environment.md

@@ -51,8 +51,12 @@ containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-l
 runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}"
 nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 # Insecure registries for containerd
-containerd_insecure_registries:
-    "{{ registry_addr }}"："{{ registry_host }}"
+containerd_registries_mirrors:
+  - prefix: "{{ registry_addr }}"
+    mirrors:
+      - host: "{{ registry_host }}"
+        capabilities: ["pull", "resolve"]
+        skip_verify: true
 
 # CentOS/Redhat/AlmaLinux/Rocky Linux
 ## Docker / Containerd

inventory/sample/group_vars/all/containerd.yml

@@ -30,17 +30,13 @@
 
 # containerd_metrics_grpc_histogram: false
 
-## An obvious use case is allowing insecure-registry access to self hosted registries.
-## Can be ipaddress and domain_name.
-## example define mirror.registry.io or 172.19.16.11:5000
-## set "name": "url". insecure url must be started http://
-## Port number is also needed if the default HTTPS port is not used.
-# containerd_insecure_registries:
-#   "localhost": "http://127.0.0.1"
-#   "172.19.16.11:5000": "http://172.19.16.11:5000"
-
-# containerd_registries:
-#   "docker.io": "https://registry-1.docker.io"
+# Registries defined within containerd.
+# containerd_registries_mirrors:
+#  - prefix: docker.io
+#    mirrors:
+#     - host: https://registry-1.docker.io
+#       capabilities: ["pull", "resolve"]
+#       skip_verify: false
 
 # containerd_max_container_log_line_size: -1
 

roles/container-engine/containerd/defaults/main.yml

@@ -50,6 +50,13 @@ containerd_metrics_grpc_histogram: false
 containerd_registries:
   "docker.io": "https://registry-1.docker.io"
 
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+
 containerd_max_container_log_line_size: -1
 
 # If enabled it will allow non root users to use port numbers <1024
@@ -74,7 +81,7 @@ containerd_limit_core: "infinity"
 containerd_limit_open_file_num: "infinity"
 containerd_limit_mem_lock: "infinity"
 
-# If enabled it will use config_path and disable use mirrors config
+# If enabled it will use config_path and config to be put in {{ containerd_cfg_dir }}/certs.d/
 containerd_use_config_path: false
 
 # OS distributions that already support containerd

roles/container-engine/containerd/tasks/main.yml

@@ -112,26 +112,20 @@
   notify: Restart containerd
 
 - name: Containerd | Configure containerd registries
-  when: containerd_use_config_path is defined and containerd_use_config_path | bool and containerd_insecure_registries is defined
+  when: containerd_registries_mirrors is defined
   block:
-    - name: Containerd ｜ Create registry directories
+    - name: Containerd | Create registry directories
       file:
-        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}"
+        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}"
         state: directory
         mode: 0755
-        recurse: true
-      with_dict: "{{ containerd_insecure_registries }}"
-    - name: Containerd ｜ Write hosts.toml file
-      blockinfile:
-        path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}/hosts.toml"
+      loop: "{{ containerd_registries_mirrors }}"
+    - name: Containerd | Write hosts.toml file
+      template:
+        src: hosts.toml.j2
+        dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml"
         mode: 0640
-        create: true
-        block: |
-          server = "{{ item.value }}"
-          [host."{{ item.value }}"]
-            capabilities = ["pull", "resolve", "push"]
-            skip_verify = true
-      with_dict: "{{ containerd_insecure_registries }}"
+      loop: "{{ containerd_registries_mirrors }}"
 
 # you can sometimes end up in a state where everything is installed
 # but containerd was not started / enabled

roles/container-engine/containerd/templates/config.toml.j2

@@ -51,18 +51,18 @@ oom_score = {{ containerd_oom_score }}
       config_path = "{{ containerd_cfg_dir }}/certs.d"
 {% else %}
       [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-{% for registry, addr in containerd_registries.items() %}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
-          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+{% set insecure_registries_addr = [] %}
+{% for registry in containerd_registries_mirrors %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry.prefix }}"]
+{% set endpoint = [] %}
+{% for mirror in registry.mirrors %}
+{% if endpoint.append(mirror.host) %}{% endif %}
+{% if mirror.skip_verify is defined and mirror.skip_verify|bool %}{% if insecure_registries_addr.append(mirror.host | urlsplit('netloc')) %}{% endif %}{% endif %}
 {% endfor %}
-{% if containerd_insecure_registries is defined and containerd_insecure_registries|length>0 %}
-{% for registry, addr in containerd_insecure_registries.items() %}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
-          endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
+          endpoint = ["{{ ( endpoint | unique ) | join('","') }}"]
 {% endfor %}
-{% endif %}
-{% for addr in containerd_insecure_registries.values() | flatten | unique %}
-        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr | urlsplit('netloc') }}".tls]
+{% for addr in insecure_registries_addr | unique %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls]
           insecure_skip_verify = true
 {% endfor %}
 {% endif %}

roles/container-engine/containerd/templates/hosts.toml.j2

@@ -0,0 +1,8 @@
+server = "https://{{ item.prefix }}"
+{% for mirror in item.mirrors %}
+[host."{{ mirror.host }}"]
+  capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"]
+{% if mirror.skip_verify is defined %}
+  skip_verify = {{ mirror.skip_verify | default('false') | string | lower }}
+{% endif %}
+{% endfor %}

roles/container-engine/nerdctl/templates/nerdctl.toml.j2

@@ -6,5 +6,4 @@ snapshotter       = "{{ nerdctl_snapshotter | default('overlayfs') }}"
 cni_path          = "/opt/cni/bin"
 cni_netconfpath   = "/etc/cni/net.d"
 cgroup_manager    = "{{ kubelet_cgroup_driver | default('systemd') }}"
-insecure_registry = {{ (containerd_insecure_registries is defined and containerd_insecure_registries|length>0) | bool | lower }}
-hosts_dir         = ["/etc/containerd/certs.d"]
+hosts_dir         = ["{{ containerd_cfg_dir }}/certs.d"]

roles/download/defaults/main/main.yml

@@ -57,7 +57,7 @@ download_retries: 4
 docker_image_pull_command: "{{ docker_bin_dir }}/docker pull"
 docker_image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \",\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \",\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\n' ','"
 nerdctl_image_info_command: "{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\n' ','"
-nerdctl_image_pull_command: "{{ bin_dir }}/nerdctl -n k8s.io pull --quiet {{ nerdctl_extra_flags }}"
+nerdctl_image_pull_command: "{{ bin_dir }}/nerdctl -n k8s.io pull --quiet"
 crictl_image_info_command: "{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\n' ','"
 crictl_image_pull_command: "{{ bin_dir }}/crictl pull"
 
@@ -72,9 +72,6 @@ image_info_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localh
 # Arch of Docker images and needed packages
 image_arch: "{{ host_architecture | default('amd64') }}"
 
-# Nerdctl insecure flag set
-nerdctl_extra_flags: '{%- if containerd_insecure_registries is defined and containerd_insecure_registries | length > 0 -%}--insecure-registry{%- else -%}{%- endif -%}'
-
 # Versions
 kubeadm_version: "{{ kube_version }}"
 crun_version: 1.8.5

roles/kubespray-defaults/defaults/main.yaml

@@ -356,15 +356,6 @@ docker_plugins: []
 # Containerd options - thse are relevant when container_manager == 'containerd'
 containerd_use_systemd_cgroup: true
 
-## An obvious use case is allowing insecure-registry access to self hosted registries.
-## Can be ipaddress and domain_name.
-## example define mirror.registry.io or 172.19.16.11:5000
-## Port number is also needed if the default HTTPS port is not used.
-# containerd_insecure_registries:
-#   "mirror.registry.io":"http://mirror.registry.io"
-#   "172.19.16.11:5000":"http://172.19.16.11:5000"
-containerd_insecure_registries: {}
-
 # Containerd conf default dir
 containerd_storage_dir: "/var/lib/containerd"
 containerd_state_dir: "/run/containerd"

tests/common/_docker_hub_registry_mirror.yml

@@ -10,6 +10,16 @@ containerd_registries:
     - "https://mirror.gcr.io"
     - "https://registry-1.docker.io"
 
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+      - host: https://registry-1.docker.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+
 containerd_max_container_log_line_size: -1
 
 crio_registries:

tests/files/packet_ubuntu22-calico-aio.yml

@@ -11,8 +11,17 @@ auto_renew_certificates: true
 kube_proxy_mode: iptables
 enable_nodelocaldns: False
 
-containerd_insecure_registries:
-  "172.19.16.11:5000": "http://172.19.16.11:5000"
-
 containerd_registries:
-  "docker.io": "https://mirror.gcr.io"
+  "docker.io": "https://mirror.gcr.io"
+
+containerd_registries_mirrors:
+  - prefix: docker.io
+    mirrors:
+      - host: https://mirror.gcr.io
+        capabilities: ["pull", "resolve"]
+        skip_verify: false
+  - prefix: 172.19.16.11:5000
+    mirrors:
+      - host: http://172.19.16.11:5000
+        capabilities: ["pull", "resolve", "push"]
+        skip_verify: true