add manage offline files script (#8956)

Signed-off-by: bo.jiang <bo.jiang@daocloud.io>

.gitignore

@@ -3,6 +3,7 @@
 **/vagrant_ansible_inventory
 *.iml
 temp
+offline-files
 .idea
 .vscode
 .tox

contrib/offline/README.md

@@ -45,3 +45,21 @@ temp
 
 In some cases you may want to update some component version, you can declare version variables in ansible inventory file or group_vars,
 then run `./generate_list.sh -i [inventory_file]` to update file.list and images.list.
+
+## manage-offline-files.sh
+
+This script will download all files according to `temp/files.list` and run nginx container to provide offline file download.
+
+Step(1) generate `files.list`
+
+```shell
+./generate_list.sh
+```
+
+Step(2) download files and run nginx container
+
+```shell
+./manage-offline-files.sh
+```
+
+when nginx container is running, it can be accessed through <http://127.0.0.1:8080/>.

contrib/offline/manage-offline-files.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+CURRENT_DIR=$(cd $(dirname $0); pwd)
+OFFLINE_FILES_DIR="${CURRENT_DIR}/offline-files"
+FILES_LIST=${FILES_LIST:-"${CURRENT_DIR}/temp/files.list"}
+NGINX_PORT=8080
+
+# download files
+if [ ! -f ${FILES_LIST} ]; then
+    echo "${FILES_LIST} should exist."
+    exit 1
+fi
+rm -rf ${OFFLINE_FILES_DIR}
+mkdir  ${OFFLINE_FILES_DIR}
+wget -x -P ${OFFLINE_FILES_DIR} -i ${FILES_LIST}
+
+# run nginx container server
+if command -v nerdctl 1>/dev/null 2>&1; then
+    runtime="nerdctl"
+elif command -v podman 1>/dev/null 2>&1; then
+    runtime="podman"
+elif command -v docker 1>/dev/null 2>&1; then
+    runtime="docker"
+else
+    echo "No supported container runtime found"
+    exit 1
+fi
+sudo "${runtime}" container inspect nginx >/dev/null 2>&1
+if [ $? -ne 0 ]; then
+    sudo "${runtime}" run \
+        --restart=always -d -p ${NGINX_PORT}:80 \
+        --volume ${OFFLINE_FILES_DIR}:/usr/share/nginx/html/download \
+        --volume "$(pwd)"/nginx.conf:/etc/nginx/nginx.conf \
+        --name nginx nginx:alpine
+fi

contrib/offline/nginx.conf

@@ -0,0 +1,39 @@
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+include /usr/share/nginx/modules/*.conf;
+events {
+    worker_connections 1024;
+}
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+    access_log  /var/log/nginx/access.log  main;
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+    default_type        application/octet-stream;
+    include /etc/nginx/conf.d/*.conf;
+    server {
+        listen       80 default_server;
+        listen       [::]:80 default_server;
+        server_name  _;
+        include /etc/nginx/default.d/*.conf;
+        location / {
+            root    /usr/share/nginx/html/download;
+        autoindex on;
+        autoindex_exact_size off;
+        autoindex_localtime on;
+        }
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+}

inventory/sample/group_vars/all/offline.yml

@@ -18,44 +18,45 @@
 # quay_image_repo: "{{ registry_host }}"
 
 ## Kubernetes components
-# kubeadm_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubeadm"
+# kubeadm_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
-# kubectl_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubectl"
+# kubectl_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
-# kubelet_download_url: "{{ files_repo }}/kubernetes/{{ kube_version }}/kubelet"
+# kubelet_download_url: "{{ files_repo }}/storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 
 ## CNI Plugins
-# cni_download_url: "{{ files_repo }}/kubernetes/cni/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+# cni_download_url: "{{ files_repo }}/github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 
 ## cri-tools
-# crictl_download_url: "{{ files_repo }}/kubernetes/cri-tools/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# crictl_download_url: "{{ files_repo }}/github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## [Optional] etcd: only if you **DON'T** use etcd_deployment=host
-# etcd_download_url: "{{ files_repo }}/kubernetes/etcd/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
+# etcd_download_url: "{{ files_repo }}/github.com/etcd-io/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] Calico: If using Calico network plugin
-# calicoctl_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_download_url: "{{ files_repo }}/github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+# calicoctl_alternate_download_url: "{{ files_repo }}/github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 # [Optional] Calico with kdd: If using Calico network plugin with kdd datastore
-# calico_crds_download_url: "{{ files_repo }}/kubernetes/calico/{{ calico_version }}.tar.gz"
+# calico_crds_download_url: "{{ files_repo }}/github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
 
 # [Optional] Flannel: If using Falnnel network plugin
 # flannel_cni_download_url: "{{ files_repo }}/kubernetes/flannel/{{ flannel_cni_version }}/flannel-{{ image_arch }}"
 
 # [Optional] helm: only if you set helm_enabled: true
-# helm_download_url: "{{ files_repo }}/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
+# helm_download_url: "{{ files_repo }}/get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 
 # [Optional] crun: only if you set crun_enabled: true
-# crun_download_url: "{{ files_repo }}/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
+# crun_download_url: "{{ files_repo }}/github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
 
 # [Optional] kata: only if you set kata_containers_enabled: true
-# kata_containers_download_url: "{{ files_repo }}/kata-containers/runtime/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
+# kata_containers_download_url: "{{ files_repo }}/github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
 
 # [Optional] cri-o: only if you set container_manager: crio
 # crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
 # crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
 
 # [Optional] runc,containerd: only if you set container_runtime: containerd
-# runc_download_url: "{{ files_repo }}/{{ runc_version }}/runc.{{ image_arch }}"
+# runc_download_url: "{{ files_repo }}/github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
-# containerd_download_url: "{{ files_repo }}/containerd/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
+# containerd_download_url: "{{ files_repo }}/github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
-# nerdctl_download_url: "{{ files_repo }}/nerdctl/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+# nerdctl_download_url: "{{ files_repo }}/github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 
 ## CentOS/Redhat/AlmaLinux
 ### For EL7, base and extras repo must be available, for EL8, baseos and appstream