kubernetes-sigs
/
kubespray
mirror of https://github.com/kubernetes-sigs/kubespray.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

scripts: refactor scripts/download_hash.sh (#10713) 

The new version brings the following improvements:
- remove having to resort to python python to limit tags (it it slower than
the sh equivalent as python has a somewhat significant startup time).

- Introduce a concept of min version so that it can only get Kubernetes
version supported by Kubespray.

- Fix an issue with kata changing their file scheme (the arch
  specifically)

- Now download sha256/sha256sum files if provided rather than
  downloading the full file and computing the hash

- A few minor style tweaks

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr.fr>
pull/10998/head
Arthur Outhenin-Chalandre 2024-03-11 12:53:26 +01:00 committed by GitHub
parent 69bf6639f3
commit 43c1e3b15e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 147 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

213
scripts/download_hash.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash #!/usr/bin/env bash
set -o errexit set -o errexit
set -o pipefail set -o pipefail
@ -8,23 +8,62 @@ fi
checksums_file="$(git rev-parse --show-toplevel)/roles/kubespray-defaults/defaults/main/checksums.yml" checksums_file="$(git rev-parse --show-toplevel)/roles/kubespray-defaults/defaults/main/checksums.yml"
downloads_folder=/tmp/kubespray_binaries downloads_folder=/tmp/kubespray_binaries
default_file="$(git rev-parse --show-toplevel)/roles/kubespray-defaults/defaults/main/main.yml"
kube_min_version="$(grep kube_version_min_required ${default_file} | sed -E 's|kube_version_min_required: v(.*)|\1|g')"
function get_versions { function filter_version() {
while read version; do
if [[ "${version}" =~ ^v?[0-9]*\.[0-9]*\.[0-9]*$ ]]; then
echo "${version}"
fi
done < /dev/stdin
}
function min_version() {
local min_version="$1"
local func_filter="${2:-filter_version}"
while read version; do
if _vercmp "${version#v}" '>=' "${min_version}"; then
echo "${version}"
fi
done | "${func_filter}"
}
function limit_version() {
local number_versions="${1:-7}"
local func_filter="${2:-filter_version}"
"${func_filter}" | head -n "${number_versions}"
}
function gvisor_version_filter() {
while read version; do
echo "${version}" | sed -E 's|^release-(.*)\..*$|\1|'
done | head -n 8
}
function get_versions() {
local type="$1" local type="$1"
local name="$2" local name="$2"
# NOTE: Limit in the number of versions to be register in the checksums file local version_func="${3:-limit_version}"
local limit="${3:-7}" if [ "$#" -ge 3 ]; then
local python_app="${4:-"import sys,re;tags=[tag.rstrip() for tag in sys.stdin if re.match(\'^v?(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$\',tag)];print(\' \'.join(tags[:$limit]))"}" shift 3
else
shift 2
fi
local version="" local version=""
local attempt_counter=0 local attempt_counter=0
readonly max_attempts=5 readonly max_attempts=5
until [ "$version" ]; do until [ "$version" ]; do
version=$("_get_$type" "$name" "$python_app") version=$("_get_$type" "$name" "${version_func}" "$@")
if [ "$version" ]; then if _vercmp "${version#v}" '<' "${min_version}"; then
continue
elif [ "$version" ] ; then
break break
elif [ ${attempt_counter} -eq ${max_attempts} ]; then elif [ ${attempt_counter} -eq ${max_attempts} ]; then
echo "Max attempts reached" echo "Max attempts reached" >&2
exit 1 exit 1
fi fi
attempt_counter=$((attempt_counter + 1)) attempt_counter=$((attempt_counter + 1))
@ -34,18 +73,19 @@ function get_versions {
echo "${version}" echo "${version}"
} }
function _get_github_tags { function _get_github_tags() {
local repo="$1" local repo="$1"
local python_app="$2" local version_func="$2"
shift 2
# The number of results per page (max 100). # The number of results per page (max 50).
tags="$(curl -s "https://api.github.com/repos/$repo/tags?per_page=100")" tags="$(curl -s "https://api.github.com/repos/$repo/tags?per_page=50")"
if [ "$tags" ]; then if [ "$tags" ]; then
echo "$tags" | grep -Po '"name":.*?[^\\]",' | awk -F '"' '{print $4}' | python -c "$python_app" echo "$tags" | grep -Po '"name":.*?[^\\]",' | awk -F '"' '{print $4}' | "$version_func" "$@"
fi fi
} }
function _vercmp { function _vercmp() {
local v1=$1 local v1=$1
local op=$2 local op=$2
local v2=$3 local v2=$3
@ -83,13 +123,13 @@ function _vercmp {
esac esac
} }
function get_checksums { function get_checksums() {
local binary="$1" local binary="$1"
local version_exceptions="cri_dockerd_archive nerdctl_archive containerd_archive youki" local version_exceptions="cri_dockerd_archive nerdctl_archive containerd_archive youki"
declare -A skip_archs=( declare -A skip_archs=(
["crio_archive"]="arm ppc64le" ["crio_archive"]="arm ppc64le"
["calicoctl_binary"]="arm" ["calicoctl_binary"]="arm"
["ciliumcli_binary"]="ppc64le" ["ciliumcli_binary"]="arm ppc64le"
["etcd_binary"]="arm" ["etcd_binary"]="arm"
["cri_dockerd_archive"]="arm ppc64le" ["cri_dockerd_archive"]="arm ppc64le"
["runc"]="arm" ["runc"]="arm"
@ -113,7 +153,7 @@ function get_checksums {
done done
} }
function get_krew_archive_checksums { function get_krew_archive_checksums() {
declare -A archs=( declare -A archs=(
["linux"]="arm arm64 amd64" ["linux"]="arm arm64 amd64"
["darwin"]="arm64 amd64" ["darwin"]="arm64 amd64"
@ -134,14 +174,14 @@ function get_krew_archive_checksums {
done done
} }
function get_calico_crds_archive_checksums { function get_calico_crds_archive_checksums() {
echo "calico_crds_archive_checksums:" | tee --append "$checksums_file" echo "calico_crds_archive_checksums:" | tee --append "$checksums_file"
for version in "$@"; do for version in "$@"; do
echo " $version: $(_get_checksum "calico_crds_archive" "$version")" | tee --append "$checksums_file" echo " $version: $(_get_checksum "calico_crds_archive" "$version")" | tee --append "$checksums_file"
done done
} }
function get_containerd_archive_checksums { function get_containerd_archive_checksums() {
declare -A support_version_history=( declare -A support_version_history=(
["arm"]="2" ["arm"]="2"
["arm64"]="1.6.0" ["arm64"]="1.6.0"
@ -159,7 +199,7 @@ function get_containerd_archive_checksums {
done done
} }
function get_k8s_checksums { function get_k8s_checksums() {
local binary=$1 local binary=$1
echo "${binary}_checksums:" | tee --append "$checksums_file" echo "${binary}_checksums:" | tee --append "$checksums_file"
@ -176,7 +216,36 @@ function get_k8s_checksums {
done done
} }
function _get_checksum { function get_crictl_checksums() {
local binary=$1
echo "${binary}_checksums:" | tee --append "$checksums_file"
echo " arm:" | tee --append "$checksums_file"
for version in "${@:2}"; do
_vercmp "${version#v}" '<' "1.29" && checksum=$(_get_checksum "$binary" "$version" "arm") || checksum=0
echo " ${version}: $checksum" | tee --append "$checksums_file"
done
for arch in arm64 amd64 ppc64le; do
echo " $arch:" | tee --append "$checksums_file"
for version in "${@:2}"; do
echo " ${version}: $(_get_checksum "$binary" "$version" "$arch")" | tee --append "$checksums_file"
done
done
}
# Note: kata changed their arch starting at version 3.2.0
function get_arch_kata() {
local version="${1}"
local arch="${2}"
if _vercmp "${version}" '<' '3.2.0'; then
echo "${arch//amd64/x86_64}"
else
echo "${arch}"
fi
}
function _get_checksum() {
local binary="$1" local binary="$1"
local version="$2" local version="$2"
local arch="${3:-amd64}" local arch="${3:-amd64}"
@ -187,73 +256,85 @@ function _get_checksum {
readonly github_archive_url="$github_url/%s/archive/%s" readonly github_archive_url="$github_url/%s/archive/%s"
readonly google_url="https://storage.googleapis.com" readonly google_url="https://storage.googleapis.com"
readonly release_url="https://dl.k8s.io" readonly release_url="https://dl.k8s.io"
readonly k8s_url="$release_url/release/$version/bin/$os/$arch/%s" readonly k8s_url="$release_url/release/$version/bin/$os/$arch/%s.sha256"
# Download URLs # Download URLs
declare -A urls=( declare -A urls=(
["crictl"]="$(printf "$github_releases_url" "kubernetes-sigs/cri-tools" "crictl-$version-$os-$arch.tar.gz")" ["crictl"]="$(printf "$github_releases_url" "kubernetes-sigs/cri-tools" "crictl-$version-$os-$arch.tar.gz.sha256")"
["crio_archive"]="$google_url/cri-o/artifacts/cri-o.$arch.$version.tar.gz" ["crio_archive"]="$google_url/cri-o/artifacts/cri-o.$arch.$version.tar.gz"
["kubelet"]="$(printf "$k8s_url" "kubelet")" ["kubelet"]="$(printf "$k8s_url" "kubelet")"
["kubectl"]="$(printf "$k8s_url" "kubectl")" ["kubectl"]="$(printf "$k8s_url" "kubectl")"
["kubeadm"]="$(printf "$k8s_url" "kubeadm")" ["kubeadm"]="$(printf "$k8s_url" "kubeadm")"
["etcd_binary"]="$(printf "$github_releases_url" "etcd-io/etcd" "etcd-$version-$os-$arch.tar.gz")" ["etcd_binary"]="$(printf "$github_releases_url" "etcd-io/etcd" "etcd-$version-$os-$arch.tar.gz")"
["cni_binary"]="$(printf "$github_releases_url" "containernetworking/plugins" "cni-plugins-$os-$arch-$version.tgz")" ["cni_binary"]="$(printf "$github_releases_url" "containernetworking/plugins" "cni-plugins-$os-$arch-$version.tgz.sha256")"
["calicoctl_binary"]="$(printf "$github_releases_url" "projectcalico/calico" "calicoctl-$os-$arch")" ["calicoctl_binary"]="$(printf "$github_releases_url" "projectcalico/calico" "calicoctl-$os-$arch")"
["ciliumcli_binary"]="$(printf "$github_releases_url" "cilium/cilium-cli" "cilium-$os-$arch.tar.gz")" ["ciliumcli_binary"]="$(printf "$github_releases_url" "cilium/cilium-cli" "cilium-$os-$arch.tar.gz.sha256sum")"
["calico_crds_archive"]="$(printf "$github_archive_url" "projectcalico/calico" "$version.tar.gz")" ["calico_crds_archive"]="$(printf "$github_archive_url" "projectcalico/calico" "$version.tar.gz")"
["krew_archive"]="$(printf "$github_releases_url" "kubernetes-sigs/krew" "krew-${os}_$arch.tar.gz")" ["krew_archive"]="$(printf "$github_releases_url" "kubernetes-sigs/krew" "krew-${os}_$arch.tar.gz")"
["helm_archive"]="https://get.helm.sh/helm-$version-$os-$arch.tar.gz" ["helm_archive"]="https://get.helm.sh/helm-$version-$os-$arch.tar.gz"
["cri_dockerd_archive"]="$(printf "$github_releases_url" "Mirantis/cri-dockerd" "cri-dockerd-${version#v}.$arch.tgz")" ["cri_dockerd_archive"]="$(printf "$github_releases_url" "Mirantis/cri-dockerd" "cri-dockerd-${version#v}.$arch.tgz")"
["runc"]="$(printf "$github_releases_url" "opencontainers/runc" "runc.$arch")" ["runc"]="$(printf "$github_releases_url" "opencontainers/runc" "runc.sha256sum")"
["crun"]="$(printf "$github_releases_url" "containers/crun" "crun-$version-$os-$arch")" ["crun"]="$(printf "$github_releases_url" "containers/crun" "crun-$version-$os-$arch")"
["youki"]="$(printf "$github_releases_url" "containers/youki" "youki_$([ $version == "v0.0.1" ] && echo "v0_0_1" || echo "${version#v}" | sed 's|\.|_|g')_$os.tar.gz")" ["youki"]="$(printf "$github_releases_url" "containers/youki" "youki_$([ $version == "v0.0.1" ] && echo "v0_0_1" || echo "${version#v}" | sed 's|\.|_|g')_$os.tar.gz")"
["kata_containers_binary"]="$(printf "$github_releases_url" "kata-containers/kata-containers" "kata-static-$version-${arch//amd64/x86_64}.tar.xz")" ["kata_containers_binary"]="$(printf "$github_releases_url" "kata-containers/kata-containers" "kata-static-$version-$(get_arch_kata "${version}" "${arch}").tar.xz")"
["gvisor_runsc_binary"]="$(printf "$google_url/gvisor/releases/release/$version/%s/runsc" "$(echo "$arch" | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')")" ["gvisor_runsc_binary"]="$(printf "$google_url/gvisor/releases/release/$version/%s/runsc" "$(echo "$arch" | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')")"
["gvisor_containerd_shim_binary"]="$(printf "$google_url/gvisor/releases/release/$version/%s/containerd-shim-runsc-v1" "$(echo "$arch" | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')")" ["gvisor_containerd_shim_binary"]="$(printf "$google_url/gvisor/releases/release/$version/%s/containerd-shim-runsc-v1" "$(echo "$arch" | sed -e 's/amd64/x86_64/' -e 's/arm64/aarch64/')")"
["nerdctl_archive"]="$(printf "$github_releases_url" "containerd/nerdctl" "nerdctl-${version#v}-$os-$([ "$arch" == "arm" ] && echo "arm-v7" || echo "$arch" ).tar.gz")" ["nerdctl_archive"]="$(printf "$github_releases_url" "containerd/nerdctl" "nerdctl-${version#v}-$os-$([ "$arch" == "arm" ] && echo "arm-v7" || echo "$arch" ).tar.gz")"
["containerd_archive"]="$(printf "$github_releases_url" "containerd/containerd" "containerd-${version#v}-$os-$arch.tar.gz")" ["containerd_archive"]="$(printf "$github_releases_url" "containerd/containerd" "containerd-${version#v}-$os-$arch.tar.gz.sha256sum")"
["skopeo_binary"]="$(printf "$github_releases_url" "lework/skopeo-binary" "skopeo-$os-$arch")" ["skopeo_binary"]="$(printf "$github_releases_url" "lework/skopeo-binary" "skopeo-$os-$arch.sha256")"
["yq"]="$(printf "$github_releases_url" "mikefarah/yq" "yq_${os}_$arch")" ["yq"]="$(printf "$github_releases_url" "mikefarah/yq" "yq_${os}_$arch")"
) )
mkdir -p "$(dirname $target)" mkdir -p "$(dirname $target)"
[ -f "$target" ] || curl -LfSs -o "${target}" "${urls[$binary]}" [ -f "$target" ] || curl -LfSs -o "${target}" "${urls[$binary]}"
sha256sum ${target} | awk '{print $1}' if echo "${urls[$binary]}" | grep -qi sha256sum; then
local hashes="$(cat "${target}")"
if [ "$(echo "${hashes}" | wc -l)" -gt 1 ]; then
hashes="$(echo "${hashes}" | grep -- "${arch}")"
fi
if [ "$(echo "${hashes}" | wc -l)" -gt 1 ]; then
hashes="$(echo "${hashes}" | grep -- "${os}")"
fi
if [ "$(echo "${hashes}" | wc -l)" -gt 1 ]; then
echo "more than 1 hash" >&2
echo "${hashes}" >&2
exit 1
fi
echo "${hashes}" | awk '{print $1}'
elif echo "${urls[$binary]}" | grep -qi sha256; then
cat "${target}" | awk '{print $1}'
else
sha256sum ${target} | awk '{print $1}'
fi
} }
function main { mkdir -p "$(dirname "$checksums_file")"
mkdir -p "$(dirname "$checksums_file")" echo "---" | tee "$checksums_file"
echo "---" | tee "$checksums_file" get_crictl_checksums crictl $(get_versions github_tags kubernetes-sigs/cri-tools min_version "${kube_min_version}")
get_checksums crictl $(get_versions github_tags kubernetes-sigs/cri-tools 4) get_checksums crio_archive $(get_versions github_tags cri-o/cri-o min_version "${kube_min_version}")
get_checksums crio_archive $(get_versions github_tags cri-o/cri-o) kubernetes_versions=$(get_versions github_tags kubernetes/kubernetes min_version "${kube_min_version}")
kubernetes_versions=$(get_versions github_tags kubernetes/kubernetes 25) echo "# Checksum" | tee --append "$checksums_file"
echo "# Checksum" | tee --append "$checksums_file" echo "# Kubernetes versions above Kubespray's current target version are untested and should be used with caution." | tee --append "$checksums_file"
echo "# Kubernetes versions above Kubespray's current target version are untested and should be used with caution." | tee --append "$checksums_file" get_k8s_checksums kubelet $kubernetes_versions
get_k8s_checksums kubelet $kubernetes_versions get_checksums kubectl $kubernetes_versions
get_checksums kubectl $kubernetes_versions get_k8s_checksums kubeadm $kubernetes_versions
get_k8s_checksums kubeadm $kubernetes_versions get_checksums etcd_binary $(get_versions github_tags etcd-io/etcd)
get_checksums etcd_binary $(get_versions github_tags etcd-io/etcd) get_checksums cni_binary $(get_versions github_tags containernetworking/plugins)
get_checksums cni_binary $(get_versions github_tags containernetworking/plugins) calico_versions=$(get_versions github_tags projectcalico/calico limit_version 20)
calico_versions=$(get_versions github_tags projectcalico/calico 20) get_checksums calicoctl_binary $calico_versions
get_checksums calicoctl_binary $calico_versions get_checksums ciliumcli_binary $(get_versions github_tags cilium/cilium-cli limit_version 10)
get_checksums ciliumcli_binary $(get_versions github_tags cilium/cilium-cli 10) get_calico_crds_archive_checksums $calico_versions
get_calico_crds_archive_checksums $calico_versions get_krew_archive_checksums $(get_versions github_tags kubernetes-sigs/krew limit_version 2)
get_krew_archive_checksums $(get_versions github_tags kubernetes-sigs/krew 2) get_checksums helm_archive $(get_versions github_tags helm/helm)
get_checksums helm_archive $(get_versions github_tags helm/helm) get_checksums cri_dockerd_archive $(get_versions github_tags Mirantis/cri-dockerd)
get_checksums cri_dockerd_archive $(get_versions github_tags Mirantis/cri-dockerd) get_checksums runc $(get_versions github_tags opencontainers/runc limit_version 5)
get_checksums runc $(get_versions github_tags opencontainers/runc 5) get_checksums crun $(get_versions github_tags containers/crun)
get_checksums crun $(get_versions github_tags containers/crun) get_checksums youki $(get_versions github_tags containers/youki)
get_checksums youki $(get_versions github_tags containers/youki) get_checksums kata_containers_binary $(get_versions github_tags kata-containers/kata-containers)
get_checksums kata_containers_binary $(get_versions github_tags kata-containers/kata-containers 10) gvisor_versions=$(get_versions github_tags google/gvisor gvisor_version_filter)
gvisor_versions=$(get_versions github_tags google/gvisor 0 "import sys,re;tags=[tag[8:16] for tag in sys.stdin if re.match('^release-?(0|[1-9]\d*)\.(0|[1-9]\d*)$',tag)];print(' '.join(tags[:9]))") get_checksums gvisor_runsc_binary $gvisor_versions
get_checksums gvisor_runsc_binary $gvisor_versions get_checksums gvisor_containerd_shim_binary $gvisor_versions
get_checksums gvisor_containerd_shim_binary $gvisor_versions get_checksums nerdctl_archive $(get_versions github_tags containerd/nerdctl)
get_checksums nerdctl_archive $(get_versions github_tags containerd/nerdctl) get_containerd_archive_checksums $(get_versions github_tags containerd/containerd limit_version 30)
get_containerd_archive_checksums $(get_versions github_tags containerd/containerd 30) get_checksums skopeo_binary $(get_versions github_tags lework/skopeo-binary)
get_checksums skopeo_binary $(get_versions github_tags lework/skopeo-binary) get_checksums yq $(get_versions github_tags mikefarah/yq)
get_checksums yq $(get_versions github_tags mikefarah/yq)
}
if [[ ${__name__:-"__main__"} == "__main__" ]]; then
main
fi