Download hash script: auto discover versions (#10849)

* Download patches version automatically from a minor

* Automate versions discovery for hash download

* Small refactoring

scripts/download_hash.py

@@ -6,8 +6,11 @@
 
 import sys
 
+from itertools import count
+from collections import defaultdict
 import requests
 from ruamel.yaml import YAML
+from packaging.version import Version
 
 CHECKSUMS_YML = "../roles/kubespray-defaults/defaults/main/checksums.yml"
 
@@ -23,33 +26,37 @@ def open_checksums_yaml():
     return data, yaml
 
 
-def download_hash(versions):
+def download_hash(minors):
     architectures = ["arm", "arm64", "amd64", "ppc64le"]
     downloads = ["kubelet", "kubectl", "kubeadm"]
 
     data, yaml = open_checksums_yaml()
+    if not minors:
+        minors = {'.'.join(minor.split('.')[:-1]) for minor in data["kubelet_checksums"]["amd64"].keys()}
 
     for download in downloads:
         checksum_name = f"{download}_checksums"
+        data[checksum_name] = defaultdict(dict, data[checksum_name])
         for arch in architectures:
-            for version in versions:
+            for minor in minors:
-                if not version.startswith("v"):
+                if not minor.startswith("v"):
-                    version = f"v{version}"
+                    minor = f"v{minor}"
-                url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}.sha256"
+                for release in (f"{minor}.{patch}" for patch in count(start=0, step=1)):
-                hash_file = requests.get(url, allow_redirects=True)
+                    if release in data[checksum_name][arch]:
-                if hash_file.status_code == 404:
+                        continue
-                    print(f"Unable to find hash file for release {version} (arch: {arch})")
+                    hash_file = requests.get(f"https://dl.k8s.io/release/{release}/bin/linux/{arch}/{download}.sha256", allow_redirects=True)
-                    continue
+                    if hash_file.status_code == 404:
-                if hash_file.status_code != 200:
+                        print(f"Unable to find {download} hash file for release {release} (arch: {arch})")
-                    raise Exception(f"Received a non-200 HTTP response code: {hash_file.status_code} (arch: {arch}, version: {version})")
+                        break
-                sha256sum = hash_file.content.decode().strip()
+                    hash_file.raise_for_status()
-                if len(sha256sum) != 64:
+                    sha256sum = hash_file.content.decode().strip()
-                    raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (arch: {arch}, version: {version})")
+                    if len(sha256sum) != 64:
-                if checksum_name not in data:
+                        raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: 1.{minor}.{patch})")
-                    data[checksum_name] = {}
+                    data[checksum_name][arch][release] = sha256sum
-                if arch not in data[checksum_name]:
+        data[checksum_name] = {arch : {r : releases[r] for r in sorted(releases.keys(),
-                    data[checksum_name][arch] = {}
+                                                  key=lambda v : Version(v[1:]),
-                data[checksum_name][arch][version] = sha256sum
+                                                  reverse=True)}
+                               for arch, releases in data[checksum_name].items()}
 
     with open(CHECKSUMS_YML, "w") as checksums_yml:
         yaml.dump(data, checksums_yml)
@@ -61,12 +68,7 @@ def usage():
 
 
 def main(argv=None):
-    if not argv:
+    download_hash(sys.argv[1:])
-        argv = sys.argv[1:]
-    if not argv:
-        usage()
-        return 1
-    download_hash(argv)
     return 0