download_hash: cache request for 'multi-hash' files

This avoid re-downloading the same file for different arch and
re-parsing it
pull/11513/head
Max Gautier 2024-09-06 16:21:20 +02:00
parent 1b1045c0e2
commit dec4e711d1
No known key found for this signature in database
1 changed files with 31 additions and 13 deletions

View File

@ -8,6 +8,7 @@ import sys
from itertools import count, groupby from itertools import count, groupby
from collections import defaultdict from collections import defaultdict
from functools import cache
import argparse import argparse
import requests import requests
from ruamel.yaml import YAML from ruamel.yaml import YAML
@ -100,6 +101,20 @@ def download_hash(only_downloads: [str]) -> None:
data, yaml = open_checksums_yaml() data, yaml = open_checksums_yaml()
s = requests.Session() s = requests.Session()
@cache
def _get_hash_by_arch(download: str, version: str) -> {str: str}:
hash_file = s.get(downloads[download].format(
version = version,
os = "linux",
),
allow_redirects=True)
if hash_file.status_code == 404:
print(f"Unable to find {download} hash file for version {version} at {hash_file.url}")
return None
hash_file.raise_for_status()
return download_hash_extract[download](hash_file.content.decode())
for download, url in (downloads if only_downloads == [] for download, url in (downloads if only_downloads == []
else {k:downloads[k] for k in downloads.keys() & only_downloads}).items(): else {k:downloads[k] for k in downloads.keys() & only_downloads}).items():
checksum_name = f"{download}_checksums" checksum_name = f"{download}_checksums"
@ -125,6 +140,14 @@ def download_hash(only_downloads: [str]) -> None:
# to find new versions # to find new versions
if version in versions and versions[version] != 0: if version in versions and versions[version] != 0:
continue continue
if download in download_hash_extract:
hashes = _get_hash_by_arch(download, version)
if hashes == None:
break
sha256sum = hashes.get(arch)
if sha256sum == None:
break
else:
hash_file = s.get(downloads[download].format( hash_file = s.get(downloads[download].format(
version = version, version = version,
os = "linux", os = "linux",
@ -135,12 +158,7 @@ def download_hash(only_downloads: [str]) -> None:
print(f"Unable to find {download} hash file for version {version} (arch: {arch}) at {hash_file.url}") print(f"Unable to find {download} hash file for version {version} (arch: {arch}) at {hash_file.url}")
break break
hash_file.raise_for_status() hash_file.raise_for_status()
sha256sum = hash_file.content.decode() sha256sum = hash_file.content.decode().split()[0]
if download in download_hash_extract:
sha256sum = download_hash_extract[download](sha256sum).get(arch)
if sha256sum == None:
break
sha256sum = sha256sum.split()[0]
if len(sha256sum) != 64: if len(sha256sum) != 64:
raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: {version}, checksum: '{sha256sum}')") raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: {version}, checksum: '{sha256sum}')")