From 8717c5ad4b4319a877afa4109146dfd5c240f74c Mon Sep 17 00:00:00 2001 From: Gondermann Date: Thu, 10 Aug 2023 16:43:14 +0200 Subject: [PATCH] Force update script to download the image and calculate SHA512 Previously the precomputed SHA256 was used for most images. Since we want to check the hash against the openstack backend, we need to have the SHA512. Sadly, most images creators do not provide us with that hash pre-computed. The only solution is to download the images and computing the hash at runtime. Signed-off-by: Gondermann --- openstack_image_manager/update.py | 76 +++++++++++++++++-------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/openstack_image_manager/update.py b/openstack_image_manager/update.py index a22e2c2c..8227cf5c 100644 --- a/openstack_image_manager/update.py +++ b/openstack_image_manager/update.py @@ -7,8 +7,9 @@ import shutil import sys import time +import hashlib +import math from urllib.parse import urlparse -from urllib.request import urlopen from loguru import logger from minio import Minio @@ -73,6 +74,11 @@ def mirror_image(image, latest_url, CONF): ) os.remove(filename) +def size_clean(size): + size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB") + i = int(math.floor(math.log(size, 1024))) + s = size / 1024 ** i + return f"{s:.2f} {size_name[i]}" def update_image(image, CONF): name = image["name"] @@ -88,54 +94,54 @@ def update_image(image, CONF): logger.info(f"Getting checksums from {latest_checksum_url}") result = requests.get(latest_checksum_url) - checksums = {} - checksum_type = "sha256" + hash_obj = hashlib.new("sha512") + file_headers = None + with requests.get(url=latest_url, stream=True, timeout=30) as response: + if response.status_code != 200: + logger.error(f"Downloading image '{name}' failed with error code {response.status_code}") + return None + + file_headers = response.headers + file_size = int(file_headers["Content-Length"]) + logger.info(f"Image size {size_clean(file_size)}") + + downloadedBytes = 0 + lastProgress = 0 + for chunk in response.iter_content(chunk_size=8192): + downloadedBytes += 8192 + progressPercent = (downloadedBytes / file_size) * 100 + progress = round(min(max(progressPercent, 0), 100)) + if progress - lastProgress >= 5: + logger.info(f"Downloading image: {progress}%") + lastProgress = progress + + hash_obj.update(chunk) + + sha512_value = hash_obj.hexdigest() + filename_pattern = None if image["shortname"] in ["centos-stream-8", "centos-stream-9", "centos-7"]: filename_pattern = latest_filename.replace("HEREBE", "") filename_pattern = filename_pattern.replace("DRAGONS", "") - elif image["shortname"] in ["debian-10", "debian-11", "debian-12"]: - checksum_type = "sha512" + new_latest_filename_list = [] for line in result.text.split("\n"): - if image["shortname"] in ["rocky-8", "rocky-9"]: - splitted_line = re.split("\s+", line) # noqa W605 - if splitted_line[0] == "SHA256": - checksums[latest_filename] = splitted_line[3] - elif image["shortname"] in [ - "ubuntu-14.04", - "ubuntu-16.04", - "ubuntu-16.04-minimal", - "ubuntu-18.04", - "ubuntu-18.04-minimal", - "ubuntu-20.04", - "ubuntu-20.04-minimal", - "ubuntu-22.04", - "ubuntu-22.04-minimal", - ]: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1][1:]] = splitted_line[0] - elif image["shortname"] in ["centos-7"]: + if image["shortname"] in ["centos-7"]: splitted_line = re.split("\s+", line) # noqa W605 if len(splitted_line) == 2: if re.search(filename_pattern, splitted_line[1]): - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1]) elif image["shortname"] in ["centos-stream-8", "centos-stream-9"]: splitted_line = re.split("\s+", line) # noqa W605 if splitted_line[0] == "SHA256" and re.search( filename_pattern, splitted_line[1][1:-1] ): - checksums[splitted_line[1][1:-1]] = splitted_line[3] - else: - splitted_line = re.split("\s+", line) # noqa W605 - if len(splitted_line) == 2: - checksums[splitted_line[1]] = splitted_line[0] + new_latest_filename_list.append(splitted_line[1][1:-1]) if filename_pattern: - new_latest_filename = natsorted(checksums.keys())[-1] + new_latest_filename = natsorted(new_latest_filename_list)[-1] new_latest_url = latest_url.replace(latest_filename, new_latest_filename) logger.info(f"Latest URL is now {new_latest_url}") @@ -144,7 +150,7 @@ def update_image(image, CONF): latest_filename = new_latest_filename latest_url = new_latest_url - current_checksum = f"{checksum_type}:{checksums[latest_filename]}" + current_checksum = f"sha512:{sha512_value}" logger.info(f"Checksum of current {latest_filename} is {current_checksum}") try: @@ -166,9 +172,8 @@ def update_image(image, CONF): if latest_checksum != current_checksum: logger.info(f"Checking {latest_url}") - conn = urlopen(latest_url, timeout=30) struct = time.strptime( - conn.headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" + file_headers["last-modified"], "%a, %d %b %Y %H:%M:%S %Z" ) dt = datetime.fromtimestamp(time.mktime(struct)) @@ -240,7 +245,8 @@ def main( for index, image in enumerate(data["images"]): if "latest_url" in image: updated_image = update_image(image, CONF) - data["images"][index] = updated_image + if updated_image: + data["images"][index] = updated_image with open(p, "w+") as fp: ryaml = ruamel.yaml.YAML()