Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve performance of python script for checksum download #10335

Merged
merged 1 commit into from
Jan 23, 2024
Merged

Improve performance of python script for checksum download #10335

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 22 additions & 13 deletions scripts/download_hash.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,21 @@
# run this script to update roles/kubespray-defaults/defaults/main/download.yml
# with new hashes.

import hashlib
import sys

import requests
from ruamel.yaml import YAML

MAIN_YML = "../roles/kubespray-defaults/defaults/main/download.yml"
CHECKSUMS_YML = "../roles/kubespray-defaults/defaults/main/checksums.yml"

def open_main_yaml():
def open_checksums_yaml():
yaml = YAML()
yaml.explicit_start = True
yaml.preserve_quotes = True
yaml.width = 4096

with open(MAIN_YML, "r") as main_yml:
data = yaml.load(main_yml)
with open(CHECKSUMS_YML, "r") as checksums_yml:
data = yaml.load(checksums_yml)

return data, yaml

Expand All @@ -28,23 +27,33 @@ def download_hash(versions):
architectures = ["arm", "arm64", "amd64", "ppc64le"]
downloads = ["kubelet", "kubectl", "kubeadm"]

data, yaml = open_main_yaml()
data, yaml = open_checksums_yaml()

for download in downloads:
checksum_name = f"{download}_checksums"
for arch in architectures:
for version in versions:
if not version.startswith("v"):
version = f"v{version}"
url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}"
download_file = requests.get(url, allow_redirects=True)
download_file.raise_for_status()
sha256sum = hashlib.sha256(download_file.content).hexdigest()
url = f"https://dl.k8s.io/release/{version}/bin/linux/{arch}/{download}.sha256"
hash_file = requests.get(url, allow_redirects=True)
if hash_file.status_code == 404:
print(f"Unable to find hash file for release {version} (arch: {arch})")
continue
if hash_file.status_code != 200:
raise Exception(f"Received a non-200 HTTP response code: {hash_file.status_code} (arch: {arch}, version: {version})")
sha256sum = hash_file.content.decode().strip()
if len(sha256sum) != 64:
raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (arch: {arch}, version: {version})")
if checksum_name not in data:
data[checksum_name] = {}
if arch not in data[checksum_name]:
data[checksum_name][arch] = {}
data[checksum_name][arch][version] = sha256sum

with open(MAIN_YML, "w") as main_yml:
yaml.dump(data, main_yml)
print(f"\n\nUpdated {MAIN_YML}\n")
with open(CHECKSUMS_YML, "w") as checksums_yml:
yaml.dump(data, checksums_yml)
print(f"\n\nUpdated {CHECKSUMS_YML}\n")


def usage():
Expand Down