diff --git a/WORKSPACE b/WORKSPACE index 469d7dcada3..8446e1f6cdc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,13 @@ workspace(name = "workerd") +load("@//build/deps:gen/build_deps.bzl", build_deps_gen = "deps_gen") + +build_deps_gen() + +load("@//build/deps:gen/deps.bzl", "deps_gen") + +deps_gen() + # ======================================================================================== # Bazel basics @@ -8,27 +16,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file" NODE_VERSION = "20.14.0" -http_archive( - name = "bazel_skylib", - sha256 = "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz", - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz", - ], -) - load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") bazel_skylib_workspace() -# Needed for objc_library starting with bazel 7. - -http_archive( - name = "build_bazel_apple_support", - sha256 = "c31ce8e531b50ef1338392ee29dd3db3689668701ec3237b9c61e26a1937ab07", - url = "https://github.com/bazelbuild/apple_support/releases/download/1.16.0/apple_support.1.16.0.tar.gz", -) - load( "@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies", @@ -44,25 +35,6 @@ bazel_features_deps() # ======================================================================================== # Simple dependencies -http_archive( - name = "capnp-cpp", - integrity = "sha256-u4TajPQnM3yQIcLUghhNnEUslu0o3q0VdY3jRoPq7yM=", - strip_prefix = "capnproto-capnproto-6446b72/c++", - type = "tgz", - urls = ["https://github.com/capnproto/capnproto/tarball/6446b721a9860eebccf9d3c73b27610491359b5a"], -) - -http_archive( - name = "ssl", - integrity = "sha256-fAgiuoIKLTwR0cwrUz6gwiAiSxGsUKtW2XzgtftXswM=", - strip_prefix = "google-boringssl-c08ccc9", - type = "tgz", - # from main-with-bazel branch. Later boringssl versions have bazel support on the main branch, - # so using the custom branch with a different directory structure will no longer be needed in - # the future. - urls = ["https://github.com/google/boringssl/tarball/c08ccc9ed166a82b92edd70ab215ae1f2501e838"], -) - http_archive( name = "sqlite3", build_file = "//:build/BUILD.sqlite3", @@ -77,13 +49,6 @@ http_archive( url = "https://sqlite.org/2023/sqlite-src-3440000.zip", ) -http_archive( - name = "rules_python", - integrity = "sha256-d4quqz5s/VbWgcifXBDXrWv40vGnLeneVbIwgbLTFhg=", - strip_prefix = "rules_python-0.34.0", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.34.0/rules_python-0.34.0.tar.gz", -) - load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") py_repositories() diff --git a/build/deps/build_deps.jsonc b/build/deps/build_deps.jsonc new file mode 100644 index 00000000000..da715bf228c --- /dev/null +++ b/build/deps/build_deps.jsonc @@ -0,0 +1,34 @@ +{ + "$schema": "deps.schema.json", + "repositories": [ + { + "name": "bazel_skylib", + "type": "github_release", + "owner": "bazelbuild", + "repo": "bazel-skylib", + "file_regex": "\\.tar\\.gz$" + }, + { + "name": "rules_license", + "type": "github_release", + "owner": "bazelbuild", + "repo": "rules_license", + "file_regex": "\\.tar\\.gz$" + }, + { + "name": "rules_python", + "type": "github_release", + "owner": "bazelbuild", + "repo": "rules_python", + "file_regex": "\\.tar\\.gz$" + }, + { + // Needed for objc_library starting with bazel 7 + "name": "build_bazel_apple_support", + "type": "github_release", + "owner": "bazelbuild", + "repo": "apple_support", + "file_regex": "\\.tar\\.gz$" + } + ] +} diff --git a/build/deps/deps.jsonc b/build/deps/deps.jsonc new file mode 100644 index 00000000000..cb0500f8f07 --- /dev/null +++ b/build/deps/deps.jsonc @@ -0,0 +1,22 @@ +{ + "$schema": "deps.schema.json", + "repositories": [ + { + "name": "capnp-cpp", + "type": "github_tarball", + "owner": "capnproto", + "repo": "capnproto", + "branch": "v2", + "extra_strip_prefix": "/c++" + }, + { + "name": "ssl", + "type": "github_tarball", + "owner": "google", + "repo": "boringssl", + "branch": "master-with-bazel", + // todo: head requires a trivial fix + "freeze_commit": "c08ccc9ed166a82b92edd70ab215ae1f2501e838" + } + ] +} diff --git a/build/deps/deps.schema.json b/build/deps/deps.schema.json new file mode 100644 index 00000000000..02aeec9c585 --- /dev/null +++ b/build/deps/deps.schema.json @@ -0,0 +1,203 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Dependencies", + "description": "Description of external dependencies for the project", + "type": "object", + "properties": { + "repositories": { + "type": "array", + "items": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "const": "github_tarball" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "freeze_commit": { + "type": "string" + }, + "freeze_sha256": { + "type": "string" + }, + "build_file": { + "type": "string" + }, + "build_file_content": { + "type": "string" + }, + "patches": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_mapping": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "branch": { + "type": "string" + }, + "extra_strip_prefix": { + "type": "string" + } + }, + "required": [ + "type", + "name", + "owner", + "repo" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "github_release" + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "file_regex": { + "type": "string" + }, + "file_type": { + "type": "string", + "enum": [ + "archive", + "executable" + ] + }, + "freeze_version": { + "type": "string" + }, + "freeze_sha256": { + "type": "string" + }, + "strip_prefix": { + "type": "string" + }, + "build_file": { + "type": "string" + }, + "build_file_content": { + "type": "string" + }, + "patches": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_mapping": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "type", + "name", + "owner", + "repo", + "file_regex" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "git_clone" + }, + "name": { + "type": "string" + }, + "url": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "build_file": { + "type": "string" + }, + "build_file_content": { + "type": "string" + }, + "freeze_commit": { + "type": "string" + }, + "patches": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_mapping": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "required": [ + "type", + "name", + "url" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "const": "crate" + }, + "name": { + "type": "string" + }, + "build_file": { + "type": "string" + }, + "freeze_version": { + "type": "string" + } + }, + "required": [ + "type", + "name", + "build_file" + ], + "additionalProperties": false + } + ] + } + } + }, + "required": [ + "repositories" + ] +} diff --git a/build/deps/gen/build_deps.bzl b/build/deps/gen/build_deps.bzl new file mode 100644 index 00000000000..430964e2945 --- /dev/null +++ b/build/deps/gen/build_deps.bzl @@ -0,0 +1,12 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//build/deps:gen/dep_bazel_skylib.bzl", "dep_bazel_skylib") +load("@//build/deps:gen/dep_build_bazel_apple_support.bzl", "dep_build_bazel_apple_support") +load("@//build/deps:gen/dep_rules_license.bzl", "dep_rules_license") +load("@//build/deps:gen/dep_rules_python.bzl", "dep_rules_python") + +def deps_gen(): + dep_bazel_skylib() + dep_rules_license() + dep_rules_python() + dep_build_bazel_apple_support() diff --git a/build/deps/gen/dep_bazel_skylib.bzl b/build/deps/gen/dep_bazel_skylib.bzl new file mode 100644 index 00000000000..e5f2ea91116 --- /dev/null +++ b/build/deps/gen/dep_bazel_skylib.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +TAG_NAME = "1.7.1" +URL = "https://api.github.com/repos/bazelbuild/bazel-skylib/tarball/1.7.1" +STRIP_PREFIX = "bazelbuild-bazel-skylib-27d429d" +SHA256 = "46e81737e39440cc0364d9ad14212c462b0b5b4395ab7ed0d73cb880604a7fe1" +TYPE = "tgz" + +def dep_bazel_skylib(): + http_archive( + name = "bazel_skylib", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/dep_build_bazel_apple_support.bzl b/build/deps/gen/dep_build_bazel_apple_support.bzl new file mode 100644 index 00000000000..209bcba5734 --- /dev/null +++ b/build/deps/gen/dep_build_bazel_apple_support.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +TAG_NAME = "1.17.0" +URL = "https://api.github.com/repos/bazelbuild/apple_support/tarball/1.17.0" +STRIP_PREFIX = "bazelbuild-apple_support-07dd08d" +SHA256 = "f90a92ad6bdb245523f63dae5e65ebaef7e0087c6651236fc83987cb03937dc3" +TYPE = "tgz" + +def dep_build_bazel_apple_support(): + http_archive( + name = "build_bazel_apple_support", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/dep_capnp_cpp.bzl b/build/deps/gen/dep_capnp_cpp.bzl new file mode 100644 index 00000000000..d4da0e0bb97 --- /dev/null +++ b/build/deps/gen/dep_capnp_cpp.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +URL = "https://github.com/capnproto/capnproto/tarball/cd7b20ad0dbd0b1917d0dc3178f7ba05639ea65d" +STRIP_PREFIX = "capnproto-capnproto-cd7b20a/c++" +SHA256 = "2cf78cbc45ab306a7d71de0eac62f1317a976a3be75339a1df80abd187edd7a3" +TYPE = "tgz" +COMMIT = "cd7b20ad0dbd0b1917d0dc3178f7ba05639ea65d" + +def dep_capnp_cpp(): + http_archive( + name = "capnp-cpp", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/dep_rules_license.bzl b/build/deps/gen/dep_rules_license.bzl new file mode 100644 index 00000000000..bb0021d5bed --- /dev/null +++ b/build/deps/gen/dep_rules_license.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +TAG_NAME = "1.0.0" +URL = "https://api.github.com/repos/bazelbuild/rules_license/tarball/1.0.0" +STRIP_PREFIX = "bazelbuild-rules_license-f85e7d6" +SHA256 = "f6cb6d78b176f768f241813c3205a0931e258328e3885c40506c0c7492f83efe" +TYPE = "tgz" + +def dep_rules_license(): + http_archive( + name = "rules_license", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/dep_rules_python.bzl b/build/deps/gen/dep_rules_python.bzl new file mode 100644 index 00000000000..58c7bb4deb1 --- /dev/null +++ b/build/deps/gen/dep_rules_python.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +TAG_NAME = "0.36.0" +URL = "https://api.github.com/repos/bazelbuild/rules_python/tarball/0.36.0" +STRIP_PREFIX = "bazelbuild-rules_python-387c2f6" +SHA256 = "6d895b4acb68b89d0a97bb0079513724e753acf0185a55fbb6e1a76837a85d18" +TYPE = "tgz" + +def dep_rules_python(): + http_archive( + name = "rules_python", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/dep_ssl.bzl b/build/deps/gen/dep_ssl.bzl new file mode 100644 index 00000000000..e2476ee42f8 --- /dev/null +++ b/build/deps/gen/dep_ssl.bzl @@ -0,0 +1,18 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//:build/http.bzl", "http_archive") + +URL = "https://github.com/google/boringssl/tarball/c08ccc9ed166a82b92edd70ab215ae1f2501e838" +STRIP_PREFIX = "google-boringssl-c08ccc9" +SHA256 = "7c0822ba820a2d3c11d1cc2b533ea0c220224b11ac50ab56d97ce0b5fb57b303" +TYPE = "tgz" +COMMIT = "c08ccc9ed166a82b92edd70ab215ae1f2501e838" + +def dep_ssl(): + http_archive( + name = "ssl", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256, + ) diff --git a/build/deps/gen/deps.bzl b/build/deps/gen/deps.bzl new file mode 100644 index 00000000000..39406138399 --- /dev/null +++ b/build/deps/gen/deps.bzl @@ -0,0 +1,8 @@ +# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT + +load("@//build/deps:gen/dep_capnp_cpp.bzl", "dep_capnp_cpp") +load("@//build/deps:gen/dep_ssl.bzl", "dep_ssl") + +def deps_gen(): + dep_capnp_cpp() + dep_ssl() diff --git a/build/deps/update-deps.py b/build/deps/update-deps.py new file mode 100755 index 00000000000..42a0c40a360 --- /dev/null +++ b/build/deps/update-deps.py @@ -0,0 +1,527 @@ +#!/usr/bin/python3 +""" +Usage: update-deps.py [dep_name] +""" +import datetime +import hashlib +import io +import json +import os +import re +import subprocess +import sys +import tarfile +import urllib.request +import zipfile +from pathlib import Path + +TARGET_FILTER = None if len(sys.argv) < 2 else sys.argv[1] + +SCRIPT_DIR = Path(__file__).resolve().parent +if "BUILD_WORKSPACE_DIRECTORY" in os.environ: + SCRIPT_DIR = Path(os.environ["BUILD_WORKSPACE_DIRECTORY"]) / "build" / "deps" + +DEPS_JSON = SCRIPT_DIR / "deps.jsonc" +BUILD_DEPS_JSON = SCRIPT_DIR / "build_deps.jsonc" +GEN_DIR = SCRIPT_DIR / "gen" +DEPS_BZL = GEN_DIR / "deps.bzl" +BUILD_DEPS_BZL = GEN_DIR / "build_deps.bzl" + + +TOP = "# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT\n" + +GITHUB_TAR_URL_TEMPLATE = "https://github.com/{owner}/{repo}/tarball/{commit}" + +GITHUB_RELEASE_FILE_URL_TEMPLATE = ( + "https://github.com/{owner}/{repo}/releases/download/v{version}/{file}" +) + +HTTP_ARCHIVE_TEMPLATE = ( + TOP + + """ +load("@//:build/http.bzl", "http_archive") + +URL = "{url}" +STRIP_PREFIX = "{prefix}" +SHA256 = "{sha256}" +TYPE = "{type}"{extra_exports} + +def {macro_name}(): + http_archive( + name = "{name}", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256,{extra_attrs} + ) +""" +) + +GITHUB_RELEASE_ARCHIVE_TEMPLATE = ( + TOP + + """ +load("@//:build/http.bzl", "http_archive") + +TAG_NAME = "{tag_name}" +URL = "{url}" +STRIP_PREFIX = "{prefix}" +SHA256 = "{sha256}" +TYPE = "{type}" + +def {macro_name}(): + http_archive( + name = "{name}", + url = URL, + strip_prefix = STRIP_PREFIX, + type = TYPE, + sha256 = SHA256,{extras} + ) +""" +) + +GITHUB_RELEASE_FILE_TEMPLATE = ( + TOP + + """ +load("@//:build/http.bzl", "http_file") + +TAG_NAME = "{tag_name}" +URL = "{url}" +SHA256 = "{sha256}" + +def {macro_name}(): + http_file( + name = "{name}", + url = URL, + executable = True, + sha256 = SHA256,{extras} + ) +""" +) + +NEW_GIT_REPOSITORY_TEMPLATE = ( + TOP + + """ +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +URL = "{url}" +COMMIT = "{commit}" + +def {macro_name}(): + git_repository( + name = "{name}", + remote = URL, + commit = COMMIT,{extras} + ) +""" +) + +V8_REVISION_TEMPLATE = ( + TOP + + """ +V8_COMMIT="{v8_commit}" +""" +) + +GITHUB_ACCESS_TOKEN = "" + + +def macro_name(repo): + return "dep_" + repo["name"].replace("-", "_") + + +class RateLimitedException(Exception): + pass + +class AssetsException(Exception): + pass + +class UnsupportedException(Exception): + pass + +def github_urlopen(url): + """ + A wrapper around urllib.request.urlopen() which parses GitHub rate limit errors and + provides a more human-friendly explanation. + """ + if GITHUB_ACCESS_TOKEN != "": + url = urllib.request.Request(url) + url.add_header("Authorization", f"Bearer {GITHUB_ACCESS_TOKEN}") + try: + return urllib.request.urlopen(url) + except urllib.error.HTTPError as e: + reset_ts = e.headers["x-ratelimit-reset"] + if e.code != 403 or not reset_ts: + raise + reset_dt = datetime.datetime.fromtimestamp(int(reset_ts)) + reset_iso_utc = reset_dt.astimezone(datetime.UTC).isoformat(" ") + reset_iso_local = reset_dt.isoformat(" ") + raise RateLimitedException( + f""" +We have been rate-limited by GitHub. We can make API calls again at: + {reset_iso_utc} UTC ({reset_iso_local} local time). +""" + + """ +You can try re-running the script and specifying an access token since authenticated +GitHub API requests have a higher rate limit. +""" + if GITHUB_ACCESS_TOKEN == "" + else "" + ) from e + + +def github_last_commit(repo): + owner = repo["owner"] + github_repo = repo["repo"] + branch = repo.get("branch", "master") + api_url = f"https://api.github.com/repos/{owner}/{github_repo}/commits/{branch}" + commits = json.loads(github_urlopen(api_url).read()) + return commits["sha"] + + +def get_url_content_sha256(url): + return hashlib.sha256(urllib.request.urlopen(url).read()).hexdigest() + + +def extra_attributes(repo): + extras = "" + if "build_file" in repo: + extras += f"\n build_file = \"{repo['build_file']}\"," + if "build_file_content" in repo: + extras += f"\n build_file_content = \"{repo['build_file_content']}\"," + if "repo_mapping" in repo: + extras += f"\n repo_mapping = {json.dumps(repo['repo_mapping'])}," + if "patches" in repo: + patches = str(repo["patches"]).replace("'", '"') + extras += f"\n patches = {patches}," + extras += '\n patch_args = ["-p1"],' + + return extras + + +def gen_github_tarball(repo): + owner = repo["owner"] + github_repo = repo["repo"] + + commit = github_last_commit(repo) + if "freeze_commit" in repo: + if repo["freeze_commit"] != commit: + print( + "frozen, update available ", + repo["freeze_commit"][:7], + " -> ", + commit[:7], + end="", + ) + commit = repo["freeze_commit"] + else: + print(commit[:7], end="") + + prefix = f"{owner}-{github_repo}-{commit[:7]}" + if "extra_strip_prefix" in repo: + prefix = prefix + repo["extra_strip_prefix"] + + url = GITHUB_TAR_URL_TEMPLATE.format( + owner=owner, + repo=github_repo, + commit=commit, + ) + + if "freeze_sha256" in repo: + sha256 = repo["freeze_sha256"] + else: + sha256 = get_url_content_sha256(url) + + extra_exports = f'\nCOMMIT = "{commit}"' + if "patches" in repo: + extra_exports += f"\nPATCHES = {repo['patches']}".replace("'", '"') + + return HTTP_ARCHIVE_TEMPLATE.format( + name=repo["name"], + url=url, + prefix=prefix, + sha256=sha256, + macro_name=macro_name(repo), + extra_attrs=extra_attributes(repo), + extra_exports=extra_exports, + type="tgz", + ) + + +def github_last_release(repo): + owner = repo["owner"] + github_repo = repo["repo"] + api_url = f"https://api.github.com/repos/{owner}/{github_repo}/releases/latest" + return json.loads(github_urlopen(api_url).read()) + + +def github_release(repo, tag_name): + owner = repo["owner"] + github_repo = repo["repo"] + api_url = ( + f"https://api.github.com/repos/{owner}/{github_repo}/releases/tags/{tag_name}" + ) + return json.loads(github_urlopen(api_url).read()) + + +def gen_github_release(repo): + try: + release = github_last_release(repo) + except urllib.error.HTTPError as e: + # If a repo only has pre-releases, github_last_release will throw a 404 error. + # In that case, we must specify a "freeze_version". + if e.code != 404 or "freeze_version" not in repo: + raise + release = None + + if "freeze_version" in repo: + frozen_release = github_release(repo, repo["freeze_version"]) + if release is not None and frozen_release["tag_name"] != release["tag_name"]: + print( + "frozen, update available: {} -> {}".format( + frozen_release["tag_name"], release["tag_name"] + ), + end="", + ) + release = frozen_release + else: + print(release["tag_name"], end="") + + assets = release["assets"] + if "file_regex" in repo: + file_regex = re.compile(repo["file_regex"]) + assets = [a for a in assets if file_regex.match(a["name"])] + + if len(assets) == 0: + if "tarball_url" in release: + url = release["tarball_url"] + else: + raise AssetsException("No assets found: " + json.dumps(release)) + elif len(assets) > 1: + raise AssetsException( + "Too many assets, use more specific file_regex: " + + str([a["name"] for a in assets]) + ) + else: + asset = assets[0] + url = asset["browser_download_url"] + + type = "tgz" + if url.endswith(".zip"): + type = "zip" + elif url.endswith(".xz"): + type = "xz" + elif url.endswith(".tar.bz2"): + type = "tar.bz2" + + content = urllib.request.urlopen(url).read() + + if "freeze_sha256" in repo: + sha256 = repo["freeze_sha256"] + else: + sha256 = hashlib.sha256(content).hexdigest() + + file_type = repo.get("file_type", "archive") + if file_type == "archive": + if "strip_prefix" in repo: + prefix = repo["strip_prefix"] + elif url.endswith(".zip"): + with zipfile.ZipFile(io.BytesIO(content)) as zip: + prefix = os.path.commonprefix(zip.namelist()) + else: + with tarfile.open(fileobj=io.BytesIO(content)) as tgz: + prefix = os.path.commonprefix(tgz.getnames()) + + return GITHUB_RELEASE_ARCHIVE_TEMPLATE.format( + name=repo["name"], + url=url, + prefix=prefix, + sha256=sha256, + macro_name=macro_name(repo), + type=type, + extras=extra_attributes(repo), + tag_name=release["tag_name"], + ) + elif file_type == "executable": + return GITHUB_RELEASE_FILE_TEMPLATE.format( + name=repo["name"], + url=url, + sha256=sha256, + macro_name=macro_name(repo), + extras=extra_attributes(repo), + tag_name=release["tag_name"], + ) + else: + raise UnsupportedException("Unsupported file_type: " + file_type) + + +def gen_git_clone(repo): + url = repo["url"] + + # We used to clone the repository here to get a shallow_since timestamp, but based + # on # https://github.com/bazelbuild/bazel/issues/12857 it is unclear if this is + # actually helpful. + ls_remote = subprocess.run( + ["git", "ls-remote", url, repo["branch"]], capture_output=True, text=True + ) + ls_remote.check_returncode() + commit = ls_remote.stdout.strip().split()[0] + + if "freeze_commit" in repo: + freeze_commit = repo["freeze_commit"] + if freeze_commit != commit: + print( + "frozen, update available ", + repo["freeze_commit"][:7], + " -> ", + commit[:7], + end="", + ) + commit = freeze_commit + else: + print(commit[:7], end="") + + return NEW_GIT_REPOSITORY_TEMPLATE.format( + name=repo["name"], + macro_name=macro_name(repo), + url=url, + commit=commit, + extras=extra_attributes(repo), + ) + + +def gen_crate(repo): + name = repo["name"] + + # find crate's latest version + crates_json_url = f"https://crates.io/api/v1/crates/{name}" + crate = json.loads(urllib.request.urlopen(crates_json_url).read()) + version = crate["versions"][0] + + if "freeze_version" in repo: + last_version = version + version = next( + v for v in crate["versions"] if v["num"] == repo["freeze_version"] + ) + if last_version["num"] != version["num"]: + print( + "frozen, update available ", + version["num"], + " -> ", + last_version["num"], + end="", + ) + else: + print(version["num"], end="") + + prefix = f"{name}-{version['num']}" + url = f"https://crates.io{version['dl_path']}" + extra_exports = f"\nVERSION = \"{version['num']}\"" + + return HTTP_ARCHIVE_TEMPLATE.format( + name=repo["name"], + url=url, + prefix=prefix, + sha256=version["checksum"], + macro_name=macro_name(repo), + extra_attrs=extra_attributes(repo), + extra_exports=extra_exports, + type="tgz", + ) + + +def gen_repo_str(repo): + if repo["type"] == "github_tarball": + return gen_github_tarball(repo) + elif repo["type"] == "github_release": + return gen_github_release(repo) + elif repo["type"] == "git_clone": + return gen_git_clone(repo) + elif repo["type"] == "crate": + return gen_crate(repo) + else: + raise UnsupportedException(f"Unsupported repo type: {repo['type']}") + + +def gen_repo_bzl(repo): + print("Checking", repo["name"], "... ", end="", flush=True) + if TARGET_FILTER is not None and not repo["name"].startswith(TARGET_FILTER): + print("skipped") + return + with (GEN_DIR / f"{macro_name(repo)}.bzl").open("w") as bzl_file: + bzl_file.write(gen_repo_str(repo)) + print() + + +def gen_deps_bzl(deps, deps_bzl): + deps_bzl_content = TOP + macro_names = [macro_name(repo) for repo in deps["repositories"]] + + for name in sorted(macro_names): + # Buildifier prefers load statements to be sorted alphabetically. + deps_bzl_content += f'\nload("@//build/deps:gen/{name}.bzl", "{name}")' + deps_bzl_content += "\n\ndef deps_gen():\n" + for name in macro_names: + deps_bzl_content += f" {name}()\n" + + with deps_bzl.open("w") as f: + f.write(deps_bzl_content) + + +def process_deps(deps, deps_bzl): + for repo in deps["repositories"]: + gen_repo_bzl(repo) + gen_deps_bzl(deps, deps_bzl) + + +def strip_comments(text): + # capture string literals first, comments send + regex = re.compile(r"(\".*\")|(//.*$)", re.MULTILINE) + return regex.sub( + lambda match: "" if match.group(2) is not None else match.group(1), text + ) + + +def read_access_token(): + # TODO(someday): It is somewhat inconvenient for the user to have to specify + # the value of the # access token. We should either use a python library to redirect + # the user to authenticate via # the browser via an appropriate oauth flow or + # use `gh auth login` and `gh api`. + if TARGET_FILTER is None: + global GITHUB_ACCESS_TOKEN + if sys.stdin.isatty(): + print( + """Follow these steps to obtain a GitHub API access token with +appropriate permissions: + +1. On github.com, go to + Settings > Developer Settings > Personal access tokens > Fine-grained tokens. +2. Generate a new token with default settings. +""" + ) + print( + "Please enter GitHub API access token (or empty to skip): ", + end="", + flush=True, + ) + GITHUB_ACCESS_TOKEN = sys.stdin.readline().strip("\n") + else: + GITHUB_ACCESS_TOKEN = "" + + +def run(): + read_access_token() + + if TARGET_FILTER is None: + for f in GEN_DIR.glob("*.bzl"): + f.unlink() + + with DEPS_JSON.open() as deps_json: + json_text = strip_comments(deps_json.read()) + process_deps(json.loads(json_text), DEPS_BZL) + + with BUILD_DEPS_JSON.open() as deps_json: + json_text = strip_comments(deps_json.read()) + process_deps(json.loads(json_text), BUILD_DEPS_BZL) + + +run() diff --git a/build/http.bzl b/build/http.bzl new file mode 100644 index 00000000000..c0e311d7e30 --- /dev/null +++ b/build/http.bzl @@ -0,0 +1,4 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive", _http_file = "http_file") + +http_archive = _http_archive +http_file = _http_file diff --git a/justfile b/justfile index 06e60d1f4ca..b4def595e99 100644 --- a/justfile +++ b/justfile @@ -42,3 +42,7 @@ format: internal-pr: ./tools/unix/create-internal-pr.sh + +# update dependencies with a given prefix (all by default) +update-deps prefix="": + ./build/deps/update-deps.py {{prefix}}