diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index ac65e7a3b99736..e5d57bab33acd2 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2744,7 +2744,7 @@ "moduleExtensions": { "//:extensions.bzl%bazel_android_deps": { "general": { - "bzlTransitiveDigest": "EBPshmASCYr3Po+LywYJajCUdE2qG2E1+ffXTN7dDnk=", + "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2890,10 +2890,10 @@ }, "//:extensions.bzl%bazel_build_deps": { "general": { - "bzlTransitiveDigest": "EBPshmASCYr3Po+LywYJajCUdE2qG2E1+ffXTN7dDnk=", + "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", "recordedFileInputs": { "@@//MODULE.bazel": "dea29024bdcd8f84856d5187b9d45ade34a8a0349387397a95abec1c998a3e30", - "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "6eb7a8bfc1274585712c029c8fd1e2a9a3fa089b30cf6f91cff6823580cf2f5c" + "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "3eab5b792e20a12c2b731306c97d6dd32ef40d25ac5d2a4d7500cb2d416b4517" }, "recordedDirentsInputs": {}, "envVariables": {}, @@ -3265,7 +3265,7 @@ }, "//:extensions.bzl%bazel_test_deps": { "general": { - "bzlTransitiveDigest": "EBPshmASCYr3Po+LywYJajCUdE2qG2E1+ffXTN7dDnk=", + "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -3445,7 +3445,7 @@ }, "//tools/android:android_extensions.bzl%remote_android_tools_extensions": { "general": { - "bzlTransitiveDigest": "7AujGJs5Ol86xb0COHH2NGNawngyUwI8KmJ0yVuSZ+E=", + "bzlTransitiveDigest": "OBuiEhPTsmGJLplgSLRwqDPybt0jns4dOHdzYzOJ1aM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -3472,7 +3472,7 @@ }, "//tools/test:extensions.bzl%remote_coverage_tools_extension": { "general": { - "bzlTransitiveDigest": "93Butk3OEer7nGZxtwV/qwe+zpX6eMh1OIi+8Lq+6nY=", + "bzlTransitiveDigest": "yd81gjOpH9Vd/KF71OifnQucGLfgWXfE+PqqamoczPk=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 5fbf8db6875187..37e8ab23ce636b 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -792,6 +792,17 @@ sh_test( shard_count = 3, ) +sh_test( + name = "external_remote_file_test", + size = "small", + srcs = ["external_remote_file_test.sh"], + data = [ + ":test-deps", + "@bazel_tools//tools/bash/runfiles", + ], + shard_count = 3, +) + sh_test( name = "external_path_test", size = "medium", diff --git a/src/test/shell/bazel/external_remote_file_test.sh b/src/test/shell/bazel/external_remote_file_test.sh new file mode 100755 index 00000000000000..7fc3a291035ca6 --- /dev/null +++ b/src/test/shell/bazel/external_remote_file_test.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tests the patching functionality of external repositories. + +set -euo pipefail +# --- begin runfiles.bash initialization --- +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else + echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" + exit 1 +fi +# --- end runfiles.bash initialization --- + +source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \ + || { echo "integration_test_setup.sh not found!" >&2; exit 1; } + +# `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux". +# `tr` converts all upper case letters to lower case. +# `case` matches the result if the `uname | tr` expression to string prefixes +# that use the same wildcards as names do in Bash, i.e. "msys*" matches strings +# starting with "msys", and "*" matches everything (it's the default case). +case "$(uname -s | tr [:upper:] [:lower:])" in +msys*) + # As of 2019-01-15, Bazel on Windows only supports MSYS Bash. + declare -r is_windows=true + ;; +*) + declare -r is_windows=false + ;; +esac + +if "$is_windows"; then + # Disable MSYS path conversion that converts path-looking command arguments to + # Windows paths (even if they arguments are not in fact paths). + export MSYS_NO_PATHCONV=1 + export MSYS2_ARG_CONV_EXCL="*" +fi + + +if $is_windows; then + export MSYS_NO_PATHCONV=1 + export MSYS2_ARG_CONV_EXCL="*" +fi + +function set_up() { + WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX") + cd "${WRKDIR}" + write_default_lockfile "MODULE.bazel.lock" + # create an archive file with files interesting for patching + mkdir hello_world-0.1.2 + cat > hello_world-0.1.2/hello_world.c <<'EOF' +#include +int main() { + printf("Hello, world!\n"); + return 0; +} +EOF + zip hello_world.zip hello_world-0.1.2/* + rm -rf hello_world-0.1.2 +} + +function get_extrepourl() { + if $is_windows; then + echo "file:///$(cygpath -m $1)" + else + echo "file://$1" + fi +} + + +test_overlay_remote_file_multiple_segments() { + EXTREPODIR=`pwd` + EXTREPOURL="$(get_extrepourl ${EXTREPODIR})" + + # Generate the remote files to overlay + mkdir -p child + cat > child/foo_bar.c <<'EOF' +#include +int main() { + printf("Foo, Bar!\n"); + return 0; +} +EOF + cat > child/BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "foo_bar", + srcs = ["foo_bar.c"], +) +EOF + touch BUILD.bazel + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "hello_world", + srcs = ["hello_world.c"], +) +EOF + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "hello_world", + srcs = ["hello_world.c"], +) +EOF + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < $TEST_log 2>&1 && fail "Expected to fail" + expect_log "but wanted sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFY=" +} + +test_overlay_remote_file_without_integrity() { + EXTREPODIR=`pwd` + EXTREPOURL="$(get_extrepourl ${EXTREPODIR})" + + # Generate the remote files to overlay + cat > BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "hello_world", + srcs = ["hello_world.c"], +) +EOF + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "hello_world", + srcs = ["hello_world.c"], +) +EOF + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < $TEST_log 2>&1 && fail "Expected to fail" + expect_log "Error in download: Cannot write outside of the repository directory" +} + +test_overlay_remote_file_disallow_absolute_path() { + EXTREPODIR=`pwd` + EXTREPOURL="$(get_extrepourl ${EXTREPODIR})" + + cat > BUILD.bazel <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "hello_world", + srcs = ["hello_world.c"], +) +EOF + touch WORKSPACE + + mkdir main + cd main + cat > WORKSPACE < $TEST_log 2>&1 && fail "Expected to fail" + expect_log "Error in download: Cannot write outside of the repository directory" +} + +run_suite "external remote file tests" diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock index 0d46ed2e24bcca..071ada1d2f608a 100644 --- a/src/test/tools/bzlmod/MODULE.bazel.lock +++ b/src/test/tools/bzlmod/MODULE.bazel.lock @@ -1145,7 +1145,7 @@ }, "@@bazel_tools//tools/android:android_extensions.bzl%remote_android_tools_extensions": { "general": { - "bzlTransitiveDigest": "7AujGJs5Ol86xb0COHH2NGNawngyUwI8KmJ0yVuSZ+E=", + "bzlTransitiveDigest": "OBuiEhPTsmGJLplgSLRwqDPybt0jns4dOHdzYzOJ1aM=", "usagesDigest": "dw7C7ErPIkjYn3yY10kLuOatAlp+XNoJ1RbOr/1/3Tg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1238,7 +1238,7 @@ }, "@@bazel_tools//tools/test:extensions.bzl%remote_coverage_tools_extension": { "general": { - "bzlTransitiveDigest": "93Butk3OEer7nGZxtwV/qwe+zpX6eMh1OIi+8Lq+6nY=", + "bzlTransitiveDigest": "yd81gjOpH9Vd/KF71OifnQucGLfgWXfE+PqqamoczPk=", "usagesDigest": "fsG10p+ckw98K4MfcQu0IYsMwidhFcUesWaQcPdj7vY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1303,7 +1303,7 @@ }, "@@rules_java~//java:extensions.bzl%toolchains": { "general": { - "bzlTransitiveDigest": "g6ePEnDe+cica1jeM4cr5pdiFxzK27/ZwwgBz0qcotU=", + "bzlTransitiveDigest": "luQz5wg3waFxw7wF3Jevqd/dQVN9RQxuHioSnhWMWbU=", "usagesDigest": "eBHBwdDAsDtvJ8WuWgXTHIEveUYrhzF+zdOY4oGoDP8=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -1869,7 +1869,7 @@ }, "@@rules_jvm_external~//:extensions.bzl%maven": { "general": { - "bzlTransitiveDigest": "s/CmQ+qQgXPq+FR5/0BRBmZ1xJgvIA9uQa2rGcsiSUA=", + "bzlTransitiveDigest": "SLfIGbNRszU0h8ge0IMO3/eI3rU4GFZKXkA2TRilJmw=", "usagesDigest": "zel/el3Hu7qA5ChuCSQc5Vo46WXLZmMwpH8aUfhD5wM=", "recordedFileInputs": { "@@rules_jvm_external~//rules_jvm_external_deps_install.json": "10442a5ae27d9ff4c2003e5ab71643bf0d8b48dcf968b4173fa274c3232a8c06" @@ -2893,7 +2893,7 @@ }, "@@rules_jvm_external~//:non-module-deps.bzl%non_module_deps": { "general": { - "bzlTransitiveDigest": "lN2hsTr54Jl4C0fITlL6g7MlqCCtezLQgeENrmdNBtg=", + "bzlTransitiveDigest": "oHEz4DXnwk2aTh/PnIWUE3E1lOge1JPTJOlZOSOr3wI=", "usagesDigest": "auoSltLJ8wMZ3iqVKYWQsyC9CgcCDt85yHgTXFuQEvE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2921,7 +2921,7 @@ }, "@@rules_python~//python/extensions:python.bzl%python": { "general": { - "bzlTransitiveDigest": "V8RFcR2uLgEQnCHK7DYDVnthwCmRsY0zbyChXOGKGGE=", + "bzlTransitiveDigest": "8RLGOLaDCviFdE0nDT5TmqaQd1w6Cm2OXWVzHbcIcxQ=", "usagesDigest": "gGVuRK4jgFR/T75mVSEYMFObc6AY82n+c1P5B+5hUkE=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -2951,7 +2951,7 @@ }, "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { "general": { - "bzlTransitiveDigest": "WppdGtl0faMPVX6wx7nN1JlV+cw9XwxYaGfXa5nWObM=", + "bzlTransitiveDigest": "HyR82V0QIYQqdGYFWjm85H/5Up3KvA+wg1c684HcFpE=", "usagesDigest": "/sVN6+OrINK3/oVrgWGYeYt+N1f1J1KnGaHR2AUoIMY=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/tools/build_defs/repo/http.bzl b/tools/build_defs/repo/http.bzl index 1b7fd22a835367..f08019ca862e8b 100644 --- a/tools/build_defs/repo/http.bzl +++ b/tools/build_defs/repo/http.bzl @@ -45,6 +45,7 @@ load( ) load( ":utils.bzl", + "download_remote_files", "get_auth", "patch", "update_attrs", @@ -133,6 +134,8 @@ def _http_archive_impl(ctx): auth = get_auth(ctx, all_urls) download_info = ctx.download_and_extract( + # TODO(fzakaria): all_urls here has the remote_patch URL which is incorrect + # I believe this to be a file all_urls, ctx.attr.add_prefix, ctx.attr.sha256, @@ -143,6 +146,8 @@ def _http_archive_impl(ctx): integrity = ctx.attr.integrity, ) workspace_and_buildfile(ctx) + + download_remote_files(ctx, auth = auth) patch(ctx, auth = auth) return _update_integrity_attr(ctx, _http_archive_attrs, download_info) @@ -298,6 +303,21 @@ following: `"zip"`, `"jar"`, `"war"`, `"aar"`, `"tar"`, `"tar.gz"`, `"tgz"`, "patch command line tool if `patch_tool` attribute is specified or there are " + "arguments other than `-p` in `patch_args` attribute.", ), + "remote_file_urls": attr.string_list_dict( + default = {}, + doc = + "A map of relative paths (key) to a list of URLs (value) that are to be downloaded " + + "and made available as overlaid files on the repo. This is useful when you want " + + "to add WORKSPACE or BUILD.bazel files atop an existing repository. The files " + + "are downloaded before applying the patches in the `patches` attribute and the list of URLs " + + "should all be possible mirrors of the same file. The URLs are tried in order until one succeeds. ", + ), + "remote_file_integrity": attr.string_dict( + default = {}, + doc = + "A map of file relative paths (key) to its integrity value (value). These relative paths should map " + + "to the files (key) in the `remote_file_urls` attribute.", + ), "remote_patches": attr.string_dict( default = {}, doc = diff --git a/tools/build_defs/repo/utils.bzl b/tools/build_defs/repo/utils.bzl index 89bcf0f0a00e51..9620853446454a 100644 --- a/tools/build_defs/repo/utils.bzl +++ b/tools/build_defs/repo/utils.bzl @@ -91,6 +91,34 @@ def _download_patch(ctx, patch_url, integrity, auth): ) return patch_path +def download_remote_files(ctx, auth = None): + """Utility function for downloading remote files. + + This rule is intended to be used in the implementation function of + a repository rule. It assumes the parameters `remote_file_urls` and + `remote_file_integrity` to be present in `ctx.attr`. + + Args: + ctx: The repository context of the repository rule calling this utility + function. + auth: An optional dict specifying authentication information for some of the URLs. + """ + pending = [ + ctx.download( + remote_file_urls, + path, + canonical_id = ctx.attr.canonical_id, + auth = auth, + integrity = ctx.attr.remote_file_integrity.get(path, ""), + block = False, + ) + for path, remote_file_urls in ctx.attr.remote_file_urls.items() + ] + + # Wait until the requests are done + for p in pending: + p.wait() + def patch(ctx, patches = None, patch_cmds = None, patch_cmds_win = None, patch_tool = None, patch_args = None, auth = None): """Implementation of patching an already extracted repository.