From 0cff50f78e53bf557af7d0673c21d9916afaebc0 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <yves.stan.lecornec@tweag.io> Date: Fri, 10 Dec 2021 17:50:24 +0100 Subject: [PATCH 1/7] bazel_4: add default PATH for local actions if it is not set. https://github.com/NixOS/nixpkgs/issues/94222 fixup1 --- .../bazel/bazel_4/actions_path.patch | 21 +++++++++++++++++++ .../build-managers/bazel/bazel_4/default.nix | 5 +++++ 2 files changed, 26 insertions(+) create mode 100644 pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch b/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch new file mode 100644 index 0000000000000..bade7fdb71654 --- /dev/null +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch @@ -0,0 +1,21 @@ +diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java +index 6fff2af..7e2877e 100755 +--- a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java ++++ b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java +@@ -47,6 +47,16 @@ public final class PosixLocalEnvProvider implements LocalEnvProvider { + Map<String, String> env, BinTools binTools, String fallbackTmpDir) { + ImmutableMap.Builder<String, String> result = ImmutableMap.builder(); + result.putAll(Maps.filterKeys(env, k -> !k.equals("TMPDIR"))); ++ ++ // In case we are running on NixOS. ++ // If bash is called with an unset PATH on this platform, ++ // it will set it to /no-such-path and default tools will be missings. ++ // See, https://github.com/NixOS/nixpkgs/issues/94222 ++ // So we ensure that minimal dependencies are present. ++ if (!env.containsKey("PATH")){ ++ result.put("PATH", "@actionsPathPatch@"); ++ } ++ + String p = clientEnv.get("TMPDIR"); + if (Strings.isNullOrEmpty(p)) { + // Do not use `fallbackTmpDir`, use `/tmp` instead. This way if the user didn't export TMPDIR diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 3dd40ad1f8a4d..09d967a599aef 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -208,6 +208,11 @@ stdenv.mkDerivation rec { strictActionEnvPatch = defaultShellPath; }) + (substituteAll { + src = ./actions_path.patch; + actionsPathPatch = defaultShellPath; + }) + # bazel reads its system bazelrc in /etc # override this path to a builtin one (substituteAll { From be9385ce5944090152568ed8973bf1442b7a2cc7 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <yves.stan.lecornec@tweag.io> Date: Fri, 10 Dec 2021 17:59:55 +0100 Subject: [PATCH 2/7] bazel_4: remove customBash script In order to remove duplicates in PATH when run_shell is called, the customBash script is removed. This is consistent with how other platform behave: only look in the local environment if PATH is not set, but it may break builds that explicitly expects a different local environment. --- .../build-managers/bazel/bazel_4/default.nix | 41 ++++--------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 09d967a599aef..64200c58fd8df 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -363,32 +363,6 @@ stdenv.mkDerivation rec { # Bazel starts a local server and needs to bind a local address. __darwinAllowLocalNetworking = true; - # Bazel expects several utils to be available in Bash even without PATH. Hence this hack. - customBash = writeCBin "bash" '' - #include <stdio.h> - #include <stdlib.h> - #include <string.h> - #include <unistd.h> - - extern char **environ; - - int main(int argc, char *argv[]) { - char *path = getenv("PATH"); - char *pathToAppend = "${defaultShellPath}"; - char *newPath; - if (path != NULL) { - int length = strlen(path) + 1 + strlen(pathToAppend) + 1; - newPath = malloc(length * sizeof(char)); - snprintf(newPath, length, "%s:%s", path, pathToAppend); - } else { - newPath = pathToAppend; - } - setenv("PATH", newPath, 1); - execve("${bash}/bin/bash", argv, environ); - return 0; - } - ''; - postPatch = let darwinPatches = '' @@ -461,8 +435,8 @@ stdenv.mkDerivation rec { # We default to python3 where possible. See also `postFixup` where # python3 is added to $out/nix-support substituteInPlace "$path" \ - --replace /bin/bash ${customBash}/bin/bash \ - --replace "/usr/bin/env bash" ${customBash}/bin/bash \ + --replace /bin/bash ${bash}/bin/bash \ + --replace "/usr/bin/env bash" ${bash}/bin/bash \ --replace "/usr/bin/env python" ${python3}/bin/python \ --replace /usr/bin/env ${coreutils}/bin/env \ --replace /bin/true ${coreutils}/bin/true @@ -470,17 +444,17 @@ stdenv.mkDerivation rec { # bazel test runner include references to /bin/bash substituteInPlace tools/build_rules/test_rules.bzl \ - --replace /bin/bash ${customBash}/bin/bash + --replace /bin/bash ${bash}/bin/bash for i in $(find tools/cpp/ -type f) do substituteInPlace $i \ - --replace /bin/bash ${customBash}/bin/bash + --replace /bin/bash ${bash}/bin/bash done # Fixup scripts that generate scripts. Not fixed up by patchShebangs below. substituteInPlace scripts/bootstrap/compile.sh \ - --replace /bin/bash ${customBash}/bin/bash + --replace /bin/bash ${bash}/bin/bash # add nix environment vars to .bazelrc cat >> .bazelrc <<EOF @@ -548,7 +522,6 @@ stdenv.mkDerivation rec { unzip makeWrapper which - customBash ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx CoreFoundation CoreServices Foundation ]; # Bazel makes extensive use of symlinks in the WORKSPACE. @@ -578,7 +551,7 @@ stdenv.mkDerivation rec { # Note that .bazelversion is always correct and is based on bazel-* # executable name, version checks should work fine export EMBED_LABEL="${version}- (@non-git)" - ${customBash}/bin/bash ./bazel_src/compile.sh + ${bash}/bin/bash ./bazel_src/compile.sh ./bazel_src/scripts/generate_bash_completion.sh \ --bazel=./bazel_src/output/bazel \ --output=./bazel_src/output/bazel-complete.bash \ @@ -672,7 +645,7 @@ stdenv.mkDerivation rec { # Save paths to hardcoded dependencies so Nix can detect them. postFixup = '' mkdir -p $out/nix-support - echo "${customBash} ${defaultShellPath}" >> $out/nix-support/depends + echo "${defaultShellPath}" >> $out/nix-support/depends # The templates get tar’d up into a .jar, # so nix can’t detect python is needed in the runtime closure # Some of the scripts explicitly depend on Python 2.7. Otherwise, we From 2aea3ebc9cd82a07b0915a700880824939251273 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <yves.stan.lecornec@tweag.io> Date: Wed, 15 Dec 2021 15:53:56 +0100 Subject: [PATCH 3/7] bazel_4: add which binary to test buildInputs The which binary is used by the tests to find python location from the PATH. --- pkgs/development/tools/build-managers/bazel/bazel_4/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 64200c58fd8df..6bce4393e1267 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -240,7 +240,7 @@ stdenv.mkDerivation rec { runLocal = name: attrs: script: let attrs' = removeAttrs attrs [ "buildInputs" ]; - buildInputs = [ python3 ] ++ (attrs.buildInputs or []); + buildInputs = [ python3 which ] ++ (attrs.buildInputs or []); in runCommandCC name ({ inherit buildInputs; From b147f2111da702cbb5614aeec855da1b0abd0836 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <ylecornec@users.noreply.github.com> Date: Wed, 15 Dec 2021 17:07:30 +0100 Subject: [PATCH 4/7] bazel_4: add default PATH for local actions if it is not set (darwin) Co-Authored-By: Uri Baghin <33242106+uri-canva@users.noreply.github.com> --- .../bazel/bazel_4/actions_path.patch | 22 ++++++++++++++++++- .../build-managers/bazel/bazel_4/default.nix | 22 ++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch b/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch index bade7fdb71654..1fa1e57483339 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/actions_path.patch @@ -1,5 +1,5 @@ diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java -index 6fff2af..7e2877e 100755 +index 6fff2af..7e2877e 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java @@ -47,6 +47,16 @@ public final class PosixLocalEnvProvider implements LocalEnvProvider { @@ -19,3 +19,23 @@ index 6fff2af..7e2877e 100755 String p = clientEnv.get("TMPDIR"); if (Strings.isNullOrEmpty(p)) { // Do not use `fallbackTmpDir`, use `/tmp` instead. This way if the user didn't export TMPDIR +index 95642767c6..39d3c62461 100644 +--- a/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java ++++ b/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java +@@ -74,6 +74,16 @@ public final class XcodeLocalEnvProvider implements LocalEnvProvider { + + ImmutableMap.Builder<String, String> newEnvBuilder = ImmutableMap.builder(); + newEnvBuilder.putAll(Maps.filterKeys(env, k -> !k.equals("TMPDIR"))); ++ ++ // In case we are running on NixOS. ++ // If bash is called with an unset PATH on this platform, ++ // it will set it to /no-such-path and default tools will be missings. ++ // See, https://github.com/NixOS/nixpkgs/issues/94222 ++ // So we ensure that minimal dependencies are present. ++ if (!env.containsKey("PATH")){ ++ newEnvBuilder.put("PATH", "@actionsPathPatch@"); ++ } ++ + String p = clientEnv.get("TMPDIR"); + if (Strings.isNullOrEmpty(p)) { + // Do not use `fallbackTmpDir`, use `/tmp` instead. This way if the user didn't export TMPDIR diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 6bce4393e1267..cabd0b512f7ff 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -103,7 +103,22 @@ let # ], # ) # - [ bash coreutils findutils gawk gnugrep gnutar gnused gzip which unzip file zip python27 python3 ]; + [ + bash + coreutils + file + findutils + gawk + gnugrep + gnused + gnutar + gzip + python27 + python3 + unzip + which + zip + ]; # Java toolchain used for the build and tests javaToolchain = "@bazel_tools//tools/jdk:toolchain_${buildJdkName}"; @@ -516,12 +531,13 @@ stdenv.mkDerivation rec { # when a command can’t be found in a bazel build, you might also # need to add it to `defaultShellPath`. nativeBuildInputs = [ + coreutils installShellFiles - zip + makeWrapper python3 unzip - makeWrapper which + zip ] ++ lib.optionals (stdenv.isDarwin) [ cctools libcxx CoreFoundation CoreServices Foundation ]; # Bazel makes extensive use of symlinks in the WORKSPACE. From b55758e4f564af896f59377a2d5b34ca83af2665 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <ylecornec@users.noreply.github.com> Date: Wed, 15 Dec 2021 17:11:38 +0100 Subject: [PATCH 5/7] bazel_4: make some tests more verbose to help debugging Co-Authored-By: Uri Baghin <33242106+uri-canva@users.noreply.github.com> --- .../tools/build-managers/bazel/bazel_4/default.nix | 4 ++++ pkgs/development/tools/build-managers/bazel/cpp-test.nix | 2 ++ pkgs/development/tools/build-managers/bazel/java-test.nix | 2 ++ pkgs/development/tools/build-managers/bazel/protobuf-test.nix | 2 ++ 4 files changed, 10 insertions(+) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index cabd0b512f7ff..9b933b6db6582 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -490,6 +490,8 @@ stdenv.mkDerivation rec { build --host_javabase='@local_jdk//:jdk' build --host_java_toolchain='${javaToolchain}' build --verbose_failures + build --curses=no + build --sandbox_debug EOF # add the same environment vars to compile.sh @@ -502,6 +504,8 @@ stdenv.mkDerivation rec { -e "/\$command \\\\$/a --host_javabase='@local_jdk//:jdk' \\\\" \ -e "/\$command \\\\$/a --host_java_toolchain='${javaToolchain}' \\\\" \ -e "/\$command \\\\$/a --verbose_failures \\\\" \ + -e "/\$command \\\\$/a --curses=no \\\\" \ + -e "/\$command \\\\$/a --sandbox_debug \\\\" \ -i scripts/bootstrap/compile.sh # This is necessary to avoid: diff --git a/pkgs/development/tools/build-managers/bazel/cpp-test.nix b/pkgs/development/tools/build-managers/bazel/cpp-test.nix index f4e03abdbc947..3f3faae25e2ec 100644 --- a/pkgs/development/tools/build-managers/bazel/cpp-test.nix +++ b/pkgs/development/tools/build-managers/bazel/cpp-test.nix @@ -44,6 +44,8 @@ let ${bazel}/bin/bazel \ build --verbose_failures \ --distdir=${distDir} \ + --curses=no \ + --sandbox_debug \ //... ''; }; diff --git a/pkgs/development/tools/build-managers/bazel/java-test.nix b/pkgs/development/tools/build-managers/bazel/java-test.nix index 11931a197c0c2..9641a95c33b8b 100644 --- a/pkgs/development/tools/build-managers/bazel/java-test.nix +++ b/pkgs/development/tools/build-managers/bazel/java-test.nix @@ -50,6 +50,8 @@ let --java_toolchain='@bazel_tools//tools/jdk:toolchain_hostjdk8' \ --javabase='@local_jdk//:jdk' \ --verbose_failures \ + --curses=no \ + --sandbox_debug \ //:ProjectRunner ''; }; diff --git a/pkgs/development/tools/build-managers/bazel/protobuf-test.nix b/pkgs/development/tools/build-managers/bazel/protobuf-test.nix index 3858a681659b2..d01e18887248d 100644 --- a/pkgs/development/tools/build-managers/bazel/protobuf-test.nix +++ b/pkgs/development/tools/build-managers/bazel/protobuf-test.nix @@ -169,6 +169,8 @@ let --java_toolchain='@bazel_tools//tools/jdk:toolchain_hostjdk8' \ --javabase='@local_jdk//:jdk' \ --verbose_failures \ + --curses=no \ + --sandbox_debug \ //... ''; }; From ee62812f01b7c5437047168a2d413f97b2c37057 Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <yves.stan.lecornec@tweag.io> Date: Thu, 16 Dec 2021 09:47:16 +0100 Subject: [PATCH 6/7] bazel_4: add default tools to buildInputs (to be accessible from repository rules) Previously the customBash wrapper added the default tools to the PATH of commands from repository rules (which are run in the same environment as Bazel). --- .../tools/build-managers/bazel/bazel_4/default.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 9b933b6db6582..17e924004f57a 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -75,7 +75,7 @@ let for i in ${builtins.toString srcDeps}; do cp $i $out/$(stripHash $i); done ''; - defaultShellPath = lib.makeBinPath + defaultShellUtils = # Keep this list conservative. For more exotic tools, prefer to use # @rules_nixpkgs to pull in tools from the nix repository. Example: # @@ -120,6 +120,8 @@ let zip ]; + defaultShellPath = lib.makeBinPath defaultShellUtils; + # Java toolchain used for the build and tests javaToolchain = "@bazel_tools//tools/jdk:toolchain_${buildJdkName}"; @@ -527,10 +529,7 @@ stdenv.mkDerivation rec { in lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches + genericPatches; - buildInputs = [ - buildJdk - python3 - ]; + buildInputs = [buildJdk] ++ defaultShellUtils; # when a command can’t be found in a bazel build, you might also # need to add it to `defaultShellPath`. From cd842a9e42c348f5a9f126a538ceddd4cd2cd41e Mon Sep 17 00:00:00 2001 From: Yves-Stan Le Cornec <yves.stan.lecornec@tweag.io> Date: Fri, 17 Dec 2021 08:47:00 +0100 Subject: [PATCH 7/7] bazel_4: remove duplicated python paths from the nix-support/depends file. --- .../tools/build-managers/bazel/bazel_4/default.nix | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix index 17e924004f57a..ce26ebc23eae4 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_4/default.nix @@ -103,6 +103,9 @@ let # ], # ) # + # Some of the scripts explicitly depend on Python 2.7. Otherwise, we + # default to using python3. Therefore, both python27 and python3 are + # runtime dependencies. [ bash coreutils @@ -662,16 +665,10 @@ stdenv.mkDerivation rec { ''; # Save paths to hardcoded dependencies so Nix can detect them. + # This is needed because the templates get tar’d up into a .jar. postFixup = '' mkdir -p $out/nix-support echo "${defaultShellPath}" >> $out/nix-support/depends - # The templates get tar’d up into a .jar, - # so nix can’t detect python is needed in the runtime closure - # Some of the scripts explicitly depend on Python 2.7. Otherwise, we - # default to using python3. Therefore, both python27 and python3 are - # runtime dependencies. - echo "${python27}" >> $out/nix-support/depends - echo "${python3}" >> $out/nix-support/depends '' + lib.optionalString stdenv.isDarwin '' echo "${cctools}" >> $out/nix-support/depends '';