Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bazel_7: init at 7.0.0 #262152

Merged
merged 40 commits into from
Jan 1, 2024
Merged
Show file tree
Hide file tree
Changes from 34 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
8cc99ed
bazel_7: start by copying bazel_6
layus Oct 20, 2023
78dd0ba
bazel_7: init at 7.0.0-pre.20230917.3
layus Oct 19, 2023
e0a1a73
bazel_7: Simplify callBazelTest
layus Oct 20, 2023
03f458b
bazel_7: restrict USER hack to darwin
layus Oct 20, 2023
d1f60e0
bazel_7: split tests for clarity
layus Oct 20, 2023
9b820d9
bazel_7: refactor and comment
layus Oct 20, 2023
30cbfd4
bazel_7: More refactors
layus Oct 20, 2023
022befe
Get bazel building on all major platforms, with tests
layus Oct 23, 2023
b09a742
bazel_7: Filter bzlmod dependencies by name predicate
layus Oct 23, 2023
7051773
bazel_7: Separate test deps (without binaries)
layus Oct 24, 2023
229bce1
bazel_7: Cleanup patchPhase
layus Oct 24, 2023
be69c18
Rework repository_cache, lockfile generation, and protobuf tests as a…
layus Oct 28, 2023
622804a
Add 7.0.0rc2, but some patches need an update
layus Oct 28, 2023
05ecc09
Bump to 7.0.0rc2
layus Oct 29, 2023
5151e14
bazel_7: cleanup changes made to common tests
layus Oct 30, 2023
90bfff1
bazel_7: cleanup changes made to common patches
layus Oct 30, 2023
058102d
bazel_7: cleanup protobuf tests
layus Oct 30, 2023
a4cf97e
bazel_7: cleanup unused files
layus Oct 30, 2023
0c13686
bazel_7: rename protobuf test lockfile
layus Oct 30, 2023
7dd831b
bazel_7: rename protobuf test lockfile, part 2
layus Oct 30, 2023
207fb75
bazel_7: darwin (sandbox) fixes
layus Oct 30, 2023
4d54ca8
bazel_7: 7.0.0rc2 -> 7.0.0rc7
layus Dec 8, 2023
306e8ea
bazel_7: fix darwin compilation with CLang 16 & recent nixpkgs/master
boltzmannrain Dec 13, 2023
2a92a23
bazel_7: 7.0.0rc7 -> 7.0.0
layus Dec 13, 2023
f8eaea1
bazel_7: address most review comments
layus Dec 13, 2023
5fc2602
bazel_7: restore xcode_locator patch from no-arc patch
layus Dec 13, 2023
6d93cdd
bazel_7: restore xcode_locator no-arc fix, still needed
layus Dec 13, 2023
0d222a3
bazel_7: re-enable arc in xcode_locator command
layus Dec 14, 2023
a30a2c9
bazel_7: fix cpp-test with bzlmod, for linux
layus Dec 14, 2023
459e800
bazel_7: fix protobuf test on darwin
layus Dec 14, 2023
f35225d
bazel_7: backport bazel_6 bash fixes for remote execution
layus Dec 18, 2023
1795976
bazel_7: Unse runJDK for runtime
layus Dec 18, 2023
eaa9b99
fixup! bazel_7: backport bazel_6 bash fixes for remote execution
layus Dec 18, 2023
2a4700c
bazel_7: Add regression test for empty lockfiles
layus Dec 18, 2023
bfbc35e
fixup! bazel_7: backport bazel_6 bash fixes for remote execution
layus Dec 18, 2023
bc3cba3
bazel_7: Cleanup tests a bit
layus Dec 28, 2023
166e47c
bazel_7: improve lockfile parsing
layus Dec 31, 2023
87716d0
bazel_7: fix remaining tests
layus Dec 31, 2023
e8e45bc
bazel_7: Check-in the main lockfile
layus Dec 31, 2023
7916e16
fixup! bazel_7: Check-in the main lockfile
layus Dec 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{ writeText, bazel, runLocal, bazelTest, distDir }:
{ writeText, bazel, runLocal, bazelTest, distDir, extraBazelArgs ? ""}:

# Tests that certain executables are available in bazel-executed bash shells.

Expand Down Expand Up @@ -35,7 +35,7 @@ let
inherit workspaceDir;

bazelScript = ''
${bazel}/bin/bazel build :tool_usage --distdir=${distDir}
${bazel}/bin/bazel build :tool_usage --distdir=${distDir} ${extraBazelArgs}
cp bazel-bin/output.txt $out
echo "Testing content" && [ "$(cat $out | wc -l)" == "2" ] && echo "OK"
'';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ stdenv.mkDerivation rec {
sha256 = "1mm4awx6sa0myiz9j4hwp71rpr7yh8vihf3zm15n2ii6xb82r31k";
};

in (lib.optionalSttrs (!stdenv.hostPlatform.isDarwin) {
in (lib.optionalAttrs (!stdenv.hostPlatform.isDarwin) {
# `extracted` doesn’t work on darwin
shebang = callPackage ../shebang-test.nix { inherit runLocal extracted bazelTest distDir; bazel = bazel_self;};
}) // {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java
index 8284eff943..a820037968 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/local/PosixLocalEnvProvider.java
@@ -47,6 +47,16 @@ public final class PosixLocalEnvProvider implements LocalEnvProvider {
Map<String, String> env, BinTools binTools, String fallbackTmpDir) {
ImmutableMap.Builder<String, String> result = ImmutableMap.builder();
result.putAll(Maps.filterKeys(env, k -> !k.equals("TMPDIR")));
+
+ // In case we are running on NixOS.
+ // If bash is called with an unset PATH on this platform,
+ // it will set it to /no-such-path and default tools will be missings.
+ // See, https://github.com/NixOS/nixpkgs/issues/94222
+ // So we ensure that minimal dependencies are present.
+ if (!env.containsKey("PATH")){
+ result.put("PATH", "@actionsPathPatch@");
+ }
+
String p = clientEnv.get("TMPDIR");
if (Strings.isNullOrEmpty(p)) {
// Do not use `fallbackTmpDir`, use `/tmp` instead. This way if the user didn't export TMPDIR
diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java b/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java
index 8f230b1b1d..2e4c7d26b7 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/local/XcodeLocalEnvProvider.java
@@ -75,6 +75,16 @@ public final class XcodeLocalEnvProvider implements LocalEnvProvider {

ImmutableMap.Builder<String, String> newEnvBuilder = ImmutableMap.builder();
newEnvBuilder.putAll(Maps.filterKeys(env, k -> !k.equals("TMPDIR")));
+
+ // In case we are running on NixOS.
+ // If bash is called with an unset PATH on this platform,
+ // it will set it to /no-such-path and default tools will be missings.
+ // See, https://github.com/NixOS/nixpkgs/issues/94222
+ // So we ensure that minimal dependencies are present.
+ if (!env.containsKey("PATH")){
+ newEnvBuilder.put("PATH", "@actionsPathPatch@");
+ }
+
String p = clientEnv.get("TMPDIR");
if (Strings.isNullOrEmpty(p)) {
// Do not use `fallbackTmpDir`, use `/tmp` instead. This way if the user didn't export TMPDIR
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
{ lib
, rnix-hashes
, runCommand
, fetchurl
# The path to the right MODULE.bazel.lock
, lockfile
# A predicate used to select only some dependencies based on their name
, requiredDepNamePredicate ? _: true
, canonicalIds ? true
}:
let
modules = builtins.fromJSON (builtins.readFile lockfile);
modulesVersion = modules.lockFileVersion;

# a foldl' for json values
foldlJSON = op: acc: value:
let
# preorder, visit the current node first
acc' = op acc value;

# then visit child values, ignoring attribute names
children =
if builtins.isList value then
lib.foldl' (foldlJSON op) acc' value
else if builtins.isAttrs value then
lib.foldlAttrs (_acc: _name: foldlJSON op _acc) acc' value
else
acc';
in
# like foldl', force evaluation of intermediate results
builtins.seq acc' children;

# remove the "--" prefix, abusing undocumented negative substring length
sanitize = str:
if modulesVersion < 3
then builtins.substring 2 (-1) str
else str;

extract_source = f: acc: value:
# We take any "attributes" object that has a "sha256" field. Every value
# under "attributes" is assumed to be an object, and all the "attributes"
# with a "sha256" field are assumed to have either a "urls" or "url" field.
#
# We add them to the `acc`umulator:
#
# acc // {
# "ffad2b06ef2e09d040...fc8e33706bb01634" = fetchurl {
# name = "source";
# sha256 = "ffad2b06ef2e09d040...fc8e33706bb01634";
# urls = [
# "https://mirror.bazel.build/github.com/golang/library.zip",
# "https://github.com/golang/library.zip"
# ];
# };
# }
let
attrs = value.attributes;
entry = hash: urls: name: {
${hash} = fetchurl {
name = "source"; # just like fetch*, to get some deduplication
inherit urls;
sha256 = hash;
passthru.sha256 = hash;
passthru.source_name = name;
passthru.urls = urls;
};
};
insert = acc: hash: urls: name:
acc // entry (sanitize hash) (map sanitize urls) (sanitize name);
accWithRemotePatches = lib.foldlAttrs
(acc: url: hash: insert acc hash [ url ] attrs.name)
acc
(attrs.remote_patches or { });
accWithNewSource = insert
accWithRemotePatches
(attrs.integrity or attrs.sha256)
(attrs.urls or [ attrs.url ])
attrs.name;
in
if builtins.isAttrs value && value ? attributes
&& (attrs ? sha256 || attrs ? integrity)
&& f attrs.name
then accWithNewSource
else acc;

requiredSourcePredicate = n: requiredDepNamePredicate (sanitize n);
requiredDeps = foldlJSON (extract_source requiredSourcePredicate) { } modules;

command = ''
mkdir -p $out/content_addressable/sha256
cd $out
'' + lib.concatMapStrings
(drv: ''
filename=$(basename "${lib.head drv.urls}")
echo Bundling $filename ${lib.optionalString (drv?source_name) "from ${drv.source_name}"}

# 1. --repository_cache format:
# 1.a. A file under a content-hash directory
hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2)
mkdir -p content_addressable/sha256/$hash
ln -sfn ${drv} content_addressable/sha256/$hash/file

# 1.b. a canonicalId marker based on the download urls
# Bazel uses these to avoid reusing a stale hash when the urls have changed.
canonicalId="${lib.concatStringsSep " " drv.urls}"
canonicalIdHash=$(echo -n "$canonicalId" | sha256sum | cut -d" " -f1)
echo -n "$canonicalId" > content_addressable/sha256/$hash/id-$canonicalIdHash

# 2. --distdir format:
# Just a file with the right basename
# Mostly to keep old tests happy, and because symlinks cost nothing.
# This is brittle because of expected file name conflicts
ln -sn ${drv} $filename || true
'')
(builtins.attrValues requiredDeps)
;

repository_cache = runCommand "bazel-repository-cache" { } command;

in
repository_cache
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
###############################################################################
# Bazel now uses Bzlmod by default to manage external dependencies.
# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
#
# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
###############################################################################

Loading