🚀 Introduce local remote execution toolchains

.bazelrc

@@ -25,5 +25,31 @@ common --enable_bzlmod
 common --registry=https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/
 common --registry=https://raw.githubusercontent.com/eomii/bazel-eomii-registry/main/
 
+# Make sure to use the correct java runtime.
+build --java_runtime_version=rbe_jdk
+build --tool_java_runtime_version=rbe_jdk
+
+# Always act as if using remote execution.
+build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build --define=EXECUTOR=remote
+
+# Remote optimizations.
+build --experimental_remote_cache_compression
+build --experimental_remote_build_event_upload=minimal
+build --remote_download_minimal
+build --nolegacy_important_outputs
+
+# Better profiling.
+build --slim_profile
+build --experimental_profile_include_primary_output
+build --experimental_profile_include_target_label
+
+# This config should only be used to test remote execution locally.
+build:rbe_local --experimental_docker_image=localhost:5000/rules_ll_remote:latest
+build:rbe_local --noexperimental_docker_use_customized_images
+build:rbe_local --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:rbe_local --experimental_enable_docker_sandbox
+build:rbe_local --experimental_docker_verbose
+
 # Allow user-side customization.
 try-import %workspace%/.bazelrc.user

.gitignore

@@ -4,6 +4,9 @@ bazel-*
 # Don't ignore the bazel-wrapper directory.
 !bazel-wrapper/
 
+# Generated during the rbegen invocation or when building a flake output.
+result
+
 # Locally generated by direnv.
 .direnv/
 

MODULE.bazel

@@ -4,11 +4,16 @@ module(
     compatibility_level = 0,
 )
 
-register_execution_platforms("@rules_ll//ll:ll_linux_exec_platform")
-register_toolchains("@rules_ll//ll:ll_toolchain")
+register_execution_platforms("@rules_ll//rbe/default/config:platform")
+register_toolchains(
+    "@rules_ll//ll:ll_toolchain",
+    "@rules_ll//rbe/default/config:cc-toolchain",
+    "@rules_ll//rbe/default/java:all",
+)
 
 # Platform support.
 bazel_dep(name = "platforms", version = "0.0.6")
+bazel_dep(name = "rules_cc", version = "0.0.6")
 
 # Various utility functions such as path manipulations and templating.
 bazel_dep(name = "bazel_skylib", version = "1.4.1")

bazel-wrapper/default.nix

@@ -3,6 +3,7 @@
 {
 
   env = ll_env;
+  version = bazel.version;
   baze_ll = pkgs.writeShellScriptBin "bazel" ''
 
 # The default compilation environment.

examples/.bazelrc

@@ -25,5 +25,31 @@ common --enable_bzlmod
 common --registry=https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/
 common --registry=https://raw.githubusercontent.com/eomii/bazel-eomii-registry/main/
 
+# Make sure to use the correct java runtime.
+build --java_runtime_version=rbe_jdk
+build --tool_java_runtime_version=rbe_jdk
+
+# Always act as if using remote execution.
+build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
+build --define=EXECUTOR=remote
+
+# Remote optimizations.
+build --experimental_remote_cache_compression
+build --experimental_remote_build_event_upload=minimal
+build --remote_download_minimal
+build --nolegacy_important_outputs
+
+# Better profiling.
+build --slim_profile
+build --experimental_profile_include_primary_output
+build --experimental_profile_include_target_label
+
+# This config should only be used to test remote execution locally.
+build:rbe_local --experimental_docker_image=localhost:5000/rules_ll_remote:latest
+build:rbe_local --noexperimental_docker_use_customized_images
+build:rbe_local --spawn_strategy=docker --strategy=Javac=docker --genrule_strategy=docker
+build:rbe_local --experimental_enable_docker_sandbox
+build:rbe_local --experimental_docker_verbose
+
 # Allow user-side customization.
 try-import %workspace%/.bazelrc.user

flake.nix

@@ -110,9 +110,45 @@
           }
         );
 
+        bazelToolchains = import ./rbe/default.nix { inherit pkgs; };
+
+        rbegen = pkgs.writeShellScriptBin "rbegen" ''
+          # Make sure to only use this tool from the root directory of rules_ll.
+          # You need a local docker registry running to regenerate the rbe
+          # configuration:
+          # docker run -d -p 5000:5000 --restart=always --name registry registry:2
+
+          # Deliberately build the image manually and use "result" instead of
+          # the derivation to prevent nix from building the image unless
+          # explicitly asked to.
+
+          nix build .#ci-image
+
+          ${pkgs.skopeo}/bin/skopeo \
+              --insecure-policy \
+              copy \
+              --dest-tls-verify=false \
+              "docker-archive://$(realpath result)" \
+              "docker://localhost:5000/rules_ll_remote"
+
+          ${bazelToolchains}/bin/rbe_configs_gen \
+              --toolchain_container=localhost:5000/rules_ll_remote \
+              --exec_os=linux \
+              --target_os=linux \
+              --bazel_version=${wrappedBazel.version} \
+              --output_src_root=$(pwd) \
+              --output_config_path=rbe/default \
+              --bazel_path=${wrappedBazel.baze_ll}/bin/bazel \
+              --cpp_env_json=${self}/rbe/rbeconfig.json
+        '';
+
       in
       {
 
+        packages = {
+          ci-image = import ./rbe/image.nix { inherit pkgs wrappedBazel; };
+        };
+
         checks = {
           pre-commit-check = pre-commit-hooks-nix.lib.${system}.run {
             src = ./.;
@@ -124,6 +160,7 @@
           default = llShell {
             unfree = true;
             packages = [
+              rbegen
               pkgs.git
               (pkgs.python3.withPackages (pylib: [
                 pylib.mkdocs-material

ll/BUILD.bazel

@@ -66,26 +66,6 @@ filegroup(
     ],
 )
 
-# Constraints.
-
-constraint_setting(name = "ll_exec")
-
-constraint_value(
-    name = "ll_exec_linux",
-    constraint_setting = ":ll_exec",
-)
-
-# Platforms.
-
-platform(
-    name = "ll_linux_exec_platform",
-    constraint_values = [
-        "@platforms//os:linux",
-        "@platforms//cpu:x86_64",
-        ":ll_exec_linux",
-    ],
-)
-
 # Configurations.
 
 string_flag(
@@ -144,7 +124,11 @@ ll_toolchain(
         ":bootstrap": None,
         "//conditions:default": "@llvm-project//libcxx:libll_cxx",
     }),
-    exec_compatible_with = [":ll_exec_linux"],
+    exec_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+        "@bazel_tools//tools/cpp:clang",
+    ],
     hip_libraries = select({
         ":hip_amdgpu": [
             "@hip//:headers",
@@ -212,7 +196,11 @@ ll_toolchain(
 
 toolchain(
     name = "ll_toolchain",
-    exec_compatible_with = [":ll_exec_linux"],
+    exec_compatible_with = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+        "@bazel_tools//tools/cpp:clang",
+    ],
     toolchain = ":ll_toolchain_x86_64-unknown-linux-gnu",
     toolchain_type = ":toolchain_type",
     visibility = ["//visibility:public"],

rbe/README.md

@@ -0,0 +1,28 @@
+<!-- vale alex.ProfanityUnlikely = NO -->
+# Remote execution toolchains
+
+Generated toolchains for `rules_java` and `rules_cc` compatible with the remote
+execution container.
+
+## Generating the sources
+
+Regenerating the toolchains requires a local container registry:
+
+```bash
+docker run -d -p 5000:5000 --restart=always --name registry registry:2
+```
+
+From the root directory of `rules_ll`, invoke the `rbegen` tool to build the
+remote execution image, copy it to the local container registry and regenerate
+the toolchain configurations:
+<!-- vale alex.ProfanityUnlikely = NO -->
+
+```bash
+rbegen
+```
+
+To stop the local container registry:
+
+```bash
+docker container stop registry && docker container rm -v registry
+```

rbe/default.nix

@@ -0,0 +1,22 @@
+{ pkgs, }:
+
+pkgs.buildGoModule rec {
+  pname = "bazel-toolchains";
+  version = "5.1.2";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "bazelbuild";
+    repo = "bazel-toolchains";
+    rev = "v${version}";
+    sha256 = "sha256-J1RFrDGBF7YR5O4D/kNNu6fkxImHpLR+fxhp+R1MaGE=";
+  };
+
+  vendorSha256 = "sha256-E6PylI2prXCXqOUYgYi5nZ4qptqOqbcaOquDfEkhaQ4=";
+
+  meta = with pkgs.lib; {
+    description = "Generate Bazel toolchain configs for remote execution.";
+    homepage = "https://github.com/bazelbuild/bazel-toolchains";
+    license = licenses.asl20;
+    maintainers = [ maintainers.aaronmondal ];
+  };
+}