⬆️ Bump LRE toolchains

This adapts the new overlay logic.

.bazelversion

@@ -1 +1 @@
-8.0.0-pre.20240516.1
+8.0.0

MODULE.bazel

@@ -8,15 +8,19 @@ register_toolchains("@rules_ll//ll:ll_toolchain")
 
 # Platform support.
 bazel_dep(name = "platforms", version = "0.0.10")
-bazel_dep(name = "rules_cc", version = "0.0.9")
+bazel_dep(name = "rules_cc", version = "0.0.17")
 
 # Various utility functions such as path manipulations and templating.
-bazel_dep(name = "bazel_skylib", version = "1.7.0")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 
 # Documentation. These should be dev_dependencies, but that doesn't work at the
 # moment. This is a bug.
 bazel_dep(name = "rules_java", version = "7.6.1", dev_dependency = False)
-bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = False)
+bazel_dep(name = "stardoc", version = "0.7.2", dev_dependency = False)
+
+# TODO(aaronmondal): We don't actually use this. Fix LRE upstream to make this
+#                    import optional.
+bazel_dep(name = "rules_rust", version = "0.56.0")
 
 # The LLVM project. We override the specific commit below.
 bazel_dep(name = "llvm-project-overlay", version = "17-init-bcr.3")

devtools/ll.nix

@@ -33,7 +33,7 @@ elif [[ "$1" == "module" ]]; then
 elif [[ "$1" == "up" ]]; then
   ${ll_up}/bin/up
 elif [[ "$1" == "down" ]]; then
-  native down
+  ${native}/bin/native down
 else
 
 printf '

devtools/up.nix

@@ -3,13 +3,14 @@
 let
   # Packages needed for the script
   kubectl = pkgs.kubectl;
+  flux = pkgs.fluxcd;
   curl = pkgs.curl;
   git = pkgs.git;
   kustomize = pkgs.kustomize;
   nix = pkgs.nix;
 
   # The specific commit to use
-  nativelinkCommit = "481226be52a84ad5a6b990cc48e9f97512d8ccd2";
+  nativelinkCommit = "b1df876fd64d60d5d1b6cb15a50e934923ab82bf";
 
   # Base URL for GitHub access
   githubBaseUrl = "github:TraceMachina/nativelink/";
@@ -30,25 +31,65 @@ pkgs.writeShellScriptBin "up" ''
   ${kubectl}/bin/kubectl apply -k \
     https://github.com/TraceMachina/nativelink//deploy/kubernetes-example?ref=${nativelinkCommit}
 
-  # Wait for PipelineRuns to start
-  until ${kubectl}/bin/kubectl get pipelinerun \
-      -l tekton.dev/pipeline=rebuild-nativelink | grep -q 'NAME'; do
-    echo "Waiting for PipelineRuns to start..."
+  # Wait for Tekton
+  ${flux}/bin/flux reconcile kustomization -n default \
+    --timeout=15m \
+    nativelink-tekton-resources
+
+  # Wait for Flux Alerts
+  ${flux}/bin/flux reconcile kustomization -n default \
+      --timeout=15m \
+      nativelink-alert-core && \
+    ${flux}/bin/flux reconcile kustomization -n default \
+      --timeout=15m \
+      nativelink-alert-worker-init && \
+    ${flux}/bin/flux reconcile kustomization -n default \
+      --timeout=15m \
+      nativelink-alert-lre-cc
+
+  ${kubectl}/bin/kubectl apply -f - << 'EOF'
+  apiVersion: source.toolkit.fluxcd.io/v1
+  kind: GitRepository
+  metadata:
+    name: dummy-repository
+    namespace: default
+  spec:
+    interval: 2m
+    url: https://github.com/TraceMachina/nativelink
+    ref:
+      branch: main
+  EOF
+
+  # Wait for PipelineRuns
+  until pr=$(${kubectl}/bin/kubectl get pipelinerun -o name | \
+             grep rebuild-nativelink-run-); do
+    echo "Waiting for pipeline to be created..."
     sleep 1
   done
 
+  echo "Found pipeline: $pr"
+  ${kubectl}/bin/kubectl wait --for=create $pr
+
   # Wait for the pipeline to succeed
   ${kubectl}/bin/kubectl wait \
     --for=condition=Succeeded \
-    --timeout=30m \
+    --timeout=45m \
     pipelinerun \
-        -l tekton.dev/pipeline=rebuild-nativelink
+    -l tekton.dev/pipeline=rebuild-nativelink
 
-  # TODO(aaronmondal): Find a better solution.
-  sleep 10
+  # Wait for NativeLink Kustomization
+  ${flux}/bin/flux reconcile kustomization -n default \
+    --timeout=15m \
+    nativelink-core
+
+  # Wait for worker Kustomization
+  ${flux}/bin/flux reconcile kustomization -n default \
+    --timeout=15m \
+    nativelink-lre-cc
 
   # Monitor the deployment rollout
-  ${kubectl}/bin/kubectl rollout status deploy/nativelink-cas
-  ${kubectl}/bin/kubectl rollout status deploy/nativelink-scheduler
+  ${kubectl}/bin/kubectl rollout status deploy/nativelink
   ${kubectl}/bin/kubectl rollout status deploy/nativelink-worker-lre-cc
+
+  echo "nativelink_ip=$(kubectl get gtw nativelink-gateway -o=jsonpath='{.status.addresses[0].value}')"
 ''

examples/.bazelversion

@@ -1 +1 @@
-8.0.0-pre.20240516.1
+8.0.0

examples/MODULE.bazel

@@ -1,4 +1,4 @@
-bazel_dep(name = "bazel_skylib", version = "1.7.0")
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
 bazel_dep(name = "rules_ll", version = "0")
 local_path_override(
     module_name = "rules_ll",

examples/flake.nix

@@ -23,7 +23,7 @@
       inputs.flake-utils.follows = "flake-utils";
     };
     nativelink = {
-      url = "github:TraceMachina/nativelink/481226be52a84ad5a6b990cc48e9f97512d8ccd2";
+      url = "github:TraceMachina/nativelink/b1df876fd64d60d5d1b6cb15a50e934923ab82bf";
       inputs.flake-utils.follows = "flake-utils";
       inputs.flake-parts.follows = "flake-parts";
       inputs.pre-commit-hooks.follows = "pre-commit-hooks";

flake.nix

@@ -20,7 +20,7 @@
     };
     nativelink = {
       # Note: Keep this commit in sync with the LRE commit in `ll/init.bzl`.
-      url = "github:TraceMachina/nativelink/481226be52a84ad5a6b990cc48e9f97512d8ccd2";
+      url = "github:TraceMachina/nativelink/b1df876fd64d60d5d1b6cb15a50e934923ab82bf";
 
       # This repository provides the autogenerated LRE toolchains which are
       # dependent on the nixpkgs version in the nativelink repository. To keep
@@ -69,12 +69,16 @@
           {
             _module.args.pkgs = import self.inputs.nixpkgs {
               inherit system;
+              overlays = [
+                nativelink.overlays.lre
+                (import ./patches/nixpkgs-disable-ratehammering-pulumi-tests.nix)
+              ];
               # CUDA support
               config.allowUnfree = true;
               config.cudaSupport = true;
             };
             local-remote-execution.settings = {
-              inherit (nativelink.packages.${system}.lre-cc.meta) Env;
+              inherit (pkgs.lre.lre-cc.meta) Env;
             };
             pre-commit.settings = {
               hooks = import ./pre-commit-hooks.nix { inherit pkgs; };
@@ -92,11 +96,12 @@
               # TODO(aaronmondal): The nativelink devcluster mounts the current
               # git repository into the kind nodes and derives the lre-cc worker
               # tag from this target. Consider changing this upstream.
-              lre-cc = nativelink.packages.${system}.lre-cc;
+              lre-cc = pkgs.lre.lre-cc;
               ll = import ./devtools/ll.nix {
                 inherit pkgs;
                 native = inputs.nativelink.packages.${system}.native-cli;
               };
+              inherit (inputs.nativelink.packages.${system}) nativelink-worker-lre-cc;
             };
             devShells.default = pkgs.mkShell {
               nativeBuildInputs =

ll/init.bzl

@@ -17,11 +17,11 @@ def _initialize_rules_ll_impl(_):
     http_archive(
         name = "local-remote-execution",
         urls = [
-            "https://github.com/TraceMachina/nativelink/archive/481226be52a84ad5a6b990cc48e9f97512d8ccd2.zip",
+            "https://github.com/TraceMachina/nativelink/archive/b1df876fd64d60d5d1b6cb15a50e934923ab82bf.zip",
         ],
-        integrity = "sha256-W65Ai9H1iuinfTVga/hvZegQGbSeLzqyEmrWXNOHpwE=",
+        integrity = "sha256-5q2yanUApHq7Gb9TVG4qKA33WcccldmvdXv4RPO2Udc=",
         # Note: Keep this in sync with `flake.nix` and `devtools/up.sh`.
-        strip_prefix = "nativelink-481226be52a84ad5a6b990cc48e9f97512d8ccd2/local-remote-execution",
+        strip_prefix = "nativelink-b1df876fd64d60d5d1b6cb15a50e934923ab82bf/local-remote-execution",
     )
 
     http_archive(

patches/nixpkgs-disable-ratehammering-pulumi-tests.nix

@@ -0,0 +1,46 @@
+_final: prev: {
+  pulumi = prev.pulumi.overrideAttrs (_old: {
+    version = "3.137.0";
+    src = prev.fetchFromGitHub {
+      owner = "pulumi";
+      repo = "pulumi";
+      rev = "v3.137.0";
+      hash = "sha256-U1BubJIqQu+tAYQz8ojVrJpx6ZbMFEarSCmmuahG6ys=";
+      name = "pulumi"; # Some tests rely on checkout directory name
+    };
+    vendorHash = "sha256-Q9NTlgd+rOD5vmbmeAepIARvFtNQT/IFLPeaFC74E7c=";
+
+    # Remove old patch that's no longer needed
+    patches = [ ];
+
+    preCheck =
+      ''
+        # The tests require `version.Version` to be unset
+        ldflags=''${ldflags//"$importpathFlags"/}
+        # Create some placeholders for plugins used in tests
+        for name in pulumi-{resource-pkg{A,B},-pkgB}; do
+          ln -s ${prev.coreutils}/bin/true "$name"
+        done
+        # Code generation tests also download dependencies from network
+        rm codegen/{docs,dotnet,go,nodejs,python,schema}/*_test.go
+        # Azure tests are not being skipped when an Azure token is not provided
+        rm secrets/cloud/azure_test.go
+        rm -R codegen/{dotnet,go,nodejs,python}/gen_program_test
+        unset PULUMI_ACCESS_TOKEN
+        export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=true
+      ''
+      + prev.lib.optionalString prev.stdenv.hostPlatform.isDarwin ''
+        export PULUMI_HOME=$(mktemp -d)
+      '';
+
+    doCheck = false;
+  });
+
+  pulumiPackages =
+    prev.pulumiPackages
+    // {
+      pulumi-language-go = prev.pulumiPackages.pulumi-language-go.overrideAttrs (_old: {
+        vendorHash = "sha256-Bk/JxFYBnd+bOlExJKGMIl2oUHPg3ViIVBqKBojoEm4=";
+      });
+    };
+}

third-party-overlays/rocr.BUILD.bazel

@@ -156,7 +156,7 @@ ll_library(
         "src/core/runtime/amd_blit_shaders_v2.h",
         "src/loader/executable.hpp",
         "src/loader/AMDHSAKernelDescriptor.h",
-    ]),
+    ], allow_empty = True),
     compile_flags = HSA_CXX_FLAGS,
     compile_string_flags = ["@rules_ll//ll:LL_AMD_INCLUDES"],
     defines = CORE_RUNTIME_TARGET_DEFINES + [