Rollup merge of #98526 - jyn514:download-llvm-outside-checkout, r=Mar…

…k-Simulacrum

Allow using `download-ci-llvm = true` outside the git checkout

`@bjorn3` noticed that this is already allowed today when download-llvm is disabled, but breaks with it enabled:
```
$ ./rust2/x.py build
fatal: not a git repository (or any of the parent directories): .git
thread 'main' panicked at 'command did not execute successfully: "git" "rev-list" "[email protected]" "-n1" "--first-parent" "HEAD" "--" "/home/jnelson/rust-lang/rust2/src/llvm-project" "/home/jnelson/rust-lang/rust2/src/bootstrap/download-ci-llvm-stamp" "/home/jnelson/rust-lang/rust2/src/version"
expected success, got: exit status: 128', src/bootstrap/native.rs:134:20
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
```

Support it too for consistency. It's unclear to me when anyone would need to use this, but `@bjorn3`
feels we should support it, and it's not much additional effort to get it working.

src/bootstrap/config.rs

@@ -1128,11 +1128,7 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit = download_ci_rustc_commit(
-                &config.stage0_metadata,
-                rust.download_rustc,
-                config.verbose > 0,
-            );
+            config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc);
         } else {
             config.rust_profile_use = flags.rust_profile_use;
             config.rust_profile_generate = flags.rust_profile_generate;
@@ -1304,6 +1300,15 @@ impl Config {
         config
     }
 
+    /// A git invocation which runs inside the source directory.
+    ///
+    /// Use this rather than `Command::new("git")` in order to support out-of-tree builds.
+    pub(crate) fn git(&self) -> Command {
+        let mut git = Command::new("git");
+        git.current_dir(&self.src);
+        git
+    }
+
     /// Try to find the relative path of `bindir`, otherwise return it in full.
     pub fn bindir_relative(&self) -> &Path {
         let bindir = &self.bindir;
@@ -1453,9 +1458,8 @@ fn threads_from_config(v: u32) -> u32 {
 
 /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
 fn download_ci_rustc_commit(
-    stage0_metadata: &Stage0Metadata,
+    config: &Config,
     download_rustc: Option<StringOrBool>,
-    verbose: bool,
 ) -> Option<String> {
     // If `download-rustc` is not set, default to rebuilding.
     let if_unchanged = match download_rustc {
@@ -1468,17 +1472,18 @@ fn download_ci_rustc_commit(
     };
 
     // Handle running from a directory other than the top level
-    let top_level = output(Command::new("git").args(&["rev-parse", "--show-toplevel"]));
+    let top_level = output(config.git().args(&["rev-parse", "--show-toplevel"]));
     let top_level = top_level.trim_end();
     let compiler = format!("{top_level}/compiler/");
     let library = format!("{top_level}/library/");
 
     // Look for a version to compare to based on the current commit.
     // Only commits merged by bors will have CI artifacts.
     let merge_base = output(
-        Command::new("git")
+        config
+            .git()
             .arg("rev-list")
-            .arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
+            .arg(format!("--author={}", config.stage0_metadata.config.git_merge_commit_email))
             .args(&["-n1", "--first-parent", "HEAD"]),
     );
     let commit = merge_base.trim_end();
@@ -1491,13 +1496,14 @@ fn download_ci_rustc_commit(
     }
 
     // Warn if there were changes to the compiler or standard library since the ancestor commit.
-    let has_changes = !t!(Command::new("git")
+    let has_changes = !t!(config
+        .git()
         .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
         .status())
     .success();
     if has_changes {
         if if_unchanged {
-            if verbose {
+            if config.verbose > 0 {
                 println!(
                     "warning: saw changes to compiler/ or library/ since {commit}; \
                           ignoring `download-rustc`"

src/bootstrap/format.rs

@@ -72,7 +72,9 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) {
         Err(_) => false,
     };
     if git_available {
-        let in_working_tree = match Command::new("git")
+        let in_working_tree = match build
+            .config
+            .git()
             .arg("rev-parse")
             .arg("--is-inside-work-tree")
             .stdout(Stdio::null())
@@ -84,10 +86,7 @@ pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) {
         };
         if in_working_tree {
             let untracked_paths_output = output(
-                Command::new("git")
-                    .arg("status")
-                    .arg("--porcelain")
-                    .arg("--untracked-files=normal"),
+                build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"),
             );
             let untracked_paths = untracked_paths_output
                 .lines()

src/bootstrap/lib.rs

@@ -644,7 +644,8 @@ impl Build {
             return;
         }
         let output = output(
-            Command::new("git")
+            self.config
+                .git()
                 .args(&["config", "--file"])
                 .arg(&self.config.src.join(".gitmodules"))
                 .args(&["--get-regexp", "path"]),
@@ -1281,12 +1282,12 @@ impl Build {
         // That's our beta number!
         // (Note that we use a `..` range, not the `...` symmetric difference.)
         let count = output(
-            Command::new("git")
+            self.config
+                .git()
                 .arg("rev-list")
                 .arg("--count")
                 .arg("--merges")
-                .arg("refs/remotes/origin/master..HEAD")
-                .current_dir(&self.src),
+                .arg("refs/remotes/origin/master..HEAD"),
         );
         let n = count.trim().parse().unwrap();
         self.prerelease_version.set(Some(n));

src/bootstrap/native.rs

@@ -118,7 +118,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
     if !config.llvm_from_ci {
         return;
     }
-    let mut rev_list = Command::new("git");
+    let mut rev_list = config.git();
     rev_list.args(&[
         PathBuf::from("rev-list"),
         format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),

src/bootstrap/setup.rs

@@ -136,7 +136,7 @@ pub fn setup(config: &Config, profile: Profile) {
 
     println!();
 
-    t!(install_git_hook_maybe(&config.src));
+    t!(install_git_hook_maybe(&config));
 
     println!();
 
@@ -302,7 +302,7 @@ pub fn interactive_path() -> io::Result<Profile> {
 }
 
 // install a git hook to automatically run tidy --bless, if they want
-fn install_git_hook_maybe(src_path: &Path) -> io::Result<()> {
+fn install_git_hook_maybe(config: &Config) -> io::Result<()> {
     let mut input = String::new();
     println!(
         "Rust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality.
@@ -328,13 +328,12 @@ undesirable, simply delete the `pre-push` file from .git/hooks."
     };
 
     if should_install {
-        let src = src_path.join("src").join("etc").join("pre-push.sh");
-        let git = t!(Command::new("git").args(&["rev-parse", "--git-common-dir"]).output().map(
-            |output| {
+        let src = config.src.join("src").join("etc").join("pre-push.sh");
+        let git =
+            t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| {
                 assert!(output.status.success(), "failed to run `git`");
                 PathBuf::from(t!(String::from_utf8(output.stdout)).trim())
-            }
-        ));
+            }));
         let dst = git.join("hooks").join("pre-push");
         match fs::hard_link(src, &dst) {
             Err(e) => eprintln!(

src/bootstrap/toolstate.rs

@@ -1,5 +1,6 @@
 use crate::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::util::t;
+use crate::Config;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::env;
@@ -96,14 +97,9 @@ fn print_error(tool: &str, submodule: &str) {
     std::process::exit(3);
 }
 
-fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>) {
+fn check_changed_files(toolstates: &HashMap<Box<str>, ToolState>, config: &Config) {
     // Changed files
-    let output = std::process::Command::new("git")
-        .arg("diff")
-        .arg("--name-status")
-        .arg("HEAD")
-        .arg("HEAD^")
-        .output();
+    let output = config.git().arg("diff").arg("--name-status").arg("HEAD").arg("HEAD^").output();
     let output = match output {
         Ok(o) => o,
         Err(e) => {
@@ -182,8 +178,8 @@ impl Step for ToolStateCheck {
             std::process::exit(1);
         }
 
-        check_changed_files(&toolstates);
-        checkout_toolstate_repo();
+        check_changed_files(&toolstates, &builder.config);
+        checkout_toolstate_repo(&builder.config);
         let old_toolstate = read_old_toolstate();
 
         for (tool, _) in STABLE_TOOLS.iter() {
@@ -229,7 +225,7 @@ impl Step for ToolStateCheck {
         }
 
         if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() {
-            commit_toolstate_change(&toolstates);
+            commit_toolstate_change(&toolstates, &builder.config);
         }
     }
 
@@ -302,16 +298,17 @@ fn toolstate_repo() -> String {
 const TOOLSTATE_DIR: &str = "rust-toolstate";
 
 /// Checks out the toolstate repo into `TOOLSTATE_DIR`.
-fn checkout_toolstate_repo() {
+fn checkout_toolstate_repo(config: &Config) {
     if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") {
-        prepare_toolstate_config(&token);
+        prepare_toolstate_config(&token, config);
     }
     if Path::new(TOOLSTATE_DIR).exists() {
         eprintln!("Cleaning old toolstate directory...");
         t!(fs::remove_dir_all(TOOLSTATE_DIR));
     }
 
-    let status = Command::new("git")
+    let status = config
+        .git()
         .arg("clone")
         .arg("--depth=1")
         .arg(toolstate_repo())
@@ -327,17 +324,17 @@ fn checkout_toolstate_repo() {
 }
 
 /// Sets up config and authentication for modifying the toolstate repo.
-fn prepare_toolstate_config(token: &str) {
-    fn git_config(key: &str, value: &str) {
-        let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status();
+fn prepare_toolstate_config(token: &str, config: &Config) {
+    let git_config = |key: &str, value: &str| {
+        let status = config.git().arg("config").arg("--global").arg(key).arg(value).status();
         let success = match status {
             Ok(s) => s.success(),
             Err(_) => false,
         };
         if !success {
             panic!("git config key={} value={} failed (status: {:?})", key, value, status);
         }
-    }
+    };
 
     // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date
     // as well.
@@ -383,14 +380,14 @@ fn read_old_toolstate() -> Vec<RepoState> {
 ///
 ///       * See <https://help.github.com/articles/about-commit-email-addresses/>
 ///           if a private email by GitHub is wanted.
-fn commit_toolstate_change(current_toolstate: &ToolstateData) {
+fn commit_toolstate_change(current_toolstate: &ToolstateData, config: &Config) {
     let message = format!("({} CI update)", OS.expect("linux/windows only"));
     let mut success = false;
     for _ in 1..=5 {
         // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo.
         // This does *not* change the "current toolstate"; that only happens post-landing
         // via `src/ci/docker/publish_toolstate.sh`.
-        publish_test_results(&current_toolstate);
+        publish_test_results(&current_toolstate, config);
 
         // `git commit` failing means nothing to commit.
         let status = t!(Command::new("git")
@@ -444,8 +441,8 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData) {
 /// These results will later be promoted to `latest.json` by the
 /// `publish_toolstate.py` script if the PR passes all tests and is merged to
 /// master.
-fn publish_test_results(current_toolstate: &ToolstateData) {
-    let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output());
+fn publish_test_results(current_toolstate: &ToolstateData, config: &Config) {
+    let commit = t!(config.git().arg("rev-parse").arg("HEAD").output());
     let commit = t!(String::from_utf8(commit.stdout));
 
     let toolstate_serialized = t!(serde_json::to_string(&current_toolstate));