From 8c506f9dae5664fb04bf070f1422df5308d90cb6 Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Fri, 9 Feb 2018 13:40:23 -0700 Subject: [PATCH 1/5] Implement excluding a build-step via --exclude --- src/bootstrap/builder.rs | 31 ++++++++++++++++++++++++++----- src/bootstrap/config.rs | 2 ++ src/bootstrap/flags.rs | 7 +++++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 74dd4a6fa0144..b65c20e398e8c 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -119,7 +119,28 @@ impl StepDescription { } } - fn maybe_run(&self, builder: &Builder, path: Option<&Path>) { + fn maybe_run(&self, builder: &Builder, should_run: &ShouldRun, path: Option<&Path>) { + if let Some(path) = path { + if builder.config.exclude.iter().any(|e| e == path) { + eprintln!("Skipping {:?} because this path is excluded", path); + return; + } else if !builder.config.exclude.is_empty() { + eprintln!("{:?} not skipped -- not in {:?}", path, builder.config.exclude); + } + } else { + if !should_run.paths.is_empty() { + if should_run.paths.iter().all(|p| builder.config.exclude.contains(&p)) { + eprintln!("Skipping because all of its paths ({:?}) are excluded", + should_run.paths); + return; + } else if should_run.paths.len() > 1 { + for path in &should_run.paths { + self.maybe_run(builder, should_run, Some(path)); + } + return; + } + } + } let build = builder.build; let hosts = if self.only_build_targets || self.only_build { build.build_triple() @@ -160,7 +181,7 @@ impl StepDescription { if paths.is_empty() { for (desc, should_run) in v.iter().zip(should_runs) { if desc.default && should_run.is_really_default { - desc.maybe_run(builder, None); + desc.maybe_run(builder, &should_run, None); } } } else { @@ -169,7 +190,7 @@ impl StepDescription { for (desc, should_run) in v.iter().zip(&should_runs) { if should_run.run(path) { attempted_run = true; - desc.maybe_run(builder, Some(path)); + desc.maybe_run(builder, &should_run, Some(path)); } } @@ -208,13 +229,13 @@ impl<'a> ShouldRun<'a> { pub fn krate(mut self, name: &str) -> Self { for (_, krate_path) in self.builder.crates(name) { - self.paths.insert(PathBuf::from(krate_path)); + self.paths.insert(t!(env::current_dir()).join(krate_path)); } self } pub fn path(mut self, path: &str) -> Self { - self.paths.insert(PathBuf::from(path)); + self.paths.insert(t!(env::current_dir()).join(path)); self } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 4f4fd14ae8cab..812ca6d64fb6a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -56,6 +56,7 @@ pub struct Config { pub sanitizers: bool, pub profiler: bool, pub ignore_git: bool, + pub exclude: Vec, pub run_host_only: bool, @@ -311,6 +312,7 @@ impl Config { let flags = Flags::parse(&args); let file = flags.config.clone(); let mut config = Config::default(); + config.exclude = flags.exclude; config.llvm_enabled = true; config.llvm_optimize = true; config.llvm_version_check = true; diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 478e496078add..465ebf846d3dd 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -42,6 +42,7 @@ pub struct Flags { pub jobs: Option, pub cmd: Subcommand, pub incremental: bool, + pub exclude: Vec, } pub enum Subcommand { @@ -109,6 +110,7 @@ To learn more about a subcommand, run `./x.py -h`"); opts.optopt("", "build", "build target of the stage0 compiler", "BUILD"); opts.optmulti("", "host", "host targets to build", "HOST"); opts.optmulti("", "target", "target targets to build", "TARGET"); + opts.optmulti("", "exclude", "build paths to exclude", "PATH"); opts.optopt("", "on-fail", "command to run on failure", "CMD"); opts.optopt("", "stage", "stage to build", "N"); opts.optopt("", "keep-stage", "stage to keep without recompiling", "N"); @@ -358,10 +360,9 @@ Arguments: stage = Some(1); } - let cwd = t!(env::current_dir()); let src = matches.opt_str("src").map(PathBuf::from) .or_else(|| env::var_os("SRC").map(PathBuf::from)) - .unwrap_or(cwd); + .unwrap_or(cwd.clone()); Flags { verbose: matches.opt_count("verbose"), @@ -378,6 +379,8 @@ Arguments: jobs: matches.opt_str("jobs").map(|j| j.parse().unwrap()), cmd, incremental: matches.opt_present("incremental"), + exclude: split(matches.opt_strs("exclude")) + .into_iter().map(|p| cwd.join(p)).collect::>(), } } } From 11580b7be3a830ccea569b64545ec77008c452ac Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Sat, 10 Feb 2018 18:15:34 -0700 Subject: [PATCH 2/5] Make exclude and paths relative to root of checkout --- src/bootstrap/flags.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 465ebf846d3dd..8a38fedc6136d 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -275,7 +275,12 @@ Arguments: }; // Get any optional paths which occur after the subcommand let cwd = t!(env::current_dir()); - let paths = matches.free[1..].iter().map(|p| cwd.join(p)).collect::>(); + let src = matches.opt_str("src").map(PathBuf::from) + .or_else(|| env::var_os("SRC").map(PathBuf::from)) + .unwrap_or(cwd.clone()); + let paths = matches.free[1..].iter().map(|p| { + cwd.join(p).strip_prefix(&src).expect("paths passed to be inside checkout").into() + }).collect::>(); let cfg_file = matches.opt_str("config").map(PathBuf::from).or_else(|| { if fs::metadata("config.toml").is_ok() { @@ -360,10 +365,6 @@ Arguments: stage = Some(1); } - let src = matches.opt_str("src").map(PathBuf::from) - .or_else(|| env::var_os("SRC").map(PathBuf::from)) - .unwrap_or(cwd.clone()); - Flags { verbose: matches.opt_count("verbose"), stage, @@ -375,12 +376,14 @@ Arguments: target: split(matches.opt_strs("target")) .into_iter().map(|x| INTERNER.intern_string(x)).collect::>(), config: cfg_file, - src, jobs: matches.opt_str("jobs").map(|j| j.parse().unwrap()), cmd, incremental: matches.opt_present("incremental"), exclude: split(matches.opt_strs("exclude")) - .into_iter().map(|p| cwd.join(p)).collect::>(), + .into_iter().map(|p| { + cwd.join(p).strip_prefix(&src).expect("paths to be inside checkout").into() + }).collect::>(), + src, } } } From 55c36e37443e6fa4252157d4bc47b49b3d6e201d Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Sat, 10 Feb 2018 18:18:41 -0700 Subject: [PATCH 3/5] Remove DontDistWithMiri struct and instead just directly check it in builder --- src/bootstrap/builder.rs | 9 +++++++-- src/bootstrap/dist.rs | 25 ------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b65c20e398e8c..32cc1d7223dbe 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -286,8 +286,7 @@ impl<'a> Builder<'a> { doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook), Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo, - dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign, - dist::DontDistWithMiriEnabled), + dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign), Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls, install::Rustfmt, install::Analysis, install::Src, install::Rustc), } @@ -344,6 +343,12 @@ impl<'a> Builder<'a> { stack: RefCell::new(Vec::new()), }; + if kind == Kind::Dist { + assert!(!build.config.test_miri, "Do not distribute with miri enabled.\n\ + The distributed libraries would include all MIR (increasing binary size). + The distributed MIR would include validation statements."); + } + StepDescription::run(&Builder::get_step_descriptions(builder.kind), &builder, paths); } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 460fb016f16ea..e7aed7eb4fead 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -1233,31 +1233,6 @@ impl Step for Rustfmt { } } - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct DontDistWithMiriEnabled; - -impl Step for DontDistWithMiriEnabled { - type Output = PathBuf; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun) -> ShouldRun { - let build_miri = run.builder.build.config.test_miri; - run.default_condition(build_miri) - } - - fn make_run(run: RunConfig) { - run.builder.ensure(DontDistWithMiriEnabled); - } - - fn run(self, _: &Builder) -> PathBuf { - panic!("Do not distribute with miri enabled.\n\ - The distributed libraries would include all MIR (increasing binary size). - The distributed MIR would include validation statements."); - } -} - - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Extended { stage: u32, From f104b120595d21e8aef311bc0057b3f854cddfc0 Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Sun, 11 Feb 2018 09:51:58 -0700 Subject: [PATCH 4/5] Change Step to be invoked with a path when in default mode. Previously, a Step would be able to tell on its own when it was invoked "by-default" (that is, `./x.py test` was called instead of `./x.py test some/path`). This commit replaces that functionality, invoking each Step with each of the paths it has specified as "should be invoked by." For example, if a step calls `path("src/tools/cargo")` and `path("src/doc/cargo")` then it's make_run will be called twice, with "src/tools/cargo" and "src/doc/cargo." This makes it so that default handling logic is in builder, instead of spread across various Steps. However, this meant that some Step specifications needed to be updated, since for example `rustdoc` can be built by `./x.py build src/librustdoc` or `./x.py build src/tools/rustdoc`. A `PathSet` abstraction is added that handles this: now, each Step can not only list `path(...)` but also `paths(&[a, b, ...])` which will make it so that we don't invoke it with each of the individual paths, instead invoking it with the first path in the list (though this shouldn't be depended on). Future work likely consists of implementing a better/easier way for a given Step to work with "any" crate in-tree, especially those that want to run tests, build, or check crates in the std, test, or rustc crate trees. Currently this is rather painful to do as most of the logic is duplicated across should_run and make_run. It seems likely this can be abstracted away into builder somehow. --- src/bootstrap/builder.rs | 125 ++++++--- src/bootstrap/check.rs | 6 +- src/bootstrap/compile.rs | 10 +- src/bootstrap/doc.rs | 2 +- src/bootstrap/lib.rs | 37 +-- src/bootstrap/native.rs | 4 +- src/bootstrap/test.rs | 543 +++++++++++++++++++++------------------ 7 files changed, 415 insertions(+), 312 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 32cc1d7223dbe..cbd303e114066 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -95,7 +95,7 @@ pub struct RunConfig<'a> { pub builder: &'a Builder<'a>, pub host: Interned, pub target: Interned, - pub path: Option<&'a Path>, + pub path: &'a Path, } struct StepDescription { @@ -105,6 +105,28 @@ struct StepDescription { only_build: bool, should_run: fn(ShouldRun) -> ShouldRun, make_run: fn(RunConfig), + name: &'static str, +} + +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] +struct PathSet { + set: BTreeSet, +} + +impl PathSet { + fn one>(path: P) -> PathSet { + let mut set = BTreeSet::new(); + set.insert(path.into()); + PathSet { set } + } + + fn has(&self, needle: &Path) -> bool { + self.set.iter().any(|p| p.ends_with(needle)) + } + + fn path(&self) -> &Path { + self.set.iter().next().unwrap() + } } impl StepDescription { @@ -116,30 +138,17 @@ impl StepDescription { only_build: S::ONLY_BUILD, should_run: S::should_run, make_run: S::make_run, + name: unsafe { ::std::intrinsics::type_name::() }, } } - fn maybe_run(&self, builder: &Builder, should_run: &ShouldRun, path: Option<&Path>) { - if let Some(path) = path { - if builder.config.exclude.iter().any(|e| e == path) { - eprintln!("Skipping {:?} because this path is excluded", path); - return; - } else if !builder.config.exclude.is_empty() { - eprintln!("{:?} not skipped -- not in {:?}", path, builder.config.exclude); - } - } else { - if !should_run.paths.is_empty() { - if should_run.paths.iter().all(|p| builder.config.exclude.contains(&p)) { - eprintln!("Skipping because all of its paths ({:?}) are excluded", - should_run.paths); - return; - } else if should_run.paths.len() > 1 { - for path in &should_run.paths { - self.maybe_run(builder, should_run, Some(path)); - } - return; - } - } + fn maybe_run(&self, builder: &Builder, pathset: &PathSet) { + if builder.config.exclude.iter().any(|e| pathset.has(e)) { + eprintln!("Skipping {:?} because it is excluded", pathset); + return; + } else if !builder.config.exclude.is_empty() { + eprintln!("{:?} not skipped for {:?} -- not in {:?}", pathset, + self.name, builder.config.exclude); } let build = builder.build; let hosts = if self.only_build_targets || self.only_build { @@ -165,7 +174,7 @@ impl StepDescription { for target in targets { let run = RunConfig { builder, - path, + path: pathset.path(), host: *host, target: *target, }; @@ -178,19 +187,28 @@ impl StepDescription { let should_runs = v.iter().map(|desc| { (desc.should_run)(ShouldRun::new(builder)) }).collect::>(); + + // sanity checks on rules + for (desc, should_run) in v.iter().zip(&should_runs) { + assert!(!should_run.paths.is_empty(), + "{:?} should have at least one pathset", desc.name); + } + if paths.is_empty() { for (desc, should_run) in v.iter().zip(should_runs) { if desc.default && should_run.is_really_default { - desc.maybe_run(builder, &should_run, None); + for pathset in &should_run.paths { + desc.maybe_run(builder, pathset); + } } } } else { for path in paths { let mut attempted_run = false; for (desc, should_run) in v.iter().zip(&should_runs) { - if should_run.run(path) { + if let Some(pathset) = should_run.pathset_for_path(path) { attempted_run = true; - desc.maybe_run(builder, &should_run, Some(path)); + desc.maybe_run(builder, pathset); } } @@ -206,7 +224,7 @@ impl StepDescription { pub struct ShouldRun<'a> { pub builder: &'a Builder<'a>, // use a BTreeSet to maintain sort order - paths: BTreeSet, + paths: BTreeSet, // If this is a default rule, this is an additional constraint placed on // it's run. Generally something like compiler docs being enabled. @@ -227,15 +245,35 @@ impl<'a> ShouldRun<'a> { self } + // Unlike `krate` this will create just one pathset. As such, it probably shouldn't actually + // ever be used, but as we transition to having all rules properly handle passing krate(...) by + // actually doing something different for every crate passed. + pub fn all_krates(mut self, name: &str) -> Self { + let mut set = BTreeSet::new(); + for krate in self.builder.in_tree_crates(name) { + set.insert(PathBuf::from(&krate.path)); + } + self.paths.insert(PathSet { set }); + self + } + pub fn krate(mut self, name: &str) -> Self { - for (_, krate_path) in self.builder.crates(name) { - self.paths.insert(t!(env::current_dir()).join(krate_path)); + for krate in self.builder.in_tree_crates(name) { + self.paths.insert(PathSet::one(&krate.path)); } self } - pub fn path(mut self, path: &str) -> Self { - self.paths.insert(t!(env::current_dir()).join(path)); + // single, non-aliased path + pub fn path(self, path: &str) -> Self { + self.paths(&[path]) + } + + // multiple aliases for the same job + pub fn paths(mut self, paths: &[&str]) -> Self { + self.paths.insert(PathSet { + set: paths.iter().map(PathBuf::from).collect(), + }); self } @@ -244,8 +282,8 @@ impl<'a> ShouldRun<'a> { self } - fn run(&self, path: &Path) -> bool { - self.paths.iter().any(|p| path.ends_with(p)) + fn pathset_for_path(&self, path: &Path) -> Option<&PathSet> { + self.paths.iter().find(|pathset| pathset.has(path)) } } @@ -275,11 +313,16 @@ impl<'a> Builder<'a> { tool::RustInstaller, tool::Cargo, tool::Rls, tool::Rustdoc, tool::Clippy, native::Llvm, tool::Rustfmt, tool::Miri), Kind::Check => describe!(check::Std, check::Test, check::Rustc), - Kind::Test => describe!(test::Tidy, test::Bootstrap, test::DefaultCompiletest, - test::HostCompiletest, test::Crate, test::CrateLibrustc, test::Rustdoc, - test::Linkcheck, test::Cargotest, test::Cargo, test::Rls, test::Docs, - test::ErrorIndex, test::Distcheck, test::Rustfmt, test::Miri, test::Clippy, - test::RustdocJS, test::RustdocTheme), + Kind::Test => describe!(test::Tidy, test::Bootstrap, test::Ui, test::RunPass, + test::CompileFail, test::ParseFail, test::RunFail, test::RunPassValgrind, + test::MirOpt, test::Codegen, test::CodegenUnits, test::Incremental, test::Debuginfo, + test::UiFullDeps, test::RunPassFullDeps, test::RunFailFullDeps, + test::CompileFailFullDeps, test::IncrementalFullDeps, test::Rustdoc, test::Pretty, + test::RunPassPretty, test::RunFailPretty, test::RunPassValgrindPretty, + test::RunPassFullDepsPretty, test::RunFailFullDepsPretty, test::RunMake, + test::Crate, test::CrateLibrustc, test::Rustdoc, test::Linkcheck, test::Cargotest, + test::Cargo, test::Rls, test::Docs, test::ErrorIndex, test::Distcheck, + test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme), Kind::Bench => describe!(test::Crate, test::CrateLibrustc), Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook, doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon, @@ -317,8 +360,10 @@ impl<'a> Builder<'a> { should_run = (desc.should_run)(should_run); } let mut help = String::from("Available paths:\n"); - for path in should_run.paths { - help.push_str(format!(" ./x.py {} {}\n", subcommand, path.display()).as_str()); + for pathset in should_run.paths { + for path in pathset.set { + help.push_str(format!(" ./x.py {} {}\n", subcommand, path.display()).as_str()); + } } Some(help) } diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index ede403491d7fc..767ee4016c6f1 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -26,7 +26,7 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/libstd").krate("std") + run.all_krates("std") } fn make_run(run: RunConfig) { @@ -67,7 +67,7 @@ impl Step for Rustc { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/librustc").krate("rustc-main") + run.all_krates("rustc-main") } fn make_run(run: RunConfig) { @@ -114,7 +114,7 @@ impl Step for Test { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/libtest").krate("test") + run.all_krates("test") } fn make_run(run: RunConfig) { diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 1d5e11c5d6d41..2dcc0e0e7cd9f 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -48,7 +48,7 @@ impl Step for Std { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/libstd").krate("std") + run.all_krates("std") } fn make_run(run: RunConfig) { @@ -320,7 +320,7 @@ impl Step for Test { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/libtest").krate("test") + run.all_krates("test") } fn make_run(run: RunConfig) { @@ -436,7 +436,7 @@ impl Step for Rustc { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/librustc").krate("rustc-main") + run.all_krates("rustc-main") } fn make_run(run: RunConfig) { @@ -593,7 +593,7 @@ impl Step for CodegenBackend { const DEFAULT: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/librustc_trans") + run.all_krates("rustc_trans") } fn make_run(run: RunConfig) { @@ -828,7 +828,7 @@ impl Step for Assemble { type Output = Compiler; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/rustc") + run.all_krates("rustc-main") } /// Prepare a new compiler from the artifacts in `stage` diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 6a75fc5112f5c..55d9723527e6d 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -429,7 +429,7 @@ impl Step for Std { fn should_run(run: ShouldRun) -> ShouldRun { let builder = run.builder; - run.krate("std").default_condition(builder.build.config.docs) + run.all_krates("std").default_condition(builder.build.config.docs) } fn make_run(run: RunConfig) { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 83c270865c0b7..afd740ce54845 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -113,9 +113,8 @@ //! More documentation can be found in each respective module below, and you can //! also check out the `src/bootstrap/README.md` file for more information. -#![deny(warnings)] -#![allow(stable_features)] -#![feature(associated_consts)] +//#![deny(warnings)] +#![feature(core_intrinsics)] #[macro_use] extern crate build_helper; @@ -267,6 +266,18 @@ struct Crate { bench_step: String, } +impl Crate { + fn is_local(&self, build: &Build) -> bool { + self.path.starts_with(&build.config.src) && + !self.path.to_string_lossy().ends_with("_shim") + } + + fn local_path(&self, build: &Build) -> PathBuf { + assert!(self.is_local(build)); + self.path.strip_prefix(&build.config.src).unwrap().into() + } +} + /// The various "modes" of invoking Cargo. /// /// These entries currently correspond to the various output directories of the @@ -949,22 +960,18 @@ impl Build { } } - /// Get a list of crates from a root crate. - /// - /// Returns Vec<(crate, path to crate, is_root_crate)> - fn crates(&self, root: &str) -> Vec<(Interned, &Path)> { - let interned = INTERNER.intern_string(root.to_owned()); + fn in_tree_crates(&self, root: &str) -> Vec<&Crate> { let mut ret = Vec::new(); - let mut list = vec![interned]; + let mut list = vec![INTERNER.intern_str(root)]; let mut visited = HashSet::new(); while let Some(krate) = list.pop() { let krate = &self.crates[&krate]; - // If we can't strip prefix, then out-of-tree path - let path = krate.path.strip_prefix(&self.src).unwrap_or(&krate.path); - ret.push((krate.name, path)); - for dep in &krate.deps { - if visited.insert(dep) && dep != "build_helper" { - list.push(*dep); + if krate.is_local(self) { + ret.push(krate); + for dep in &krate.deps { + if visited.insert(dep) && dep != "build_helper" { + list.push(*dep); + } } } } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 2ea026244034f..3c91cf3ecc7b5 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -51,9 +51,7 @@ impl Step for Llvm { } fn make_run(run: RunConfig) { - let emscripten = run.path.map(|p| { - p.ends_with("llvm-emscripten") - }).unwrap_or(false); + let emscripten = run.path.ends_with("llvm-emscripten"); run.builder.ensure(Llvm { target: run.target, emscripten, diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index f6b95f0bf9744..64ede4f4ecc88 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -13,7 +13,6 @@ //! This file implements the various regression test suites that we execute on //! our CI. -use std::collections::HashSet; use std::env; use std::ffi::OsString; use std::iter; @@ -26,6 +25,7 @@ use std::io::Read; use build_helper::{self, output}; use builder::{Kind, RunConfig, ShouldRun, Builder, Compiler, Step}; +use Crate as CargoCrate; use cache::{INTERNER, Interned}; use compile; use dist; @@ -550,181 +550,214 @@ fn testdir(build: &Build, host: Interned) -> PathBuf { build.out.join(host).join("test") } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct Test { - path: &'static str, - mode: &'static str, - suite: &'static str, +macro_rules! default_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); + } } -static DEFAULT_COMPILETESTS: &[Test] = &[ - Test { path: "src/test/ui", mode: "ui", suite: "ui" }, - Test { path: "src/test/run-pass", mode: "run-pass", suite: "run-pass" }, - Test { path: "src/test/compile-fail", mode: "compile-fail", suite: "compile-fail" }, - Test { path: "src/test/parse-fail", mode: "parse-fail", suite: "parse-fail" }, - Test { path: "src/test/run-fail", mode: "run-fail", suite: "run-fail" }, - Test { - path: "src/test/run-pass-valgrind", - mode: "run-pass-valgrind", - suite: "run-pass-valgrind" - }, - Test { path: "src/test/mir-opt", mode: "mir-opt", suite: "mir-opt" }, - Test { path: "src/test/codegen", mode: "codegen", suite: "codegen" }, - Test { path: "src/test/codegen-units", mode: "codegen-units", suite: "codegen-units" }, - Test { path: "src/test/incremental", mode: "incremental", suite: "incremental" }, - - // What this runs varies depending on the native platform being apple - Test { path: "src/test/debuginfo", mode: "debuginfo-XXX", suite: "debuginfo" }, -]; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct DefaultCompiletest { - compiler: Compiler, - target: Interned, - mode: &'static str, - suite: &'static str, +macro_rules! host_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); + } } -impl Step for DefaultCompiletest { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(mut run: ShouldRun) -> ShouldRun { - for test in DEFAULT_COMPILETESTS { - run = run.path(test.path); +macro_rules! test { + ($name:ident { + path: $path:expr, + mode: $mode:expr, + suite: $suite:expr, + default: $default:expr, + host: $host:expr + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: Interned, } - run - } - fn make_run(run: RunConfig) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $host; - let test = run.path.map(|path| { - DEFAULT_COMPILETESTS.iter().find(|&&test| { - path.ends_with(test.path) - }).unwrap_or_else(|| { - panic!("make_run in compile test to receive test path, received {:?}", path); - }) - }); - - if let Some(test) = test { - run.builder.ensure(DefaultCompiletest { - compiler, - target: run.target, - mode: test.mode, - suite: test.suite, - }); - } else { - for test in DEFAULT_COMPILETESTS { - run.builder.ensure(DefaultCompiletest { - compiler, - target: run.target, - mode: test.mode, - suite: test.suite - }); + fn should_run(run: ShouldRun) -> ShouldRun { + run.path($path) } - } - } - - fn run(self, builder: &Builder) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: self.mode, - suite: self.suite, - }) - } -} - -// Also default, but host-only. -static HOST_COMPILETESTS: &[Test] = &[ - Test { path: "src/test/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }, - Test { path: "src/test/run-pass-fulldeps", mode: "run-pass", suite: "run-pass-fulldeps" }, - Test { path: "src/test/run-fail-fulldeps", mode: "run-fail", suite: "run-fail-fulldeps" }, - Test { - path: "src/test/compile-fail-fulldeps", - mode: "compile-fail", - suite: "compile-fail-fulldeps", - }, - Test { - path: "src/test/incremental-fulldeps", - mode: "incremental", - suite: "incremental-fulldeps", - }, - Test { path: "src/test/rustdoc", mode: "rustdoc", suite: "rustdoc" }, - - Test { path: "src/test/pretty", mode: "pretty", suite: "pretty" }, - Test { path: "src/test/run-pass/pretty", mode: "pretty", suite: "run-pass" }, - Test { path: "src/test/run-fail/pretty", mode: "pretty", suite: "run-fail" }, - Test { path: "src/test/run-pass-valgrind/pretty", mode: "pretty", suite: "run-pass-valgrind" }, - Test { path: "src/test/run-pass-fulldeps/pretty", mode: "pretty", suite: "run-pass-fulldeps" }, - Test { path: "src/test/run-fail-fulldeps/pretty", mode: "pretty", suite: "run-fail-fulldeps" }, - Test { path: "src/test/run-make", mode: "run-make", suite: "run-make" }, -]; -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct HostCompiletest { - compiler: Compiler, - target: Interned, - mode: &'static str, - suite: &'static str, -} + fn make_run(run: RunConfig) { + let compiler = run.builder.compiler(run.builder.top_stage, run.host); -impl Step for HostCompiletest { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(mut run: ShouldRun) -> ShouldRun { - for test in HOST_COMPILETESTS { - run = run.path(test.path); - } - run - } - - fn make_run(run: RunConfig) { - let compiler = run.builder.compiler(run.builder.top_stage, run.host); - - let test = run.path.map(|path| { - HOST_COMPILETESTS.iter().find(|&&test| { - path.ends_with(test.path) - }).unwrap_or_else(|| { - panic!("make_run in compile test to receive test path, received {:?}", path); - }) - }); - - if let Some(test) = test { - run.builder.ensure(HostCompiletest { - compiler, - target: run.target, - mode: test.mode, - suite: test.suite, - }); - } else { - for test in HOST_COMPILETESTS { - if test.mode == "pretty" { - continue; - } - run.builder.ensure(HostCompiletest { + run.builder.ensure($name { compiler, target: run.target, - mode: test.mode, - suite: test.suite }); } - } - } - fn run(self, builder: &Builder) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: self.mode, - suite: self.suite, - }) + fn run(self, builder: &Builder) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: $mode, + suite: $suite, + }) + } + } } } +default_test!(Ui { + path: "src/test/ui", + mode: "ui", + suite: "ui" +}); + +default_test!(RunPass { + path: "src/test/run-pass", + mode: "run-pass", + suite: "run-pass" +}); + +default_test!(CompileFail { + path: "src/test/compile-fail", + mode: "compile-fail", + suite: "compile-fail" +}); + +default_test!(ParseFail { + path: "src/test/parse-fail", + mode: "parse-fail", + suite: "parse-fail" +}); + +default_test!(RunFail { + path: "src/test/run-fail", + mode: "run-fail", + suite: "run-fail" +}); + +default_test!(RunPassValgrind { + path: "src/test/run-pass-valgrind", + mode: "run-pass-valgrind", + suite: "run-pass-valgrind" +}); + +default_test!(MirOpt { + path: "src/test/mir-opt", + mode: "mir-opt", + suite: "mir-opt" +}); + +default_test!(Codegen { + path: "src/test/codegen", + mode: "codegen", + suite: "codegen" +}); + +default_test!(CodegenUnits { + path: "src/test/codegen-units", + mode: "codegen-units", + suite: "codegen-units" +}); + +default_test!(Incremental { + path: "src/test/incremental", + mode: "incremental", + suite: "incremental" +}); + +default_test!(Debuginfo { + path: "src/test/debuginfo", + // What this runs varies depending on the native platform being apple + mode: "debuginfo-XXX", + suite: "debuginfo" +}); + +host_test!(UiFullDeps { + path: "src/test/ui-fulldeps", + mode: "ui", + suite: "ui-fulldeps" +}); + +host_test!(RunPassFullDeps { + path: "src/test/run-pass-fulldeps", + mode: "run-pass", + suite: "run-pass-fulldeps" +}); + +host_test!(RunFailFullDeps { + path: "src/test/run-fail-fulldeps", + mode: "run-fail", + suite: "run-fail-fulldeps" +}); + +host_test!(CompileFailFullDeps { + path: "src/test/compile-fail-fulldeps", + mode: "compile-fail", + suite: "compile-fail-fulldeps" +}); + +host_test!(IncrementalFullDeps { + path: "src/test/incremental-fulldeps", + mode: "incremental", + suite: "incremental-fulldeps" +}); + +host_test!(Rustdoc { + path: "src/test/rustdoc", + mode: "rustdoc", + suite: "rustdoc" +}); + +test!(Pretty { + path: "src/test/pretty", + mode: "pretty", + suite: "pretty", + default: false, + host: true +}); +test!(RunPassPretty { + path: "src/test/run-pass/pretty", + mode: "pretty", + suite: "run-pass", + default: false, + host: true +}); +test!(RunFailPretty { + path: "src/test/run-fail/pretty", + mode: "pretty", + suite: "run-fail", + default: false, + host: true +}); +test!(RunPassValgrindPretty { + path: "src/test/run-pass-valgrind/pretty", + mode: "pretty", + suite: "run-pass-valgrind", + default: false, + host: true +}); +test!(RunPassFullDepsPretty { + path: "src/test/run-pass-fulldeps/pretty", + mode: "pretty", + suite: "run-pass-fulldeps", + default: false, + host: true +}); +test!(RunFailFullDepsPretty { + path: "src/test/run-fail-fulldeps/pretty", + mode: "pretty", + suite: "run-fail-fulldeps", + default: false, + host: true +}); + +host_test!(RunMake { + path: "src/test/run-make", + mode: "run-make", + suite: "run-make" +}); + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] struct Compiletest { compiler: Compiler, @@ -902,7 +935,7 @@ impl Step for Compiletest { } } if suite == "run-make" && !build.config.llvm_enabled { - println!("Ignoring run-make test suite as they generally don't work without LLVM"); + println!("Ignoring run-make test suite as they generally dont work without LLVM"); return; } @@ -1099,7 +1132,7 @@ pub struct CrateLibrustc { compiler: Compiler, target: Interned, test_kind: TestKind, - krate: Option>, + krate: Interned, } impl Step for CrateLibrustc { @@ -1115,35 +1148,26 @@ impl Step for CrateLibrustc { let builder = run.builder; let compiler = builder.compiler(builder.top_stage, run.host); - let make = |name: Option>| { - let test_kind = if builder.kind == Kind::Test { - TestKind::Test - } else if builder.kind == Kind::Bench { - TestKind::Bench - } else { - panic!("unexpected builder.kind in crate: {:?}", builder.kind); - }; - - builder.ensure(CrateLibrustc { - compiler, - target: run.target, - test_kind, - krate: name, - }); - }; + for krate in builder.in_tree_crates("rustc-main") { + if run.path.ends_with(&krate.path) { + let test_kind = if builder.kind == Kind::Test { + TestKind::Test + } else if builder.kind == Kind::Bench { + TestKind::Bench + } else { + panic!("unexpected builder.kind in crate: {:?}", builder.kind); + }; - if let Some(path) = run.path { - for (name, krate_path) in builder.crates("rustc-main") { - if path.ends_with(krate_path) { - make(Some(name)); - } + builder.ensure(CrateLibrustc { + compiler, + target: run.target, + test_kind, + krate: krate.name, + }); } - } else { - make(None); } } - fn run(self, builder: &Builder) { builder.ensure(Crate { compiler: self.compiler, @@ -1155,28 +1179,96 @@ impl Step for CrateLibrustc { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateNotDefault { + compiler: Compiler, + target: Interned, + test_kind: TestKind, + krate: &'static str, +} + +impl Step for CrateNotDefault { + type Output = (); + + fn should_run(run: ShouldRun) -> ShouldRun { + run.path("src/liballoc_jemalloc") + .path("src/librustc_asan") + .path("src/librustc_lsan") + .path("src/librustc_msan") + .path("src/librustc_tsan") + } + + fn make_run(run: RunConfig) { + let builder = run.builder; + let compiler = builder.compiler(builder.top_stage, run.host); + + let test_kind = if builder.kind == Kind::Test { + TestKind::Test + } else if builder.kind == Kind::Bench { + TestKind::Bench + } else { + panic!("unexpected builder.kind in crate: {:?}", builder.kind); + }; + + builder.ensure(CrateNotDefault { + compiler, + target: run.target, + test_kind, + krate: match run.path { + _ if run.path.ends_with("src/liballoc_jemalloc") => "alloc_jemalloc", + _ if run.path.ends_with("src/librustc_asan") => "rustc_asan", + _ if run.path.ends_with("src/librustc_lsan") => "rustc_lsan", + _ if run.path.ends_with("src/librustc_msan") => "rustc_msan", + _ if run.path.ends_with("src/librustc_tsan") => "rustc_tsan", + _ => panic!("unexpected path {:?}", run.path), + }, + }); + } + + fn run(self, builder: &Builder) { + builder.ensure(Crate { + compiler: self.compiler, + target: self.target, + mode: Mode::Libstd, + test_kind: self.test_kind, + krate: INTERNER.intern_str(self.krate), + }); + } +} + + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] pub struct Crate { compiler: Compiler, target: Interned, mode: Mode, test_kind: TestKind, - krate: Option>, + krate: Interned, } impl Step for Crate { type Output = (); const DEFAULT: bool = true; - fn should_run(run: ShouldRun) -> ShouldRun { - run.krate("std").krate("test") + fn should_run(mut run: ShouldRun) -> ShouldRun { + let builder = run.builder; + run = run.krate("test"); + for krate in run.builder.in_tree_crates("std") { + if krate.is_local(&run.builder) && + !krate.name.contains("jemalloc") && + !(krate.name.starts_with("rustc_") && krate.name.ends_with("san")) && + krate.name != "dlmalloc" { + run = run.path(krate.local_path(&builder).to_str().unwrap()); + } + } + run } fn make_run(run: RunConfig) { let builder = run.builder; let compiler = builder.compiler(builder.top_stage, run.host); - let make = |mode: Mode, name: Option>| { + let make = |mode: Mode, krate: &CargoCrate| { let test_kind = if builder.kind == Kind::Test { TestKind::Test } else if builder.kind == Kind::Bench { @@ -1190,29 +1282,24 @@ impl Step for Crate { target: run.target, mode, test_kind, - krate: name, + krate: krate.name, }); }; - if let Some(path) = run.path { - for (name, krate_path) in builder.crates("std") { - if path.ends_with(krate_path) { - make(Mode::Libstd, Some(name)); - } + for krate in builder.in_tree_crates("std") { + if run.path.ends_with(&krate.local_path(&builder)) { + make(Mode::Libstd, krate); } - for (name, krate_path) in builder.crates("test") { - if path.ends_with(krate_path) { - make(Mode::Libtest, Some(name)); - } + } + for krate in builder.in_tree_crates("test") { + if run.path.ends_with(&krate.local_path(&builder)) { + make(Mode::Libtest, krate); } - } else { - make(Mode::Libstd, None); - make(Mode::Libtest, None); } } - /// Run all unit tests plus documentation tests for an entire crate DAG defined - /// by a `Cargo.toml` + /// Run all unit tests plus documentation tests for a given crate defined + /// by a `Cargo.toml` (single manifest) /// /// This is what runs tests for crates like the standard library, compiler, etc. /// It essentially is the driver for running `cargo test`. @@ -1241,27 +1328,23 @@ impl Step for Crate { }; let mut cargo = builder.cargo(compiler, mode, target, test_kind.subcommand()); - let (name, root) = match mode { + match mode { Mode::Libstd => { compile::std_cargo(build, &compiler, target, &mut cargo); - ("libstd", "std") } Mode::Libtest => { compile::test_cargo(build, &compiler, target, &mut cargo); - ("libtest", "test") } Mode::Librustc => { builder.ensure(compile::Rustc { compiler, target }); compile::rustc_cargo(build, &mut cargo); - ("librustc", "rustc-main") } _ => panic!("can only test libraries"), }; - let root = INTERNER.intern_string(String::from(root)); let _folder = build.fold_output(|| { - format!("{}_stage{}-{}", test_kind.subcommand(), compiler.stage, name) + format!("{}_stage{}-{}", test_kind.subcommand(), compiler.stage, krate) }); - println!("{} {} stage{} ({} -> {})", test_kind, name, compiler.stage, + println!("{} {} stage{} ({} -> {})", test_kind, krate, compiler.stage, &compiler.host, target); // Build up the base `cargo test` command. @@ -1273,37 +1356,7 @@ impl Step for Crate { cargo.arg("--no-fail-fast"); } - match krate { - Some(krate) => { - cargo.arg("-p").arg(krate); - } - None => { - let mut visited = HashSet::new(); - let mut next = vec![root]; - while let Some(name) = next.pop() { - // Right now jemalloc and the sanitizer crates are - // target-specific crate in the sense that it's not present - // on all platforms. Custom skip it here for now, but if we - // add more this probably wants to get more generalized. - // - // Also skip `build_helper` as it's not compiled normally - // for target during the bootstrap and it's just meant to be - // a helper crate, not tested. If it leaks through then it - // ends up messing with various mtime calculations and such. - if !name.contains("jemalloc") && - *name != *"build_helper" && - !(name.starts_with("rustc_") && name.ends_with("san")) && - name != "dlmalloc" { - cargo.arg("-p").arg(&format!("{}:0.0.0", name)); - } - for dep in build.crates[&name].deps.iter() { - if visited.insert(dep) { - next.push(*dep); - } - } - } - } - } + cargo.arg("-p").arg(krate); // The tests are going to run with the *target* libraries, so we need to // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. @@ -1355,18 +1408,18 @@ impl Step for Crate { } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustdoc { +pub struct CrateRustdoc { host: Interned, test_kind: TestKind, } -impl Step for Rustdoc { +impl Step for CrateRustdoc { type Output = (); const DEFAULT: bool = true; const ONLY_HOSTS: bool = true; fn should_run(run: ShouldRun) -> ShouldRun { - run.path("src/librustdoc").path("src/tools/rustdoc") + run.paths(&["src/librustdoc", "src/tools/rustdoc"]) } fn make_run(run: RunConfig) { @@ -1380,7 +1433,7 @@ impl Step for Rustdoc { panic!("unexpected builder.kind in crate: {:?}", builder.kind); }; - builder.ensure(Rustdoc { + builder.ensure(CrateRustdoc { host: run.host, test_kind, }); From a64575c3bd81f6f169eff22a4884984c9c5b36c0 Mon Sep 17 00:00:00 2001 From: Mark Simulacrum Date: Tue, 13 Feb 2018 18:42:26 -0700 Subject: [PATCH 5/5] Fix default Steps without paths. Some Steps are by-default run but don't have any paths associated with them. We need to have at least one PathSet per each Step, though, so we add an empty one on calls to `never()`. --- src/bootstrap/builder.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index cbd303e114066..c00138d85e149 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -95,7 +95,7 @@ pub struct RunConfig<'a> { pub builder: &'a Builder<'a>, pub host: Interned, pub target: Interned, - pub path: &'a Path, + pub path: PathBuf, } struct StepDescription { @@ -114,6 +114,10 @@ struct PathSet { } impl PathSet { + fn empty() -> PathSet { + PathSet { set: BTreeSet::new() } + } + fn one>(path: P) -> PathSet { let mut set = BTreeSet::new(); set.insert(path.into()); @@ -124,8 +128,8 @@ impl PathSet { self.set.iter().any(|p| p.ends_with(needle)) } - fn path(&self) -> &Path { - self.set.iter().next().unwrap() + fn path(&self, builder: &Builder) -> PathBuf { + self.set.iter().next().unwrap_or(&builder.build.src).to_path_buf() } } @@ -174,7 +178,7 @@ impl StepDescription { for target in targets { let run = RunConfig { builder, - path: pathset.path(), + path: pathset.path(builder), host: *host, target: *target, }; @@ -278,7 +282,8 @@ impl<'a> ShouldRun<'a> { } // allows being more explicit about why should_run in Step returns the value passed to it - pub fn never(self) -> ShouldRun<'a> { + pub fn never(mut self) -> ShouldRun<'a> { + self.paths.insert(PathSet::empty()); self }