Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rework how bootstrap tools are built #58897

Merged
merged 2 commits into from
Mar 21, 2019
Merged

Rework how bootstrap tools are built #58897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9efc93c
Tools built by the bootstrap compiler must be built by it
Mark-Simulacrum Mar 3, 2019
03718ed
Permit getting stage 0 rustdoc
Mark-Simulacrum Mar 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions src/bootstrap/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,28 +668,27 @@ impl<'a> Builder<'a> {
.map(|entry| entry.path())
}

pub fn rustdoc(&self, host: Interned<String>) -> PathBuf {
self.ensure(tool::Rustdoc { host })
pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
self.ensure(tool::Rustdoc { compiler })
}

pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
let compiler = self.compiler(self.top_stage, host);
cmd.env("RUSTC_STAGE", compiler.stage.to_string())
.env("RUSTC_SYSROOT", self.sysroot(compiler))
// Note that this is *not* the sysroot_libdir because rustdoc must be linked
// equivalently to rustc.
.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler))
.env("CFG_RELEASE_CHANNEL", &self.config.channel)
.env("RUSTDOC_REAL", self.rustdoc(host))
.env("RUSTDOC_REAL", self.rustdoc(compiler))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this line fixed #58587. After it std is documented with rustdoc stage1 with stage1 libraries.

.env("RUSTDOC_CRATE_VERSION", self.rust_version())
.env("RUSTC_BOOTSTRAP", "1");

// Remove make-related flags that can cause jobserver problems.
cmd.env_remove("MAKEFLAGS");
cmd.env_remove("MFLAGS");

if let Some(linker) = self.linker(host) {
if let Some(linker) = self.linker(compiler.host) {
cmd.env("RUSTC_TARGET_LINKER", linker);
}
cmd
Expand Down Expand Up @@ -751,7 +750,7 @@ impl<'a> Builder<'a> {
// This is the intended out directory for compiler documentation.
my_out = self.compiler_doc_out(target);
}
let rustdoc = self.rustdoc(compiler.host);
let rustdoc = self.rustdoc(compiler);
self.clear_if_dirty(&my_out, &rustdoc);
} else if cmd != "test" {
match mode {
Expand Down Expand Up @@ -897,7 +896,7 @@ impl<'a> Builder<'a> {
.env(
"RUSTDOC_REAL",
if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
self.rustdoc(compiler.host)
self.rustdoc(compiler)
} else {
PathBuf::from("/path/to/nowhere/rustdoc/not/required")
},
Expand Down
2 changes: 1 addition & 1 deletion src/bootstrap/dist.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@ impl Step for Rustc {
t!(fs::create_dir_all(image.join("bin")));
builder.cp_r(&src.join("bin"), &image.join("bin"));

builder.install(&builder.rustdoc(compiler.host), &image.join("bin"), 0o755);
builder.install(&builder.rustdoc(compiler), &image.join("bin"), 0o755);

// Copy runtime DLLs needed by the compiler
if libdir != "bin" {
Expand Down
14 changes: 9 additions & 5 deletions src/bootstrap/doc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ fn invoke_rustdoc(
let footer = builder.src.join("src/doc/footer.inc");
let version_info = out.join("version_info.html");

let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);

let out = out.join("book");

Expand Down Expand Up @@ -415,7 +415,7 @@ impl Step for Standalone {
}

let html = out.join(filename).with_extension("html");
let rustdoc = builder.rustdoc(compiler.host);
let rustdoc = builder.rustdoc(compiler);
if up_to_date(&path, &html) &&
up_to_date(&footer, &html) &&
up_to_date(&favicon, &html) &&
Expand All @@ -425,7 +425,7 @@ impl Step for Standalone {
continue
}

let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);
cmd.arg("--html-after-content").arg(&footer)
.arg("--html-before-content").arg(&version_info)
.arg("--html-in-header").arg(&favicon)
Expand Down Expand Up @@ -824,7 +824,7 @@ impl Step for Rustdoc {
builder.ensure(Rustc { stage, target });

// Build rustdoc.
builder.ensure(tool::Rustdoc { host: compiler.host });
builder.ensure(tool::Rustdoc { compiler: compiler });

// Symlink compiler docs to the output directory of rustdoc documentation.
let out_dir = builder.stage_out(compiler, Mode::ToolRustc)
Expand Down Expand Up @@ -883,7 +883,11 @@ impl Step for ErrorIndex {
builder.info(&format!("Documenting error index ({})", target));
let out = builder.doc_out(target);
t!(fs::create_dir_all(&out));
let mut index = builder.tool_cmd(Tool::ErrorIndex);
let compiler = builder.compiler(2, builder.config.build);
let mut index = tool::ErrorIndex::command(
builder,
compiler,
);
index.arg("html");
index.arg(out.join("error-index.html"));

Expand Down
18 changes: 9 additions & 9 deletions src/bootstrap/test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ impl Step for Cargotest {
cmd.arg(&builder.initial_cargo)
.arg(&out_dir)
.env("RUSTC", builder.rustc(compiler))
.env("RUSTDOC", builder.rustdoc(compiler.host)),
.env("RUSTDOC", builder.rustdoc(compiler)),
);
}
}
Expand Down Expand Up @@ -414,7 +414,6 @@ impl Step for Miri {

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct CompiletestTest {
stage: u32,
host: Interned<String>,
}

Expand All @@ -427,16 +426,14 @@ impl Step for CompiletestTest {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(CompiletestTest {
stage: run.builder.top_stage,
host: run.target,
});
}

/// Runs `cargo test` for compiletest.
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let host = self.host;
let compiler = builder.compiler(stage, host);
let compiler = builder.compiler(0, host);

let mut cargo = tool::prepare_tool_cargo(builder,
compiler,
Expand Down Expand Up @@ -563,7 +560,7 @@ impl Step for RustdocTheme {
builder.sysroot_libdir(self.compiler, self.compiler.host),
)
.env("CFG_RELEASE_CHANNEL", &builder.config.channel)
.env("RUSTDOC_REAL", builder.rustdoc(self.compiler.host))
.env("RUSTDOC_REAL", builder.rustdoc(self.compiler))
.env("RUSTDOC_CRATE_VERSION", builder.rust_version())
.env("RUSTC_BOOTSTRAP", "1");
if let Some(linker) = builder.linker(self.compiler.host) {
Expand Down Expand Up @@ -998,7 +995,7 @@ impl Step for Compiletest {
|| (mode == "ui" && is_rustdoc_ui)
{
cmd.arg("--rustdoc-path")
.arg(builder.rustdoc(compiler.host));
.arg(builder.rustdoc(compiler));
}

cmd.arg("--src-base")
Expand Down Expand Up @@ -1426,7 +1423,10 @@ impl Step for ErrorIndex {
t!(fs::create_dir_all(&dir));
let output = dir.join("error-index.md");

let mut tool = builder.tool_cmd(Tool::ErrorIndex);
let mut tool = tool::ErrorIndex::command(
builder,
builder.compiler(compiler.stage, builder.config.build),
);
tool.arg("markdown")
.arg(&output)
.env("CFG_BUILD", &builder.config.build)
Expand All @@ -1451,7 +1451,7 @@ fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) ->
}

builder.info(&format!("doc tests for: {}", markdown.display()));
let mut cmd = builder.rustdoc_cmd(compiler.host);
let mut cmd = builder.rustdoc_cmd(compiler);
builder.add_rust_test_threads(&mut cmd);
cmd.arg("--test");
cmd.arg(markdown);
Expand Down
112 changes: 73 additions & 39 deletions src/bootstrap/tool.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,9 @@ pub fn prepare_tool_cargo(
cargo
}

macro_rules! tool {
macro_rules! bootstrap_tool {
($(
$name:ident, $path:expr, $tool_name:expr, $mode:expr
$name:ident, $path:expr, $tool_name:expr
$(,llvm_tools = $llvm:expr)*
$(,is_external_tool = $external:expr)*
;
Expand All @@ -266,10 +266,7 @@ macro_rules! tool {

impl Tool {
pub fn get_mode(&self) -> Mode {
let mode = match self {
$(Tool::$name => $mode,)+
};
mode
Mode::ToolBootstrap
}

/// Whether this tool requires LLVM to run
Expand All @@ -282,27 +279,15 @@ macro_rules! tool {

impl<'a> Builder<'a> {
pub fn tool_exe(&self, tool: Tool) -> PathBuf {
let stage = self.tool_default_stage(tool);
match tool {
$(Tool::$name =>
self.ensure($name {
compiler: self.compiler(stage, self.config.build),
compiler: self.compiler(0, self.config.build),
target: self.config.build,
}),
)+
}
}

pub fn tool_default_stage(&self, tool: Tool) -> u32 {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can. Otherwise compile
// everything else in stage0 as there's no need to rebootstrap
// everything.
match tool {
Tool::ErrorIndex if self.top_stage >= 2 => self.top_stage,
_ => 0,
}
}
}

$(
Expand All @@ -321,7 +306,8 @@ macro_rules! tool {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure($name {
compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
// snapshot compiler
compiler: run.builder.compiler(0, run.builder.config.build),
target: run.target,
});
}
Expand All @@ -331,7 +317,7 @@ macro_rules! tool {
compiler: self.compiler,
target: self.target,
tool: $tool_name,
mode: $mode,
mode: Mode::ToolBootstrap,
path: $path,
is_optional_tool: false,
source_type: if false $(|| $external)* {
Expand All @@ -347,21 +333,67 @@ macro_rules! tool {
}
}

tool!(
Rustbook, "src/tools/rustbook", "rustbook", Mode::ToolBootstrap;
ErrorIndex, "src/tools/error_index_generator", "error_index_generator", Mode::ToolRustc;
UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen", Mode::ToolBootstrap;
Tidy, "src/tools/tidy", "tidy", Mode::ToolBootstrap;
Linkchecker, "src/tools/linkchecker", "linkchecker", Mode::ToolBootstrap;
CargoTest, "src/tools/cargotest", "cargotest", Mode::ToolBootstrap;
Compiletest, "src/tools/compiletest", "compiletest", Mode::ToolBootstrap, llvm_tools = true;
BuildManifest, "src/tools/build-manifest", "build-manifest", Mode::ToolBootstrap;
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client", Mode::ToolBootstrap;
RustInstaller, "src/tools/rust-installer", "fabricate", Mode::ToolBootstrap,
is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes", Mode::ToolBootstrap;
bootstrap_tool!(
Rustbook, "src/tools/rustbook", "rustbook";
UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen";
Tidy, "src/tools/tidy", "tidy";
Linkchecker, "src/tools/linkchecker", "linkchecker";
CargoTest, "src/tools/cargotest", "cargotest";
Compiletest, "src/tools/compiletest", "compiletest", llvm_tools = true;
BuildManifest, "src/tools/build-manifest", "build-manifest";
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
);

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct ErrorIndex {
pub compiler: Compiler,
}

impl ErrorIndex {
pub fn command(builder: &Builder<'_>, compiler: Compiler) -> Command {
let mut cmd = Command::new(builder.ensure(ErrorIndex {
compiler
}));
add_lib_path(
vec![PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))],
&mut cmd,
);
cmd
}
}

impl Step for ErrorIndex {
type Output = PathBuf;

fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/error_index_generator")
}

fn make_run(run: RunConfig<'_>) {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can.
let stage = if run.builder.top_stage >= 2 { run.builder.top_stage } else { 0 };
run.builder.ensure(ErrorIndex {
compiler: run.builder.compiler(stage, run.builder.config.build),
});
}

fn run(self, builder: &Builder<'_>) -> PathBuf {
builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.compiler.host,
tool: "error_index_generator",
mode: Mode::ToolRustc,
path: "src/tools/error_index_generator",
is_optional_tool: false,
source_type: SourceType::InTree,
extra_features: Vec::new(),
}).expect("expected to build -- essential tool")
}
}

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct RemoteTestServer {
pub compiler: Compiler,
Expand Down Expand Up @@ -398,7 +430,9 @@ impl Step for RemoteTestServer {

#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Rustdoc {
pub host: Interned<String>,
/// This should only ever be 0 or 2.
/// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here.
pub compiler: Compiler,
}

impl Step for Rustdoc {
Expand All @@ -412,12 +446,12 @@ impl Step for Rustdoc {

fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustdoc {
host: run.host,
compiler: run.builder.compiler(run.builder.top_stage, run.host),
});
}

fn run(self, builder: &Builder<'_>) -> PathBuf {
let target_compiler = builder.compiler(builder.top_stage, self.host);
let target_compiler = self.compiler;
if target_compiler.stage == 0 {
if !target_compiler.is_snapshot(builder) {
panic!("rustdoc in stage 0 must be snapshot rustdoc");
Expand Down Expand Up @@ -625,7 +659,7 @@ impl<'a> Builder<'a> {
/// `host`.
pub fn tool_cmd(&self, tool: Tool) -> Command {
let mut cmd = Command::new(self.tool_exe(tool));
let compiler = self.compiler(self.tool_default_stage(tool), self.config.build);
let compiler = self.compiler(0, self.config.build);
self.prepare_tool_cmd(compiler, tool, &mut cmd);
cmd
}
Expand All @@ -637,7 +671,7 @@ impl<'a> Builder<'a> {
fn prepare_tool_cmd(&self, compiler: Compiler, tool: Tool, cmd: &mut Command) {
let host = &compiler.host;
let mut lib_paths: Vec<PathBuf> = vec![
if compiler.stage == 0 && tool != Tool::ErrorIndex {
if compiler.stage == 0 {
self.build.rustc_snapshot_libdir()
} else {
PathBuf::from(&self.sysroot_libdir(compiler, compiler.host))
Expand Down