From 178652a2988d9ecd478a4116237e1d7ea6b777f7 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 20 Mar 2018 02:06:38 +0000 Subject: [PATCH 1/4] Add support to rustbuild for a 'rustc docs' component tarball --- src/bootstrap/builder.rs | 12 ++-- src/bootstrap/dist.rs | 69 +++++++++++++++++--- src/bootstrap/doc.rs | 133 ++++++++++++++++++++++++++++++--------- src/bootstrap/lib.rs | 5 ++ 4 files changed, 178 insertions(+), 41 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 675d3dd437eef..9f779da6f65b3 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -316,11 +316,13 @@ impl<'a> Builder<'a> { test::Rustfmt, test::Miri, test::Clippy, test::RustdocJS, test::RustdocTheme), Kind::Bench => describe!(test::Crate, test::CrateLibrustc), Kind::Doc => describe!(doc::UnstableBook, doc::UnstableBookGen, doc::TheBook, - doc::Standalone, doc::Std, doc::Test, doc::Rustc, doc::ErrorIndex, doc::Nomicon, - doc::Reference, doc::Rustdoc, doc::RustByExample, doc::CargoBook), - Kind::Dist => describe!(dist::Docs, dist::Mingw, dist::Rustc, dist::DebuggerScripts, - dist::Std, dist::Analysis, dist::Src, dist::PlainSourceTarball, dist::Cargo, - dist::Rls, dist::Rustfmt, dist::Extended, dist::HashSign), + doc::Standalone, doc::Std, doc::Test, doc::WhitelistedRustc, doc::Rustc, + doc::ErrorIndex, doc::Nomicon, doc::Reference, doc::Rustdoc, doc::RustByExample, + doc::CargoBook), + Kind::Dist => describe!(dist::Docs, dist::RustcDocs, dist::Mingw, dist::Rustc, + dist::DebuggerScripts, dist::Std, dist::Analysis, dist::Src, + dist::PlainSourceTarball, dist::Cargo, dist::Rls, dist::Rustfmt, dist::Extended, + dist::HashSign), Kind::Install => describe!(install::Docs, install::Std, install::Cargo, install::Rls, install::Rustfmt, install::Analysis, install::Src, install::Rustc), } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index dcb572416594e..e25a7e1852586 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -102,7 +102,7 @@ impl Step for Docs { let dst = image.join("share/doc/rust/html"); t!(fs::create_dir_all(&dst)); - let src = build.out.join(host).join("doc"); + let src = build.doc_out(host); cp_r(&src, &dst); let mut cmd = rust_installer(builder); @@ -120,14 +120,69 @@ impl Step for Docs { build.run(&mut cmd); t!(fs::remove_dir_all(&image)); - // As part of this step, *also* copy the docs directory to a directory which - // buildbot typically uploads. - if host == build.build { - let dst = distdir(build).join("doc").join(build.rust_package_vers()); - t!(fs::create_dir_all(&dst)); - cp_r(&src, &dst); + distdir(build).join(format!("{}-{}.tar.gz", name, host)) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDocs { + pub stage: u32, + pub host: Interned, +} + +impl Step for RustcDocs { + type Output = PathBuf; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + run.path("src/librustc") + } + + fn make_run(run: RunConfig) { + run.builder.ensure(RustcDocs { + stage: run.builder.top_stage, + host: run.target, + }); + } + + /// Builds the `rustc-docs` installer component. + fn run(self, builder: &Builder) -> PathBuf { + let build = builder.build; + let host = self.host; + + let name = pkgname(build, "rustc-docs"); + + println!("Dist compiler docs ({})", host); + if !build.config.compiler_docs { + println!("\tskipping - compiler docs disabled"); + return distdir(build).join(format!("{}-{}.tar.gz", name, host)); } + builder.default_doc(None); + + let image = tmpdir(build).join(format!("{}-{}-image", name, host)); + let _ = fs::remove_dir_all(&image); + + let dst = image.join("share/doc/rustc/html"); + t!(fs::create_dir_all(&dst)); + let src = build.compiler_doc_out(host); + cp_r(&src, &dst); + + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rustc-Documentation") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=Rustc-documentation-is-installed.") + .arg("--image-dir").arg(&image) + .arg("--work-dir").arg(&tmpdir(build)) + .arg("--output-dir").arg(&distdir(build)) + .arg(format!("--package-name={}-{}", name, host)) + .arg("--component-name=rustc-docs") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg("--bulk-dirs=share/doc/rustc/html"); + build.run(&mut cmd); + t!(fs::remove_dir_all(&image)); + distdir(build).join(format!("{}-{}.tar.gz", name, host)) } } diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 5bc582b3507bb..ca45adb4649af 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -483,21 +483,17 @@ impl Step for Std { let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "doc"); compile::std_cargo(builder, &compiler, target, &mut cargo); - // We don't want to build docs for internal std dependencies unless - // in compiler-docs mode. When not in that mode, we whitelist the crates - // for which docs must be built. - if !build.config.compiler_docs { - cargo.arg("--no-deps"); - for krate in &["alloc", "core", "std", "std_unicode"] { - cargo.arg("-p").arg(krate); - // Create all crate output directories first to make sure rustdoc uses - // relative links. - // FIXME: Cargo should probably do this itself. - t!(fs::create_dir_all(out_dir.join(krate))); - } + // Keep a whitelist so we do not build internal stdlib crates, these will be + // build by the rustc step later if enabled. + cargo.arg("--no-deps"); + for krate in &["alloc", "core", "std", "std_unicode"] { + cargo.arg("-p").arg(krate); + // Create all crate output directories first to make sure rustdoc uses + // relative links. + // FIXME: Cargo should probably do this itself. + t!(fs::create_dir_all(out_dir.join(krate))); } - build.run(&mut cargo); cp_r(&my_out, &out); } @@ -563,6 +559,81 @@ impl Step for Test { } } +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct WhitelistedRustc { + stage: u32, + target: Interned, +} + +impl Step for WhitelistedRustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun) -> ShouldRun { + let builder = run.builder; + run.krate("rustc-main").default_condition(builder.build.config.docs) + } + + fn make_run(run: RunConfig) { + run.builder.ensure(WhitelistedRustc { + stage: run.builder.top_stage, + target: run.target, + }); + } + + /// Generate whitelisted compiler crate documentation. + /// + /// This will generate all documentation for crates that are whitelisted + /// to be included in the standard documentation. This documentation is + /// included in the standard Rust documentation, so we should always + /// document it and symlink to merge with the rest of the std and test + /// documentation. We don't build other compiler documentation + /// here as we want to be able to keep it separate from the standard + /// documentation. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder) { + let build = builder.build; + let stage = self.stage; + let target = self.target; + println!("Documenting stage{} whitelisted compiler ({})", stage, target); + let out = build.doc_out(target); + t!(fs::create_dir_all(&out)); + let compiler = builder.compiler(stage, build.build); + let rustdoc = builder.rustdoc(compiler.host); + let compiler = if build.force_use_stage1(compiler, target) { + builder.compiler(1, compiler.host) + } else { + compiler + }; + + // Build libstd docs so that we generate relative links + builder.ensure(Std { stage, target }); + + builder.ensure(compile::Rustc { compiler, target }); + let out_dir = build.stage_out(compiler, Mode::Librustc) + .join(target).join("doc"); + + // See docs in std above for why we symlink + let my_out = build.crate_doc_out(target); + build.clear_if_dirty(&my_out, &rustdoc); + t!(symlink_dir_force(&my_out, &out_dir)); + + let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc"); + compile::rustc_cargo(build, &mut cargo); + + // We don't want to build docs for internal compiler dependencies in this + // step (there is another step for that). Therefore, we whitelist the crates + // for which docs must be built. + cargo.arg("--no-deps"); + for krate in &["proc_macro"] { + cargo.arg("-p").arg(krate); + } + + build.run(&mut cargo); + cp_r(&my_out, &out); + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Rustc { stage: u32, @@ -586,16 +657,18 @@ impl Step for Rustc { }); } - /// Generate all compiler documentation. + /// Generate compiler documentation. /// - /// This will generate all documentation for the compiler libraries and their - /// dependencies. This is largely just a wrapper around `cargo doc`. + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. fn run(self, builder: &Builder) { let build = builder.build; let stage = self.stage; let target = self.target; println!("Documenting stage{} compiler ({})", stage, target); - let out = build.doc_out(target); + let out = build.compiler_doc_out(target); t!(fs::create_dir_all(&out)); let compiler = builder.compiler(stage, build.build); let rustdoc = builder.rustdoc(compiler.host); @@ -605,6 +678,11 @@ impl Step for Rustc { compiler }; + if !build.config.compiler_docs { + println!("\tskipping - compiler docs disabled"); + return; + } + // Build libstd docs so that we generate relative links builder.ensure(Std { stage, target }); @@ -613,6 +691,12 @@ impl Step for Rustc { .join(target).join("doc"); // See docs in std above for why we symlink + // + // This step must happen after other documentation steps. This + // invariant ensures that compiler documentation is not included + // in the standard documentation tarballs but that all the + // documentation from the standard documentation tarballs is included + // in the compiler documentation tarball. let my_out = build.crate_doc_out(target); build.clear_if_dirty(&my_out, &rustdoc); t!(symlink_dir_force(&my_out, &out_dir)); @@ -620,18 +704,9 @@ impl Step for Rustc { let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc"); compile::rustc_cargo(build, &mut cargo); - if build.config.compiler_docs { - // src/rustc/Cargo.toml contains a bin crate called rustc which - // would otherwise overwrite the docs for the real rustc lib crate. - cargo.arg("-p").arg("rustc_driver"); - } else { - // Like with libstd above if compiler docs aren't enabled then we're not - // documenting internal dependencies, so we have a whitelist. - cargo.arg("--no-deps"); - for krate in &["proc_macro"] { - cargo.arg("-p").arg(krate); - } - } + // src/rustc/Cargo.toml contains a bin crate called rustc which + // would otherwise overwrite the docs for the real rustc lib crate. + cargo.arg("-p").arg("rustc_driver"); build.run(&mut cargo); cp_r(&my_out, &out); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index b778ba33d89cc..b2c8ac24d72d8 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -511,6 +511,11 @@ impl Build { self.out.join(&*target).join("doc") } + /// Output directory for all documentation for a target + fn compiler_doc_out(&self, target: Interned) -> PathBuf { + self.out.join(&*target).join("compiler-doc") + } + /// Output directory for some generated md crate documentation for a target (temporary) fn md_doc_out(&self, target: Interned) -> Interned { INTERNER.intern_path(self.out.join(&*target).join("md-doc")) From 1392179cdc9c96e75353771959c246553b918d57 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 20 Mar 2018 02:07:16 +0000 Subject: [PATCH 2/4] Configure the dist-x86_64-linux builder to produce compiler documentation --- src/ci/docker/dist-x86_64-linux/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/dist-x86_64-linux/Dockerfile b/src/ci/docker/dist-x86_64-linux/Dockerfile index 3b98b0aa926bd..28c97e8c6dbf9 100644 --- a/src/ci/docker/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/dist-x86_64-linux/Dockerfile @@ -84,7 +84,8 @@ ENV HOSTS=x86_64-unknown-linux-gnu ENV RUST_CONFIGURE_ARGS \ --enable-full-tools \ --enable-sanitizers \ - --enable-profiler + --enable-profiler \ + --enable-compiler-docs ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS # This is the only builder which will create source tarballs From 1b0e9f5af9283b003f5fce6f19ede145c1776684 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 22 Mar 2018 14:57:28 +0000 Subject: [PATCH 3/4] Only generate documentation for local rustc crates. --- src/bootstrap/doc.rs | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index ca45adb4649af..e525bdb98fc0c 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -17,12 +17,13 @@ //! Everything here is basically just a shim around calling either `rustbook` or //! `rustdoc`. +use std::collections::HashSet; use std::fs::{self, File}; use std::io::prelude::*; use std::io; use std::path::{PathBuf, Path}; -use Mode; +use {Build, Mode}; use build_helper::up_to_date; use util::{cp_r, symlink_dir}; @@ -704,15 +705,41 @@ impl Step for Rustc { let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc"); compile::rustc_cargo(build, &mut cargo); - // src/rustc/Cargo.toml contains a bin crate called rustc which - // would otherwise overwrite the docs for the real rustc lib crate. - cargo.arg("-p").arg("rustc_driver"); + // Only include compiler crates, no dependencies of those, such as `libc`. + cargo.arg("--no-deps"); + + // Find dependencies for top level crates. + let mut compiler_crates = HashSet::new(); + for root_crate in &["rustc", "rustc_driver"] { + let interned_root_crate = INTERNER.intern_str(root_crate); + find_compiler_crates(&build, &interned_root_crate, &mut compiler_crates); + } + + for krate in &compiler_crates { + cargo.arg("-p").arg(krate); + } build.run(&mut cargo); cp_r(&my_out, &out); } } +fn find_compiler_crates( + build: &Build, + name: &Interned, + crates: &mut HashSet> +) { + // Add current crate. + crates.insert(*name); + + // Look for dependencies. + for dep in build.crates.get(name).unwrap().deps.iter() { + if build.crates.get(dep).unwrap().is_local(build) { + find_compiler_crates(build, dep, crates); + } + } +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct ErrorIndex { target: Interned, From 73fa6d52ed99651940267affa941a1069cc20f35 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 22 Mar 2018 20:49:05 +0000 Subject: [PATCH 4/4] Remove std/test documentation from compiler docs. --- src/bootstrap/dist.rs | 4 ++-- src/bootstrap/doc.rs | 16 ++++------------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e25a7e1852586..5563b153f5aff 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -163,7 +163,7 @@ impl Step for RustcDocs { let image = tmpdir(build).join(format!("{}-{}-image", name, host)); let _ = fs::remove_dir_all(&image); - let dst = image.join("share/doc/rustc/html"); + let dst = image.join("share/doc/rust/html"); t!(fs::create_dir_all(&dst)); let src = build.compiler_doc_out(host); cp_r(&src, &dst); @@ -179,7 +179,7 @@ impl Step for RustcDocs { .arg(format!("--package-name={}-{}", name, host)) .arg("--component-name=rustc-docs") .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg("--bulk-dirs=share/doc/rustc/html"); + .arg("--bulk-dirs=share/doc/rust/html"); build.run(&mut cmd); t!(fs::remove_dir_all(&image)); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index e525bdb98fc0c..44073a5b07572 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -690,17 +690,10 @@ impl Step for Rustc { builder.ensure(compile::Rustc { compiler, target }); let out_dir = build.stage_out(compiler, Mode::Librustc) .join(target).join("doc"); - - // See docs in std above for why we symlink - // - // This step must happen after other documentation steps. This - // invariant ensures that compiler documentation is not included - // in the standard documentation tarballs but that all the - // documentation from the standard documentation tarballs is included - // in the compiler documentation tarball. - let my_out = build.crate_doc_out(target); - build.clear_if_dirty(&my_out, &rustdoc); - t!(symlink_dir_force(&my_out, &out_dir)); + // We do not symlink to the same shared folder that already contains std library + // documentation from previous steps as we do not want to include that. + build.clear_if_dirty(&out, &rustdoc); + t!(symlink_dir_force(&out, &out_dir)); let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "doc"); compile::rustc_cargo(build, &mut cargo); @@ -720,7 +713,6 @@ impl Step for Rustc { } build.run(&mut cargo); - cp_r(&my_out, &out); } }