Auto merge of #7350 - alexcrichton:mock-std, r=ehuss

Improve test suite for `-Zbuild-std`

This commit is aimed directly at rust-lang/wg-cargo-std-aware#33 and in
general making the `-Zbuild-std` tests more robust. The main change here
is that a new source tree is checked in, `tests/testsuite/mock-std`,
which mirrors rust-lang/rust's own tree for libstd. This mock tree is as
empty as it can be, ideally duplicating almost nothing but for not
requiring duplication of Cargo metadata about patches and such.

The end result here looks like:

* All `-Zbuild-std` tests are now run in parallel
* All tests run much more quickly since they're compiling tiny crates
  instead of actually compiling libstd/libcore
* No tests require network access
* We verify that crates have access to the "custom" libraries
  that we build

Coverage of tests is not currently expanded, but it's hoped that we
could add that shortly afterwards. Coverage has actually gone down
slightly since the custom target test was commented out temporarily and
the full integration test of running `-Zbuild-std` isn't run on CI any
more.

Closes rust-lang/wg-cargo-std-aware#33

Cargo.toml

@@ -103,6 +103,7 @@ features = [
 
 [dev-dependencies]
 cargo-test-macro = { path = "crates/cargo-test-macro", version = "0.1.0" }
+cargo-test-support = { path = "crates/cargo-test-support", version = "0.1.0" }
 
 [[bin]]
 name = "cargo"

azure-pipelines.yml

@@ -43,6 +43,7 @@ jobs:
       x86_64-msvc:
         TOOLCHAIN: stable-x86_64-pc-windows-msvc
         OTHER_TARGET: i686-pc-windows-msvc
+
 - job: rustfmt
   pool:
     vmImage: ubuntu-16.04
@@ -54,6 +55,8 @@ jobs:
       displayName: "Check rustfmt (cargo)"
     - bash: cd crates/cargo-test-macro && cargo fmt --all -- --check
       displayName: "Check rustfmt (cargo-test-macro)"
+    - bash: cd crates/cargo-test-support && cargo fmt --all -- --check
+      displayName: "Check rustfmt (cargo-test-support)"
     - bash: cd crates/crates-io && cargo fmt --all -- --check
       displayName: "Check rustfmt (crates-io)"
     - bash: cd crates/resolver-tests && cargo fmt --all -- --check
@@ -71,6 +74,20 @@ jobs:
   variables:
     TOOLCHAIN: stable
 
+- job: build_std
+  pool:
+    vmImage: ubuntu-16.04
+  steps:
+    - template: ci/azure-install-rust.yml
+    - bash: rustup component add rust-src
+      displayName: "Install rust-src"
+    - bash: cargo build
+    - bash: cargo test --test build-std
+      displayName: "tests"
+  variables:
+    TOOLCHAIN: nightly
+    CARGO_RUN_BUILD_STD_TESTS: 1
+
 - job: docs
   pool:
     vmImage: ubuntu-16.04
@@ -88,4 +105,3 @@ jobs:
       displayName: "Build mdbook documentation"
   variables:
     TOOLCHAIN: stable
-

ci/azure-test-all.yml

@@ -16,11 +16,6 @@ steps:
 - bash: rustup component add clippy || echo "clippy not available"
   displayName: "Install clippy (maybe)"
 
-# This is needed for standard library tests.
-- bash: rustup component add rust-src
-  condition: and(succeeded(), eq(variables['TOOLCHAIN'], 'nightly'))
-  displayName: "Install rust-src (maybe)"
-
 # Deny warnings on CI to avoid warnings getting into the codebase, and note the
 # `force-system-lib-on-osx` which is intended to fix compile issues on OSX where
 # compiling curl from source on OSX yields linker errors on Azure.
@@ -31,3 +26,6 @@ steps:
 # fix the link errors.
 - bash: cargo test --features 'deny-warnings curl/force-system-lib-on-osx'
   displayName: "cargo test"
+
+- bash: cargo test -p cargo-test-support
+  displayName: "cargo test -p cargo-test-support"

crates/cargo-test-macro/src/lib.rs

@@ -3,7 +3,7 @@ extern crate proc_macro;
 use proc_macro::*;
 
 #[proc_macro_attribute]
-pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
+pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
     let span = Span::call_site();
     let mut ret = TokenStream::new();
     ret.extend(Some(TokenTree::from(Punct::new('#', Spacing::Alone))));
@@ -13,6 +13,8 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
         test.into(),
     ))));
 
+    let build_std = contains_ident(&attr, "build_std");
+
     for token in item {
         let group = match token {
             TokenTree::Group(g) => {
@@ -29,25 +31,20 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
             }
         };
 
-        let mut new_body = vec![
-            TokenTree::from(Ident::new("let", span)),
-            TokenTree::from(Ident::new("_test_guard", span)),
-            TokenTree::from(Punct::new('=', Spacing::Alone)),
-            TokenTree::from(Ident::new("crate", span)),
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-            TokenTree::from(Ident::new("support", span)),
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-            TokenTree::from(Ident::new("paths", span)),
-            TokenTree::from(Punct::new(':', Spacing::Joint)),
-            TokenTree::from(Punct::new(':', Spacing::Alone)),
-            TokenTree::from(Ident::new("init_root", span)),
-            TokenTree::from(Group::new(Delimiter::Parenthesis, TokenStream::new())),
-            TokenTree::from(Punct::new(';', Spacing::Alone)),
-        ]
-        .into_iter()
-        .collect::<TokenStream>();
+        let mut new_body =
+            to_token_stream("let _test_guard = cargo_test_support::paths::init_root();");
+
+        // If this is a `build_std` test (aka `tests/build-std/*.rs`) then they
+        // only run on nightly and they only run when specifically instructed to
+        // on CI.
+        if build_std {
+            let ts = to_token_stream("if !cargo_test_support::is_nightly() { return }");
+            new_body.extend(ts);
+            let ts = to_token_stream(
+                "if std::env::var(\"CARGO_RUN_BUILD_STD_TESTS\").is_err() { return }",
+            );
+            new_body.extend(ts);
+        }
         new_body.extend(group.stream());
         ret.extend(Some(TokenTree::from(Group::new(
             group.delimiter(),
@@ -57,3 +54,14 @@ pub fn cargo_test(_attr: TokenStream, item: TokenStream) -> TokenStream {
 
     return ret;
 }
+
+fn contains_ident(t: &TokenStream, ident: &str) -> bool {
+    t.clone().into_iter().any(|t| match t {
+        TokenTree::Ident(i) => i.to_string() == ident,
+        _ => false,
+    })
+}
+
+fn to_token_stream(code: &str) -> TokenStream {
+    code.parse().unwrap()
+}

crates/cargo-test-support/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "cargo-test-support"
+version = "0.1.0"
+authors = ["Alex Crichton <[email protected]>"]
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+cargo = { path = "../.." }
+cargo-test-macro = { path = "../cargo-test-macro" }
+filetime = "0.2"
+flate2 = "1.0"
+git2 = "0.10"
+glob = "0.3"
+lazy_static = "1.0"
+remove_dir_all = "0.5"
+serde_json = "1.0"
+tar = "0.4"
+url = "2.0"

tests/testsuite/support/cross_compile.rs → crates/cargo-test-support/src/cross_compile.rs

@@ -1,10 +1,9 @@
+use crate::{basic_bin_manifest, main_file, project};
 use std::env;
 use std::process::Command;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
 
-use crate::support::{basic_bin_manifest, main_file, project};
-
 pub fn disabled() -> bool {
     // First, disable if `./configure` requested so.
     match env::var("CFG_DISABLE_CROSS_TESTS") {

tests/testsuite/support/git.rs → crates/cargo-test-support/src/git.rs

@@ -38,15 +38,13 @@ use some of the helper functions in this file to interact with the repository.
 
 */
 
+use crate::{path2url, project, Project, ProjectBuilder};
+use git2;
 use std::fs::{self, File};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
-
-use git2;
 use url::Url;
 
-use crate::support::{path2url, project, Project, ProjectBuilder};
-
 #[must_use]
 pub struct RepoBuilder {
     repo: git2::Repository,

tests/testsuite/support/install.rs → crates/cargo-test-support/src/install.rs

@@ -1,8 +1,7 @@
+use crate::paths;
 use std::env::consts::EXE_SUFFIX;
 use std::path::{Path, PathBuf};
 
-use crate::support::paths;
-
 /// Used by `cargo install` tests to assert an executable binary
 /// has been installed. Example usage:
 ///

tests/testsuite/support/mod.rs → crates/cargo-test-support/src/lib.rs

@@ -125,6 +125,7 @@ use url::Url;
 
 use self::paths::CargoPathExt;
 
+#[macro_export]
 macro_rules! t {
     ($e:expr) => {
         match $e {
@@ -134,6 +135,8 @@ macro_rules! t {
     };
 }
 
+pub use cargo_test_macro::cargo_test;
+
 pub mod cross_compile;
 pub mod git;
 pub mod paths;
@@ -409,7 +412,7 @@ impl Project {
     ///             .with_stdout("bar\n")
     ///             .run();
     pub fn process<T: AsRef<OsStr>>(&self, program: T) -> Execs {
-        let mut p = crate::support::process(program);
+        let mut p = process(program);
         p.cwd(self.root());
         execs().with_process_builder(p)
     }
@@ -1425,7 +1428,7 @@ pub fn lines_match(expected: &str, mut actual: &str) -> bool {
     actual.is_empty() || expected.ends_with("[..]")
 }
 
-#[cargo_test]
+#[test]
 fn lines_match_works() {
     assert!(lines_match("a b", "a b"));
     assert!(lines_match("a[..]b", "a b"));

tests/testsuite/support/paths.rs → crates/cargo-test-support/src/paths.rs

@@ -1,3 +1,5 @@
+use filetime::{self, FileTime};
+use lazy_static::lazy_static;
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::env;
@@ -7,9 +9,6 @@ use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Mutex;
 
-use filetime::{self, FileTime};
-use lazy_static::lazy_static;
-
 static CARGO_INTEGRATION_TEST_DIR: &str = "cit";
 
 lazy_static! {

tests/testsuite/support/publish.rs → crates/cargo-test-support/src/publish.rs

@@ -1,12 +1,10 @@
+use crate::registry::{self, alt_api_path};
+use crate::{find_json_mismatch, lines_match};
+use flate2::read::GzDecoder;
 use std::collections::{HashMap, HashSet};
 use std::fs::File;
 use std::io::{self, prelude::*, SeekFrom};
 use std::path::{Path, PathBuf};
-
-use crate::support::registry::{self, alt_api_path};
-use crate::support::{find_json_mismatch, lines_match};
-
-use flate2::read::GzDecoder;
 use tar::Archive;
 
 fn read_le_u32<R>(mut reader: R) -> io::Result<u32>

tests/testsuite/support/registry.rs → crates/cargo-test-support/src/registry.rs

@@ -1,18 +1,16 @@
-use std::collections::HashMap;
-use std::fs::{self, File};
-use std::io::prelude::*;
-use std::path::{Path, PathBuf};
-
+use crate::git::repo;
+use crate::paths;
 use cargo::sources::CRATES_IO_INDEX;
 use cargo::util::Sha256;
 use flate2::write::GzEncoder;
 use flate2::Compression;
+use std::collections::HashMap;
+use std::fs::{self, File};
+use std::io::prelude::*;
+use std::path::{Path, PathBuf};
 use tar::{Builder, Header};
 use url::Url;
 
-use crate::support::git::repo;
-use crate::support::paths;
-
 /// Gets the path to the local index pretending to be crates.io. This is a Git repo
 /// initialized with a `config.json` file pointing to `dl_path` for downloads
 /// and `api_path` for uploads.

src/cargo/core/compiler/standard_lib.rs

@@ -7,6 +7,7 @@ use crate::core::{Dependency, PackageId, PackageSet, Resolve, SourceId, Workspac
 use crate::ops::{self, Packages};
 use crate::util::errors::CargoResult;
 use std::collections::{HashMap, HashSet};
+use std::env;
 use std::path::PathBuf;
 
 /// Parse the `-Zbuild-std` flag.
@@ -148,6 +149,10 @@ pub fn generate_std_roots<'a>(
 }
 
 fn detect_sysroot_src_path(ws: &Workspace<'_>) -> CargoResult<PathBuf> {
+    if let Some(s) = env::var_os("__CARGO_TESTS_ONLY_SRC_ROOT") {
+        return Ok(s.into());
+    }
+
     // NOTE: This is temporary until we figure out how to acquire the source.
     // If we decide to keep the sysroot probe, then BuildConfig will need to
     // be restructured so that the TargetInfo is created earlier and passed