Change to new, experimental cargo feature resolver

The [new cargo feature resolver](rust-lang/cargo#7820)
won't unify features across build deps, dev deps, and targets. This solves our
problem of needing to work around feature unification to avoid Clone
implementations on private key material.

This commit puts back our true dependency information into the
Cargo.tomls. Specifically, it adds dev-dependencies that enable features that
aren't enabled on normal dependencies. This will cause the feature unification
to happen when using the old resolver, but the build will be correct under the
new resolver.

In order to maintain correct builds in CI, this commit also changes CI to use
the nightly cargo with `-Z features=all` but still preserving the rustc
toolchain specified in `rustc-toolchain`. Local developer builds will likely
still use the `rustc-toolchain` version of cargo with the old resolver, but
this shouldn't cause any problems for development.

.circleci/config.yml

@@ -15,14 +15,6 @@ executors:
     resource_class: small
 
 commands:
-  rust_setup:
-    description: Set rustc version
-    steps:
-      - run:
-          name: Set rustc version
-          command: |
-            rustup default stable
-            rustup update stable
   print_versions:
     description: Version Info
     steps:
@@ -40,6 +32,7 @@ commands:
             echo 'export LIBRA_DUMP_LOGS=1' >> $BASH_ENV
             echo 'export CARGO_INCREMENTAL=0' >> $BASH_ENV
             echo 'export CI_TIMEOUT="timeout 40m"' >> $BASH_ENV
+            echo 'export RUST_NIGHTLY=nightly-2020-03-18' >> $BASH_ENV
   install_deps:
     steps:
       - run:
@@ -48,14 +41,20 @@ commands:
             sudo apt-get update
             sudo apt-get install -y cmake curl clang llvm
             rustup component add clippy rustfmt
+            rustup toolchain install $RUST_NIGHTLY
+      - run:
+          name: Set cargo Environment
+          command: |
+            echo 'export CARGO=$(rustup which --toolchain $RUST_NIGHTLY cargo)' >> $BASH_ENV
+            echo 'export CARGOFLAGS=-Zfeatures=all' >> $BASH_ENV
   install_code_coverage_deps:
     steps:
       - run:
           name: Install grcov and lcov
           command: |
             sudo apt-get update
             sudo apt-get install lcov
-            cargo install --force grcov
+            $(CARGO) install --force grcov
   install_docker_linter:
     steps:
       - run:
@@ -64,11 +63,6 @@ commands:
              export HADOLINT=${HOME}/hadolint
              export HADOLINT_VER=v1.17.4
              curl -sL -o ${HADOLINT} "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VER}/hadolint-$(uname -s)-$(uname -m)" && chmod 700 ${HADOLINT}
-  install_rust_nightly_toolchain:
-    steps:
-      - run:
-          name: Install nightly toolchain for features not in beta/stable
-          command: rustup install nightly
   find_dockerfile_changes:
     steps:
       - run:
@@ -82,7 +76,6 @@ commands:
   build_setup:
     steps:
       - checkout
-      - rust_setup
       - print_versions
       - env_setup
       - install_deps
@@ -104,37 +97,37 @@ jobs:
       - run:
           name: Linting
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo x lint
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xclippy --workspace --all-targets
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo xfmt --check
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || cargo install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
-            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $(cargo guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS x lint
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS xclippy --workspace --all-targets
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS xfmt --check
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || $CARGO $CARGOFLAGS install cargo-guppy --git http://github.com/calibra/cargo-guppy --rev 8b2bc45c0cd6323a7a2b8170ddad6d2a5b79047b
+            [[ $CIRCLE_NODE_INDEX =~ [1234] ]] || [[ -z $($CARGO $CARGOFLAGS guppy dups --target x86_64-unknown-linux-gnu --kind directthirdparty) ]]
       - run:
           name: Build Release
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 cargo build -j 16 --release
+            [[ $CIRCLE_NODE_INDEX =~ [0234] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 --release
       - run:
           name: Build Dev
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-swarm
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cluster-test
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p libra-fuzzer
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p language_benchmarks
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p cost-synthesis
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 cargo build -j 16 -p test-generation
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-swarm
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cluster-test
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p libra-fuzzer
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p language_benchmarks
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p cost-synthesis
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CARGO $CARGOFLAGS build -j 16 -p test-generation
       - run:
           name: Run All Non Flaky Unit Tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
+            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGS test --all-features --workspace --exclude libra-node --exclude libra-crypto --exclude testsuite --exclude consensus
       - run:
           name: Run Cryptography Unit Tests with the formally verified backend
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT cargo test --features='std fiat_u64_backend fuzzing' --no-default-features )
+            [[ $CIRCLE_NODE_INDEX =~ [0134] ]] || ( RUST_BACKTRACE=1 cd crypto/crypto && $CI_TIMEOUT $CARGO $CARGOFLAGS test --features='std fiat_u64_backend fuzzing' --no-default-features )
       - run:
           name: Run All End to End Tests
           command: |
-            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT cargo x test --package testsuite -- --test-threads 1
+            [[ $CIRCLE_NODE_INDEX =~ [0124] ]] || RUST_BACKTRACE=1 $CI_TIMEOUT $CARGO $CARGOFLAGSx test --package testsuite -- --test-threads 1
       - run:
           name: Run Quarantined Unit Tests 3 (consensus) times
           command: |
@@ -155,16 +148,16 @@ jobs:
     steps:
       - build_setup
       - run:
-          name: Install Cargo Audit
+          name: Install cargo-audit
           command: |
-            cargo install --force cargo-audit
+            $CARGO $CARGOFLAGS install --force cargo-audit
       - run:
           # NOTE ignored advisory rules
           # RUSTSEC-2018-0015 - term
           # RUSTSEC-2019-0031 - spin
           name: Audit crates
           command: |
-            cargo audit --deny-warnings \
+            $CARGO $CARGOFLAGS audit --deny-warnings \
               --ignore RUSTSEC-2018-0015 \
               --ignore RUSTSEC-2019-0031
       - build_teardown
@@ -174,7 +167,6 @@ jobs:
     steps:
       - build_setup
       - install_code_coverage_deps
-      - install_rust_nightly_toolchain
       - run:
           name: Setup code coverage output
           command: echo "export CODECOV_OUTPUT=codecov" >> $BASH_ENV

Cargo.toml

@@ -108,7 +108,8 @@ members = [
 
 # NOTE: These members should never include crates that require fuzzing
 # features or test features. These are the crates we want built with no extra
-# test-only code included.
+# test-only code included. These are essentially the main libra release
+# binaries.
 default-members = [
     "client/cli",
     "language/compiler",

admission_control/admission-control-service/Cargo.toml

@@ -32,6 +32,12 @@ serde_json = "1.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"
+proptest = "0.9.4"
+libra-mempool = { path = "../../mempool", version = "0.1.0", features = ["fuzzing"] }
+libra-proptest-helpers = { path = "../../common/proptest-helpers" }
+libra-prost-ext = { path = "../../common/prost-ext", version = "0.1.0" }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
+storage-service = { path = "../../storage/storage-service" }
 vm-validator = { path = "../../vm-validator", version = "0.1.0" }
 
 [features]

client/cli/Cargo.toml

@@ -42,6 +42,7 @@ transaction-builder = { path = "../../language/transaction-builder", version = "
 
 [dev-dependencies]
 proptest = "0.9.2"
+libra-crypto = { path = "../../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

config/Cargo.toml

@@ -29,6 +29,9 @@ libra-logger = { path = "../common/logger", version = "0.1.0" }
 libra-temppath = { path = "../common/temppath", version = "0.1.0" }
 libra-types = { path = "../types", version = "0.1.0" }
 
+[dev-dependencies]
+libra-crypto = { path = "../crypto/crypto", version = "0.1.0", features = ["fuzzing"] }
+
 [features]
 default = []
 fuzzing = ["libra-crypto/fuzzing", "libra-types/fuzzing"]

consensus/Cargo.toml

@@ -31,7 +31,7 @@ prometheus = { version = "0.8.0", default-features = false }
 proptest = { version = "0.9.4", optional = true }
 
 channel = { path = "../common/channel", version = "0.1.0" }
-consensus-types = { path = "consensus-types", version = "0.1.0", default-features = false }
+consensus-types = { path = "consensus-types", version = "0.1.0" }
 crash-handler = { path = "../common/crash-handler", version = "0.1.0" }
 debug-interface = { path = "../common/debug-interface", version = "0.1.0" }
 executor = { path = "../execution/executor", version = "0.1.0" }
@@ -57,6 +57,9 @@ cached = "0.12.0"
 proptest = "0.9.4"
 tempfile = "3.1.0"
 
+consensus-types = { path = "consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../config", version = "0.1.0", features = ["fuzzing"] }
+libra-mempool = { path = "../mempool", version = "0.1.0", features = ["fuzzing"] }
 vm-genesis = { path = "../language/tools/vm-genesis", version = "0.1.0" }
 vm-validator = { path = "../vm-validator", version = "0.1.0" }
 

consensus/consensus-types/Cargo.toml

@@ -21,6 +21,7 @@ libra-types = { path = "../../types", version = "0.1.0" }
 
 [dev-dependencies]
 proptest = "0.9.4"
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

consensus/safety-rules/Cargo.toml

@@ -28,6 +28,8 @@ workspace-builder = { path = "../../common/workspace-builder", version = "0.1.0"
 criterion = "0.3"
 rand = { version = "0.6.5", default-features = false }
 tempfile = "3.1.0"
+consensus-types = { path = "../consensus-types", version = "0.1.0", features = ["fuzzing"] }
+libra-config = { path = "../../config", version = "0.1.0", features = ["fuzzing"] }
 
 [[bench]]
 name = "safety_rules"

consensus/src/chained_bft/chained_bft_smr_test.rs

@@ -44,7 +44,6 @@ struct SMRNode {
     commit_cb_receiver: mpsc::UnboundedReceiver<LedgerInfoWithSignatures>,
     storage: Arc<MockStorage<TestPayload>>,
     state_sync: mpsc::UnboundedReceiver<Vec<usize>>,
-    shared_mempool: MockSharedMempool,
 }
 
 impl SMRNode {
@@ -75,7 +74,7 @@ impl SMRNode {
         let (state_sync_client, state_sync) = mpsc::unbounded();
         let (commit_cb_sender, commit_cb_receiver) = mpsc::unbounded::<LedgerInfoWithSignatures>();
         let shared_mempool = MockSharedMempool::new(None);
-        let consensus_to_mempool_sender = shared_mempool.consensus_sender.clone();
+        let consensus_to_mempool_sender = shared_mempool.consensus_sender;
 
         let mut smr = ChainedBftSMR::new(
             network_sender,
@@ -101,7 +100,6 @@ impl SMRNode {
             commit_cb_receiver,
             storage,
             state_sync,
-            shared_mempool,
         }
     }
 

consensus/src/chained_bft/event_processor_test.rs

@@ -49,7 +49,7 @@ use libra_types::{
     block_info::BlockInfo,
     ledger_info::LedgerInfoWithSignatures,
     validator_signer::ValidatorSigner,
-    validator_verifier::{random_validator_verifier, ValidatorVerifier},
+    validator_verifier::{random_validator_verifier},
 };
 use network::peer_manager::{
     conn_status_channel, ConnectionRequestSender, PeerManagerRequestSender,
@@ -65,7 +65,6 @@ pub struct NodeSetup {
     storage: Arc<MockStorage<TestPayload>>,
     signer: ValidatorSigner,
     proposer_author: Author,
-    validators: Arc<ValidatorVerifier>,
     safety_rules_manager: SafetyRulesManager<TestPayload>,
 }
 
@@ -143,7 +142,7 @@ impl NodeSetup {
         playground.add_node(author, consensus_tx, network_reqs_rx, conn_mgr_reqs_rx);
 
         let (self_sender, self_receiver) = channel::new_test(8);
-        let network = NetworkSender::new(author, network_sender, self_sender, validators.clone());
+        let network = NetworkSender::new(author, network_sender, self_sender, validators);
 
         let (task, _receiver) = NetworkTask::<TestPayload>::new(network_events, self_receiver);
 
@@ -199,7 +198,6 @@ impl NodeSetup {
             storage,
             signer,
             proposer_author,
-            validators,
             safety_rules_manager,
         }
     }

consensus/src/chained_bft/test_utils/mod.rs

@@ -61,20 +61,6 @@ pub fn build_simple_tree() -> (
     (vec![genesis_block, a1, a2, a3, b1, b2, c1], block_store)
 }
 
-pub fn build_chain() -> Vec<Arc<ExecutedBlock<TestPayload>>> {
-    let mut inserter = TreeInserter::default();
-    let block_store = inserter.block_store();
-    let genesis = block_store.root();
-    let a1 = inserter.insert_block_with_qc(certificate_for_genesis(), &genesis, 1);
-    let a2 = inserter.insert_block(&a1, 2, None);
-    let a3 = inserter.insert_block(&a2, 3, Some(genesis.block_info()));
-    let a4 = inserter.insert_block(&a3, 4, Some(a1.block_info()));
-    let a5 = inserter.insert_block(&a4, 5, Some(a2.block_info()));
-    let a6 = inserter.insert_block(&a5, 6, Some(a3.block_info()));
-    let a7 = inserter.insert_block(&a6, 7, Some(a4.block_info()));
-    vec![genesis, a1, a2, a3, a4, a5, a6, a7]
-}
-
 pub fn build_empty_tree() -> Arc<BlockStore<TestPayload>> {
     let (initial_data, storage) = EmptyStorage::start_for_testing();
     Arc::new(BlockStore::new(

execution/executor/Cargo.toml

@@ -42,7 +42,8 @@ executor-utils = { path = "../executor-utils", version = "0.1.0" }
 storage-service = { path = "../../storage/storage-service", version = "0.1.0" }
 stdlib = { path = "../../language/stdlib", version = "0.1.0" }
 transaction-builder = { path = "../../language/transaction-builder", version = "0.1.0" }
-
+libra-config = { path = "../../config", version = "0.1.0", features = ["fuzzing"] }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 vm-genesis = { path = "../../language/tools/vm-genesis", version = "0.1.0" }
 
 [features]

json-rpc/Cargo.toml

@@ -37,6 +37,12 @@ libra-temppath = { path = "../common/temppath", version = "0.1.0", optional = tr
 storage-proto = { path = "../storage/storage-proto", version = "0.1.0" }
 
 [dev-dependencies]
+proptest = "0.9.2"
+libra-mempool = { path = "../mempool", version = "0.1.0", features = ["fuzzing"] }
+libra-proptest-helpers = { path = "../common/proptest-helpers" }
+libra-temppath = { path = "../common/temppath", version = "0.1.0" }
+libra-types = { path = "../types", version = "0.1.0", features = ["fuzzing"] }
+libradb = { path = "../storage/libradb", version = "0.1.0", features = ["fuzzing"] }
 vm-validator = { path = "../vm-validator", version = "0.1.0" }
 
 [features]

language/libra-vm/Cargo.toml

@@ -36,6 +36,7 @@ serde = { version = "1.0.105", default-features = false }
 
 [dev-dependencies]
 proptest = "0.9"
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

language/move-vm/types/Cargo.toml

@@ -24,6 +24,7 @@ move-core-types = { path = "../../move-core/types", version = "0.1.0" }
 
 [dev-dependencies]
 proptest = "0.9"
+vm = { path = "../../vm", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

language/vm/Cargo.toml

@@ -30,8 +30,9 @@ num-variants = { path = "../../common/num-variants", version = "0.1.0" }
 [dev-dependencies]
 proptest = "0.9"
 proptest-derive = "0.1.1"
-libra-proptest-helpers = { path = "../../common/proptest-helpers", version = "0.1.0" }
 serde_json = "1"
+libra-proptest-helpers = { path = "../../common/proptest-helpers", version = "0.1.0" }
+libra-types = { path = "../../types", version = "0.1.0", features = ["fuzzing"] }
 
 [features]
 default = []

mempool/Cargo.toml

@@ -44,6 +44,7 @@ storage-service = { path = "../storage/storage-service", version = "0.1.0", opti
 [dev-dependencies]
 parity-multiaddr = { version = "0.7.2", default-features = false }
 rand = "0.6.5"
+storage-service = { path = "../storage/storage-service", version = "0.1.0", features = ["fuzzing"] }
 
 [build-dependencies]
 tonic-build = "0.1"

mempool/src/lib.rs

@@ -60,7 +60,7 @@
 extern crate prometheus;
 
 /// This module provides mocks of shared mempool for tests.
-#[cfg(feature = "fuzzing")]
+#[cfg(any(test, feature = "fuzzing"))]
 pub mod mocks;
 pub use shared_mempool::{
     bootstrap, generate_reconfig_subscription, CommitNotification, CommitResponse,

network/Cargo.toml

@@ -43,6 +43,9 @@ proptest = { version = "0.9.4", default-features = true, optional = true }
 [dev-dependencies]
 criterion = "=0.3.1"
 noise = { path = "noise", version = "0.1.0", features = ["testing"] }
+proptest = { version = "0.9.4", default-features = false }
+libra-proptest-helpers = { path = "../common/proptest-helpers", version = "0.1.0" }
+libra-types = { path = "../types", version = "0.1.0", features = ["fuzzing"] }
 socket-bench-server = { path = "socket-bench-server", version = "0.1.0" }
 
 [features]