Stream the dep-graph to a file instead of storing it in-memory.

compiler/rustc_incremental/src/assert_dep_graph.rs

@@ -40,8 +40,9 @@ use rustc_graphviz as dot;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
-use rustc_middle::dep_graph::debug::{DepNodeFilter, EdgeFilter};
-use rustc_middle::dep_graph::{DepGraphQuery, DepKind, DepNode, DepNodeExt};
+use rustc_middle::dep_graph::{
+    DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter,
+};
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::{sym, Symbol};
@@ -54,7 +55,7 @@ use std::io::{BufWriter, Write};
 pub fn assert_dep_graph(tcx: TyCtxt<'_>) {
     tcx.dep_graph.with_ignore(|| {
         if tcx.sess.opts.debugging_opts.dump_dep_graph {
-            dump_graph(tcx);
+            tcx.dep_graph.with_query(dump_graph);
         }
 
         if !tcx.sess.opts.debugging_opts.query_dep_graph {
@@ -200,29 +201,29 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou
         }
         return;
     }
-    let query = tcx.dep_graph.query();
-    for &(_, source_def_id, ref source_dep_node) in if_this_changed {
-        let dependents = query.transitive_predecessors(source_dep_node);
-        for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need {
-            if !dependents.contains(&target_dep_node) {
-                tcx.sess.span_err(
-                    target_span,
-                    &format!(
-                        "no path from `{}` to `{}`",
-                        tcx.def_path_str(source_def_id),
-                        target_pass
-                    ),
-                );
-            } else {
-                tcx.sess.span_err(target_span, "OK");
+    tcx.dep_graph.with_query(|query| {
+        for &(_, source_def_id, ref source_dep_node) in if_this_changed {
+            let dependents = query.transitive_predecessors(source_dep_node);
+            for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need {
+                if !dependents.contains(&target_dep_node) {
+                    tcx.sess.span_err(
+                        target_span,
+                        &format!(
+                            "no path from `{}` to `{}`",
+                            tcx.def_path_str(source_def_id),
+                            target_pass
+                        ),
+                    );
+                } else {
+                    tcx.sess.span_err(target_span, "OK");
+                }
             }
         }
-    }
+    });
 }
 
-fn dump_graph(tcx: TyCtxt<'_>) {
+fn dump_graph(query: &DepGraphQuery) {
     let path: String = env::var("RUST_DEP_GRAPH").unwrap_or_else(|_| "dep_graph".to_string());
-    let query = tcx.dep_graph.query();
 
     let nodes = match env::var("RUST_DEP_GRAPH_FILTER") {
         Ok(string) => {

compiler/rustc_incremental/src/lib.rs

@@ -14,7 +14,7 @@ mod assert_dep_graph;
 pub mod assert_module_sources;
 mod persist;
 
-pub use assert_dep_graph::assert_dep_graph;
+use assert_dep_graph::assert_dep_graph;
 pub use persist::copy_cgu_workproduct_to_incr_comp_cache_dir;
 pub use persist::delete_workproduct_files;
 pub use persist::finalize_session_directory;
@@ -26,4 +26,4 @@ pub use persist::prepare_session_directory;
 pub use persist::save_dep_graph;
 pub use persist::save_work_product_index;
 pub use persist::LoadResult;
-pub use persist::{load_dep_graph, DepGraphFuture};
+pub use persist::{build_dep_graph, load_dep_graph, DepGraphFuture};

compiler/rustc_incremental/src/persist/dirty_clean.rs

@@ -14,7 +14,6 @@
 //! the required condition is not met.
 
 use rustc_ast::{self as ast, Attribute, NestedMetaItem};
-use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -381,39 +380,18 @@ impl DirtyCleanVisitor<'tcx> {
     fn assert_dirty(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_dirty({:?})", dep_node);
 
-        let current_fingerprint = self.get_fingerprint(&dep_node);
-        let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
-
-        if current_fingerprint == prev_fingerprint {
+        if self.tcx.dep_graph.is_green(&dep_node) {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess
                 .span_err(item_span, &format!("`{}` should be dirty but is not", dep_node_str));
         }
     }
 
-    fn get_fingerprint(&self, dep_node: &DepNode) -> Option<Fingerprint> {
-        if self.tcx.dep_graph.dep_node_exists(dep_node) {
-            let dep_node_index = self.tcx.dep_graph.dep_node_index_of(dep_node);
-            Some(self.tcx.dep_graph.fingerprint_of(dep_node_index))
-        } else {
-            None
-        }
-    }
-
     fn assert_clean(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_clean({:?})", dep_node);
 
-        let current_fingerprint = self.get_fingerprint(&dep_node);
-        let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
-
-        // if the node wasn't previously evaluated and now is (or vice versa),
-        // then the node isn't actually clean or dirty.
-        if (current_fingerprint == None) ^ (prev_fingerprint == None) {
-            return;
-        }
-
-        if current_fingerprint != prev_fingerprint {
+        if self.tcx.dep_graph.is_red(&dep_node) {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess

compiler/rustc_incremental/src/persist/fs.rs

@@ -122,6 +122,7 @@ mod tests;
 
 const LOCK_FILE_EXT: &str = ".lock";
 const DEP_GRAPH_FILENAME: &str = "dep-graph.bin";
+const STAGING_DEP_GRAPH_FILENAME: &str = "dep-graph.part.bin";
 const WORK_PRODUCTS_FILENAME: &str = "work-products.bin";
 const QUERY_CACHE_FILENAME: &str = "query-cache.bin";
 
@@ -134,6 +135,9 @@ const INT_ENCODE_BASE: usize = base_n::CASE_INSENSITIVE;
 pub fn dep_graph_path(sess: &Session) -> PathBuf {
     in_incr_comp_dir_sess(sess, DEP_GRAPH_FILENAME)
 }
+pub fn staging_dep_graph_path(sess: &Session) -> PathBuf {
+    in_incr_comp_dir_sess(sess, STAGING_DEP_GRAPH_FILENAME)
+}
 pub fn dep_graph_path_from(incr_comp_session_dir: &Path) -> PathBuf {
     in_incr_comp_dir(incr_comp_session_dir, DEP_GRAPH_FILENAME)
 }

compiler/rustc_incremental/src/persist/load.rs

@@ -5,7 +5,7 @@ use rustc_hir::definitions::Definitions;
 use rustc_middle::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::query::OnDiskCache;
 use rustc_serialize::opaque::Decoder;
-use rustc_serialize::Decodable as RustcDecodable;
+use rustc_serialize::Decodable;
 use rustc_session::Session;
 use std::path::Path;
 
@@ -120,7 +120,7 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
             // Decode the list of work_products
             let mut work_product_decoder = Decoder::new(&work_products_data[..], start_pos);
             let work_products: Vec<SerializedWorkProduct> =
-                RustcDecodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
+                Decodable::decode(&mut work_product_decoder).unwrap_or_else(|e| {
                     let msg = format!(
                         "Error decoding `work-products` from incremental \
                                     compilation session directory: {}",

compiler/rustc_incremental/src/persist/mod.rs

@@ -18,6 +18,7 @@ pub use fs::prepare_session_directory;
 pub use load::load_query_result_cache;
 pub use load::LoadResult;
 pub use load::{load_dep_graph, DepGraphFuture};
+pub use save::build_dep_graph;
 pub use save::save_dep_graph;
 pub use save::save_work_product_index;
 pub use work_product::copy_cgu_workproduct_to_incr_comp_cache_dir;

compiler/rustc_incremental/src/persist/save.rs

@@ -1,6 +1,6 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::join;
-use rustc_middle::dep_graph::{DepGraph, WorkProduct, WorkProductId};
+use rustc_middle::dep_graph::{DepGraph, PreviousDepGraph, WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_serialize::Encodable as RustcEncodable;
@@ -15,6 +15,9 @@ use super::file_format;
 use super::fs::*;
 use super::work_product;
 
+/// Save and dump the DepGraph.
+///
+/// No query must be invoked after this function.
 pub fn save_dep_graph(tcx: TyCtxt<'_>) {
     debug!("save_dep_graph()");
     tcx.dep_graph.with_ignore(|| {
@@ -29,23 +32,41 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
 
         let query_cache_path = query_cache_path(sess);
         let dep_graph_path = dep_graph_path(sess);
+        let staging_dep_graph_path = staging_dep_graph_path(sess);
+
+        sess.time("assert_dep_graph", || crate::assert_dep_graph(tcx));
+        sess.time("check_dirty_clean", || dirty_clean::check_dirty_clean_annotations(tcx));
+
+        if sess.opts.debugging_opts.incremental_info {
+            tcx.dep_graph.print_incremental_info()
+        }
 
         join(
             move || {
                 sess.time("incr_comp_persist_result_cache", || {
                     save_in(sess, query_cache_path, "query cache", |e| encode_query_cache(tcx, e));
                 });
             },
-            || {
+            move || {
                 sess.time("incr_comp_persist_dep_graph", || {
-                    save_in(sess, dep_graph_path, "dependency graph", |e| {
-                        sess.time("incr_comp_encode_dep_graph", || encode_dep_graph(tcx, e))
-                    });
+                    if let Err(err) = tcx.dep_graph.encode(&tcx.sess.prof) {
+                        sess.err(&format!(
+                            "failed to write dependency graph to `{}`: {}",
+                            staging_dep_graph_path.display(),
+                            err
+                        ));
+                    }
+                    if let Err(err) = fs::rename(&staging_dep_graph_path, &dep_graph_path) {
+                        sess.err(&format!(
+                            "failed to move dependency graph from `{}` to `{}`: {}",
+                            staging_dep_graph_path.display(),
+                            dep_graph_path.display(),
+                            err
+                        ));
+                    }
                 });
             },
         );
-
-        dirty_clean::check_dirty_clean_annotations(tcx);
     })
 }
 
@@ -92,7 +113,7 @@ pub fn save_work_product_index(
     });
 }
 
-fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
+pub(crate) fn save_in<F>(sess: &Session, path_buf: PathBuf, name: &str, encode: F)
 where
     F: FnOnce(&mut FileEncoder) -> FileEncodeResult,
 {
@@ -144,21 +165,6 @@ where
     debug!("save: data written to disk successfully");
 }
 
-fn encode_dep_graph(tcx: TyCtxt<'_>, encoder: &mut FileEncoder) -> FileEncodeResult {
-    // First encode the commandline arguments hash
-    tcx.sess.opts.dep_tracking_hash().encode(encoder)?;
-
-    if tcx.sess.opts.debugging_opts.incremental_info {
-        tcx.dep_graph.print_incremental_info();
-    }
-
-    // There is a tiny window between printing the incremental info above and encoding the dep
-    // graph below in which the dep graph could change, thus making the printed incremental info
-    // slightly out of date. If this matters to you, please feel free to submit a patch. :)
-
-    tcx.sess.time("incr_comp_encode_serialized_dep_graph", || tcx.dep_graph.encode(encoder))
-}
-
 fn encode_work_product_index(
     work_products: &FxHashMap<WorkProductId, WorkProduct>,
     encoder: &mut FileEncoder,
@@ -177,3 +183,56 @@ fn encode_work_product_index(
 fn encode_query_cache(tcx: TyCtxt<'_>, encoder: &mut FileEncoder) -> FileEncodeResult {
     tcx.sess.time("incr_comp_serialize_result_cache", || tcx.serialize_query_result_cache(encoder))
 }
+
+pub fn build_dep_graph(
+    sess: &Session,
+    prev_graph: PreviousDepGraph,
+    prev_work_products: FxHashMap<WorkProductId, WorkProduct>,
+) -> Option<DepGraph> {
+    if sess.opts.incremental.is_none() {
+        // No incremental compilation.
+        return None;
+    }
+
+    // Stream the dep-graph to an alternate file, to avoid overwriting anything in case of errors.
+    let path_buf = staging_dep_graph_path(sess);
+
+    let mut encoder = match FileEncoder::new(&path_buf) {
+        Ok(encoder) => encoder,
+        Err(err) => {
+            sess.err(&format!(
+                "failed to create dependency graph at `{}`: {}",
+                path_buf.display(),
+                err
+            ));
+            return None;
+        }
+    };
+
+    if let Err(err) = file_format::write_file_header(&mut encoder, sess.is_nightly_build()) {
+        sess.err(&format!(
+            "failed to write dependency graph header to `{}`: {}",
+            path_buf.display(),
+            err
+        ));
+        return None;
+    }
+
+    // First encode the commandline arguments hash
+    if let Err(err) = sess.opts.dep_tracking_hash().encode(&mut encoder) {
+        sess.err(&format!(
+            "failed to write dependency graph hash `{}`: {}",
+            path_buf.display(),
+            err
+        ));
+        return None;
+    }
+
+    Some(DepGraph::new(
+        prev_graph,
+        prev_work_products,
+        encoder,
+        sess.opts.debugging_opts.query_dep_graph,
+        sess.opts.debugging_opts.incremental_info,
+    ))
+}

compiler/rustc_interface/src/passes.rs

@@ -1021,9 +1021,6 @@ pub fn start_codegen<'tcx>(
         rustc_symbol_mangling::test::report_symbol_names(tcx);
     }
 
-    tcx.sess.time("assert_dep_graph", || rustc_incremental::assert_dep_graph(tcx));
-    tcx.sess.time("serialize_dep_graph", || rustc_incremental::save_dep_graph(tcx));
-
     info!("Post-codegen\n{:?}", tcx.debug_stats());
 
     if tcx.sess.opts.output_types.contains_key(&OutputType::Mir) {

compiler/rustc_interface/src/queries.rs

@@ -207,7 +207,13 @@ impl<'tcx> Queries<'tcx> {
                                 })
                                 .open(self.session())
                         });
-                    DepGraph::new(prev_graph, prev_work_products)
+
+                    rustc_incremental::build_dep_graph(
+                        self.session(),
+                        prev_graph,
+                        prev_work_products,
+                    )
+                    .unwrap_or_else(DepGraph::new_disabled)
                 }
             })
         })
@@ -435,6 +441,9 @@ impl Compiler {
             if self.session().opts.debugging_opts.query_stats {
                 gcx.enter(rustc_query_impl::print_stats);
             }
+
+            self.session()
+                .time("serialize_dep_graph", || gcx.enter(rustc_incremental::save_dep_graph));
         }
 
         _timer = Some(self.session().timer("free_global_ctxt"));

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -8,8 +8,8 @@ use rustc_session::Session;
 mod dep_node;
 
 pub use rustc_query_system::dep_graph::{
-    debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, SerializedDepNodeIndex,
-    WorkProduct, WorkProductId,
+    debug::DepNodeFilter, hash_result, DepContext, DepNodeColor, DepNodeIndex,
+    SerializedDepNodeIndex, WorkProduct, WorkProductId,
 };
 
 crate use dep_node::make_compile_codegen_unit;
@@ -20,6 +20,7 @@ pub type TaskDeps = rustc_query_system::dep_graph::TaskDeps<DepKind>;
 pub type DepGraphQuery = rustc_query_system::dep_graph::DepGraphQuery<DepKind>;
 pub type PreviousDepGraph = rustc_query_system::dep_graph::PreviousDepGraph<DepKind>;
 pub type SerializedDepGraph = rustc_query_system::dep_graph::SerializedDepGraph<DepKind>;
+pub type EdgeFilter = rustc_query_system::dep_graph::debug::EdgeFilter<DepKind>;
 
 impl rustc_query_system::dep_graph::DepKind for DepKind {
     const NULL: Self = DepKind::Null;

compiler/rustc_query_impl/src/plumbing.rs

@@ -477,10 +477,7 @@ macro_rules! define_queries {
                         return
                     }
 
-                    debug_assert!(tcx.dep_graph
-                                     .node_color(dep_node)
-                                     .map(|c| c.is_green())
-                                     .unwrap_or(false));
+                    debug_assert!(tcx.dep_graph.is_green(dep_node));
 
                     let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
                     if queries::$name::cache_on_disk(tcx, &key, None) {