From 5384ce8dcba5fed7ba4c240617182667da74620a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 16 Apr 2019 10:45:01 +0200 Subject: [PATCH 1/9] Fix inherent_impls --- .../coherence/inherent_impls.rs | 33 +++---------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/src/librustc_typeck/coherence/inherent_impls.rs b/src/librustc_typeck/coherence/inherent_impls.rs index 6088c03fc0681..0b04160522047 100644 --- a/src/librustc_typeck/coherence/inherent_impls.rs +++ b/src/librustc_typeck/coherence/inherent_impls.rs @@ -7,7 +7,6 @@ //! `tcx.inherent_impls(def_id)`). That value, however, //! is computed by selecting an idea from this table. -use rustc::dep_graph::DepKind; use rustc::hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc::hir; use rustc::hir::itemlikevisit::ItemLikeVisitor; @@ -36,35 +35,11 @@ pub fn crate_inherent_impls<'tcx>( pub fn inherent_impls<'tcx>(tcx: TyCtxt<'tcx>, ty_def_id: DefId) -> &'tcx [DefId] { assert!(ty_def_id.is_local()); - // NB. Until we adopt the red-green dep-tracking algorithm (see - // [the plan] for details on that), we do some hackery here to get - // the dependencies correct. Basically, we use a `with_ignore` to - // read the result we want. If we didn't have the `with_ignore`, - // we would wind up with a dependency on the entire crate, which - // we don't want. Then we go and add dependencies on all the impls - // in the result (which is what we wanted). - // - // The result is a graph with an edge from `Hir(I)` for every impl - // `I` defined on some type `T` to `CoherentInherentImpls(T)`, - // thus ensuring that if any of those impls change, the set of - // inherent impls is considered dirty. - // - // [the plan]: https://github.com/rust-lang/rust-roadmap/issues/4 - - let result = tcx.dep_graph.with_ignore(|| { - let crate_map = tcx.crate_inherent_impls(ty_def_id.krate); - match crate_map.inherent_impls.get(&ty_def_id) { - Some(v) => &v[..], - None => &[], - } - }); - - for &impl_def_id in &result[..] { - let def_path_hash = tcx.def_path_hash(impl_def_id); - tcx.dep_graph.read(def_path_hash.to_dep_node(DepKind::Hir)); + let crate_map = tcx.crate_inherent_impls(ty_def_id.krate); + match crate_map.inherent_impls.get(&ty_def_id) { + Some(v) => &v[..], + None => &[], } - - result } struct InherentCollect<'tcx> { From 5afd52351e8b3f2ec54793f7e256ba985084cdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 15 Mar 2019 12:17:11 +0100 Subject: [PATCH 2/9] Add an AtomicCell abstraction --- src/librustc_data_structures/Cargo.toml | 1 + src/librustc_data_structures/sync.rs | 71 ++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/librustc_data_structures/Cargo.toml b/src/librustc_data_structures/Cargo.toml index cd792d31187bd..fc58da4335025 100644 --- a/src/librustc_data_structures/Cargo.toml +++ b/src/librustc_data_structures/Cargo.toml @@ -19,6 +19,7 @@ rustc_cratesio_shim = { path = "../librustc_cratesio_shim" } serialize = { path = "../libserialize" } graphviz = { path = "../libgraphviz" } cfg-if = "0.1.2" +crossbeam-utils = { version = "0.6.5", features = ["nightly"] } stable_deref_trait = "1.0.0" rayon = { version = "0.2.0", package = "rustc-rayon" } rayon-core = { version = "0.2.0", package = "rustc-rayon-core" } diff --git a/src/librustc_data_structures/sync.rs b/src/librustc_data_structures/sync.rs index 73247c1469efd..183fab095b9ce 100644 --- a/src/librustc_data_structures/sync.rs +++ b/src/librustc_data_structures/sync.rs @@ -67,6 +67,71 @@ cfg_if! { use std::ops::Add; use std::panic::{resume_unwind, catch_unwind, AssertUnwindSafe}; + #[derive(Debug)] + pub struct AtomicCell(Cell); + + impl AtomicCell { + #[inline] + pub fn new(v: T) -> Self { + AtomicCell(Cell::new(v)) + } + + #[inline] + pub fn get_mut(&mut self) -> &mut T { + self.0.get_mut() + } + } + + impl AtomicCell { + pub fn into_inner(self) -> T { + self.0.into_inner() + } + + #[inline] + pub fn load(&self) -> T { + self.0.get() + } + + #[inline] + pub fn store(&self, val: T) { + self.0.set(val) + } + + pub fn swap(&self, val: T) -> T { + self.0.replace(val) + } + } + + impl AtomicCell { + pub fn compare_and_swap(&self, current: T, new: T) -> T { + match self.compare_exchange(current, new) { + Ok(v) => v, + Err(v) => v, + } + } + + pub fn compare_exchange(&self, + current: T, + new: T) + -> Result { + let read = self.0.get(); + if read == current { + self.0.set(new); + Ok(read) + } else { + Err(read) + } + } + } + + impl + Copy> AtomicCell { + pub fn fetch_add(&self, val: T) -> T { + let old = self.0.get(); + self.0.set(old + val); + old + } + } + #[derive(Debug)] pub struct Atomic(Cell); @@ -77,7 +142,7 @@ cfg_if! { } } - impl Atomic { + impl Atomic { pub fn into_inner(self) -> T { self.0.into_inner() } @@ -95,7 +160,9 @@ cfg_if! { pub fn swap(&self, val: T, _: Ordering) -> T { self.0.replace(val) } + } + impl Atomic { pub fn compare_exchange(&self, current: T, new: T, @@ -271,6 +338,8 @@ cfg_if! { pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; + pub use crossbeam_utils::atomic::AtomicCell; + pub use std::sync::Arc as Lrc; pub use std::sync::Weak as Weak; From 136783d7da4840817d4be25d1bb0f1316d2cc673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 18 Jun 2019 03:38:45 +0200 Subject: [PATCH 3/9] Update Cargo.lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 40b8cf507e97c..6d705ded6ca57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,6 +2893,7 @@ name = "rustc_data_structures" version = "0.0.0" dependencies = [ "cfg-if 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "ena 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "graphviz 0.0.0", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", From 241aad38425da2d06dd00f7ac2376c86573d5245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 19 Apr 2019 18:49:15 +0200 Subject: [PATCH 4/9] Refactor diagnostic emission for green nodes --- src/librustc/dep_graph/graph.rs | 79 ++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index c2e3c12cea8ed..35bd7ce9e73de 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -7,6 +7,7 @@ use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering}; use std::env; use std::hash::Hash; use std::collections::hash_map::Entry; +use std::mem; use crate::ty::{self, TyCtxt}; use crate::util::common::{ProfileQueriesMsg, profq_msg}; use parking_lot::{Mutex, Condvar}; @@ -61,11 +62,11 @@ struct DepGraphData { colors: DepNodeColorMap, - /// A set of loaded diagnostics that have been emitted. - emitted_diagnostics: Mutex>, + /// A set of loaded diagnostics that is in the progress of being emitted. + emitting_diagnostics: Mutex>, /// Used to wait for diagnostics to be emitted. - emitted_diagnostics_cond_var: Condvar, + emitting_diagnostics_cond_var: Condvar, /// When we load, there may be `.o` files, cached MIR, or other such /// things available to us. If we find that they are not dirty, we @@ -99,8 +100,8 @@ impl DepGraph { previous_work_products: prev_work_products, dep_node_debug: Default::default(), current: Lock::new(CurrentDepGraph::new(prev_graph_node_count)), - emitted_diagnostics: Default::default(), - emitted_diagnostics_cond_var: Condvar::new(), + emitting_diagnostics: Default::default(), + emitting_diagnostics_cond_var: Condvar::new(), previous: prev_graph, colors: DepNodeColorMap::new(prev_graph_node_count), loaded_from_cache: Default::default(), @@ -744,7 +745,7 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently - let (dep_node_index, did_allocation) = { + let dep_node_index = { let mut current = data.current.borrow_mut(); // Copy the fingerprint from the previous graph, @@ -758,17 +759,22 @@ impl DepGraph { // ... emitting any stored diagnostic ... + // FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere + // Maybe store a list on disk and encode this fact in the DepNodeState let diagnostics = tcx.queries.on_disk_cache - .load_diagnostics(tcx, prev_dep_node_index); + .load_diagnostics(tcx, prev_dep_node_index); if unlikely!(diagnostics.len() > 0) { self.emit_diagnostics( tcx, data, dep_node_index, - did_allocation, + prev_dep_node_index, diagnostics ); + } else { + // Avoid calling the destructor, since LLVM fails to optimize it away + mem::forget(diagnostics); } // ... and finally storing a "Green" entry in the color map. @@ -784,8 +790,8 @@ impl DepGraph { Some(dep_node_index) } - /// Atomically emits some loaded diagnotics, assuming that this only gets called with - /// `did_allocation` set to `true` on a single thread. + /// Atomically emits some loaded diagnotics. + /// This may be called concurrently on multiple threads for the same dep node. #[cold] #[inline(never)] fn emit_diagnostics<'tcx>( @@ -793,36 +799,49 @@ impl DepGraph { tcx: TyCtxt<'tcx>, data: &DepGraphData, dep_node_index: DepNodeIndex, - did_allocation: bool, + prev_dep_node_index: SerializedDepNodeIndex, diagnostics: Vec, ) { - if did_allocation || !cfg!(parallel_compiler) { - // Only the thread which did the allocation emits the error messages - let handle = tcx.sess.diagnostic(); + let mut emitting = data.emitting_diagnostics.lock(); + + if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) { + // The node is already green so diagnostics must have been emitted already + return; + } + + if emitting.insert(dep_node_index) { + // We were the first to insert the node in the set so this thread + // must emit the diagnostics and signal other potentially waiting + // threads after. + mem::drop(emitting); // Promote the previous diagnostics to the current session. tcx.queries.on_disk_cache - .store_diagnostics(dep_node_index, diagnostics.clone().into()); + .store_diagnostics(dep_node_index, diagnostics.clone().into()); + + let handle = tcx.sess.diagnostic(); for diagnostic in diagnostics { DiagnosticBuilder::new_diagnostic(handle, diagnostic).emit(); } - #[cfg(parallel_compiler)] - { - // Mark the diagnostics and emitted and wake up waiters - data.emitted_diagnostics.lock().insert(dep_node_index); - data.emitted_diagnostics_cond_var.notify_all(); - } + // Mark the node as green now that diagnostics are emitted + data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + + // Remove the node from the set + data.emitting_diagnostics.lock().remove(&dep_node_index); + + // Wake up waiters + data.emitting_diagnostics_cond_var.notify_all(); } else { - // The other threads will wait for the diagnostics to be emitted + // We must wait for the other thread to finish emitting the diagnostic - let mut emitted_diagnostics = data.emitted_diagnostics.lock(); loop { - if emitted_diagnostics.contains(&dep_node_index) { + data.emitting_diagnostics_cond_var.wait(&mut emitting); + if data.colors + .get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) { break; } - data.emitted_diagnostics_cond_var.wait(&mut emitted_diagnostics); } } } @@ -1038,7 +1057,7 @@ impl CurrentDepGraph { hash: self.anon_id_seed.combine(hasher.finish()), }; - self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO).0 + self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO) } fn alloc_node( @@ -1048,7 +1067,7 @@ impl CurrentDepGraph { fingerprint: Fingerprint ) -> DepNodeIndex { debug_assert!(!self.node_to_node_index.contains_key(&dep_node)); - self.intern_node(dep_node, edges, fingerprint).0 + self.intern_node(dep_node, edges, fingerprint) } fn intern_node( @@ -1056,11 +1075,11 @@ impl CurrentDepGraph { dep_node: DepNode, edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint - ) -> (DepNodeIndex, bool) { + ) -> DepNodeIndex { debug_assert_eq!(self.node_to_node_index.len(), self.data.len()); match self.node_to_node_index.entry(dep_node) { - Entry::Occupied(entry) => (*entry.get(), false), + Entry::Occupied(entry) => *entry.get(), Entry::Vacant(entry) => { let dep_node_index = DepNodeIndex::new(self.data.len()); self.data.push(DepNodeData { @@ -1069,7 +1088,7 @@ impl CurrentDepGraph { fingerprint }); entry.insert(dep_node_index); - (dep_node_index, true) + dep_node_index } } } From 9b7150ac2d210a6c2b2a6f30b55af46beca1c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 12 Jun 2019 14:39:12 +0200 Subject: [PATCH 5/9] Use sharded maps for interning and queries --- src/librustc/ty/context.rs | 42 ++++---- src/librustc/ty/query/config.rs | 4 +- src/librustc/ty/query/on_disk_cache.rs | 12 +-- src/librustc/ty/query/plumbing.rs | 44 ++++---- src/librustc_data_structures/interner.rs | 58 ---------- src/librustc_data_structures/lib.rs | 2 +- src/librustc_data_structures/sharded.rs | 129 +++++++++++++++++++++++ 7 files changed, 183 insertions(+), 108 deletions(-) delete mode 100644 src/librustc_data_structures/interner.rs create mode 100644 src/librustc_data_structures/sharded.rs diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index ce785f9a0246b..bf8df59bed0e1 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -48,7 +48,6 @@ use crate::util::common::ErrorReported; use crate::util::nodemap::{DefIdMap, DefIdSet, ItemLocalMap, ItemLocalSet}; use crate::util::nodemap::{FxHashMap, FxHashSet}; use errors::DiagnosticBuilder; -use rustc_data_structures::interner::HashInterner; use smallvec::SmallVec; use rustc_data_structures::stable_hasher::{HashStable, hash_stable_hashmap, StableHasher, StableHasherResult, @@ -56,6 +55,7 @@ use rustc_data_structures::stable_hasher::{HashStable, hash_stable_hashmap, use arena::SyncDroplessArena; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use rustc_data_structures::sync::{Lrc, Lock, WorkerLocal}; +use rustc_data_structures::sharded::ShardedHashMap; use std::any::Any; use std::borrow::Borrow; use std::cmp::Ordering; @@ -93,7 +93,7 @@ impl AllArenas { } } -type InternedSet<'tcx, T> = Lock, ()>>; +type InternedSet<'tcx, T> = ShardedHashMap, ()>; pub struct CtxtInterners<'tcx> { /// The arena that types, regions, etc are allocated from @@ -147,7 +147,7 @@ impl<'tcx> CtxtInterners<'tcx> { // determine that all contents are in the global tcx. // See comments on Lift for why we can't use that. if flags.flags.intersects(ty::TypeFlags::KEEP_IN_LOCAL_TCX) { - local.type_.borrow_mut().intern(st, |st| { + local.type_.intern(st, |st| { let ty_struct = TyS { sty: st, flags: flags.flags, @@ -171,7 +171,7 @@ impl<'tcx> CtxtInterners<'tcx> { Interned(local.arena.alloc(ty_struct)) }).0 } else { - global.type_.borrow_mut().intern(st, |st| { + global.type_.intern(st, |st| { let ty_struct = TyS { sty: st, flags: flags.flags, @@ -964,7 +964,7 @@ impl<'tcx> CommonTypes<'tcx> { impl<'tcx> CommonLifetimes<'tcx> { fn new(interners: &CtxtInterners<'tcx>) -> CommonLifetimes<'tcx> { let mk = |r| { - interners.region.borrow_mut().intern(r, |r| { + interners.region.intern(r, |r| { Interned(interners.arena.alloc(r)) }).0 }; @@ -980,7 +980,7 @@ impl<'tcx> CommonLifetimes<'tcx> { impl<'tcx> CommonConsts<'tcx> { fn new(interners: &CtxtInterners<'tcx>, types: &CommonTypes<'tcx>) -> CommonConsts<'tcx> { let mk_const = |c| { - interners.const_.borrow_mut().intern(c, |c| { + interners.const_.intern(c, |c| { Interned(interners.arena.alloc(c)) }).0 }; @@ -1096,14 +1096,14 @@ pub struct GlobalCtxt<'tcx> { /// Data layout specification for the current target. pub data_layout: TargetDataLayout, - stability_interner: Lock>, + stability_interner: ShardedHashMap<&'tcx attr::Stability, ()>, /// Stores the value of constants (and deduplicates the actual memory) - allocation_interner: Lock>, + allocation_interner: ShardedHashMap<&'tcx Allocation, ()>, pub alloc_map: Lock>, - layout_interner: Lock>, + layout_interner: ShardedHashMap<&'tcx LayoutDetails, ()>, /// A general purpose channel to throw data out the back towards LLVM worker /// threads. @@ -1148,7 +1148,7 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn intern_const_alloc(self, alloc: Allocation) -> &'tcx Allocation { - self.allocation_interner.borrow_mut().intern(alloc, |alloc| { + self.allocation_interner.intern(alloc, |alloc| { self.arena.alloc(alloc) }) } @@ -1162,13 +1162,13 @@ impl<'tcx> TyCtxt<'tcx> { } pub fn intern_stability(self, stab: attr::Stability) -> &'tcx attr::Stability { - self.stability_interner.borrow_mut().intern(stab, |stab| { + self.stability_interner.intern(stab, |stab| { self.arena.alloc(stab) }) } pub fn intern_layout(self, layout: LayoutDetails) -> &'tcx LayoutDetails { - self.layout_interner.borrow_mut().intern(layout, |layout| { + self.layout_interner.intern(layout, |layout| { self.arena.alloc(layout) }) } @@ -2110,7 +2110,9 @@ macro_rules! sty_debug_print { }; $(let mut $variant = total;)* - for &Interned(t) in tcx.interners.type_.borrow().keys() { + let shards = tcx.interners.type_.lock_shards(); + let types = shards.iter().flat_map(|shard| shard.keys()); + for &Interned(t) in types { let variant = match t.sty { ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | ty::Float(..) | ty::Str | ty::Never => continue, @@ -2161,11 +2163,11 @@ impl<'tcx> TyCtxt<'tcx> { Generator, GeneratorWitness, Dynamic, Closure, Tuple, Bound, Param, Infer, UnnormalizedProjection, Projection, Opaque, Foreign); - println!("InternalSubsts interner: #{}", self.interners.substs.borrow().len()); - println!("Region interner: #{}", self.interners.region.borrow().len()); - println!("Stability interner: #{}", self.stability_interner.borrow().len()); - println!("Allocation interner: #{}", self.allocation_interner.borrow().len()); - println!("Layout interner: #{}", self.layout_interner.borrow().len()); + println!("InternalSubsts interner: #{}", self.interners.substs.len()); + println!("Region interner: #{}", self.interners.region.len()); + println!("Stability interner: #{}", self.stability_interner.len()); + println!("Allocation interner: #{}", self.allocation_interner.len()); + println!("Layout interner: #{}", self.layout_interner.len()); } } @@ -2298,7 +2300,7 @@ macro_rules! intern_method { // determine that all contents are in the global tcx. // See comments on Lift for why we can't use that. if ($keep_in_local_tcx)(&v) { - self.interners.$name.borrow_mut().intern_ref(key, || { + self.interners.$name.intern_ref(key, || { // Make sure we don't end up with inference // types/regions in the global tcx. if self.is_global() { @@ -2310,7 +2312,7 @@ macro_rules! intern_method { Interned($alloc_method(&self.interners.arena, v)) }).0 } else { - self.global_interners.$name.borrow_mut().intern_ref(key, || { + self.global_interners.$name.intern_ref(key, || { Interned($alloc_method(&self.global_interners.arena, v)) }).0 } diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index 13d93f173e845..36d893e993a09 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -11,7 +11,7 @@ use crate::util::profiling::ProfileCategory; use std::borrow::Cow; use std::hash::Hash; use std::fmt::Debug; -use rustc_data_structures::sync::Lock; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::fingerprint::Fingerprint; use crate::ich::StableHashingContext; @@ -31,7 +31,7 @@ pub(crate) trait QueryAccessors<'tcx>: QueryConfig<'tcx> { fn query(key: Self::Key) -> Query<'tcx>; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_cache<'a>(tcx: TyCtxt<'tcx>) -> &'a Lock>; + fn query_cache<'a>(tcx: TyCtxt<'tcx>) -> &'a Sharded>; fn to_dep_node(tcx: TyCtxt<'tcx>, key: &Self::Key) -> DepNode; diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs index 982886f0f1573..f617e03222d37 100644 --- a/src/librustc/ty/query/on_disk_cache.rs +++ b/src/librustc/ty/query/on_disk_cache.rs @@ -224,9 +224,9 @@ impl<'sess> OnDiskCache<'sess> { // const eval is special, it only encodes successfully evaluated constants use crate::ty::query::QueryAccessors; - let cache = const_eval::query_cache(tcx).borrow(); - assert!(cache.active.is_empty()); - for (key, entry) in cache.results.iter() { + let shards = const_eval::query_cache(tcx).lock_shards(); + assert!(shards.iter().all(|shard| shard.active.is_empty())); + for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) { use crate::ty::query::config::QueryDescription; if const_eval::cache_on_disk(tcx, key.clone()) { if let Ok(ref value) = entry.value { @@ -1087,9 +1087,9 @@ where unsafe { ::std::intrinsics::type_name::() }); time_ext(tcx.sess.time_extended(), Some(tcx.sess), desc, || { - let map = Q::query_cache(tcx).borrow(); - assert!(map.active.is_empty()); - for (key, entry) in map.results.iter() { + let shards = Q::query_cache(tcx).lock_shards(); + assert!(shards.iter().all(|shard| shard.active.is_empty())); + for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) { if Q::cache_on_disk(tcx, key.clone()) { let dep_node = SerializedDepNodeIndex::new(entry.index.index()); diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 0f158d2982a20..2b67841314de8 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -17,6 +17,7 @@ use errors::Diagnostic; use errors::FatalError; use rustc_data_structures::fx::{FxHashMap}; use rustc_data_structures::sync::{Lrc, Lock}; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::thin_vec::ThinVec; #[cfg(not(parallel_compiler))] use rustc_data_structures::cold_path; @@ -90,7 +91,7 @@ macro_rules! profq_query_msg { /// A type representing the responsibility to execute the job in the `job` field. /// This will poison the relevant query if dropped. pub(super) struct JobOwner<'a, 'tcx, Q: QueryDescription<'tcx> + 'a> { - cache: &'a Lock>, + cache: &'a Sharded>, key: Q::Key, job: Lrc>, } @@ -107,7 +108,7 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { pub(super) fn try_get(tcx: TyCtxt<'tcx>, span: Span, key: &Q::Key) -> TryGetJob<'a, 'tcx, Q> { let cache = Q::query_cache(tcx); loop { - let mut lock = cache.borrow_mut(); + let mut lock = cache.get_shard_by_value(key).lock(); if let Some(value) = lock.results.get(key) { profq_msg!(tcx, ProfileQueriesMsg::CacheHit); tcx.sess.profiler(|p| p.record_query_hit(Q::NAME)); @@ -191,7 +192,7 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> JobOwner<'a, 'tcx, Q> { let value = QueryValue::new(result.clone(), dep_node_index); { - let mut lock = cache.borrow_mut(); + let mut lock = cache.get_shard_by_value(&key).lock(); lock.active.remove(&key); lock.results.insert(key, value); } @@ -215,7 +216,8 @@ impl<'a, 'tcx, Q: QueryDescription<'tcx>> Drop for JobOwner<'a, 'tcx, Q> { #[cold] fn drop(&mut self) { // Poison the query so jobs waiting on it panic - self.cache.borrow_mut().active.insert(self.key.clone(), QueryResult::Poisoned); + let shard = self.cache.get_shard_by_value(&self.key); + shard.lock().active.insert(self.key.clone(), QueryResult::Poisoned); // Also signal the completion of the job, so waiters // will continue execution self.job.signal_complete(); @@ -683,7 +685,7 @@ macro_rules! define_queries_inner { use std::mem; #[cfg(parallel_compiler)] use ty::query::job::QueryResult; - use rustc_data_structures::sync::Lock; + use rustc_data_structures::sharded::Sharded; use crate::{ rustc_data_structures::stable_hasher::HashStable, rustc_data_structures::stable_hasher::StableHasherResult, @@ -715,18 +717,17 @@ macro_rules! define_queries_inner { pub fn collect_active_jobs(&self) -> Vec>> { let mut jobs = Vec::new(); - // We use try_lock here since we are only called from the + // We use try_lock_shards here since we are only called from the // deadlock handler, and this shouldn't be locked. $( - jobs.extend( - self.$name.try_lock().unwrap().active.values().filter_map(|v| - if let QueryResult::Started(ref job) = *v { - Some(job.clone()) - } else { - None - } - ) - ); + let shards = self.$name.try_lock_shards().unwrap(); + jobs.extend(shards.iter().flat_map(|shard| shard.active.values().filter_map(|v| + if let QueryResult::Started(ref job) = *v { + Some(job.clone()) + } else { + None + } + ))); )* jobs @@ -748,26 +749,27 @@ macro_rules! define_queries_inner { fn stats<'tcx, Q: QueryConfig<'tcx>>( name: &'static str, - map: &QueryCache<'tcx, Q> + map: &Sharded>, ) -> QueryStats { + let map = map.lock_shards(); QueryStats { name, #[cfg(debug_assertions)] - cache_hits: map.cache_hits, + cache_hits: map.iter().map(|shard| shard.cache_hits).sum(), #[cfg(not(debug_assertions))] cache_hits: 0, key_size: mem::size_of::(), key_type: unsafe { type_name::() }, value_size: mem::size_of::(), value_type: unsafe { type_name::() }, - entry_count: map.results.len(), + entry_count: map.iter().map(|shard| shard.results.len()).sum(), } } $( queries.push(stats::>( stringify!($name), - &*self.$name.lock() + &self.$name, )); )* @@ -939,7 +941,7 @@ macro_rules! define_queries_inner { } #[inline(always)] - fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a Lock> { + fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a Sharded> { &tcx.queries.$name } @@ -1066,7 +1068,7 @@ macro_rules! define_queries_struct { providers: IndexVec>, fallback_extern_providers: Box>, - $($(#[$attr])* $name: Lock>>,)* + $($(#[$attr])* $name: Sharded>>,)* } }; } diff --git a/src/librustc_data_structures/interner.rs b/src/librustc_data_structures/interner.rs deleted file mode 100644 index 36ccbb704a733..0000000000000 --- a/src/librustc_data_structures/interner.rs +++ /dev/null @@ -1,58 +0,0 @@ -use std::hash::Hash; -use std::hash::BuildHasher; -use std::hash::Hasher; -use std::collections::HashMap; -use std::collections::hash_map::RawEntryMut; -use std::borrow::Borrow; - -pub trait HashInterner { - fn intern_ref K>(&mut self, value: &Q, make: F) -> K - where K: Borrow, - Q: Hash + Eq; - - fn intern K>(&mut self, value: Q, make: F) -> K - where K: Borrow, - Q: Hash + Eq; -} - -impl HashInterner for HashMap { - #[inline] - fn intern_ref K>(&mut self, value: &Q, make: F) -> K - where K: Borrow, - Q: Hash + Eq - { - let mut hasher = self.hasher().build_hasher(); - value.hash(&mut hasher); - let hash = hasher.finish(); - let entry = self.raw_entry_mut().from_key_hashed_nocheck(hash, value); - - match entry { - RawEntryMut::Occupied(e) => *e.key(), - RawEntryMut::Vacant(e) => { - let v = make(); - e.insert_hashed_nocheck(hash, v, ()); - v - } - } - } - - #[inline] - fn intern K>(&mut self, value: Q, make: F) -> K - where K: Borrow, - Q: Hash + Eq - { - let mut hasher = self.hasher().build_hasher(); - value.hash(&mut hasher); - let hash = hasher.finish(); - let entry = self.raw_entry_mut().from_key_hashed_nocheck(hash, &value); - - match entry { - RawEntryMut::Occupied(e) => *e.key(), - RawEntryMut::Vacant(e) => { - let v = make(value); - e.insert_hashed_nocheck(hash, v, ()); - v - } - } - } -} diff --git a/src/librustc_data_structures/lib.rs b/src/librustc_data_structures/lib.rs index a1d7ab8856daa..9cd3bfcb944f9 100644 --- a/src/librustc_data_structures/lib.rs +++ b/src/librustc_data_structures/lib.rs @@ -79,7 +79,6 @@ pub mod flock; pub mod fx; pub mod graph; pub mod indexed_vec; -pub mod interner; pub mod jobserver; pub mod obligation_forest; pub mod owning_ref; @@ -91,6 +90,7 @@ pub use ena::snapshot_vec; pub mod sorted_map; #[macro_use] pub mod stable_hasher; pub mod sync; +pub mod sharded; pub mod tiny_list; pub mod thin_vec; pub mod transitive_relation; diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs new file mode 100644 index 0000000000000..e59db31fe043b --- /dev/null +++ b/src/librustc_data_structures/sharded.rs @@ -0,0 +1,129 @@ +use std::hash::{Hasher, Hash}; +use std::mem; +use std::borrow::Borrow; +use std::collections::hash_map::RawEntryMut; +use crate::fx::{FxHasher, FxHashMap}; +use crate::sync::{Lock, LockGuard}; + +#[derive(Clone, Default)] +#[cfg_attr(parallel_compiler, repr(align(64)))] +struct CacheAligned(T); + +#[cfg(parallel_compiler)] +const SHARD_BITS: usize = 5; + +#[cfg(not(parallel_compiler))] +const SHARD_BITS: usize = 0; + +const SHARDS: usize = 1 << SHARD_BITS; + +/// An array of cache-line aligned inner locked structures with convenience methods. +#[derive(Clone)] +pub struct Sharded { + shards: [CacheAligned>; SHARDS], +} + +impl Default for Sharded { + #[inline] + fn default() -> Self { + let mut shards: mem::MaybeUninit<[CacheAligned>; SHARDS]> = + mem::MaybeUninit::uninit(); + let first = shards.as_mut_ptr() as *mut CacheAligned>; + unsafe { + for i in 0..SHARDS { + first.add(i).write(CacheAligned(Lock::new(T::default()))); + } + Sharded { + shards: shards.assume_init(), + } + } + } +} + +impl Sharded { + #[inline] + pub fn get_shard_by_value(&self, val: &K) -> &Lock { + if SHARDS == 1 { + &self.shards[0].0 + } else { + self.get_shard_by_hash(make_hash(val)) + } + } + + #[inline] + pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { + let hash_len = mem::size_of::(); + // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. + // hashbrown also uses the lowest bits, so we can't use those + let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; + let i = bits % SHARDS; + &self.shards[i].0 + } + + pub fn lock_shards(&self) -> Vec> { + (0..SHARDS).map(|i| self.shards[i].0.lock()).collect() + } + + pub fn try_lock_shards(&self) -> Option>> { + let mut result = Vec::with_capacity(SHARDS); + for i in 0..SHARDS { + result.push(self.shards[i].0.try_lock()?); + } + Some(result) + } +} + +pub type ShardedHashMap = Sharded>; + +impl ShardedHashMap { + pub fn len(&self) -> usize{ + self.lock_shards().iter().map(|shard| shard.len()).sum() + } +} + +impl ShardedHashMap { + #[inline] + pub fn intern_ref(&self, value: &Q, make: impl FnOnce() -> K) -> K + where K: Borrow, + Q: Hash + Eq + { + let hash = make_hash(value); + let mut shard = self.get_shard_by_hash(hash).lock(); + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(); + e.insert_hashed_nocheck(hash, v, ()); + v + } + } + } + + #[inline] + pub fn intern(&self, value: Q, make: impl FnOnce(Q) -> K) -> K + where K: Borrow, + Q: Hash + Eq + { + let hash = make_hash(&value); + let mut shard = self.get_shard_by_hash(hash).lock(); + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(value); + e.insert_hashed_nocheck(hash, v, ()); + v + } + } + } +} + +#[inline] +fn make_hash(val: &K) -> u64 { + let mut state = FxHasher::default(); + val.hash(&mut state); + state.finish() +} From cd2747e351f54d99ebc4c2ea54c4f79db6e9fce3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 13 Jun 2019 22:42:24 +0200 Subject: [PATCH 6/9] Use more fine grained locks for the dep graph --- src/librustc/dep_graph/graph.rs | 123 ++++++++++++++---------------- src/librustc/ty/query/plumbing.rs | 4 +- 2 files changed, 59 insertions(+), 68 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 35bd7ce9e73de..c986fed9ec9dd 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -3,7 +3,8 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; -use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering}; +use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering}; +use std::sync::atomic::Ordering::{Acquire, SeqCst}; use std::env; use std::hash::Hash; use std::collections::hash_map::Entry; @@ -54,7 +55,7 @@ struct DepGraphData { /// tracking. The `current` field is the dependency graph of only the /// current compilation session: We don't merge the previous dep-graph into /// current one anymore. - current: Lock, + current: CurrentDepGraph, /// The dep-graph from the previous compilation session. It contains all /// nodes and edges as well as all fingerprints of nodes that have them. @@ -77,7 +78,7 @@ struct DepGraphData { dep_node_debug: Lock>, // Used for testing, only populated when -Zquery-dep-graph is specified. - loaded_from_cache: Lock>, + loaded_from_cache: Lock>, } pub fn hash_result(hcx: &mut StableHashingContext<'_>, result: &R) -> Option @@ -99,7 +100,7 @@ impl DepGraph { data: Some(Lrc::new(DepGraphData { previous_work_products: prev_work_products, dep_node_debug: Default::default(), - current: Lock::new(CurrentDepGraph::new(prev_graph_node_count)), + current: CurrentDepGraph::new(prev_graph_node_count), emitting_diagnostics: Default::default(), emitting_diagnostics_cond_var: Condvar::new(), previous: prev_graph, @@ -122,13 +123,13 @@ impl DepGraph { } pub fn query(&self) -> DepGraphQuery { - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); - let nodes: Vec<_> = current_dep_graph.data.iter().map(|n| n.node).collect(); + let data = self.data.as_ref().unwrap().current.data.lock(); + let nodes: Vec<_> = data.iter().map(|n| n.node).collect(); let mut edges = Vec::new(); - for (from, edge_targets) in current_dep_graph.data.iter() + for (from, edge_targets) in data.iter() .map(|d| (d.node, &d.edges)) { for &edge_target in edge_targets.iter() { - let to = current_dep_graph.data[edge_target].node; + let to = data[edge_target].node; edges.push((from, to)); } } @@ -207,7 +208,7 @@ impl DepGraph { read_set: Default::default(), }), |data, key, fingerprint, task| { - data.borrow_mut().complete_task(key, task.unwrap(), fingerprint) + data.complete_task(key, task.unwrap(), fingerprint) }, hash_result) } @@ -228,7 +229,7 @@ impl DepGraph { self.with_task_impl(key, cx, input, true, identity_fn, |_| None, |data, key, fingerprint, _| { - data.borrow_mut().alloc_node(key, SmallVec::new(), fingerprint) + data.alloc_node(key, SmallVec::new(), fingerprint) }, hash_result::) } @@ -241,7 +242,7 @@ impl DepGraph { no_tcx: bool, task: fn(C, A) -> R, create_task: fn(DepNode) -> Option, - finish_task_and_alloc_depnode: fn(&Lock, + finish_task_and_alloc_depnode: fn(&CurrentDepGraph, DepNode, Fingerprint, Option) -> DepNodeIndex, @@ -363,7 +364,6 @@ impl DepGraph { (r, task_deps.into_inner()) }); let dep_node_index = data.current - .borrow_mut() .complete_anon_task(dep_kind, task_deps); (result, dep_node_index) } else { @@ -387,8 +387,7 @@ impl DepGraph { self.with_task_impl(key, cx, arg, false, task, |_| None, |data, key, fingerprint, _| { - let mut current = data.borrow_mut(); - current.alloc_node(key, smallvec![], fingerprint) + data.alloc_node(key, smallvec![], fingerprint) }, hash_result) } @@ -396,9 +395,9 @@ impl DepGraph { #[inline] pub fn read(&self, v: DepNode) { if let Some(ref data) = self.data { - let current = data.current.borrow_mut(); - if let Some(&dep_node_index) = current.node_to_node_index.get(&v) { - std::mem::drop(current); + let map = data.current.node_to_node_index.lock(); + if let Some(dep_node_index) = map.get(&v).copied() { + std::mem::drop(map); data.read_index(dep_node_index); } else { bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind) @@ -419,8 +418,8 @@ impl DepGraph { .as_ref() .unwrap() .current - .borrow_mut() .node_to_node_index + .lock() .get(dep_node) .cloned() .unwrap() @@ -429,7 +428,7 @@ impl DepGraph { #[inline] pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { if let Some(ref data) = self.data { - data.current.borrow_mut().node_to_node_index.contains_key(dep_node) + data.current.node_to_node_index.lock().contains_key(dep_node) } else { false } @@ -437,8 +436,8 @@ impl DepGraph { #[inline] pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { - let current = self.data.as_ref().expect("dep graph enabled").current.borrow_mut(); - current.data[dep_node_index].fingerprint + let data = self.data.as_ref().expect("dep graph enabled").current.data.lock(); + data[dep_node_index].fingerprint } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { @@ -492,32 +491,29 @@ impl DepGraph { pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> { if cfg!(debug_assertions) { - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); + let current_dep_graph = &self.data.as_ref().unwrap().current; - Some((current_dep_graph.total_read_count, - current_dep_graph.total_duplicate_read_count)) + Some((current_dep_graph.total_read_count.load(Acquire), + current_dep_graph.total_duplicate_read_count.load(Acquire))) } else { None } } pub fn serialize(&self) -> SerializedDepGraph { - let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); + let data = self.data.as_ref().unwrap().current.data.lock(); let fingerprints: IndexVec = - current_dep_graph.data.iter().map(|d| d.fingerprint).collect(); + data.iter().map(|d| d.fingerprint).collect(); let nodes: IndexVec = - current_dep_graph.data.iter().map(|d| d.node).collect(); + data.iter().map(|d| d.node).collect(); - let total_edge_count: usize = current_dep_graph.data.iter() - .map(|d| d.edges.len()) - .sum(); + let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum(); let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); let mut edge_list_data = Vec::with_capacity(total_edge_count); - for (current_dep_node_index, edges) in current_dep_graph.data.iter_enumerated() - .map(|(i, d)| (i, &d.edges)) { + for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; // This should really just be a memcpy :/ edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); @@ -746,15 +742,13 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently let dep_node_index = { - let mut current = data.current.borrow_mut(); - // Copy the fingerprint from the previous graph, // so we don't have to recompute it let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index); // We allocating an entry for the node in the current dependency graph and // adding all the appropriate edges imported from the previous graph - current.intern_node(*dep_node, current_deps, fingerprint) + data.current.intern_node(*dep_node, current_deps, fingerprint) }; // ... emitting any stored diagnostic ... @@ -889,23 +883,21 @@ impl DepGraph { } } - pub fn mark_loaded_from_cache(&self, dep_node_index: DepNodeIndex, state: bool) { - debug!("mark_loaded_from_cache({:?}, {})", - self.data.as_ref().unwrap().current.borrow().data[dep_node_index].node, - state); + pub fn mark_loaded_from_cache(&self, dep_node: DepNode, state: bool) { + debug!("mark_loaded_from_cache({:?}, {})", dep_node, state); self.data .as_ref() .unwrap() .loaded_from_cache .borrow_mut() - .insert(dep_node_index, state); + .insert(dep_node, state); } + // FIXME: Remove this since it appears to be unused? pub fn was_loaded_from_cache(&self, dep_node: &DepNode) -> Option { let data = self.data.as_ref().unwrap(); - let dep_node_index = data.current.borrow().node_to_node_index[dep_node]; - data.loaded_from_cache.borrow().get(&dep_node_index).cloned() + data.loaded_from_cache.borrow().get(&dep_node).cloned() } } @@ -962,8 +954,8 @@ struct DepNodeData { } pub(super) struct CurrentDepGraph { - data: IndexVec, - node_to_node_index: FxHashMap, + data: Lock>, + node_to_node_index: Lock>, #[allow(dead_code)] forbidden_edge: Option, @@ -980,8 +972,8 @@ pub(super) struct CurrentDepGraph { /// the `DepGraph` is created. anon_id_seed: Fingerprint, - total_read_count: u64, - total_duplicate_read_count: u64, + total_read_count: AtomicU64, + total_duplicate_read_count: AtomicU64, } impl CurrentDepGraph { @@ -1015,20 +1007,20 @@ impl CurrentDepGraph { let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; CurrentDepGraph { - data: IndexVec::with_capacity(new_node_count_estimate), - node_to_node_index: FxHashMap::with_capacity_and_hasher( + data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)), + node_to_node_index: Lock::new(FxHashMap::with_capacity_and_hasher( new_node_count_estimate, Default::default(), - ), + )), anon_id_seed: stable_hasher.finish(), forbidden_edge, - total_read_count: 0, - total_duplicate_read_count: 0, + total_read_count: AtomicU64::new(0), + total_duplicate_read_count: AtomicU64::new(0), } } fn complete_task( - &mut self, + &self, node: DepNode, task_deps: TaskDeps, fingerprint: Fingerprint @@ -1036,7 +1028,7 @@ impl CurrentDepGraph { self.alloc_node(node, task_deps.reads, fingerprint) } - fn complete_anon_task(&mut self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex { + fn complete_anon_task(&self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex { debug_assert!(!kind.is_eval_always()); let mut hasher = StableHasher::new(); @@ -1061,28 +1053,27 @@ impl CurrentDepGraph { } fn alloc_node( - &mut self, + &self, dep_node: DepNode, edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> DepNodeIndex { - debug_assert!(!self.node_to_node_index.contains_key(&dep_node)); + debug_assert!(!self.node_to_node_index.lock().contains_key(&dep_node)); self.intern_node(dep_node, edges, fingerprint) } fn intern_node( - &mut self, + &self, dep_node: DepNode, edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> DepNodeIndex { - debug_assert_eq!(self.node_to_node_index.len(), self.data.len()); - - match self.node_to_node_index.entry(dep_node) { + match self.node_to_node_index.lock().entry(dep_node) { Entry::Occupied(entry) => *entry.get(), Entry::Vacant(entry) => { - let dep_node_index = DepNodeIndex::new(self.data.len()); - self.data.push(DepNodeData { + let mut data = self.data.lock(); + let dep_node_index = DepNodeIndex::new(data.len()); + data.push(DepNodeData { node: dep_node, edges, fingerprint @@ -1101,7 +1092,7 @@ impl DepGraphData { if let Some(task_deps) = icx.task_deps { let mut task_deps = task_deps.lock(); if cfg!(debug_assertions) { - self.current.lock().total_read_count += 1; + self.current.total_read_count.fetch_add(1, SeqCst); } if task_deps.read_set.insert(source) { task_deps.reads.push(source); @@ -1109,9 +1100,9 @@ impl DepGraphData { #[cfg(debug_assertions)] { if let Some(target) = task_deps.node { - let graph = self.current.lock(); - if let Some(ref forbidden_edge) = graph.forbidden_edge { - let source = graph.data[source].node; + let data = self.current.data.lock(); + if let Some(ref forbidden_edge) = self.current.forbidden_edge { + let source = data[source].node; if forbidden_edge.test(&source, &target) { bug!("forbidden edge {:?} -> {:?} created", source, @@ -1121,7 +1112,7 @@ impl DepGraphData { } } } else if cfg!(debug_assertions) { - self.current.lock().total_duplicate_read_count += 1; + self.current.total_duplicate_read_count.fetch_add(1, SeqCst); } } }) diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index 2b67841314de8..ffc17cdf5c7fd 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -492,7 +492,7 @@ impl<'tcx> TyCtxt<'tcx> { } if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { - self.dep_graph.mark_loaded_from_cache(dep_node_index, true); + self.dep_graph.mark_loaded_from_cache(*dep_node, true); } result @@ -568,7 +568,7 @@ impl<'tcx> TyCtxt<'tcx> { profq_msg!(self, ProfileQueriesMsg::ProviderEnd); if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { - self.dep_graph.mark_loaded_from_cache(dep_node_index, false); + self.dep_graph.mark_loaded_from_cache(dep_node, false); } if dep_node.kind != crate::dep_graph::DepKind::Null { From b424c099e1ba72bedd1986d1e8701dc1e7838f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 13 Jun 2019 23:14:44 +0200 Subject: [PATCH 7/9] Use a sharded dep node to dep node index map --- src/librustc/dep_graph/graph.rs | 29 +++++++++++++++++------- src/librustc_data_structures/sharded.rs | 30 +++++++++++++++++++------ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index c986fed9ec9dd..a96869a1659f6 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -4,6 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering}; +use rustc_data_structures::sharded::{self, Sharded}; use std::sync::atomic::Ordering::{Acquire, SeqCst}; use std::env; use std::hash::Hash; @@ -395,7 +396,7 @@ impl DepGraph { #[inline] pub fn read(&self, v: DepNode) { if let Some(ref data) = self.data { - let map = data.current.node_to_node_index.lock(); + let map = data.current.node_to_node_index.get_shard_by_value(&v).lock(); if let Some(dep_node_index) = map.get(&v).copied() { std::mem::drop(map); data.read_index(dep_node_index); @@ -419,6 +420,7 @@ impl DepGraph { .unwrap() .current .node_to_node_index + .get_shard_by_value(dep_node) .lock() .get(dep_node) .cloned() @@ -428,7 +430,11 @@ impl DepGraph { #[inline] pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { if let Some(ref data) = self.data { - data.current.node_to_node_index.lock().contains_key(dep_node) + data.current + .node_to_node_index + .get_shard_by_value(&dep_node) + .lock() + .contains_key(dep_node) } else { false } @@ -609,7 +615,11 @@ impl DepGraph { #[cfg(not(parallel_compiler))] { - debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node)); + debug_assert!(!data.current + .node_to_node_index + .get_shard_by_value(dep_node) + .lock() + .contains_key(dep_node)); debug_assert!(data.colors.get(prev_dep_node_index).is_none()); } @@ -955,7 +965,7 @@ struct DepNodeData { pub(super) struct CurrentDepGraph { data: Lock>, - node_to_node_index: Lock>, + node_to_node_index: Sharded>, #[allow(dead_code)] forbidden_edge: Option, @@ -1008,8 +1018,8 @@ impl CurrentDepGraph { CurrentDepGraph { data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)), - node_to_node_index: Lock::new(FxHashMap::with_capacity_and_hasher( - new_node_count_estimate, + node_to_node_index: Sharded::new(|| FxHashMap::with_capacity_and_hasher( + new_node_count_estimate / sharded::SHARDS, Default::default(), )), anon_id_seed: stable_hasher.finish(), @@ -1058,7 +1068,10 @@ impl CurrentDepGraph { edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> DepNodeIndex { - debug_assert!(!self.node_to_node_index.lock().contains_key(&dep_node)); + debug_assert!(!self.node_to_node_index + .get_shard_by_value(&dep_node) + .lock() + .contains_key(&dep_node)); self.intern_node(dep_node, edges, fingerprint) } @@ -1068,7 +1081,7 @@ impl CurrentDepGraph { edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> DepNodeIndex { - match self.node_to_node_index.lock().entry(dep_node) { + match self.node_to_node_index.get_shard_by_value(&dep_node).lock().entry(dep_node) { Entry::Occupied(entry) => *entry.get(), Entry::Vacant(entry) => { let mut data = self.data.lock(); diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs index e59db31fe043b..7b3d105ff8496 100644 --- a/src/librustc_data_structures/sharded.rs +++ b/src/librustc_data_structures/sharded.rs @@ -2,6 +2,7 @@ use std::hash::{Hasher, Hash}; use std::mem; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; +use smallvec::SmallVec; use crate::fx::{FxHasher, FxHashMap}; use crate::sync::{Lock, LockGuard}; @@ -15,7 +16,7 @@ const SHARD_BITS: usize = 5; #[cfg(not(parallel_compiler))] const SHARD_BITS: usize = 0; -const SHARDS: usize = 1 << SHARD_BITS; +pub const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. #[derive(Clone)] @@ -26,21 +27,36 @@ pub struct Sharded { impl Default for Sharded { #[inline] fn default() -> Self { + Self::new(|| T::default()) + } +} + +impl Sharded { + #[inline] + pub fn new(mut value: impl FnMut() -> T) -> Self { + // Create a vector of the values we want + let mut values: SmallVec<[_; SHARDS]> = (0..SHARDS).map(|_| { + CacheAligned(Lock::new(value())) + }).collect(); + + // Create an unintialized array let mut shards: mem::MaybeUninit<[CacheAligned>; SHARDS]> = mem::MaybeUninit::uninit(); - let first = shards.as_mut_ptr() as *mut CacheAligned>; + unsafe { - for i in 0..SHARDS { - first.add(i).write(CacheAligned(Lock::new(T::default()))); - } + // Copy the values into our array + let first = shards.as_mut_ptr() as *mut CacheAligned>; + values.as_ptr().copy_to_nonoverlapping(first, SHARDS); + + // Ignore the content of the vector + values.set_len(0); + Sharded { shards: shards.assume_init(), } } } -} -impl Sharded { #[inline] pub fn get_shard_by_value(&self, val: &K) -> &Lock { if SHARDS == 1 { From fcd003f246dffbd78631c6028b7d05d0fb8559b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Fri, 14 Jun 2019 01:20:27 +0200 Subject: [PATCH 8/9] Use a sharded structure for the dep node index to dep node data map --- src/librustc/dep_graph/graph.rs | 105 +++++++++++++++++++----- src/librustc_data_structures/sharded.rs | 5 ++ 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index a96869a1659f6..9158c53d92c79 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -3,11 +3,15 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; -use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering}; +use rustc_data_structures::sync::{ + Lrc, Lock, LockGuard, MappedLockGuard, AtomicU32, AtomicU64, Ordering +}; use rustc_data_structures::sharded::{self, Sharded}; use std::sync::atomic::Ordering::{Acquire, SeqCst}; use std::env; +use std::iter; use std::hash::Hash; +use std::convert::TryFrom; use std::collections::hash_map::Entry; use std::mem; use crate::ty::{self, TyCtxt}; @@ -124,7 +128,15 @@ impl DepGraph { } pub fn query(&self) -> DepGraphQuery { - let data = self.data.as_ref().unwrap().current.data.lock(); + let current = &self.data.as_ref().unwrap().current; + let shards = current.data.lock_shards(); + let node_count = current.node_count.load(Acquire) as usize; + let data: IndexVec = (0..node_count).map(|i| { + let shard = i % sharded::SHARDS; + let inner = i / sharded::SHARDS; + &shards[shard][inner] + }).collect(); + let nodes: Vec<_> = data.iter().map(|n| n.node).collect(); let mut edges = Vec::new(); for (from, edge_targets) in data.iter() @@ -442,8 +454,7 @@ impl DepGraph { #[inline] pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { - let data = self.data.as_ref().expect("dep graph enabled").current.data.lock(); - data[dep_node_index].fingerprint + self.data.as_ref().expect("dep graph enabled").current.data(dep_node_index).fingerprint } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { @@ -507,25 +518,32 @@ impl DepGraph { } pub fn serialize(&self) -> SerializedDepGraph { - let data = self.data.as_ref().unwrap().current.data.lock(); + let current = &self.data.as_ref().unwrap().current; + let shards = current.data.lock_shards(); + let node_count = current.node_count.load(Acquire) as usize; + let data = || (0..node_count).map(|i| { + let shard = i % sharded::SHARDS; + let inner = i / sharded::SHARDS; + &shards[shard][inner] + }); let fingerprints: IndexVec = - data.iter().map(|d| d.fingerprint).collect(); + data().map(|d| d.fingerprint).collect(); let nodes: IndexVec = - data.iter().map(|d| d.node).collect(); + data().map(|d| d.node).collect(); - let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum(); + let total_edge_count: usize = data().map(|d| d.edges.len()).sum(); let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); let mut edge_list_data = Vec::with_capacity(total_edge_count); - for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) { + for (current_dep_node_index, edges) in data().enumerate().map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; // This should really just be a memcpy :/ edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); let end = edge_list_data.len() as u32; - debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len()); + debug_assert_eq!(current_dep_node_index, edge_list_indices.len()); edge_list_indices.push((start, end)); } @@ -964,7 +982,14 @@ struct DepNodeData { } pub(super) struct CurrentDepGraph { - data: Lock>, + /// The current node count. Used to allocate an index before storing it in the + /// `data` and `node_to_node_index` field below. + node_count: AtomicU64, + + /// Maps from a `DepNodeIndex` to `DepNodeData`. The lowest bits of `DepNodeIndex` determines + /// which shard is used and the higher bits are the index into the vector. + data: Sharded>, + node_to_node_index: Sharded>, #[allow(dead_code)] forbidden_edge: Option, @@ -1017,7 +1042,8 @@ impl CurrentDepGraph { let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; CurrentDepGraph { - data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)), + node_count: AtomicU64::new(0), + data: Sharded::new(|| Vec::with_capacity(new_node_count_estimate / sharded::SHARDS)), node_to_node_index: Sharded::new(|| FxHashMap::with_capacity_and_hasher( new_node_count_estimate / sharded::SHARDS, Default::default(), @@ -1081,20 +1107,56 @@ impl CurrentDepGraph { edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint ) -> DepNodeIndex { - match self.node_to_node_index.get_shard_by_value(&dep_node).lock().entry(dep_node) { + let (index, inserted) = match self.node_to_node_index + .get_shard_by_value(&dep_node) + .lock() + .entry(dep_node) { Entry::Occupied(entry) => *entry.get(), Entry::Vacant(entry) => { - let mut data = self.data.lock(); - let dep_node_index = DepNodeIndex::new(data.len()); - data.push(DepNodeData { - node: dep_node, - edges, - fingerprint - }); + let index = self.node_count.fetch_add(1, SeqCst); + // Cast to u32 to ensure we didn't overflow. + let index = u32::try_from(index).unwrap(); + + let dep_node_index = DepNodeIndex::new(index as usize); entry.insert(dep_node_index); dep_node_index } + }; + + if inserted { + let dep_node_data = DepNodeData { + node: dep_node, + edges, + fingerprint + }; + let inner_index = index.as_usize() / sharded::SHARDS; + let mut data = self.data.get_shard_by_index(index.as_usize()).lock(); + let len = data.len(); + if likely!(len == inner_index) { + data.push(dep_node_data) + } else { + let dummy_data = DepNodeData { + node: DepNode::new_no_params(DepKind::Null), + edges: SmallVec::default(), + fingerprint: Fingerprint::ZERO, + }; + if inner_index >= len { + data.extend(iter::repeat(dummy_data).take(inner_index - len + 1)); + } + data[inner_index] = dep_node_data; + } } + + (index, inserted) + } + + fn data( + &self, + index: DepNodeIndex, + ) -> MappedLockGuard<'_, DepNodeData> { + LockGuard::map(self.data.get_shard_by_index(index.as_usize()).lock(), |vec| { + &mut vec[index.as_usize() / sharded::SHARDS] + }) } } @@ -1113,9 +1175,8 @@ impl DepGraphData { #[cfg(debug_assertions)] { if let Some(target) = task_deps.node { - let data = self.current.data.lock(); if let Some(ref forbidden_edge) = self.current.forbidden_edge { - let source = data[source].node; + let source = self.current.data(source).node; if forbidden_edge.test(&source, &target) { bug!("forbidden edge {:?} -> {:?} created", source, diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs index 7b3d105ff8496..b1bf9df99760f 100644 --- a/src/librustc_data_structures/sharded.rs +++ b/src/librustc_data_structures/sharded.rs @@ -57,6 +57,11 @@ impl Sharded { } } + #[inline] + pub fn get_shard_by_index(&self, index: usize) -> &Lock { + &self.shards[index % SHARDS].0 + } + #[inline] pub fn get_shard_by_value(&self, val: &K) -> &Lock { if SHARDS == 1 { From 290912ab0e75d02ff9b5eb6378c9a5aced919a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 16 Apr 2019 09:59:55 +0200 Subject: [PATCH 9/9] Make dep node indices persistent between sessions --- src/librustc/dep_graph/graph.rs | 618 ++++++++++++++--------- src/librustc/dep_graph/mod.rs | 4 +- src/librustc/dep_graph/prev.rs | 55 +- src/librustc/dep_graph/serialized.rs | 19 +- src/librustc/query/mod.rs | 2 +- src/librustc/ty/query/config.rs | 7 +- src/librustc/ty/query/on_disk_cache.rs | 30 +- src/librustc/ty/query/plumbing.rs | 20 +- src/librustc_data_structures/sharded.rs | 27 +- src/librustc_incremental/persist/load.rs | 26 +- src/librustc_interface/queries.rs | 4 +- src/librustc_macros/src/query.rs | 4 +- 12 files changed, 492 insertions(+), 324 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 9158c53d92c79..5585b43844973 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -4,7 +4,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::indexed_vec::{Idx, IndexVec}; use smallvec::SmallVec; use rustc_data_structures::sync::{ - Lrc, Lock, LockGuard, MappedLockGuard, AtomicU32, AtomicU64, Ordering + Lrc, Lock, LockGuard, MappedLockGuard, AtomicU64, AtomicCell }; use rustc_data_structures::sharded::{self, Sharded}; use std::sync::atomic::Ordering::{Acquire, SeqCst}; @@ -24,7 +24,7 @@ use super::debug::EdgeFilter; use super::dep_node::{DepNode, DepKind, WorkProductId}; use super::query::DepGraphQuery; use super::safe::DepGraphSafe; -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +use super::serialized::SerializedDepGraph; use super::prev::PreviousDepGraph; #[derive(Clone)] @@ -43,14 +43,38 @@ impl DepNodeIndex { #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum DepNodeColor { Red, - Green(DepNodeIndex) + Green, } impl DepNodeColor { pub fn is_green(self) -> bool { match self { DepNodeColor::Red => false, - DepNodeColor::Green(_) => true, + DepNodeColor::Green => true, + } + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub enum DepNodeState { + /// The dep node index is invalid and does not refer to any dep node. + Invalid, + + /// The node is from the previous session, but its state is unknown + Unknown, + + Red, + + Green, +} + +impl DepNodeState { + pub fn color(self) -> Option { + match self { + DepNodeState::Invalid => bug!(), + DepNodeState::Unknown => None, + DepNodeState::Red => Some(DepNodeColor::Red), + DepNodeState::Green => Some(DepNodeColor::Green), } } } @@ -96,20 +120,39 @@ where Some(stable_hasher.finish()) } -impl DepGraph { - pub fn new(prev_graph: PreviousDepGraph, - prev_work_products: FxHashMap) -> DepGraph { - let prev_graph_node_count = prev_graph.node_count(); +pub struct DepGraphArgs { + pub prev_graph: PreviousDepGraph, + pub prev_work_products: FxHashMap, + pub state: IndexVec>, + pub current: CurrentDepGraph, +} +impl Default for DepGraphArgs { + fn default() -> Self { + let prev_graph = Default::default(); + Self { + current: CurrentDepGraph::new(&prev_graph), + prev_work_products: Default::default(), + state: Default::default(), + prev_graph, + } + } +} + +impl DepGraph { + pub fn new(args: DepGraphArgs) -> DepGraph { + let colors = DepNodeColorMap { + values: args.state, + }; DepGraph { data: Some(Lrc::new(DepGraphData { - previous_work_products: prev_work_products, + previous_work_products: args.prev_work_products, dep_node_debug: Default::default(), - current: CurrentDepGraph::new(prev_graph_node_count), + current: args.current, emitting_diagnostics: Default::default(), emitting_diagnostics_cond_var: Condvar::new(), - previous: prev_graph, - colors: DepNodeColorMap::new(prev_graph_node_count), + colors, + previous: args.prev_graph, loaded_from_cache: Default::default(), })), } @@ -128,15 +171,18 @@ impl DepGraph { } pub fn query(&self) -> DepGraphQuery { - let current = &self.data.as_ref().unwrap().current; + let data = self.data.as_ref().unwrap(); + let current = &data.current; let shards = current.data.lock_shards(); - let node_count = current.node_count.load(Acquire) as usize; + let node_count = data.node_count(); let data: IndexVec = (0..node_count).map(|i| { let shard = i % sharded::SHARDS; let inner = i / sharded::SHARDS; &shards[shard][inner] }).collect(); + // FIXME: Only use Red/Green nodes here. + let nodes: Vec<_> = data.iter().map(|n| n.node).collect(); let mut edges = Vec::new(); for (from, edge_targets) in data.iter() @@ -220,9 +266,6 @@ impl DepGraph { reads: SmallVec::new(), read_set: Default::default(), }), - |data, key, fingerprint, task| { - data.complete_task(key, task.unwrap(), fingerprint) - }, hash_result) } @@ -241,9 +284,6 @@ impl DepGraph { self.with_task_impl(key, cx, input, true, identity_fn, |_| None, - |data, key, fingerprint, _| { - data.alloc_node(key, SmallVec::new(), fingerprint) - }, hash_result::) } @@ -255,10 +295,6 @@ impl DepGraph { no_tcx: bool, task: fn(C, A) -> R, create_task: fn(DepNode) -> Option, - finish_task_and_alloc_depnode: fn(&CurrentDepGraph, - DepNode, - Fingerprint, - Option) -> DepNodeIndex, hash_result: impl FnOnce(&mut StableHashingContext<'_>, &R) -> Option, ) -> (R, DepNodeIndex) where @@ -299,49 +335,65 @@ impl DepGraph { let current_fingerprint = hash_result(&mut hcx, &result); - let dep_node_index = finish_task_and_alloc_depnode( - &data.current, - key, - current_fingerprint.unwrap_or(Fingerprint::ZERO), - task_deps.map(|lock| lock.into_inner()), - ); + let deps = task_deps.map(|lock| lock.into_inner().reads).unwrap_or(SmallVec::new()); + let fingerprint = current_fingerprint.unwrap_or(Fingerprint::ZERO); let print_status = cfg!(debug_assertions) && hcx.sess().opts.debugging_opts.dep_tasks; - // Determine the color of the new DepNode. - if let Some(prev_index) = data.previous.node_to_index_opt(&key) { + let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) { let prev_fingerprint = data.previous.fingerprint_by_index(prev_index); + // Determine the color of the DepNode. let color = if let Some(current_fingerprint) = current_fingerprint { if current_fingerprint == prev_fingerprint { if print_status { eprintln!("[task::green] {:?}", key); } - DepNodeColor::Green(dep_node_index) + DepNodeState::Green } else { if print_status { eprintln!("[task::red] {:?}", key); } - DepNodeColor::Red + DepNodeState::Red } } else { if print_status { eprintln!("[task::unknown] {:?}", key); } // Mark the node as Red if we can't hash the result - DepNodeColor::Red + DepNodeState::Red }; - debug_assert!(data.colors.get(prev_index).is_none(), - "DepGraph::with_task() - Duplicate DepNodeColor \ - insertion for {:?}", key); + debug_assert_eq!( + data.colors.get(prev_index).color(), + None, + "DepGraph::with_task() - Duplicate DepNodeState insertion for {:?}", + key + ); data.colors.insert(prev_index, color); + + data.current.update_node( + prev_index, + key, + deps, + fingerprint + ); + + prev_index } else { if print_status { eprintln!("[task::new] {:?}", key); } - } + + data.current.new_node( + data, + key, + deps, + fingerprint, + &data.previous, + ) + }; (result, dep_node_index) } else { @@ -377,7 +429,7 @@ impl DepGraph { (r, task_deps.into_inner()) }); let dep_node_index = data.current - .complete_anon_task(dep_kind, task_deps); + .complete_anon_task(data, dep_kind, task_deps); (result, dep_node_index) } else { (op(), DepNodeIndex::INVALID) @@ -399,9 +451,6 @@ impl DepGraph { { self.with_task_impl(key, cx, arg, false, task, |_| None, - |data, key, fingerprint, _| { - data.alloc_node(key, smallvec![], fingerprint) - }, hash_result) } @@ -446,7 +495,8 @@ impl DepGraph { .node_to_node_index .get_shard_by_value(&dep_node) .lock() - .contains_key(dep_node) + .get(dep_node) + .map_or(false, |index| data.current(*index)) } else { false } @@ -454,18 +504,15 @@ impl DepGraph { #[inline] pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { - self.data.as_ref().expect("dep graph enabled").current.data(dep_node_index).fingerprint + let data = self.data.as_ref().expect("dep graph enabled"); + assert!(data.current(dep_node_index)); + data.current.data(dep_node_index).fingerprint } pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } - #[inline] - pub fn prev_dep_node_index_of(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { - self.data.as_ref().unwrap().previous.node_to_index(dep_node) - } - /// Checks whether a previous work product exists for `v` and, if /// so, return the path that leads to it. Used to skip doing work. pub fn previous_work_product(&self, v: &WorkProductId) -> Option { @@ -518,29 +565,59 @@ impl DepGraph { } pub fn serialize(&self) -> SerializedDepGraph { - let current = &self.data.as_ref().unwrap().current; + let data = self.data.as_ref().unwrap(); + + let current = &data.current; let shards = current.data.lock_shards(); - let node_count = current.node_count.load(Acquire) as usize; - let data = || (0..node_count).map(|i| { + let node_count = data.node_count(); + let node_data_iter = || (0..node_count).map(|i| { let shard = i % sharded::SHARDS; let inner = i / sharded::SHARDS; &shards[shard][inner] }); - let fingerprints: IndexVec = - data().map(|d| d.fingerprint).collect(); - let nodes: IndexVec = - data().map(|d| d.node).collect(); + let fingerprints: IndexVec = + node_data_iter().map(|d| d.fingerprint).collect(); + let mut nodes: IndexVec = + node_data_iter().map(|d| d.node).collect(); + + // Invalidate dep nodes with unknown state as these cannot safely + // be marked green in the next session. One of the dependencies of the + // unknown node may have changed in this session (and is currently marked red), + // but might be green again in the next session, which may cause the unknown node + // to incorrectly be marked green in the next session, even though one of its dependencies + // did actually change. + for index in data.colors.values.indices() { + match data.colors.get(index) { + // In order to this invalidation to be safe, none of the valid nodes can + // point to unknown nodes. + DepNodeState::Unknown => { + // Change the node kind to Null so we know this node is invalidated when + // we load the dep graph + nodes[index] = DepNode::new_no_params(DepKind::Null); + // FIXME: Also clear edges + } - let total_edge_count: usize = data().map(|d| d.edges.len()).sum(); + // For green nodes, we either executed the query (which always uses valid nodes) + // or we marked it as green because all its dependencies are green and valid. + DepNodeState::Green | + // Red nodes were always exexuted. + DepNodeState::Red | + // We don't need to invalidate already invalid nodes + DepNodeState::Invalid => {}, + } + } + + let total_edge_count: usize = node_data_iter().map(|d| d.edges.len()).sum(); let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); let mut edge_list_data = Vec::with_capacity(total_edge_count); - for (current_dep_node_index, edges) in data().enumerate().map(|(i, d)| (i, &d.edges)) { + for (current_dep_node_index, edges) in node_data_iter().enumerate() + .map(|(i, d)| (i, &d.edges)) { let start = edge_list_data.len() as u32; // This should really just be a memcpy :/ - edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); + edge_list_data.extend(edges.iter().map(|i| DepNodeIndex::new(i.index()))); let end = edge_list_data.len() as u32; debug_assert_eq!(current_dep_node_index, edge_list_indices.len()); @@ -561,7 +638,7 @@ impl DepGraph { pub fn node_color(&self, dep_node: &DepNode) -> Option { if let Some(ref data) = self.data { if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) { - return data.colors.get(prev_index) + return data.colors.get(prev_index).color() } else { // This is a node that did not exist in the previous compilation // session, so we consider it to be red. @@ -580,11 +657,11 @@ impl DepGraph { &self, tcx: TyCtxt<'_>, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { - self.try_mark_green(tcx, dep_node).map(|(prev_index, dep_node_index)| { + ) -> Option { + self.try_mark_green(tcx, dep_node).map(|prev_index| { debug_assert!(self.is_green(&dep_node)); - self.read_index(dep_node_index); - (prev_index, dep_node_index) + self.read_index(prev_index); + prev_index }) } @@ -592,7 +669,7 @@ impl DepGraph { &self, tcx: TyCtxt<'_>, dep_node: &DepNode, - ) -> Option<(SerializedDepNodeIndex, DepNodeIndex)> { + ) -> Option { debug_assert!(!dep_node.kind.is_eval_always()); // Return None if the dep graph is disabled @@ -602,59 +679,125 @@ impl DepGraph { let prev_index = data.previous.node_to_index_opt(dep_node)?; match data.colors.get(prev_index) { - Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)), - Some(DepNodeColor::Red) => None, - None => { + DepNodeState::Invalid => bug!(), + DepNodeState::Green => Some(prev_index), + DepNodeState::Red => None, + DepNodeState::Unknown => { // This DepNode and the corresponding query invocation existed // in the previous compilation session too, so we can try to // mark it as green by recursively marking all of its // dependencies green. - self.try_mark_previous_green( + if self.try_mark_previous_green( tcx.global_tcx(), data, prev_index, &dep_node - ).map(|dep_node_index| { - (prev_index, dep_node_index) - }) + ) { + Some(prev_index) + } else { + None + } } } } + /// Try to force a dep node to execute and see if it's green + fn try_force_previous_green( + &self, + tcx: TyCtxt<'_>, + data: &DepGraphData, + dep_node_index: DepNodeIndex, + ) -> bool { + let dep_node = &data.previous.index_to_node(dep_node_index); + + match dep_node.kind { + DepKind::Hir | + DepKind::HirBody | + DepKind::CrateMetadata => { + if dep_node.extract_def_id(tcx).is_none() { + // If the node does not exist anymore, we + // just fail to mark green. + return false + } else { + // If the node does exist, it should have + // been pre-allocated. + bug!("DepNode {:?} should have been \ + pre-allocated but wasn't.", + dep_node) + } + } + _ => { + // For other kinds of nodes it's OK to be + // forced. + } + } + + debug!("try_force_previous_green({:?}) --- trying to force", dep_node); + if crate::ty::query::force_from_dep_node(tcx, dep_node) { + match data.colors.get(dep_node_index) { + DepNodeState::Green => { + debug!("try_force_previous_green({:?}) --- managed to \ + FORCE to green", + dep_node); + true + } + DepNodeState::Red => { + debug!( + "try_force_previous_green({:?}) - END - was red after forcing", + dep_node + ); + false + } + DepNodeState::Invalid => bug!(), + DepNodeState::Unknown => { + if !tcx.sess.has_errors() { + bug!("try_force_previous_green() - Forcing the DepNode \ + should have set its color") + } else { + // If the query we just forced has resulted + // in some kind of compilation error, we + // don't expect that the corresponding + // dep-node color has been updated. + // A query cycle which does not panic is one + // such error. + false + } + } + } + } else { + // The DepNode could not be forced. + debug!("try_force_previous_green({:?}) - END - could not be forced", dep_node); + false + } + } + /// Try to mark a dep-node which existed in the previous compilation session as green. fn try_mark_previous_green<'tcx>( &self, tcx: TyCtxt<'tcx>, data: &DepGraphData, - prev_dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, + // FIXME: Remove this, only used in debug statements dep_node: &DepNode, - ) -> Option { + ) -> bool { debug!("try_mark_previous_green({:?}) - BEGIN", dep_node); #[cfg(not(parallel_compiler))] - { - debug_assert!(!data.current - .node_to_node_index - .get_shard_by_value(dep_node) - .lock() - .contains_key(dep_node)); - debug_assert!(data.colors.get(prev_dep_node_index).is_none()); - } + debug_assert!(data.colors.get(dep_node_index).color().is_none()); // We never try to mark eval_always nodes as green debug_assert!(!dep_node.kind.is_eval_always()); - debug_assert_eq!(data.previous.index_to_node(prev_dep_node_index), *dep_node); - - let prev_deps = data.previous.edge_targets_from(prev_dep_node_index); + debug_assert_eq!(data.previous.index_to_node(dep_node_index), *dep_node); - let mut current_deps = SmallVec::new(); + let prev_deps = data.previous.edge_targets_from(dep_node_index); for &dep_dep_node_index in prev_deps { let dep_dep_node_color = data.colors.get(dep_dep_node_index); match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { + DepNodeState::Invalid => bug!(), + DepNodeState::Green => { // This dependency has been marked as green before, we are // still fine and can continue with checking the other // dependencies. @@ -662,9 +805,8 @@ impl DepGraph { be immediately green", dep_node, data.previous.index_to_node(dep_dep_node_index)); - current_deps.push(node_index); } - Some(DepNodeColor::Red) => { + DepNodeState::Red => { // We found a dependency the value of which has changed // compared to the previous compilation session. We cannot // mark the DepNode as green and also don't need to bother @@ -673,91 +815,32 @@ impl DepGraph { immediately red", dep_node, data.previous.index_to_node(dep_dep_node_index)); - return None + return false } - None => { + DepNodeState::Unknown => { let dep_dep_node = &data.previous.index_to_node(dep_dep_node_index); // We don't know the state of this dependency. If it isn't // an eval_always node, let's try to mark it green recursively. if !dep_dep_node.kind.is_eval_always() { - debug!("try_mark_previous_green({:?}) --- state of dependency {:?} \ - is unknown, trying to mark it green", dep_node, - dep_dep_node); - - let node_index = self.try_mark_previous_green( - tcx, - data, - dep_dep_node_index, - dep_dep_node - ); - if let Some(node_index) = node_index { - debug!("try_mark_previous_green({:?}) --- managed to MARK \ - dependency {:?} as green", dep_node, dep_dep_node); - current_deps.push(node_index); - continue; - } - } else { - match dep_dep_node.kind { - DepKind::Hir | - DepKind::HirBody | - DepKind::CrateMetadata => { - if dep_dep_node.extract_def_id(tcx).is_none() { - // If the node does not exist anymore, we - // just fail to mark green. - return None - } else { - // If the node does exist, it should have - // been pre-allocated. - bug!("DepNode {:?} should have been \ - pre-allocated but wasn't.", - dep_dep_node) - } - } - _ => { - // For other kinds of nodes it's OK to be - // forced. - } - } + // We don't know the state of this dependency. + // We known it is not an eval_always node, since those get marked as `Invalid`. + // Let's try to mark it green recursively. + if self.try_mark_previous_green( + tcx, + data, + dep_dep_node_index, + dep_dep_node + ) { + debug!("try_mark_previous_green({:?}) --- managed to MARK \ + dependency {:?} as green", dep_node, dep_dep_node); + continue; + } } // We failed to mark it green, so we try to force the query. - debug!("try_mark_previous_green({:?}) --- trying to force \ - dependency {:?}", dep_node, dep_dep_node); - if crate::ty::query::force_from_dep_node(tcx, dep_dep_node) { - let dep_dep_node_color = data.colors.get(dep_dep_node_index); - - match dep_dep_node_color { - Some(DepNodeColor::Green(node_index)) => { - debug!("try_mark_previous_green({:?}) --- managed to \ - FORCE dependency {:?} to green", - dep_node, dep_dep_node); - current_deps.push(node_index); - } - Some(DepNodeColor::Red) => { - debug!("try_mark_previous_green({:?}) - END - \ - dependency {:?} was red after forcing", - dep_node, - dep_dep_node); - return None - } - None => { - if !tcx.sess.has_errors() { - bug!("try_mark_previous_green() - Forcing the DepNode \ - should have set its color") - } else { - // If the query we just forced has resulted - // in some kind of compilation error, we - // don't expect that the corresponding - // dep-node color has been updated. - } - } - } - } else { - // The DepNode could not be forced. - debug!("try_mark_previous_green({:?}) - END - dependency {:?} \ - could not be forced", dep_node, dep_dep_node); - return None + if !self.try_force_previous_green(tcx, data, dep_dep_node_index) { + return false; } } } @@ -769,29 +852,23 @@ impl DepGraph { // There may be multiple threads trying to mark the same dep node green concurrently - let dep_node_index = { - // Copy the fingerprint from the previous graph, - // so we don't have to recompute it - let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index); - - // We allocating an entry for the node in the current dependency graph and - // adding all the appropriate edges imported from the previous graph - data.current.intern_node(*dep_node, current_deps, fingerprint) - }; + #[cfg(not(parallel_compiler))] + debug_assert_eq!(data.colors.get(dep_node_index).color(), None, + "DepGraph::try_mark_previous_green() - Duplicate DepNodeState \ + insertion for {:?}", dep_node); // ... emitting any stored diagnostic ... // FIXME: Store the fact that a node has diagnostics in a bit in the dep graph somewhere // Maybe store a list on disk and encode this fact in the DepNodeState let diagnostics = tcx.queries.on_disk_cache - .load_diagnostics(tcx, prev_dep_node_index); + .load_diagnostics(tcx, dep_node_index); if unlikely!(diagnostics.len() > 0) { self.emit_diagnostics( tcx, data, dep_node_index, - prev_dep_node_index, diagnostics ); } else { @@ -801,15 +878,12 @@ impl DepGraph { // ... and finally storing a "Green" entry in the color map. // Multiple threads can all write the same color here - #[cfg(not(parallel_compiler))] - debug_assert!(data.colors.get(prev_dep_node_index).is_none(), - "DepGraph::try_mark_previous_green() - Duplicate DepNodeColor \ - insertion for {:?}", dep_node); - data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + data.colors.insert(dep_node_index, DepNodeState::Green); debug!("try_mark_previous_green({:?}) - END - successfully marked as green", dep_node); - Some(dep_node_index) + + true } /// Atomically emits some loaded diagnotics. @@ -821,12 +895,11 @@ impl DepGraph { tcx: TyCtxt<'tcx>, data: &DepGraphData, dep_node_index: DepNodeIndex, - prev_dep_node_index: SerializedDepNodeIndex, diagnostics: Vec, ) { let mut emitting = data.emitting_diagnostics.lock(); - if data.colors.get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) { + if data.colors.get(dep_node_index) == DepNodeState::Green { // The node is already green so diagnostics must have been emitted already return; } @@ -848,7 +921,7 @@ impl DepGraph { } // Mark the node as green now that diagnostics are emitted - data.colors.insert(prev_dep_node_index, DepNodeColor::Green(dep_node_index)); + data.colors.insert(dep_node_index, DepNodeState::Green); // Remove the node from the set data.emitting_diagnostics.lock().remove(&dep_node_index); @@ -860,8 +933,7 @@ impl DepGraph { loop { data.emitting_diagnostics_cond_var.wait(&mut emitting); - if data.colors - .get(prev_dep_node_index) == Some(DepNodeColor::Green(dep_node_index)) { + if data.colors.get(dep_node_index) == DepNodeState::Green { break; } } @@ -887,7 +959,7 @@ impl DepGraph { let data = self.data.as_ref().unwrap(); data.colors.values.indices().filter_map(|prev_index| { match data.colors.get(prev_index) { - Some(DepNodeColor::Green(_)) => { + DepNodeState::Green => { let dep_node = data.previous.index_to_node(prev_index); if dep_node.cache_on_disk(tcx) { Some(dep_node) @@ -895,8 +967,15 @@ impl DepGraph { None } } - None | - Some(DepNodeColor::Red) => { + + // There cannot be results stored for invalid indices. + DepNodeState::Invalid | + + // Unknown nodes are unused, so we don't want to promote these and we would + // not to mark their colors in order to do so anyway. + DepNodeState::Unknown | + + DepNodeState::Red => { // We can skip red nodes because a node can only be marked // as red if the query result was recomputed and thus is // already in memory. @@ -981,10 +1060,11 @@ struct DepNodeData { fingerprint: Fingerprint, } -pub(super) struct CurrentDepGraph { - /// The current node count. Used to allocate an index before storing it in the +pub struct CurrentDepGraph { + /// The numbers of nodes allocated in this session. + /// Used to calculate an `DepNodeIndex` before storing it in the /// `data` and `node_to_node_index` field below. - node_count: AtomicU64, + nodes_allocated: AtomicU64, /// Maps from a `DepNodeIndex` to `DepNodeData`. The lowest bits of `DepNodeIndex` determines /// which shard is used and the higher bits are the index into the vector. @@ -1012,7 +1092,7 @@ pub(super) struct CurrentDepGraph { } impl CurrentDepGraph { - fn new(prev_graph_node_count: usize) -> CurrentDepGraph { + pub fn new(prev_graph: &PreviousDepGraph) -> CurrentDepGraph { use std::time::{SystemTime, UNIX_EPOCH}; let duration = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -1039,12 +1119,26 @@ impl CurrentDepGraph { // that we hopefully don't have to re-allocate during this compilation // session. The over-allocation is 2% plus a small constant to account // for the fact that in very small crates 2% might not be enough. - let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; + let new_node_count_estimate = (prev_graph.node_count() * 102) / 100 + 200; + + let mut current = CurrentDepGraph { + nodes_allocated: AtomicU64::new(0), + data: Sharded::new(|i| { + let mut data = Vec::with_capacity(new_node_count_estimate / sharded::SHARDS); + + // Fill in with the node data from the previous session which belongs to this shard + for idx in (0..prev_graph.node_count()).skip(i).step_by(sharded::SHARDS) { + let idx = DepNodeIndex::new(idx); + data.push(DepNodeData { + node: prev_graph.index_to_node(idx), + fingerprint: prev_graph.fingerprint_by_index(idx), + edges: prev_graph.edge_targets_from(idx).iter().copied().collect() + }); + } - CurrentDepGraph { - node_count: AtomicU64::new(0), - data: Sharded::new(|| Vec::with_capacity(new_node_count_estimate / sharded::SHARDS)), - node_to_node_index: Sharded::new(|| FxHashMap::with_capacity_and_hasher( + data + }), + node_to_node_index: Sharded::new(|_| FxHashMap::with_capacity_and_hasher( new_node_count_estimate / sharded::SHARDS, Default::default(), )), @@ -1052,19 +1146,22 @@ impl CurrentDepGraph { forbidden_edge, total_read_count: AtomicU64::new(0), total_duplicate_read_count: AtomicU64::new(0), + }; + + // Fill in with the node to index map from the previous session + for (node, idx) in prev_graph.index.iter() { + current.node_to_node_index.get_shard_by_value_mut(node).insert(*node, *idx); } + + current } - fn complete_task( + fn complete_anon_task( &self, - node: DepNode, - task_deps: TaskDeps, - fingerprint: Fingerprint + data: &DepGraphData, + kind: DepKind, + task_deps: TaskDeps ) -> DepNodeIndex { - self.alloc_node(node, task_deps.reads, fingerprint) - } - - fn complete_anon_task(&self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex { debug_assert!(!kind.is_eval_always()); let mut hasher = StableHasher::new(); @@ -1085,24 +1182,45 @@ impl CurrentDepGraph { hash: self.anon_id_seed.combine(hasher.finish()), }; - self.intern_node(target_dep_node, task_deps.reads, Fingerprint::ZERO) + self.intern_node(data, target_dep_node, task_deps.reads, Fingerprint::ZERO) } - fn alloc_node( + fn new_node( &self, + data: &DepGraphData, dep_node: DepNode, edges: SmallVec<[DepNodeIndex; 8]>, - fingerprint: Fingerprint + fingerprint: Fingerprint, + previous: &PreviousDepGraph, ) -> DepNodeIndex { + debug_assert!(previous.node_to_index_opt(&dep_node).is_none()); debug_assert!(!self.node_to_node_index .get_shard_by_value(&dep_node) .lock() .contains_key(&dep_node)); - self.intern_node(dep_node, edges, fingerprint) + self.intern_node(data, dep_node, edges, fingerprint) + } + + fn update_node( + &self, + index: DepNodeIndex, + dep_node: DepNode, + edges: SmallVec<[DepNodeIndex; 8]>, + fingerprint: Fingerprint, + ) { + // Update the edges and fingerprint for this dep node + let inner_index = index.as_usize() / sharded::SHARDS; + let mut data = self.data.get_shard_by_index(index.as_usize()).lock(); + data[inner_index] = DepNodeData { + node: dep_node, + edges, + fingerprint + }; } fn intern_node( &self, + data: &DepGraphData, dep_node: DepNode, edges: SmallVec<[DepNodeIndex; 8]>, fingerprint: Fingerprint @@ -1111,15 +1229,14 @@ impl CurrentDepGraph { .get_shard_by_value(&dep_node) .lock() .entry(dep_node) { - Entry::Occupied(entry) => *entry.get(), + Entry::Occupied(entry) => (*entry.get(), false), Entry::Vacant(entry) => { - let index = self.node_count.fetch_add(1, SeqCst); + let allocation = self.nodes_allocated.fetch_add(1, SeqCst); // Cast to u32 to ensure we didn't overflow. - let index = u32::try_from(index).unwrap(); - - let dep_node_index = DepNodeIndex::new(index as usize); + let allocation = u32::try_from(allocation).unwrap(); + let dep_node_index = data.index_from_allocation(allocation as usize); entry.insert(dep_node_index); - dep_node_index + (dep_node_index, true) } }; @@ -1147,7 +1264,7 @@ impl CurrentDepGraph { } } - (index, inserted) + index } fn data( @@ -1161,6 +1278,36 @@ impl CurrentDepGraph { } impl DepGraphData { + fn current(&self, dep_node_index: DepNodeIndex) -> bool { + if dep_node_index.as_usize() < self.previous.node_count() { + // From this session if it's colored + self.colors.get(dep_node_index).color().is_some() + } else { + // From this session + true + } + } + + /// Create a DepNodeIndex from the n-th allocation in this session, starting from 0 + fn index_from_allocation(&self, allocation: usize) -> DepNodeIndex { + if allocation < self.previous.unused.len() { + self.previous.unused[allocation] + } else { + let index = self.previous.node_count() + (allocation - self.previous.unused.len()); + DepNodeIndex::new(index) + } + } + + /// Get the total number of dep node indices, including unused ones + fn node_count(&self) -> usize { + let allocated = usize::try_from(self.current.nodes_allocated.load(Acquire)).unwrap(); + if allocated <= self.previous.unused.len() { + self.previous.node_count() + } else { + self.previous.node_count() + (allocated - self.previous.unused.len()) + } + } + fn read_index(&self, source: DepNodeIndex) { ty::tls::with_context_opt(|icx| { let icx = if let Some(icx) = icx { icx } else { return }; @@ -1200,37 +1347,18 @@ pub struct TaskDeps { read_set: FxHashSet, } -// A data structure that stores Option values as a contiguous -// array, using one u32 per entry. struct DepNodeColorMap { - values: IndexVec, + values: IndexVec>, } -const COMPRESSED_NONE: u32 = 0; -const COMPRESSED_RED: u32 = 1; -const COMPRESSED_FIRST_GREEN: u32 = 2; - impl DepNodeColorMap { - fn new(size: usize) -> DepNodeColorMap { - DepNodeColorMap { - values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect(), - } - } - - fn get(&self, index: SerializedDepNodeIndex) -> Option { - match self.values[index].load(Ordering::Acquire) { - COMPRESSED_NONE => None, - COMPRESSED_RED => Some(DepNodeColor::Red), - value => Some(DepNodeColor::Green(DepNodeIndex::from_u32( - value - COMPRESSED_FIRST_GREEN - ))) - } + #[inline] + fn get(&self, index: DepNodeIndex) -> DepNodeState { + self.values[index].load() } - fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) { - self.values[index].store(match color { - DepNodeColor::Red => COMPRESSED_RED, - DepNodeColor::Green(index) => index.as_u32() + COMPRESSED_FIRST_GREEN, - }, Ordering::Release) + #[inline] + fn insert(&self, index: DepNodeIndex, state: DepNodeState) { + self.values[index].store(state) } } diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index 1535e6d349cf1..4451c91365036 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -11,9 +11,9 @@ pub mod cgu_reuse_tracker; pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig}; pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId, RecoverKey, label_strs}; pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex, DepNodeColor, TaskDeps, hash_result}; -pub use self::graph::WorkProductFileKind; +pub use self::graph::{DepGraphArgs, WorkProductFileKind, CurrentDepGraph}; pub use self::prev::PreviousDepGraph; pub use self::query::DepGraphQuery; pub use self::safe::AssertDepGraphSafe; pub use self::safe::DepGraphSafe; -pub use self::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +pub use self::serialized::SerializedDepGraph; diff --git a/src/librustc/dep_graph/prev.rs b/src/librustc/dep_graph/prev.rs index d971690bbe317..887a2d85fa551 100644 --- a/src/librustc/dep_graph/prev.rs +++ b/src/librustc/dep_graph/prev.rs @@ -1,43 +1,70 @@ use crate::ich::Fingerprint; use rustc_data_structures::fx::FxHashMap; -use super::dep_node::DepNode; -use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex}; +use rustc_data_structures::indexed_vec::IndexVec; +use rustc_data_structures::sync::AtomicCell; +use super::dep_node::{DepNode, DepKind}; +use super::graph::{DepNodeIndex, DepNodeState}; +use super::serialized::SerializedDepGraph; -#[derive(Debug, RustcEncodable, RustcDecodable, Default)] +#[derive(Debug, Default)] pub struct PreviousDepGraph { data: SerializedDepGraph, - index: FxHashMap, + pub(super) index: FxHashMap, + pub(super) unused: Vec, } impl PreviousDepGraph { - pub fn new(data: SerializedDepGraph) -> PreviousDepGraph { + pub fn new_and_state( + data: SerializedDepGraph + ) -> (PreviousDepGraph, IndexVec>) { + let mut unused = Vec::new(); + + let state: IndexVec<_, _> = data.nodes.iter_enumerated().map(|(index, node)| { + if node.kind == DepKind::Null { + // There might be `DepKind::Null` nodes due to thread-local dep node indices + // that didn't get assigned anything. + // We also changed outdated nodes to `DepKind::Null`. + unused.push(index); + AtomicCell::new(DepNodeState::Invalid) + } else { + AtomicCell::new(DepNodeState::Unknown) + } + }).collect(); + let index: FxHashMap<_, _> = data.nodes .iter_enumerated() - .map(|(idx, &dep_node)| (dep_node, idx)) + .filter_map(|(idx, &dep_node)| { + if dep_node.kind == DepKind::Null { + None + } else { + Some((dep_node, idx)) + } + }) .collect(); - PreviousDepGraph { data, index } + + (PreviousDepGraph { data, index, unused }, state) } #[inline] pub fn edge_targets_from( &self, - dep_node_index: SerializedDepNodeIndex - ) -> &[SerializedDepNodeIndex] { + dep_node_index: DepNodeIndex + ) -> &[DepNodeIndex] { self.data.edge_targets_from(dep_node_index) } #[inline] - pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode { + pub fn index_to_node(&self, dep_node_index: DepNodeIndex) -> DepNode { self.data.nodes[dep_node_index] } #[inline] - pub fn node_to_index(&self, dep_node: &DepNode) -> SerializedDepNodeIndex { + pub fn node_to_index(&self, dep_node: &DepNode) -> DepNodeIndex { self.index[dep_node] } #[inline] - pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { + pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option { self.index.get(dep_node).cloned() } @@ -50,12 +77,12 @@ impl PreviousDepGraph { #[inline] pub fn fingerprint_by_index(&self, - dep_node_index: SerializedDepNodeIndex) + dep_node_index: DepNodeIndex) -> Fingerprint { self.data.fingerprints[dep_node_index] } pub fn node_count(&self) -> usize { - self.index.len() + self.data.nodes.len() } } diff --git a/src/librustc/dep_graph/serialized.rs b/src/librustc/dep_graph/serialized.rs index b64f71ed908d8..81dc99d5b75fc 100644 --- a/src/librustc/dep_graph/serialized.rs +++ b/src/librustc/dep_graph/serialized.rs @@ -2,34 +2,31 @@ use crate::dep_graph::DepNode; use crate::ich::Fingerprint; -use rustc_data_structures::indexed_vec::{IndexVec, Idx}; - -newtype_index! { - pub struct SerializedDepNodeIndex { .. } -} +use rustc_data_structures::indexed_vec::IndexVec; +use super::graph::DepNodeIndex; /// Data for use when recompiling the **current crate**. #[derive(Debug, RustcEncodable, RustcDecodable, Default)] pub struct SerializedDepGraph { /// The set of all DepNodes in the graph - pub nodes: IndexVec, + pub nodes: IndexVec, /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to /// the DepNode at the same index in the nodes vector. - pub fingerprints: IndexVec, + pub fingerprints: IndexVec, /// For each DepNode, stores the list of edges originating from that /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data, /// which holds the actual DepNodeIndices of the target nodes. - pub edge_list_indices: IndexVec, + pub edge_list_indices: IndexVec, /// A flattened list of all edge targets in the graph. Edge sources are /// implicit in edge_list_indices. - pub edge_list_data: Vec, + pub edge_list_data: Vec, } impl SerializedDepGraph { #[inline] pub fn edge_targets_from(&self, - source: SerializedDepNodeIndex) - -> &[SerializedDepNodeIndex] { + source: DepNodeIndex) + -> &[DepNodeIndex] { let targets = self.edge_list_indices[source]; &self.edge_list_data[targets.0 as usize..targets.1 as usize] } diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index 10efef54526a6..7cec2e480eb77 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -2,10 +2,10 @@ use crate::ty::query::QueryDescription; use crate::ty::query::queries; use crate::ty::{self, ParamEnvAnd, Ty, TyCtxt}; use crate::ty::subst::SubstsRef; -use crate::dep_graph::SerializedDepNodeIndex; use crate::hir::def_id::{CrateNum, DefId, DefIndex}; use crate::mir; use crate::mir::interpret::GlobalId; +use crate::dep_graph::DepNodeIndex; use crate::traits; use crate::traits::query::{ CanonicalPredicateGoal, CanonicalProjectionGoal, diff --git a/src/librustc/ty/query/config.rs b/src/librustc/ty/query/config.rs index 36d893e993a09..64b0c752ee120 100644 --- a/src/librustc/ty/query/config.rs +++ b/src/librustc/ty/query/config.rs @@ -1,5 +1,4 @@ -use crate::dep_graph::SerializedDepNodeIndex; -use crate::dep_graph::DepNode; +use crate::dep_graph::{DepNode, DepNodeIndex}; use crate::hir::def_id::{CrateNum, DefId}; use crate::ty::TyCtxt; use crate::ty::query::queries; @@ -54,7 +53,7 @@ pub(crate) trait QueryDescription<'tcx>: QueryAccessors<'tcx> { false } - fn try_load_from_disk(_: TyCtxt<'tcx>, _: SerializedDepNodeIndex) -> Option { + fn try_load_from_disk(_: TyCtxt<'tcx>, _: DepNodeIndex) -> Option { bug!("QueryDescription::load_from_disk() called for an unsupported query.") } } @@ -86,7 +85,7 @@ macro_rules! impl_disk_cacheable_query( #[inline] fn try_load_from_disk(tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex) + id: DepNodeIndex) -> Option { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) } diff --git a/src/librustc/ty/query/on_disk_cache.rs b/src/librustc/ty/query/on_disk_cache.rs index f617e03222d37..5dfb6767a5e42 100644 --- a/src/librustc/ty/query/on_disk_cache.rs +++ b/src/librustc/ty/query/on_disk_cache.rs @@ -1,4 +1,4 @@ -use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use crate::dep_graph::DepNodeIndex; use crate::hir; use crate::hir::def_id::{CrateNum, DefIndex, DefId, LocalDefId, LOCAL_CRATE}; use crate::hir::map::definitions::DefPathHash; @@ -62,11 +62,11 @@ pub struct OnDiskCache<'sess> { // A map from dep-node to the position of the cached query result in // `serialized_data`. - query_result_index: FxHashMap, + query_result_index: FxHashMap, // A map from dep-node to the position of any associated diagnostics in // `serialized_data`. - prev_diagnostics_index: FxHashMap, + prev_diagnostics_index: FxHashMap, alloc_decoding_state: AllocDecodingState, } @@ -82,8 +82,8 @@ struct Footer { interpret_alloc_index: Vec, } -type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; -type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; +type EncodedQueryResultIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; +type EncodedDiagnosticsIndex = Vec<(DepNodeIndex, AbsoluteBytePos)>; type EncodedDiagnostics = Vec; #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)] @@ -230,12 +230,12 @@ impl<'sess> OnDiskCache<'sess> { use crate::ty::query::config::QueryDescription; if const_eval::cache_on_disk(tcx, key.clone()) { if let Ok(ref value) = entry.value { - let dep_node = SerializedDepNodeIndex::new(entry.index.index()); + let dep_node = DepNodeIndex::new(entry.index.index()); // Record position of the cache entry qri.push((dep_node, AbsoluteBytePos::new(enc.position()))); - // Encode the type check tables with the SerializedDepNodeIndex + // Encode the type check tables with the DepNodeIndex // as tag. enc.encode_tagged(dep_node, value)?; } @@ -253,7 +253,7 @@ impl<'sess> OnDiskCache<'sess> { let pos = AbsoluteBytePos::new(encoder.position()); // Let's make sure we get the expected type here: let diagnostics: &EncodedDiagnostics = diagnostics; - let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index()); + let dep_node_index = DepNodeIndex::new(dep_node_index.index()); encoder.encode_tagged(dep_node_index, diagnostics)?; Ok((dep_node_index, pos)) @@ -327,7 +327,7 @@ impl<'sess> OnDiskCache<'sess> { pub fn load_diagnostics<'tcx>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Vec { let diagnostics: Option = self.load_indexed( tcx, @@ -352,11 +352,11 @@ impl<'sess> OnDiskCache<'sess> { } /// Returns the cached query result if there is something in the cache for - /// the given `SerializedDepNodeIndex`; otherwise returns `None`. + /// the given `DepNodeIndex`; otherwise returns `None`. pub fn try_load_query_result<'tcx, T>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, + dep_node_index: DepNodeIndex, ) -> Option where T: Decodable, @@ -386,8 +386,8 @@ impl<'sess> OnDiskCache<'sess> { fn load_indexed<'tcx, T>( &self, tcx: TyCtxt<'tcx>, - dep_node_index: SerializedDepNodeIndex, - index: &FxHashMap, + dep_node_index: DepNodeIndex, + index: &FxHashMap, debug_tag: &'static str, ) -> Option where @@ -1091,12 +1091,12 @@ where assert!(shards.iter().all(|shard| shard.active.is_empty())); for (key, entry) in shards.iter().flat_map(|shard| shard.results.iter()) { if Q::cache_on_disk(tcx, key.clone()) { - let dep_node = SerializedDepNodeIndex::new(entry.index.index()); + let dep_node = DepNodeIndex::new(entry.index.index()); // Record position of the cache entry query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - // Encode the type check tables with the SerializedDepNodeIndex + // Encode the type check tables with the DepNodeIndex // as tag. encoder.encode_tagged(dep_node, &entry.value)?; } diff --git a/src/librustc/ty/query/plumbing.rs b/src/librustc/ty/query/plumbing.rs index ffc17cdf5c7fd..526b54c17d661 100644 --- a/src/librustc/ty/query/plumbing.rs +++ b/src/librustc/ty/query/plumbing.rs @@ -2,7 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepNodeIndex, DepNode, DepKind, SerializedDepNodeIndex}; +use crate::dep_graph::{DepNodeIndex, DepNode, DepKind}; use crate::ty::tls; use crate::ty::{self, TyCtxt}; use crate::ty::query::Query; @@ -413,10 +413,9 @@ impl<'tcx> TyCtxt<'tcx> { // try_mark_green(), so we can ignore them here. let loaded = self.start_query(job.job.clone(), None, |tcx| { let marked = tcx.dep_graph.try_mark_green_and_read(tcx, &dep_node); - marked.map(|(prev_dep_node_index, dep_node_index)| { + marked.map(|dep_node_index| { (tcx.load_from_disk_and_cache_in_memory::( key.clone(), - prev_dep_node_index, dep_node_index, &dep_node ), dep_node_index) @@ -436,7 +435,6 @@ impl<'tcx> TyCtxt<'tcx> { fn load_from_disk_and_cache_in_memory>( self, key: Q::Key, - prev_dep_node_index: SerializedDepNodeIndex, dep_node_index: DepNodeIndex, dep_node: &DepNode, ) -> Q::Value { @@ -449,7 +447,7 @@ impl<'tcx> TyCtxt<'tcx> { let result = if Q::cache_on_disk(self.global_tcx(), key.clone()) && self.sess.opts.debugging_opts.incremental_queries { self.sess.profiler(|p| p.incremental_load_result_start(Q::NAME)); - let result = Q::try_load_from_disk(self.global_tcx(), prev_dep_node_index); + let result = Q::try_load_from_disk(self.global_tcx(), dep_node_index); self.sess.profiler(|p| p.incremental_load_result_end(Q::NAME)); // We always expect to find a cached result for things that @@ -488,7 +486,7 @@ impl<'tcx> TyCtxt<'tcx> { // If -Zincremental-verify-ich is specified, re-hash results from // the cache and make sure that they have the expected fingerprint. if unlikely!(self.sess.opts.debugging_opts.incremental_verify_ich) { - self.incremental_verify_ich::(&result, dep_node, dep_node_index); + self.incremental_verify_ich::(&result, dep_node); } if unlikely!(self.sess.opts.debugging_opts.query_dep_graph) { @@ -504,24 +502,18 @@ impl<'tcx> TyCtxt<'tcx> { self, result: &Q::Value, dep_node: &DepNode, - dep_node_index: DepNodeIndex, ) { use crate::ich::Fingerprint; - assert!(Some(self.dep_graph.fingerprint_of(dep_node_index)) == - self.dep_graph.prev_fingerprint_of(dep_node), - "Fingerprint for green query instance not loaded \ - from cache: {:?}", dep_node); - debug!("BEGIN verify_ich({:?})", dep_node); let mut hcx = self.create_stable_hashing_context(); let new_hash = Q::hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO); debug!("END verify_ich({:?})", dep_node); - let old_hash = self.dep_graph.fingerprint_of(dep_node_index); + let old_hash = self.dep_graph.prev_fingerprint_of(dep_node); - assert!(new_hash == old_hash, "Found unstable fingerprints \ + assert!(Some(new_hash) == old_hash, "Found unstable fingerprints \ for {:?}", dep_node); } diff --git a/src/librustc_data_structures/sharded.rs b/src/librustc_data_structures/sharded.rs index b1bf9df99760f..d234893c305d5 100644 --- a/src/librustc_data_structures/sharded.rs +++ b/src/librustc_data_structures/sharded.rs @@ -27,16 +27,16 @@ pub struct Sharded { impl Default for Sharded { #[inline] fn default() -> Self { - Self::new(|| T::default()) + Self::new(|_| T::default()) } } impl Sharded { #[inline] - pub fn new(mut value: impl FnMut() -> T) -> Self { + pub fn new(mut value: impl FnMut(usize) -> T) -> Self { // Create a vector of the values we want - let mut values: SmallVec<[_; SHARDS]> = (0..SHARDS).map(|_| { - CacheAligned(Lock::new(value())) + let mut values: SmallVec<[_; SHARDS]> = (0..SHARDS).map(|i| { + CacheAligned(Lock::new(value(i))) }).collect(); // Create an unintialized array @@ -72,13 +72,26 @@ impl Sharded { } #[inline] - pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { + pub fn get_shard_by_value_mut(&mut self, val: &K) -> &mut T { + if SHARDS == 1 { + self.shards[0].0.get_mut() + } else { + self.shards[Self::get_shard_index_by_hash(make_hash(val))].0.get_mut() + } + } + + #[inline] + fn get_shard_index_by_hash(hash: u64) -> usize { let hash_len = mem::size_of::(); // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. // hashbrown also uses the lowest bits, so we can't use those let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; - let i = bits % SHARDS; - &self.shards[i].0 + bits % SHARDS + } + + #[inline] + pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { + &self.shards[Self::get_shard_index_by_hash(hash)].0 } pub fn lock_shards(&self) -> Vec> { diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index d9bcc0b2a83c7..af6fcccb326d4 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -1,7 +1,7 @@ //! Code to save/load the dep-graph from files. use rustc_data_structures::fx::FxHashMap; -use rustc::dep_graph::{PreviousDepGraph, SerializedDepGraph, WorkProduct, WorkProductId}; +use rustc::dep_graph::{PreviousDepGraph, DepGraphArgs, SerializedDepGraph, CurrentDepGraph}; use rustc::session::Session; use rustc::ty::TyCtxt; use rustc::ty::query::OnDiskCache; @@ -23,16 +23,14 @@ pub fn dep_graph_tcx_init<'tcx>(tcx: TyCtxt<'tcx>) { tcx.allocate_metadata_dep_nodes(); } -type WorkProductMap = FxHashMap; - pub enum LoadResult { Ok { data: T }, DataOutOfDate, Error { message: String }, } -impl LoadResult<(PreviousDepGraph, WorkProductMap)> { - pub fn open(self, sess: &Session) -> (PreviousDepGraph, WorkProductMap) { +impl LoadResult { + pub fn open(self, sess: &Session) -> DepGraphArgs { match self { LoadResult::Error { message } => { sess.warn(&message); @@ -93,7 +91,7 @@ impl MaybeAsync { } } -pub type DepGraphFuture = MaybeAsync>; +pub type DepGraphFuture = MaybeAsync>; /// Launch a thread and load the dependency graph in the background. pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { @@ -185,7 +183,21 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture { let dep_graph = SerializedDepGraph::decode(&mut decoder) .expect("Error reading cached dep-graph"); - LoadResult::Ok { data: (PreviousDepGraph::new(dep_graph), prev_work_products) } + let (prev_graph, state) = PreviousDepGraph::new_and_state(dep_graph); + let current = time_ext( + time_passes, + None, + "background setup current dep-graph", + || CurrentDepGraph::new(&prev_graph), + ); + LoadResult::Ok { + data: DepGraphArgs { + state, + prev_graph, + prev_work_products, + current, + } + } } } }) diff --git a/src/librustc_interface/queries.rs b/src/librustc_interface/queries.rs index 570509ffb2b8c..9545b513a2605 100644 --- a/src/librustc_interface/queries.rs +++ b/src/librustc_interface/queries.rs @@ -173,13 +173,13 @@ impl Compiler { Ok(match self.dep_graph_future()?.take() { None => DepGraph::new_disabled(), Some(future) => { - let (prev_graph, prev_work_products) = + let args = time(self.session(), "blocked while dep-graph loading finishes", || { future.open().unwrap_or_else(|e| rustc_incremental::LoadResult::Error { message: format!("could not decode incremental cache: {:?}", e), }).open(self.session()) }); - DepGraph::new(prev_graph, prev_work_products) + DepGraph::new(args) } }) }) diff --git a/src/librustc_macros/src/query.rs b/src/librustc_macros/src/query.rs index 0474d2a2e3b3a..79696f0e8785d 100644 --- a/src/librustc_macros/src/query.rs +++ b/src/librustc_macros/src/query.rs @@ -328,7 +328,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( #tcx: TyCtxt<'tcx>, - #id: SerializedDepNodeIndex + #id: DepNodeIndex ) -> Option { #block } @@ -339,7 +339,7 @@ fn add_query_description_impl( #[inline] fn try_load_from_disk( tcx: TyCtxt<'tcx>, - id: SerializedDepNodeIndex + id: DepNodeIndex ) -> Option { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) }