From 1e53966ad7e1badaace4622cad263f606dccd670 Mon Sep 17 00:00:00 2001 From: Alex Kirszenberg Date: Mon, 3 Jul 2023 11:08:00 +0200 Subject: [PATCH] ReverseTopological -> AdjacencyMap (vercel/turbo#5430) ### Description This PR renames the `ReverseTopological` graph store implementation to `AdjacencyMap`. The reasoning here is that we can implement any graph traversal algorithm on top of `AdjacencyMap`, not just reverse topological. This PR also adds a `reverse_topological_from_node` iterator method to only traverse the subgraph from a given node. Next.js PR: https://github.com/vercel/next.js/compare/alexkirsz/web-866-adjacency-map?expand=1 ### Testing Instructions N/A --- crates/turbo-tasks/src/graph/adjacency_map.rs | 203 ++++++++++++++++++ crates/turbo-tasks/src/graph/mod.rs | 4 +- .../src/graph/reverse_topological.rs | 128 ----------- .../turbopack-build/src/chunking_context.rs | 6 +- .../src/chunk/available_assets.rs | 8 +- crates/turbopack-core/src/chunk/mod.rs | 6 +- crates/turbopack-dev/src/chunking_context.rs | 6 +- crates/turbopack-node/src/lib.rs | 6 +- 8 files changed, 222 insertions(+), 145 deletions(-) create mode 100644 crates/turbo-tasks/src/graph/adjacency_map.rs delete mode 100644 crates/turbo-tasks/src/graph/reverse_topological.rs diff --git a/crates/turbo-tasks/src/graph/adjacency_map.rs b/crates/turbo-tasks/src/graph/adjacency_map.rs new file mode 100644 index 0000000000000..cd85614d18f5f --- /dev/null +++ b/crates/turbo-tasks/src/graph/adjacency_map.rs @@ -0,0 +1,203 @@ +use std::collections::{HashMap, HashSet}; + +use super::graph_store::{GraphNode, GraphStore}; + +/// A graph traversal that builds an adjacency map +pub struct AdjacencyMap +where + T: Eq + std::hash::Hash + Clone, +{ + adjacency_map: HashMap>, + roots: Vec, +} + +impl Default for AdjacencyMap +where + T: Eq + std::hash::Hash + Clone, +{ + fn default() -> Self { + Self::new() + } +} + +impl AdjacencyMap +where + T: Eq + std::hash::Hash + Clone, +{ + /// Creates a new adjacency map + pub fn new() -> Self { + Self { + adjacency_map: HashMap::new(), + roots: Vec::new(), + } + } + + /// Returns an iterator over the root nodes of the graph + pub fn roots(&self) -> impl Iterator { + self.roots.iter() + } + + /// Returns an iterator over the children of the given node + pub fn get(&self, node: &T) -> Option> { + self.adjacency_map.get(node).map(|vec| vec.iter()) + } +} + +impl GraphStore for AdjacencyMap +where + T: Eq + std::hash::Hash + Clone, +{ + type Node = T; + type Handle = T; + + fn insert(&mut self, from_handle: Option, node: GraphNode) -> Option<(Self::Handle, &T)> { + let vec = if let Some(from_handle) = from_handle { + self.adjacency_map + .entry(from_handle) + .or_insert_with(|| Vec::with_capacity(1)) + } else { + &mut self.roots + }; + + vec.push(node.node().clone()); + Some((node.into_node(), vec.last().unwrap())) + } +} + +impl AdjacencyMap +where + T: Eq + std::hash::Hash + Clone, +{ + /// Returns an iterator over the nodes in reverse topological order, + /// starting from the roots. + pub fn into_reverse_topological(self) -> ReverseTopologicalIter { + ReverseTopologicalIter { + adjacency_map: self.adjacency_map, + stack: self + .roots + .into_iter() + .map(|root| (ReverseTopologicalPass::Pre, root)) + .collect(), + visited: HashSet::new(), + } + } + + /// Returns an iterator over the nodes in reverse topological order, + /// starting from the given node. + pub fn reverse_topological_from_node<'graph>( + &'graph self, + node: &'graph T, + ) -> ReverseTopologicalFromNodeIter<'graph, T> { + ReverseTopologicalFromNodeIter { + adjacency_map: &self.adjacency_map, + stack: vec![(ReverseTopologicalPass::Pre, node)], + visited: HashSet::new(), + } + } +} + +#[derive(Debug)] +enum ReverseTopologicalPass { + Pre, + Post, +} + +/// An iterator over the nodes of a graph in reverse topological order, starting +/// from the roots. +pub struct ReverseTopologicalIter +where + T: Eq + std::hash::Hash + Clone, +{ + adjacency_map: HashMap>, + stack: Vec<(ReverseTopologicalPass, T)>, + visited: HashSet, +} + +impl Iterator for ReverseTopologicalIter +where + T: Eq + std::hash::Hash + Clone, +{ + type Item = T; + + fn next(&mut self) -> Option { + let current = loop { + let (pass, current) = self.stack.pop()?; + + match pass { + ReverseTopologicalPass::Post => { + break current; + } + ReverseTopologicalPass::Pre => { + if self.visited.contains(¤t) { + continue; + } + + self.visited.insert(current.clone()); + + let Some(neighbors) = self.adjacency_map.get(¤t) else { + break current; + }; + + self.stack.push((ReverseTopologicalPass::Post, current)); + self.stack.extend( + neighbors + .iter() + .map(|neighbor| (ReverseTopologicalPass::Pre, neighbor.clone())), + ); + } + } + }; + + Some(current) + } +} + +/// An iterator over the nodes of a graph in reverse topological order, starting +/// from a given node. +pub struct ReverseTopologicalFromNodeIter<'graph, T> +where + T: Eq + std::hash::Hash + Clone, +{ + adjacency_map: &'graph HashMap>, + stack: Vec<(ReverseTopologicalPass, &'graph T)>, + visited: HashSet<&'graph T>, +} + +impl<'graph, T> Iterator for ReverseTopologicalFromNodeIter<'graph, T> +where + T: Eq + std::hash::Hash + Clone, +{ + type Item = &'graph T; + + fn next(&mut self) -> Option { + let current = loop { + let (pass, current) = self.stack.pop()?; + + match pass { + ReverseTopologicalPass::Post => { + break current; + } + ReverseTopologicalPass::Pre => { + if self.visited.contains(¤t) { + continue; + } + + self.visited.insert(current); + + let Some(neighbors) = self.adjacency_map.get(current) else { + break current; + }; + + self.stack.push((ReverseTopologicalPass::Post, current)); + self.stack.extend( + neighbors + .iter() + .map(|neighbor| (ReverseTopologicalPass::Pre, neighbor)), + ); + } + } + }; + + Some(current) + } +} diff --git a/crates/turbo-tasks/src/graph/mod.rs b/crates/turbo-tasks/src/graph/mod.rs index 1b59ae689a982..e2c0754485387 100644 --- a/crates/turbo-tasks/src/graph/mod.rs +++ b/crates/turbo-tasks/src/graph/mod.rs @@ -1,14 +1,14 @@ +mod adjacency_map; mod control_flow; mod graph_store; mod graph_traversal; mod non_deterministic; -mod reverse_topological; mod visit; mod with_future; +pub use adjacency_map::AdjacencyMap; pub use control_flow::VisitControlFlow; pub use graph_store::{GraphStore, SkipDuplicates}; pub use graph_traversal::{GraphTraversal, GraphTraversalResult}; pub use non_deterministic::NonDeterministic; -pub use reverse_topological::ReverseTopological; pub use visit::Visit; diff --git a/crates/turbo-tasks/src/graph/reverse_topological.rs b/crates/turbo-tasks/src/graph/reverse_topological.rs deleted file mode 100644 index a2fb6af73bb00..0000000000000 --- a/crates/turbo-tasks/src/graph/reverse_topological.rs +++ /dev/null @@ -1,128 +0,0 @@ -use std::collections::{HashMap, HashSet}; - -use super::graph_store::{GraphNode, GraphStore}; - -/// A graph traversal that returns nodes in reverse topological order. -pub struct ReverseTopological -where - T: Eq + std::hash::Hash + Clone, -{ - adjacency_map: HashMap>, - roots: Vec, -} - -impl Default for ReverseTopological -where - T: Eq + std::hash::Hash + Clone, -{ - fn default() -> Self { - Self::new() - } -} - -impl ReverseTopological -where - T: Eq + std::hash::Hash + Clone, -{ - pub fn new() -> Self { - Self { - adjacency_map: HashMap::new(), - roots: Vec::new(), - } - } -} - -impl GraphStore for ReverseTopological -where - T: Eq + std::hash::Hash + Clone, -{ - type Node = T; - type Handle = T; - - fn insert(&mut self, from_handle: Option, node: GraphNode) -> Option<(Self::Handle, &T)> { - let vec = if let Some(from_handle) = from_handle { - self.adjacency_map - .entry(from_handle) - .or_insert_with(|| Vec::with_capacity(1)) - } else { - &mut self.roots - }; - - vec.push(node.node().clone()); - Some((node.into_node(), vec.last().unwrap())) - } -} - -#[derive(Debug)] -enum ReverseTopologicalPass { - Pre, - Post, -} - -impl IntoIterator for ReverseTopological -where - T: Eq + std::hash::Hash + Clone, -{ - type Item = T; - type IntoIter = ReverseTopologicalIntoIter; - - fn into_iter(self) -> Self::IntoIter { - ReverseTopologicalIntoIter { - adjacency_map: self.adjacency_map, - stack: self - .roots - .into_iter() - .map(|root| (ReverseTopologicalPass::Pre, root)) - .collect(), - visited: HashSet::new(), - } - } -} - -pub struct ReverseTopologicalIntoIter -where - T: Eq + std::hash::Hash + Clone, -{ - adjacency_map: HashMap>, - stack: Vec<(ReverseTopologicalPass, T)>, - visited: HashSet, -} - -impl Iterator for ReverseTopologicalIntoIter -where - T: Eq + std::hash::Hash + Clone, -{ - type Item = T; - - fn next(&mut self) -> Option { - let current = loop { - let (pass, current) = self.stack.pop()?; - - match pass { - ReverseTopologicalPass::Post => { - break current; - } - ReverseTopologicalPass::Pre => { - if self.visited.contains(¤t) { - continue; - } - - self.visited.insert(current.clone()); - - let Some(neighbors) = self.adjacency_map.get(¤t) else { - break current; - }; - - self.stack.push((ReverseTopologicalPass::Post, current)); - self.stack.extend( - neighbors - .iter() - .map(|neighbor| (ReverseTopologicalPass::Pre, neighbor.clone())), - ); - } - } - }; - - Some(current) - } -} diff --git a/crates/turbopack-build/src/chunking_context.rs b/crates/turbopack-build/src/chunking_context.rs index 65574e532ef86..41cd417078c63 100644 --- a/crates/turbopack-build/src/chunking_context.rs +++ b/crates/turbopack-build/src/chunking_context.rs @@ -1,7 +1,7 @@ use anyhow::Result; use indexmap::IndexSet; use turbo_tasks::{ - graph::{GraphTraversal, ReverseTopological}, + graph::{AdjacencyMap, GraphTraversal}, primitives::{BoolVc, StringVc}, TryJoinIterExt, Value, }; @@ -322,7 +322,7 @@ async fn get_parallel_chunks(entries: I) -> Result, { - Ok(ReverseTopological::new() + Ok(AdjacencyMap::new() .skip_duplicates() .visit(entries, |chunk: ChunkVc| async move { Ok(chunk @@ -336,7 +336,7 @@ where .await .completed()? .into_inner() - .into_iter()) + .into_reverse_topological()) } async fn get_optimized_chunks(chunks: I) -> Result diff --git a/crates/turbopack-core/src/chunk/available_assets.rs b/crates/turbopack-core/src/chunk/available_assets.rs index 6dfc58ae8b2b4..5ead0a9f14099 100644 --- a/crates/turbopack-core/src/chunk/available_assets.rs +++ b/crates/turbopack-core/src/chunk/available_assets.rs @@ -3,7 +3,7 @@ use std::iter::once; use anyhow::Result; use indexmap::IndexSet; use turbo_tasks::{ - graph::{GraphTraversal, ReverseTopological}, + graph::{AdjacencyMap, GraphTraversal}, primitives::{BoolVc, U64Vc}, TryJoinIterExt, ValueToString, }; @@ -83,7 +83,7 @@ impl AvailableAssetsVc { #[turbo_tasks::function] async fn chunkable_assets_set(root: AssetVc) -> Result { - let assets = ReverseTopological::new() + let assets = AdjacencyMap::new() .skip_duplicates() .visit(once(root), |&asset: &AssetVc| async move { Ok(asset @@ -124,5 +124,7 @@ async fn chunkable_assets_set(root: AssetVc) -> Result { }) .await .completed()?; - Ok(AssetsSetVc::cell(assets.into_inner().into_iter().collect())) + Ok(AssetsSetVc::cell( + assets.into_inner().into_reverse_topological().collect(), + )) } diff --git a/crates/turbopack-core/src/chunk/mod.rs b/crates/turbopack-core/src/chunk/mod.rs index 4b1fd67df36c3..897279326ae71 100644 --- a/crates/turbopack-core/src/chunk/mod.rs +++ b/crates/turbopack-core/src/chunk/mod.rs @@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize}; use tracing::{info_span, Span}; use turbo_tasks::{ debug::ValueDebugFormat, - graph::{GraphTraversal, GraphTraversalResult, ReverseTopological, Visit, VisitControlFlow}, + graph::{AdjacencyMap, GraphTraversal, GraphTraversalResult, Visit, VisitControlFlow}, primitives::{StringReadRef, StringVc}, trace::TraceRawVcs, TryJoinIterExt, Value, ValueToString, ValueToStringVc, @@ -586,11 +586,11 @@ where _phantom: PhantomData, }; - let GraphTraversalResult::Completed(traversal_result) = ReverseTopological::new().visit(root_edges, visit).await else { + let GraphTraversalResult::Completed(traversal_result) = AdjacencyMap::new().visit(root_edges, visit).await else { return Ok(None); }; - let graph_nodes: Vec<_> = traversal_result?.into_iter().collect(); + let graph_nodes: Vec<_> = traversal_result?.into_reverse_topological().collect(); let mut chunk_items = Vec::new(); let mut chunks = Vec::new(); diff --git a/crates/turbopack-dev/src/chunking_context.rs b/crates/turbopack-dev/src/chunking_context.rs index 1cdab38e0581c..dcf68b406dbef 100644 --- a/crates/turbopack-dev/src/chunking_context.rs +++ b/crates/turbopack-dev/src/chunking_context.rs @@ -1,7 +1,7 @@ use anyhow::Result; use indexmap::IndexSet; use turbo_tasks::{ - graph::{GraphTraversal, ReverseTopological}, + graph::{AdjacencyMap, GraphTraversal}, primitives::{BoolVc, StringVc}, TryJoinIterExt, Value, }; @@ -351,7 +351,7 @@ async fn get_parallel_chunks(entries: I) -> Result, { - Ok(ReverseTopological::new() + Ok(AdjacencyMap::new() .skip_duplicates() .visit(entries, |chunk: ChunkVc| async move { Ok(chunk @@ -365,7 +365,7 @@ where .await .completed()? .into_inner() - .into_iter()) + .into_reverse_topological()) } async fn get_optimized_chunks(chunks: I) -> Result diff --git a/crates/turbopack-node/src/lib.rs b/crates/turbopack-node/src/lib.rs index 6648db9bd7cdb..3ca91943386b6 100644 --- a/crates/turbopack-node/src/lib.rs +++ b/crates/turbopack-node/src/lib.rs @@ -11,7 +11,7 @@ pub use node_entry::{ NodeEntry, NodeEntryVc, NodeRenderingEntriesVc, NodeRenderingEntry, NodeRenderingEntryVc, }; use turbo_tasks::{ - graph::{GraphTraversal, ReverseTopological}, + graph::{AdjacencyMap, GraphTraversal}, CompletionVc, CompletionsVc, TryJoinIterExt, ValueToString, }; use turbo_tasks_env::{ProcessEnv, ProcessEnvVc}; @@ -171,7 +171,7 @@ async fn separate_assets( .await }; - let graph = ReverseTopological::new() + let graph = AdjacencyMap::new() .skip_duplicates() .visit(once(Type::Internal(intermediate_asset)), get_asset_children) .await @@ -181,7 +181,7 @@ async fn separate_assets( let mut internal_assets = IndexSet::new(); let mut external_asset_entrypoints = IndexSet::new(); - for item in graph { + for item in graph.into_reverse_topological() { match item { Type::Internal(asset) => { internal_assets.insert(asset);