From b94dfd0a78be5ca084c8545f9551b94b2c58ccc3 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:23:04 +0100 Subject: [PATCH] Turbopack: Single-graph-traversal and migrate next/dynamic (#73222) Closes PACK-3535 - Adds some infrastructure for the single-graph-traversal stuff, ~~focusing on dev with layout segment optimization for now~~ - Migrate next/dynamic collection to it - This shouldn't make any observable change to the output --- Cargo.lock | 4 + crates/next-api/Cargo.toml | 2 + crates/next-api/src/app.rs | 76 +-- crates/next-api/src/dynamic_imports.rs | 233 ++----- crates/next-api/src/lib.rs | 1 + crates/next-api/src/module_graph.rs | 646 ++++++++++++++++++ crates/next-api/src/pages.rs | 26 +- crates/next-api/src/project.rs | 73 +- .../ecmascript_client_reference_module.rs | 70 +- ...cmascript_client_reference_proxy_module.rs | 18 +- .../turbo-tasks/src/graph/adjacency_map.rs | 53 +- .../src/chunk/chunking_context.rs | 20 +- .../crates/turbopack-core/src/chunk/mod.rs | 20 +- .../turbopack-core/src/introspect/utils.rs | 6 + .../turbopack-core/src/reference/mod.rs | 41 +- .../src/references/async_module.rs | 2 +- .../src/worker_chunk/module.rs | 65 +- 17 files changed, 1067 insertions(+), 289 deletions(-) create mode 100644 crates/next-api/src/module_graph.rs diff --git a/Cargo.lock b/Cargo.lock index 84c3e0f1073f8..292ba210ba74e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,9 +3951,11 @@ name = "next-api" version = "0.1.0" dependencies = [ "anyhow", + "auto-hash-map", "futures", "indexmap 2.5.0", "next-core", + "petgraph", "regex", "serde", "serde_json", @@ -4633,6 +4635,8 @@ checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", "indexmap 1.9.3", + "serde", + "serde_derive", ] [[package]] diff --git a/crates/next-api/Cargo.toml b/crates/next-api/Cargo.toml index 431746269c50e..af30185e2ce2f 100644 --- a/crates/next-api/Cargo.toml +++ b/crates/next-api/Cargo.toml @@ -14,9 +14,11 @@ workspace = true [dependencies] anyhow = { workspace = true, features = ["backtrace"] } +auto-hash-map = { workspace = true } futures = { workspace = true } indexmap = { workspace = true } next-core = { workspace = true } +petgraph = { workspace = true, features = ["serde-1"]} regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs index 945ae7bdd430d..fc3d3b6562bca 100644 --- a/crates/next-api/src/app.rs +++ b/crates/next-api/src/app.rs @@ -40,8 +40,8 @@ use serde::{Deserialize, Serialize}; use tracing::Instrument; use turbo_rcstr::RcStr; use turbo_tasks::{ - fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexMap, FxIndexSet, ResolvedVc, - TryJoinIterExt, Value, ValueToString, Vc, + fxindexmap, fxindexset, trace::TraceRawVcs, Completion, FxIndexSet, ResolvedVc, TryJoinIterExt, + Value, ValueToString, Vc, }; use turbo_tasks_env::{CustomProcessEnv, ProcessEnv}; use turbo_tasks_fs::{File, FileContent, FileSystemPath}; @@ -69,12 +69,10 @@ use turbopack_core::{ use turbopack_ecmascript::resolve::cjs_resolve; use crate::{ - dynamic_imports::{ - collect_chunk_group, collect_evaluated_chunk_group, collect_next_dynamic_imports, - VisitedDynamicImportModules, - }, + dynamic_imports::{collect_chunk_group, collect_evaluated_chunk_group}, font::create_font_manifest, loadable_manifest::create_react_loadable_manifest, + module_graph::get_reduced_graphs_for_endpoint, nft_json::NftJsonAsset, paths::{ all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, @@ -912,7 +910,7 @@ impl AppEndpoint { None }; - let (client_dynamic_imports, client_references, client_references_chunks) = + let (next_dynamic_imports, client_references, client_references_chunks) = if process_client_components { let client_shared_chunk_group = get_app_client_shared_chunk_group( AssetIdent::from_path(this.app_project.project().project_path()) @@ -933,6 +931,15 @@ impl AppEndpoint { } let client_shared_availability_info = client_shared_chunk_group.availability_info; + let reduced_graphs = get_reduced_graphs_for_endpoint( + this.app_project.project(), + *rsc_entry, + Vc::upcast(this.app_project.client_module_context()), + ); + let next_dynamic_imports = reduced_graphs + .get_next_dynamic_imports_for_endpoint(*rsc_entry) + .await?; + let client_references = { let ServerEntries { server_component_entries, @@ -961,32 +968,6 @@ impl AppEndpoint { }; let client_references_cell = client_references.clone().cell(); - let client_dynamic_imports = { - let mut client_dynamic_imports = FxIndexMap::default(); - let mut visited_modules = VisitedDynamicImportModules::empty(); - - for refs in client_references - .client_references_by_server_component - .values() - { - let result = collect_next_dynamic_imports( - refs.iter().map(|v| **v).collect(), - Vc::upcast(this.app_project.client_module_context()), - visited_modules, - ) - .await?; - client_dynamic_imports.extend( - result - .client_dynamic_imports - .iter() - .map(|(k, v)| (*k, v.clone())), - ); - visited_modules = *result.visited_modules; - } - - client_dynamic_imports - }; - let client_references_chunks = get_app_client_references_chunks( client_references_cell, client_chunking_context, @@ -1130,7 +1111,7 @@ impl AppEndpoint { } ( - Some(client_dynamic_imports), + Some(next_dynamic_imports), Some(client_references_cell), Some(client_references_chunks), ) @@ -1310,19 +1291,11 @@ impl AppEndpoint { .await?; server_assets.insert(app_paths_manifest_output); - // create react-loadable-manifest for next/dynamic - let mut dynamic_import_modules = collect_next_dynamic_imports( - vec![*ResolvedVc::upcast(app_entry.rsc_entry)], - Vc::upcast(this.app_project.client_module_context()), - VisitedDynamicImportModules::empty(), - ) - .await? - .client_dynamic_imports - .clone(); - dynamic_import_modules.extend(client_dynamic_imports.into_iter().flatten()); let dynamic_import_entries = collect_evaluated_chunk_group( Vc::upcast(client_chunking_context), - dynamic_import_modules, + next_dynamic_imports + .as_deref() + .unwrap_or(&Default::default()), ) .await?; let loadable_manifest_output = create_react_loadable_manifest( @@ -1369,18 +1342,11 @@ impl AppEndpoint { // create react-loadable-manifest for next/dynamic let availability_info = Value::new(AvailabilityInfo::Root); - let mut dynamic_import_modules = collect_next_dynamic_imports( - vec![*ResolvedVc::upcast(app_entry.rsc_entry)], - Vc::upcast(this.app_project.client_module_context()), - VisitedDynamicImportModules::empty(), - ) - .await? - .client_dynamic_imports - .clone(); - dynamic_import_modules.extend(client_dynamic_imports.into_iter().flatten()); let dynamic_import_entries = collect_chunk_group( Vc::upcast(client_chunking_context), - dynamic_import_modules, + next_dynamic_imports + .as_deref() + .unwrap_or(&Default::default()), availability_info, ) .await?; diff --git a/crates/next-api/src/dynamic_imports.rs b/crates/next-api/src/dynamic_imports.rs index fb7eb3ff0ba04..bfc0c31db926b 100644 --- a/crates/next-api/src/dynamic_imports.rs +++ b/crates/next-api/src/dynamic_imports.rs @@ -1,20 +1,13 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use anyhow::{bail, Result}; use futures::Future; -use next_core::next_client_reference::EcmascriptClientReferenceModule; -use serde::{Deserialize, Serialize}; use swc_core::ecma::{ ast::{CallExpr, Callee, Expr, Ident, Lit}, visit::{Visit, VisitWith}, }; -use tracing::{Instrument, Level}; use turbo_rcstr::RcStr; -use turbo_tasks::{ - graph::{GraphTraversal, NonDeterministic, VisitControlFlow, VisitedNodes}, - trace::TraceRawVcs, - FxIndexMap, ReadRef, ResolvedVc, TryJoinIterExt, Value, ValueToString, Vc, -}; +use turbo_tasks::{FxIndexMap, ResolvedVc, TryFlatJoinIterExt, Value, Vc}; use turbopack_core::{ chunk::{ availability_info::AvailabilityInfo, ChunkableModule, ChunkingContext, ChunkingContextExt, @@ -23,14 +16,15 @@ use turbopack_core::{ context::AssetContext, module::Module, output::OutputAssets, - reference::primary_referenced_modules, reference_type::EcmaScriptModulesReferenceSubType, resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern}, }; use turbopack_ecmascript::{parse::ParseResult, resolve::esm_resolve, EcmascriptParsable}; +use crate::module_graph::SingleModuleGraph; + async fn collect_chunk_group_inner( - dynamic_import_entries: FxIndexMap>, DynamicImportedModules>, + dynamic_import_entries: &FxIndexMap>, DynamicImportedModules>, mut build_chunk: F, ) -> Result> where @@ -44,11 +38,11 @@ where // dynamic import. for (origin_module, dynamic_imports) in dynamic_import_entries { for (imported_raw_str, imported_module) in dynamic_imports { - let chunk = if let Some(chunk) = chunks_hash.get(&imported_raw_str) { + let chunk = if let Some(chunk) = chunks_hash.get(imported_raw_str) { *chunk } else { let Some(module) = - ResolvedVc::try_sidecast::>(imported_module).await? + ResolvedVc::try_sidecast::>(*imported_module).await? else { bail!("module must be evaluatable"); }; @@ -65,7 +59,7 @@ where }; dynamic_import_chunks - .entry(origin_module) + .entry(*origin_module) .or_insert_with(Vec::new) .push((imported_raw_str.clone(), chunk)); } @@ -76,7 +70,7 @@ where pub(crate) async fn collect_chunk_group( chunking_context: Vc>, - dynamic_import_entries: FxIndexMap>, DynamicImportedModules>, + dynamic_import_entries: &FxIndexMap>, DynamicImportedModules>, availability_info: Value, ) -> Result> { collect_chunk_group_inner(dynamic_import_entries, |module| async move { @@ -87,7 +81,7 @@ pub(crate) async fn collect_chunk_group( pub(crate) async fn collect_evaluated_chunk_group( chunking_context: Vc>, - dynamic_import_entries: FxIndexMap>, DynamicImportedModules>, + dynamic_import_entries: &FxIndexMap>, DynamicImportedModules>, ) -> Result> { collect_chunk_group_inner(dynamic_import_entries, |module| async move { if let Some(module) = Vc::try_resolve_downcast::>(module).await? { @@ -103,24 +97,7 @@ pub(crate) async fn collect_evaluated_chunk_group( .await } -#[turbo_tasks::value(shared)] -pub struct NextDynamicImportsResult { - pub client_dynamic_imports: FxIndexMap>, DynamicImportedModules>, - pub visited_modules: ResolvedVc, -} - -#[turbo_tasks::value(shared)] -pub struct VisitedDynamicImportModules(HashSet); - -#[turbo_tasks::value_impl] -impl VisitedDynamicImportModules { - #[turbo_tasks::function] - pub fn empty() -> Vc { - VisitedDynamicImportModules(Default::default()).cell() - } -} - -/// Returns a mapping of the dynamic imports for each module, if the import is +/// Returns a mapping of the dynamic imports for the module, if the import is /// wrapped in `next/dynamic`'s `dynamic()`. Refer [documentation](https://nextjs.org/docs/pages/building-your-application/optimizing/lazy-loading#with-named-exports) for the usecases. /// /// If an import is specified as dynamic, next.js does few things: @@ -143,155 +120,7 @@ impl VisitedDynamicImportModules { /// to wait until all the dynamic components are being loaded, this ensures hydration mismatch /// won't occur #[turbo_tasks::function] -pub(crate) async fn collect_next_dynamic_imports( - // `server_entries` cannot be a `Vc>` because that would compare by cell identity and - // not by value, breaking memoization. - server_entries: Vec>>, - client_asset_context: Vc>, - visited_modules: Vc, -) -> Result> { - async move { - // Traverse referenced modules graph, collect all of the dynamic imports: - // - Read the Program AST of the Module, this is the origin (A) - // - If there's `dynamic(import(B))`, then B is the module that is being imported - // Returned import mappings are in the form of - // (Module, Vec<(B, Module)>) (where B is the raw import source string, - // and Module is the actual resolved Module) - let (result, visited_modules) = NonDeterministic::new() - .skip_duplicates_with_visited_nodes(VisitedNodes(visited_modules.await?.0.clone())) - .visit( - server_entries - .iter() - .map(|module| async move { - Ok(NextDynamicVisitEntry::Module( - module.to_resolved().await?, - module.ident().to_string().await?, - )) - }) - .try_join() - .await? - .into_iter(), - NextDynamicVisit { - client_asset_context: client_asset_context.resolve().await?, - }, - ) - .await - .completed()? - .into_inner_with_visited(); - - let imported_modules_mapping = result.into_iter().filter_map(|entry| { - if let NextDynamicVisitEntry::DynamicImportsMap(dynamic_imports_map) = entry { - Some(dynamic_imports_map) - } else { - None - } - }); - - // Consolidate import mappings into a single indexmap - let mut import_mappings: FxIndexMap>, DynamicImportedModules> = - FxIndexMap::default(); - - for module_mapping in imported_modules_mapping { - let (origin_module, dynamic_imports) = &*module_mapping.await?; - import_mappings - .entry(*origin_module) - .or_insert_with(Vec::new) - .append(&mut dynamic_imports.clone()) - } - - Ok(NextDynamicImportsResult { - client_dynamic_imports: import_mappings, - visited_modules: VisitedDynamicImportModules(visited_modules.0).resolved_cell(), - } - .cell()) - } - .instrument(tracing::info_span!("collecting next/dynamic imports")) - .await -} - -#[derive(Debug, PartialEq, Eq, Hash, Clone, TraceRawVcs, Serialize, Deserialize)] -enum NextDynamicVisitEntry { - Module(ResolvedVc>, ReadRef), - DynamicImportsMap(ResolvedVc), -} - -#[turbo_tasks::value(transparent)] -struct NextDynamicVisitEntries(Vec); - -#[turbo_tasks::function] -async fn get_next_dynamic_edges( - client_asset_context: Vc>, - module: Vc>, -) -> Result> { - let dynamic_imports_map = build_dynamic_imports_map_for_module(client_asset_context, module); - - let mut edges = if Vc::try_resolve_downcast_type::(module) - .await? - .is_some() - { - vec![] - } else { - primary_referenced_modules(module) - .await? - .iter() - .map(|&referenced_module| async move { - Ok(NextDynamicVisitEntry::Module( - referenced_module, - referenced_module.ident().to_string().await?, - )) - }) - .try_join() - .await? - }; - - if let Some(dynamic_imports_map) = *dynamic_imports_map.await? { - edges.reserve_exact(1); - edges.push(NextDynamicVisitEntry::DynamicImportsMap( - dynamic_imports_map, - )); - } - Ok(Vc::cell(edges)) -} - -struct NextDynamicVisit { - client_asset_context: Vc>, -} - -impl turbo_tasks::graph::Visit for NextDynamicVisit { - type Edge = NextDynamicVisitEntry; - type EdgesIntoIter = impl Iterator; - type EdgesFuture = impl Future>; - - fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow { - match edge { - NextDynamicVisitEntry::Module(..) => VisitControlFlow::Continue(edge), - NextDynamicVisitEntry::DynamicImportsMap(_) => VisitControlFlow::Skip(edge), - } - } - - fn edges(&mut self, entry: &NextDynamicVisitEntry) -> Self::EdgesFuture { - let &NextDynamicVisitEntry::Module(module, _) = entry else { - unreachable!(); - }; - let client_asset_context = self.client_asset_context; - async move { - Ok(get_next_dynamic_edges(client_asset_context, *module) - .await? - .into_iter() - .cloned()) - } - } - - fn span(&mut self, entry: &NextDynamicVisitEntry) -> tracing::Span { - let NextDynamicVisitEntry::Module(_, name) = entry else { - unreachable!(); - }; - tracing::span!(Level::INFO, "next/dynamic visit", name = display(name)) - } -} - -#[turbo_tasks::function] -async fn build_dynamic_imports_map_for_module( +pub async fn build_dynamic_imports_map_for_module( client_asset_context: Vc>, server_module: ResolvedVc>, ) -> Result> { @@ -439,3 +268,41 @@ pub struct OptionDynamicImportsMap(Option>); pub struct DynamicImportedChunks( pub FxIndexMap>, DynamicImportedOutputAssets>, ); + +/// "app/client.js [app-ssr] (ecmascript)" -> +/// [("./dynamic", "app/dynamic.js [app-client] (ecmascript)")])] +#[turbo_tasks::value(transparent)] +pub struct DynamicImports(pub FxIndexMap>, DynamicImportedModules>); + +#[turbo_tasks::function] +pub async fn map_next_dynamic( + graph: Vc, + client_asset_context: Vc>, +) -> Result> { + let data = graph + .await? + .enumerate_nodes() + .map(|(_, node)| { + async move { + // TODO: compare module contexts instead? + let is_browser = node + .layer + .as_ref() + .is_some_and(|layer| &**layer == "app-client" || &**layer == "client"); + if !is_browser { + // Only collect in RSC and SSR + if let Some(v) = + &*build_dynamic_imports_map_for_module(client_asset_context, *node.module) + .await? + { + return Ok(Some(v.await?.clone_value())); + } + } + Ok(None) + } + }) + .try_flat_join() + .await?; + + Ok(Vc::cell(data.into_iter().collect())) +} diff --git a/crates/next-api/src/lib.rs b/crates/next-api/src/lib.rs index 95ea0e0e4d54f..17dec0cf4e7ee 100644 --- a/crates/next-api/src/lib.rs +++ b/crates/next-api/src/lib.rs @@ -12,6 +12,7 @@ pub mod global_module_id_strategy; mod instrumentation; mod loadable_manifest; mod middleware; +mod module_graph; mod nft_json; mod pages; pub mod paths; diff --git a/crates/next-api/src/module_graph.rs b/crates/next-api/src/module_graph.rs new file mode 100644 index 0000000000000..ef964abfe018c --- /dev/null +++ b/crates/next-api/src/module_graph.rs @@ -0,0 +1,646 @@ +use std::{ + collections::{HashMap, HashSet}, + future::Future, + hash::Hash, + ops::Deref, +}; + +use anyhow::{Context, Result}; +use next_core::{ + mode::NextMode, + next_client_reference::{find_server_entries, ServerEntries}, +}; +use petgraph::{ + graph::{DiGraph, NodeIndex}, + visit::{Dfs, VisitMap, Visitable}, +}; +use serde::{Deserialize, Serialize}; +use tracing::Instrument; +use turbo_rcstr::RcStr; +use turbo_tasks::{ + debug::ValueDebugFormat, + graph::{AdjacencyMap, GraphTraversal, Visit, VisitControlFlow, VisitedNodes}, + trace::{TraceRawVcs, TraceRawVcsContext}, + CollectiblesSource, FxIndexMap, ReadRef, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, + ValueToString, Vc, +}; +use turbopack_core::{ + chunk::ChunkingType, + context::AssetContext, + issue::{Issue, IssueExt}, + module::{Module, Modules}, + reference::primary_chunkable_referenced_modules, +}; + +use crate::{ + dynamic_imports::{map_next_dynamic, DynamicImports}, + project::Project, +}; + +#[turbo_tasks::value(transparent)] +#[derive(Clone, Debug)] +struct SingleModuleGraphs(pub Vec>); + +#[derive(PartialEq, Eq, Debug)] +pub enum GraphTraversalAction { + /// Continue visiting children + Continue, + /// Skip the immediate children + Skip, +} + +#[derive(Clone, Debug, Serialize, Deserialize, TraceRawVcs)] +pub struct SingleModuleGraphNode { + pub module: ResolvedVc>, + pub issues: Vec>>, + pub layer: Option>, +} +impl SingleModuleGraphNode { + fn emit_issues(&self) { + for issue in &self.issues { + issue.emit(); + } + } +} + +#[derive(Clone, Debug, ValueDebugFormat, Serialize, Deserialize)] +struct TracedDiGraph(DiGraph); +impl Default for TracedDiGraph { + fn default() -> Self { + Self(Default::default()) + } +} +impl TraceRawVcs for TracedDiGraph { + fn trace_raw_vcs(&self, trace_context: &mut TraceRawVcsContext) { + for node in self.0.node_weights() { + node.trace_raw_vcs(trace_context); + } + for edge in self.0.edge_weights() { + edge.trace_raw_vcs(trace_context); + } + } +} +impl Deref for TracedDiGraph { + type Target = DiGraph; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[turbo_tasks::value(cell = "new", eq = "manual", into = "new")] +#[derive(Clone, Default)] +pub struct SingleModuleGraph { + graph: TracedDiGraph, + // NodeIndex isn't necessarily stable, but these are first nodes in the graph, so shouldn't + // ever be involved in a swap_remove operation + // + // HashMaps have nondeterministic order, but this map is only used for lookups (in `get_entry`) + // and not iteration. + // + // This contains Vcs, but they are already contained in the graph, so no need to trace this. + #[turbo_tasks(trace_ignore)] + entries: HashMap>, NodeIndex>, +} + +#[turbo_tasks::value(transparent)] +#[derive(Clone, Debug)] +struct ModuleSet(pub HashSet>>); + +#[derive(Clone, Hash, PartialEq, Eq)] +enum SingleModuleGraphBuilderNode { + Module { + module: ResolvedVc>, + layer: Option>, + ident: ReadRef, + }, + #[allow(dead_code)] + Issues(Vec>>), +} + +impl SingleModuleGraphBuilderNode { + async fn new(module: ResolvedVc>) -> Result { + let ident = module.ident(); + Ok(Self::Module { + module, + layer: match ident.await?.layer { + Some(layer) => Some(layer.await?), + None => None, + }, + ident: ident.to_string().await?, + }) + } + fn module(&self) -> Option>> { + match self { + SingleModuleGraphBuilderNode::Module { module, .. } => Some(*module), + SingleModuleGraphBuilderNode::Issues(_) => None, + } + } +} +struct SingleModuleGraphBuilderEdge { + to: SingleModuleGraphBuilderNode, +} +struct SingleModuleGraphBuilder {} +impl Visit for SingleModuleGraphBuilder { + type Edge = SingleModuleGraphBuilderEdge; + type EdgesIntoIter = Vec; + type EdgesFuture = impl Future>; + + fn visit(&mut self, edge: Self::Edge) -> VisitControlFlow { + match edge.to { + SingleModuleGraphBuilderNode::Module { .. } => VisitControlFlow::Continue(edge.to), + SingleModuleGraphBuilderNode::Issues(_) => VisitControlFlow::Skip(edge.to), + } + } + + fn edges(&mut self, node: &SingleModuleGraphBuilderNode) -> Self::EdgesFuture { + let module = node.module(); + async move { + // This error should never occur since we always skip visiting these + let module = module.context("visiting SingleModuleGraphBuilderNode::Issues")?; + + let refs_cell = primary_chunkable_referenced_modules(*module); + let refs = refs_cell.await?; + // TODO This is currently too slow + // let refs_issues = refs_cell + // .take_collectibles::>() + // .iter() + // .map(|issue| issue.to_resolved()) + // .try_join() + // .await?; + + let edges = refs + .iter() + .flat_map(|(ty, modules)| { + if matches!(ty, ChunkingType::Traced) { + None + } else { + Some(modules.iter()) + } + }) + .flatten() + .map(|m| async move { + Ok(SingleModuleGraphBuilderEdge { + to: SingleModuleGraphBuilderNode::new(*m).await?, + }) + }) + .try_join() + .await?; + // if !refs_issues.is_empty() { + // x.push(SingleModuleGraphBuilderEdge { + // to: SingleModuleGraphBuilderNode::Issues(refs_issues), + // }); + // } + Ok(edges) + } + } + + fn span(&mut self, node: &SingleModuleGraphBuilderNode) -> tracing::Span { + match node { + SingleModuleGraphBuilderNode::Module { ident, .. } => { + tracing::info_span!("module", name = display(ident)) + } + SingleModuleGraphBuilderNode::Issues(_) => { + tracing::info_span!("issues") + } + } + } +} + +impl SingleModuleGraph { + /// Walks the graph starting from the given entries and collects all reachable nodes, skipping + /// nodes listed in `visited_modules` + /// If passed, `root` is connected to the entries and include in `self.entries`. + async fn new_inner( + root: Option>>, + entries: &Vec>>, + visited_modules: &HashSet>>, + ) -> Result> { + let mut graph = DiGraph::new(); + + let root_edges = entries + .iter() + .map(|e| async move { + Ok(SingleModuleGraphBuilderEdge { + to: SingleModuleGraphBuilderNode::new(*e).await?, + }) + }) + .try_join() + .await?; + let children_modules_iter = AdjacencyMap::new() + .skip_duplicates_with_visited_nodes(VisitedNodes( + visited_modules + .iter() + .map(|&module| SingleModuleGraphBuilderNode::new(module)) + .try_join() + .await? + .into_iter() + .collect(), + )) + .visit(root_edges, SingleModuleGraphBuilder {}) + .await + .completed()? + .into_inner(); + + let mut modules: HashMap>, NodeIndex> = HashMap::new(); + { + let _span = tracing::info_span!("build module graph").entered(); + for (parent, current) in children_modules_iter.into_breadth_first_edges() { + let parent_idx = + parent.map(|parent| *modules.get(&parent.module().unwrap()).unwrap()); + + match current { + SingleModuleGraphBuilderNode::Module { + module, + layer, + ident: _, + } => { + if let Some(idx) = modules.get(&module) { + if let Some(parent_idx) = parent_idx { + graph.add_edge(parent_idx, *idx, ()); + } + } else { + let idx = graph.add_node(SingleModuleGraphNode { + module, + issues: Default::default(), + layer, + }); + modules.insert(module, idx); + if let Some(parent_idx) = parent_idx { + graph.add_edge(parent_idx, idx, ()); + } + } + } + SingleModuleGraphBuilderNode::Issues(issues) => { + let parent_idx = parent_idx.unwrap(); + graph + .node_weight_mut(parent_idx) + .unwrap() + .issues + .extend(issues) + } + } + } + } + + let root_idx = root.and_then(|root| { + if !modules.contains_key(&root) { + let root_idx = graph.add_node(SingleModuleGraphNode { + module: root, + issues: Default::default(), + layer: None, + }); + for entry in entries { + graph.add_edge(root_idx, *modules.get(entry).unwrap(), ()); + } + Some((root, root_idx)) + } else { + None + } + }); + + Ok(SingleModuleGraph { + graph: TracedDiGraph(graph), + entries: entries + .iter() + .map(|e| (*e, *modules.get(e).unwrap())) + .chain(root_idx.into_iter()) + .collect(), + } + .cell()) + } + + fn get_entry(&self, module: ResolvedVc>) -> Result { + self.entries + .get(&module) + .copied() + .context("Couldn't find entry module in graph") + } + + pub fn enumerate_nodes( + &self, + ) -> impl Iterator + '_ { + self.graph + .node_indices() + .map(move |idx| (idx, self.graph.node_weight(idx).unwrap())) + } + + /// Traverses all reachable nodes (once) + pub fn traverse_from_entry<'a>( + &'a self, + entry: ResolvedVc>, + mut visitor: impl FnMut(&'a SingleModuleGraphNode), + ) -> Result<()> { + let entry_node = self.get_entry(entry)?; + + let mut dfs = Dfs::new(&*self.graph, entry_node); + while let Some(nx) = dfs.next(&*self.graph) { + let weight = self.graph.node_weight(nx).unwrap(); + weight.emit_issues(); + visitor(weight); + } + Ok(()) + } + + /// Traverses all reachable edges exactly once and calls the visitor with the edge source and + /// target. + /// + /// This means that target nodes can be revisited (but not recursively). + /// + /// Edges are traversed in reverse order, so recently added edges are added last. + pub fn traverse_edges_from_entry<'a>( + &'a self, + entry: ResolvedVc>, + mut visitor: impl FnMut( + (Option<&'a SingleModuleGraphNode>, &'a SingleModuleGraphNode), + ) -> GraphTraversalAction, + ) -> Result<()> { + let graph = &self.graph; + let entry_node = self.get_entry(entry)?; + + let mut stack = vec![entry_node]; + let mut discovered = graph.visit_map(); + let entry_weight = graph.node_weight(entry_node).unwrap(); + entry_weight.emit_issues(); + visitor((None, entry_weight)); + + while let Some(node) = stack.pop() { + let node_weight = graph.node_weight(node).unwrap(); + if discovered.visit(node) { + for succ in graph.neighbors(node).collect::>().into_iter().rev() { + let succ_weight = graph.node_weight(succ).unwrap(); + let action = visitor((Some(node_weight), succ_weight)); + if !discovered.is_visited(&succ) && action == GraphTraversalAction::Continue { + stack.push(succ); + } + } + } + } + + Ok(()) + } +} + +#[turbo_tasks::value_impl] +impl SingleModuleGraph { + #[turbo_tasks::function] + async fn new_with_entries(entries: Vc) -> Result> { + SingleModuleGraph::new_inner(None, &*entries.await?, &Default::default()).await + } + + /// `root` is connected to the entries and include in `self.entries`. + #[turbo_tasks::function] + async fn new_with_entries_visited( + root: ResolvedVc>, + // This must not be a Vc> to ensure layout segment optimization hits the cache + entries: Vec>>, + visited_modules: Vc, + ) -> Result> { + SingleModuleGraph::new_inner(Some(root), &entries, &*visited_modules.await?).await + } +} + +/// Implements layout segment optimization to compute a graph "chain" for each layout segment +#[turbo_tasks::function] +async fn get_module_graph_for_endpoint( + entry: ResolvedVc>, +) -> Result> { + let ServerEntries { + server_utils, + server_component_entries, + } = &*find_server_entries(*entry).await?; + + let graph = SingleModuleGraph::new_with_entries_visited( + *entry, + server_utils.iter().map(|m| **m).collect(), + Vc::cell(Default::default()), + ) + .to_resolved() + .await?; + let mut visited_modules: HashSet<_> = graph + .await? + .graph + .node_weights() + .map(|n| n.module) + .collect(); + + let mut graphs = vec![graph]; + for module in server_component_entries + .iter() + .map(|m| ResolvedVc::upcast::>(*m)) + { + let graph = SingleModuleGraph::new_with_entries_visited( + *entry, + vec![*module], + Vc::cell(visited_modules.clone()), + ) + .to_resolved() + .await?; + visited_modules.extend(graph.await?.graph.node_weights().map(|n| n.module)); + graphs.push(graph); + } + let graph = SingleModuleGraph::new_with_entries_visited( + *entry, + vec![*entry], + Vc::cell(visited_modules.clone()), + ) + .to_resolved() + .await?; + graphs.push(graph); + + Ok(Vc::cell(graphs)) +} + +#[turbo_tasks::value] +pub struct NextDynamicGraph { + is_single_page: bool, + graph: ResolvedVc, + /// RSC/SSR importer -> dynamic imports (specifier and client module) + data: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl NextDynamicGraph { + #[turbo_tasks::function] + pub async fn new_with_entries( + graph: ResolvedVc, + is_single_page: bool, + client_asset_context: Vc>, + ) -> Result> { + let mapped = map_next_dynamic(*graph, client_asset_context); + mapped.strongly_consistent().await?; + // TODO this can be removed once next/dynamic collection is moved to the transition instead + // of AST traversal + let _ = mapped.take_collectibles::>(); + + // TODO shrink graph here, using the information from + // - `mapped` (which lists the relevant nodes) + // - `graph.entries` (which lists the page/route/... entries we need to keep) + + // This would clone the graph and allow changing the node weights. We can probably get away + // with keeping the sidecar information separate from the graph itself, though. + // + // let mut reduced_modules: HashMap>, NodeIndex> = + // HashMap::new(); let mut reduced_graph = DiGraph::new(); + // for idx in graph.node_indices() { + // let weight = *graph.node_weight(idx).unwrap(); + // let new_idx = reduced_graph.add_node(weight); + // reduced_modules.insert(weight, new_idx); + // for e in graph.edges_directed(idx, petgraph::Direction::Outgoing) { + // let target_weight = *graph.node_weight(e.target()).context("Missing + // target")?; if let Some(new_target_idx) = + // reduced_modules.get(&target_weight) { + // reduced_graph.add_edge(new_idx, *new_target_idx, ()); } else { + // let new_idx = reduced_graph.add_node(target_weight); + // reduced_modules.insert(target_weight, new_idx); + // } + // } + // } + + Ok(NextDynamicGraph { + is_single_page, + graph, + data: mapped.to_resolved().await?, + } + .cell()) + } + + #[turbo_tasks::function] + pub async fn get_next_dynamic_imports_for_endpoint( + &self, + entry: ResolvedVc>, + ) -> Result> { + let span = tracing::info_span!("collect next/dynamic imports for endpoint"); + async move { + if self.is_single_page { + // The graph contains the endpoint (= `entry`) only, no need to filter. + Ok(*self.data) + } else { + // The graph contains the whole app, traverse and collect all reachable imports. + let graph = &*self.graph.await?; + let data = &self.data.await?; + + let mut result = FxIndexMap::default(); + graph.traverse_from_entry(entry, |node| { + if let Some(node_data) = data.get(&node.module) { + result.insert(node.module, node_data.clone()); + } + })?; + Ok(Vc::cell(result)) + } + } + .instrument(span) + .await + } +} + +/// The consumers of this shouldn't need to care about the exact contents since it's abstracted away +/// by the accessor functions, but +/// - In dev, contains information about the modules of the current endpoint only +/// - In prod, there is a single `ReducedGraphs` for the whole app, containing all pages +#[turbo_tasks::value] +pub struct ReducedGraphs { + next_dynamic: Vec>, + // TODO add other graphs +} + +#[turbo_tasks::value_impl] +impl ReducedGraphs { + /// Returns the dynamic imports in RSC and SSR modules for the given endpoint. + #[turbo_tasks::function] + pub async fn get_next_dynamic_imports_for_endpoint( + &self, + entry: Vc>, + ) -> Result> { + let span = tracing::info_span!("collect all next/dynamic imports for endpoint"); + async move { + if let [graph] = &self.next_dynamic[..] { + // Just a single graph, no need to merge results + Ok(graph.get_next_dynamic_imports_for_endpoint(entry)) + } else { + let result = self + .next_dynamic + .iter() + .map(|graph| async move { + Ok(graph + .get_next_dynamic_imports_for_endpoint(entry) + .await? + .iter() + .map(|(k, v)| (*k, v.clone())) + // TODO remove this collect and return an iterator instead + .collect::>()) + }) + .try_flat_join() + .await?; + + Ok(Vc::cell(result.into_iter().collect())) + } + } + .instrument(span) + .await + } +} + +#[turbo_tasks::function] +async fn get_reduced_graphs_for_endpoint_inner( + project: Vc, + entry: ResolvedVc>, + // TODO should this happen globally or per endpoint? Do they all have the same context? + client_asset_context: Vc>, +) -> Result> { + let (is_single_page, graphs) = match &*project.next_mode().await? { + NextMode::Development => ( + true, + async move { get_module_graph_for_endpoint(*entry).await } + .instrument(tracing::info_span!("module graph for endpoint")) + .await? + .clone_value(), + ), + NextMode::Build => ( + false, + vec![ + async move { + SingleModuleGraph::new_with_entries(project.get_all_entries()) + .to_resolved() + .await + } + .instrument(tracing::info_span!("module graph for app")) + .await?, + ], + ), + }; + + let next_dynamic = async move { + graphs + .iter() + .map(|graph| { + NextDynamicGraph::new_with_entries(**graph, is_single_page, client_asset_context) + .to_resolved() + }) + .try_join() + .await + } + .instrument(tracing::info_span!("generating next/dynamic graphs")) + .await?; + + Ok(ReducedGraphs { next_dynamic }.cell()) +} + +/// Generates a [ReducedGraph] for the given project and endpoint containing information that is +/// either global (module ids, chunking) or computed globally as a performance optimization (client +/// references, etc). +#[turbo_tasks::function] +pub async fn get_reduced_graphs_for_endpoint( + project: Vc, + entry: Vc>, + // TODO should this happen globally or per endpoint? Do they all have the same context? + client_asset_context: Vc>, +) -> Result> { + // TODO get rid of this function once everything inside of + // `get_reduced_graphs_for_endpoint_inner` calls `take_collectibles()` when needed + let result = get_reduced_graphs_for_endpoint_inner(project, entry, client_asset_context); + if project.next_mode().await?.is_production() { + result.strongly_consistent().await?; + let _issues = result.take_collectibles::>(); + } + Ok(result) +} diff --git a/crates/next-api/src/pages.rs b/crates/next-api/src/pages.rs index 2929fc0aae525..ee7a99d42b1f3 100644 --- a/crates/next-api/src/pages.rs +++ b/crates/next-api/src/pages.rs @@ -63,12 +63,10 @@ use turbopack_ecmascript::resolve::esm_resolve; use turbopack_nodejs::NodeJsChunkingContext; use crate::{ - dynamic_imports::{ - collect_chunk_group, collect_evaluated_chunk_group, collect_next_dynamic_imports, - DynamicImportedChunks, VisitedDynamicImportModules, - }, + dynamic_imports::{collect_chunk_group, collect_evaluated_chunk_group, DynamicImportedChunks}, font::create_font_manifest, loadable_manifest::create_react_loadable_manifest, + module_graph::get_reduced_graphs_for_endpoint, nft_json::NftJsonAsset, paths::{ all_paths_in_root, all_server_paths, get_asset_paths_from_root, get_js_paths_from_root, @@ -846,14 +844,14 @@ impl PageEndpoint { runtime, } = *self.internal_ssr_chunk_module().await?; - let dynamic_import_modules = collect_next_dynamic_imports( - vec![*ResolvedVc::upcast(ssr_module)], - this.pages_project.client_module_context(), - VisitedDynamicImportModules::empty(), - ) - .await? - .client_dynamic_imports - .clone(); + let reduced_graphs = get_reduced_graphs_for_endpoint( + this.pages_project.project(), + *ssr_module, + Vc::upcast(this.pages_project.client_module_context()), + ); + let next_dynamic_imports = reduced_graphs + .get_next_dynamic_imports_for_endpoint(*ssr_module) + .await?; let is_edge = matches!(runtime, NextRuntime::Edge); if is_edge { @@ -876,7 +874,7 @@ impl PageEndpoint { this.pages_project.project().client_chunking_context(); let dynamic_import_entries = collect_evaluated_chunk_group( Vc::upcast(client_chunking_context), - dynamic_import_modules, + &next_dynamic_imports, ) .await? .to_resolved() @@ -911,7 +909,7 @@ impl PageEndpoint { this.pages_project.project().client_chunking_context(); let dynamic_import_entries = collect_chunk_group( Vc::upcast(client_chunking_context), - dynamic_import_modules, + &next_dynamic_imports, Value::new(AvailabilityInfo::Root), ) .await? diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 884badb3897b1..4ce61355561f7 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -47,7 +47,7 @@ use turbopack_core::{ diagnostics::DiagnosticExt, file_source::FileSource, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, - module::Modules, + module::{Module, Modules}, output::{OutputAsset, OutputAssets}, resolve::{find_context_file, FindContextFileResult}, source_map::OptionSourceMap, @@ -68,7 +68,7 @@ use crate::{ instrumentation::InstrumentationEndpoint, middleware::MiddlewareEndpoint, pages::PagesProject, - route::{Endpoint, Route}, + route::{AppPageRoute, Endpoint, Route}, versioned_content_map::{OutputAssetsOperation, VersionedContentMap}, }; @@ -666,6 +666,75 @@ impl Project { get_client_compile_time_info(self.browserslist_query.clone(), self.define_env.client()) } + #[turbo_tasks::function] + pub async fn get_all_entries(self: Vc) -> Result> { + let mut modules = Vec::new(); + + async fn add_endpoint( + endpoint: Vc>, + modules: &mut Vec>>, + ) -> Result<()> { + let root_modules = endpoint.root_modules().await?; + modules.extend(root_modules.iter().copied()); + Ok(()) + } + + modules.extend(self.client_main_modules().await?.iter().copied()); + + let entrypoints = self.entrypoints().await?; + + modules.extend(self.client_main_modules().await?.iter().copied()); + add_endpoint(*entrypoints.pages_error_endpoint, &mut modules).await?; + add_endpoint(*entrypoints.pages_app_endpoint, &mut modules).await?; + add_endpoint(*entrypoints.pages_document_endpoint, &mut modules).await?; + + if let Some(middleware) = &entrypoints.middleware { + add_endpoint(middleware.endpoint, &mut modules).await?; + } + + if let Some(instrumentation) = &entrypoints.instrumentation { + let node_js = instrumentation.node_js; + let edge = instrumentation.edge; + add_endpoint(node_js, &mut modules).await?; + add_endpoint(edge, &mut modules).await?; + } + + for (_, route) in entrypoints.routes.iter() { + match route { + Route::Page { + html_endpoint, + data_endpoint: _, + } => { + add_endpoint(**html_endpoint, &mut modules).await?; + } + Route::PageApi { endpoint } => { + add_endpoint(**endpoint, &mut modules).await?; + } + Route::AppPage(page_routes) => { + for AppPageRoute { + original_name: _, + html_endpoint, + rsc_endpoint: _, + } in page_routes + { + add_endpoint(*html_endpoint, &mut modules).await?; + } + } + Route::AppRoute { + original_name: _, + endpoint, + } => { + add_endpoint(**endpoint, &mut modules).await?; + } + Route::Conflict => { + tracing::info!("WARN: conflict"); + } + } + } + + Ok(Vc::cell(modules)) + } + #[turbo_tasks::function] pub(super) async fn server_compile_time_info(self: Vc) -> Result> { let this = self.await?; diff --git a/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_module.rs b/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_module.rs index 86a9c0ab6ed8d..6cb833e0fd8fd 100644 --- a/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_module.rs +++ b/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_module.rs @@ -1,12 +1,14 @@ #![allow(rustdoc::private_intra_doc_links)] use anyhow::{bail, Result}; use turbo_rcstr::RcStr; -use turbo_tasks::{ResolvedVc, Vc}; +use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbopack_core::{ asset::{Asset, AssetContent}, + chunk::{ChunkGroupType, ChunkableModuleReference, ChunkingType, ChunkingTypeOption}, ident::AssetIdent, module::Module, - reference::{ModuleReferences, SingleModuleReference}, + reference::{ModuleReference, ModuleReferences}, + resolve::ModuleResolveResult, }; use turbopack_ecmascript::chunk::EcmascriptChunkPlaceable; @@ -70,20 +72,20 @@ impl Module for EcmascriptClientReferenceModule { #[turbo_tasks::function] async fn references(&self) -> Result> { - let client_module = ResolvedVc::upcast(self.client_module); - let ssr_module = ResolvedVc::upcast(self.ssr_module); Ok(Vc::cell(vec![ ResolvedVc::upcast( - SingleModuleReference::new( - *client_module, + EcmascriptClientReference::new( + *ResolvedVc::upcast(self.client_module), + ChunkGroupType::Evaluated, ecmascript_client_reference_client_ref_modifier(), ) .to_resolved() .await?, ), ResolvedVc::upcast( - SingleModuleReference::new( - *ssr_module, + EcmascriptClientReference::new( + *ResolvedVc::upcast(self.ssr_module), + ChunkGroupType::Entry, ecmascript_client_reference_ssr_ref_modifier(), ) .to_resolved() @@ -101,3 +103,55 @@ impl Asset for EcmascriptClientReferenceModule { bail!("EcmascriptClientReferenceModule has no content") } } + +#[turbo_tasks::value] +pub(crate) struct EcmascriptClientReference { + module: ResolvedVc>, + ty: ChunkGroupType, + description: ResolvedVc, +} + +#[turbo_tasks::value_impl] +impl EcmascriptClientReference { + #[turbo_tasks::function] + pub fn new( + module: ResolvedVc>, + ty: ChunkGroupType, + description: ResolvedVc, + ) -> Vc { + Self::cell(EcmascriptClientReference { + module, + ty, + description, + }) + } +} + +#[turbo_tasks::value_impl] +impl ChunkableModuleReference for EcmascriptClientReference { + #[turbo_tasks::function] + fn chunking_type(&self) -> Vc { + Vc::cell(Some(ChunkingType::Isolated { + _ty: self.ty, + // TODO use proper values here + _merge_tag: None, + _chunking_context: None, + })) + } +} + +#[turbo_tasks::value_impl] +impl ModuleReference for EcmascriptClientReference { + #[turbo_tasks::function] + fn resolve_reference(&self) -> Vc { + ModuleResolveResult::module(self.module).cell() + } +} + +#[turbo_tasks::value_impl] +impl ValueToString for EcmascriptClientReference { + #[turbo_tasks::function] + fn to_string(&self) -> Vc { + *self.description + } +} diff --git a/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_proxy_module.rs b/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_proxy_module.rs index 3d783ee1fc522..52a2237e4c233 100644 --- a/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_proxy_module.rs +++ b/crates/next-core/src/next_client_reference/ecmascript_client_reference/ecmascript_client_reference_proxy_module.rs @@ -7,12 +7,14 @@ use turbo_tasks::{ResolvedVc, Value, ValueToString, Vc}; use turbo_tasks_fs::File; use turbopack_core::{ asset::{Asset, AssetContent}, - chunk::{AsyncModuleInfo, ChunkItem, ChunkType, ChunkableModule, ChunkingContext}, + chunk::{ + AsyncModuleInfo, ChunkGroupType, ChunkItem, ChunkType, ChunkableModule, ChunkingContext, + }, code_builder::CodeBuilder, context::AssetContext, ident::AssetIdent, module::Module, - reference::{ModuleReferences, SingleModuleReference}, + reference::ModuleReferences, reference_type::ReferenceType, virtual_source::VirtualSource, }; @@ -24,7 +26,10 @@ use turbopack_ecmascript::{ utils::StringifyJs, }; -use super::ecmascript_client_reference_module::EcmascriptClientReferenceModule; +use crate::next_client_reference::{ + ecmascript_client_reference::ecmascript_client_reference_module::EcmascriptClientReference, + EcmascriptClientReferenceModule, +}; /// A [`EcmascriptClientReferenceProxyModule`] is used in RSC to represent /// a client or SSR asset. @@ -186,13 +191,18 @@ impl Module for EcmascriptClientReferenceProxyModule { .await? .iter() .copied() + // TODO this will break once ChunkingType::Isolated is properly implemented. + // + // We should instead merge EcmascriptClientReferenceProxyModule and + // EcmascriptClientReferenceModule into a single module .chain(once(ResolvedVc::upcast( - SingleModuleReference::new( + EcmascriptClientReference::new( Vc::upcast(EcmascriptClientReferenceModule::new( **server_module_ident, **client_module, **ssr_module, )), + ChunkGroupType::Entry, client_reference_description(), ) .to_resolved() diff --git a/turbopack/crates/turbo-tasks/src/graph/adjacency_map.rs b/turbopack/crates/turbo-tasks/src/graph/adjacency_map.rs index 63dbcf4ca0470..7d92520e62861 100644 --- a/turbopack/crates/turbo-tasks/src/graph/adjacency_map.rs +++ b/turbopack/crates/turbo-tasks/src/graph/adjacency_map.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use serde::{Deserialize, Serialize}; use turbo_tasks_macros::{TraceRawVcs, ValueDebugFormat}; @@ -88,6 +88,21 @@ where } } + /// Returns an owned iterator over all edges (node pairs) in breadth first order, + /// starting from the roots. + pub fn into_breadth_first_edges(self) -> IntoBreadthFirstEdges { + IntoBreadthFirstEdges { + adjacency_map: self.adjacency_map, + stack: self + .roots + .into_iter() + .rev() + .map(|root| (None, root)) + .collect(), + visited: HashSet::new(), + } + } + /// Returns an iterator over the nodes in reverse topological order, /// starting from the roots. pub fn reverse_topological(&self) -> ReverseTopologicalIter { @@ -174,6 +189,42 @@ where } } +pub struct IntoBreadthFirstEdges +where + T: Eq + std::hash::Hash + Clone, +{ + adjacency_map: HashMap>, + stack: VecDeque<(Option, T)>, + visited: HashSet, +} + +impl Iterator for IntoBreadthFirstEdges +where + T: Eq + std::hash::Hash + Clone, +{ + type Item = (Option, T); + + fn next(&mut self) -> Option { + let (parent, current) = self.stack.pop_front()?; + + let Some(neighbors) = self.adjacency_map.get(¤t) else { + self.visited.insert(current.clone()); + return Some((parent, current)); + }; + + if self.visited.insert(current.clone()) { + self.stack.extend( + neighbors + .iter() + .rev() + .map(|neighbor| (Some(current.clone()), neighbor.clone())), + ); + } + + Some((parent, current)) + } +} + /// An iterator over the nodes of a graph in reverse topological order, starting /// from the roots. pub struct ReverseTopologicalIter<'graph, T> diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs index 4b98c511b31a1..9e45ccedf8cc9 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs @@ -22,8 +22,6 @@ use crate::{ Copy, PartialEq, Eq, - PartialOrd, - Ord, Hash, Serialize, Deserialize, @@ -36,6 +34,24 @@ pub enum MinifyType { NoMinify, } +#[derive( + Debug, + TaskInput, + Clone, + Copy, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + TraceRawVcs, + DeterministicHash, +)] +pub enum ChunkGroupType { + Entry, + Evaluated, +} + #[turbo_tasks::value(shared)] pub struct ChunkGroupResult { pub assets: ResolvedVc, diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs index a7653fd7b6063..513b964a841a8 100644 --- a/turbopack/crates/turbopack-core/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-core/src/chunk/mod.rs @@ -34,7 +34,8 @@ use turbo_tasks_hash::DeterministicHash; use self::{availability_info::AvailabilityInfo, available_chunk_items::AvailableChunkItems}; pub use self::{ chunking_context::{ - ChunkGroupResult, ChunkingContext, ChunkingContextExt, EntryChunkGroupResult, MinifyType, + ChunkGroupResult, ChunkGroupType, ChunkingContext, ChunkingContextExt, + EntryChunkGroupResult, MinifyType, }, data::{ChunkData, ChunkDataOption, ChunksData}, evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets}, @@ -154,7 +155,6 @@ pub trait OutputChunk: Asset { /// Specifies how a chunk interacts with other chunks when building a chunk /// group #[derive( - Copy, Debug, Default, Clone, @@ -177,6 +177,14 @@ pub enum ChunkingType { /// An async loader is placed into the referencing chunk and loads the /// separate chunk group in which the module is placed. Async, + /// Create a new chunk group in a separate context, merging references with the same tag into a + /// single chunk group. It does not inherit the available modules from the parent. + // TODO implement + Isolated { + _ty: ChunkGroupType, + _merge_tag: Option, + _chunking_context: Option>>, + }, /// Module not placed in chunk group, but its references are still followed and placed into the /// chunk group. Passthrough, @@ -369,7 +377,7 @@ async fn graph_node_to_referenced_nodes( }]); }; - let Some(chunking_type) = *chunkable_module_reference.chunking_type().await? else { + let Some(chunking_type) = &*chunkable_module_reference.chunking_type().await? else { return Ok(vec![ChunkGraphEdge { key: None, node: ChunkContentGraphNode::ExternalModuleReference(reference), @@ -384,7 +392,7 @@ async fn graph_node_to_referenced_nodes( .await? .into_iter() .map(|&module| async move { - if chunking_type == ChunkingType::Traced { + if matches!(chunking_type, ChunkingType::Traced) { if *chunking_context.is_tracing_enabled().await? { return Ok(( Some(ChunkGraphEdge { @@ -492,6 +500,10 @@ async fn graph_node_to_referenced_nodes( )) } } + ChunkingType::Isolated { .. } => { + // TODO implement + Ok((None, None)) + } ChunkingType::Traced => { bail!("unreachable ChunkingType::Traced"); } diff --git a/turbopack/crates/turbopack-core/src/introspect/utils.rs b/turbopack/crates/turbopack-core/src/introspect/utils.rs index 0f4743df8ef10..03237a0c1fb49 100644 --- a/turbopack/crates/turbopack-core/src/introspect/utils.rs +++ b/turbopack/crates/turbopack-core/src/introspect/utils.rs @@ -38,6 +38,11 @@ fn passthrough_reference_ty() -> Vc { Vc::cell("passthrough reference".into()) } +#[turbo_tasks::function] +fn isolated_reference_ty() -> Vc { + Vc::cell("isolated reference".into()) +} + #[turbo_tasks::function] fn traced_reference_ty() -> Vc { Vc::cell("traced reference".into()) @@ -81,6 +86,7 @@ pub async fn children_from_module_references( key = parallel_inherit_async_reference_ty() } Some(ChunkingType::Async) => key = async_reference_ty(), + Some(ChunkingType::Isolated { .. }) => key = isolated_reference_ty(), Some(ChunkingType::Passthrough) => key = passthrough_reference_ty(), Some(ChunkingType::Traced) => key = traced_reference_ty(), } diff --git a/turbopack/crates/turbopack-core/src/reference/mod.rs b/turbopack/crates/turbopack-core/src/reference/mod.rs index 845b2d291e09f..f964ef49d028c 100644 --- a/turbopack/crates/turbopack-core/src/reference/mod.rs +++ b/turbopack/crates/turbopack-core/src/reference/mod.rs @@ -4,7 +4,7 @@ use anyhow::Result; use turbo_rcstr::RcStr; use turbo_tasks::{ graph::{AdjacencyMap, GraphTraversal}, - FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc, + FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc, }; use crate::{ @@ -270,6 +270,45 @@ pub async fn primary_referenced_modules(module: Vc>) -> Result>>; +#[turbo_tasks::value(transparent)] +pub struct ModulesWithChunkingType(Vec<(ChunkingType, ModulesVec)>); + +/// Aggregates all primary [Module]s referenced by an [Module] via [ChunkableModuleReference]s. +/// This does not include transitively references [Module]s, only includes +/// primary [Module]s referenced. +/// +/// [Module]: crate::module::Module +#[turbo_tasks::function] +pub async fn primary_chunkable_referenced_modules( + module: Vc>, +) -> Result> { + let modules = module + .references() + .await? + .iter() + .map(|reference| async { + if let Some(reference) = + ResolvedVc::try_downcast::>(*reference).await? + { + if let Some(chunking_type) = &*reference.chunking_type().await? { + let resolved = reference + .resolve_reference() + .resolve() + .await? + .primary_modules() + .await? + .clone_value(); + return Ok(Some((chunking_type.clone(), resolved))); + } + } + Ok(None) + }) + .try_flat_join() + .await?; + Ok(Vc::cell(modules)) +} + /// Walks the asset graph from multiple assets and collect all referenced /// assets. #[turbo_tasks::function] diff --git a/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs b/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs index c7923beab05dd..ec2ae4f133b65 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/async_module.rs @@ -83,7 +83,7 @@ async fn get_inherit_async_referenced_asset( let Some(r) = Vc::try_resolve_downcast::>(r).await? else { return Ok(None); }; - let Some(ty) = *r.chunking_type().await? else { + let Some(ty) = &*r.chunking_type().await? else { return Ok(None); }; if !matches!(ty, ChunkingType::ParallelInheritAsync) { diff --git a/turbopack/crates/turbopack-ecmascript/src/worker_chunk/module.rs b/turbopack/crates/turbopack-ecmascript/src/worker_chunk/module.rs index c562749c3cea6..05437d500495c 100644 --- a/turbopack/crates/turbopack-ecmascript/src/worker_chunk/module.rs +++ b/turbopack/crates/turbopack-ecmascript/src/worker_chunk/module.rs @@ -1,12 +1,16 @@ use anyhow::Result; use turbo_rcstr::RcStr; -use turbo_tasks::{ResolvedVc, Vc}; +use turbo_tasks::{ResolvedVc, ValueToString, Vc}; use turbopack_core::{ asset::{Asset, AssetContent}, - chunk::{ChunkableModule, ChunkingContext}, + chunk::{ + ChunkGroupType, ChunkableModule, ChunkableModuleReference, ChunkingContext, ChunkingType, + ChunkingTypeOption, + }, ident::AssetIdent, module::Module, - reference::{ModuleReferences, SingleModuleReference}, + reference::{ModuleReference, ModuleReferences}, + resolve::ModuleResolveResult, }; use super::chunk_item::WorkerLoaderChunkItem; @@ -36,11 +40,6 @@ impl WorkerLoaderModule { } } -#[turbo_tasks::function] -fn inner_module_reference_description() -> Vc { - Vc::cell("worker module".into()) -} - #[turbo_tasks::value_impl] impl Module for WorkerLoaderModule { #[turbo_tasks::function] @@ -51,12 +50,9 @@ impl Module for WorkerLoaderModule { #[turbo_tasks::function] async fn references(self: Vc) -> Result> { Ok(Vc::cell(vec![ResolvedVc::upcast( - SingleModuleReference::new( - *ResolvedVc::upcast(self.await?.inner), - inner_module_reference_description(), - ) - .to_resolved() - .await?, + WorkerModuleReference::new(*ResolvedVc::upcast(self.await?.inner)) + .to_resolved() + .await?, )])) } } @@ -85,3 +81,44 @@ impl ChunkableModule for WorkerLoaderModule { ) } } + +#[turbo_tasks::value] +struct WorkerModuleReference { + module: ResolvedVc>, +} + +#[turbo_tasks::value_impl] +impl WorkerModuleReference { + #[turbo_tasks::function] + pub fn new(module: ResolvedVc>) -> Vc { + Self::cell(WorkerModuleReference { module }) + } +} + +#[turbo_tasks::value_impl] +impl ChunkableModuleReference for WorkerModuleReference { + #[turbo_tasks::function] + fn chunking_type(self: Vc) -> Vc { + Vc::cell(Some(ChunkingType::Isolated { + _ty: ChunkGroupType::Evaluated, + _merge_tag: None, + _chunking_context: None, + })) + } +} + +#[turbo_tasks::value_impl] +impl ModuleReference for WorkerModuleReference { + #[turbo_tasks::function] + fn resolve_reference(&self) -> Vc { + ModuleResolveResult::module(self.module).cell() + } +} + +#[turbo_tasks::value_impl] +impl ValueToString for WorkerModuleReference { + #[turbo_tasks::function] + fn to_string(&self) -> Vc { + Vc::cell("worker module".into()) + } +}