diff --git a/crates/turbopack-ecmascript/src/chunk/content.rs b/crates/turbopack-ecmascript/src/chunk/content.rs new file mode 100644 index 0000000000000..f782d9586b7ae --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/content.rs @@ -0,0 +1,360 @@ +use std::io::Write as _; + +use anyhow::{anyhow, bail, Result}; +use indexmap::IndexSet; +use indoc::{indoc, writedoc}; +use turbo_tasks::TryJoinIterExt; +use turbo_tasks_fs::{embed_file, File, FileContent, FileSystemPathReadRef, FileSystemPathVc}; +use turbopack_core::{ + asset::AssetContentVc, + chunk::{ + chunk_content, chunk_content_split, ChunkContentResult, ChunkGroupVc, ChunkVc, + ChunkingContext, ChunkingContextVc, ModuleId, + }, + code_builder::{CodeBuilder, CodeVc}, + environment::{ChunkLoading, EnvironmentVc}, + reference::AssetReferenceVc, + source_map::{GenerateSourceMap, GenerateSourceMapVc, OptionSourceMapVc, SourceMapVc}, + version::{UpdateVc, VersionVc, VersionedContent, VersionedContentVc}, +}; + +use super::{ + evaluate::EcmascriptChunkContentEvaluateVc, + item::{EcmascriptChunkItemVc, EcmascriptChunkItems, EcmascriptChunkItemsVc}, + placeable::{EcmascriptChunkPlaceableVc, EcmascriptChunkPlaceablesVc}, + snapshot::EcmascriptChunkContentEntriesSnapshotReadRef, + version::{EcmascriptChunkVersion, EcmascriptChunkVersionVc}, +}; +use crate::utils::stringify_js; + +#[turbo_tasks::value] +pub struct EcmascriptChunkContentResult { + pub chunk_items: EcmascriptChunkItemsVc, + pub chunks: Vec<ChunkVc>, + pub async_chunk_groups: Vec<ChunkGroupVc>, + pub external_asset_references: Vec<AssetReferenceVc>, +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentResultVc { + #[turbo_tasks::function] + fn filter(self, _other: EcmascriptChunkContentResultVc) -> EcmascriptChunkContentResultVc { + todo!() + } +} + +impl From<ChunkContentResult<EcmascriptChunkItemVc>> for EcmascriptChunkContentResult { + fn from(from: ChunkContentResult<EcmascriptChunkItemVc>) -> Self { + EcmascriptChunkContentResult { + chunk_items: EcmascriptChunkItems(EcmascriptChunkItems::make_chunks(&from.chunk_items)) + .cell(), + chunks: from.chunks, + async_chunk_groups: from.async_chunk_groups, + external_asset_references: from.external_asset_references, + } + } +} + +#[turbo_tasks::function] +pub(crate) fn ecmascript_chunk_content( + context: ChunkingContextVc, + main_entries: EcmascriptChunkPlaceablesVc, + omit_entries: Option<EcmascriptChunkPlaceablesVc>, +) -> EcmascriptChunkContentResultVc { + let mut chunk_content = ecmascript_chunk_content_internal(context, main_entries); + if let Some(omit_entries) = omit_entries { + let omit_chunk_content = ecmascript_chunk_content_internal(context, omit_entries); + chunk_content = chunk_content.filter(omit_chunk_content); + } + chunk_content +} + +#[turbo_tasks::function] +async fn ecmascript_chunk_content_internal( + context: ChunkingContextVc, + entries: EcmascriptChunkPlaceablesVc, +) -> Result<EcmascriptChunkContentResultVc> { + let entries = entries.await?; + let entries = entries.iter().copied(); + + let contents = entries + .map(|entry| ecmascript_chunk_content_single_entry(context, entry)) + .collect::<Vec<_>>(); + + if contents.len() == 1 { + return Ok(contents.into_iter().next().unwrap()); + } + + let mut all_chunk_items = IndexSet::<EcmascriptChunkItemVc>::new(); + let mut all_chunks = IndexSet::<ChunkVc>::new(); + let mut all_async_chunk_groups = IndexSet::<ChunkGroupVc>::new(); + let mut all_external_asset_references = IndexSet::<AssetReferenceVc>::new(); + + for content in contents { + let EcmascriptChunkContentResult { + chunk_items, + chunks, + async_chunk_groups, + external_asset_references, + } = &*content.await?; + for chunk in chunk_items.await?.iter() { + all_chunk_items.extend(chunk.await?.iter().copied()); + } + all_chunks.extend(chunks.iter().copied()); + all_async_chunk_groups.extend(async_chunk_groups.iter().copied()); + all_external_asset_references.extend(external_asset_references.iter().copied()); + } + + let chunk_items = + EcmascriptChunkItems::make_chunks(&all_chunk_items.into_iter().collect::<Vec<_>>()); + Ok(EcmascriptChunkContentResult { + chunk_items: EcmascriptChunkItemsVc::cell(chunk_items), + chunks: all_chunks.into_iter().collect(), + async_chunk_groups: all_async_chunk_groups.into_iter().collect(), + external_asset_references: all_external_asset_references.into_iter().collect(), + } + .cell()) +} + +#[turbo_tasks::function] +async fn ecmascript_chunk_content_single_entry( + context: ChunkingContextVc, + entry: EcmascriptChunkPlaceableVc, +) -> Result<EcmascriptChunkContentResultVc> { + let asset = entry.as_asset(); + + Ok(EcmascriptChunkContentResultVc::cell( + if let Some(res) = chunk_content::<EcmascriptChunkItemVc>(context, asset, None).await? { + res + } else { + chunk_content_split::<EcmascriptChunkItemVc>(context, asset, None).await? + } + .into(), + )) +} + +#[turbo_tasks::value(serialization = "none")] +pub(super) struct EcmascriptChunkContent { + pub(super) module_factories: EcmascriptChunkContentEntriesSnapshotReadRef, + pub(super) chunk_path: FileSystemPathReadRef, + pub(super) output_root: FileSystemPathReadRef, + pub(super) evaluate: Option<EcmascriptChunkContentEvaluateVc>, + pub(super) environment: EnvironmentVc, +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentVc { + #[turbo_tasks::function] + pub(super) async fn new( + context: ChunkingContextVc, + main_entries: EcmascriptChunkPlaceablesVc, + omit_entries: Option<EcmascriptChunkPlaceablesVc>, + chunk_path: FileSystemPathVc, + evaluate: Option<EcmascriptChunkContentEvaluateVc>, + ) -> Result<Self> { + // TODO(alexkirsz) All of this should be done in a transition, otherwise we run + // the risks of values not being strongly consistent with each other. + let chunk_content = ecmascript_chunk_content(context, main_entries, omit_entries); + let chunk_content = chunk_content.await?; + let chunk_path = chunk_path.await?; + let module_factories = chunk_content.chunk_items.to_entry_snapshot().await?; + let output_root = context.output_root().await?; + Ok(EcmascriptChunkContent { + module_factories, + chunk_path, + output_root, + evaluate, + environment: context.environment(), + } + .cell()) + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentVc { + #[turbo_tasks::function] + pub(super) async fn version(self) -> Result<EcmascriptChunkVersionVc> { + let this = self.await?; + let chunk_server_path = if let Some(path) = this.output_root.get_path_to(&*this.chunk_path) + { + path + } else { + bail!( + "chunk path {} is not in output root {}", + this.chunk_path.to_string(), + this.output_root.to_string() + ); + }; + let module_factories_hashes = this + .module_factories + .iter() + .map(|entry| (entry.id.clone(), entry.hash)) + .collect(); + Ok(EcmascriptChunkVersion { + module_factories_hashes, + chunk_server_path: chunk_server_path.to_string(), + } + .cell()) + } + + #[turbo_tasks::function] + async fn code(self) -> Result<CodeVc> { + let this = self.await?; + let chunk_server_path = if let Some(path) = this.output_root.get_path_to(&*this.chunk_path) + { + path + } else { + bail!( + "chunk path {} is not in output root {}", + this.chunk_path.to_string(), + this.output_root.to_string() + ); + }; + let mut code = CodeBuilder::default(); + code += "(self.TURBOPACK = self.TURBOPACK || []).push(["; + + writeln!(code, "{}, {{", stringify_js(chunk_server_path))?; + for entry in &this.module_factories { + write!(code, "\n{}: ", &stringify_js(entry.id()))?; + code.push_code(entry.code()); + code += ","; + } + code += "\n}"; + + if let Some(evaluate) = &this.evaluate { + let evaluate = evaluate.await?; + let condition = evaluate + .ecma_chunks_server_paths + .iter() + .map(|path| format!(" && loadedChunks.has({})", stringify_js(path))) + .collect::<Vec<_>>() + .join(""); + let entries_instantiations = evaluate + .entry_modules_ids + .iter() + .map(|id| async move { + let id = id.await?; + let id = stringify_js(&id); + Ok(format!(r#"instantiateRuntimeModule({id});"#)) as Result<_> + }) + .try_join() + .await? + .join("\n"); + + // Add a runnable to the chunk that requests the entry module to ensure it gets + // executed when the chunk is evaluated. + // The condition stops the entry module from being executed while chunks it + // depend on have not yet been registered. + // The runnable will run every time a new chunk is `.push`ed to TURBOPACK, until + // all dependent chunks have been evaluated. + writedoc!( + code, + r#" + , ({{ loadedChunks, instantiateRuntimeModule }}) => {{ + if(!(true{condition})) return true; + {entries_instantiations} + }} + "# + )?; + } + code += "]);\n"; + if this.evaluate.is_some() { + // When a chunk is executed, it will either register itself with the current + // instance of the runtime, or it will push itself onto the list of pending + // chunks (`self.TURBOPACK`). + // + // When the runtime executes, it will pick up and register all pending chunks, + // and replace the list of pending chunks with itself so later chunks can + // register directly with it. + writedoc!( + code, + r#" + (() => {{ + if (!Array.isArray(globalThis.TURBOPACK)) {{ + return; + }} + "# + )?; + + let specific_runtime_code = match *this.environment.chunk_loading().await? { + ChunkLoading::None => embed_file!("js/src/runtime.none.js").await?, + ChunkLoading::NodeJs => embed_file!("js/src/runtime.nodejs.js").await?, + ChunkLoading::Dom => embed_file!("js/src/runtime.dom.js").await?, + }; + + match &*specific_runtime_code { + FileContent::NotFound => return Err(anyhow!("specific runtime code is not found")), + FileContent::Content(file) => code.push_source(file.content(), None), + }; + + let shared_runtime_code = embed_file!("js/src/runtime.js").await?; + + match &*shared_runtime_code { + FileContent::NotFound => return Err(anyhow!("shared runtime code is not found")), + FileContent::Content(file) => code.push_source(file.content(), None), + }; + + code += indoc! { r#" + })(); + "# }; + } + + if code.has_source_map() { + let filename = this.chunk_path.file_name(); + write!(code, "\n\n//# sourceMappingURL={}.map", filename)?; + } + + Ok(code.build().cell()) + } + + #[turbo_tasks::function] + async fn content(self) -> Result<AssetContentVc> { + let code = self.code().await?; + Ok(File::from(code.source_code().clone()).into()) + } +} + +#[turbo_tasks::value_impl] +impl VersionedContent for EcmascriptChunkContent { + #[turbo_tasks::function] + fn content(self_vc: EcmascriptChunkContentVc) -> AssetContentVc { + self_vc.content() + } + + #[turbo_tasks::function] + fn version(self_vc: EcmascriptChunkContentVc) -> VersionVc { + self_vc.version().into() + } + + #[turbo_tasks::function] + async fn update( + self_vc: EcmascriptChunkContentVc, + from_version: VersionVc, + ) -> Result<UpdateVc> { + update_ecmascript_chunk(self_vc, from_version).await + } +} + +#[turbo_tasks::value_impl] +impl GenerateSourceMap for EcmascriptChunkContent { + #[turbo_tasks::function] + fn generate_source_map(self_vc: EcmascriptChunkContentVc) -> SourceMapVc { + self_vc.code().generate_source_map() + } + + #[turbo_tasks::function] + async fn by_section(&self, section: &str) -> Result<OptionSourceMapVc> { + // Weirdly, the ContentSource will have already URL decoded the ModuleId, and we + // can't reparse that via serde. + if let Ok(id) = ModuleId::parse(section) { + for entry in self.module_factories.iter() { + if id == *entry.id() { + let sm = entry.code_vc.generate_source_map(); + return Ok(OptionSourceMapVc::cell(Some(sm))); + } + } + } + + Ok(OptionSourceMapVc::cell(None)) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/context.rs b/crates/turbopack-ecmascript/src/chunk/context.rs new file mode 100644 index 0000000000000..c5788d317286a --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/context.rs @@ -0,0 +1,35 @@ +use std::fmt::Write; + +use anyhow::Result; +use turbo_tasks::ValueToString; +use turbopack_core::chunk::{ChunkingContext, ChunkingContextVc, ModuleId, ModuleIdVc}; + +use super::item::EcmascriptChunkItemVc; + +#[turbo_tasks::value] +pub(super) struct EcmascriptChunkContext { + context: ChunkingContextVc, +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContextVc { + #[turbo_tasks::function] + pub fn of(context: ChunkingContextVc) -> EcmascriptChunkContextVc { + EcmascriptChunkContextVc::cell(EcmascriptChunkContext { context }) + } + + #[turbo_tasks::function] + pub async fn chunk_item_id(self, chunk_item: EcmascriptChunkItemVc) -> Result<ModuleIdVc> { + let layer = &*self.await?.context.layer().await?; + let mut s = chunk_item.to_string().await?.clone_value(); + if !layer.is_empty() { + if s.ends_with(')') { + s.pop(); + write!(s, ", {layer})")?; + } else { + write!(s, " ({layer})")?; + } + } + Ok(ModuleId::String(s).cell()) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/evaluate.rs b/crates/turbopack-ecmascript/src/chunk/evaluate.rs new file mode 100644 index 0000000000000..2bb97ca431c94 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/evaluate.rs @@ -0,0 +1,83 @@ +use anyhow::Result; +use turbopack_core::{ + asset::Asset, + chunk::{ + chunk_in_group::ChunkInGroupVc, ChunkGroupVc, ChunkingContext, ChunkingContextVc, + ModuleIdVc, + }, +}; + +use super::{ + item::EcmascriptChunkItem, + placeable::{EcmascriptChunkPlaceable, EcmascriptChunkPlaceablesVc}, + EcmascriptChunkVc, +}; + +/// Whether the ES chunk should include and evaluate a runtime. +#[turbo_tasks::value(shared)] +pub struct EcmascriptChunkEvaluate { + /// Entries that will be executed in that order only all chunks are ready. + /// These entries must be included in `main_entries` so that they are + /// available. + pub evaluate_entries: EcmascriptChunkPlaceablesVc, + /// All chunks of this chunk group need to be ready for execution to start. + /// When None, it will use a chunk group created from the current chunk. + pub chunk_group: Option<ChunkGroupVc>, +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkEvaluateVc { + #[turbo_tasks::function] + pub(super) async fn content( + self, + context: ChunkingContextVc, + origin_chunk: EcmascriptChunkVc, + ) -> Result<EcmascriptChunkContentEvaluateVc> { + let &EcmascriptChunkEvaluate { + evaluate_entries, + chunk_group, + } = &*self.await?; + let chunk_group = + chunk_group.unwrap_or_else(|| ChunkGroupVc::from_chunk(origin_chunk.into())); + let evaluate_chunks = chunk_group.chunks().await?; + let mut ecma_chunks_server_paths = Vec::new(); + let mut other_chunks_server_paths = Vec::new(); + let output_root = context.output_root().await?; + for chunk in evaluate_chunks.iter() { + if let Some(chunk_in_group) = ChunkInGroupVc::resolve_from(chunk).await? { + let chunks_server_paths = if let Some(ecma_chunk) = + EcmascriptChunkVc::resolve_from(chunk_in_group.inner()).await? + { + if ecma_chunk == origin_chunk { + continue; + } + &mut ecma_chunks_server_paths + } else { + &mut other_chunks_server_paths + }; + let chunk_path = &*chunk.path().await?; + if let Some(chunk_server_path) = output_root.get_path_to(chunk_path) { + chunks_server_paths.push(chunk_server_path.to_string()); + } + } + } + let entry_modules_ids = evaluate_entries + .await? + .iter() + .map(|entry| entry.as_chunk_item(context).id()) + .collect(); + Ok(EcmascriptChunkContentEvaluate { + ecma_chunks_server_paths, + other_chunks_server_paths, + entry_modules_ids, + } + .cell()) + } +} + +#[turbo_tasks::value] +pub(super) struct EcmascriptChunkContentEvaluate { + pub ecma_chunks_server_paths: Vec<String>, + pub other_chunks_server_paths: Vec<String>, + pub entry_modules_ids: Vec<ModuleIdVc>, +} diff --git a/crates/turbopack-ecmascript/src/chunk/item.rs b/crates/turbopack-ecmascript/src/chunk/item.rs new file mode 100644 index 0000000000000..a15120a544fcd --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/item.rs @@ -0,0 +1,127 @@ +use anyhow::Result; +use indexmap::IndexSet; +use serde::{Deserialize, Serialize}; +use turbo_tasks::{trace::TraceRawVcs, TryJoinIterExt, ValueToString, ValueToStringVc}; +use turbo_tasks_fs::{rope::Rope, FileSystemPathVc}; +use turbopack_core::{ + asset::AssetVc, + chunk::{ + ChunkItem, ChunkItemVc, ChunkableAssetVc, ChunkingContextVc, FromChunkableAsset, ModuleIdVc, + }, +}; + +use super::{ + context::EcmascriptChunkContextVc, + manifest::{chunk_asset::ManifestChunkAssetVc, loader_item::ManifestLoaderItemVc}, + placeable::EcmascriptChunkPlaceableVc, + snapshot::{ + EcmascriptChunkContentEntries, EcmascriptChunkContentEntriesSnapshot, + EcmascriptChunkContentEntriesSnapshotVc, EcmascriptChunkContentEntryVc, + }, + EcmascriptChunkPlaceable, +}; +use crate::ParseResultSourceMapVc; + +#[turbo_tasks::value(shared)] +#[derive(Default)] +pub struct EcmascriptChunkItemContent { + pub inner_code: Rope, + pub source_map: Option<ParseResultSourceMapVc>, + pub options: EcmascriptChunkItemOptions, + pub placeholder_for_future_extensions: (), +} + +#[derive(PartialEq, Eq, Default, Debug, Clone, Serialize, Deserialize, TraceRawVcs)] +pub struct EcmascriptChunkItemOptions { + pub module: bool, + pub exports: bool, + pub this: bool, + pub placeholder_for_future_extensions: (), +} + +#[turbo_tasks::value_trait] +pub trait EcmascriptChunkItem: ChunkItem + ValueToString { + fn related_path(&self) -> FileSystemPathVc; + fn content(&self) -> EcmascriptChunkItemContentVc; + fn chunking_context(&self) -> ChunkingContextVc; + fn id(&self) -> ModuleIdVc { + EcmascriptChunkContextVc::of(self.chunking_context()).chunk_item_id(*self) + } +} + +#[async_trait::async_trait] +impl FromChunkableAsset for EcmascriptChunkItemVc { + async fn from_asset(context: ChunkingContextVc, asset: AssetVc) -> Result<Option<Self>> { + if let Some(placeable) = EcmascriptChunkPlaceableVc::resolve_from(asset).await? { + return Ok(Some(placeable.as_chunk_item(context))); + } + Ok(None) + } + + async fn from_async_asset( + context: ChunkingContextVc, + asset: ChunkableAssetVc, + ) -> Result<Option<Self>> { + let chunk = ManifestChunkAssetVc::new(asset, context); + Ok(Some(ManifestLoaderItemVc::new(context, chunk).into())) + } +} + +#[turbo_tasks::value(transparent)] +pub struct EcmascriptChunkItemsChunk(Vec<EcmascriptChunkItemVc>); + +#[turbo_tasks::value(transparent)] +pub struct EcmascriptChunkItems(pub(super) Vec<EcmascriptChunkItemsChunkVc>); + +impl EcmascriptChunkItems { + pub fn make_chunks(list: &[EcmascriptChunkItemVc]) -> Vec<EcmascriptChunkItemsChunkVc> { + let size = list.len().div_ceil(100); + let chunk_items = list + .chunks(size) + .map(|chunk| EcmascriptChunkItemsChunkVc::cell(chunk.to_vec())) + .collect(); + chunk_items + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkItemsChunkVc { + #[turbo_tasks::function] + async fn to_entry_snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { + let list = self.await?; + Ok(EcmascriptChunkContentEntries( + list.iter() + .map(|chunk_item| EcmascriptChunkContentEntryVc::new(*chunk_item)) + .collect(), + ) + .cell() + .snapshot()) + } +} + +#[turbo_tasks::value(transparent)] +pub(super) struct EcmascriptChunkItemsSet(IndexSet<EcmascriptChunkItemVc>); + +#[turbo_tasks::value_impl] +impl EcmascriptChunkItemsVc { + #[turbo_tasks::function] + pub(super) async fn to_entry_snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { + let list = self.await?; + Ok(EcmascriptChunkContentEntriesSnapshot::Nested( + list.iter() + .map(|chunk| chunk.to_entry_snapshot()) + .try_join() + .await?, + ) + .cell()) + } + + #[turbo_tasks::function] + pub(super) async fn to_set(self) -> Result<EcmascriptChunkItemsSetVc> { + let mut set = IndexSet::new(); + for chunk in self.await?.iter().copied().try_join().await? { + set.extend(chunk.iter().copied()) + } + Ok(EcmascriptChunkItemsSetVc::cell(set)) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/loader.rs b/crates/turbopack-ecmascript/src/chunk/loader.rs deleted file mode 100644 index 27447cf51292f..0000000000000 --- a/crates/turbopack-ecmascript/src/chunk/loader.rs +++ /dev/null @@ -1,350 +0,0 @@ -use std::io::Write as _; - -use anyhow::{anyhow, bail, Result}; -use indexmap::IndexSet; -use turbo_tasks::{primitives::StringVc, ValueToString, ValueToStringVc}; -use turbo_tasks_fs::FileSystemPathVc; -use turbopack_core::{ - asset::{Asset, AssetContentVc, AssetVc}, - chunk::{ - ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkReferenceVc, ChunkVc, ChunkableAsset, - ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext, - ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc, ChunksVc, - }, - reference::{AssetReference, AssetReferenceVc, AssetReferencesVc}, - resolve::{ResolveResult, ResolveResultVc}, -}; - -use crate::{ - chunk::{ - EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc, - EcmascriptChunkItemVc, EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc, - EcmascriptChunkVc, EcmascriptExports, EcmascriptExportsVc, - }, - utils::stringify_js, -}; - -/// The manifest loader item is shipped in the same chunk that uses the dynamic -/// `import()` expression. Its responsibility is to load the manifest chunk from -/// the server. The dynamic import has been rewritten to import this manifest -/// loader item, which will load the manifest chunk from the server, which -/// will load all the chunks needed by the dynamic import. Finally, we'll be -/// able to import the module we're trying to dynamically import. -/// -/// Splitting the dynamic import into a quickly generate-able manifest loader -/// item and a slow-to-generate manifest chunk allows for faster incremental -/// compilation. The traversal won't be performed until the dynamic import is -/// actually reached, instead of eagerly as part of the chunk that the dynamic -/// import appears in. -#[turbo_tasks::value] -pub struct ManifestLoaderItem { - context: ChunkingContextVc, - manifest: ManifestChunkAssetVc, -} - -#[turbo_tasks::value_impl] -impl ManifestLoaderItemVc { - #[turbo_tasks::function] - pub fn new(context: ChunkingContextVc, manifest: ManifestChunkAssetVc) -> Self { - Self::cell(ManifestLoaderItem { context, manifest }) - } -} - -#[turbo_tasks::value_impl] -impl ValueToString for ManifestLoaderItem { - #[turbo_tasks::function] - fn to_string(&self) -> StringVc { - self.manifest - .path() - .parent() - .join("manifest-loader.js") - .to_string() - } -} - -#[turbo_tasks::value_impl] -impl ChunkItem for ManifestLoaderItem { - #[turbo_tasks::function] - fn references(&self) -> AssetReferencesVc { - AssetReferencesVc::cell(vec![ManifestChunkAssetReference { - manifest: self.manifest, - } - .cell() - .into()]) - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkItem for ManifestLoaderItem { - #[turbo_tasks::function] - fn chunking_context(&self) -> ChunkingContextVc { - self.context - } - - #[turbo_tasks::function] - fn related_path(&self) -> FileSystemPathVc { - self.manifest.path() - } - - #[turbo_tasks::function] - async fn content(&self) -> Result<EcmascriptChunkItemContentVc> { - let mut code = Vec::new(); - - let manifest = self.manifest.await?; - let asset = manifest.asset.as_asset(); - let chunk = self.manifest.as_chunk(self.context); - let chunk_path = &*chunk.path().await?; - - let output_root = self.context.output_root().await?; - - // We need several items in order for a dynamic import to fully load. First, we - // need the chunk path of the manifest chunk, relative from the output root. The - // chunk is a servable file, which will contain the manifest chunk item, which - // will perform the actual chunk traversal and generate load statements. - let chunk_server_path = if let Some(path) = output_root.get_path_to(chunk_path) { - path - } else { - bail!( - "chunk path {} is not in output root {}", - chunk.path().to_string().await?, - self.context.output_root().to_string().await? - ); - }; - - // We also need the manifest chunk item's id, which points to a CJS module that - // exports a promise for all of the necessary chunk loads. - let item_id = &*self.manifest.as_chunk_item(self.context).id().await?; - - // Finally, we need the id of the module that we're actually trying to - // dynamically import. - let placeable = EcmascriptChunkPlaceableVc::resolve_from(asset) - .await? - .ok_or_else(|| anyhow!("asset is not placeable in ecmascript chunk"))?; - let dynamic_id = &*placeable.as_chunk_item(self.context).id().await?; - - // TODO: a dedent macro with expression interpolation would be awesome. - write!( - code, - " -__turbopack_export_value__((__turbopack_import__) => {{ - return __turbopack_load__({chunk_server_path}).then(() => {{ - return __turbopack_require__({item_id}); - }}).then(() => __turbopack_import__({dynamic_id})); -}});", - chunk_server_path = stringify_js(chunk_server_path), - item_id = stringify_js(item_id), - dynamic_id = stringify_js(dynamic_id), - )?; - - Ok(EcmascriptChunkItemContent { - inner_code: code.into(), - ..Default::default() - } - .into()) - } -} - -/// The manifest chunk is deferred until requested by the manifest loader -/// item when the dynamic `import()` expression is reached. Its responsibility -/// is to generate a Promise that will resolve only after all the necessary -/// chunks needed by the dynamic import are loaded by the client. -/// -/// Splitting the dynamic import into a quickly generate-able manifest loader -/// item and a slow-to-generate manifest chunk allows for faster incremental -/// compilation. The traversal won't be performed until the dynamic import is -/// actually reached, instead of eagerly as part of the chunk that the dynamic -/// import appears in. -#[turbo_tasks::value(shared)] -pub struct ManifestChunkAsset { - pub asset: ChunkableAssetVc, - pub chunking_context: ChunkingContextVc, -} - -#[turbo_tasks::value_impl] -impl ManifestChunkAssetVc { - #[turbo_tasks::function] - pub fn new(asset: ChunkableAssetVc, chunking_context: ChunkingContextVc) -> Self { - Self::cell(ManifestChunkAsset { - asset, - chunking_context, - }) - } - - #[turbo_tasks::function] - async fn chunks(self) -> Result<ChunksVc> { - let this = self.await?; - let chunk_group = ChunkGroupVc::from_asset(this.asset, this.chunking_context); - Ok(chunk_group.chunks()) - } -} - -#[turbo_tasks::value_impl] -impl Asset for ManifestChunkAsset { - #[turbo_tasks::function] - fn path(&self) -> FileSystemPathVc { - self.asset.path().join("manifest-chunk.js") - } - - #[turbo_tasks::function] - fn content(&self) -> AssetContentVc { - todo!() - } - - #[turbo_tasks::function] - fn references(&self) -> AssetReferencesVc { - todo!() - } -} - -#[turbo_tasks::value_impl] -impl ChunkableAsset for ManifestChunkAsset { - #[turbo_tasks::function] - fn as_chunk(self_vc: ManifestChunkAssetVc, context: ChunkingContextVc) -> ChunkVc { - EcmascriptChunkVc::new(context, self_vc.into()).into() - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkPlaceable for ManifestChunkAsset { - #[turbo_tasks::function] - fn as_chunk_item( - self_vc: ManifestChunkAssetVc, - context: ChunkingContextVc, - ) -> EcmascriptChunkItemVc { - ManifestChunkItem { - context, - manifest: self_vc, - } - .cell() - .into() - } - - #[turbo_tasks::function] - fn get_exports(&self) -> EcmascriptExportsVc { - EcmascriptExports::Value.cell() - } -} - -/// The ManifestChunkItem generates a __turbopack_load__ call for every chunk -/// necessary to load the real asset. Once all the loads resolve, it is safe to -/// __turbopack_import__ the actual module that was dynamically imported. -#[turbo_tasks::value] -struct ManifestChunkItem { - context: ChunkingContextVc, - manifest: ManifestChunkAssetVc, -} - -#[turbo_tasks::value_impl] -impl ValueToString for ManifestChunkItem { - #[turbo_tasks::function] - async fn to_string(&self) -> Result<StringVc> { - Ok(self.manifest.path().to_string()) - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkItem for ManifestChunkItem { - #[turbo_tasks::function] - fn chunking_context(&self) -> ChunkingContextVc { - self.context - } - - #[turbo_tasks::function] - fn related_path(&self) -> FileSystemPathVc { - self.manifest.path() - } - - #[turbo_tasks::function] - async fn content(&self) -> Result<EcmascriptChunkItemContentVc> { - let chunks = self.manifest.chunks().await?; - - let mut chunk_server_paths = IndexSet::new(); - for chunk in chunks.iter() { - // The "path" in this case is the chunk's path, not the chunk item's path. - // The difference is a chunk is a file served by the dev server, and an - // item is one of several that are contained in that chunk file. - let chunk_path = &*chunk.path().await?; - // The pathname is the file path necessary to load the chunk from the server. - let output_root = self.context.output_root().await?; - let chunk_server_path = if let Some(path) = output_root.get_path_to(chunk_path) { - path - } else { - bail!( - "chunk path {} is not in output root {}", - chunk.path().to_string().await?, - self.context.output_root().to_string().await? - ); - }; - chunk_server_paths.insert(chunk_server_path.to_string()); - } - - let mut code = b"const chunks = [\n".to_vec(); - for pathname in chunk_server_paths { - writeln!(code, " {},", stringify_js(&pathname))?; - } - writeln!(code, "];")?; - - // TODO: a dedent macro would be awesome. - write!( - code, - " -__turbopack_export_value__(Promise.all(chunks.map(__turbopack_load__)));" - )?; - - Ok(EcmascriptChunkItemContent { - inner_code: code.into(), - ..Default::default() - } - .into()) - } -} - -#[turbo_tasks::value_impl] -impl ChunkItem for ManifestChunkItem { - #[turbo_tasks::function] - async fn references(&self) -> Result<AssetReferencesVc> { - let chunks = self.manifest.chunks(); - - Ok(AssetReferencesVc::cell( - chunks - .await? - .iter() - .copied() - .map(ChunkReferenceVc::new) - .map(Into::into) - .collect(), - )) - } -} - -#[turbo_tasks::value] -struct ManifestChunkAssetReference { - manifest: ManifestChunkAssetVc, -} - -#[turbo_tasks::value_impl] -impl ValueToString for ManifestChunkAssetReference { - #[turbo_tasks::function] - async fn to_string(&self) -> Result<StringVc> { - Ok(StringVc::cell(format!( - "referenced manifest {}", - self.manifest.path().to_string().await? - ))) - } -} - -#[turbo_tasks::value_impl] -impl AssetReference for ManifestChunkAssetReference { - #[turbo_tasks::function] - fn resolve_reference(&self) -> ResolveResultVc { - ResolveResult::asset(self.manifest.into()).cell() - } -} - -#[turbo_tasks::value_impl] -impl ChunkableAssetReference for ManifestChunkAssetReference { - #[turbo_tasks::function] - fn chunking_type(&self, _context: ChunkingContextVc) -> ChunkingTypeOptionVc { - ChunkingTypeOptionVc::cell(Some(ChunkingType::Separate)) - } -} diff --git a/crates/turbopack-ecmascript/src/chunk/manifest/chunk_asset.rs b/crates/turbopack-ecmascript/src/chunk/manifest/chunk_asset.rs new file mode 100644 index 0000000000000..25d1caad6004f --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/manifest/chunk_asset.rs @@ -0,0 +1,134 @@ +use anyhow::Result; +use turbo_tasks::{primitives::StringVc, ValueToString, ValueToStringVc}; +use turbo_tasks_fs::FileSystemPathVc; +use turbopack_core::{ + asset::{Asset, AssetContentVc, AssetVc}, + chunk::{ + ChunkGroupVc, ChunkVc, ChunkableAsset, ChunkableAssetReference, ChunkableAssetReferenceVc, + ChunkableAssetVc, ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc, + }, + reference::{AssetReference, AssetReferenceVc, AssetReferencesVc}, + resolve::{ResolveResult, ResolveResultVc}, +}; + +use super::chunk_item::ManifestChunkItem; +use crate::chunk::{ + item::EcmascriptChunkItemVc, + placeable::{ + EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc, EcmascriptExports, + EcmascriptExportsVc, + }, + EcmascriptChunkVc, +}; + +/// The manifest chunk is deferred until requested by the manifest loader +/// item when the dynamic `import()` expression is reached. Its responsibility +/// is to generate a Promise that will resolve only after all the necessary +/// chunks needed by the dynamic import are loaded by the client. +/// +/// Splitting the dynamic import into a quickly generate-able manifest loader +/// item and a slow-to-generate manifest chunk allows for faster incremental +/// compilation. The traversal won't be performed until the dynamic import is +/// actually reached, instead of eagerly as part of the chunk that the dynamic +/// import appears in. +#[turbo_tasks::value(shared)] +pub struct ManifestChunkAsset { + pub asset: ChunkableAssetVc, + pub chunking_context: ChunkingContextVc, +} + +#[turbo_tasks::value_impl] +impl ManifestChunkAssetVc { + #[turbo_tasks::function] + pub fn new(asset: ChunkableAssetVc, chunking_context: ChunkingContextVc) -> Self { + Self::cell(ManifestChunkAsset { + asset, + chunking_context, + }) + } + + #[turbo_tasks::function] + pub(super) async fn chunk_group(self) -> Result<ChunkGroupVc> { + let this = self.await?; + Ok(ChunkGroupVc::from_asset(this.asset, this.chunking_context)) + } +} + +#[turbo_tasks::value_impl] +impl Asset for ManifestChunkAsset { + #[turbo_tasks::function] + fn path(&self) -> FileSystemPathVc { + self.asset.path().join("manifest-chunk.js") + } + + #[turbo_tasks::function] + fn content(&self) -> AssetContentVc { + todo!() + } + + #[turbo_tasks::function] + fn references(&self) -> AssetReferencesVc { + todo!() + } +} + +#[turbo_tasks::value_impl] +impl ChunkableAsset for ManifestChunkAsset { + #[turbo_tasks::function] + fn as_chunk(self_vc: ManifestChunkAssetVc, context: ChunkingContextVc) -> ChunkVc { + EcmascriptChunkVc::new(context, self_vc.into()).into() + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkPlaceable for ManifestChunkAsset { + #[turbo_tasks::function] + fn as_chunk_item( + self_vc: ManifestChunkAssetVc, + context: ChunkingContextVc, + ) -> EcmascriptChunkItemVc { + ManifestChunkItem { + context, + manifest: self_vc, + } + .cell() + .into() + } + + #[turbo_tasks::function] + fn get_exports(&self) -> EcmascriptExportsVc { + EcmascriptExports::Value.cell() + } +} + +#[turbo_tasks::value(shared)] +pub(super) struct ManifestChunkAssetReference { + pub manifest: ManifestChunkAssetVc, +} + +#[turbo_tasks::value_impl] +impl ValueToString for ManifestChunkAssetReference { + #[turbo_tasks::function] + async fn to_string(&self) -> Result<StringVc> { + Ok(StringVc::cell(format!( + "referenced manifest {}", + self.manifest.path().to_string().await? + ))) + } +} + +#[turbo_tasks::value_impl] +impl AssetReference for ManifestChunkAssetReference { + #[turbo_tasks::function] + fn resolve_reference(&self) -> ResolveResultVc { + ResolveResult::asset(self.manifest.into()).cell() + } +} + +#[turbo_tasks::value_impl] +impl ChunkableAssetReference for ManifestChunkAssetReference { + #[turbo_tasks::function] + fn chunking_type(&self, _context: ChunkingContextVc) -> ChunkingTypeOptionVc { + ChunkingTypeOptionVc::cell(Some(ChunkingType::Separate)) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/manifest/chunk_item.rs b/crates/turbopack-ecmascript/src/chunk/manifest/chunk_item.rs new file mode 100644 index 0000000000000..43d99220b3174 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/manifest/chunk_item.rs @@ -0,0 +1,105 @@ +use anyhow::{bail, Result}; +use indexmap::IndexSet; +use indoc::formatdoc; +use turbo_tasks::{primitives::StringVc, ValueToString, ValueToStringVc}; +use turbo_tasks_fs::FileSystemPathVc; +use turbopack_core::{ + asset::Asset, + chunk::{ChunkItem, ChunkItemVc, ChunkReferenceVc, ChunkingContext, ChunkingContextVc}, + reference::AssetReferencesVc, +}; + +use super::chunk_asset::ManifestChunkAssetVc; +use crate::{ + chunk::item::{ + EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc, + EcmascriptChunkItemVc, + }, + utils::stringify_js_pretty, +}; + +/// The ManifestChunkItem generates a __turbopack_load__ call for every chunk +/// necessary to load the real asset. Once all the loads resolve, it is safe to +/// __turbopack_import__ the actual module that was dynamically imported. +#[turbo_tasks::value(shared)] +pub(super) struct ManifestChunkItem { + pub context: ChunkingContextVc, + pub manifest: ManifestChunkAssetVc, +} + +#[turbo_tasks::value_impl] +impl ValueToString for ManifestChunkItem { + #[turbo_tasks::function] + async fn to_string(&self) -> Result<StringVc> { + Ok(self.manifest.path().to_string()) + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkItem for ManifestChunkItem { + #[turbo_tasks::function] + fn chunking_context(&self) -> ChunkingContextVc { + self.context + } + + #[turbo_tasks::function] + fn related_path(&self) -> FileSystemPathVc { + self.manifest.path() + } + + #[turbo_tasks::function] + async fn content(&self) -> Result<EcmascriptChunkItemContentVc> { + let chunks = self.manifest.chunk_group().chunks().await?; + + let mut chunk_server_paths = IndexSet::new(); + for chunk in chunks.iter() { + // The "path" in this case is the chunk's path, not the chunk item's path. + // The difference is a chunk is a file served by the dev server, and an + // item is one of several that are contained in that chunk file. + let chunk_path = &*chunk.path().await?; + // The pathname is the file path necessary to load the chunk from the server. + let output_root = self.context.output_root().await?; + let chunk_server_path = if let Some(path) = output_root.get_path_to(chunk_path) { + path + } else { + bail!( + "chunk path {} is not in output root {}", + chunk.path().to_string().await?, + self.context.output_root().to_string().await? + ); + }; + chunk_server_paths.insert(chunk_server_path.to_string()); + } + + let code = formatdoc! { + r#" + __turbopack_export_value__({chunk_paths}); + "#, + chunk_paths = stringify_js_pretty(&chunk_server_paths) + }; + + Ok(EcmascriptChunkItemContent { + inner_code: code.into(), + ..Default::default() + } + .into()) + } +} + +#[turbo_tasks::value_impl] +impl ChunkItem for ManifestChunkItem { + #[turbo_tasks::function] + async fn references(&self) -> Result<AssetReferencesVc> { + let chunks = self.manifest.chunk_group().chunks(); + + Ok(AssetReferencesVc::cell( + chunks + .await? + .iter() + .copied() + .map(ChunkReferenceVc::new) + .map(Into::into) + .collect(), + )) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/manifest/loader_item.rs b/crates/turbopack-ecmascript/src/chunk/manifest/loader_item.rs new file mode 100644 index 0000000000000..ef27b2b995372 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/manifest/loader_item.rs @@ -0,0 +1,164 @@ +use std::io::Write as _; + +use anyhow::{anyhow, bail, Result}; +use indoc::writedoc; +use turbo_tasks::{primitives::StringVc, ValueToString, ValueToStringVc}; +use turbo_tasks_fs::FileSystemPathVc; +use turbopack_core::{ + asset::Asset, + chunk::{ChunkItem, ChunkItemVc, ChunkableAsset, ChunkingContext, ChunkingContextVc}, + reference::AssetReferencesVc, +}; + +use super::chunk_asset::{ManifestChunkAssetReference, ManifestChunkAssetVc}; +use crate::{ + chunk::{ + item::{ + EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc, + EcmascriptChunkItemVc, + }, + placeable::{EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc}, + }, + utils::stringify_js, +}; + +/// The manifest loader item is shipped in the same chunk that uses the dynamic +/// `import()` expression. Its responsibility is to load the manifest chunk from +/// the server. The dynamic import has been rewritten to import this manifest +/// loader item, which will load the manifest chunk from the server, which +/// will load all the chunks needed by the dynamic import. Finally, we'll be +/// able to import the module we're trying to dynamically import. +/// +/// Splitting the dynamic import into a quickly generate-able manifest loader +/// item and a slow-to-generate manifest chunk allows for faster incremental +/// compilation. The traversal won't be performed until the dynamic import is +/// actually reached, instead of eagerly as part of the chunk that the dynamic +/// import appears in. +#[turbo_tasks::value] +pub struct ManifestLoaderItem { + context: ChunkingContextVc, + manifest: ManifestChunkAssetVc, +} + +#[turbo_tasks::value_impl] +impl ManifestLoaderItemVc { + #[turbo_tasks::function] + pub fn new(context: ChunkingContextVc, manifest: ManifestChunkAssetVc) -> Self { + Self::cell(ManifestLoaderItem { context, manifest }) + } + + #[turbo_tasks::function] + async fn chunks_list_path(self) -> Result<FileSystemPathVc> { + let this = &*self.await?; + Ok(this.context.chunk_path( + this.manifest.path().parent().join("chunk-list.json"), + ".json", + )) + } +} + +#[turbo_tasks::value_impl] +impl ValueToString for ManifestLoaderItem { + #[turbo_tasks::function] + fn to_string(&self) -> StringVc { + self.manifest + .path() + .parent() + .join("manifest-loader.js") + .to_string() + } +} + +#[turbo_tasks::value_impl] +impl ChunkItem for ManifestLoaderItem { + #[turbo_tasks::function] + async fn references(self_vc: ManifestLoaderItemVc) -> Result<AssetReferencesVc> { + let this = &*self_vc.await?; + Ok(AssetReferencesVc::cell(vec![ManifestChunkAssetReference { + manifest: this.manifest, + } + .cell() + .into()])) + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkItem for ManifestLoaderItem { + #[turbo_tasks::function] + fn chunking_context(&self) -> ChunkingContextVc { + self.context + } + + #[turbo_tasks::function] + fn related_path(&self) -> FileSystemPathVc { + self.manifest.path() + } + + #[turbo_tasks::function] + async fn content(self_vc: ManifestLoaderItemVc) -> Result<EcmascriptChunkItemContentVc> { + let this = &*self_vc.await?; + let mut code = Vec::new(); + + let manifest = this.manifest.await?; + let asset = manifest.asset.as_asset(); + let chunk = this.manifest.as_chunk(this.context); + let chunk_path = &*chunk.path().await?; + + let output_root = this.context.output_root().await?; + + // We need several items in order for a dynamic import to fully load. First, we + // need the chunk path of the manifest chunk, relative from the output root. The + // chunk is a servable file, which will contain the manifest chunk item, which + // will perform the actual chunk traversal and generate load statements. + let chunk_server_path = if let Some(path) = output_root.get_path_to(chunk_path) { + path + } else { + bail!( + "chunk path {} is not in output root {}", + chunk.path().to_string().await?, + this.context.output_root().to_string().await? + ); + }; + + // We also need the manifest chunk item's id, which points to a CJS module that + // exports a promise for all of the necessary chunk loads. + let item_id = &*this.manifest.as_chunk_item(this.context).id().await?; + + // Finally, we need the id of the module that we're actually trying to + // dynamically import. + let placeable = EcmascriptChunkPlaceableVc::resolve_from(asset) + .await? + .ok_or_else(|| anyhow!("asset is not placeable in ecmascript chunk"))?; + let dynamic_id = &*placeable.as_chunk_item(this.context).id().await?; + + // This is the code that will be executed when the dynamic import is reached. + // It will load the manifest chunk, which will load all the chunks needed by + // the dynamic import, and finally we'll be able to import the module we're + // trying to dynamically import. + // This is similar to what happens when the first evaluated chunk is executed + // on first page load, but it's happening on-demand instead of eagerly. + writedoc!( + code, + r#" + __turbopack_export_value__((__turbopack_import__) => {{ + return __turbopack_load__({chunk_server_path}).then(() => {{ + return __turbopack_require__({item_id}); + }}).then((chunks_paths) => {{ + return Promise.all(chunks_paths.map((chunk_path) => __turbopack_load__(chunk_path))); + }}).then(() => {{ + return __turbopack_import__({dynamic_id}); + }}); + }}); + "#, + chunk_server_path = stringify_js(chunk_server_path), + item_id = stringify_js(item_id), + dynamic_id = stringify_js(dynamic_id) + )?; + + Ok(EcmascriptChunkItemContent { + inner_code: code.into(), + ..Default::default() + } + .into()) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/manifest/mod.rs b/crates/turbopack-ecmascript/src/chunk/manifest/mod.rs new file mode 100644 index 0000000000000..ef741f48e9166 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/manifest/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod chunk_asset; +pub(crate) mod chunk_item; +pub(crate) mod loader_item; diff --git a/crates/turbopack-ecmascript/src/chunk/mod.rs b/crates/turbopack-ecmascript/src/chunk/mod.rs index 68747c4ade076..849f6b71e96e4 100644 --- a/crates/turbopack-ecmascript/src/chunk/mod.rs +++ b/crates/turbopack-ecmascript/src/chunk/mod.rs @@ -1,57 +1,59 @@ -pub mod loader; +pub(crate) mod content; +pub(crate) mod context; +pub(crate) mod evaluate; +pub(crate) mod item; +pub(crate) mod manifest; +pub(crate) mod module_factory; pub(crate) mod optimize; -pub mod source_map; +pub(crate) mod placeable; +pub(crate) mod snapshot; +pub(crate) mod source_map; +pub(crate) mod update; +pub(crate) mod version; -use std::{fmt::Write, io::Write as _, slice::Iter}; +use std::fmt::Write; -use anyhow::{anyhow, bail, Result}; -use indexmap::{IndexMap, IndexSet}; -use indoc::indoc; -use serde::{Deserialize, Serialize}; +use anyhow::{anyhow, Result}; +use indexmap::IndexSet; use turbo_tasks::{ - primitives::{JsonValueVc, StringReadRef, StringVc, StringsVc, UsizeVc}, - trace::TraceRawVcs, + primitives::{StringReadRef, StringVc, UsizeVc}, TryJoinIterExt, ValueToString, ValueToStringVc, }; -use turbo_tasks_fs::{ - embed_file, rope::Rope, File, FileContent, FileSystemPathOptionVc, FileSystemPathVc, -}; -use turbo_tasks_hash::{encode_hex, hash_xxh3_hash64, DeterministicHasher, Xxh3Hash64Hasher}; +use turbo_tasks_fs::{FileSystemPathOptionVc, FileSystemPathVc}; +use turbo_tasks_hash::{encode_hex, DeterministicHasher, Xxh3Hash64Hasher}; use turbopack_core::{ asset::{Asset, AssetContentVc, AssetVc}, chunk::{ - chunk_content, chunk_content_split, - chunk_in_group::ChunkInGroupVc, optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc}, - Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkGroupVc, ChunkItem, ChunkItemVc, - ChunkReferenceVc, ChunkVc, ChunkableAsset, ChunkableAssetVc, ChunkingContext, - ChunkingContextVc, FromChunkableAsset, ModuleId, ModuleIdReadRef, ModuleIdVc, ModuleIdsVc, + Chunk, ChunkGroupReferenceVc, ChunkReferenceVc, ChunkVc, ChunkingContext, + ChunkingContextVc, }, - code_builder::{Code, CodeBuilder, CodeReadRef, CodeVc}, - environment::{ChunkLoading, EnvironmentVc}, introspect::{ asset::{children_from_asset_references, content_to_details, IntrospectableAssetVc}, Introspectable, IntrospectableChildrenVc, IntrospectableVc, }, - issue::{code_gen::CodeGenerationIssue, IssueSeverity}, - reference::{AssetReferenceVc, AssetReferencesVc}, - source_map::{GenerateSourceMap, GenerateSourceMapVc, OptionSourceMapVc, SourceMapVc}, - version::{ - PartialUpdate, TotalUpdate, Update, UpdateVc, Version, VersionVc, VersionedContent, - VersionedContentVc, - }, + reference::AssetReferencesVc, + source_map::{GenerateSourceMap, GenerateSourceMapVc, SourceMapVc}, + version::{VersionedContent, VersionedContentVc}, }; use self::{ - loader::{ManifestChunkAssetVc, ManifestLoaderItemVc}, + content::{ecmascript_chunk_content, EcmascriptChunkContentResultVc, EcmascriptChunkContentVc}, optimize::EcmascriptChunkOptimizerVc, source_map::EcmascriptChunkSourceMapAssetReferenceVc, }; -use crate::{ - parse::ParseResultSourceMapVc, - references::esm::EsmExportsVc, - utils::{stringify_js, FormatIter}, +pub use self::{ + evaluate::{EcmascriptChunkEvaluate, EcmascriptChunkEvaluateVc}, + item::{ + EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkItemContentVc, + EcmascriptChunkItemOptions, EcmascriptChunkItemVc, + }, + placeable::{ + EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc, EcmascriptChunkPlaceables, + EcmascriptChunkPlaceablesVc, EcmascriptExports, EcmascriptExportsVc, + }, }; +use crate::utils::FormatIter; #[turbo_tasks::value] pub struct EcmascriptChunk { @@ -184,696 +186,6 @@ pub struct EcmascriptChunkComparison { right_chunk_items: usize, } -/// Whether the ES chunk should include and evaluate a runtime. -#[turbo_tasks::value] -pub struct EcmascriptChunkEvaluate { - /// Entries that will be executed in that order only all chunks are ready. - /// These entries must be included in `main_entries` so that they are - /// available. - evaluate_entries: EcmascriptChunkPlaceablesVc, - /// All chunks of this chunk group need to be ready for execution to start. - /// When None, it will use a chunk group created from the current chunk. - chunk_group: Option<ChunkGroupVc>, -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkEvaluateVc { - #[turbo_tasks::function] - async fn content( - self, - context: ChunkingContextVc, - origin_chunk: EcmascriptChunkVc, - ) -> Result<EcmascriptChunkContentEvaluateVc> { - let &EcmascriptChunkEvaluate { - evaluate_entries, - chunk_group, - } = &*self.await?; - let chunk_group = - chunk_group.unwrap_or_else(|| ChunkGroupVc::from_chunk(origin_chunk.into())); - let evaluate_chunks = chunk_group.chunks().await?; - let mut chunks_server_paths = Vec::new(); - let output_root = context.output_root().await?; - for chunk in evaluate_chunks.iter() { - if let Some(chunk_in_group) = ChunkInGroupVc::resolve_from(chunk).await? { - if let Some(ecma_chunk) = - EcmascriptChunkVc::resolve_from(chunk_in_group.inner()).await? - { - if ecma_chunk != origin_chunk { - let chunk_path = &*chunk.path().await?; - if let Some(chunk_server_path) = output_root.get_path_to(chunk_path) { - chunks_server_paths.push(chunk_server_path.to_string()); - } - } - } - } - } - let entry_modules_ids = evaluate_entries - .await? - .iter() - .map(|entry| entry.as_chunk_item(context).id()) - .collect(); - Ok(EcmascriptChunkContentEvaluate { - chunks_server_paths: StringsVc::cell(chunks_server_paths), - entry_modules_ids: ModuleIdsVc::cell(entry_modules_ids), - } - .cell()) - } -} - -#[turbo_tasks::value] -pub struct EcmascriptChunkContentResult { - pub chunk_items: EcmascriptChunkItemsVc, - pub chunks: Vec<ChunkVc>, - pub async_chunk_groups: Vec<ChunkGroupVc>, - pub external_asset_references: Vec<AssetReferenceVc>, -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContentResultVc { - #[turbo_tasks::function] - fn filter(self, _other: EcmascriptChunkContentResultVc) -> EcmascriptChunkContentResultVc { - todo!() - } -} - -impl From<ChunkContentResult<EcmascriptChunkItemVc>> for EcmascriptChunkContentResult { - fn from(from: ChunkContentResult<EcmascriptChunkItemVc>) -> Self { - EcmascriptChunkContentResult { - chunk_items: EcmascriptChunkItems(EcmascriptChunkItems::make_chunks(&from.chunk_items)) - .cell(), - chunks: from.chunks, - async_chunk_groups: from.async_chunk_groups, - external_asset_references: from.external_asset_references, - } - } -} - -#[turbo_tasks::function] -fn ecmascript_chunk_content( - context: ChunkingContextVc, - main_entries: EcmascriptChunkPlaceablesVc, - omit_entries: Option<EcmascriptChunkPlaceablesVc>, -) -> EcmascriptChunkContentResultVc { - let mut chunk_content = ecmascript_chunk_content_internal(context, main_entries); - if let Some(omit_entries) = omit_entries { - let omit_chunk_content = ecmascript_chunk_content_internal(context, omit_entries); - chunk_content = chunk_content.filter(omit_chunk_content); - } - chunk_content -} - -#[turbo_tasks::function] -async fn ecmascript_chunk_content_internal( - context: ChunkingContextVc, - entries: EcmascriptChunkPlaceablesVc, -) -> Result<EcmascriptChunkContentResultVc> { - let entries = entries.await?; - let entries = entries.iter().copied(); - - let contents = entries - .map(|entry| ecmascript_chunk_content_single_entry(context, entry)) - .collect::<Vec<_>>(); - - if contents.len() == 1 { - return Ok(contents.into_iter().next().unwrap()); - } - - let mut all_chunk_items = IndexSet::<EcmascriptChunkItemVc>::new(); - let mut all_chunks = IndexSet::<ChunkVc>::new(); - let mut all_async_chunk_groups = IndexSet::<ChunkGroupVc>::new(); - let mut all_external_asset_references = IndexSet::<AssetReferenceVc>::new(); - - for content in contents { - let EcmascriptChunkContentResult { - chunk_items, - chunks, - async_chunk_groups, - external_asset_references, - } = &*content.await?; - for chunk in chunk_items.await?.iter() { - all_chunk_items.extend(chunk.await?.iter().copied()); - } - all_chunks.extend(chunks.iter().copied()); - all_async_chunk_groups.extend(async_chunk_groups.iter().copied()); - all_external_asset_references.extend(external_asset_references.iter().copied()); - } - - let chunk_items = - EcmascriptChunkItems::make_chunks(&all_chunk_items.into_iter().collect::<Vec<_>>()); - Ok(EcmascriptChunkContentResult { - chunk_items: EcmascriptChunkItemsVc::cell(chunk_items), - chunks: all_chunks.into_iter().collect(), - async_chunk_groups: all_async_chunk_groups.into_iter().collect(), - external_asset_references: all_external_asset_references.into_iter().collect(), - } - .cell()) -} - -#[turbo_tasks::function] -async fn ecmascript_chunk_content_single_entry( - context: ChunkingContextVc, - entry: EcmascriptChunkPlaceableVc, -) -> Result<EcmascriptChunkContentResultVc> { - let asset = entry.as_asset(); - - Ok(EcmascriptChunkContentResultVc::cell( - if let Some(res) = chunk_content::<EcmascriptChunkItemVc>(context, asset, None).await? { - res - } else { - chunk_content_split::<EcmascriptChunkItemVc>(context, asset, None).await? - } - .into(), - )) -} - -#[turbo_tasks::value(serialization = "none")] -pub struct EcmascriptChunkContent { - module_factories: EcmascriptChunkContentEntriesSnapshotReadRef, - chunk_path: FileSystemPathVc, - output_root: FileSystemPathVc, - evaluate: Option<EcmascriptChunkContentEvaluateVc>, - environment: EnvironmentVc, -} - -#[turbo_tasks::value(transparent)] -struct EcmascriptChunkContentEntries(Vec<EcmascriptChunkContentEntryVc>); - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContentEntriesVc { - #[turbo_tasks::function] - async fn snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { - Ok(EcmascriptChunkContentEntriesSnapshot::List( - self.await?.iter().copied().try_join().await?, - ) - .cell()) - } -} - -/// This is a snapshot of a list of EcmascriptChunkContentEntry represented as -/// tree of ReadRefs. -/// -/// A tree is used instead of a plain Vec to allow to reused cached parts of the -/// list when it only a few elements have changed -#[turbo_tasks::value(serialization = "none")] -enum EcmascriptChunkContentEntriesSnapshot { - List(Vec<EcmascriptChunkContentEntryReadRef>), - Nested(Vec<EcmascriptChunkContentEntriesSnapshotReadRef>), -} - -impl EcmascriptChunkContentEntriesSnapshot { - fn iter(&self) -> EcmascriptChunkContentEntriesSnapshotIterator { - match self { - EcmascriptChunkContentEntriesSnapshot::List(l) => { - EcmascriptChunkContentEntriesSnapshotIterator::List(l.iter()) - } - EcmascriptChunkContentEntriesSnapshot::Nested(n) => { - let mut it = n.iter(); - if let Some(inner) = it.next() { - EcmascriptChunkContentEntriesSnapshotIterator::Nested( - Box::new(inner.iter()), - it, - ) - } else { - EcmascriptChunkContentEntriesSnapshotIterator::Empty - } - } - } - } -} - -impl<'a> IntoIterator for &'a EcmascriptChunkContentEntriesSnapshot { - type Item = &'a EcmascriptChunkContentEntryReadRef; - - type IntoIter = EcmascriptChunkContentEntriesSnapshotIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -enum EcmascriptChunkContentEntriesSnapshotIterator<'a> { - Empty, - List(Iter<'a, EcmascriptChunkContentEntryReadRef>), - Nested( - Box<EcmascriptChunkContentEntriesSnapshotIterator<'a>>, - Iter<'a, EcmascriptChunkContentEntriesSnapshotReadRef>, - ), -} - -impl<'a> Iterator for EcmascriptChunkContentEntriesSnapshotIterator<'a> { - type Item = &'a EcmascriptChunkContentEntryReadRef; - - fn next(&mut self) -> Option<Self::Item> { - match self { - EcmascriptChunkContentEntriesSnapshotIterator::Empty => None, - EcmascriptChunkContentEntriesSnapshotIterator::List(i) => i.next(), - EcmascriptChunkContentEntriesSnapshotIterator::Nested(inner, i) => loop { - if let Some(r) = inner.next() { - return Some(r); - } - if let Some(new) = i.next() { - **inner = new.iter(); - } else { - return None; - } - }, - } - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContentVc { - #[turbo_tasks::function] - async fn new( - context: ChunkingContextVc, - main_entries: EcmascriptChunkPlaceablesVc, - omit_entries: Option<EcmascriptChunkPlaceablesVc>, - chunk_path: FileSystemPathVc, - evaluate: Option<EcmascriptChunkContentEvaluateVc>, - ) -> Result<Self> { - // TODO(alexkirsz) All of this should be done in a transition, otherwise we run - // the risks of values not being strongly consistent with each other. - let chunk_content = ecmascript_chunk_content(context, main_entries, omit_entries); - let chunk_content = chunk_content.await?; - let module_factories = chunk_content.chunk_items.to_entry_snapshot().await?; - let output_root = context.output_root(); - Ok(EcmascriptChunkContent { - module_factories, - chunk_path, - output_root, - evaluate, - environment: context.environment(), - } - .cell()) - } -} - -#[turbo_tasks::value(serialization = "none")] -struct EcmascriptChunkContentEntry { - chunk_item: EcmascriptChunkItemVc, - id: ModuleIdReadRef, - code: CodeReadRef, - code_vc: CodeVc, - hash: u64, -} - -impl EcmascriptChunkContentEntry { - fn id(&self) -> &ModuleId { - &self.id - } - - fn code(&self) -> &Code { - &self.code - } - - fn source_code(&self) -> &Rope { - self.code.source_code() - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContentEntryVc { - #[turbo_tasks::function] - async fn new(chunk_item: EcmascriptChunkItemVc) -> Result<Self> { - let content = chunk_item.content(); - let factory = match module_factory(content).resolve().await { - Ok(factory) => factory, - Err(error) => { - let id = chunk_item.id().to_string().await; - let id = id.as_ref().map_or_else(|_| "unknown", |id| &**id); - let mut error_message = - format!("An error occurred while generating the chunk item {}", id); - for err in error.chain() { - write!(error_message, "\n at {}", err)?; - } - let js_error_message = serde_json::to_string(&error_message)?; - let issue = CodeGenerationIssue { - severity: IssueSeverity::Error.cell(), - path: chunk_item.related_path(), - title: StringVc::cell("Code generation for chunk item errored".to_string()), - message: StringVc::cell(error_message), - } - .cell(); - issue.as_issue().emit(); - let mut code = CodeBuilder::default(); - code += "(() => {{\n\n"; - writeln!(code, "throw new Error({error});", error = &js_error_message)?; - code += "\n}})"; - code.build().cell() - } - }; - let id = chunk_item.id().await?; - let code = factory.await?; - let hash = hash_xxh3_hash64(code.source_code()); - Ok(EcmascriptChunkContentEntry { - chunk_item, - id, - code, - code_vc: factory, - hash, - } - .cell()) - } -} - -#[turbo_tasks::function] -async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result<CodeVc> { - let content = content.await?; - let mut args = vec![ - "r: __turbopack_require__", - "x: __turbopack_external_require__", - "i: __turbopack_import__", - "s: __turbopack_esm__", - "v: __turbopack_export_value__", - "c: __turbopack_cache__", - "l: __turbopack_load__", - "j: __turbopack_cjs__", - "p: process", - "g: global", - // HACK - "__dirname", - ]; - if content.options.module { - args.push("m: module"); - } - if content.options.exports { - args.push("e: exports"); - } - let mut code = CodeBuilder::default(); - let args = FormatIter(|| args.iter().copied().intersperse(", ")); - if content.options.this { - write!(code, "(function({{ {} }}) {{ !function() {{\n\n", args,)?; - } else { - write!(code, "(({{ {} }}) => (() => {{\n\n", args,)?; - } - - let source_map = content.source_map.map(|sm| sm.as_generate_source_map()); - code.push_source(&content.inner_code, source_map); - if content.options.this { - code += "\n}.call(this) })"; - } else { - code += "\n})())"; - } - Ok(code.build().cell()) -} - -#[derive(Serialize)] -#[serde(tag = "type")] -struct EcmascriptChunkUpdate<'a> { - added: IndexMap<&'a ModuleId, HmrUpdateEntry<'a>>, - modified: IndexMap<&'a ModuleId, HmrUpdateEntry<'a>>, - deleted: IndexSet<&'a ModuleId>, -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContentVc { - #[turbo_tasks::function] - async fn version(self) -> Result<EcmascriptChunkVersionVc> { - let module_factories_hashes = self - .await? - .module_factories - .iter() - .map(|entry| (entry.id.clone(), entry.hash)) - .collect(); - Ok(EcmascriptChunkVersion { - module_factories_hashes, - } - .cell()) - } - - #[turbo_tasks::function] - async fn code(self) -> Result<CodeVc> { - let this = self.await?; - let chunk_path = &*this.chunk_path.await?; - let chunk_server_path = if let Some(path) = this.output_root.await?.get_path_to(chunk_path) - { - path - } else { - bail!( - "chunk path {} is not in output root {}", - this.chunk_path.to_string().await?, - this.output_root.to_string().await? - ); - }; - let mut code = CodeBuilder::default(); - code += "(self.TURBOPACK = self.TURBOPACK || []).push(["; - - writeln!(code, "{}, {{", stringify_js(chunk_server_path))?; - for entry in &this.module_factories { - write!(code, "\n{}: ", &stringify_js(entry.id()))?; - code.push_code(entry.code()); - code += ","; - } - code += "\n}"; - - if let Some(evaluate) = &this.evaluate { - let evaluate = evaluate.await?; - let condition = evaluate - .chunks_server_paths - .await? - .iter() - .map(|path| format!(" && loadedChunks.has({})", stringify_js(path))) - .collect::<Vec<_>>() - .join(""); - let entries_ids = &*evaluate.entry_modules_ids.await?; - let entries_instantiations = entries_ids - .iter() - .map(|id| async move { - let id = id.await?; - let id = stringify_js(&id); - Ok(format!(r#"instantiateRuntimeModule({id});"#)) as Result<_> - }) - .try_join() - .await? - .join("\n"); - // Add a runnable to the chunk that requests the entry module to ensure it gets - // executed when the chunk is evaluated. - // The condition stops the entry module from being executed while chunks it - // depend on have not yet been registered. - // The runnable will run every time a new chunk is `.push`ed to TURBOPACK, until - // all dependent chunks have been evaluated. - write!( - code, - ", ({{ loadedChunks, instantiateRuntimeModule }}) => {{ - if(!(true{condition})) return true; - {entries_instantiations} -}}" - )?; - } - code += "]);\n"; - if this.evaluate.is_some() { - // When a chunk is executed, it will either register itself with the current - // instance of the runtime, or it will push itself onto the list of pending - // chunks (`self.TURBOPACK`). - // - // When the runtime executes, it will pick up and register all pending chunks, - // and replace the list of pending chunks with itself so later chunks can - // register directly with it. - code += indoc! { r#" - (() => { - if (!Array.isArray(globalThis.TURBOPACK)) { - return; - } - "# }; - - let specific_runtime_code = match *this.environment.chunk_loading().await? { - ChunkLoading::None => embed_file!("js/src/runtime.none.js").await?, - ChunkLoading::NodeJs => embed_file!("js/src/runtime.nodejs.js").await?, - ChunkLoading::Dom => embed_file!("js/src/runtime.dom.js").await?, - }; - - match &*specific_runtime_code { - FileContent::NotFound => return Err(anyhow!("specific runtime code is not found")), - FileContent::Content(file) => code.push_source(file.content(), None), - }; - - let shared_runtime_code = embed_file!("js/src/runtime.js").await?; - - match &*shared_runtime_code { - FileContent::NotFound => return Err(anyhow!("shared runtime code is not found")), - FileContent::Content(file) => code.push_source(file.content(), None), - }; - - code += indoc! { r#" - })(); - "# }; - } - - if code.has_source_map() { - let filename = chunk_path.file_name(); - write!(code, "\n\n//# sourceMappingURL={}.map", filename)?; - } - - Ok(code.build().cell()) - } - - #[turbo_tasks::function] - async fn content(self) -> Result<AssetContentVc> { - let code = self.code().await?; - Ok(File::from(code.source_code().clone()).into()) - } -} - -#[turbo_tasks::value_impl] -impl VersionedContent for EcmascriptChunkContent { - #[turbo_tasks::function] - fn content(self_vc: EcmascriptChunkContentVc) -> AssetContentVc { - self_vc.content() - } - - #[turbo_tasks::function] - fn version(self_vc: EcmascriptChunkContentVc) -> VersionVc { - self_vc.version().into() - } - - #[turbo_tasks::function] - async fn update( - self_vc: EcmascriptChunkContentVc, - from_version: VersionVc, - ) -> Result<UpdateVc> { - let to_version = self_vc.version(); - let from_version = - if let Some(from) = EcmascriptChunkVersionVc::resolve_from(from_version).await? { - from - } else { - return Ok(Update::Total(TotalUpdate { - to: to_version.into(), - }) - .cell()); - }; - - let to = to_version.await?; - let from = from_version.await?; - - // When to and from point to the same value we can skip comparing them. - // This will happen since `cell_local` will not clone the value, but only make - // the local cell point to the same immutable value (Arc). - if from.ptr_eq(&to) { - return Ok(Update::None.cell()); - } - - let this = self_vc.await?; - let chunk_path = &this.chunk_path.await?.path; - - // TODO(alexkirsz) This should probably be stored as a HashMap already. - let mut module_factories: IndexMap<_, _> = this - .module_factories - .iter() - .map(|entry| (entry.id(), entry)) - .collect(); - let mut added = IndexMap::new(); - let mut modified = IndexMap::new(); - let mut deleted = IndexSet::new(); - - for (id, hash) in &from.module_factories_hashes { - let id = &**id; - if let Some(entry) = module_factories.remove(id) { - if entry.hash != *hash { - modified.insert(id, HmrUpdateEntry::new(entry, chunk_path)); - } - } else { - deleted.insert(id); - } - } - - // Remaining entries are added - for (id, entry) in module_factories { - added.insert(id, HmrUpdateEntry::new(entry, chunk_path)); - } - - let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() { - Update::None - } else { - let chunk_update = EcmascriptChunkUpdate { - added, - modified, - deleted, - }; - - Update::Partial(PartialUpdate { - to: to_version.into(), - instruction: JsonValueVc::cell(serde_json::to_value(&chunk_update)?), - }) - }; - - Ok(update.into()) - } -} - -#[turbo_tasks::value_impl] -impl GenerateSourceMap for EcmascriptChunkContent { - #[turbo_tasks::function] - fn generate_source_map(self_vc: EcmascriptChunkContentVc) -> SourceMapVc { - self_vc.code().generate_source_map() - } - - #[turbo_tasks::function] - async fn by_section(&self, section: &str) -> Result<OptionSourceMapVc> { - // Weirdly, the ContentSource will have already URL decoded the ModuleId, and we - // can't reparse that via serde. - if let Ok(id) = ModuleId::parse(section) { - for entry in self.module_factories.iter() { - if id == *entry.id() { - let sm = entry.code_vc.generate_source_map(); - return Ok(OptionSourceMapVc::cell(Some(sm))); - } - } - } - - Ok(OptionSourceMapVc::cell(None)) - } -} - -#[derive(serde::Serialize)] -struct HmrUpdateEntry<'a> { - code: &'a Rope, - url: String, - map: Option<String>, -} - -impl<'a> HmrUpdateEntry<'a> { - fn new(entry: &'a EcmascriptChunkContentEntry, chunk_path: &str) -> Self { - /// serde_qs can't serialize a lone enum when it's [serde::untagged]. - #[derive(Serialize)] - struct Id<'a> { - id: &'a ModuleId, - } - let id = serde_qs::to_string(&Id { id: &entry.id }).unwrap(); - HmrUpdateEntry { - code: entry.source_code(), - url: format!("/{}?{}", chunk_path, &id), - map: entry - .code - .has_source_map() - .then(|| format!("/__turbopack_sourcemap__/{}.map?{}", chunk_path, &id)), - } - } -} - -#[turbo_tasks::value(serialization = "none")] -struct EcmascriptChunkVersion { - module_factories_hashes: IndexMap<ModuleIdReadRef, u64>, -} - -#[turbo_tasks::value_impl] -impl Version for EcmascriptChunkVersion { - #[turbo_tasks::function] - async fn id(&self) -> Result<StringVc> { - let sorted_hashes = { - let mut versions: Vec<_> = self.module_factories_hashes.values().copied().collect(); - versions.sort(); - versions - }; - let mut hasher = Xxh3Hash64Hasher::new(); - for hash in sorted_hashes { - hasher.write_value(hash); - } - let hash = hasher.finish(); - let hex_hash = encode_hex(hash); - Ok(StringVc::cell(hex_hash)) - } -} - #[turbo_tasks::value_impl] impl Chunk for EcmascriptChunk {} @@ -1000,13 +312,19 @@ impl Asset for EcmascriptChunk { // evalute only contributes to the hashed info if let Some(evaluate) = this.evaluate { let evaluate = evaluate.content(this.context, self_vc).await?; - let chunks_server_paths = evaluate.chunks_server_paths.await?; - hasher.write_usize(chunks_server_paths.len()); - for path in chunks_server_paths.iter() { + let ecma_chunks_server_paths = &evaluate.ecma_chunks_server_paths; + hasher.write_usize(ecma_chunks_server_paths.len()); + for path in ecma_chunks_server_paths.iter() { hasher.write_ref(path); need_hash = true; } - let entry_modules_ids = evaluate.entry_modules_ids.await?; + let other_chunks_server_paths = &evaluate.other_chunks_server_paths; + hasher.write_usize(other_chunks_server_paths.len()); + for path in other_chunks_server_paths.iter() { + hasher.write_ref(path); + need_hash = true; + } + let entry_modules_ids = &evaluate.entry_modules_ids; hasher.write_usize(entry_modules_ids.len()); for id in entry_modules_ids.iter() { hasher.write_value(id.await?); @@ -1148,166 +466,3 @@ impl GenerateSourceMap for EcmascriptChunk { self_vc.chunk_content().generate_source_map() } } - -#[turbo_tasks::value] -struct EcmascriptChunkContentEvaluate { - chunks_server_paths: StringsVc, - entry_modules_ids: ModuleIdsVc, -} - -#[turbo_tasks::value] -pub struct EcmascriptChunkContext { - context: ChunkingContextVc, -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkContextVc { - #[turbo_tasks::function] - pub fn of(context: ChunkingContextVc) -> EcmascriptChunkContextVc { - EcmascriptChunkContextVc::cell(EcmascriptChunkContext { context }) - } - - #[turbo_tasks::function] - pub async fn chunk_item_id(self, chunk_item: EcmascriptChunkItemVc) -> Result<ModuleIdVc> { - let layer = &*self.await?.context.layer().await?; - let mut s = chunk_item.to_string().await?.clone_value(); - if !layer.is_empty() { - if s.ends_with(')') { - s.pop(); - write!(s, ", {layer})")?; - } else { - write!(s, " ({layer})")?; - } - } - Ok(ModuleId::String(s).cell()) - } -} - -#[turbo_tasks::value(shared)] -pub enum EcmascriptExports { - EsmExports(EsmExportsVc), - CommonJs, - Value, - None, -} - -#[turbo_tasks::value_trait] -pub trait EcmascriptChunkPlaceable: ChunkableAsset + Asset { - fn as_chunk_item(&self, context: ChunkingContextVc) -> EcmascriptChunkItemVc; - fn get_exports(&self) -> EcmascriptExportsVc; -} - -#[turbo_tasks::value(transparent)] -pub struct EcmascriptChunkPlaceables(Vec<EcmascriptChunkPlaceableVc>); - -#[turbo_tasks::value_impl] -impl EcmascriptChunkPlaceablesVc { - #[turbo_tasks::function] - pub fn empty() -> Self { - Self::cell(Vec::new()) - } -} - -#[turbo_tasks::value(shared)] -#[derive(Default)] -pub struct EcmascriptChunkItemContent { - pub inner_code: Rope, - pub source_map: Option<ParseResultSourceMapVc>, - pub options: EcmascriptChunkItemOptions, - pub placeholder_for_future_extensions: (), -} - -#[derive(PartialEq, Eq, Default, Debug, Clone, Serialize, Deserialize, TraceRawVcs)] -pub struct EcmascriptChunkItemOptions { - pub module: bool, - pub exports: bool, - pub this: bool, - pub placeholder_for_future_extensions: (), -} - -#[turbo_tasks::value_trait] -pub trait EcmascriptChunkItem: ChunkItem + ValueToString { - fn related_path(&self) -> FileSystemPathVc; - fn content(&self) -> EcmascriptChunkItemContentVc; - fn chunking_context(&self) -> ChunkingContextVc; - fn id(&self) -> ModuleIdVc { - EcmascriptChunkContextVc::of(self.chunking_context()).chunk_item_id(*self) - } -} - -#[async_trait::async_trait] -impl FromChunkableAsset for EcmascriptChunkItemVc { - async fn from_asset(context: ChunkingContextVc, asset: AssetVc) -> Result<Option<Self>> { - if let Some(placeable) = EcmascriptChunkPlaceableVc::resolve_from(asset).await? { - return Ok(Some(placeable.as_chunk_item(context))); - } - Ok(None) - } - - async fn from_async_asset( - context: ChunkingContextVc, - asset: ChunkableAssetVc, - ) -> Result<Option<Self>> { - let chunk = ManifestChunkAssetVc::new(asset, context); - Ok(Some(ManifestLoaderItemVc::new(context, chunk).into())) - } -} - -#[turbo_tasks::value(transparent)] -pub struct EcmascriptChunkItemsChunk(Vec<EcmascriptChunkItemVc>); - -#[turbo_tasks::value(transparent)] -pub struct EcmascriptChunkItems(Vec<EcmascriptChunkItemsChunkVc>); - -impl EcmascriptChunkItems { - pub fn make_chunks(list: &[EcmascriptChunkItemVc]) -> Vec<EcmascriptChunkItemsChunkVc> { - let size = list.len().div_ceil(100); - let chunk_items = list - .chunks(size) - .map(|chunk| EcmascriptChunkItemsChunkVc::cell(chunk.to_vec())) - .collect(); - chunk_items - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkItemsChunkVc { - #[turbo_tasks::function] - async fn to_entry_snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { - let list = self.await?; - Ok(EcmascriptChunkContentEntries( - list.iter() - .map(|chunk_item| EcmascriptChunkContentEntryVc::new(*chunk_item)) - .collect(), - ) - .cell() - .snapshot()) - } -} - -#[turbo_tasks::value_impl] -impl EcmascriptChunkItemsVc { - #[turbo_tasks::function] - async fn to_entry_snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { - let list = self.await?; - Ok(EcmascriptChunkContentEntriesSnapshot::Nested( - list.iter() - .map(|chunk| chunk.to_entry_snapshot()) - .try_join() - .await?, - ) - .cell()) - } - - #[turbo_tasks::function] - async fn to_set(self) -> Result<EcmascriptChunkItemsSetVc> { - let mut set = IndexSet::new(); - for chunk in self.await?.iter().copied().try_join().await? { - set.extend(chunk.iter().copied()) - } - Ok(EcmascriptChunkItemsSetVc::cell(set)) - } -} - -#[turbo_tasks::value(transparent)] -pub struct EcmascriptChunkItemsSet(IndexSet<EcmascriptChunkItemVc>); diff --git a/crates/turbopack-ecmascript/src/chunk/module_factory.rs b/crates/turbopack-ecmascript/src/chunk/module_factory.rs new file mode 100644 index 0000000000000..8fb82390491aa --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/module_factory.rs @@ -0,0 +1,48 @@ +use std::io::Write; + +use anyhow::Result; +use turbopack_core::code_builder::{CodeBuilder, CodeVc}; + +use super::item::EcmascriptChunkItemContentVc; +use crate::utils::FormatIter; + +#[turbo_tasks::function] +pub(super) async fn module_factory(content: EcmascriptChunkItemContentVc) -> Result<CodeVc> { + let content = content.await?; + let mut args = vec![ + "r: __turbopack_require__", + "x: __turbopack_external_require__", + "i: __turbopack_import__", + "s: __turbopack_esm__", + "v: __turbopack_export_value__", + "c: __turbopack_cache__", + "l: __turbopack_load__", + "j: __turbopack_cjs__", + "p: process", + "g: global", + // HACK + "__dirname", + ]; + if content.options.module { + args.push("m: module"); + } + if content.options.exports { + args.push("e: exports"); + } + let mut code = CodeBuilder::default(); + let args = FormatIter(|| args.iter().copied().intersperse(", ")); + if content.options.this { + write!(code, "(function({{ {} }}) {{ !function() {{\n\n", args,)?; + } else { + write!(code, "(({{ {} }}) => (() => {{\n\n", args,)?; + } + + let source_map = content.source_map.map(|sm| sm.as_generate_source_map()); + code.push_source(&content.inner_code, source_map); + if content.options.this { + code += "\n}.call(this) })"; + } else { + code += "\n})())"; + } + Ok(code.build().cell()) +} diff --git a/crates/turbopack-ecmascript/src/chunk/optimize.rs b/crates/turbopack-ecmascript/src/chunk/optimize.rs index 4bb5b67e2c6c2..a3e93bf9abc8a 100644 --- a/crates/turbopack-ecmascript/src/chunk/optimize.rs +++ b/crates/turbopack-ecmascript/src/chunk/optimize.rs @@ -11,8 +11,7 @@ use turbopack_core::chunk::{ ChunkGroupVc, ChunkVc, ChunkingContextVc, ChunksVc, }; -use super::{EcmascriptChunkPlaceablesVc, EcmascriptChunkVc}; -use crate::chunk::EcmascriptChunkEvaluate; +use super::{evaluate::EcmascriptChunkEvaluate, EcmascriptChunkPlaceablesVc, EcmascriptChunkVc}; #[turbo_tasks::value] pub struct EcmascriptChunkOptimizer(ChunkingContextVc); diff --git a/crates/turbopack-ecmascript/src/chunk/placeable.rs b/crates/turbopack-ecmascript/src/chunk/placeable.rs new file mode 100644 index 0000000000000..3233e25a2677a --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/placeable.rs @@ -0,0 +1,33 @@ +use anyhow::Result; +use turbopack_core::{ + asset::{Asset, AssetVc}, + chunk::{ChunkableAsset, ChunkableAssetVc, ChunkingContextVc}, +}; + +use super::item::EcmascriptChunkItemVc; +use crate::references::esm::EsmExportsVc; + +#[turbo_tasks::value_trait] +pub trait EcmascriptChunkPlaceable: ChunkableAsset + Asset { + fn as_chunk_item(&self, context: ChunkingContextVc) -> EcmascriptChunkItemVc; + fn get_exports(&self) -> EcmascriptExportsVc; +} + +#[turbo_tasks::value(transparent)] +pub struct EcmascriptChunkPlaceables(Vec<EcmascriptChunkPlaceableVc>); + +#[turbo_tasks::value_impl] +impl EcmascriptChunkPlaceablesVc { + #[turbo_tasks::function] + pub fn empty() -> Self { + Self::cell(Vec::new()) + } +} + +#[turbo_tasks::value(shared)] +pub enum EcmascriptExports { + EsmExports(EsmExportsVc), + CommonJs, + Value, + None, +} diff --git a/crates/turbopack-ecmascript/src/chunk/snapshot.rs b/crates/turbopack-ecmascript/src/chunk/snapshot.rs new file mode 100644 index 0000000000000..b9336fa3c01a5 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/snapshot.rs @@ -0,0 +1,162 @@ +use std::{fmt::Write, io::Write as _, slice::Iter}; + +use anyhow::Result; +use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString}; +use turbo_tasks_hash::hash_xxh3_hash64; +use turbopack_core::{ + chunk::{ModuleId, ModuleIdReadRef}, + code_builder::{Code, CodeBuilder, CodeReadRef, CodeVc}, + issue::{code_gen::CodeGenerationIssue, IssueSeverity}, +}; + +use super::{item::EcmascriptChunkItemVc, module_factory::module_factory, EcmascriptChunkItem}; + +#[turbo_tasks::value(transparent)] +pub(super) struct EcmascriptChunkContentEntries(pub(super) Vec<EcmascriptChunkContentEntryVc>); + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentEntriesVc { + #[turbo_tasks::function] + pub async fn snapshot(self) -> Result<EcmascriptChunkContentEntriesSnapshotVc> { + Ok(EcmascriptChunkContentEntriesSnapshot::List( + self.await?.iter().copied().try_join().await?, + ) + .cell()) + } +} + +/// This is a snapshot of a list of EcmascriptChunkContentEntry represented as +/// tree of ReadRefs. +/// +/// A tree is used instead of a plain Vec to allow to reused cached parts of the +/// list when it only a few elements have changed +#[turbo_tasks::value(serialization = "none", shared)] +pub(super) enum EcmascriptChunkContentEntriesSnapshot { + List(Vec<EcmascriptChunkContentEntryReadRef>), + Nested(Vec<EcmascriptChunkContentEntriesSnapshotReadRef>), +} + +impl EcmascriptChunkContentEntriesSnapshot { + pub(super) fn iter(&self) -> EcmascriptChunkContentEntriesSnapshotIterator { + match self { + EcmascriptChunkContentEntriesSnapshot::List(l) => { + EcmascriptChunkContentEntriesSnapshotIterator::List(l.iter()) + } + EcmascriptChunkContentEntriesSnapshot::Nested(n) => { + let mut it = n.iter(); + if let Some(inner) = it.next() { + EcmascriptChunkContentEntriesSnapshotIterator::Nested( + Box::new(inner.iter()), + it, + ) + } else { + EcmascriptChunkContentEntriesSnapshotIterator::Empty + } + } + } + } +} + +impl<'a> IntoIterator for &'a EcmascriptChunkContentEntriesSnapshot { + type Item = &'a EcmascriptChunkContentEntryReadRef; + + type IntoIter = EcmascriptChunkContentEntriesSnapshotIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +pub(super) enum EcmascriptChunkContentEntriesSnapshotIterator<'a> { + Empty, + List(Iter<'a, EcmascriptChunkContentEntryReadRef>), + Nested( + Box<EcmascriptChunkContentEntriesSnapshotIterator<'a>>, + Iter<'a, EcmascriptChunkContentEntriesSnapshotReadRef>, + ), +} + +impl<'a> Iterator for EcmascriptChunkContentEntriesSnapshotIterator<'a> { + type Item = &'a EcmascriptChunkContentEntryReadRef; + + fn next(&mut self) -> Option<Self::Item> { + match self { + EcmascriptChunkContentEntriesSnapshotIterator::Empty => None, + EcmascriptChunkContentEntriesSnapshotIterator::List(i) => i.next(), + EcmascriptChunkContentEntriesSnapshotIterator::Nested(inner, i) => loop { + if let Some(r) = inner.next() { + return Some(r); + } + if let Some(new) = i.next() { + **inner = new.iter(); + } else { + return None; + } + }, + } + } +} + +#[turbo_tasks::value(serialization = "none")] +pub(super) struct EcmascriptChunkContentEntry { + pub chunk_item: EcmascriptChunkItemVc, + pub id: ModuleIdReadRef, + pub code: CodeReadRef, + pub code_vc: CodeVc, + pub hash: u64, +} + +impl EcmascriptChunkContentEntry { + pub fn id(&self) -> &ModuleId { + &self.id + } + + pub fn code(&self) -> &Code { + &self.code + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkContentEntryVc { + #[turbo_tasks::function] + pub async fn new(chunk_item: EcmascriptChunkItemVc) -> Result<Self> { + let content = chunk_item.content(); + let factory = match module_factory(content).resolve().await { + Ok(factory) => factory, + Err(error) => { + let id = chunk_item.id().to_string().await; + let id = id.as_ref().map_or_else(|_| "unknown", |id| &**id); + let mut error_message = + format!("An error occurred while generating the chunk item {}", id); + for err in error.chain() { + write!(error_message, "\n at {}", err)?; + } + let js_error_message = serde_json::to_string(&error_message)?; + let issue = CodeGenerationIssue { + severity: IssueSeverity::Error.cell(), + path: chunk_item.related_path(), + title: StringVc::cell("Code generation for chunk item errored".to_string()), + message: StringVc::cell(error_message), + } + .cell(); + issue.as_issue().emit(); + let mut code = CodeBuilder::default(); + code += "(() => {{\n\n"; + writeln!(code, "throw new Error({error});", error = &js_error_message)?; + code += "\n}})"; + code.build().cell() + } + }; + let id = chunk_item.id().await?; + let code = factory.await?; + let hash = hash_xxh3_hash64(code.source_code()); + Ok(EcmascriptChunkContentEntry { + chunk_item, + id, + code, + code_vc: factory, + hash, + } + .cell()) + } +} diff --git a/crates/turbopack-ecmascript/src/chunk/update.rs b/crates/turbopack-ecmascript/src/chunk/update.rs new file mode 100644 index 0000000000000..4f471c927f554 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/update.rs @@ -0,0 +1,84 @@ +use anyhow::Result; +use indexmap::IndexMap; +use turbopack_core::{chunk::ModuleIdReadRef, code_builder::CodeReadRef}; + +use super::{content::EcmascriptChunkContentVc, version::EcmascriptChunkVersionVc}; + +#[turbo_tasks::value] +pub(super) struct EcmascriptChunkUpdate { + pub added: IndexMap<ModuleIdReadRef, (u64, CodeReadRef)>, + pub deleted: IndexMap<ModuleIdReadRef, u64>, + pub modified: IndexMap<ModuleIdReadRef, CodeReadRef>, +} + +pub(super) async fn update_ecmascript_chunk( + content: EcmascriptChunkContentVc, + from_version: EcmascriptChunkVersionVc, +) -> Result<UpdateVc> { + let to_version = self_vc.version(); + let from_version = + if let Some(from) = EcmascriptChunkVersionVc::resolve_from(from_version).await? { + from + } else { + return Ok(Update::Total(TotalUpdate { + to: to_version.into(), + }) + .cell()); + }; + + let to = to_version.await?; + let from = from_version.await?; + + // When to and from point to the same value we can skip comparing them. + // This will happen since `cell_local` will not clone the value, but only make + // the local cell point to the same immutable value (Arc). + if from.ptr_eq(&to) { + return Ok(Update::None.cell()); + } + + let this = self_vc.await?; + let chunk_path = &this.chunk_path.await?.path; + + // TODO(alexkirsz) This should probably be stored as a HashMap already. + let mut module_factories: IndexMap<_, _> = this + .module_factories + .iter() + .map(|entry| (entry.id(), entry)) + .collect(); + let mut added = IndexMap::new(); + let mut modified = IndexMap::new(); + let mut deleted = IndexSet::new(); + + for (id, hash) in &from.module_factories_hashes { + let id = &**id; + if let Some(entry) = module_factories.remove(id) { + if entry.hash != *hash { + modified.insert(id, HmrUpdateEntry::new(entry, chunk_path)); + } + } else { + deleted.insert(id); + } + } + + // Remaining entries are added + for (id, entry) in module_factories { + added.insert(id, HmrUpdateEntry::new(entry, chunk_path)); + } + + let update = if added.is_empty() && modified.is_empty() && deleted.is_empty() { + Update::None + } else { + let chunk_update = EcmascriptChunkUpdate { + added, + modified, + deleted, + }; + + Update::Partial(PartialUpdate { + to: to_version.into(), + instruction: JsonValueVc::cell(serde_json::to_value(&chunk_update)?), + }) + }; + + Ok(update.into()) +} diff --git a/crates/turbopack-ecmascript/src/chunk/version.rs b/crates/turbopack-ecmascript/src/chunk/version.rs new file mode 100644 index 0000000000000..7d09f4c07cc92 --- /dev/null +++ b/crates/turbopack-ecmascript/src/chunk/version.rs @@ -0,0 +1,37 @@ +use anyhow::Result; +use indexmap::IndexMap; +use turbo_tasks::primitives::StringVc; +use turbo_tasks_hash::{encode_hex, Xxh3Hash64Hasher}; +use turbopack_core::{ + chunk::ModuleIdReadRef, + version::{Version, VersionVc}, +}; + +#[turbo_tasks::value(shared, serialization = "none")] +pub(super) struct EcmascriptChunkVersion { + pub(super) chunk_server_path: String, + pub(super) module_factories_hashes: IndexMap<ModuleIdReadRef, u64>, +} + +#[turbo_tasks::value_impl] +impl Version for EcmascriptChunkVersion { + #[turbo_tasks::function] + fn id(&self) -> StringVc { + let mut hasher = Xxh3Hash64Hasher::new(); + let sorted_hashes = { + let mut hashes: Vec<_> = self + .module_factories_hashes + .values() + .map(|hash| *hash) + .collect(); + hashes.sort(); + hashes + }; + for hash in sorted_hashes { + hasher.write_value(hash); + } + let hash = hasher.finish(); + let hex_hash = encode_hex(hash); + StringVc::cell(hex_hash) + } +} diff --git a/crates/turbopack-ecmascript/src/utils.rs b/crates/turbopack-ecmascript/src/utils.rs index 7a8a33acb3444..66e4386df4a62 100644 --- a/crates/turbopack-ecmascript/src/utils.rs +++ b/crates/turbopack-ecmascript/src/utils.rs @@ -72,6 +72,15 @@ where serde_json::to_string(s).unwrap() } +/// Converts a serializable value into a pretty-printed valid JavaScript +/// expression. +pub fn stringify_js_pretty<T>(s: &T) -> String +where + T: Serialize + ?Sized, +{ + serde_json::to_string_pretty(s).unwrap() +} + pub struct FormatIter<T: Iterator, F: Fn() -> T>(pub F); macro_rules! format_iter {