Skip to content

Commit

Permalink
Merge EcmascriptChunkUpdates before sending them to the client (ver…
Browse files Browse the repository at this point in the history
…cel/turborepo#3975)

This diff:
* introduces the `VersionedContentMerger` trait, which allows for
merging the updates of versioned contents within the same chunk group;
* implements this for `EcmascriptChunkContent`/`EcmascriptChunkUpdate`,
turning them into
`EcmascriptMergedChunkContent`/`EcmascriptMergedChunkUpdate`;
* creates a new `ChunkList` asset which is capable of merging chunk
updates of chunks within the same chunk group, and create such an asset
for dynamic chunks (through manifest/loader_item.rs) and chunk group
files assets.

This fixes a bunch of edge cases related to HMR:
* HMR of dynamic imports now works;
* Chunks getting added/deleted/renamed now also works with HMR, since
we're listening to updates at the chunk group level;
* CSS chunks get reloaded in the right order, with respect for
precedence.

There are still known edge cases with HMR:
* CSS chunks added through HMR are not inserted at the right position to
respect precedence (WEB-652).
* Update aggregation is disabled because we don't want a critical issue
to stop all HMR (WEB-582) . This would be fixed by applying aggregated
updates when dismissing the error modal, but there are some edge cases
with this too (e.g. what happens when an HMR update also causes an error
on top of an existing error).
  • Loading branch information
alexkirsz authored Mar 9, 2023
1 parent aa61628 commit 6f2be77
Show file tree
Hide file tree
Showing 164 changed files with 60,271 additions and 2,614 deletions.
420 changes: 306 additions & 114 deletions crates/next-core/js/src/dev/hmr-client.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/app-renderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import { headersFromEntries } from "@vercel/turbopack-next/internal/headers";
import { parse, ParsedUrlQuery } from "node:querystring";

globalThis.__next_require__ = (data) => {
const [, , ssr_id] = JSON.parse(data);
const [, , , ssr_id] = JSON.parse(data);
return __turbopack_require__(ssr_id);
};
globalThis.__next_chunk_load__ = () => Promise.resolve();
Expand Down
3 changes: 2 additions & 1 deletion crates/next-core/js/src/entry/app/hydrate.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ window.next = {
};

globalThis.__next_require__ = (data) => {
const [client_id] = JSON.parse(data);
const [client_id, chunks, chunkListPath] = JSON.parse(data);
__turbopack_register_chunk_list__(chunkListPath, chunks);
return __turbopack_require__(client_id);
};
globalThis.__next_chunk_load__ = __turbopack_load__;
Expand Down
1 change: 1 addition & 0 deletions crates/next-core/js/src/entry/app/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ export = Anything;

export const __turbopack_module_id__: string | number;
export const chunks: string[];
export const chunkListPath: string;
6 changes: 4 additions & 2 deletions crates/next-core/js/src/entry/app/server-to-client-ssr.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/
import { __turbopack_module_id__ as id } from "CLIENT_MODULE";

// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";

export default createProxy(JSON.stringify([client_id, chunks, id]));
export default createProxy(
JSON.stringify([client_id, chunks, chunkListPath, id])
);
4 changes: 2 additions & 2 deletions crates/next-core/js/src/entry/app/server-to-client.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";

// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";

export default createProxy(JSON.stringify([client_id, chunks]));
export default createProxy(JSON.stringify([client_id, chunks, chunkListPath]));
2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/fallback.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ subscribeToUpdate(
},
},
(update) => {
if (update.type === "restart") {
if (update.type === "restart" || update.type === "notFound") {
location.reload();
}
}
Expand Down
2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/next-hydrate.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) {
path: "_next/static/development/_devPagesManifest.json",
},
(update) => {
if (["restart", "partial"].includes(update.type)) {
if (["restart", "notFound", "partial"].includes(update.type)) {
return;
}

Expand Down
4 changes: 4 additions & 0 deletions crates/next-core/js/types/globals.d.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
declare global {
function __turbopack_require__(name: any): any;
function __turbopack_load__(path: string): any;
function __turbopack_register_chunk_list__(
chunkListPath: string,
chunksPaths: string[]
): any;
function __webpack_require__(name: any): any;
var __webpack_public_path__: string | undefined;
var __DEV_MIDDLEWARE_MATCHERS: any[];
Expand Down
4 changes: 2 additions & 2 deletions crates/next-core/src/app_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -411,7 +411,7 @@ async fn create_app_source_for_directory(
segments: layouts,
} => {
let LayoutSegment { target, .. } = *segment.await?;
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);

sources.push(create_node_rendered_source(
Expand Down Expand Up @@ -443,7 +443,7 @@ async fn create_app_source_for_directory(
route,
..
} => {
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);

sources.push(create_node_api_source(
Expand Down
1 change: 1 addition & 0 deletions crates/next-core/src/next_client/transition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ impl Transition for NextClientTransition {
asset: asset.into(),
chunking_context: self.client_chunking_context,
base_path: self.server_root.join("_next"),
server_root: self.server_root,
runtime_entries: Some(runtime_entries),
};

Expand Down
93 changes: 67 additions & 26 deletions crates/next-core/src/next_client_chunks/with_chunks.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::Result;
use anyhow::{bail, Result};
use indoc::formatdoc;
use serde_json::Value;
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc};
use turbo_tasks_fs::FileSystemPathVc;
Expand All @@ -13,14 +14,15 @@ use turbopack::ecmascript::{
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
chunk::{
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContextVc,
ChunkingType, ChunkingTypeOptionVc,
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext,
ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc,
},
ident::AssetIdentVc,
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;

use super::in_chunking_context_asset::InChunkingContextAsset;

Expand Down Expand Up @@ -93,6 +95,25 @@ struct WithChunksChunkItem {
inner: WithChunksAssetVc,
}

#[turbo_tasks::value_impl]
impl WithChunksChunkItemVc {
#[turbo_tasks::function]
async fn chunk_list_path(self) -> Result<FileSystemPathVc> {
let this = self.await?;
Ok(this.inner_context.chunk_list_path(this.inner.ident()))
}

#[turbo_tasks::function]
async fn chunk_group(self) -> Result<ChunkGroupVc> {
let this = self.await?;
let inner = this.inner.await?;
Ok(ChunkGroupVc::from_asset(
inner.asset.into(),
this.inner_context,
))
}
}

#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for WithChunksChunkItem {
#[turbo_tasks::function]
Expand All @@ -101,29 +122,40 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
}

#[turbo_tasks::function]
async fn content(&self) -> Result<EcmascriptChunkItemContentVc> {
let inner = self.inner.await?;
let group = ChunkGroupVc::from_asset(inner.asset.into(), self.inner_context);
async fn content(self_vc: WithChunksChunkItemVc) -> Result<EcmascriptChunkItemContentVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
let group = self_vc.chunk_group();
let chunks = group.chunks().await?;
let server_root = inner.server_root.await?;
let mut client_chunks = Vec::new();

let chunk_list_path = self_vc.chunk_list_path().await?;
let chunk_list_path = if let Some(path) = server_root.get_path_to(&chunk_list_path) {
path
} else {
bail!("could not get path to chunk list");
};

for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? {
if let Some(path) = server_root.get_path_to(&chunk_path) {
client_chunks.push(Value::String(path.to_string()));
}
}
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.inner_context).id().await?);
let module_id = stringify_js(&*inner.asset.as_chunk_item(this.inner_context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: format!(
"__turbopack_esm__({{
default: () => {},
chunks: () => chunks
}});
const chunks = {};
",
inner_code: formatdoc! {
r#"
__turbopack_esm__({{
default: () => {},
chunks: () => {},
chunkListPath: () => {},
}});
"#,
module_id,
Value::Array(client_chunks)
)
stringify_js_pretty(&client_chunks),
stringify_js(&chunk_list_path),
}
.into(),
..Default::default()
}
Expand All @@ -139,18 +171,27 @@ impl ChunkItem for WithChunksChunkItem {
}

#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
Ok(AssetReferencesVc::cell(vec![WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.inner_context,
async fn references(self_vc: WithChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: this.inner_context,
}
.cell()
.into(),
}
.cell()
.into(),
}
.cell()
.into()]))
ChunkListReferenceVc::new(
inner.server_root,
self_vc.chunk_group(),
self_vc.chunk_list_path(),
)
.into(),
]))
}
}

Expand Down
14 changes: 9 additions & 5 deletions crates/next-core/src/next_client_component/with_client_chunks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ use turbopack_core::{
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;

use crate::next_client_chunks::in_chunking_context_asset::InChunkingContextAsset;

Expand Down Expand Up @@ -129,15 +130,17 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem {
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: formatdoc!(
// We store the chunks in a binding, otherwise a new array would be created every
// time the export binding is read.
r#"
__turbopack_esm__({{
default: () => __turbopack_import__({}),
chunks: () => chunks
chunks: () => chunks,
}});
const chunks = {};
"#,
module_id,
stringify_js(&client_chunks)
stringify_js_pretty(&client_chunks),
)
.into(),
..Default::default()
Expand All @@ -154,13 +157,14 @@ impl ChunkItem for WithClientChunksChunkItem {
}

#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
async fn references(self_vc: WithClientChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithClientChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.context,
chunking_context: this.context,
}
.cell()
.into(),
Expand Down
1 change: 1 addition & 0 deletions crates/next-core/src/next_edge/transition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ impl Transition for NextEdgeTransition {
asset: new_asset.into(),
chunking_context: self.edge_chunking_context,
base_path: self.output_path,
server_root: self.output_path,
runtime_entries: None,
};

Expand Down
27 changes: 19 additions & 8 deletions crates/next-core/src/page_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::io::Write;

use anyhow::{bail, Result};
use indexmap::indexmap;
use turbo_tasks::{primitives::StringVc, Value};
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, Value};
use turbo_tasks_fs::{rope::RopeBuilder, File, FileContent, FileSystemPathVc};
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
Expand Down Expand Up @@ -116,17 +116,28 @@ impl Asset for PageLoaderAsset {
let this = &*self_vc.await?;

let chunks = self_vc.get_page_chunks().await?;

let mut data = Vec::with_capacity(chunks.len());
for chunk in chunks.iter() {
let path = chunk.path().await?;
data.push(serde_json::Value::String(path.path.clone()));
}
let server_root = this.server_root.await?;

let chunk_paths: Vec<_> = chunks
.iter()
.map(|chunk| {
let server_root = server_root.clone();
async move {
Ok(server_root
.get_path_to(&*chunk.path().await?)
.map(|path| path.to_string()))
}
})
.try_join()
.await?
.into_iter()
.flatten()
.collect();

let content = format!(
"__turbopack_load_page_chunks__({}, {})\n",
stringify_js(&this.pathname.await?),
serde_json::Value::Array(data)
stringify_js(&chunk_paths)
);

Ok(AssetContentVc::from(File::from(content)))
Expand Down
5 changes: 3 additions & 2 deletions crates/next-core/src/page_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ async fn create_page_source_for_file(
Value::new(ClientContextType::Pages { pages_dir }),
);

let pathname = pathname_for_path(server_root, server_path, true);
let pathname = pathname_for_path(server_root, server_path, true, false);
let route_matcher = NextParamsMatcherVc::new(pathname);

Ok(if is_api_path {
Expand All @@ -370,8 +370,9 @@ async fn create_page_source_for_file(
runtime_entries,
)
} else {
let data_pathname = pathname_for_path(server_root, server_path, true, true);
let data_route_matcher =
NextPrefixSuffixParamsMatcherVc::new(pathname, "_next/data/development/", ".json");
NextPrefixSuffixParamsMatcherVc::new(data_pathname, "_next/data/development/", ".json");

let ssr_entry = SsrEntry {
context: server_context,
Expand Down
11 changes: 8 additions & 3 deletions crates/next-core/src/util.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ pub async fn pathname_for_path(
server_root: FileSystemPathVc,
server_path: FileSystemPathVc,
has_extension: bool,
data: bool,
) -> Result<StringVc> {
let server_path_value = &*server_path.await?;
let path = if let Some(path) = server_root.await?.get_path_to(server_path_value) {
Expand All @@ -46,10 +47,14 @@ pub async fn pathname_for_path(
} else {
path
};
let path = if path == "index" {
""
let path = if data {
path
} else {
path.strip_suffix("/index").unwrap_or(path)
if path == "index" {
""
} else {
path.strip_suffix("/index").unwrap_or(path)
}
};

Ok(StringVc::cell(path.to_string()))
Expand Down
Loading

0 comments on commit 6f2be77

Please sign in to comment.