Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge EcmascriptChunkUpdates before sending them to the client #3975

Merged
merged 1 commit into from
Mar 9, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
420 changes: 306 additions & 114 deletions crates/next-core/js/src/dev/hmr-client.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/app-renderer.tsx
Original file line number Diff line number Diff line change
@@ -37,7 +37,7 @@ import { headersFromEntries } from "@vercel/turbopack-next/internal/headers";
import { parse, ParsedUrlQuery } from "node:querystring";

globalThis.__next_require__ = (data) => {
const [, , ssr_id] = JSON.parse(data);
const [, , , ssr_id] = JSON.parse(data);
return __turbopack_require__(ssr_id);
};
globalThis.__next_chunk_load__ = () => Promise.resolve();
3 changes: 2 additions & 1 deletion crates/next-core/js/src/entry/app/hydrate.tsx
Original file line number Diff line number Diff line change
@@ -19,7 +19,8 @@ window.next = {
};

globalThis.__next_require__ = (data) => {
const [client_id] = JSON.parse(data);
const [client_id, chunks, chunkListPath] = JSON.parse(data);
__turbopack_register_chunk_list__(chunkListPath, chunks);
return __turbopack_require__(client_id);
};
globalThis.__next_chunk_load__ = __turbopack_load__;
1 change: 1 addition & 0 deletions crates/next-core/js/src/entry/app/index.d.ts
Original file line number Diff line number Diff line change
@@ -5,3 +5,4 @@ export = Anything;

export const __turbopack_module_id__: string | number;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need this file anymore since we don't have "." imports anymore?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed. Instead, we could declare our "PAGE"/"INNER_ASSETS" modules (probably would need to rename them to "@vercel/inner/page" or "@turbopack/inner/page").

export const chunks: string[];
export const chunkListPath: string;
6 changes: 4 additions & 2 deletions crates/next-core/js/src/entry/app/server-to-client-ssr.tsx
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@ import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/
import { __turbopack_module_id__ as id } from "CLIENT_MODULE";

// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";

export default createProxy(JSON.stringify([client_id, chunks, id]));
export default createProxy(
JSON.stringify([client_id, chunks, chunkListPath, id])
);
4 changes: 2 additions & 2 deletions crates/next-core/js/src/entry/app/server-to-client.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createProxy } from "next/dist/build/webpack/loaders/next-flight-loader/module-proxy";

// @ts-expect-error CLIENT_CHUNKS is provided by rust
import client_id, { chunks } from "CLIENT_CHUNKS";
import client_id, { chunks, chunkListPath } from "CLIENT_CHUNKS";

export default createProxy(JSON.stringify([client_id, chunks]));
export default createProxy(JSON.stringify([client_id, chunks, chunkListPath]));
alexkirsz marked this conversation as resolved.
Show resolved Hide resolved
2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/fallback.tsx
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@ subscribeToUpdate(
},
},
(update) => {
if (update.type === "restart") {
if (update.type === "restart" || update.type === "notFound") {
location.reload();
}
}
2 changes: 1 addition & 1 deletion crates/next-core/js/src/entry/next-hydrate.tsx
Original file line number Diff line number Diff line change
@@ -97,7 +97,7 @@ function subscribeToPageManifest({ assetPrefix }: { assetPrefix: string }) {
path: "_next/static/development/_devPagesManifest.json",
},
(update) => {
if (["restart", "partial"].includes(update.type)) {
if (["restart", "notFound", "partial"].includes(update.type)) {
return;
}

4 changes: 4 additions & 0 deletions crates/next-core/js/types/globals.d.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
declare global {
function __turbopack_require__(name: any): any;
function __turbopack_load__(path: string): any;
function __turbopack_register_chunk_list__(
chunkListPath: string,
chunksPaths: string[]
): any;
Comment on lines 3 to +7
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we merge these and offer a single method:

function __turbopack_load_chunks__(
    chunkPaths: string[]
    chunkListPath: string,
  ): any;

which loads all the chunks at once and registers the chunk list.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think __turbopack_load__ is still useful on its own for loading separate assets which don't have a chunk list.

function __webpack_require__(name: any): any;
var __webpack_public_path__: string | undefined;
var __DEV_MIDDLEWARE_MATCHERS: any[];
4 changes: 2 additions & 2 deletions crates/next-core/src/app_source.rs
Original file line number Diff line number Diff line change
@@ -411,7 +411,7 @@ async fn create_app_source_for_directory(
segments: layouts,
} => {
let LayoutSegment { target, .. } = *segment.await?;
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);

sources.push(create_node_rendered_source(
@@ -443,7 +443,7 @@ async fn create_app_source_for_directory(
route,
..
} => {
let pathname = pathname_for_path(server_root, url, false);
let pathname = pathname_for_path(server_root, url, false, false);
let params_matcher = NextParamsMatcherVc::new(pathname);

sources.push(create_node_api_source(
1 change: 1 addition & 0 deletions crates/next-core/src/next_client/transition.rs
Original file line number Diff line number Diff line change
@@ -95,6 +95,7 @@ impl Transition for NextClientTransition {
asset: asset.into(),
chunking_context: self.client_chunking_context,
base_path: self.server_root.join("_next"),
server_root: self.server_root,
runtime_entries: Some(runtime_entries),
};

93 changes: 67 additions & 26 deletions crates/next-core/src/next_client_chunks/with_chunks.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use anyhow::Result;
use anyhow::{bail, Result};
use indoc::formatdoc;
use serde_json::Value;
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, ValueToString, ValueToStringVc};
use turbo_tasks_fs::FileSystemPathVc;
@@ -13,14 +14,15 @@ use turbopack::ecmascript::{
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
chunk::{
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContextVc,
ChunkingType, ChunkingTypeOptionVc,
Chunk, ChunkGroupVc, ChunkItem, ChunkItemVc, ChunkListReferenceVc, ChunkVc, ChunkableAsset,
ChunkableAssetReference, ChunkableAssetReferenceVc, ChunkableAssetVc, ChunkingContext,
ChunkingContextVc, ChunkingType, ChunkingTypeOptionVc,
},
ident::AssetIdentVc,
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;

use super::in_chunking_context_asset::InChunkingContextAsset;

@@ -93,6 +95,25 @@ struct WithChunksChunkItem {
inner: WithChunksAssetVc,
}

#[turbo_tasks::value_impl]
impl WithChunksChunkItemVc {
#[turbo_tasks::function]
async fn chunk_list_path(self) -> Result<FileSystemPathVc> {
let this = self.await?;
Ok(this.inner_context.chunk_list_path(this.inner.ident()))
}

#[turbo_tasks::function]
async fn chunk_group(self) -> Result<ChunkGroupVc> {
let this = self.await?;
let inner = this.inner.await?;
Ok(ChunkGroupVc::from_asset(
inner.asset.into(),
this.inner_context,
))
}
}

#[turbo_tasks::value_impl]
impl EcmascriptChunkItem for WithChunksChunkItem {
#[turbo_tasks::function]
@@ -101,29 +122,40 @@ impl EcmascriptChunkItem for WithChunksChunkItem {
}

#[turbo_tasks::function]
async fn content(&self) -> Result<EcmascriptChunkItemContentVc> {
let inner = self.inner.await?;
let group = ChunkGroupVc::from_asset(inner.asset.into(), self.inner_context);
async fn content(self_vc: WithChunksChunkItemVc) -> Result<EcmascriptChunkItemContentVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
let group = self_vc.chunk_group();
let chunks = group.chunks().await?;
let server_root = inner.server_root.await?;
let mut client_chunks = Vec::new();

let chunk_list_path = self_vc.chunk_list_path().await?;
let chunk_list_path = if let Some(path) = server_root.get_path_to(&chunk_list_path) {
path
} else {
bail!("could not get path to chunk list");
};

for chunk_path in chunks.iter().map(|c| c.path()).try_join().await? {
if let Some(path) = server_root.get_path_to(&chunk_path) {
client_chunks.push(Value::String(path.to_string()));
}
}
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.inner_context).id().await?);
let module_id = stringify_js(&*inner.asset.as_chunk_item(this.inner_context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: format!(
"__turbopack_esm__({{
default: () => {},
chunks: () => chunks
}});
const chunks = {};
",
inner_code: formatdoc! {
r#"
__turbopack_esm__({{
default: () => {},
chunks: () => {},
chunkListPath: () => {},
}});
"#,
module_id,
Value::Array(client_chunks)
)
stringify_js_pretty(&client_chunks),
stringify_js(&chunk_list_path),
}
.into(),
..Default::default()
}
@@ -139,18 +171,27 @@ impl ChunkItem for WithChunksChunkItem {
}

#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
Ok(AssetReferencesVc::cell(vec![WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.inner_context,
async fn references(self_vc: WithChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: this.inner_context,
}
.cell()
.into(),
}
.cell()
.into(),
}
.cell()
.into()]))
ChunkListReferenceVc::new(
inner.server_root,
self_vc.chunk_group(),
self_vc.chunk_list_path(),
)
.into(),
]))
}
}

Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ use turbopack_core::{
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
};
use turbopack_ecmascript::utils::stringify_js_pretty;

use crate::next_client_chunks::in_chunking_context_asset::InChunkingContextAsset;

@@ -129,15 +130,17 @@ impl EcmascriptChunkItem for WithClientChunksChunkItem {
let module_id = stringify_js(&*inner.asset.as_chunk_item(self.context).id().await?);
Ok(EcmascriptChunkItemContent {
inner_code: formatdoc!(
// We store the chunks in a binding, otherwise a new array would be created every
// time the export binding is read.
r#"
__turbopack_esm__({{
default: () => __turbopack_import__({}),
chunks: () => chunks
chunks: () => chunks,
}});
const chunks = {};
"#,
module_id,
stringify_js(&client_chunks)
stringify_js_pretty(&client_chunks),
)
.into(),
..Default::default()
@@ -154,13 +157,14 @@ impl ChunkItem for WithClientChunksChunkItem {
}

#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
let inner = self.inner.await?;
async fn references(self_vc: WithClientChunksChunkItemVc) -> Result<AssetReferencesVc> {
let this = self_vc.await?;
let inner = this.inner.await?;
Ok(AssetReferencesVc::cell(vec![
WithClientChunksAssetReference {
asset: InChunkingContextAsset {
asset: inner.asset,
chunking_context: self.context,
chunking_context: this.context,
}
.cell()
.into(),
1 change: 1 addition & 0 deletions crates/next-core/src/next_edge/transition.rs
Original file line number Diff line number Diff line change
@@ -112,6 +112,7 @@ impl Transition for NextEdgeTransition {
asset: new_asset.into(),
chunking_context: self.edge_chunking_context,
base_path: self.output_path,
server_root: self.output_path,
runtime_entries: None,
};

27 changes: 19 additions & 8 deletions crates/next-core/src/page_loader.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ use std::io::Write;

use anyhow::{bail, Result};
use indexmap::indexmap;
use turbo_tasks::{primitives::StringVc, Value};
use turbo_tasks::{primitives::StringVc, TryJoinIterExt, Value};
use turbo_tasks_fs::{rope::RopeBuilder, File, FileContent, FileSystemPathVc};
use turbopack_core::{
asset::{Asset, AssetContentVc, AssetVc},
@@ -116,17 +116,28 @@ impl Asset for PageLoaderAsset {
let this = &*self_vc.await?;

let chunks = self_vc.get_page_chunks().await?;

let mut data = Vec::with_capacity(chunks.len());
for chunk in chunks.iter() {
let path = chunk.path().await?;
data.push(serde_json::Value::String(path.path.clone()));
}
let server_root = this.server_root.await?;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way the page loader asset computed paths was incorrect before, as it wasn't basing them upon the server root path. But this doesn't really have any measurable effect today as we don't support base path.


let chunk_paths: Vec<_> = chunks
.iter()
.map(|chunk| {
let server_root = server_root.clone();
async move {
Ok(server_root
.get_path_to(&*chunk.path().await?)
.map(|path| path.to_string()))
}
})
.try_join()
.await?
.into_iter()
.flatten()
.collect();

let content = format!(
"__turbopack_load_page_chunks__({}, {})\n",
stringify_js(&this.pathname.await?),
serde_json::Value::Array(data)
stringify_js(&chunk_paths)
);

Ok(AssetContentVc::from(File::from(content)))
5 changes: 3 additions & 2 deletions crates/next-core/src/page_source.rs
Original file line number Diff line number Diff line change
@@ -346,7 +346,7 @@ async fn create_page_source_for_file(
Value::new(ClientContextType::Pages { pages_dir }),
);

let pathname = pathname_for_path(server_root, server_path, true);
let pathname = pathname_for_path(server_root, server_path, true, false);
let route_matcher = NextParamsMatcherVc::new(pathname);

Ok(if is_api_path {
@@ -370,8 +370,9 @@ async fn create_page_source_for_file(
runtime_entries,
)
} else {
let data_pathname = pathname_for_path(server_root, server_path, true, true);
let data_route_matcher =
NextPrefixSuffixParamsMatcherVc::new(pathname, "_next/data/development/", ".json");
NextPrefixSuffixParamsMatcherVc::new(data_pathname, "_next/data/development/", ".json");

let ssr_entry = SsrEntry {
context: server_context,
11 changes: 8 additions & 3 deletions crates/next-core/src/util.rs
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@ pub async fn pathname_for_path(
server_root: FileSystemPathVc,
server_path: FileSystemPathVc,
has_extension: bool,
data: bool,
) -> Result<StringVc> {
let server_path_value = &*server_path.await?;
let path = if let Some(path) = server_root.await?.get_path_to(server_path_value) {
@@ -46,10 +47,14 @@ pub async fn pathname_for_path(
} else {
path
};
let path = if path == "index" {
""
let path = if data {
path
} else {
path.strip_suffix("/index").unwrap_or(path)
if path == "index" {
""
} else {
path.strip_suffix("/index").unwrap_or(path)
}
};

Ok(StringVc::cell(path.to_string()))
Loading