From c3f0bf1b8b3d8938ec2e62d0acb7ab0a67c1ca8c Mon Sep 17 00:00:00 2001 From: Hana Date: Fri, 29 Nov 2024 15:45:03 +0800 Subject: [PATCH] perf: move from `Buffer` to zero-copy `BufferSlice` --- crates/node_binding/binding.d.ts | 16 +- .../node_binding/src/plugins/interceptor.rs | 4 +- .../src/compilation/mod.rs | 22 ++- crates/rspack_binding_values/src/module.rs | 16 +- crates/rspack_binding_values/src/source.rs | 185 +++++++++++++++--- packages/rspack/src/Compilation.ts | 8 +- packages/rspack/src/util/source.ts | 6 +- 7 files changed, 202 insertions(+), 55 deletions(-) diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index 05244078341..c05214e8a33 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -47,7 +47,7 @@ export declare class EntryOptionsDto { export type EntryOptionsDTO = EntryOptionsDto export declare class JsCompilation { - updateAsset(filename: string, newSourceOrFunction: JsCompatSource | ((source: JsCompatSource) => JsCompatSource), assetInfoUpdateOrFunction?: JsAssetInfo | ((assetInfo: JsAssetInfo) => JsAssetInfo)): void + updateAsset(filename: string, newSourceOrFunction: JsCompatSource | ((source: JsCompatSourceOwned) => JsCompatSourceOwned), assetInfoUpdateOrFunction?: JsAssetInfo | ((assetInfo: JsAssetInfo) => JsAssetInfo)): void getAssets(): Readonly[] getAsset(name: string): JsAsset | null getAssetSource(name: string): JsCompatSource | null @@ -526,11 +526,23 @@ export interface JsCodegenerationResults { map: Record> } +/** + * Zero copy `JsCompatSource` slice shared between Rust and Node.js if buffer is used. + * + * It can only be used in non-async context and the lifetime is bound to the fn closure. + * + * If you want to use Node.js Buffer in async context or want to extend the lifetime, use `JsCompatSourceOwned` instead. + */ export interface JsCompatSource { source: string | Buffer map?: string } +export interface JsCompatSourceOwned { + source: string | Buffer + map?: string +} + export interface JsCreateData { request: string userRequest: string @@ -781,7 +793,7 @@ export interface JsRuntimeGlobals { } export interface JsRuntimeModule { - source?: JsCompatSource + source?: JsCompatSourceOwned moduleIdentifier: string constructorName: string name: string diff --git a/crates/node_binding/src/plugins/interceptor.rs b/crates/node_binding/src/plugins/interceptor.rs index 1a54b82bb7d..5589cca40ac 100644 --- a/crates/node_binding/src/plugins/interceptor.rs +++ b/crates/node_binding/src/plugins/interceptor.rs @@ -20,7 +20,7 @@ use rspack_binding_values::{ JsFactorizeOutput, JsModuleWrapper, JsNormalModuleFactoryCreateModuleArgs, JsResolveArgs, JsResolveForSchemeArgs, JsResolveForSchemeOutput, JsResolveOutput, JsRuntimeGlobals, JsRuntimeModule, JsRuntimeModuleArg, JsRuntimeRequirementInTreeArg, - JsRuntimeRequirementInTreeResult, ToJsCompatSource, + JsRuntimeRequirementInTreeResult, ToJsCompatSourceOwned, }; use rspack_collections::IdentifierSet; use rspack_core::{ @@ -1256,7 +1256,7 @@ impl CompilationRuntimeModule for CompilationRuntimeModuleTap { source: Some( module .generate(compilation)? - .to_js_compat_source() + .to_js_compat_source_owned() .unwrap_or_else(|err| panic!("Failed to generate runtime module source: {err}")), ), module_identifier: module.identifier().to_string(), diff --git a/crates/rspack_binding_values/src/compilation/mod.rs b/crates/rspack_binding_values/src/compilation/mod.rs index cf53a60c1b1..3eecc10f6bd 100644 --- a/crates/rspack_binding_values/src/compilation/mod.rs +++ b/crates/rspack_binding_values/src/compilation/mod.rs @@ -25,14 +25,13 @@ use rspack_plugin_runtime::RuntimeModuleFromJs; use super::{JsFilename, PathWithInfo}; use crate::utils::callbackify; use crate::JsAddingRuntimeModule; +use crate::JsCompatSource; use crate::JsModuleGraph; use crate::JsModuleWrapper; use crate::JsStatsOptimizationBailout; use crate::LocalJsFilename; -use crate::ToJsCompatSource; -use crate::{ - chunk::JsChunk, JsAsset, JsAssetInfo, JsChunkGroup, JsCompatSource, JsPathData, JsStats, -}; +use crate::ToJsCompatSource as _; +use crate::{chunk::JsChunk, JsAsset, JsAssetInfo, JsChunkGroup, JsPathData, JsStats}; use crate::{JsRspackDiagnostic, JsRspackError}; #[napi] @@ -72,10 +71,11 @@ impl JsCompilation { #[napi] impl JsCompilation { #[napi( - ts_args_type = r#"filename: string, newSourceOrFunction: JsCompatSource | ((source: JsCompatSource) => JsCompatSource), assetInfoUpdateOrFunction?: JsAssetInfo | ((assetInfo: JsAssetInfo) => JsAssetInfo)"# + ts_args_type = r#"filename: string, newSourceOrFunction: JsCompatSource | ((source: JsCompatSourceOwned) => JsCompatSourceOwned), assetInfoUpdateOrFunction?: JsAssetInfo | ((assetInfo: JsAssetInfo) => JsAssetInfo)"# )] pub fn update_asset( &mut self, + env: &Env, filename: String, new_source_or_function: Either>, asset_info_update_or_function: Option< @@ -90,8 +90,8 @@ impl JsCompilation { let new_source = match new_source_or_function { Either::A(new_source) => new_source.into(), Either::B(new_source_fn) => { - let js_compat_source: JsCompatSource = - new_source_fn.call(original_source.to_js_compat_source()?)?; + let js_compat_source = + new_source_fn.call(original_source.to_js_compat_source(env)?)?; js_compat_source.into() } }; @@ -147,13 +147,17 @@ impl JsCompilation { } #[napi] - pub fn get_asset_source(&self, name: String) -> Result> { + pub fn get_asset_source<'a>( + &self, + env: &'a Env, + name: String, + ) -> Result>> { let compilation = self.as_ref()?; compilation .assets() .get(&name) - .and_then(|v| v.source.as_ref().map(|s| s.to_js_compat_source())) + .and_then(|v| v.source.as_ref().map(|s| s.to_js_compat_source(env))) .transpose() } diff --git a/crates/rspack_binding_values/src/module.rs b/crates/rspack_binding_values/src/module.rs index eb7eb26afc5..089baa9e94c 100644 --- a/crates/rspack_binding_values/src/module.rs +++ b/crates/rspack_binding_values/src/module.rs @@ -11,8 +11,11 @@ use rspack_plugin_runtime::RuntimeModuleFromJs; use rspack_util::source_map::SourceMapKind; use rustc_hash::FxHashMap as HashMap; -use super::{JsCompatSource, ToJsCompatSource}; -use crate::{JsChunk, JsCodegenerationResults, JsDependenciesBlockWrapper, JsDependencyWrapper}; +use super::JsCompatSourceOwned; +use crate::{ + JsChunk, JsCodegenerationResults, JsCompatSource, JsDependenciesBlockWrapper, + JsDependencyWrapper, ToJsCompatSource, +}; #[derive(Default)] #[napi(object)] @@ -74,11 +77,14 @@ impl JsModule { } #[napi(getter)] - pub fn original_source(&mut self) -> napi::Result> { + pub fn original_source<'a>( + &mut self, + env: &'a Env, + ) -> napi::Result, ()>> { let module = self.as_ref()?; Ok(match module.original_source() { - Some(source) => match source.to_js_compat_source().ok() { + Some(source) => match source.to_js_compat_source(env).ok() { Some(s) => Either::A(s), None => Either::B(()), }, @@ -397,7 +403,7 @@ pub struct JsExecuteModuleArg { #[derive(Default)] #[napi(object)] pub struct JsRuntimeModule { - pub source: Option, + pub source: Option, pub module_identifier: String, pub constructor_name: String, pub name: String, diff --git a/crates/rspack_binding_values/src/source.rs b/crates/rspack_binding_values/src/source.rs index ba922b1f669..2dd374b7f6d 100644 --- a/crates/rspack_binding_values/src/source.rs +++ b/crates/rspack_binding_values/src/source.rs @@ -7,15 +7,49 @@ use rspack_core::rspack_sources::{ }; use rspack_napi::napi::bindgen_prelude::*; +/// Zero copy `JsCompatSource` slice shared between Rust and Node.js if buffer is used. +/// +/// It can only be used in non-async context and the lifetime is bound to the fn closure. +/// +/// If you want to use Node.js Buffer in async context or want to extend the lifetime, use `JsCompatSourceOwned` instead. +#[napi(object)] +pub struct JsCompatSource<'s> { + pub source: Either>, + pub map: Option, +} + +impl<'s> From> for BoxSource { + fn from(value: JsCompatSource<'s>) -> Self { + match value.source { + Either::A(string) => { + if let Some(map) = value.map { + match SourceMap::from_slice(map.as_ref()).ok() { + Some(source_map) => SourceMapSource::new(WithoutOriginalOptions { + value: string, + name: "inmemory://from js", + source_map, + }) + .boxed(), + None => RawSource::from(string).boxed(), + } + } else { + RawSource::from(string).boxed() + } + } + Either::B(buffer) => RawSource::from(buffer.to_vec()).boxed(), + } + } +} + #[napi(object)] #[derive(Clone)] -pub struct JsCompatSource { +pub struct JsCompatSourceOwned { pub source: Either, pub map: Option, } -impl From for BoxSource { - fn from(value: JsCompatSource) -> Self { +impl From for BoxSource { + fn from(value: JsCompatSourceOwned) -> Self { match value.source { Either::A(string) => { if let Some(map) = value.map { @@ -38,14 +72,14 @@ impl From for BoxSource { } pub trait ToJsCompatSource { - fn to_js_compat_source(&self) -> Result; + fn to_js_compat_source(&self, env: &Env) -> Result; } impl ToJsCompatSource for RawSource { - fn to_js_compat_source(&self) -> Result { + fn to_js_compat_source(&self, env: &Env) -> Result { Ok(JsCompatSource { source: if self.is_buffer() { - Either::B(self.buffer().to_vec().into()) + Either::B(BufferSlice::from_data(env, self.buffer())?) } else { Either::A(self.source().to_string()) }, @@ -55,36 +89,36 @@ impl ToJsCompatSource for RawSource { } impl ToJsCompatSource for ReplaceSource { - fn to_js_compat_source(&self) -> Result { + fn to_js_compat_source(&self, env: &Env) -> Result { Ok(JsCompatSource { - source: Either::A(self.source().to_string()), + source: Either::B(BufferSlice::from_data(env, self.source().as_bytes())?), map: to_webpack_map(self)?, }) } } impl ToJsCompatSource for CachedSource { - fn to_js_compat_source(&self) -> Result { - self.original().to_js_compat_source() + fn to_js_compat_source(&self, env: &Env) -> Result { + self.original().to_js_compat_source(env) } } impl ToJsCompatSource for Arc { - fn to_js_compat_source(&self) -> Result { - (**self).to_js_compat_source() + fn to_js_compat_source(&self, env: &Env) -> Result { + (**self).to_js_compat_source(env) } } impl ToJsCompatSource for Box { - fn to_js_compat_source(&self) -> Result { - (**self).to_js_compat_source() + fn to_js_compat_source(&self, env: &Env) -> Result { + (**self).to_js_compat_source(env) } } macro_rules! impl_default_to_compat_source { ($ident:ident) => { impl ToJsCompatSource for $ident { - fn to_js_compat_source(&self) -> Result { + fn to_js_compat_source(&self, _env: &Env) -> Result { Ok(JsCompatSource { source: Either::A(self.source().to_string()), map: to_webpack_map(self)?, @@ -98,41 +132,132 @@ impl_default_to_compat_source!(SourceMapSource); impl_default_to_compat_source!(ConcatSource); impl_default_to_compat_source!(OriginalSource); -fn to_webpack_map(source: &dyn Source) -> Result> { - let map = source.map(&MapOptions::default()); +impl ToJsCompatSource for dyn Source + '_ { + fn to_js_compat_source(&self, env: &Env) -> Result { + if let Some(raw_source) = self.as_any().downcast_ref::() { + raw_source.to_js_compat_source(env) + } else if let Some(cached_source) = self.as_any().downcast_ref::>() { + cached_source.to_js_compat_source(env) + } else if let Some(cached_source) = self + .as_any() + .downcast_ref::>>() + { + cached_source.to_js_compat_source(env) + } else if let Some(cached_source) = self + .as_any() + .downcast_ref::>>() + { + cached_source.to_js_compat_source(env) + } else if let Some(source) = self.as_any().downcast_ref::>() { + source.to_js_compat_source(env) + } else if let Some(source) = self.as_any().downcast_ref::>() { + source.to_js_compat_source(env) + } else { + // If it's not a `RawSource` related type, then we regards it as a `Source` type. + Ok(JsCompatSource { + source: Either::A(self.source().to_string()), + map: to_webpack_map(self)?, + }) + } + } +} - map - .map(|m| m.to_json()) - .transpose() - .map_err(|err| napi::Error::from_reason(err.to_string())) +pub trait ToJsCompatSourceOwned { + fn to_js_compat_source_owned(&self) -> Result; } -impl ToJsCompatSource for dyn Source + '_ { - fn to_js_compat_source(&self) -> Result { +impl ToJsCompatSourceOwned for RawSource { + fn to_js_compat_source_owned(&self) -> Result { + Ok(JsCompatSourceOwned { + source: if self.is_buffer() { + Either::B(self.buffer().to_vec().into()) + } else { + Either::A(self.source().to_string()) + }, + map: to_webpack_map(self)?, + }) + } +} + +impl ToJsCompatSourceOwned for ReplaceSource { + fn to_js_compat_source_owned(&self) -> Result { + Ok(JsCompatSourceOwned { + source: Either::A(self.source().to_string()), + map: to_webpack_map(self)?, + }) + } +} + +impl ToJsCompatSourceOwned for CachedSource { + fn to_js_compat_source_owned(&self) -> Result { + self.original().to_js_compat_source_owned() + } +} + +impl ToJsCompatSourceOwned for Arc { + fn to_js_compat_source_owned(&self) -> Result { + (**self).to_js_compat_source_owned() + } +} + +impl ToJsCompatSourceOwned for Box { + fn to_js_compat_source_owned(&self) -> Result { + (**self).to_js_compat_source_owned() + } +} + +macro_rules! impl_default_to_compat_source { + ($ident:ident) => { + impl ToJsCompatSourceOwned for $ident { + fn to_js_compat_source_owned(&self) -> Result { + Ok(JsCompatSourceOwned { + source: Either::A(self.source().to_string()), + map: to_webpack_map(self)?, + }) + } + } + }; +} + +impl_default_to_compat_source!(SourceMapSource); +impl_default_to_compat_source!(ConcatSource); +impl_default_to_compat_source!(OriginalSource); + +impl ToJsCompatSourceOwned for dyn Source + '_ { + fn to_js_compat_source_owned(&self) -> Result { if let Some(raw_source) = self.as_any().downcast_ref::() { - raw_source.to_js_compat_source() + raw_source.to_js_compat_source_owned() } else if let Some(cached_source) = self.as_any().downcast_ref::>() { - cached_source.to_js_compat_source() + cached_source.to_js_compat_source_owned() } else if let Some(cached_source) = self .as_any() .downcast_ref::>>() { - cached_source.to_js_compat_source() + cached_source.to_js_compat_source_owned() } else if let Some(cached_source) = self .as_any() .downcast_ref::>>() { - cached_source.to_js_compat_source() + cached_source.to_js_compat_source_owned() } else if let Some(source) = self.as_any().downcast_ref::>() { - source.to_js_compat_source() + source.to_js_compat_source_owned() } else if let Some(source) = self.as_any().downcast_ref::>() { - source.to_js_compat_source() + source.to_js_compat_source_owned() } else { // If it's not a `RawSource` related type, then we regards it as a `Source` type. - Ok(JsCompatSource { + Ok(JsCompatSourceOwned { source: Either::A(self.source().to_string()), map: to_webpack_map(self)?, }) } } } + +fn to_webpack_map(source: &dyn Source) -> Result> { + let map = source.map(&MapOptions::default()); + + map + .map(|m| m.to_json()) + .transpose() + .map_err(|err| napi::Error::from_reason(err.to_string())) +} diff --git a/packages/rspack/src/Compilation.ts b/packages/rspack/src/Compilation.ts index 81aff64a33b..48f714f5033 100644 --- a/packages/rspack/src/Compilation.ts +++ b/packages/rspack/src/Compilation.ts @@ -10,7 +10,7 @@ import type * as binding from "@rspack/binding"; import { type ExternalObject, - type JsCompatSource, + type JsCompatSourceOwned, type JsCompilation, type JsModule, type JsPathData, @@ -605,12 +605,12 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si | ((assetInfo: AssetInfo) => AssetInfo) ) { let compatNewSourceOrFunction: - | JsCompatSource - | ((source: JsCompatSource) => JsCompatSource); + | JsCompatSourceOwned + | ((source: JsCompatSourceOwned) => JsCompatSourceOwned); if (typeof newSourceOrFunction === "function") { compatNewSourceOrFunction = function newSourceFunction( - source: JsCompatSource + source: JsCompatSourceOwned ) { return JsSource.__to_binding( newSourceOrFunction(JsSource.__from_binding(source)) diff --git a/packages/rspack/src/util/source.ts b/packages/rspack/src/util/source.ts index b80e04948d2..f222deea1bd 100644 --- a/packages/rspack/src/util/source.ts +++ b/packages/rspack/src/util/source.ts @@ -1,8 +1,8 @@ -import type { JsCompatSource } from "@rspack/binding"; +import type { JsCompatSourceOwned } from "@rspack/binding"; import { RawSource, Source, SourceMapSource } from "webpack-sources"; class JsSource extends Source { - static __from_binding(source: JsCompatSource): Source { + static __from_binding(source: JsCompatSourceOwned): Source { if (source.source instanceof Buffer) { // @ts-expect-error: webpack-sources can accept buffer as source, // see: https://github.com/webpack/webpack-sources/blob/9f98066311d53a153fdc7c633422a1d086528027/lib/RawSource.js#L12 @@ -20,7 +20,7 @@ class JsSource extends Source { ); } - static __to_binding(source: Source): JsCompatSource { + static __to_binding(source: Source): JsCompatSourceOwned { if (source instanceof RawSource) { // @ts-expect-error: The 'isBuffer' method exists on 'RawSource' in 'webpack-sources', if (source.isBuffer()) {