From 22fae289545f16c032b8380b47bb670783d2af27 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Wed, 11 Oct 2023 11:48:14 +0200 Subject: [PATCH 01/12] debug wasm build optimisation --- .../utils/wasm-builder/src/wasm_project.rs | 143 ++++++++++++------ 1 file changed, 97 insertions(+), 46 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 849af853c6da..8ce8e1213ae6 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -132,9 +132,14 @@ pub(crate) fn create_and_compile( features_to_enable, ); - let profile = build_project(&project, default_rustflags, cargo_cmd); - let (wasm_binary, wasm_binary_compressed, bloaty) = - compact_wasm_file(&project, profile, project_cargo_toml, wasm_binary_name); + let build_config = BuildConfiguration::detect(&project); + build_wasm(&build_config.wasm_build_profile, &project, default_rustflags, cargo_cmd); + let (wasm_binary, wasm_binary_compressed, bloaty) = maybe_compact_wasm_and_copy_blobs( + &project, + &build_config, + project_cargo_toml, + wasm_binary_name, + ); if check_for_runtime_version_section { ensure_runtime_version_wasm_section_exists(bloaty.wasm_binary_bloaty_path()); @@ -539,8 +544,8 @@ fn create_project( wasm_project_folder } -/// The cargo profile that is used to build the wasm project. -#[derive(Debug, EnumIter)] +/// A rustc profile. +#[derive(Clone, Debug, EnumIter, PartialEq)] enum Profile { /// The `--profile dev` profile. Debug, @@ -551,17 +556,63 @@ enum Profile { } impl Profile { - /// Create a profile by detecting which profile is used for the main build. + /// The name of the profile as supplied to the cargo `--profile` cli option. + fn name(&self) -> &'static str { + match self { + Self::Debug => "dev", + Self::Release => "release", + Self::Production => "production", + } + } + + /// The sub directory within `target` where cargo places the build output. + /// + /// # Note + /// + /// Usually this is the same as [`Self::name`] with the exception of the debug + /// profile which is called `dev`. + fn directory(&self) -> &'static str { + match self { + Self::Debug => "debug", + _ => self.name(), + } + } + + /// Whether the resulting binary should be compacted and compressed. + fn wants_compact(&self) -> bool { + !matches!(self, Self::Debug) + } +} + +/// The build configuration for this build. +#[derive(Debug)] +struct BuildConfiguration { + /// The profile that is used to build the main project. + pub cargo_build_profile: Profile, + /// The profile to use to build the wasm project. + pub wasm_build_profile: Profile, +} + +impl BuildConfiguration { + /// Create a [`BuildConfiguration`] by detecting which profile is used for the main build and + /// checking any env var overrides. /// /// We cannot easily determine the profile that is used by the main cargo invocation /// because the `PROFILE` environment variable won't contain any custom profiles like /// "production". It would only contain the builtin profile where the custom profile /// inherits from. This is why we inspect the build path to learn which profile is used. /// + /// When not overriden by a env variable we always default to building wasm with the `Release` + /// profile even when the main build uses the debug build. This is because wasm built with the + /// `Debug` profile is too slow for normal development activities and almost never intended. + /// + /// When cargo is building in `--profile dev`, user likely intends to compile fast, so we don't + /// bother producing compact or compressed blobs. + /// /// # Note /// /// Can be overriden by setting [`crate::WASM_BUILD_TYPE_ENV`]. - fn detect(wasm_project: &Path) -> Profile { + fn detect(wasm_project: &Path) -> Self { let (name, overriden) = if let Ok(name) = env::var(crate::WASM_BUILD_TYPE_ENV) { (name, true) } else { @@ -585,7 +636,8 @@ impl Profile { .to_string(); (name, false) }; - match (Profile::iter().find(|p| p.directory() == name), overriden) { + let cargo_build_profile = Profile::iter().find(|p| p.directory() == name); + let wasm_build_profile = match (cargo_build_profile.clone(), overriden) { // When not overriden by a env variable we default to using the `Release` profile // for the wasm build even when the main build uses the debug build. This // is because the `Debug` profile is too slow for normal development activities. @@ -615,35 +667,12 @@ impl Profile { ); process::exit(1); }, + }; + BuildConfiguration { + cargo_build_profile: cargo_build_profile.unwrap_or(Profile::Release), + wasm_build_profile, } } - - /// The name of the profile as supplied to the cargo `--profile` cli option. - fn name(&self) -> &'static str { - match self { - Self::Debug => "dev", - Self::Release => "release", - Self::Production => "production", - } - } - - /// The sub directory within `target` where cargo places the build output. - /// - /// # Note - /// - /// Usually this is the same as [`Self::name`] with the exception of the debug - /// profile which is called `dev`. - fn directory(&self) -> &'static str { - match self { - Self::Debug => "debug", - _ => self.name(), - } - } - - /// Whether the resulting binary should be compacted and compressed. - fn wants_compact(&self) -> bool { - !matches!(self, Self::Debug) - } } /// Check environment whether we should build without network @@ -651,12 +680,13 @@ fn offline_build() -> bool { env::var(OFFLINE).map_or(false, |v| v == "true") } -/// Build the project to create the WASM binary. -fn build_project( +/// Build the project and create the WASM binary. +fn build_wasm( + wasm_build_profile: &Profile, project: &Path, default_rustflags: &str, cargo_cmd: CargoCommandVersioned, -) -> Profile { +) { let manifest_path = project.join("Cargo.toml"); let mut build_cmd = cargo_cmd.command(); @@ -685,9 +715,8 @@ fn build_project( build_cmd.arg("--color=always"); } - let profile = Profile::detect(project); build_cmd.arg("--profile"); - build_cmd.arg(profile.name()); + build_cmd.arg(wasm_build_profile.name()); if offline_build() { build_cmd.arg("--offline"); @@ -698,16 +727,19 @@ fn build_project( println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); match build_cmd.status().map(|s| s.success()) { - Ok(true) => profile, + Ok(true) => {}, // Use `process.exit(1)` to have a clean error output. _ => process::exit(1), } } -/// Compact the WASM binary using `wasm-gc` and compress it using zstd. -fn compact_wasm_file( +/// If the cargo build profile is not [`Profile::Debug`], spend the additional time to create +/// a compacted and compressed wasm binary. +/// +/// Then, copy the wasm binaries to the intended directory. +fn maybe_compact_wasm_and_copy_blobs( project: &Path, - profile: Profile, + build_config: &BuildConfiguration, cargo_manifest: &Path, out_name: Option, ) -> (Option, Option, WasmBinaryBloaty) { @@ -715,26 +747,45 @@ fn compact_wasm_file( let out_name = out_name.unwrap_or_else(|| default_out_name.clone()); let in_path = project .join("target/wasm32-unknown-unknown") - .join(profile.directory()) + .join(build_config.wasm_build_profile.directory()) .join(format!("{}.wasm", default_out_name)); - let (wasm_compact_path, wasm_compact_compressed_path) = if profile.wants_compact() { + // When cargo is running cargo in `--profile dev` the user wants speed, so skip producing + // compact and compressed blobs. + let (wasm_compact_path, wasm_compact_compressed_path) = if build_config + .cargo_build_profile + .wants_compact() + { let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name,)); + let start = std::time::Instant::now(); wasm_opt::OptimizationOptions::new_opt_level_0() .mvp_features_only() .debug_info(true) .add_pass(wasm_opt::Pass::StripDwarf) .run(&in_path, &wasm_compact_path) .expect("Failed to compact generated WASM binary."); + println!( + "{} {}", + colorize_info_message("Compacted wasm in"), + colorize_info_message(format!("{:?}", start.elapsed()).as_str()) + ); let wasm_compact_compressed_path = project.join(format!("{}.compact.compressed.wasm", out_name)); + let start = std::time::Instant::now(); if compress_wasm(&wasm_compact_path, &wasm_compact_compressed_path) { + println!( + "{} {}", + colorize_info_message("Compressed wasm in"), + colorize_info_message(format!("{:?}", start.elapsed()).as_str()) + ); (Some(WasmBinary(wasm_compact_path)), Some(WasmBinary(wasm_compact_compressed_path))) } else { + println!("{}", colorize_info_message("Skipping wasm compression")); (Some(WasmBinary(wasm_compact_path)), None) } } else { + println!("{}", colorize_info_message("Skipping wasm compaction and compression")); (None, None) }; From 0035c21a86428e81855335441f2522f6f2d386b7 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Wed, 11 Oct 2023 12:26:27 +0200 Subject: [PATCH 02/12] cleanup --- substrate/utils/wasm-builder/src/wasm_project.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 8ce8e1213ae6..3ff7542e7bc0 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -726,10 +726,9 @@ fn build_wasm( println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd); println!("{} {}", colorize_info_message("Using rustc version:"), cargo_cmd.rustc_version()); - match build_cmd.status().map(|s| s.success()) { - Ok(true) => {}, - // Use `process.exit(1)` to have a clean error output. - _ => process::exit(1), + // Use `process::exit(1)` to have a clean error output. + if build_cmd.status().map(|s| s.success()).is_err() { + process::exit(1); } } From 1995cf2fc42cc67e56fd30c04dcf5404e6d57fbb Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Wed, 11 Oct 2023 14:44:02 +0200 Subject: [PATCH 03/12] remove derive partialeq --- substrate/utils/wasm-builder/src/wasm_project.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 3ff7542e7bc0..0a09d2d48382 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -545,7 +545,7 @@ fn create_project( } /// A rustc profile. -#[derive(Clone, Debug, EnumIter, PartialEq)] +#[derive(Clone, Debug, EnumIter)] enum Profile { /// The `--profile dev` profile. Debug, From edb604c08b76e8396eaafcbad3078c17c76dbcec Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 10:48:15 +1100 Subject: [PATCH 04/12] Update substrate/utils/wasm-builder/src/wasm_project.rs Co-authored-by: Koute --- substrate/utils/wasm-builder/src/wasm_project.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 0a09d2d48382..862d4a643d26 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -587,10 +587,10 @@ impl Profile { /// The build configuration for this build. #[derive(Debug)] struct BuildConfiguration { - /// The profile that is used to build the main project. - pub cargo_build_profile: Profile, - /// The profile to use to build the wasm project. - pub wasm_build_profile: Profile, + /// The profile that is used to build the outer project. + pub outer_build_profile: Profile, + /// The profile to use to build the runtime blob. + pub blob_build_profile: Profile, } impl BuildConfiguration { From 3cd37bed31451a8b09da2d5b31c7a1ab6333b125 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 10:49:25 +1100 Subject: [PATCH 05/12] rename build_wasm to build_blob --- substrate/utils/wasm-builder/src/wasm_project.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 5af1e55edfec..0f3d5c5690c5 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -133,7 +133,7 @@ pub(crate) fn create_and_compile( ); let build_config = BuildConfiguration::detect(&project); - build_wasm(&build_config.wasm_build_profile, &project, default_rustflags, cargo_cmd); + build_blob(&build_config.wasm_build_profile, &project, default_rustflags, cargo_cmd); let (wasm_binary, wasm_binary_compressed, bloaty) = maybe_compact_wasm_and_copy_blobs( &project, &build_config, @@ -680,8 +680,8 @@ fn offline_build() -> bool { env::var(OFFLINE).map_or(false, |v| v == "true") } -/// Build the project and create the WASM binary. -fn build_wasm( +/// Build the project and create the runtime blob. +fn build_blob( wasm_build_profile: &Profile, project: &Path, default_rustflags: &str, From 898cc42c2a153226b82733efce12f0de8fad12ff Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 10:51:32 +1100 Subject: [PATCH 06/12] finish build config renaming --- .../utils/wasm-builder/src/wasm_project.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 0f3d5c5690c5..88eda95250f9 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -133,7 +133,7 @@ pub(crate) fn create_and_compile( ); let build_config = BuildConfiguration::detect(&project); - build_blob(&build_config.wasm_build_profile, &project, default_rustflags, cargo_cmd); + build_blob(&build_config.blob_build_profile, &project, default_rustflags, cargo_cmd); let (wasm_binary, wasm_binary_compressed, bloaty) = maybe_compact_wasm_and_copy_blobs( &project, &build_config, @@ -636,8 +636,8 @@ impl BuildConfiguration { .to_string(); (name, false) }; - let cargo_build_profile = Profile::iter().find(|p| p.directory() == name); - let wasm_build_profile = match (cargo_build_profile.clone(), overriden) { + let outer_build_profile = Profile::iter().find(|p| p.directory() == name); + let blob_build_profile = match (outer_build_profile.clone(), overriden) { // When not overriden by a env variable we default to using the `Release` profile // for the wasm build even when the main build uses the debug build. This // is because the `Debug` profile is too slow for normal development activities. @@ -669,8 +669,8 @@ impl BuildConfiguration { }, }; BuildConfiguration { - cargo_build_profile: cargo_build_profile.unwrap_or(Profile::Release), - wasm_build_profile, + outer_build_profile: outer_build_profile.unwrap_or(Profile::Release), + blob_build_profile, } } } @@ -682,7 +682,7 @@ fn offline_build() -> bool { /// Build the project and create the runtime blob. fn build_blob( - wasm_build_profile: &Profile, + blob_build_profile: &Profile, project: &Path, default_rustflags: &str, cargo_cmd: CargoCommandVersioned, @@ -716,7 +716,7 @@ fn build_blob( } build_cmd.arg("--profile"); - build_cmd.arg(wasm_build_profile.name()); + build_cmd.arg(blob_build_profile.name()); if offline_build() { build_cmd.arg("--offline"); @@ -746,13 +746,13 @@ fn maybe_compact_wasm_and_copy_blobs( let out_name = out_name.unwrap_or_else(|| default_out_name.clone()); let in_path = project .join("target/wasm32-unknown-unknown") - .join(build_config.wasm_build_profile.directory()) + .join(build_config.blob_build_profile.directory()) .join(format!("{}.wasm", default_out_name)); // When cargo is running cargo in `--profile dev` the user wants speed, so skip producing // compact and compressed blobs. let (wasm_compact_path, wasm_compact_compressed_path) = if build_config - .cargo_build_profile + .outer_build_profile .wants_compact() { let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name,)); From be34e513173929a8057555ebe4ae8ff173525f1b Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 11:26:57 +1100 Subject: [PATCH 07/12] koute refactor suggestion --- substrate/utils/wasm-builder/src/builder.rs | 4 +- .../utils/wasm-builder/src/wasm_project.rs | 216 +++++++++--------- 2 files changed, 112 insertions(+), 108 deletions(-) diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs index e0a30644b231..9c1655d85623 100644 --- a/substrate/utils/wasm-builder/src/builder.rs +++ b/substrate/utils/wasm-builder/src/builder.rs @@ -273,9 +273,9 @@ fn build_project( ); let (wasm_binary, wasm_binary_bloaty) = if let Some(wasm_binary) = wasm_binary { - (wasm_binary.wasm_binary_path_escaped(), bloaty.wasm_binary_bloaty_path_escaped()) + (wasm_binary.wasm_binary_path_escaped(), bloaty.bloaty_path_escaped()) } else { - (bloaty.wasm_binary_bloaty_path_escaped(), bloaty.wasm_binary_bloaty_path_escaped()) + (bloaty.bloaty_path_escaped(), bloaty.bloaty_path_escaped()) }; crate::write_file_if_changed( diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 88eda95250f9..d83ea674ef49 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -48,13 +48,13 @@ fn colorize_info_message(message: &str) -> String { pub struct WasmBinaryBloaty(PathBuf); impl WasmBinaryBloaty { - /// Returns the escaped path to the bloaty wasm binary. - pub fn wasm_binary_bloaty_path_escaped(&self) -> String { + /// Returns the escaped path to the bloaty binary. + pub fn bloaty_path_escaped(&self) -> String { self.0.display().to_string().escape_default().to_string() } - /// Returns the path to the wasm binary. - pub fn wasm_binary_bloaty_path(&self) -> &Path { + /// Returns the path to the binary. + pub fn bloaty_path(&self) -> &Path { &self.0 } } @@ -110,83 +110,106 @@ fn crate_metadata(cargo_manifest: &Path) -> Metadata { /// /// # Returns /// -/// The path to the compact WASM binary and the bloaty WASM binary. +/// The path to the compact runtime binary and the bloaty runtime binary. pub(crate) fn create_and_compile( project_cargo_toml: &Path, default_rustflags: &str, cargo_cmd: CargoCommandVersioned, features_to_enable: Vec, - wasm_binary_name: Option, + runtime_blob_name: Option, check_for_runtime_version_section: bool, ) -> (Option, WasmBinaryBloaty) { - let wasm_workspace_root = get_wasm_workspace_root(); - let wasm_workspace = wasm_workspace_root.join("wbuild"); + let runtime_workspace_root = get_wasm_workspace_root(); + let runtime_workspace = runtime_workspace_root.join("wbuild"); let crate_metadata = crate_metadata(project_cargo_toml); let project = create_project( project_cargo_toml, - &wasm_workspace, + &runtime_workspace, &crate_metadata, crate_metadata.workspace_root.as_ref(), features_to_enable, ); let build_config = BuildConfiguration::detect(&project); - build_blob(&build_config.blob_build_profile, &project, default_rustflags, cargo_cmd); - let (wasm_binary, wasm_binary_compressed, bloaty) = maybe_compact_wasm_and_copy_blobs( - &project, - &build_config, - project_cargo_toml, - wasm_binary_name, - ); + + // Build the bloaty runtime blob + build_bloaty_blob(&build_config.blob_build_profile, &project, default_rustflags, cargo_cmd); + + // Get the name of the bloaty runtime blob. + let default_bloaty_name = get_blob_name(project_cargo_toml); + let bloaty_blob_name = runtime_blob_name.unwrap_or_else(|| default_bloaty_name.clone()); + + // Try to compact and compress the bloaty blob, if the *outer* profile wants it. + // + // This is because, by default the inner profile will be set to `Release` even when the outer + // profile is `Debug`, because the blob built in `Debug` profile is too slow for normal + // development activities. + let (compact_blob_path, compact_compressed_blob_path) = + match build_config.outer_build_profile.wants_compact() { + true => { + let compact_blob_path = + compact_wasm(&project, project_cargo_toml, &bloaty_blob_name); + let compact_compressed_blob_path = compact_blob_path + .as_ref() + .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); + (compact_blob_path, compact_compressed_blob_path) + }, + false => { + println!("{}", colorize_info_message("Skipping wasm compaction and compression")); + (None, None) + }, + }; + + let bloaty_blob_binary = copy_bloaty_blob(&project, project_cargo_toml, &bloaty_blob_name); if check_for_runtime_version_section { - ensure_runtime_version_wasm_section_exists(bloaty.wasm_binary_bloaty_path()); + ensure_runtime_version_wasm_section_exists(bloaty_blob_binary.bloaty_path()); } - wasm_binary + compact_blob_path .as_ref() - .map(|wasm_binary| copy_wasm_to_target_directory(project_cargo_toml, wasm_binary)); + .map(|wasm_binary| copy_blob_to_target_directory(project_cargo_toml, wasm_binary)); - wasm_binary_compressed.as_ref().map(|wasm_binary_compressed| { - copy_wasm_to_target_directory(project_cargo_toml, wasm_binary_compressed) + compact_compressed_blob_path.as_ref().map(|wasm_binary_compressed| { + copy_blob_to_target_directory(project_cargo_toml, wasm_binary_compressed) }); - let final_wasm_binary = wasm_binary_compressed.or(wasm_binary); + let final_blob_binary = compact_compressed_blob_path.or(compact_blob_path); generate_rerun_if_changed_instructions( project_cargo_toml, &project, - &wasm_workspace, - final_wasm_binary.as_ref(), - &bloaty, + &runtime_workspace, + final_blob_binary.as_ref(), + &bloaty_blob_binary, ); - if let Err(err) = adjust_mtime(&bloaty, final_wasm_binary.as_ref()) { - build_helper::warning!("Error while adjusting the mtime of the wasm binaries: {}", err) + if let Err(err) = adjust_mtime(&bloaty_blob_binary, final_blob_binary.as_ref()) { + build_helper::warning!("Error while adjusting the mtime of the blob binaries: {}", err) } - (final_wasm_binary, bloaty) + (final_blob_binary, bloaty_blob_binary) } -/// Ensures that the `runtime_version` wasm section exists in the given wasm file. +/// Ensures that the `runtime_version` section exists in the given blob. /// /// If the section can not be found, it will print an error and exit the builder. -fn ensure_runtime_version_wasm_section_exists(wasm: &Path) { - let wasm_blob = fs::read(wasm).expect("`{wasm}` was just written and should exist; qed"); +fn ensure_runtime_version_wasm_section_exists(blob_path: &Path) { + let blob = fs::read(blob_path).expect("`{blob_path}` was just written and should exist; qed"); - let module: Module = match deserialize_buffer(&wasm_blob) { + let module: Module = match deserialize_buffer(&blob) { Ok(m) => m, Err(e) => { - println!("Failed to deserialize `{}`: {e:?}", wasm.display()); + println!("Failed to deserialize `{}`: {e:?}", blob_path.display()); process::exit(1); }, }; if !module.custom_sections().any(|cs| cs.name() == "runtime_version") { println!( - "Couldn't find the `runtime_version` wasm section. \ + "Couldn't find the `runtime_version` section. \ Please ensure that you are using the `sp_version::runtime_version` attribute macro!" ); process::exit(1); @@ -213,7 +236,7 @@ fn adjust_mtime( let metadata = fs::metadata(invoked_timestamp)?; let mtime = filetime::FileTime::from_last_modification_time(&metadata); - filetime::set_file_mtime(bloaty_wasm.wasm_binary_bloaty_path(), mtime)?; + filetime::set_file_mtime(bloaty_wasm.bloaty_path(), mtime)?; if let Some(binary) = compressed_or_compact_wasm.as_ref() { filetime::set_file_mtime(binary.wasm_binary_path(), mtime)?; } @@ -284,8 +307,8 @@ fn get_crate_name(cargo_manifest: &Path) -> String { .expect("Package name exists; qed") } -/// Returns the name for the wasm binary. -fn get_wasm_binary_name(cargo_manifest: &Path) -> String { +/// Returns the name for the blob binary. +fn get_blob_name(cargo_manifest: &Path) -> String { get_crate_name(cargo_manifest).replace('-', "_") } @@ -506,7 +529,7 @@ fn create_project( ) -> PathBuf { let crate_name = get_crate_name(project_cargo_toml); let crate_path = project_cargo_toml.parent().expect("Parent path exists; qed"); - let wasm_binary = get_wasm_binary_name(project_cargo_toml); + let wasm_binary = get_blob_name(project_cargo_toml); let wasm_project_folder = wasm_workspace.join(&crate_name); fs::create_dir_all(wasm_project_folder.join("src")) @@ -680,8 +703,8 @@ fn offline_build() -> bool { env::var(OFFLINE).map_or(false, |v| v == "true") } -/// Build the project and create the runtime blob. -fn build_blob( +/// Build the project and create the bloaty runtime blob. +fn build_bloaty_blob( blob_build_profile: &Profile, project: &Path, default_rustflags: &str, @@ -732,84 +755,65 @@ fn build_blob( } } -/// If the cargo build profile is not [`Profile::Debug`], spend the additional time to create -/// a compacted and compressed wasm binary. -/// -/// Then, copy the wasm binaries to the intended directory. -fn maybe_compact_wasm_and_copy_blobs( - project: &Path, - build_config: &BuildConfiguration, - cargo_manifest: &Path, - out_name: Option, -) -> (Option, Option, WasmBinaryBloaty) { - let default_out_name = get_wasm_binary_name(cargo_manifest); - let out_name = out_name.unwrap_or_else(|| default_out_name.clone()); +fn compact_wasm(project: &Path, cargo_manifest: &Path, out_name: &str) -> Option { + let default_out_name = get_blob_name(cargo_manifest); let in_path = project .join("target/wasm32-unknown-unknown") - .join(build_config.blob_build_profile.directory()) .join(format!("{}.wasm", default_out_name)); - // When cargo is running cargo in `--profile dev` the user wants speed, so skip producing - // compact and compressed blobs. - let (wasm_compact_path, wasm_compact_compressed_path) = if build_config - .outer_build_profile - .wants_compact() - { - let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name,)); - let start = std::time::Instant::now(); - wasm_opt::OptimizationOptions::new_opt_level_0() - .mvp_features_only() - .debug_info(true) - .add_pass(wasm_opt::Pass::StripDwarf) - .run(&in_path, &wasm_compact_path) - .expect("Failed to compact generated WASM binary."); - println!( - "{} {}", - colorize_info_message("Compacted wasm in"), - colorize_info_message(format!("{:?}", start.elapsed()).as_str()) - ); - - let wasm_compact_compressed_path = - project.join(format!("{}.compact.compressed.wasm", out_name)); - let start = std::time::Instant::now(); - if compress_wasm(&wasm_compact_path, &wasm_compact_compressed_path) { - println!( - "{} {}", - colorize_info_message("Compressed wasm in"), - colorize_info_message(format!("{:?}", start.elapsed()).as_str()) - ); - (Some(WasmBinary(wasm_compact_path)), Some(WasmBinary(wasm_compact_compressed_path))) - } else { - println!("{}", colorize_info_message("Skipping wasm compression")); - (Some(WasmBinary(wasm_compact_path)), None) - } - } else { - println!("{}", colorize_info_message("Skipping wasm compaction and compression")); - (None, None) - }; + let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name)); + let start = std::time::Instant::now(); + wasm_opt::OptimizationOptions::new_opt_level_0() + .mvp_features_only() + .debug_info(true) + .add_pass(wasm_opt::Pass::StripDwarf) + .run(&in_path, &wasm_compact_path) + .expect("Failed to compact generated WASM binary."); + println!( + "{} {}", + colorize_info_message("Compacted wasm in"), + colorize_info_message(format!("{:?}", start.elapsed()).as_str()) + ); + Some(WasmBinary(wasm_compact_path)) +} +fn copy_bloaty_blob(project: &Path, cargo_manifest: &Path, out_name: &str) -> WasmBinaryBloaty { + let default_out_name = get_blob_name(cargo_manifest); + let in_path = project + .join("target/wasm32-unknown-unknown") + .join(format!("{}.wasm", default_out_name)); let bloaty_path = project.join(format!("{}.wasm", out_name)); - fs::copy(in_path, &bloaty_path).expect("Copying the bloaty file to the project dir."); - (wasm_compact_path, wasm_compact_compressed_path, WasmBinaryBloaty(bloaty_path)) + fs::copy(in_path, &bloaty_path).expect("Copying the bloaty blob to the project dir."); + WasmBinaryBloaty(bloaty_path) } -fn compress_wasm(wasm_binary_path: &Path, compressed_binary_out_path: &Path) -> bool { +fn try_compress_blob(compact_blob_path: &Path, out_name: &str) -> Option { use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT; - let data = fs::read(wasm_binary_path).expect("Failed to read WASM binary"); + let project = compact_blob_path.parent().expect("blob path should have a parent directory"); + let compact_compressed_blob_path = + project.join(format!("{}.compact.compressed.wasm", out_name)); + + let start = std::time::Instant::now(); + let data = fs::read(compact_blob_path).expect("Failed to read WASM binary"); if let Some(compressed) = sp_maybe_compressed_blob::compress(&data, CODE_BLOB_BOMB_LIMIT) { - fs::write(compressed_binary_out_path, &compressed[..]) + fs::write(&compact_compressed_blob_path, &compressed[..]) .expect("Failed to write WASM binary"); - true + println!( + "{} {}", + colorize_info_message("Compressed blob in"), + colorize_info_message(format!("{:?}", start.elapsed()).as_str()) + ); + Some(WasmBinary(compact_compressed_blob_path)) } else { build_helper::warning!( - "Writing uncompressed wasm. Exceeded maximum size {}", + "Writing uncompressed blob. Exceeded maximum size {}", CODE_BLOB_BOMB_LIMIT, ); - - false + println!("{}", colorize_info_message("Skipping blob compression")); + None } } @@ -919,7 +923,7 @@ fn generate_rerun_if_changed_instructions( packages.iter().for_each(package_rerun_if_changed); compressed_or_compact_wasm.map(|w| rerun_if_changed(w.wasm_binary_path())); - rerun_if_changed(bloaty_wasm.wasm_binary_bloaty_path()); + rerun_if_changed(bloaty_wasm.bloaty_path()); // Register our env variables println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV); @@ -952,9 +956,9 @@ fn package_rerun_if_changed(package: &DeduplicatePackage) { .for_each(rerun_if_changed); } -/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY` environment +/// Copy the blob binary to the target directory set in `WASM_TARGET_DIRECTORY` environment /// variable. If the variable is not set, this is a no-op. -fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) { +fn copy_blob_to_target_directory(cargo_manifest: &Path, blob_binary: &WasmBinary) { let target_dir = match env::var(crate::WASM_TARGET_DIRECTORY) { Ok(path) => PathBuf::from(path), Err(_) => return, @@ -973,8 +977,8 @@ fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary fs::create_dir_all(&target_dir).expect("Creates `WASM_TARGET_DIRECTORY`."); fs::copy( - wasm_binary.wasm_binary_path(), - target_dir.join(format!("{}.wasm", get_wasm_binary_name(cargo_manifest))), + blob_binary.wasm_binary_path(), + target_dir.join(format!("{}.wasm", get_blob_name(cargo_manifest))), ) - .expect("Copies WASM binary to `WASM_TARGET_DIRECTORY`."); + .expect("Copies blob binary to `WASM_TARGET_DIRECTORY`."); } From 96456eb28edeb810d8e0f9eeb408b884cafa4242 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 12:15:24 +1100 Subject: [PATCH 08/12] fix paths --- .../utils/wasm-builder/src/wasm_project.rs | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index d83ea674ef49..6fce741ffc7e 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -116,7 +116,7 @@ pub(crate) fn create_and_compile( default_rustflags: &str, cargo_cmd: CargoCommandVersioned, features_to_enable: Vec, - runtime_blob_name: Option, + bloaty_blob_name: Option, check_for_runtime_version_section: bool, ) -> (Option, WasmBinaryBloaty) { let runtime_workspace_root = get_wasm_workspace_root(); @@ -138,8 +138,11 @@ pub(crate) fn create_and_compile( build_bloaty_blob(&build_config.blob_build_profile, &project, default_rustflags, cargo_cmd); // Get the name of the bloaty runtime blob. - let default_bloaty_name = get_blob_name(project_cargo_toml); - let bloaty_blob_name = runtime_blob_name.unwrap_or_else(|| default_bloaty_name.clone()); + let default_bloaty_blob_name = get_blob_name(project_cargo_toml); + let bloaty_blob_name = bloaty_blob_name.unwrap_or_else(|| default_bloaty_blob_name.clone()); + + let bloaty_blob_binary = + copy_bloaty_blob(&project, &build_config.blob_build_profile, &bloaty_blob_name); // Try to compact and compress the bloaty blob, if the *outer* profile wants it. // @@ -149,8 +152,12 @@ pub(crate) fn create_and_compile( let (compact_blob_path, compact_compressed_blob_path) = match build_config.outer_build_profile.wants_compact() { true => { - let compact_blob_path = - compact_wasm(&project, project_cargo_toml, &bloaty_blob_name); + let compact_blob_path = compact_wasm( + &project, + &build_config.blob_build_profile, + project_cargo_toml, + &bloaty_blob_name, + ); let compact_compressed_blob_path = compact_blob_path .as_ref() .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); @@ -162,8 +169,6 @@ pub(crate) fn create_and_compile( }, }; - let bloaty_blob_binary = copy_bloaty_blob(&project, project_cargo_toml, &bloaty_blob_name); - if check_for_runtime_version_section { ensure_runtime_version_wasm_section_exists(bloaty_blob_binary.bloaty_path()); } @@ -755,10 +760,16 @@ fn build_bloaty_blob( } } -fn compact_wasm(project: &Path, cargo_manifest: &Path, out_name: &str) -> Option { +fn compact_wasm( + project: &Path, + inner_profile: &Profile, + cargo_manifest: &Path, + out_name: &str, +) -> Option { let default_out_name = get_blob_name(cargo_manifest); let in_path = project .join("target/wasm32-unknown-unknown") + .join(inner_profile.directory()) .join(format!("{}.wasm", default_out_name)); let wasm_compact_path = project.join(format!("{}.compact.wasm", out_name)); @@ -777,14 +788,14 @@ fn compact_wasm(project: &Path, cargo_manifest: &Path, out_name: &str) -> Option Some(WasmBinary(wasm_compact_path)) } -fn copy_bloaty_blob(project: &Path, cargo_manifest: &Path, out_name: &str) -> WasmBinaryBloaty { - let default_out_name = get_blob_name(cargo_manifest); +fn copy_bloaty_blob(project: &Path, inner_profile: &Profile, out_name: &str) -> WasmBinaryBloaty { let in_path = project .join("target/wasm32-unknown-unknown") - .join(format!("{}.wasm", default_out_name)); - let bloaty_path = project.join(format!("{}.wasm", out_name)); + .join(inner_profile.directory()) + .join(format!("{}.wasm", out_name)); - fs::copy(in_path, &bloaty_path).expect("Copying the bloaty blob to the project dir."); + let bloaty_path = project.join(format!("{}.wasm", out_name)); + fs::copy(in_path, &bloaty_path).expect("Copying the bloaty file to the project dir."); WasmBinaryBloaty(bloaty_path) } From c1430e9b5b8eaa59eebb4be31dcdafe688715032 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 15:11:27 +1100 Subject: [PATCH 09/12] use if instead of match --- .../utils/wasm-builder/src/wasm_project.rs | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 6fce741ffc7e..c7e9598da214 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -150,23 +150,20 @@ pub(crate) fn create_and_compile( // profile is `Debug`, because the blob built in `Debug` profile is too slow for normal // development activities. let (compact_blob_path, compact_compressed_blob_path) = - match build_config.outer_build_profile.wants_compact() { - true => { - let compact_blob_path = compact_wasm( - &project, - &build_config.blob_build_profile, - project_cargo_toml, - &bloaty_blob_name, - ); - let compact_compressed_blob_path = compact_blob_path - .as_ref() - .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); - (compact_blob_path, compact_compressed_blob_path) - }, - false => { - println!("{}", colorize_info_message("Skipping wasm compaction and compression")); - (None, None) - }, + if build_config.outer_build_profile.wants_compact() { + let compact_blob_path = compact_wasm( + &project, + &build_config.blob_build_profile, + project_cargo_toml, + &bloaty_blob_name, + ); + let compact_compressed_blob_path = compact_blob_path + .as_ref() + .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); + (compact_blob_path, compact_compressed_blob_path) + } else { + println!("{}", colorize_info_message("Skipping wasm compaction and compression")); + (None, None) }; if check_for_runtime_version_section { From 7f55d20d84e9903e047bb537d19e3e2a1031e395 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 15:12:33 +1100 Subject: [PATCH 10/12] remove unnecesarry log --- substrate/utils/wasm-builder/src/wasm_project.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index c7e9598da214..49674225b796 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -162,7 +162,6 @@ pub(crate) fn create_and_compile( .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); (compact_blob_path, compact_compressed_blob_path) } else { - println!("{}", colorize_info_message("Skipping wasm compaction and compression")); (None, None) }; From b7a21a0255f3f39d6f1fc312b369a4fa1df72efa Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 16:11:32 +1100 Subject: [PATCH 11/12] fix out name override --- .../utils/wasm-builder/src/wasm_project.rs | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 49674225b796..e92276084fbc 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -116,7 +116,7 @@ pub(crate) fn create_and_compile( default_rustflags: &str, cargo_cmd: CargoCommandVersioned, features_to_enable: Vec, - bloaty_blob_name: Option, + bloaty_blob_out_name_override: Option, check_for_runtime_version_section: bool, ) -> (Option, WasmBinaryBloaty) { let runtime_workspace_root = get_wasm_workspace_root(); @@ -138,11 +138,16 @@ pub(crate) fn create_and_compile( build_bloaty_blob(&build_config.blob_build_profile, &project, default_rustflags, cargo_cmd); // Get the name of the bloaty runtime blob. - let default_bloaty_blob_name = get_blob_name(project_cargo_toml); - let bloaty_blob_name = bloaty_blob_name.unwrap_or_else(|| default_bloaty_blob_name.clone()); + let bloaty_blob_default_name = get_blob_name(project_cargo_toml); + let bloaty_blob_out_name = + bloaty_blob_out_name_override.unwrap_or_else(|| bloaty_blob_default_name.clone()); - let bloaty_blob_binary = - copy_bloaty_blob(&project, &build_config.blob_build_profile, &bloaty_blob_name); + let bloaty_blob_binary = copy_bloaty_blob( + &project, + &build_config.blob_build_profile, + &bloaty_blob_default_name, + &bloaty_blob_out_name, + ); // Try to compact and compress the bloaty blob, if the *outer* profile wants it. // @@ -155,11 +160,11 @@ pub(crate) fn create_and_compile( &project, &build_config.blob_build_profile, project_cargo_toml, - &bloaty_blob_name, + &bloaty_blob_out_name, ); let compact_compressed_blob_path = compact_blob_path .as_ref() - .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_name)); + .and_then(|p| try_compress_blob(&p.0, &bloaty_blob_out_name)); (compact_blob_path, compact_compressed_blob_path) } else { (None, None) @@ -784,12 +789,18 @@ fn compact_wasm( Some(WasmBinary(wasm_compact_path)) } -fn copy_bloaty_blob(project: &Path, inner_profile: &Profile, out_name: &str) -> WasmBinaryBloaty { +fn copy_bloaty_blob( + project: &Path, + inner_profile: &Profile, + in_name: &str, + out_name: &str, +) -> WasmBinaryBloaty { let in_path = project .join("target/wasm32-unknown-unknown") .join(inner_profile.directory()) - .join(format!("{}.wasm", out_name)); + .join(format!("{}.wasm", in_name)); + dbg!(&in_path); let bloaty_path = project.join(format!("{}.wasm", out_name)); fs::copy(in_path, &bloaty_path).expect("Copying the bloaty file to the project dir."); WasmBinaryBloaty(bloaty_path) From d5af3784e858b00f8e9ee4e9e2f68b9b07fac020 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Tue, 24 Oct 2023 16:45:30 +1100 Subject: [PATCH 12/12] remove dbg --- substrate/utils/wasm-builder/src/wasm_project.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index e92276084fbc..c41e0935d750 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -800,7 +800,6 @@ fn copy_bloaty_blob( .join(inner_profile.directory()) .join(format!("{}.wasm", in_name)); - dbg!(&in_path); let bloaty_path = project.join(format!("{}.wasm", out_name)); fs::copy(in_path, &bloaty_path).expect("Copying the bloaty file to the project dir."); WasmBinaryBloaty(bloaty_path)