From 6bd68fcceb1d0708ce898fb92275119e2658d64b Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Sun, 14 Apr 2024 11:35:01 +0200 Subject: [PATCH 01/25] Run filecheck on dest-prop/branch.rs --- tests/mir-opt/dest-prop/branch.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/mir-opt/dest-prop/branch.rs b/tests/mir-opt/dest-prop/branch.rs index cd55130728570..b1741bb2395c3 100644 --- a/tests/mir-opt/dest-prop/branch.rs +++ b/tests/mir-opt/dest-prop/branch.rs @@ -1,4 +1,3 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY //! Tests that assignment in both branches of an `if` are eliminated. //@ unit-test: DestinationPropagation @@ -12,6 +11,10 @@ fn cond() -> bool { // EMIT_MIR branch.foo.DestinationPropagation.diff fn foo() -> i32 { + // CHECK-LABEL: fn foo( + // CHECK: debug y => [[y:_.*]]; + // CHECK: [[y]] = val() + // CHECK-NOT: [[y]] = {{_.*}}; let x = val(); let y = if cond() { From f238eba6211635d80ff0c31e7a89e2d38e528cfc Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Sun, 14 Apr 2024 11:35:23 +0200 Subject: [PATCH 02/25] Run filecheck on dest-prop/copy_propagation.rs --- .../mir-opt/dest-prop/copy_propagation_arg.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/mir-opt/dest-prop/copy_propagation_arg.rs b/tests/mir-opt/dest-prop/copy_propagation_arg.rs index f84b5fde8d88b..2ea5c098ff40a 100644 --- a/tests/mir-opt/dest-prop/copy_propagation_arg.rs +++ b/tests/mir-opt/dest-prop/copy_propagation_arg.rs @@ -1,4 +1,3 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY // Check that DestinationPropagation does not propagate an assignment to a function argument // (doing so can break usages of the original argument value) @@ -9,18 +8,29 @@ fn dummy(x: u8) -> u8 { // EMIT_MIR copy_propagation_arg.foo.DestinationPropagation.diff fn foo(mut x: u8) { + // CHECK-LABEL: fn foo( + // CHECK: debug x => [[x:_.*]]; + // CHECK: dummy(move [[x]]) + // CHECK: [[x]] = move {{_.*}}; // calling `dummy` to make a use of `x` that copyprop cannot eliminate x = dummy(x); // this will assign a local to `x` } // EMIT_MIR copy_propagation_arg.bar.DestinationPropagation.diff fn bar(mut x: u8) { + // CHECK-LABEL: fn bar( + // CHECK: debug x => [[x:_.*]]; + // CHECK: dummy(move [[x]]) + // CHECK: [[x]] = const 5_u8; dummy(x); x = 5; } // EMIT_MIR copy_propagation_arg.baz.DestinationPropagation.diff fn baz(mut x: i32) -> i32 { + // CHECK-LABEL: fn baz( + // CHECK: debug x => [[x:_.*]]; + // CHECK-NOT: [[x]] = {{_.*}} // self-assignment to a function argument should be eliminated x = x; x @@ -28,6 +38,12 @@ fn baz(mut x: i32) -> i32 { // EMIT_MIR copy_propagation_arg.arg_src.DestinationPropagation.diff fn arg_src(mut x: i32) -> i32 { + // CHECK-LABEL: fn arg_src( + // CHECK: debug x => [[x:_.*]]; + // CHECK: debug y => [[y:_.*]]; + // CHECK: [[y]] = [[x]] + // CHECK: [[x]] = const 123_i32; + // CHECK-NOT: {{_.*}} = [[y]]; let y = x; x = 123; // Don't propagate this assignment to `y` y From 853311c35811ea8a05baf97aae895b017b83cc23 Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Sun, 14 Apr 2024 11:35:37 +0200 Subject: [PATCH 03/25] Run filecheck on dest-prop/cycle.rs --- tests/mir-opt/dest-prop/cycle.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/mir-opt/dest-prop/cycle.rs b/tests/mir-opt/dest-prop/cycle.rs index e6663956d7822..41d9dd81253fe 100644 --- a/tests/mir-opt/dest-prop/cycle.rs +++ b/tests/mir-opt/dest-prop/cycle.rs @@ -1,4 +1,3 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY //! Tests that cyclic assignments don't hang DestinationPropagation, and result in reasonable code. //@ unit-test: DestinationPropagation @@ -8,6 +7,10 @@ fn val() -> i32 { // EMIT_MIR cycle.main.DestinationPropagation.diff fn main() { + // CHECK-LABEL: main( + // CHECK: debug x => [[x:_.*]]; + // CHECK: [[x]] = val() + // CHECK-NOT: [[x]] = {{_.*}}; let mut x = val(); let y = x; let z = y; From 069209034dfdaf520dc14aa4f0bfab935da0032c Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Sun, 14 Apr 2024 11:35:55 +0200 Subject: [PATCH 04/25] Run filecheck on dest-prop/dead_stores_79191.rs and dead_stores_better.rs --- tests/mir-opt/dest-prop/dead_stores_79191.rs | 8 +++++++- tests/mir-opt/dest-prop/dead_stores_better.rs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/mir-opt/dest-prop/dead_stores_79191.rs b/tests/mir-opt/dest-prop/dead_stores_79191.rs index b3e370966d05d..85411b17569da 100644 --- a/tests/mir-opt/dest-prop/dead_stores_79191.rs +++ b/tests/mir-opt/dest-prop/dead_stores_79191.rs @@ -1,4 +1,3 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY //@ unit-test: DestinationPropagation @@ -8,6 +7,13 @@ fn id(x: T) -> T { // EMIT_MIR dead_stores_79191.f.DestinationPropagation.after.mir fn f(mut a: usize) -> usize { + // CHECK-LABEL: fn f( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[b]] = [[a]]; + // CHECK: [[a]] = const 5_usize; + // CHECK: [[a]] = move [[b]]; + // CHECK: id::(move [[a]]) let b = a; a = 5; a = b; diff --git a/tests/mir-opt/dest-prop/dead_stores_better.rs b/tests/mir-opt/dest-prop/dead_stores_better.rs index c241d71594bab..3d2af421bc237 100644 --- a/tests/mir-opt/dest-prop/dead_stores_better.rs +++ b/tests/mir-opt/dest-prop/dead_stores_better.rs @@ -1,4 +1,3 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY // This is a copy of the `dead_stores_79191` test, except that we turn on DSE. This demonstrates // that that pass enables this one to do more optimizations. @@ -12,6 +11,13 @@ fn id(x: T) -> T { // EMIT_MIR dead_stores_better.f.DestinationPropagation.after.mir pub fn f(mut a: usize) -> usize { + // CHECK-LABEL: fn f( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[b]] = [[a]]; + // CHECK: [[a]] = const 5_usize; + // CHECK: [[a]] = move [[b]]; + // CHECK: id::(move [[a]]) let b = a; a = 5; a = b; From f0f867e2d3d220ae8a11bfe83d2545ec1a3a19d2 Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Sun, 14 Apr 2024 11:36:24 +0200 Subject: [PATCH 05/25] Run filecheck on dest-prop/simple.rs --- tests/mir-opt/dest-prop/simple.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/mir-opt/dest-prop/simple.rs b/tests/mir-opt/dest-prop/simple.rs index 4aa6b6a4876e4..203f44a38261d 100644 --- a/tests/mir-opt/dest-prop/simple.rs +++ b/tests/mir-opt/dest-prop/simple.rs @@ -1,9 +1,12 @@ -// skip-filecheck // EMIT_MIR_FOR_EACH_PANIC_STRATEGY //! Copy of `nrvo-simple.rs`, to ensure that full dest-prop handles it too. //@ unit-test: DestinationPropagation // EMIT_MIR simple.nrvo.DestinationPropagation.diff fn nrvo(init: fn(&mut [u8; 1024])) -> [u8; 1024] { + // CHECK-LABEL: fn nrvo( + // CHECK: debug init => [[init:_.*]]; + // CHECK-NOT: {{_.*}} = [[init]]; + // CHECK: move [[init]](move {{_.*}}) let mut buf = [0; 1024]; init(&mut buf); buf From 2d5a4832f30d246b81c775aa5bf9003997568b65 Mon Sep 17 00:00:00 2001 From: CastilloDel Date: Wed, 26 Jun 2024 19:39:37 +0200 Subject: [PATCH 06/25] Acknowledge comments --- tests/mir-opt/dest-prop/copy_propagation_arg.rs | 2 +- tests/mir-opt/dest-prop/simple.rs | 3 +++ tests/mir-opt/dest-prop/union.rs | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/mir-opt/dest-prop/copy_propagation_arg.rs b/tests/mir-opt/dest-prop/copy_propagation_arg.rs index 2ea5c098ff40a..5d99920ebc467 100644 --- a/tests/mir-opt/dest-prop/copy_propagation_arg.rs +++ b/tests/mir-opt/dest-prop/copy_propagation_arg.rs @@ -30,7 +30,7 @@ fn bar(mut x: u8) { fn baz(mut x: i32) -> i32 { // CHECK-LABEL: fn baz( // CHECK: debug x => [[x:_.*]]; - // CHECK-NOT: [[x]] = {{_.*}} + // CHECK-NOT: [[x]] = // self-assignment to a function argument should be eliminated x = x; x diff --git a/tests/mir-opt/dest-prop/simple.rs b/tests/mir-opt/dest-prop/simple.rs index 203f44a38261d..a2c0e976b5fc2 100644 --- a/tests/mir-opt/dest-prop/simple.rs +++ b/tests/mir-opt/dest-prop/simple.rs @@ -5,8 +5,11 @@ fn nrvo(init: fn(&mut [u8; 1024])) -> [u8; 1024] { // CHECK-LABEL: fn nrvo( // CHECK: debug init => [[init:_.*]]; + // CHECK: debug buf => [[buf:_.*]]; + // CHECK: [[buf]] = [const 0_u8; 1024]; // CHECK-NOT: {{_.*}} = [[init]]; // CHECK: move [[init]](move {{_.*}}) + // CHECK: {{_.*}} = [[buf]] let mut buf = [0; 1024]; init(&mut buf); buf diff --git a/tests/mir-opt/dest-prop/union.rs b/tests/mir-opt/dest-prop/union.rs index abd1f1b2c937a..54c284e5b5529 100644 --- a/tests/mir-opt/dest-prop/union.rs +++ b/tests/mir-opt/dest-prop/union.rs @@ -8,6 +8,8 @@ fn val() -> u32 { // EMIT_MIR union.main.DestinationPropagation.diff fn main() { + // CHECK-LABEL: fn args( + // CHECK: {{_.*}} = Un { us: const 1_u32 }; union Un { us: u32, } From a5dc082d6f51b1df9dc0bf0daf365ccf933f46d5 Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Fri, 5 Jul 2024 15:43:12 +0000 Subject: [PATCH 07/25] Use windows_targets macro for alloc --- library/std/src/sys/pal/windows/alloc.rs | 129 +++++++++++------------ library/std/src/sys/pal/windows/c.rs | 2 +- 2 files changed, 64 insertions(+), 67 deletions(-) diff --git a/library/std/src/sys/pal/windows/alloc.rs b/library/std/src/sys/pal/windows/alloc.rs index 24c237b5eb03d..e6cbdb6ef7d64 100644 --- a/library/std/src/sys/pal/windows/alloc.rs +++ b/library/std/src/sys/pal/windows/alloc.rs @@ -4,7 +4,7 @@ use crate::alloc::{GlobalAlloc, Layout, System}; use crate::ffi::c_void; use crate::ptr; use crate::sync::atomic::{AtomicPtr, Ordering}; -use crate::sys::c; +use crate::sys::c::{self, windows_targets}; use crate::sys::common::alloc::{realloc_fallback, MIN_ALIGN}; use core::mem::MaybeUninit; @@ -17,74 +17,71 @@ mod tests; // Flag to indicate that the memory returned by `HeapAlloc` should be zeroed. const HEAP_ZERO_MEMORY: c::DWORD = 0x00000008; -#[link(name = "kernel32")] -extern "system" { - // Get a handle to the default heap of the current process, or null if the operation fails. - // - // SAFETY: Successful calls to this function within the same process are assumed to - // always return the same handle, which remains valid for the entire lifetime of the process. - // - // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-getprocessheap - fn GetProcessHeap() -> c::HANDLE; +// Get a handle to the default heap of the current process, or null if the operation fails. +// +// SAFETY: Successful calls to this function within the same process are assumed to +// always return the same handle, which remains valid for the entire lifetime of the process. +// +// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-getprocessheap +windows_targets::link!("kernel32.dll" "system" fn GetProcessHeap() -> c::HANDLE); - // Allocate a block of `dwBytes` bytes of memory from a given heap `hHeap`. - // The allocated memory may be uninitialized, or zeroed if `dwFlags` is - // set to `HEAP_ZERO_MEMORY`. - // - // Returns a pointer to the newly-allocated memory or null if the operation fails. - // The returned pointer will be aligned to at least `MIN_ALIGN`. - // - // SAFETY: - // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. - // - `dwFlags` must be set to either zero or `HEAP_ZERO_MEMORY`. - // - // Note that `dwBytes` is allowed to be zero, contrary to some other allocators. - // - // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapalloc - fn HeapAlloc(hHeap: c::HANDLE, dwFlags: c::DWORD, dwBytes: c::SIZE_T) -> c::LPVOID; +// Allocate a block of `dwBytes` bytes of memory from a given heap `hHeap`. +// The allocated memory may be uninitialized, or zeroed if `dwFlags` is +// set to `HEAP_ZERO_MEMORY`. +// +// Returns a pointer to the newly-allocated memory or null if the operation fails. +// The returned pointer will be aligned to at least `MIN_ALIGN`. +// +// SAFETY: +// - `hHeap` must be a non-null handle returned by `GetProcessHeap`. +// - `dwFlags` must be set to either zero or `HEAP_ZERO_MEMORY`. +// +// Note that `dwBytes` is allowed to be zero, contrary to some other allocators. +// +// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapalloc +windows_targets::link!("kernel32.dll" "system" fn HeapAlloc(hheap: c::HANDLE, dwflags: u32, dwbytes: usize) -> *mut core::ffi::c_void); - // Reallocate a block of memory behind a given pointer `lpMem` from a given heap `hHeap`, - // to a block of at least `dwBytes` bytes, either shrinking the block in place, - // or allocating at a new location, copying memory, and freeing the original location. - // - // Returns a pointer to the reallocated memory or null if the operation fails. - // The returned pointer will be aligned to at least `MIN_ALIGN`. - // If the operation fails the given block will never have been freed. - // - // SAFETY: - // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. - // - `dwFlags` must be set to zero. - // - `lpMem` must be a non-null pointer to an allocated block returned by `HeapAlloc` or - // `HeapReAlloc`, that has not already been freed. - // If the block was successfully reallocated at a new location, pointers pointing to - // the freed memory, such as `lpMem`, must not be dereferenced ever again. - // - // Note that `dwBytes` is allowed to be zero, contrary to some other allocators. - // - // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heaprealloc - fn HeapReAlloc( - hHeap: c::HANDLE, - dwFlags: c::DWORD, - lpMem: c::LPVOID, - dwBytes: c::SIZE_T, - ) -> c::LPVOID; +// Reallocate a block of memory behind a given pointer `lpMem` from a given heap `hHeap`, +// to a block of at least `dwBytes` bytes, either shrinking the block in place, +// or allocating at a new location, copying memory, and freeing the original location. +// +// Returns a pointer to the reallocated memory or null if the operation fails. +// The returned pointer will be aligned to at least `MIN_ALIGN`. +// If the operation fails the given block will never have been freed. +// +// SAFETY: +// - `hHeap` must be a non-null handle returned by `GetProcessHeap`. +// - `dwFlags` must be set to zero. +// - `lpMem` must be a non-null pointer to an allocated block returned by `HeapAlloc` or +// `HeapReAlloc`, that has not already been freed. +// If the block was successfully reallocated at a new location, pointers pointing to +// the freed memory, such as `lpMem`, must not be dereferenced ever again. +// +// Note that `dwBytes` is allowed to be zero, contrary to some other allocators. +// +// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heaprealloc +windows_targets::link!("kernel32.dll" "system" fn HeapReAlloc( + hheap: c::HANDLE, + dwflags : u32, + lpmem: *const core::ffi::c_void, + dwbytes: usize +) -> *mut core::ffi::c_void); - // Free a block of memory behind a given pointer `lpMem` from a given heap `hHeap`. - // Returns a nonzero value if the operation is successful, and zero if the operation fails. - // - // SAFETY: - // - `hHeap` must be a non-null handle returned by `GetProcessHeap`. - // - `dwFlags` must be set to zero. - // - `lpMem` must be a pointer to an allocated block returned by `HeapAlloc` or `HeapReAlloc`, - // that has not already been freed. - // If the block was successfully freed, pointers pointing to the freed memory, such as `lpMem`, - // must not be dereferenced ever again. - // - // Note that `lpMem` is allowed to be null, which will not cause the operation to fail. - // - // See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapfree - fn HeapFree(hHeap: c::HANDLE, dwFlags: c::DWORD, lpMem: c::LPVOID) -> c::BOOL; -} +// Free a block of memory behind a given pointer `lpMem` from a given heap `hHeap`. +// Returns a nonzero value if the operation is successful, and zero if the operation fails. +// +// SAFETY: +// - `hHeap` must be a non-null handle returned by `GetProcessHeap`. +// - `dwFlags` must be set to zero. +// - `lpMem` must be a pointer to an allocated block returned by `HeapAlloc` or `HeapReAlloc`, +// that has not already been freed. +// If the block was successfully freed, pointers pointing to the freed memory, such as `lpMem`, +// must not be dereferenced ever again. +// +// Note that `lpMem` is allowed to be null, which will not cause the operation to fail. +// +// See https://docs.microsoft.com/windows/win32/api/heapapi/nf-heapapi-heapfree +windows_targets::link!("kernel32.dll" "system" fn HeapFree(hheap: c::HANDLE, dwflags: u32, lpmem: *const core::ffi::c_void) -> c::BOOL); // Cached handle to the default heap of the current process. // Either a non-null handle returned by `GetProcessHeap`, or null when not yet initialized or `GetProcessHeap` failed. diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index 27aa35f69f1bf..7dfda4f714c77 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -13,7 +13,7 @@ use crate::os::raw::{c_char, c_long, c_longlong, c_uint, c_ulong, c_ushort, c_vo use crate::os::windows::io::{AsRawHandle, BorrowedHandle}; use crate::ptr; -mod windows_targets; +pub(super) mod windows_targets; mod windows_sys; pub use windows_sys::*; From e136f08a6f4f784d1a1751aced5bf919bb094436 Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Fri, 5 Jul 2024 14:04:26 +0000 Subject: [PATCH 08/25] Add experimental raw-dylib feature to std For Windows, this allows defining imports without needing the user to have import libraries. It's intended for this to become the default. --- library/std/Cargo.toml | 4 ++++ .../std/src/sys/pal/windows/c/windows_targets.rs | 13 +++++++++++++ library/sysroot/Cargo.toml | 1 + 3 files changed, 18 insertions(+) diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 358510b8f77d8..b991b1cf22dd8 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -87,6 +87,10 @@ std_detect_file_io = ["std_detect/std_detect_file_io"] std_detect_dlsym_getauxval = ["std_detect/std_detect_dlsym_getauxval"] std_detect_env_override = ["std_detect/std_detect_env_override"] +# Enable using raw-dylib for Windows imports. +# This will eventually be the default. +windows_raw_dylib = [] + [package.metadata.fortanix-sgx] # Maximum possible number of threads when testing threads = 125 diff --git a/library/std/src/sys/pal/windows/c/windows_targets.rs b/library/std/src/sys/pal/windows/c/windows_targets.rs index 56c563462d366..252bceb70942b 100644 --- a/library/std/src/sys/pal/windows/c/windows_targets.rs +++ b/library/std/src/sys/pal/windows/c/windows_targets.rs @@ -3,6 +3,18 @@ //! This is a simple wrapper around an `extern` block with a `#[link]` attribute. //! It's very roughly equivalent to the windows-targets crate. +#[cfg(feature = "windows_raw_dylib")] +pub macro link { + ($library:literal $abi:literal $($link_name:literal)? $(#[$doc:meta])? fn $($function:tt)*) => ( + #[cfg_attr(not(target_arch = "x86"), link(name = $library, kind = "raw-dylib", modifiers = "+verbatim"))] + #[cfg_attr(target_arch = "x86", link(name = $library, kind = "raw-dylib", modifiers = "+verbatim", import_name_type = "undecorated"))] + extern $abi { + $(#[link_name=$link_name])? + pub fn $($function)*; + } + ) +} +#[cfg(not(feature = "windows_raw_dylib"))] pub macro link { ($library:literal $abi:literal $($link_name:literal)? $(#[$doc:meta])? fn $($function:tt)*) => ( // Note: the windows-targets crate uses a pre-built Windows.lib import library which we don't @@ -17,6 +29,7 @@ pub macro link { ) } +#[cfg(not(feature = "windows_raw_dylib"))] #[link(name = "advapi32")] #[link(name = "ntdll")] #[link(name = "userenv")] diff --git a/library/sysroot/Cargo.toml b/library/sysroot/Cargo.toml index 1ddacd92e6b94..169eeeca8c2e8 100644 --- a/library/sysroot/Cargo.toml +++ b/library/sysroot/Cargo.toml @@ -27,3 +27,4 @@ profiler = ["std/profiler"] std_detect_file_io = ["std/std_detect_file_io"] std_detect_dlsym_getauxval = ["std/std_detect_dlsym_getauxval"] std_detect_env_override = ["std/std_detect_env_override"] +windows_raw_dylib = ["std/windows_raw_dylib"] From 48192701e0385e2f045da79d34bd5e856cf32496 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 7 Jul 2024 00:07:08 +0300 Subject: [PATCH 09/25] use "bootstrap" instead of "rustbuild" in comments and docs Signed-off-by: onur-ozkan --- INSTALL.md | 2 +- compiler/rustc_codegen_ssa/src/back/linker.rs | 2 +- compiler/rustc_session/src/config.rs | 2 +- config.example.toml | 8 ++++---- src/bootstrap/README.md | 18 +++++++++--------- src/bootstrap/bootstrap.py | 2 +- src/bootstrap/mk/Makefile.in | 2 +- src/bootstrap/src/bin/main.rs | 2 +- src/bootstrap/src/core/build_steps/compile.rs | 6 +++--- src/bootstrap/src/core/build_steps/doc.rs | 2 +- src/bootstrap/src/core/build_steps/test.rs | 2 +- src/bootstrap/src/core/builder.rs | 8 ++++---- src/bootstrap/src/core/builder/tests.rs | 2 +- src/bootstrap/src/core/config/config.rs | 5 ++--- src/bootstrap/src/core/config/flags.rs | 2 +- src/bootstrap/src/core/sanity.rs | 2 +- src/bootstrap/src/lib.rs | 4 ++-- src/bootstrap/src/utils/helpers.rs | 2 +- src/ci/scripts/install-clang.sh | 2 +- .../x86_64-fortanix-unknown-sgx.md | 2 +- src/tools/build_helper/src/ci.rs | 2 +- src/tools/lint-docs/src/groups.rs | 2 +- src/tools/lint-docs/src/lib.rs | 2 +- .../crates/rust-analyzer/src/version.rs | 2 +- src/tools/rust-installer/src/combiner.rs | 2 +- src/tools/rust-installer/test.sh | 2 +- 26 files changed, 44 insertions(+), 45 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9619ec2ce5cfa..f660cfb0d7c38 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -215,7 +215,7 @@ python x.py build Right now, building Rust only works with some known versions of Visual Studio. If you have a more recent version installed and the build system doesn't -understand, you may need to force rustbuild to use an older version. +understand, you may need to force bootstrap to use an older version. This can be done by manually calling the appropriate vcvars file before running the bootstrap. diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 2bd5dfdce83ee..dd134ebbe6b18 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -398,7 +398,7 @@ impl<'a> GccLinker<'a> { self.link_arg("-dylib"); // Note that the `osx_rpath_install_name` option here is a hack - // purely to support rustbuild right now, we should get a more + // purely to support bootstrap right now, we should get a more // principled solution at some point to force the compiler to pass // the right `-Wl,-install_name` with an `@rpath` in it. if self.sess.opts.cg.rpath || self.sess.opts.unstable_opts.osx_rpath_install_name { diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 41c99f7edeeff..e748d1ff47b63 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2620,7 +2620,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M // This is the location used by the `rust-src` `rustup` component. let mut candidate = sysroot.join("lib/rustlib/src/rust"); if let Ok(metadata) = candidate.symlink_metadata() { - // Replace the symlink rustbuild creates, with its destination. + // Replace the symlink bootstrap creates, with its destination. // We could try to use `fs::canonicalize` instead, but that might // produce unnecessarily verbose path. if metadata.file_type().is_symlink() { diff --git a/config.example.toml b/config.example.toml index 679abcdc7771b..68c632d91cdec 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,6 +1,6 @@ # Sample TOML configuration file for building Rust. # -# To configure rustbuild, run `./configure` or `./x.py setup`. +# To configure bootstrap, run `./configure` or `./x.py setup`. # See https://rustc-dev-guide.rust-lang.org/building/how-to-build-and-run.html#create-a-configtoml for more information. # # All options are commented out by default in this file, and they're commented @@ -109,7 +109,7 @@ # increases the size of binaries and consequently the memory required by # each linker process. # If set to 0, linker invocations are treated like any other job and -# controlled by rustbuild's -j parameter. +# controlled by bootstrap's -j parameter. #link-jobs = 0 # Whether to build LLVM as a dynamically linked library (as opposed to statically linked). @@ -371,11 +371,11 @@ # Useful for modifying only the stage2 compiler without having to pass `--keep-stage 0` each time. #local-rebuild = false -# Print out how long each rustbuild step took (mostly intended for CI and +# Print out how long each bootstrap step took (mostly intended for CI and # tracking over time) #print-step-timings = false -# Print out resource usage data for each rustbuild step, as defined by the Unix +# Print out resource usage data for each bootstrap step, as defined by the Unix # struct rusage. (Note that this setting is completely unstable: the data it # captures, what platforms it supports, the format of its associated output, and # this setting's very existence, are all subject to change.) diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index fb3c862704306..0ac58645d2dfc 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -1,7 +1,7 @@ -# rustbuild - Bootstrapping Rust +# Bootstrapping Rust This README is aimed at helping to explain how Rust is bootstrapped, -and some of the technical details of the build system. +and some of the technical details of the bootstrap build system. Note that this README only covers internal information, not how to use the tool. Please check [bootstrapping dev guide][bootstrapping-dev-guide] for further information. @@ -12,17 +12,17 @@ Please check [bootstrapping dev guide][bootstrapping-dev-guide] for further info The build system defers most of the complicated logic of managing invocations of rustc and rustdoc to Cargo itself. However, moving through various stages -and copying artifacts is still necessary for it to do. Each time rustbuild +and copying artifacts is still necessary for it to do. Each time bootstrap is invoked, it will iterate through the list of predefined steps and execute each serially in turn if it matches the paths passed or is a default rule. -For each step, rustbuild relies on the step internally being incremental and -parallel. Note, though, that the `-j` parameter to rustbuild gets forwarded +For each step, bootstrap relies on the step internally being incremental and +parallel. Note, though, that the `-j` parameter to bootstrap gets forwarded to appropriate test harnesses and such. ## Build phases -The rustbuild build system goes through a few phases to actually build the -compiler. What actually happens when you invoke rustbuild is: +Bootstrap build system goes through a few phases to actually build the +compiler. What actually happens when you invoke bootstrap is: 1. The entry point script (`x` for unix like systems, `x.ps1` for windows systems, `x.py` cross-platform) is run. This script is responsible for downloading the stage0 @@ -151,9 +151,9 @@ build/ stage3/ ``` -## Extending rustbuild +## Extending bootstrap -When you use the bootstrap system, you'll call it through the entry point script +When you use bootstrap, you'll call it through the entry point script (`x`, `x.ps1`, or `x.py`). However, most of the code lives in `src/bootstrap`. `bootstrap` has a difficult problem: it is written in Rust, but yet it is run before the Rust compiler is built! To work around this, there are two components diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 7e47b373ff9df..4e8e0fd2532f1 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -1038,7 +1038,7 @@ def build_triple(self): def check_vendored_status(self): """Check that vendoring is configured properly""" - # keep this consistent with the equivalent check in rustbuild: + # keep this consistent with the equivalent check in bootstrap: # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if os.getuid() == 0: diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index cab37e0da4736..f85aefcb268d3 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -20,7 +20,7 @@ all: $(Q)$(BOOTSTRAP) doc --stage 2 $(BOOTSTRAP_ARGS) help: - $(Q)echo 'Welcome to the rustbuild build system!' + $(Q)echo 'Welcome to bootstrap, the Rust build system!' $(Q)echo $(Q)echo This makefile is a thin veneer over the ./x.py script located $(Q)echo in this directory. To get the full power of the build system diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs index 44fb1911fc6d3..a7d21ba6ae127 100644 --- a/src/bootstrap/src/bin/main.rs +++ b/src/bootstrap/src/bin/main.rs @@ -1,4 +1,4 @@ -//! rustbuild, the Rust build system +//! bootstrap, the Rust build system //! //! This is the entry point for the build system used to compile the `rustc` //! compiler. Lots of documentation can be found in the `README.md` file in the diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 01b25224de4cd..9d79baf45e5c5 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1,7 +1,7 @@ //! Implementation of compiling various phases of the compiler and standard //! library. //! -//! This module contains some of the real meat in the rustbuild build system +//! This module contains some of the real meat in the bootstrap build system //! which is where Cargo is used to compile the standard library, libtest, and //! the compiler. This module is also responsible for assembling the sysroot as it //! goes along from the output of the previous stage. @@ -819,8 +819,8 @@ pub struct Rustc { pub compiler: Compiler, /// Whether to build a subset of crates, rather than the whole compiler. /// - /// This should only be requested by the user, not used within rustbuild itself. - /// Using it within rustbuild can lead to confusing situation where lints are replayed + /// This should only be requested by the user, not used within bootstrap itself. + /// Using it within bootstrap can lead to confusing situation where lints are replayed /// in two different steps. crates: Vec, } diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 823e842693e96..38d4f0bed4ccd 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -1,4 +1,4 @@ -//! Documentation generation for rustbuilder. +//! Documentation generation for bootstrap. //! //! This module implements generation for all bits and pieces of documentation //! for the Rust project. This notably includes suites like the rust book, the diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index dc53bd3cfa785..5ea711ebcd602 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -2993,7 +2993,7 @@ impl Step for Bootstrap { // https://github.com/rust-lang/rust/issues/49215 cmd.env("RUSTFLAGS", flags); } - // rustbuild tests are racy on directory creation so just run them one at a time. + // bootstrap tests are racy on directory creation so just run them one at a time. // Since there's not many this shouldn't be a problem. run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); } diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 4da912994c3ee..0ea28728e0d45 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -1627,11 +1627,11 @@ impl<'a> Builder<'a> { } // This tells Cargo (and in turn, rustc) to output more complete - // dependency information. Most importantly for rustbuild, this + // dependency information. Most importantly for bootstrap, this // includes sysroot artifacts, like libstd, which means that we don't - // need to track those in rustbuild (an error prone process!). This + // need to track those in bootstrap (an error prone process!). This // feature is currently unstable as there may be some bugs and such, but - // it represents a big improvement in rustbuild's reliability on + // it represents a big improvement in bootstrap's reliability on // rebuilds, so we're using it here. // // For some additional context, see #63470 (the PR originally adding @@ -1643,7 +1643,7 @@ impl<'a> Builder<'a> { // Restrict the allowed features so we don't depend on nightly // accidentally. // - // binary-dep-depinfo is used by rustbuild itself for all + // binary-dep-depinfo is used by bootstrap itself for all // compilations. // // Lots of tools depend on proc_macro2 and proc-macro-error. diff --git a/src/bootstrap/src/core/builder/tests.rs b/src/bootstrap/src/core/builder/tests.rs index aa119b8c69912..97c9ece0036ea 100644 --- a/src/bootstrap/src/core/builder/tests.rs +++ b/src/bootstrap/src/core/builder/tests.rs @@ -266,7 +266,7 @@ mod defaults { // rustdoc/rustcc/std here (the user only requested a host=B build, so // there's not really a need for us to build for target A in this case // (since we're producing stage 1 libraries/binaries). But currently - // rustbuild is just a bit buggy here; this should be fixed though. + // bootstrap is just a bit buggy here; this should be fixed though. assert_eq!( first(cache.all::()), &[ diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index 10ac6c93e9a43..508f7350b51c7 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -658,8 +658,7 @@ impl Merge for TomlConfig { } } -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of -// rustbuild. +// We are using a decl macro instead of a derive proc macro here to reduce the compile time of bootstrap. macro_rules! define_config { ($(#[$attr:meta])* struct $name:ident { $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* @@ -704,7 +703,7 @@ macro_rules! define_config { // The following is a trimmed version of what serde_derive generates. All parts not relevant // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of rustbuild. + // compile time of bootstrap. impl<'de> Deserialize<'de> for $name { fn deserialize(deserializer: D) -> Result where diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs index aeb608a9ea26b..19f752da81c13 100644 --- a/src/bootstrap/src/core/config/flags.rs +++ b/src/bootstrap/src/core/config/flags.rs @@ -1,4 +1,4 @@ -//! Command-line interface of the rustbuild build system. +//! Command-line interface of the bootstrap build system. //! //! This module implements the command-line parsing of the build system which //! has various flags to configure how it's run. diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 78862ccb8cd68..c70f879568d14 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -1,4 +1,4 @@ -//! Sanity checking performed by rustbuild before actually executing anything. +//! Sanity checking performed by bootstrap before actually executing anything. //! //! This module contains the implementation of ensuring that the build //! environment looks reasonable before progressing. This will verify that diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index ffdd9bc885a62..e08eb4e62bb9a 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1,9 +1,9 @@ -//! Implementation of rustbuild, the Rust build system. +//! Implementation of bootstrap, the Rust build system. //! //! This module, and its descendants, are the implementation of the Rust build //! system. Most of this build system is backed by Cargo but the outer layer //! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: +//! builds, building artifacts like LLVM, etc. The goals of bootstrap are: //! //! * To be an easily understandable, easily extensible, and maintainable build //! system. diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 2575b7939b44c..a52b4bbf98867 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -1,4 +1,4 @@ -//! Various utility functions used throughout rustbuild. +//! Various utility functions used throughout bootstrap. //! //! Simple things like testing the various filesystem operations here and there, //! not a lot of interesting happenings here unfortunately. diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 24b9904d65c25..6103aa61248ae 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -40,7 +40,7 @@ if isMacOS; then # our own clang can figure out the correct include path on its own. ciCommandSetEnv SDKROOT "$(xcrun --sdk macosx --show-sdk-path)" - # Configure `AR` specifically so rustbuild doesn't try to infer it as + # Configure `AR` specifically so bootstrap doesn't try to infer it as # `clang-ar` by accident. ciCommandSetEnv AR "ar" elif isWindows && ! isKnownToBeMingwBuild; then diff --git a/src/doc/rustc/src/platform-support/x86_64-fortanix-unknown-sgx.md b/src/doc/rustc/src/platform-support/x86_64-fortanix-unknown-sgx.md index 97b5827c1443f..33e1c44e6d351 100644 --- a/src/doc/rustc/src/platform-support/x86_64-fortanix-unknown-sgx.md +++ b/src/doc/rustc/src/platform-support/x86_64-fortanix-unknown-sgx.md @@ -46,7 +46,7 @@ on how to setup a development and runtime environment. As a tier 2 target, the target is built by the Rust project. -You can configure rustbuild like so: +You can configure bootstrap like so: ```toml [build] diff --git a/src/tools/build_helper/src/ci.rs b/src/tools/build_helper/src/ci.rs index 09489b0d9b7da..233fed4151c19 100644 --- a/src/tools/build_helper/src/ci.rs +++ b/src/tools/build_helper/src/ci.rs @@ -25,7 +25,7 @@ impl CiEnv { /// If in a CI environment, forces the command to run with colors. pub fn force_coloring_in_ci(self, cmd: &mut Command) { if self != CiEnv::None { - // Due to use of stamp/docker, the output stream of rustbuild is not + // Due to use of stamp/docker, the output stream of bootstrap is not // a TTY in CI, so coloring is by-default turned off. // The explicit `TERM=xterm` environment is needed for // `--color always` to actually work. This env var was lost when diff --git a/src/tools/lint-docs/src/groups.rs b/src/tools/lint-docs/src/groups.rs index f246d71d499b4..9eaa234bfaf30 100644 --- a/src/tools/lint-docs/src/groups.rs +++ b/src/tools/lint-docs/src/groups.rs @@ -37,7 +37,7 @@ impl<'a> LintExtractor<'a> { .map_err(|e| format!("could not read {}: {}", groups_path.display(), e))?; let new_contents = contents.replace("{{groups-table}}", &self.make_groups_table(lints, &groups)?); - // Delete the output because rustbuild uses hard links in its copies. + // Delete the output because bootstrap uses hard links in its copies. let _ = fs::remove_file(&groups_path); fs::write(&groups_path, new_contents) .map_err(|e| format!("could not write to {}: {}", groups_path.display(), e))?; diff --git a/src/tools/lint-docs/src/lib.rs b/src/tools/lint-docs/src/lib.rs index 72d6a495e7e79..48e2dd098769e 100644 --- a/src/tools/lint-docs/src/lib.rs +++ b/src/tools/lint-docs/src/lib.rs @@ -532,7 +532,7 @@ impl<'a> LintExtractor<'a> { } add_rename_redirect(level, &mut result); let out_path = self.out_path.join("listing").join(level.doc_filename()); - // Delete the output because rustbuild uses hard links in its copies. + // Delete the output because bootstrap uses hard links in its copies. let _ = fs::remove_file(&out_path); fs::write(&out_path, result) .map_err(|e| format!("could not write to {}: {}", out_path.display(), e))?; diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/version.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/version.rs index 1e829299e6f3c..2ff967416c006 100644 --- a/src/tools/rust-analyzer/crates/rust-analyzer/src/version.rs +++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/version.rs @@ -15,7 +15,7 @@ pub struct VersionInfo { pub version: &'static str, /// The release channel we were built for (stable/beta/nightly/dev). /// - /// `None` if not built via rustbuild. + /// `None` if not built via bootstrap. pub release_channel: Option<&'static str>, /// Information about the Git repository we may have been built from. /// diff --git a/src/tools/rust-installer/src/combiner.rs b/src/tools/rust-installer/src/combiner.rs index c211b34850a07..e7020980dc3d6 100644 --- a/src/tools/rust-installer/src/combiner.rs +++ b/src/tools/rust-installer/src/combiner.rs @@ -109,7 +109,7 @@ impl Combiner { .with_context(|| format!("failed to read components in '{}'", input_tarball))?; for component in pkg_components.split_whitespace() { // All we need to do is copy the component directory. We could - // move it, but rustbuild wants to reuse the unpacked package + // move it, but bootstrap wants to reuse the unpacked package // dir for OS-specific installers on macOS and Windows. let component_dir = package_dir.join(component); create_dir(&component_dir)?; diff --git a/src/tools/rust-installer/test.sh b/src/tools/rust-installer/test.sh index 16b05c66197c8..7b3e745600629 100755 --- a/src/tools/rust-installer/test.sh +++ b/src/tools/rust-installer/test.sh @@ -974,7 +974,7 @@ combined_remains() { --package-name=rust \ --input-tarballs="$OUT_DIR/rustc.tar.gz,$OUT_DIR/cargo.tar.gz,$OUT_DIR/rust-docs.tar.gz" for component in rustc cargo rust-docs; do - # rustbuild wants the original extracted package intact too + # bootstrap wants the original extracted package intact too try test -d "$WORK_DIR/$component/$component" try test -d "$WORK_DIR/rust/$component" done From 99721c84691b3ee7339d307c92a762414527143d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 16:53:10 +1000 Subject: [PATCH 10/25] Clear `inner_attr_ranges` regularly. There's a comment saying we don't do it for performance reasons, but it doesn't actually affect performance. The commit also tweaks the control flow, to make clearer that two code paths are mutually exclusive. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 38f18022e3c58..e9aece13b2e68 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -352,15 +352,10 @@ impl<'a> Parser<'a> { let target = AttrsTarget { attrs: final_attrs.iter().cloned().collect(), tokens }; self.capture_state.replace_ranges.push((start_pos..end_pos, Some(target))); self.capture_state.replace_ranges.extend(inner_attr_replace_ranges); - } - - // Only clear our `replace_ranges` when we're finished capturing entirely. - if matches!(self.capture_state.capturing, Capturing::No) { + } else if matches!(self.capture_state.capturing, Capturing::No) { + // Only clear the ranges once we've finished capturing entirely. self.capture_state.replace_ranges.clear(); - // We don't clear `inner_attr_ranges`, as doing so repeatedly - // had a measurable performance impact. Most inner attributes that - // we insert will get removed - when we drop the parser, we'll free - // up the memory used by any attributes that we didn't remove from the map. + self.capture_state.inner_attr_ranges.clear(); } Ok(ret) } From a47ae57a187eb9537fe22a50692088cc655f357f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 14:47:12 +1000 Subject: [PATCH 11/25] Use an `@` pattern to shorten some code. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 38f18022e3c58..a627bc89aa02f 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -325,12 +325,9 @@ impl<'a> Parser<'a> { replace_ranges, }); - // If we support tokens at all - if let Some(target_tokens) = ret.tokens_mut() { - if target_tokens.is_none() { - // Store our newly captured tokens into the AST node. - *target_tokens = Some(tokens.clone()); - } + // If we support tokens and don't already have them, store the newly captured tokens. + if let Some(target_tokens @ None) = ret.tokens_mut() { + *target_tokens = Some(tokens.clone()); } let final_attrs = ret.attrs(); From b16201317eefa9054d545dc5237b1bc830448258 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 17:41:05 +1000 Subject: [PATCH 12/25] Use iterator normally in `make_attr_token_stream`. In a `for` loop, instead of a `while` loop. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a627bc89aa02f..16261feca91d0 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -367,7 +367,7 @@ impl<'a> Parser<'a> { /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// close delims. fn make_attr_token_stream( - mut iter: impl Iterator, + iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { #[derive(Debug)] @@ -377,8 +377,7 @@ fn make_attr_token_stream( inner: Vec, } let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; - let mut token_and_spacing = iter.next(); - while let Some((token, spacing)) = token_and_spacing { + for (token, spacing) in iter { match token { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { stack @@ -416,7 +415,6 @@ fn make_attr_token_stream( .push(AttrTokenTree::AttrsTarget(target)), FlatToken::Empty => {} } - token_and_spacing = iter.next(); } let mut final_buf = stack.pop().expect("Missing final buf!"); if break_last_token { From a88c4d67d9ea8a9f450638f207be867fa984bfce Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 18:05:18 +1000 Subject: [PATCH 13/25] Split the stack in `make_attr_token_stream`. It makes for shorter code, and fewer allocations. --- .../rustc_parse/src/parser/attr_wrapper.rs | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 16261feca91d0..5a4259eb1e76f 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -376,18 +376,19 @@ fn make_attr_token_stream( open_delim_sp: Option<(Delimiter, Span, Spacing)>, inner: Vec, } - let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; + // The stack always has at least one element. Storing it separately makes for shorter code. + let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; + let mut stack_rest = vec![]; for (token, spacing) in iter { match token { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { - stack - .push(FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }); + stack_rest.push(mem::replace( + &mut stack_top, + FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, + )); } FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { - let frame_data = stack - .pop() - .unwrap_or_else(|| panic!("Token stack was empty for token: {token:?}")); - + let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); assert_eq!( open_delim, delim, @@ -397,28 +398,18 @@ fn make_attr_token_stream( let dspacing = DelimSpacing::new(open_spacing, spacing); let stream = AttrTokenStream::new(frame_data.inner); let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); - stack - .last_mut() - .unwrap_or_else(|| panic!("Bottom token frame is missing for token: {token:?}")) - .inner - .push(delimited); + stack_top.inner.push(delimited); + } + FlatToken::Token(token) => stack_top.inner.push(AttrTokenTree::Token(token, spacing)), + FlatToken::AttrsTarget(target) => { + stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) } - FlatToken::Token(token) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::Token(token, spacing)), - FlatToken::AttrsTarget(target) => stack - .last_mut() - .expect("Bottom token frame is missing!") - .inner - .push(AttrTokenTree::AttrsTarget(target)), FlatToken::Empty => {} } } - let mut final_buf = stack.pop().expect("Missing final buf!"); + if break_last_token { - let last_token = final_buf.inner.pop().unwrap(); + let last_token = stack_top.inner.pop().unwrap(); if let AttrTokenTree::Token(last_token, spacing) = last_token { let unglued_first = last_token.kind.break_two_token_op().unwrap().0; @@ -426,14 +417,14 @@ fn make_attr_token_stream( let mut first_span = last_token.span.shrink_to_lo(); first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); - final_buf + stack_top .inner .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing)); } else { panic!("Unexpected last token {last_token:?}") } } - AttrTokenStream::new(final_buf.inner) + AttrTokenStream::new(stack_top.inner) } // Some types are used a lot. Make sure they don't unintentionally get bigger. From f5527949f2e819c056551a25d8fab7008facf6da Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 8 Jul 2024 19:32:21 +1000 Subject: [PATCH 14/25] Move `Spacing` into `FlatToken`. It's only needed for the `FlatToken::Token` variant. This makes things a little more concise. --- .../rustc_parse/src/parser/attr_wrapper.rs | 28 ++++++++----------- compiler/rustc_parse/src/parser/mod.rs | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 5a4259eb1e76f..67d2424bb48a3 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -103,11 +103,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // produce an empty `TokenStream` if no calls were made, and omit the // final token otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain(iter::repeat_with(|| { - let token = cursor_snapshot.next(); - (FlatToken::Token(token.0), token.1) - })) + let tokens = iter::once(FlatToken::Token(self.start_token.clone())) + .chain(iter::repeat_with(|| FlatToken::Token(cursor_snapshot.next()))) .take(self.num_calls as usize); if self.replace_ranges.is_empty() { @@ -156,11 +153,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { (range.start as usize)..(range.end as usize), target .into_iter() - .map(|target| (FlatToken::AttrsTarget(target), Spacing::Alone)) - .chain( - iter::repeat((FlatToken::Empty, Spacing::Alone)) - .take(range.len() - target_len), - ), + .map(|target| FlatToken::AttrsTarget(target)) + .chain(iter::repeat(FlatToken::Empty).take(range.len() - target_len)), ); } make_attr_token_stream(tokens.into_iter(), self.break_last_token) @@ -367,7 +361,7 @@ impl<'a> Parser<'a> { /// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and /// close delims. fn make_attr_token_stream( - iter: impl Iterator, + iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { #[derive(Debug)] @@ -379,15 +373,15 @@ fn make_attr_token_stream( // The stack always has at least one element. Storing it separately makes for shorter code. let mut stack_top = FrameData { open_delim_sp: None, inner: vec![] }; let mut stack_rest = vec![]; - for (token, spacing) in iter { - match token { - FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { + for flat_token in iter { + match flat_token { + FlatToken::Token((Token { kind: TokenKind::OpenDelim(delim), span }, spacing)) => { stack_rest.push(mem::replace( &mut stack_top, FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] }, )); } - FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { + FlatToken::Token((Token { kind: TokenKind::CloseDelim(delim), span }, spacing)) => { let frame_data = mem::replace(&mut stack_top, stack_rest.pop().unwrap()); let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap(); assert_eq!( @@ -400,7 +394,9 @@ fn make_attr_token_stream( let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream); stack_top.inner.push(delimited); } - FlatToken::Token(token) => stack_top.inner.push(AttrTokenTree::Token(token, spacing)), + FlatToken::Token((token, spacing)) => { + stack_top.inner.push(AttrTokenTree::Token(token, spacing)) + } FlatToken::AttrsTarget(target) => { stack_top.inner.push(AttrTokenTree::AttrsTarget(target)) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 45ca267fe5d1e..1482c665a16b7 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -1607,7 +1607,7 @@ pub(crate) fn make_unclosed_delims_error( enum FlatToken { /// A token - this holds both delimiter (e.g. '{' and '}') /// and non-delimiter tokens - Token(Token), + Token((Token, Spacing)), /// Holds the `AttrsTarget` for an AST node. The `AttrsTarget` is inserted /// directly into the constructed `AttrTokenStream` as an /// `AttrTokenTree::AttrsTarget`. From 2a3e22bf04b41c6116cb1e29ab7d8daeed8e63db Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 24 Jun 2024 12:30:44 -0700 Subject: [PATCH 15/25] Promote the `wasm32-wasip2` target to Tier 2 This commit promotes the `wasm32-wasip2` Rust target to tier 2 as proposed in rust-lang/compiler-team#760. There are two major changes in this PR: 1. The `dist-various-2` container, which already produces the other WASI targets, now has an extra target added for `wasm32-wasip2`. 2. A new `wasm-component-ld` binary is added to all host toolchains when LLD is enabled. This is the linker used for the `wasm32-wasip2` target. This new linker is added for all host toolchains to ensure that all host toolchains can produce the `wasm32-wasip2` target. This is similar to how `rust-lld` was originally included for all host toolchains to be able to produce WebAssembly output when the targets were first added. The new linker is developed [here][wasm-component-ld] and is pulled in via a crates.io-based dependency to the tree here. [wasm-component-ld]: https://github.com/bytecodealliance/wasm-component-ld --- Cargo.lock | 157 +++++++++++++++++- Cargo.toml | 4 + src/bootstrap/src/core/build_steps/compile.rs | 15 ++ src/bootstrap/src/core/build_steps/tool.rs | 28 ++++ .../host-x86_64/dist-various-2/Dockerfile | 1 + src/tools/tidy/src/deps.rs | 5 + src/tools/wasm-component-ld/Cargo.toml | 13 ++ src/tools/wasm-component-ld/README.md | 62 +++++++ src/tools/wasm-component-ld/src/main.rs | 9 + 9 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 src/tools/wasm-component-ld/Cargo.toml create mode 100644 src/tools/wasm-component-ld/README.md create mode 100644 src/tools/wasm-component-ld/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index afeb9faec097e..f426c0e370402 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1671,6 +1671,7 @@ dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", "rustc-std-workspace-core", + "serde", ] [[package]] @@ -1880,6 +1881,12 @@ dependencies = [ "syn 2.0.67", ] +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + [[package]] name = "ident_case" version = "1.0.1" @@ -2109,6 +2116,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" +[[package]] +name = "lexopt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" + [[package]] name = "libc" version = "0.2.155" @@ -2623,7 +2636,7 @@ dependencies = [ "indexmap", "memchr", "ruzstd 0.5.0", - "wasmparser", + "wasmparser 0.118.2", ] [[package]] @@ -3424,7 +3437,7 @@ dependencies = [ "object 0.34.0", "regex", "similar", - "wasmparser", + "wasmparser 0.118.2", ] [[package]] @@ -3805,7 +3818,7 @@ dependencies = [ "thin-vec", "thorin-dwp", "tracing", - "wasm-encoder", + "wasm-encoder 0.200.0", "windows", ] @@ -5248,6 +5261,15 @@ dependencies = [ "color-eyre", ] +[[package]] +name = "spdx" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47317bbaf63785b53861e1ae2d11b80d6b624211d42cb20efcd210ee6f8a14bc" +dependencies = [ + "smallvec", +] + [[package]] name = "spdx-expression" version = "0.5.2" @@ -6296,6 +6318,28 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm-component-ld" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "314d932d5e84c9678751b85498b1482b2f32f185744e449d3ce0b1d400376dad" +dependencies = [ + "anyhow", + "clap", + "lexopt", + "tempfile", + "wasmparser 0.210.0", + "wat", + "wit-component", +] + +[[package]] +name = "wasm-component-ld-wrapper" +version = "0.1.0" +dependencies = [ + "wasm-component-ld", +] + [[package]] name = "wasm-encoder" version = "0.200.0" @@ -6305,6 +6349,40 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-encoder" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e3764d9d6edabd8c9e16195e177be0d20f6ab942ad18af52860f12f82bc59a" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.211.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7d931a1120ef357f32b74547646b6fa68ea25e377772b72874b131a9ed70d4" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "012729d1294907fcb0866f08460ab95426a6d0b176a599619b84cac7653452b4" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.210.0", + "wasmparser 0.210.0", +] + [[package]] name = "wasmparser" version = "0.118.2" @@ -6315,6 +6393,42 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7bbcd21e7581619d9f6ca00f8c4f08f1cacfe58bf63f83af57cd0476f1026f5" +dependencies = [ + "ahash", + "bitflags 2.5.0", + "hashbrown", + "indexmap", + "semver", + "serde", +] + +[[package]] +name = "wast" +version = "211.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25506dd82d00da6b14a87436b3d52b1d264083fa79cdb72a0d1b04a8595ccaa" +dependencies = [ + "bumpalo", + "leb128", + "memchr", + "unicode-width", + "wasm-encoder 0.211.1", +] + +[[package]] +name = "wat" +version = "1.211.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb716ca6c86eecac2d82541ffc39860118fc0af9309c4f2670637bea2e1bdd7d" +dependencies = [ + "wast", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6542,6 +6656,43 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-component" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a450bdb5d032acf1fa0865451fa0c6f50e62f2d31eaa8dba967c2e2d068694a4" +dependencies = [ + "anyhow", + "bitflags 2.5.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.210.0", + "wasm-metadata", + "wasmparser 0.210.0", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.210.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a965cbd439af19a4b44a54a97ab8957d86f02d01320efc9e31c1d3605c6710" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.210.0", +] + [[package]] name = "writeable" version = "0.5.5" diff --git a/Cargo.toml b/Cargo.toml index 93c520b0d689d..ce87a8c20b77d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "src/tools/opt-dist", "src/tools/coverage-dump", "src/tools/rustc-perf-wrapper", + "src/tools/wasm-component-ld", ] exclude = [ @@ -104,6 +105,9 @@ rustc-demangle.debug = 0 [profile.release.package.lld-wrapper] debug = 0 strip = true +[profile.release.package.wasm-component-ld-wrapper] +debug = 0 +strip = true [patch.crates-io] # See comments in `library/rustc-std-workspace-core/README.md` for what's going on diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 11a7a40453516..175ef52b33fce 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1829,6 +1829,21 @@ impl Step for Assemble { &self_contained_lld_dir.join(exe(name, target_compiler.host)), ); } + + // In addition to `rust-lld` also install `wasm-component-ld` when + // LLD is enabled. This is a relatively small binary that primarily + // delegates to the `rust-lld` binary for linking and then runs + // logic to create the final binary. This is used by the + // `wasm32-wasip2` target of Rust. + let wasm_component_ld_exe = + builder.ensure(crate::core::build_steps::tool::WasmComponentLd { + compiler: build_compiler, + target: target_compiler.host, + }); + builder.copy_link( + &wasm_component_ld_exe, + &libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), + ); } if builder.config.llvm_enabled(target_compiler.host) { diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index ad92a01bce7f3..e19a05371a7f2 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -701,6 +701,34 @@ impl Step for LldWrapper { } } +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct WasmComponentLd { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for WasmComponentLd { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "wasm-component-ld", + mode: Mode::ToolStd, + path: "src/tools/wasm-component-ld", + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + cargo_args: Vec::new(), + }) + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustAnalyzer { pub compiler: Compiler, diff --git a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile index e3cb396b78297..962484593b4be 100644 --- a/src/ci/docker/host-x86_64/dist-various-2/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-2/Dockerfile @@ -112,6 +112,7 @@ ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,wasm32-wasi ENV TARGETS=$TARGETS,wasm32-wasip1 ENV TARGETS=$TARGETS,wasm32-wasip1-threads +ENV TARGETS=$TARGETS,wasm32-wasip2 ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,x86_64-pc-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 82fa43f581fde..871855df7bebe 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -95,7 +95,12 @@ const EXCEPTIONS: ExceptionList = &[ ("self_cell", "Apache-2.0"), // rustc (fluent translations) ("snap", "BSD-3-Clause"), // rustc ("wasm-encoder", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wasm-metadata", "Apache-2.0 WITH LLVM-exception"), // rustc ("wasmparser", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wast", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wat", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wit-component", "Apache-2.0 WITH LLVM-exception"), // rustc + ("wit-parser", "Apache-2.0 WITH LLVM-exception"), // rustc // tidy-alphabetical-end ]; diff --git a/src/tools/wasm-component-ld/Cargo.toml b/src/tools/wasm-component-ld/Cargo.toml new file mode 100644 index 0000000000000..91ff19ad9fcb0 --- /dev/null +++ b/src/tools/wasm-component-ld/Cargo.toml @@ -0,0 +1,13 @@ +# See the `README.md` in this directory for what this tool is. + +[package] +name = "wasm-component-ld-wrapper" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "wasm-component-ld" +path = "src/main.rs" + +[dependencies] +wasm-component-ld = "0.5.4" diff --git a/src/tools/wasm-component-ld/README.md b/src/tools/wasm-component-ld/README.md new file mode 100644 index 0000000000000..54608a2dea1d4 --- /dev/null +++ b/src/tools/wasm-component-ld/README.md @@ -0,0 +1,62 @@ +# `wasm-component-ld` + +This wrapper is a wrapper around the [`wasm-component-ld`] crates.io crate. That +crate. That crate is itself a thin wrapper around two pieces: + +* `wasm-ld` - the LLVM-based linker distributed as part of LLD and packaged in + Rust as `rust-lld`. +* [`wit-component`] - a Rust crate for creating a [WebAssembly Component] from a + core wasm module. + +This linker is used for Rust's `wasm32-wasip2` target to natively output a +component instead of a core WebAssembly module, unlike other WebAssembly +targets. If you're confused about any of this here's an FAQ-style explanation of +what's going on here: + +* **What's a component?** - It's a proposal to the WebAssembly standard + primarily developed at this time by out-of-browser use cases of WebAssembly. + You can find high-level documentation [here][component docs]. + +* **What's WASIp2?** - Not to be confused with WASIp1, WASIp0, + `wasi_snapshot_preview1`, or `wasi_unstable`, it's a version of WASI. Released + in January 2024 it's the first version of WASI defined in terms of the + component model. + +* **Why does this need its own linker?** - like any target that Rust has the + `wasm32-wasip2` target needs a linker. What makes this different from other + WebAssembly targets is that WASIp2 is defined at the component level, not core + WebAssembly level. This means that filesystem functions take a `string` + instead of `i32 i32`, for example. This means that the raw output of LLVM and + `wasm-ld`, a core WebAssembly module, is not suitable. + +* **Isn't writing a linker really hard?** - Generally, yes, but this linker + works by first asking `wasm-ld` to do all the hard work. It invokes `wasm-ld` + and then uses the output core WebAssembly module to create a component. + +* **How do you create a component from a core module?** - this is the purpose of + the [`wit-component`] crate, notably the `ComponentEncoder` type. This uses + component type information embedded in the core module and a general set of + conventions/guidelines with what the core module imports/exports. A component + is then hooked up to codify all of these conventions in a component itself. + +* **Why not require users to run `wit-component` themselves?** - while possible + it adds friction to the usage `wasm32-wasip2` target. More importantly though + the "module only" output of the `wasm32-wasip2` target is not ready right now. + The standard library still imports from `wasi_snapshot_preview1` and it will + take time to migrate all usage to WASIp2. + +* **What exactly does this linker do?** - the `wasm-component-ld` has the same + CLI interface and flags as `wasm-ld`, plus some more that are + component-specific. These flags are used to forward most flags to `wasm-ld` to + produce a core wasm module. After the core wasm module is produced the + `wit-component` crate will read custom sections in the final binary which + contain component type information. After merging all this type information + together a component is produced which wraps the core module. + +If you've got any other questions about this linker or its operation don't +hesitate to reach out to the maintainers of the `wasm32-wasip2` target. + +[`wasm-component-ld`]: https://crates.io/crates/wasm-component-ld +[`wit-component`]: https://crates.io/crates/wit-component +[WebAssembly Component]: https://github.com/webassembly/component-model +[component docs]: https://component-model.bytecodealliance.org/ diff --git a/src/tools/wasm-component-ld/src/main.rs b/src/tools/wasm-component-ld/src/main.rs new file mode 100644 index 0000000000000..caaac651c4c7b --- /dev/null +++ b/src/tools/wasm-component-ld/src/main.rs @@ -0,0 +1,9 @@ +// See the `README.md` in this directory for what this tool is. + +// The source for this crate lives at +// https://github.com/bytecodealliance/wasm-component-ld and the binary is +// independently used in other projects such as `wasi-sdk` so the `main` +// function is just reexported here to delegate. A Cargo dependency is used to +// facilitate version management in the Rust repository and work well with +// vendored/offline builds. +use wasm_component_ld::main; From 1afdd45881d39f6af434fba77ae07e68e1b62bce Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 9 Jul 2024 10:26:33 -0700 Subject: [PATCH 16/25] Update how wasm-component-ld is built Reuse preexisting macro and switch it to a "bootstrap tool" to try to resolve build issues. --- src/bootstrap/src/core/build_steps/tool.rs | 29 +--------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index e19a05371a7f2..d62166d8f0472 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -337,6 +337,7 @@ bootstrap_tool!( RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; CoverageDump, "src/tools/coverage-dump", "coverage-dump"; RustcPerfWrapper, "src/tools/rustc-perf-wrapper", "rustc-perf-wrapper"; + WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld"; ); #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -701,34 +702,6 @@ impl Step for LldWrapper { } } -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct WasmComponentLd { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for WasmComponentLd { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "wasm-component-ld", - mode: Mode::ToolStd, - path: "src/tools/wasm-component-ld", - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - cargo_args: Vec::new(), - }) - } -} - #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct RustAnalyzer { pub compiler: Compiler, From 4cd6eee894ec87de26b8dc212363c23f1db70c62 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 9 Jul 2024 13:02:51 -0700 Subject: [PATCH 17/25] Unconditionally use stage0 build compiler --- src/bootstrap/src/core/build_steps/compile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 175ef52b33fce..5650fea1b8124 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1837,7 +1837,7 @@ impl Step for Assemble { // `wasm32-wasip2` target of Rust. let wasm_component_ld_exe = builder.ensure(crate::core::build_steps::tool::WasmComponentLd { - compiler: build_compiler, + compiler: build_compiler.with_stage(0), target: target_compiler.host, }); builder.copy_link( From 8a390bae06dbb5686b581797b8f46cb0353dc255 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 14:41:39 +1000 Subject: [PATCH 18/25] Change empty replace range condition. The new condition is equivalent in practice, but it's much more obvious that it would result in an empty range, because the condition lines up with the contents of the iterator. --- .../rustc_parse/src/parser/attr_wrapper.rs | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 67d2424bb48a3..e90568f177c26 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -295,21 +295,22 @@ impl<'a> Parser<'a> { let num_calls = end_pos - start_pos; - // If we have no attributes, then we will never need to - // use any replace ranges. - let replace_ranges: Box<[ReplaceRange]> = if ret.attrs().is_empty() && !self.capture_cfg { - Box::new([]) - } else { - // Grab any replace ranges that occur *inside* the current AST node. - // We will perform the actual replacement when we convert the `LazyAttrTokenStream` - // to an `AttrTokenStream`. - self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] - .iter() - .cloned() - .chain(inner_attr_replace_ranges.iter().cloned()) - .map(|(range, data)| ((range.start - start_pos)..(range.end - start_pos), data)) - .collect() - }; + // This is hot enough for `deep-vector` that checking the conditions for an empty iterator + // is measurably faster than actually executing the iterator. + let replace_ranges: Box<[ReplaceRange]> = + if replace_ranges_start == replace_ranges_end && inner_attr_replace_ranges.is_empty() { + Box::new([]) + } else { + // Grab any replace ranges that occur *inside* the current AST node. + // We will perform the actual replacement when we convert the `LazyAttrTokenStream` + // to an `AttrTokenStream`. + self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] + .iter() + .cloned() + .chain(inner_attr_replace_ranges.iter().cloned()) + .map(|(range, data)| ((range.start - start_pos)..(range.end - start_pos), data)) + .collect() + }; let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl { start_token, From fee152556fda2260a17fa24cea21d18b57c8460d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 14:51:41 +1000 Subject: [PATCH 19/25] Rework `Attribute::get_tokens`. Returning `Vec` works better for the call sites than returning `TokenStream`. --- compiler/rustc_ast/src/attr/mod.rs | 23 ++++++++++------------- compiler/rustc_ast/src/tokenstream.rs | 9 +++++---- compiler/rustc_expand/src/config.rs | 4 +--- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 088ae9ba44102..d2c7b1c0753da 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -202,21 +202,18 @@ impl Attribute { } } - // Named `get_tokens` to distinguish it from the `::tokens` method. - pub fn get_tokens(&self) -> TokenStream { - match &self.kind { - AttrKind::Normal(normal) => TokenStream::new( - normal - .tokens - .as_ref() - .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) - .to_attr_token_stream() - .to_token_trees(), - ), - &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone( + pub fn token_trees(&self) -> Vec { + match self.kind { + AttrKind::Normal(ref normal) => normal + .tokens + .as_ref() + .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) + .to_attr_token_stream() + .to_token_trees(), + AttrKind::DocComment(comment_kind, data) => vec![TokenTree::token_alone( token::DocComment(comment_kind, self.style, data), self.span, - ), + )], } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index ee068f19332a5..1f2adde2570d7 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -225,11 +225,12 @@ impl AttrTokenStream { // properly implemented - we always synthesize fake tokens, // so we never reach this code. - let mut stream = TokenStream::default(); + let mut tts = vec![]; for inner_attr in inner_attrs { - stream.push_stream(inner_attr.get_tokens()); + tts.extend(inner_attr.token_trees()); } - stream.push_stream(delim_tokens.clone()); + tts.extend(delim_tokens.0.iter().cloned()); + let stream = TokenStream::new(tts); *tree = TokenTree::Delimited(*span, *spacing, *delim, stream); found = true; break; @@ -242,7 +243,7 @@ impl AttrTokenStream { ); } for attr in outer_attrs { - res.extend(attr.get_tokens().0.iter().cloned()); + res.extend(attr.token_trees()); } res.extend(target_tokens); } diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 40e16b4511575..8d796500dcc67 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -292,8 +292,6 @@ impl<'a> StripUnconfigured<'a> { attr: &Attribute, (item, item_span): (ast::AttrItem, Span), ) -> Attribute { - let orig_tokens = attr.get_tokens(); - // We are taking an attribute of the form `#[cfg_attr(pred, attr)]` // and producing an attribute of the form `#[attr]`. We // have captured tokens for `attr` itself, but we need to @@ -302,7 +300,7 @@ impl<'a> StripUnconfigured<'a> { // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // for `attr` when we expand it to `#[attr]` - let mut orig_trees = orig_tokens.trees(); + let mut orig_trees = attr.token_trees().into_iter(); let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap().clone() else { From d8b6aa6d0dabc0c102f16f9f9bb35f687a63101c Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 15:11:57 +1000 Subject: [PATCH 20/25] Use `cfg_attr` as a name more. In various functions where the attribute being processed is known to be a `#[cfg_attr(...)]` attribute. I find this a helpful reminder. --- compiler/rustc_expand/src/config.rs | 22 +++++++++++----------- compiler/rustc_parse/src/lib.rs | 6 +++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 8d796500dcc67..13414f990fba9 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -253,9 +253,9 @@ impl<'a> StripUnconfigured<'a> { /// Gives a compiler warning when the `cfg_attr` contains no attributes and /// is in the original source file. Gives a compiler error if the syntax of /// the attribute is incorrect. - pub(crate) fn expand_cfg_attr(&self, attr: &Attribute, recursive: bool) -> Vec { + pub(crate) fn expand_cfg_attr(&self, cfg_attr: &Attribute, recursive: bool) -> Vec { let Some((cfg_predicate, expanded_attrs)) = - rustc_parse::parse_cfg_attr(attr, &self.sess.psess) + rustc_parse::parse_cfg_attr(cfg_attr, &self.sess.psess) else { return vec![]; }; @@ -264,7 +264,7 @@ impl<'a> StripUnconfigured<'a> { if expanded_attrs.is_empty() { self.sess.psess.buffer_lint( rustc_lint_defs::builtin::UNUSED_ATTRIBUTES, - attr.span, + cfg_attr.span, ast::CRATE_NODE_ID, BuiltinLintDiag::CfgAttrNoAttributes, ); @@ -280,16 +280,16 @@ impl<'a> StripUnconfigured<'a> { // `#[cfg_attr(false, cfg_attr(true, some_attr))]`. expanded_attrs .into_iter() - .flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(attr, item))) + .flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(cfg_attr, item))) .collect() } else { - expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(attr, item)).collect() + expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(cfg_attr, item)).collect() } } fn expand_cfg_attr_item( &self, - attr: &Attribute, + cfg_attr: &Attribute, (item, item_span): (ast::AttrItem, Span), ) -> Attribute { // We are taking an attribute of the form `#[cfg_attr(pred, attr)]` @@ -300,11 +300,11 @@ impl<'a> StripUnconfigured<'a> { // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // for `attr` when we expand it to `#[attr]` - let mut orig_trees = attr.token_trees().into_iter(); + let mut orig_trees = cfg_attr.token_trees().into_iter(); let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap().clone() else { - panic!("Bad tokens for attribute {attr:?}"); + panic!("Bad tokens for attribute {cfg_attr:?}"); }; // We don't really have a good span to use for the synthesized `[]` @@ -318,12 +318,12 @@ impl<'a> StripUnconfigured<'a> { .unwrap_or_else(|| panic!("Missing tokens for {item:?}")) .to_attr_token_stream(), ); - let trees = if attr.style == AttrStyle::Inner { + let trees = if cfg_attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap().clone() else { - panic!("Bad tokens for attribute {attr:?}"); + panic!("Bad tokens for attribute {cfg_attr:?}"); }; vec![ AttrTokenTree::Token(pound_token, Spacing::Joint), @@ -338,7 +338,7 @@ impl<'a> StripUnconfigured<'a> { &self.sess.psess.attr_id_generator, item, tokens, - attr.style, + cfg_attr.style, item_span, ); if attr.has_name(sym::crate_type) { diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index 5522127be83ef..4454747ea0212 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -157,14 +157,14 @@ pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> Tok } pub fn parse_cfg_attr( - attr: &Attribute, + cfg_attr: &Attribute, psess: &ParseSess, ) -> Option<(MetaItem, Vec<(AttrItem, Span)>)> { const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]"; const CFG_ATTR_NOTE_REF: &str = "for more information, visit \ "; - match attr.get_normal_item().args { + match cfg_attr.get_normal_item().args { ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens }) if !tokens.is_empty() => { @@ -180,7 +180,7 @@ pub fn parse_cfg_attr( } _ => { psess.dcx().emit_err(errors::MalformedCfgAttr { - span: attr.span, + span: cfg_attr.span, sugg: CFG_ATTR_GRAMMAR_HELP, }); } From d6ebbbfcb203b969f9b6e02998efd9badbe885d6 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 15:47:02 +1000 Subject: [PATCH 21/25] Factor out `AttrsTarget` flattening code. This commit does the following. - Pulls the code out of `AttrTokenStream::to_token_trees` into a new function `attrs_and_tokens_to_token_trees`. - Simplifies `TokenStream::from_ast` by calling the new function. This is nicer than the old way, which created a temporary `AttrTokenStream` containing a single `AttrsTarget` (which required some cloning) just to call `to_token_trees` on it. (It is good to remove this use of `AttrsTarget` which isn't related to `cfg_attr` expansion.) --- compiler/rustc_ast/src/tokenstream.rs | 132 +++++++++++++------------- compiler/rustc_expand/src/config.rs | 5 +- 2 files changed, 72 insertions(+), 65 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 1f2adde2570d7..1f69a7dc35666 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -16,7 +16,7 @@ use crate::ast::{AttrStyle, StmtKind}; use crate::ast_traits::{HasAttrs, HasTokens}; use crate::token::{self, Delimiter, Nonterminal, Token, TokenKind}; -use crate::AttrVec; +use crate::{AttrVec, Attribute}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{self, Lrc}; @@ -179,11 +179,10 @@ impl AttrTokenStream { AttrTokenStream(Lrc::new(tokens)) } - /// Converts this `AttrTokenStream` to a plain `Vec`. - /// During conversion, `AttrTokenTree::AttrsTarget` get 'flattened' - /// back to a `TokenStream` of the form `outer_attr attr_target`. - /// If there are inner attributes, they are inserted into the proper - /// place in the attribute target tokens. + /// Converts this `AttrTokenStream` to a plain `Vec`. During + /// conversion, any `AttrTokenTree::AttrsTarget` gets "flattened" back to a + /// `TokenStream`, as described in the comment on + /// `attrs_and_tokens_to_token_trees`. pub fn to_token_trees(&self) -> Vec { let mut res = Vec::with_capacity(self.0.len()); for tree in self.0.iter() { @@ -200,52 +199,7 @@ impl AttrTokenStream { )) } AttrTokenTree::AttrsTarget(target) => { - let idx = target - .attrs - .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); - let (outer_attrs, inner_attrs) = target.attrs.split_at(idx); - - let mut target_tokens = target.tokens.to_attr_token_stream().to_token_trees(); - if !inner_attrs.is_empty() { - let mut found = false; - // Check the last two trees (to account for a trailing semi) - for tree in target_tokens.iter_mut().rev().take(2) { - if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree { - // Inner attributes are only supported on extern blocks, functions, - // impls, and modules. All of these have their inner attributes - // placed at the beginning of the rightmost outermost braced group: - // e.g. fn foo() { #![my_attr] } - // - // Therefore, we can insert them back into the right location - // without needing to do any extra position tracking. - // - // Note: Outline modules are an exception - they can - // have attributes like `#![my_attr]` at the start of a file. - // Support for custom attributes in this position is not - // properly implemented - we always synthesize fake tokens, - // so we never reach this code. - - let mut tts = vec![]; - for inner_attr in inner_attrs { - tts.extend(inner_attr.token_trees()); - } - tts.extend(delim_tokens.0.iter().cloned()); - let stream = TokenStream::new(tts); - *tree = TokenTree::Delimited(*span, *spacing, *delim, stream); - found = true; - break; - } - } - - assert!( - found, - "Failed to find trailing delimited group in: {target_tokens:?}" - ); - } - for attr in outer_attrs { - res.extend(attr.token_trees()); - } - res.extend(target_tokens); + attrs_and_tokens_to_token_trees(&target.attrs, &target.tokens, &mut res); } } } @@ -253,6 +207,64 @@ impl AttrTokenStream { } } +// Converts multiple attributes and the tokens for a target AST node into token trees, and appends +// them to `res`. +// +// Example: if the AST node is "fn f() { blah(); }", then: +// - Simple if no attributes are present, e.g. "fn f() { blah(); }" +// - Simple if only outer attribute are present, e.g. "#[outer1] #[outer2] fn f() { blah(); }" +// - Trickier if inner attributes are present, because they must be moved within the AST node's +// tokens, e.g. "#[outer] fn f() { #![inner] blah() }" +fn attrs_and_tokens_to_token_trees( + attrs: &[Attribute], + target_tokens: &LazyAttrTokenStream, + res: &mut Vec, +) { + let idx = attrs.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); + let (outer_attrs, inner_attrs) = attrs.split_at(idx); + + // Add outer attribute tokens. + for attr in outer_attrs { + res.extend(attr.token_trees()); + } + + // Add target AST node tokens. + res.extend(target_tokens.to_attr_token_stream().to_token_trees()); + + // Insert inner attribute tokens. + if !inner_attrs.is_empty() { + let mut found = false; + // Check the last two trees (to account for a trailing semi) + for tree in res.iter_mut().rev().take(2) { + if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree { + // Inner attributes are only supported on extern blocks, functions, + // impls, and modules. All of these have their inner attributes + // placed at the beginning of the rightmost outermost braced group: + // e.g. fn foo() { #![my_attr] } + // + // Therefore, we can insert them back into the right location + // without needing to do any extra position tracking. + // + // Note: Outline modules are an exception - they can + // have attributes like `#![my_attr]` at the start of a file. + // Support for custom attributes in this position is not + // properly implemented - we always synthesize fake tokens, + // so we never reach this code. + let mut tts = vec![]; + for inner_attr in inner_attrs { + tts.extend(inner_attr.token_trees()); + } + tts.extend(delim_tokens.0.iter().cloned()); + let stream = TokenStream::new(tts); + *tree = TokenTree::Delimited(*span, *spacing, *delim, stream); + found = true; + break; + } + } + assert!(found, "Failed to find trailing delimited group in: {res:?}"); + } +} + /// Stores the tokens for an attribute target, along /// with its attributes. /// @@ -438,18 +450,10 @@ impl TokenStream { } pub fn from_ast(node: &(impl HasAttrs + HasTokens + fmt::Debug)) -> TokenStream { - let Some(tokens) = node.tokens() else { - panic!("missing tokens for node: {:?}", node); - }; - let attrs = node.attrs(); - let attr_stream = if attrs.is_empty() { - tokens.to_attr_token_stream() - } else { - let target = - AttrsTarget { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; - AttrTokenStream::new(vec![AttrTokenTree::AttrsTarget(target)]) - }; - TokenStream::new(attr_stream.to_token_trees()) + let tokens = node.tokens().unwrap_or_else(|| panic!("missing tokens for node: {:?}", node)); + let mut tts = vec![]; + attrs_and_tokens_to_token_trees(node.attrs(), tokens, &mut tts); + TokenStream::new(tts) } pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 13414f990fba9..323cea1af4acf 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -283,7 +283,10 @@ impl<'a> StripUnconfigured<'a> { .flat_map(|item| self.process_cfg_attr(&self.expand_cfg_attr_item(cfg_attr, item))) .collect() } else { - expanded_attrs.into_iter().map(|item| self.expand_cfg_attr_item(cfg_attr, item)).collect() + expanded_attrs + .into_iter() + .map(|item| self.expand_cfg_attr_item(cfg_attr, item)) + .collect() } } From 478ba5902629b0a4dda914e81167e6b3b44c3a51 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 10 Jul 2024 16:29:48 +1000 Subject: [PATCH 22/25] Add some comments. Explaining things that took me some time to work out. --- compiler/rustc_ast/src/tokenstream.rs | 5 ++++- compiler/rustc_expand/src/config.rs | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 1f69a7dc35666..a92ef575777c7 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -269,11 +269,14 @@ fn attrs_and_tokens_to_token_trees( /// with its attributes. /// /// This is constructed during parsing when we need to capture -/// tokens. +/// tokens, for `cfg` and `cfg_attr` attributes. /// /// For example, `#[cfg(FALSE)] struct Foo {}` would /// have an `attrs` field containing the `#[cfg(FALSE)]` attr, /// and a `tokens` field storing the (unparsed) tokens `struct Foo {}` +/// +/// The `cfg`/`cfg_attr` processing occurs in +/// `StripUnconfigured::configure_tokens`. #[derive(Clone, Debug, Encodable, Decodable)] pub struct AttrsTarget { /// Attributes, both outer and inner. diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 323cea1af4acf..9da4aa84db525 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -187,6 +187,7 @@ impl<'a> StripUnconfigured<'a> { .iter() .filter_map(|tree| match tree.clone() { AttrTokenTree::AttrsTarget(mut target) => { + // Expand any `cfg_attr` attributes. target.attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr)); if self.in_cfg(&target.attrs) { @@ -195,6 +196,8 @@ impl<'a> StripUnconfigured<'a> { ); Some(AttrTokenTree::AttrsTarget(target)) } else { + // Remove the target if there's a `cfg` attribute and + // the condition isn't satisfied. None } } From 7fc69436a1226b254b3b6f9ed8c6d54965e0cfbf Mon Sep 17 00:00:00 2001 From: Ben Kimock Date: Fri, 12 Jul 2024 13:49:41 -0400 Subject: [PATCH 23/25] Use ManuallyDrop in BufWriter::into_parts --- library/std/src/io/buffered/bufwriter.rs | 14 +++++++------- library/std/src/io/buffered/tests.rs | 10 ++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/library/std/src/io/buffered/bufwriter.rs b/library/std/src/io/buffered/bufwriter.rs index 2d13230ffbabd..1768bb05ddbcb 100644 --- a/library/std/src/io/buffered/bufwriter.rs +++ b/library/std/src/io/buffered/bufwriter.rs @@ -3,7 +3,7 @@ use crate::fmt; use crate::io::{ self, ErrorKind, IntoInnerError, IoSlice, Seek, SeekFrom, Write, DEFAULT_BUF_SIZE, }; -use crate::mem; +use crate::mem::{self, ManuallyDrop}; use crate::ptr; /// Wraps a writer and buffers its output. @@ -164,13 +164,13 @@ impl BufWriter { /// assert_eq!(&buffered_data.unwrap(), b"ata"); /// ``` #[stable(feature = "bufwriter_into_parts", since = "1.56.0")] - pub fn into_parts(mut self) -> (W, Result, WriterPanicked>) { - let buf = mem::take(&mut self.buf); - let buf = if !self.panicked { Ok(buf) } else { Err(WriterPanicked { buf }) }; + pub fn into_parts(self) -> (W, Result, WriterPanicked>) { + let mut this = ManuallyDrop::new(self); + let buf = mem::take(&mut this.buf); + let buf = if !this.panicked { Ok(buf) } else { Err(WriterPanicked { buf }) }; - // SAFETY: forget(self) prevents double dropping inner - let inner = unsafe { ptr::read(&self.inner) }; - mem::forget(self); + // SAFETY: double-drops are prevented by putting `this` in a ManuallyDrop that is never dropped + let inner = unsafe { ptr::read(&this.inner) }; (inner, buf) } diff --git a/library/std/src/io/buffered/tests.rs b/library/std/src/io/buffered/tests.rs index ee0db30e22c2e..ab66deaf31d22 100644 --- a/library/std/src/io/buffered/tests.rs +++ b/library/std/src/io/buffered/tests.rs @@ -1067,3 +1067,13 @@ fn bufreader_full_initialize() { // But we initialized the whole buffer! assert_eq!(reader.initialized(), reader.capacity()); } + +/// This is a regression test for https://github.com/rust-lang/rust/issues/127584. +#[test] +fn bufwriter_aliasing() { + use crate::io::{BufWriter, Cursor}; + let mut v = vec![0; 1024]; + let c = Cursor::new(&mut v); + let w = BufWriter::new(Box::new(c)); + let _ = w.into_parts(); +} From fc0d1dc99bc579b07ad59733d74e9ec6e823d6e9 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 13 Jul 2024 10:04:33 +0300 Subject: [PATCH 24/25] use `ModeToolBootstrap` for run-make-support's crate tests We don't need to ensure std (and rustc) for testing run-make-support's unit tests. Using stage 0 compiler is already enough and speeds up `x test run-make-support` invocations on a clean build. Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 0b60587bb792b..33517c5e2e440 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1325,13 +1325,12 @@ impl Step for CrateRunMakeSupport { /// Runs `cargo test` for run-make-support. fn run(self, builder: &Builder<'_>) { let host = self.host; - let compiler = builder.compiler(builder.top_stage, host); + let compiler = builder.compiler(0, host); - builder.ensure(compile::Std::new(compiler, host)); let mut cargo = tool::prepare_tool_cargo( builder, compiler, - Mode::ToolStd, + Mode::ToolBootstrap, host, "test", "src/tools/run-make-support", From 41070bd9381bc0604171eba9308869eecd94b20b Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 13 Jul 2024 19:58:36 +0300 Subject: [PATCH 25/25] explain why we use in-tree std for compiletest Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 33517c5e2e440..1fa58040a1195 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -686,6 +686,8 @@ impl Step for CompiletestTest { let mut cargo = tool::prepare_tool_cargo( builder, compiler, + // compiletest uses libtest internals; make it use the in-tree std to make sure it never breaks + // when std sources change. Mode::ToolStd, host, "test",