rustc panic! after refactoring a const variable using find

[frochet]

I believe I've triggered the following rustc panic after trying to refactor a pub const variable recursively from the root directory of the code:

find . \( -type d -name .git -prune \) -o -type f -print0 | xargs -0 sed -i 's/MY_VAR_NAME/MY_NEW_VAR_NAME/g'

cargo clean resolved it.

rustc --version --verbose:

rustc 1.76.0 (07dca489a 2024-02-04)
binary: rustc
commit-hash: 07dca489ac2d933c78d3c5158e3f43beefeb02ce
commit-date: 2024-02-04
host: x86_64-unknown-linux-gnu
release: 1.76.0
LLVM version: 17.0.6

Error output

thread 'rustc' panicked at
compiler/rustc_middle/src/query/on_disk_cache.rs:525:5:
assertion `left == right` failed
   left: 106
  right: 1002111927320821928687967599834759150
stack backtrace:
    0:     0x7c6a80d8b6f6 -
std::backtrace_rs::backtrace::libunwind::trace::hbee8a7973eeb6c93
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/../../backtrace/src/backtrace/libunwind.rs:104:5
    1:     0x7c6a80d8b6f6 -
std::backtrace_rs::backtrace::trace_unsynchronized::hc8ac75eea3aa6899
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
    2:     0x7c6a80d8b6f6 -
std::sys_common::backtrace::_print_fmt::hc7f3e3b5298b1083
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs:68:5
    3:     0x7c6a80d8b6f6 -
<std::sys_common::backtrace::_print::DisplayBacktrace as
core::fmt::Display>::fmt::hbb235daedd7c6190
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs:44:22
    4:     0x7c6a80dddf40 - core::fmt::rt::Argument::fmt::h76c38a80d925a410
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/fmt/rt.rs:142:9
    5:     0x7c6a80dddf40 - core::fmt::write::h3ed6aeaa977c8e45
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/fmt/mod.rs:1120:17
    6:     0x7c6a80d7f53f - std::io::Write::write_fmt::h78b18af5775fedb5
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/io/mod.rs:1810:15
    7:     0x7c6a80d8b4d4 -
std::sys_common::backtrace::_print::h5d645a07e0fcfdbb
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs:47:5
    8:     0x7c6a80d8b4d4 -
std::sys_common::backtrace::print::h85035a511aafe7a8
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs:34:9
    9:     0x7c6a80d8e267 -
std::panicking::default_hook::{{closure}}::hcce8cea212785a25
   10:     0x7c6a80d8dfc9 - std::panicking::default_hook::hf5fcb0f213fe709a
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs:292:9
   11:     0x7c6a83a1461c -
std[79729d9c385e1623]::panicking::update_hook::<alloc[6df67106ebca92c0]::boxed::Box<rustc_driver_impl[66ed8fdbde15dc6c]::install_ice_hook::{closure#0}>>::{closure#0}
   12:     0x7c6a80d8e9b6 - <alloc::boxed::Box<F,A> as
core::ops::function::Fn<Args>>::call::hbc5ccf4eb663e1e5
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs:2029:9
   13:     0x7c6a80d8e9b6 -
std::panicking::rust_panic_with_hook::h095fccf1dc9379ee
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs:783:13
   14:     0x7c6a80d8e702 -
std::panicking::begin_panic_handler::{{closure}}::h032ba12139b353db
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs:657:13
   15:     0x7c6a80d8bbf6 -
std::sys_common::backtrace::__rust_end_short_backtrace::h9259bc2ff8fd0f76
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys_common/backtrace.rs:171:18
   16:     0x7c6a80d8e460 - rust_begin_unwind
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/panicking.rs:645:5
   17:     0x7c6a80dda645 - core::panicking::panic_fmt::h784f20a50eaab275
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panicking.rs:72:14
   18:     0x7c6a80ddabdb -
core::panicking::assert_failed_inner::hbf94b40c37b92af0
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/core/src/panicking.rs:342:17
   19:     0x7c6a83e26247 -
core[f975038e3cc9791c]::panicking::assert_failed::<u128, u128>
   20:     0x7c6a85c4a531 -
<rustc_middle[e0f609162babcb92]::query::on_disk_cache::OnDiskCache>::new
   21:     0x7c6a8594bba2 -
rustc_incremental[9a504be5dec41bc5]::persist::load::load_query_result_cache
   22:     0x7c6a8594bdaa -
rustc_interface[bf5cafa581ab7832]::passes::create_global_ctxt
   23:     0x7c6a85af2af0 -
<rustc_interface[bf5cafa581ab7832]::queries::Queries>::global_ctxt
   24:     0x7c6a858fe568 -
rustc_interface[bf5cafa581ab7832]::interface::run_compiler::<core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>,
rustc_driver_impl[66ed8fdbde15dc6c]::run_compiler::{closure#0}>::{closure#0}
   25:     0x7c6a85a2c7db -
std[79729d9c385e1623]::sys_common::backtrace::__rust_begin_short_backtrace::<rustc_interface[bf5cafa581ab7832]::util::run_in_thread_with_globals<rustc_interface[bf5cafa581ab7832]::interface::run_compiler<core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>,
rustc_driver_impl[66ed8fdbde15dc6c]::run_compiler::{closure#0}>::{closure#0},
core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>>::{closure#0}::{closure#0},
core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>>
   26:     0x7c6a85a2c639 -
<<std[79729d9c385e1623]::thread::Builder>::spawn_unchecked_<rustc_interface[bf5cafa581ab7832]::util::run_in_thread_with_globals<rustc_interface[bf5cafa581ab7832]::interface::run_compiler<core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>,
rustc_driver_impl[66ed8fdbde15dc6c]::run_compiler::{closure#0}>::{closure#0},
core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>>::{closure#0}::{closure#0},
core[f975038e3cc9791c]::result::Result<(),
rustc_span[7a149c27976a99e7]::ErrorGuaranteed>>::{closure#1} as
core[f975038e3cc9791c]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
   27:     0x7c6a80d988e5 - <alloc::boxed::Box<F,A> as
core::ops::function::FnOnce<Args>>::call_once::h12de4fc57affb195
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs:2015:9
   28:     0x7c6a80d988e5 - <alloc::boxed::Box<F,A> as
core::ops::function::FnOnce<Args>>::call_once::h3c619f45059d5cf1
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/alloc/src/boxed.rs:2015:9
   29:     0x7c6a80d988e5 -
std::sys::unix::thread::Thread::new::thread_start::hbac657605e4b7389
                                at
/rustc/07dca489ac2d933c78d3c5158e3f43beefeb02ce/library/std/src/sys/unix/thread.rs:108:17
   30:     0x7c6a80a94ac3 - start_thread
                                at ./nptl/pthread_create.c:442:8
   31:     0x7c6a80b26850 - __GI___clone3
                                at
./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
   32:                0x0 - <unknown>

[lqd]

Is the code available so we can try to reproduce the issue?

[frochet]

I confirm I can reliably trigger the crash by building the project, using the command above to change the var name, and building again.

The repository is currently in private, and I expect to open it in the coming weeks. If you like, I can try to reproduce on a dummy project.

[lqd]

If you are able to, yes it'd be helpful, otherwise we won't be able to do much to investigate this.

[saethlin]

This looks like another situation where we fail to load a compiler output file because the file has been truncated, so the footer is invalid. Wise of whoever made the footer tag so distinctive, I can easily search for the tag in the issue tracker and it looks like we have 7 issues like this open and 4 were closed with a comment that makes it clear that the cause of the crash was never debugged or understood.

[saethlin]

Ah this is on 1.76. I'm inclined to not trust ICE reports about truncated files before 1.77, because before then I/O errors aren't reported as fatal, permitting later compilations to read output from a crashed compiler: #119510

Of course it's very possible that this issue is reporting a new and real bug, but this issue is based on a compiler with known-broken error reporting.

[saethlin]

@frochet After an ICE, how much free disk space is available?

[frochet]

@saethlin I still had 16GB free. But it looks like #119510 could indeed explain it. I'll try compiling with a more recent rustc tonight.

@lqd I tried a few things on a dummy project but could not reproduce. I'll let you know when I open publicly.

[frochet]

@saethlin, @lqd, FYI it also crashes in a reproducible way on rustc 1.80.0-nightly (f705de5 2024-04-30).

[saethlin]

In your initial report you said

cargo clean resolved it.

But also you can cause the compiler to crash in a reproducible way. What are you doing to reproduce the crash? And what filesystem is your target directory on?

[frochet]

In your initial report you said

cargo clean resolved it.

But also you can cause the compiler to crash in a reproducible way. What are you doing to reproduce the crash? And what filesystem is your target directory on?

I can reproduce by changing the var name again e.g., doing these 4 steps is reproducible:

1. cargo build (works)
2. find . ( -type d -name .git -prune ) -o -type f -print0 | xargs -0 sed -i 's/MY_VAR_NAME/MY_NEW_VAR_NAME/g' (this touches the target directory as well, my initial mistake)
3. cargo build (crashes)
4. cargo clean
5. find . ( -type d -name .git -prune ) -o -type f -print0 | xargs -0 sed -i 's/MY_NEW_VAR_NAME/MY_VAR_NAME/g' (restoring to retry from step 1)

The filesystem is ext4.

[lqd]

Does the issue reproduce if you don't modify the target dir and its internal files?

[frochet]

No, the steps above while pruning the target directory too does not lead to a crash. I used the following:

find . \( -type d -name .git -prune -o -type d -name target -prune \) -o -type f -print0 | xargs -0 sed -i 's/MY_VAR_NAME/MY_NEW_VAR_NAME/g'

Note that I forgot a step in the above description, where I restore the initial variable name (edited)

[saethlin]

(I'm copy-pasting this comment into a lot of issues, which will probably be uniquely obvious on this issue)

As far as I can tell, the ICE reported in this issue is caused by another process damaging rustc's build artifacts or by a buggy filesystem. That other process could be anything from an IDE running the compiler at the same time as a manual build to a find-and-replace that accidentally edits files in the target directory.

#124686 contains a mitigation for this situation; the compiler now has a crude way to detect and ignore damaged incremental compilation files. You will still get a warning about the damaged file, but the build should proceed normally. That change shipped in Rust 1.80 which released today, so I am closing this issue and a few others which as far as I can tell all have the same root cause. You shouldn't see this ICE anymore, but please file a new issue if you see something like it on 1.80 or later.