From e68f3039d4f2b12dcfc348ebb50fd7855e6d7fd1 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Wed, 17 Jan 2024 13:53:25 -0500 Subject: [PATCH] Optimize large array creation in const-eval This changes repeated memcpy's to a memset for the case that we're propagating a single byte into a region of memory. --- .../rustc_const_eval/src/interpret/memory.rs | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 7ff970661d6f1..3afd14eb57435 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -1209,21 +1209,28 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { throw_ub_custom!(fluent::const_eval_copy_nonoverlapping_overlapping); } } + } - for i in 0..num_copies { - ptr::copy( - src_bytes, - dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication - size.bytes_usize(), - ); + let size_in_bytes = size.bytes_usize(); + // For particularly large arrays (where this is perf-sensitive) it's common that + // we're writing a single byte repeatedly. So, optimize that case to a memset. + if size_in_bytes == 1 && num_copies >= 1 { + // SAFETY: `src_bytes` would be read from anyway by copies below (num_copies >= 1). + // Since size_in_bytes = 1, then the `init.no_bytes_init()` check above guarantees + // that this read at type `u8` is OK -- it must be an initialized byte. + let value = *src_bytes; + dest_bytes.write_bytes(value, (size * num_copies).bytes_usize()); + } else if src_alloc_id == dest_alloc_id { + let mut dest_ptr = dest_bytes; + for _ in 0..num_copies { + ptr::copy(src_bytes, dest_ptr, size_in_bytes); + dest_ptr = dest_ptr.add(size_in_bytes); } } else { - for i in 0..num_copies { - ptr::copy_nonoverlapping( - src_bytes, - dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication - size.bytes_usize(), - ); + let mut dest_ptr = dest_bytes; + for _ in 0..num_copies { + ptr::copy_nonoverlapping(src_bytes, dest_ptr, size_in_bytes); + dest_ptr = dest_ptr.add(size_in_bytes); } } }