Auto merge of rust-lang#126733 - ZhuUx:llvm-19-adapt, r=Zalathar

[Coverage][MCDC] Adapt mcdc to llvm 19

Related issue: rust-lang#126672

Also finish task 4 at rust-lang#124144

[llvm rust-lang#82448](llvm/llvm-project#82448) has introduced some break changes into mcdc, causing incompatibility between llvm 18 and 19. This draft adapts to that change and gives up supporting for llvm-18.

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1679,16 +1679,21 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
+        bitmap_bits: &'ll Value,
     ) {
-        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes);
+        debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bits);
+
+        assert!(
+            crate::llvm_util::get_version() >= (19, 0, 0),
+            "MCDC intrinsics require LLVM 19 or later"
+        );
 
         let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
             &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes];
+        let args = &[fn_name, hash, bitmap_bits];
         let args = self.check_call("call", llty, llfn, args);
 
         unsafe {
@@ -1708,28 +1713,25 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         &mut self,
         fn_name: &'ll Value,
         hash: &'ll Value,
-        bitmap_bytes: &'ll Value,
         bitmap_index: &'ll Value,
         mcdc_temp: &'ll Value,
     ) {
         debug!(
-            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp
+            "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?})",
+            fn_name, hash, bitmap_index, mcdc_temp
+        );
+        assert!(
+            crate::llvm_util::get_version() >= (19, 0, 0),
+            "MCDC intrinsics require LLVM 19 or later"
         );
 
         let llfn =
             unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) };
         let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-            ],
+            &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_ptr()],
             self.cx.type_void(),
         );
-        let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp];
+        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
         let args = self.check_call("call", llty, llfn, args);
         unsafe {
             let _ = llvm::LLVMRustBuildCall(
@@ -1745,41 +1747,15 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
     }
 
-    pub(crate) fn mcdc_condbitmap_update(
-        &mut self,
-        fn_name: &'ll Value,
-        hash: &'ll Value,
-        cond_loc: &'ll Value,
-        mcdc_temp: &'ll Value,
-        bool_value: &'ll Value,
-    ) {
-        debug!(
-            "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})",
-            fn_name, hash, cond_loc, mcdc_temp, bool_value
-        );
-        let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) };
-        let llty = self.cx.type_func(
-            &[
-                self.cx.type_ptr(),
-                self.cx.type_i64(),
-                self.cx.type_i32(),
-                self.cx.type_ptr(),
-                self.cx.type_i1(),
-            ],
-            self.cx.type_void(),
+    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
+        debug!("mcdc_condbitmap_update() with args ({:?}, {:?})", cond_index, mcdc_temp);
+        assert!(
+            crate::llvm_util::get_version() >= (19, 0, 0),
+            "MCDC intrinsics require LLVM 19 or later"
         );
-        let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value];
-        self.check_call("call", llty, llfn, args);
-        unsafe {
-            let _ = llvm::LLVMRustBuildCall(
-                self.llbuilder,
-                llty,
-                llfn,
-                args.as_ptr() as *const &llvm::Value,
-                args.len() as c_uint,
-                [].as_ptr(),
-                0 as c_uint,
-            );
-        }
+        let align = self.tcx.data_layout.i32_align.abi;
+        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
+        let new_tv_index = self.add(current_tv_index, cond_index);
+        self.store(new_tv_index, mcdc_temp, align);
     }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs

@@ -111,7 +111,7 @@ enum RegionKind {
 }
 
 mod mcdc {
-    use rustc_middle::mir::coverage::{ConditionInfo, DecisionInfo};
+    use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo};
 
     /// Must match the layout of `LLVMRustMCDCDecisionParameters`.
     #[repr(C)]
@@ -167,12 +167,13 @@ mod mcdc {
 
     impl From<ConditionInfo> for BranchParameters {
         fn from(value: ConditionInfo) -> Self {
+            let to_llvm_cond_id = |cond_id: Option<ConditionId>| {
+                cond_id.and_then(|id| LLVMConditionId::try_from(id.as_usize()).ok()).unwrap_or(-1)
+            };
+            let ConditionInfo { condition_id, true_next_id, false_next_id } = value;
             Self {
-                condition_id: value.condition_id.as_u32() as LLVMConditionId,
-                condition_ids: [
-                    value.false_next_id.as_u32() as LLVMConditionId,
-                    value.true_next_id.as_u32() as LLVMConditionId,
-                ],
+                condition_id: to_llvm_cond_id(Some(condition_id)),
+                condition_ids: [to_llvm_cond_id(false_next_id), to_llvm_cond_id(true_next_id)],
             }
         }
     }

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -98,14 +98,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
         };
 
         // If there are no MC/DC bitmaps to set up, return immediately.
-        if function_coverage_info.mcdc_bitmap_bytes == 0 {
+        if function_coverage_info.mcdc_bitmap_bits == 0 {
             return;
         }
 
         let fn_name = self.get_pgo_func_name_var(instance);
         let hash = self.const_u64(function_coverage_info.function_source_hash);
-        let bitmap_bytes = self.const_u32(function_coverage_info.mcdc_bitmap_bytes);
-        self.mcdc_parameters(fn_name, hash, bitmap_bytes);
+        let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32);
+        self.mcdc_parameters(fn_name, hash, bitmap_bits);
 
         // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps.
         let mut cond_bitmaps = vec![];
@@ -185,35 +185,28 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
             CoverageKind::ExpressionUsed { id } => {
                 func_coverage.mark_expression_id_seen(id);
             }
-            CoverageKind::CondBitmapUpdate { id, value, decision_depth } => {
+            CoverageKind::CondBitmapUpdate { index, decision_depth } => {
                 drop(coverage_map);
-                assert_ne!(
-                    id.as_u32(),
-                    0,
-                    "ConditionId of evaluated conditions should never be zero"
-                );
                 let cond_bitmap = coverage_context
                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                     .expect("mcdc cond bitmap should have been allocated for updating");
-                let cond_loc = bx.const_i32(id.as_u32() as i32 - 1);
-                let bool_value = bx.const_bool(value);
-                let fn_name = bx.get_pgo_func_name_var(instance);
-                let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value);
+                let cond_index = bx.const_i32(index as i32);
+                bx.mcdc_condbitmap_update(cond_index, cond_bitmap);
             }
             CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => {
                 drop(coverage_map);
                 let cond_bitmap = coverage_context
                                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)
                                     .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap");
-                let bitmap_bytes = function_coverage_info.mcdc_bitmap_bytes;
-                assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range");
+                assert!(
+                    bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits,
+                    "bitmap index of the decision out of range"
+                );
 
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                let bitmap_bytes = bx.const_u32(bitmap_bytes);
                 let bitmap_index = bx.const_u32(bitmap_idx);
-                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap);
+                bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap);
             }
         }
     }

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1614,7 +1614,6 @@ unsafe extern "C" {
     pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value;
     pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value;
 
     pub fn LLVMRustBuildCall<'a>(
         B: &Builder<'a>,

compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp

@@ -88,38 +88,7 @@ struct LLVMRustMCDCParameters {
   LLVMRustMCDCBranchParameters BranchParameters;
 };
 
-// LLVM representations for `MCDCParameters` evolved from LLVM 18 to 19.
-// Look at representations in 18
-// https://github.com/rust-lang/llvm-project/blob/66a2881a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L253-L263
-// and representations in 19
-// https://github.com/llvm/llvm-project/blob/843cc474faefad1d639f4c44c1cf3ad7dbda76c8/llvm/include/llvm/ProfileData/Coverage/MCDCTypes.h
-#if LLVM_VERSION_LT(19, 0)
-static coverage::CounterMappingRegion::MCDCParameters
-fromRust(LLVMRustMCDCParameters Params) {
-  auto parameter = coverage::CounterMappingRegion::MCDCParameters{};
-  switch (Params.Tag) {
-  case LLVMRustMCDCParametersTag::None:
-    return parameter;
-  case LLVMRustMCDCParametersTag::Decision:
-    parameter.BitmapIdx =
-        static_cast<unsigned>(Params.DecisionParameters.BitmapIdx),
-    parameter.NumConditions =
-        static_cast<unsigned>(Params.DecisionParameters.NumConditions);
-    return parameter;
-  case LLVMRustMCDCParametersTag::Branch:
-    parameter.ID = static_cast<coverage::CounterMappingRegion::MCDCConditionID>(
-        Params.BranchParameters.ConditionID),
-    parameter.FalseID =
-        static_cast<coverage::CounterMappingRegion::MCDCConditionID>(
-            Params.BranchParameters.ConditionIDs[0]),
-    parameter.TrueID =
-        static_cast<coverage::CounterMappingRegion::MCDCConditionID>(
-            Params.BranchParameters.ConditionIDs[1]);
-    return parameter;
-  }
-  report_fatal_error("Bad LLVMRustMCDCParametersTag!");
-}
-#else
+#if LLVM_VERSION_GE(19, 0)
 static coverage::mcdc::Parameters fromRust(LLVMRustMCDCParameters Params) {
   switch (Params.Tag) {
   case LLVMRustMCDCParametersTag::None:
@@ -214,13 +183,17 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer(
            RustMappingRegions, NumMappingRegions)) {
     MappingRegions.emplace_back(
         fromRust(Region.Count), fromRust(Region.FalseCount),
-#if LLVM_VERSION_LT(19, 0)
-        // LLVM 19 may move this argument to last.
-        fromRust(Region.MCDCParameters),
+#if LLVM_VERSION_GE(18, 0) && LLVM_VERSION_LT(19, 0)
+        coverage::CounterMappingRegion::MCDCParameters{},
 #endif
         Region.FileID, Region.ExpandedFileID, // File IDs, then region info.
         Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd,
-        fromRust(Region.Kind));
+        fromRust(Region.Kind)
+#if LLVM_VERSION_GE(19, 0)
+            ,
+        fromRust(Region.MCDCParameters)
+#endif
+    );
   }
 
   std::vector<coverage::CounterExpression> Expressions;

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1539,23 +1539,21 @@ LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
 
 extern "C" LLVMValueRef
 LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) {
+#if LLVM_VERSION_GE(19, 0)
   return wrap(llvm::Intrinsic::getDeclaration(
       unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters));
+#else
+  report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions");
+#endif
 }
 
 extern "C" LLVMValueRef
 LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) {
+#if LLVM_VERSION_GE(19, 0)
   return wrap(llvm::Intrinsic::getDeclaration(
       unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update));
-}
-
-extern "C" LLVMValueRef
-LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(LLVMModuleRef M) {
-#if LLVM_VERSION_LT(19, 0)
-  return wrap(llvm::Intrinsic::getDeclaration(
-      unwrap(M), llvm::Intrinsic::instrprof_mcdc_condbitmap_update));
 #else
-  report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions");
+  report_fatal_error("LLVM 19.0 is required for mcdc intrinsic functions");
 #endif
 }