JIT: fix gc hole in peephole optimizations

We cannot safely peephole instructions that straddle a gc enable boundary.
Detecting when this might happen is a bit subtle; currently we rely on
`emitForceNewIG` to be set.

Add a new utility 'emitCanPeepholeLastIns` to centralize the logic that
decides whether basing current emission on `emitLastIns` is safe.

Closed #77661.

src/coreclr/jit/emit.h

@@ -2179,6 +2179,13 @@ class emitter
 
     instrDesc* emitLastIns;
 
+    // Check if a peephole opt involving emitLastIns is unsafe
+    // emitForceNewIG here prevents peepholes from crossing nogc boundaries.
+    bool emitCannotPeepholeLastIns()
+    {
+        return emitForceNewIG || ((emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0));
+    }
+
 #ifdef TARGET_ARMARCH
     instrDesc* emitLastMemBarrier;
 #endif

src/coreclr/jit/emitarm64.cpp

@@ -15940,7 +15940,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN
         return false;
     }
 
-    const bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0);
+    const bool canOptimize = !emitCannotPeepholeLastIns();
 
     if (dst == src)
     {
@@ -15960,7 +15960,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN
         else if (isGeneralRegisterOrSP(dst) && (size == EA_4BYTE))
         {
             // See if the previous instruction already cleared upper 4 bytes for us unintentionally
-            if (!isFirstInstrInBlock && (emitLastIns != nullptr) && (emitLastIns->idReg1() == dst) &&
+            if (canOptimize && (emitLastIns != nullptr) && (emitLastIns->idReg1() == dst) &&
                 (emitLastIns->idOpSize() == size) && emitLastIns->idInsIs(INS_ldr, INS_ldrh, INS_ldrb))
             {
                 JITDUMP("\n -- suppressing mov because ldr already cleared upper 4 bytes\n");
@@ -15969,7 +15969,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN
         }
     }
 
-    if (!isFirstInstrInBlock && // Don't optimize if instruction is not the first instruction in IG.
+    if (canOptimize && // Don't optimize if instruction is not the first instruction in IG.
         (emitLastIns != nullptr) &&
         (emitLastIns->idIns() == INS_mov) && // Don't optimize if last instruction was not 'mov'.
         (emitLastIns->idOpSize() == size))   // Don't optimize if operand size is different than previous instruction.
@@ -16048,9 +16048,7 @@ bool emitter::IsRedundantMov(instruction ins, emitAttr size, regNumber dst, regN
 bool emitter::IsRedundantLdStr(
     instruction ins, regNumber reg1, regNumber reg2, ssize_t imm, emitAttr size, insFormat fmt)
 {
-    bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0);
-
-    if (((ins != INS_ldr) && (ins != INS_str)) || (isFirstInstrInBlock) || (emitLastIns == nullptr))
+    if (((ins != INS_ldr) && (ins != INS_str)) || emitCannotPeepholeLastIns())
     {
         return false;
     }

src/coreclr/jit/emitxarch.cpp

@@ -317,7 +317,7 @@ bool emitter::AreUpper32BitsZero(regNumber reg)
     // If there are no instructions in this IG, we can look back at
     // the previous IG's instructions if this IG is an extension.
     //
-    if ((emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0))
+    if (emitCannotPeepholeLastIns())
     {
         return false;
     }
@@ -398,7 +398,7 @@ bool emitter::AreFlagsSetToZeroCmp(regNumber reg, emitAttr opSize, genTreeOps tr
     }
 
     // Don't look back across IG boundaries (possible control flow)
-    if (emitCurIGinsCnt == 0 && ((emitCurIG->igFlags & IGF_EXTEND) == 0))
+    if (emitCannotPeepholeLastIns())
     {
         return false;
     }
@@ -480,8 +480,7 @@ bool emitter::AreFlagsSetForSignJumpOpt(regNumber reg, emitAttr opSize, GenTree*
         return false;
     }
 
-    // Don't look back across IG boundaries (possible control flow)
-    if (emitCurIGinsCnt == 0 && ((emitCurIG->igFlags & IGF_EXTEND) == 0))
+    if (emitCannotPeepholeLastIns())
     {
         return false;
     }
@@ -4698,12 +4697,10 @@ bool emitter::IsRedundantMov(
         return true;
     }
 
-    bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0);
-
     // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in
     // functionality even if their actual identifier differs and we should optimize these
 
-    if (isFirstInstrInBlock ||               // Don't optimize if instruction is the first instruction in IG.
+    if (emitCannotPeepholeLastIns() ||       // Don't optimize if instruction is the first instruction in IG.
         (emitLastIns == nullptr) ||          // or if a last instruction doesn't exist
         (emitLastIns->idIns() != ins) ||     // or if the instruction is different from the last instruction
         (emitLastIns->idOpSize() != size) || // or if the operand size is different from the last instruction
@@ -7343,13 +7340,10 @@ bool emitter::IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size,
         return false;
     }
 
-    bool hasSideEffect = HasSideEffect(ins, size);
-
-    bool isFirstInstrInBlock = (emitCurIGinsCnt == 0) && ((emitCurIG->igFlags & IGF_EXTEND) == 0);
     // TODO-XArch-CQ: Certain instructions, such as movaps vs movups, are equivalent in
     // functionality even if their actual identifier differs and we should optimize these
 
-    if (isFirstInstrInBlock ||             // Don't optimize if instruction is the first instruction in IG.
+    if (emitCannotPeepholeLastIns() ||     // Don't optimize if instruction is the first instruction in IG.
         (emitLastIns == nullptr) ||        // or if a last instruction doesn't exist
         (emitLastIns->idIns() != ins) ||   // or if the instruction is different from the last instruction
         (emitLastIns->idOpSize() != size)) // or if the operand size is different from the last instruction
@@ -7367,6 +7361,8 @@ bool emitter::IsRedundantStackMov(instruction ins, insFormat fmt, emitAttr size,
     int       varNum   = emitLastIns->idAddr()->iiaLclVar.lvaVarNum();
     int       lastOffs = emitLastIns->idAddr()->iiaLclVar.lvaOffset();
 
+    const bool hasSideEffect = HasSideEffect(ins, size);
+
     // Check if the last instruction and current instructions use the same register and local memory.
     if (varNum == varx && lastReg1 == ireg && lastOffs == offs)
     {