From 4347119ebd48bcb92b78f233aae6e91afdf26850 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Wed, 21 Apr 2021 09:33:30 -0700 Subject: [PATCH] JIT: revise criteria for inline candidates with intrinsics or folding opportunities Look for IL patterns in an inlinee that are indicative of type folding. Also track when an inlinee has calls that are intrinsics. Use this to boost the inlinee profitability estimate. Also, allow an aggressive inline inline candidate method to go over the budget when it contains type folding or intrinsics (and so the method may be simpler than it appears). Remove the old criteria (top-level inline) which was harder to reason about. Closes #43761. Closes #41692. Closes #51587. Closes #47434. --- src/coreclr/jit/fgbasic.cpp | 54 ++++++++++++++++++++++++++++++-- src/coreclr/jit/inline.def | 2 ++ src/coreclr/jit/inlinepolicy.cpp | 36 +++++++++++++++------ src/coreclr/jit/inlinepolicy.h | 4 +++ 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 7992d3b696a31..6f600c75c0de3 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -853,7 +853,6 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed const bool isPreJit = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT); const bool resolveTokens = makeInlineObservations && (isTier1 || isPreJit); unsigned retBlocks = 0; - unsigned intrinsicCalls = 0; int prefixFlags = 0; int value = 0; @@ -967,7 +966,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed if (isJitIntrinsic) { - intrinsicCalls++; + compInlineResult->Note(InlineObservation::CALLEE_INTRINSIC_CALL); + ni = lookupNamedIntrinsic(methodHnd); switch (ni) @@ -1487,6 +1487,56 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed retBlocks++; break; + case CEE_BOX: + { + // Look for evidence of type specialization patterns. + // + // box + br + // box + isinst + br + // + // Todo: share pattern recognition with impBoxPatternMatch somehow + // + if (makeInlineObservations && (codeAddr < codeEndp)) + { + switch (codeAddr[0]) + { + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + // box + br_true/false + if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) + { + compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD); + } + + case CEE_ISINST: + if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp) + { + const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken); + switch (nextCodeAddr[0]) + { + // box + isinst + br_true/false + case CEE_BRTRUE: + case CEE_BRTRUE_S: + case CEE_BRFALSE: + case CEE_BRFALSE_S: + if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp) + { + compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD); + } + break; + default: + break; + } + } + + default: + break; + } + } + } + default: break; } diff --git a/src/coreclr/jit/inline.def b/src/coreclr/jit/inline.def index 3c3ef462a150f..d13a36c95d4e5 100644 --- a/src/coreclr/jit/inline.def +++ b/src/coreclr/jit/inline.def @@ -80,6 +80,7 @@ INLINE_OBSERVATION(HAS_PINNED_LOCALS, bool, "has pinned locals", INLINE_OBSERVATION(HAS_SIMD, bool, "has SIMD arg, local, or ret", INFORMATION, CALLEE) INLINE_OBSERVATION(HAS_SWITCH, bool, "has switch", INFORMATION, CALLEE) INLINE_OBSERVATION(IL_CODE_SIZE, int, "number of bytes of IL", INFORMATION, CALLEE) +INLINE_OBSERVATION(INTRINSIC_CALL, bool, "intrinsic call", INFORMATION, CALLEE) INLINE_OBSERVATION(IS_CLASS_CTOR, bool, "class constructor", INFORMATION, CALLEE) INLINE_OBSERVATION(IS_DISCRETIONARY_INLINE, bool, "can inline, check heuristics", INFORMATION, CALLEE) INLINE_OBSERVATION(IS_FORCE_INLINE, bool, "aggressive inline attribute", INFORMATION, CALLEE) @@ -94,6 +95,7 @@ INLINE_OBSERVATION(OPCODE_NORMED, int, "next opcode in IL stream" INLINE_OBSERVATION(NUMBER_OF_ARGUMENTS, int, "number of arguments", INFORMATION, CALLEE) INLINE_OBSERVATION(NUMBER_OF_BASIC_BLOCKS, int, "number of basic blocks", INFORMATION, CALLEE) INLINE_OBSERVATION(NUMBER_OF_LOCALS, int, "number of locals", INFORMATION, CALLEE) +INLINE_OBSERVATION(POSSIBLE_TYPE_FOLD, bool, "possible type fold branch", INFORMATION, CALLEE) INLINE_OBSERVATION(RANDOM_ACCEPT, bool, "random accept", INFORMATION, CALLEE) INLINE_OBSERVATION(UNSUPPORTED_OPCODE, bool, "unsupported opcode", INFORMATION, CALLEE) diff --git a/src/coreclr/jit/inlinepolicy.cpp b/src/coreclr/jit/inlinepolicy.cpp index bd8b363e977f8..4de25d5d0f1c7 100644 --- a/src/coreclr/jit/inlinepolicy.cpp +++ b/src/coreclr/jit/inlinepolicy.cpp @@ -312,6 +312,14 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value) m_ArgFeedsRangeCheck++; break; + case InlineObservation::CALLEE_INTRINSIC_CALL: + m_IntrinsicCalls++; + break; + + case InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD: + m_PossibleTypeFolds++; + break; + case InlineObservation::CALLEE_HAS_SWITCH: case InlineObservation::CALLEE_UNSUPPORTED_OPCODE: propagate = true; @@ -472,20 +480,16 @@ bool DefaultPolicy::BudgetCheck() const if (overBudget) { - // If the candidate is a forceinline and the callsite is - // not too deep, allow the inline even if it goes over budget. - // - // For now, "not too deep" means a top-level inline. Note - // depth 0 is used for the root method, so inline candidate depth - // will be 1 or more. + // If the candidate is a forceinline and we've seen evidence + // that the callee might not be as expensive as it appears, + // allow the inline, even if it goes over budget. // assert(m_IsForceInlineKnown); - assert(m_CallsiteDepth > 0); - const bool allowOverBudget = m_IsForceInline && (m_CallsiteDepth == 1); + const bool hasEvidence = (m_IntrinsicCalls > 0) || (m_PossibleTypeFolds > 0); - if (allowOverBudget) + if (hasEvidence && m_IsForceInline) { - JITDUMP("Allowing over-budget top-level forceinline\n"); + JITDUMP("Allowing over-budget forceinline\n"); } else { @@ -733,6 +737,18 @@ double DefaultPolicy::DetermineMultiplier() JITDUMP("\nPrejit root candidate has arg that feeds a conditional. Multiplier increased to %g.", multiplier); } + if (m_IntrinsicCalls > 0) + { + multiplier += 3.0; + JITDUMP("\nPossible intrinsic simplifications. Multiplier increased to %g.", multiplier); + } + + if (m_PossibleTypeFolds > 0) + { + multiplier += 3.0; + JITDUMP("\nPossible type folding opportunities. Multiplier increased to %g.", multiplier); + } + switch (m_CallsiteFrequency) { case InlineCallsiteFrequency::RARE: diff --git a/src/coreclr/jit/inlinepolicy.h b/src/coreclr/jit/inlinepolicy.h index 3e0d3440a8e85..2fc31e89c2657 100644 --- a/src/coreclr/jit/inlinepolicy.h +++ b/src/coreclr/jit/inlinepolicy.h @@ -96,6 +96,8 @@ class DefaultPolicy : public LegalPolicy , m_ArgFeedsConstantTest(0) , m_ArgFeedsRangeCheck(0) , m_ConstantArgFeedsConstantTest(0) + , m_IntrinsicCalls(0) + , m_PossibleTypeFolds(0) , m_CalleeNativeSizeEstimate(0) , m_CallsiteNativeSizeEstimate(0) , m_IsForceInline(false) @@ -164,6 +166,8 @@ class DefaultPolicy : public LegalPolicy unsigned m_ArgFeedsConstantTest; unsigned m_ArgFeedsRangeCheck; unsigned m_ConstantArgFeedsConstantTest; + unsigned m_IntrinsicCalls; + unsigned m_PossibleTypeFolds; int m_CalleeNativeSizeEstimate; int m_CallsiteNativeSizeEstimate; bool m_IsForceInline : 1;