Skip to content

Commit

Permalink
JIT: revise criteria for inline candidates with intrinsics or folding…
Browse files Browse the repository at this point in the history 
… opportunities

Look for IL patterns in an inlinee that are indicative of type folding. Also track
when an inlinee has calls that are intrinsics.

Use this to boost the inlinee profitability estimate.

Also, allow an aggressive inline inline candidate method to go over the budget when
it contains type folding or intrinsics (and so the method may be simpler than it appears).
Remove the old criteria (top-level inline) which was harder to reason about.

Closes dotnet#43761.
Closes dotnet#41692.
Closes dotnet#51587.
Closes dotnet#47434.
  • Loading branch information
@AndyAyersMS
AndyAyersMS committed Apr 21, 2021
1 parent d7e2b8f commit 4347119
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 12 deletions.
54 changes: 52 additions & 2 deletions src/coreclr/jit/fgbasic.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -853,7 +853,6 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
const bool isPreJit = opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT);
const bool resolveTokens = makeInlineObservations && (isTier1 || isPreJit);
unsigned retBlocks = 0;
unsigned intrinsicCalls = 0;
int prefixFlags = 0;
int value = 0;

Expand Down Expand Up @@ -967,7 +966,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed

if (isJitIntrinsic)
{
intrinsicCalls++;
compInlineResult->Note(InlineObservation::CALLEE_INTRINSIC_CALL);

ni = lookupNamedIntrinsic(methodHnd);

switch (ni)
Expand Down Expand Up @@ -1487,6 +1487,56 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
retBlocks++;
break;

case CEE_BOX:
{
// Look for evidence of type specialization patterns.
//
// box + br
// box + isinst + br
//
// Todo: share pattern recognition with impBoxPatternMatch somehow
//
if (makeInlineObservations && (codeAddr < codeEndp))
{
switch (codeAddr[0])
{
case CEE_BRTRUE:
case CEE_BRTRUE_S:
case CEE_BRFALSE:
case CEE_BRFALSE_S:
// box + br_true/false
if ((codeAddr + ((codeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp)
{
compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD);
}

case CEE_ISINST:
if (codeAddr + 1 + sizeof(mdToken) + 1 <= codeEndp)
{
const BYTE* nextCodeAddr = codeAddr + 1 + sizeof(mdToken);
switch (nextCodeAddr[0])
{
// box + isinst + br_true/false
case CEE_BRTRUE:
case CEE_BRTRUE_S:
case CEE_BRFALSE:
case CEE_BRFALSE_S:
if ((nextCodeAddr + ((nextCodeAddr[0] >= CEE_BRFALSE) ? 5 : 2)) <= codeEndp)
{
compInlineResult->Note(InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD);
}
break;
default:
break;
}
}

default:
break;
}
}
}

default:
break;
}
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/inline.def
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ INLINE_OBSERVATION(HAS_PINNED_LOCALS, bool, "has pinned locals",
INLINE_OBSERVATION(HAS_SIMD, bool, "has SIMD arg, local, or ret", INFORMATION, CALLEE)
INLINE_OBSERVATION(HAS_SWITCH, bool, "has switch", INFORMATION, CALLEE)
INLINE_OBSERVATION(IL_CODE_SIZE, int, "number of bytes of IL", INFORMATION, CALLEE)
INLINE_OBSERVATION(INTRINSIC_CALL, bool, "intrinsic call", INFORMATION, CALLEE)
INLINE_OBSERVATION(IS_CLASS_CTOR, bool, "class constructor", INFORMATION, CALLEE)
INLINE_OBSERVATION(IS_DISCRETIONARY_INLINE, bool, "can inline, check heuristics", INFORMATION, CALLEE)
INLINE_OBSERVATION(IS_FORCE_INLINE, bool, "aggressive inline attribute", INFORMATION, CALLEE)
Expand All @@ -94,6 +95,7 @@ INLINE_OBSERVATION(OPCODE_NORMED, int, "next opcode in IL stream"
INLINE_OBSERVATION(NUMBER_OF_ARGUMENTS, int, "number of arguments", INFORMATION, CALLEE)
INLINE_OBSERVATION(NUMBER_OF_BASIC_BLOCKS, int, "number of basic blocks", INFORMATION, CALLEE)
INLINE_OBSERVATION(NUMBER_OF_LOCALS, int, "number of locals", INFORMATION, CALLEE)
INLINE_OBSERVATION(POSSIBLE_TYPE_FOLD, bool, "possible type fold branch", INFORMATION, CALLEE)
INLINE_OBSERVATION(RANDOM_ACCEPT, bool, "random accept", INFORMATION, CALLEE)
INLINE_OBSERVATION(UNSUPPORTED_OPCODE, bool, "unsupported opcode", INFORMATION, CALLEE)

Expand Down
36 changes: 26 additions & 10 deletions src/coreclr/jit/inlinepolicy.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,14 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value)
m_ArgFeedsRangeCheck++;
break;

case InlineObservation::CALLEE_INTRINSIC_CALL:
m_IntrinsicCalls++;
break;

case InlineObservation::CALLEE_POSSIBLE_TYPE_FOLD:
m_PossibleTypeFolds++;
break;

case InlineObservation::CALLEE_HAS_SWITCH:
case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
propagate = true;
Expand Down Expand Up @@ -472,20 +480,16 @@ bool DefaultPolicy::BudgetCheck() const

if (overBudget)
{
// If the candidate is a forceinline and the callsite is
// not too deep, allow the inline even if it goes over budget.
//
// For now, "not too deep" means a top-level inline. Note
// depth 0 is used for the root method, so inline candidate depth
// will be 1 or more.
// If the candidate is a forceinline and we've seen evidence
// that the callee might not be as expensive as it appears,
// allow the inline, even if it goes over budget.
//
assert(m_IsForceInlineKnown);
assert(m_CallsiteDepth > 0);
const bool allowOverBudget = m_IsForceInline && (m_CallsiteDepth == 1);
const bool hasEvidence = (m_IntrinsicCalls > 0) || (m_PossibleTypeFolds > 0);

if (allowOverBudget)
if (hasEvidence && m_IsForceInline)
{
JITDUMP("Allowing over-budget top-level forceinline\n");
JITDUMP("Allowing over-budget forceinline\n");
}
else
{
Expand Down Expand Up @@ -733,6 +737,18 @@ double DefaultPolicy::DetermineMultiplier()
JITDUMP("\nPrejit root candidate has arg that feeds a conditional. Multiplier increased to %g.", multiplier);
}

if (m_IntrinsicCalls > 0)
{
multiplier += 3.0;
JITDUMP("\nPossible intrinsic simplifications. Multiplier increased to %g.", multiplier);
}

if (m_PossibleTypeFolds > 0)
{
multiplier += 3.0;
JITDUMP("\nPossible type folding opportunities. Multiplier increased to %g.", multiplier);
}

switch (m_CallsiteFrequency)
{
case InlineCallsiteFrequency::RARE:
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/jit/inlinepolicy.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ class DefaultPolicy : public LegalPolicy
, m_ArgFeedsConstantTest(0)
, m_ArgFeedsRangeCheck(0)
, m_ConstantArgFeedsConstantTest(0)
, m_IntrinsicCalls(0)
, m_PossibleTypeFolds(0)
, m_CalleeNativeSizeEstimate(0)
, m_CallsiteNativeSizeEstimate(0)
, m_IsForceInline(false)
Expand Down Expand Up @@ -164,6 +166,8 @@ class DefaultPolicy : public LegalPolicy
unsigned m_ArgFeedsConstantTest;
unsigned m_ArgFeedsRangeCheck;
unsigned m_ConstantArgFeedsConstantTest;
unsigned m_IntrinsicCalls;
unsigned m_PossibleTypeFolds;
int m_CalleeNativeSizeEstimate;
int m_CallsiteNativeSizeEstimate;
bool m_IsForceInline : 1;
Expand Down

0 comments on commit 4347119

Please sign in to comment.