diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 38838f1e6b8fd..e6e2abeb059c2 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -1585,8 +1585,9 @@ BasicBlock* BasicBlock::New(Compiler* compiler) block->bbNatLoopNum = BasicBlock::NOT_IN_LOOP; - block->bbPreorderNum = 0; - block->bbPostorderNum = 0; + block->bbPreorderNum = 0; + block->bbPostorderNum = 0; + block->bbNewPostorderNum = 0; return block; } diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index cec551fa6038e..93fdc0358ce83 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -1220,8 +1220,9 @@ struct BasicBlock : private LIR::Range void* bbSparseCountInfo; // Used early on by fgIncorporateEdgeCounts - unsigned bbPreorderNum; // the block's preorder number in the graph (1...fgMaxBBNum] - unsigned bbPostorderNum; // the block's postorder number in the graph (1...fgMaxBBNum] + unsigned bbPreorderNum; // the block's preorder number in the graph (1...fgMaxBBNum] + unsigned bbPostorderNum; // the block's postorder number in the graph (1...fgMaxBBNum] + unsigned bbNewPostorderNum; // the block's postorder number in the graph [0...postOrderCount) IL_OFFSET bbCodeOffs; // IL offset of the beginning of the block IL_OFFSET bbCodeOffsEnd; // IL offset past the end of the block. Thus, the [bbCodeOffs..bbCodeOffsEnd) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index bd7f1088ba8c7..24df83d90f363 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5765,6 +5765,9 @@ void Compiler::RecomputeLoopInfo() optSetBlockWeights(); // Rebuild the loop tree annotations themselves optFindLoops(); + + m_dfsTree = fgComputeDfs(); + optFindNewLoops(); } /*****************************************************************************/ diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 41c9a138cbf0a..315fd2d0988e2 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -2085,7 +2085,7 @@ class FlowGraphNaturalLoop friend class FlowGraphNaturalLoops; // The DFS tree that contains the loop blocks. - const FlowGraphDfsTree* m_tree; + const FlowGraphDfsTree* m_dfsTree; // The header block; dominates all other blocks in the loop, and is the // only block branched to from outside the loop. @@ -2094,12 +2094,15 @@ class FlowGraphNaturalLoop // Parent loop. By loop properties, well-scopedness is always guaranteed. // That is, the parent loop contains all blocks of this loop. FlowGraphNaturalLoop* m_parent = nullptr; + // First child loop. + FlowGraphNaturalLoop* m_child = nullptr; + // Sibling child loop, in reverse post order of the header blocks. + FlowGraphNaturalLoop* m_sibling = nullptr; // Bit vector of blocks in the loop; each index is the RPO index a block, // with the head block's RPO index subtracted. BitVec m_blocks; - - // Side of m_blocks. + // Size of m_blocks. unsigned m_blocksSize = 0; // Edges from blocks inside the loop back to the header. @@ -2116,7 +2119,7 @@ class FlowGraphNaturalLoop // Can be used to store additional annotations for this loop on the side. unsigned m_index = 0; - FlowGraphNaturalLoop(const FlowGraphDfsTree* tree, BasicBlock* head); + FlowGraphNaturalLoop(const FlowGraphDfsTree* dfsTree, BasicBlock* head); unsigned LoopBlockBitVecIndex(BasicBlock* block); bool TryGetLoopBlockBitVecIndex(BasicBlock* block, unsigned* pIndex); @@ -2138,7 +2141,7 @@ class FlowGraphNaturalLoop const FlowGraphDfsTree* GetDfsTree() const { - return m_tree; + return m_dfsTree; } FlowGraphNaturalLoop* GetParent() const @@ -2146,6 +2149,16 @@ class FlowGraphNaturalLoop return m_parent; } + FlowGraphNaturalLoop* GetChild() const + { + return m_child; + } + + FlowGraphNaturalLoop* GetSibling() const + { + return m_sibling; + } + unsigned GetIndex() const { return m_index; @@ -2166,7 +2179,26 @@ class FlowGraphNaturalLoop return m_exitEdges; } + FlowEdge* BackEdge(unsigned index) + { + assert(index < m_backEdges.size()); + return m_backEdges[index]; + } + + FlowEdge* EntryEdge(unsigned index) + { + assert(index < m_entryEdges.size()); + return m_entryEdges[index]; + } + + FlowEdge* ExitEdge(unsigned index) + { + assert(index < m_exitEdges.size()); + return m_exitEdges[index]; + } + bool ContainsBlock(BasicBlock* block); + bool ContainsLoop(FlowGraphNaturalLoop* childLoop); unsigned NumLoopBlocks(); @@ -2199,7 +2231,7 @@ class FlowGraphNaturalLoop // class FlowGraphNaturalLoops { - const FlowGraphDfsTree* m_dfs; + const FlowGraphDfsTree* m_dfsTree; // Collection of loops that were found. jitstd::vector m_loops; @@ -2212,6 +2244,11 @@ class FlowGraphNaturalLoops static bool FindNaturalLoopBlocks(FlowGraphNaturalLoop* loop, jitstd::list& worklist); public: + const FlowGraphDfsTree* GetDfsTree() + { + return m_dfsTree; + } + size_t NumLoops() { return m_loops.size(); @@ -2222,7 +2259,8 @@ class FlowGraphNaturalLoops return m_improperLoopHeaders > 0; } - FlowGraphNaturalLoop* GetLoopFromHeader(BasicBlock* header); + FlowGraphNaturalLoop* GetLoopByIndex(unsigned index); + FlowGraphNaturalLoop* GetLoopByHeader(BasicBlock* header); bool IsLoopBackEdge(FlowEdge* edge); bool IsLoopExitEdge(FlowEdge* edge); @@ -2290,14 +2328,14 @@ class FlowGraphDominatorTree template friend class NewDomTreeVisitor; - const FlowGraphDfsTree* m_dfs; - const DomTreeNode* m_tree; + const FlowGraphDfsTree* m_dfsTree; + const DomTreeNode* m_domTree; const unsigned* m_preorderNum; const unsigned* m_postorderNum; - FlowGraphDominatorTree(const FlowGraphDfsTree* dfs, const DomTreeNode* tree, const unsigned* preorderNum, const unsigned* postorderNum) - : m_dfs(dfs) - , m_tree(tree) + FlowGraphDominatorTree(const FlowGraphDfsTree* dfsTree, const DomTreeNode* domTree, const unsigned* preorderNum, const unsigned* postorderNum) + : m_dfsTree(dfsTree) + , m_domTree(domTree) , m_preorderNum(preorderNum) , m_postorderNum(postorderNum) { @@ -2308,9 +2346,65 @@ class FlowGraphDominatorTree BasicBlock* Intersect(BasicBlock* block, BasicBlock* block2); bool Dominates(BasicBlock* dominator, BasicBlock* dominated); - static FlowGraphDominatorTree* Build(const FlowGraphDfsTree* dfs); + static FlowGraphDominatorTree* Build(const FlowGraphDfsTree* dfsTree); +}; + +// Represents a reverse mapping from block back to its (most nested) containing loop. +class BlockToNaturalLoopMap +{ + FlowGraphNaturalLoops* m_loops; + // Array from postorder num -> index of most-nested loop containing the + // block, or UINT_MAX if no loop contains it. + unsigned* m_indices; + + BlockToNaturalLoopMap(FlowGraphNaturalLoops* loops, unsigned* indices) + : m_loops(loops), m_indices(indices) + { + } + +public: + FlowGraphNaturalLoop* GetLoop(BasicBlock* block); + + static BlockToNaturalLoopMap* Build(FlowGraphNaturalLoops* loops); }; +enum class FieldKindForVN +{ + SimpleStatic, + WithBaseAddr +}; + +typedef JitHashTable, FieldKindForVN> FieldHandleSet; + +typedef JitHashTable, bool> ClassHandleSet; + +// Represents a distillation of the useful side effects that occur inside a loop. +// Used by VN to be able to reason more precisely when entering loops. +struct LoopSideEffects +{ + // The loop contains an operation that we assume has arbitrary memory side + // effects. If this is set, the fields below may not be accurate (since + // they become irrelevant.) + bool HasMemoryHavoc[MemoryKindCount]; + // The set of variables that are IN or OUT during the execution of this loop + VARSET_TP VarInOut; + // The set of variables that are USE or DEF during the execution of this loop. + VARSET_TP VarUseDef; + // This has entries for all static field and object instance fields modified + // in the loop. + FieldHandleSet* FieldsModified = nullptr; + // Bits set indicate the set of sz array element types such that + // arrays of that type are modified + // in the loop. + ClassHandleSet* ArrayElemTypesModified = nullptr; + bool ContainsCall = false; + + LoopSideEffects(); + + void AddVariableLiveness(Compiler* comp, BasicBlock* block); + void AddModifiedField(Compiler* comp, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind); + void AddModifiedElemType(Compiler* comp, CORINFO_CLASS_HANDLE structHnd); +}; // The following holds information about instr offsets in terms of generated code. @@ -4849,11 +4943,20 @@ class Compiler unsigned fgBBNumMax; // The max bbNum that has been assigned to basic blocks unsigned fgDomBBcount; // # of BBs for which we have dominator and reachability information BasicBlock** fgBBReversePostorder; // Blocks in reverse postorder - FlowGraphDfsTree* m_dfs; + + FlowGraphDfsTree* m_dfsTree; + // The next members are annotations on the flow graph used during the + // optimization phases. They are invalidated once RBO runs and modifies the + // flow graph. FlowGraphNaturalLoops* m_loops; struct LoopDsc; LoopDsc** m_newToOldLoop; FlowGraphNaturalLoop** m_oldToNewLoop; + LoopSideEffects* m_loopSideEffects; + BlockToNaturalLoopMap* m_blockToLoop; + // Dominator tree used by SSA construction and copy propagation (the two are expected to use the same tree + // in order to avoid the need for SSA reconstruction and an "out of SSA" phase). + FlowGraphDominatorTree* fgSsaDomTree; // After the dominance tree is computed, we cache a DFS preorder number and DFS postorder number to compute // dominance queries in O(1). fgDomTreePreOrder and fgDomTreePostOrder are arrays giving the block's preorder and @@ -4864,10 +4967,6 @@ class Compiler unsigned* fgDomTreePreOrder; unsigned* fgDomTreePostOrder; - // Dominator tree used by SSA construction and copy propagation (the two are expected to use the same tree - // in order to avoid the need for SSA reconstruction and an "out of SSA" phase). - FlowGraphDominatorTree* fgSsaDomTree; - bool fgBBVarSetsInited; // Track how many artificial ref counts we've added to fgEntryBB (for OSR) @@ -5441,10 +5540,10 @@ class Compiler // Perform value-numbering for the trees in "blk". void fgValueNumberBlock(BasicBlock* blk); - // Requires that "entryBlock" is the entry block of loop "loopNum", and that "loopNum" is the + // Requires that "entryBlock" is the header block of "loop" and that "loop" is the // innermost loop of which "entryBlock" is the entry. Returns the value number that should be // assumed for the memoryKind at the start "entryBlk". - ValueNum fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, BasicBlock* entryBlock, unsigned loopNum); + ValueNum fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, BasicBlock* entryBlock, FlowGraphNaturalLoop* loop); // Called when an operation (performed by "tree", described by "msg") may cause the GcHeap to be mutated. // As GcHeap is a subset of ByrefExposed, this will also annotate the ByrefExposed mutation. @@ -6627,6 +6726,13 @@ class Compiler // Previous decisions on loop-invariance of value numbers in the current loop. VNSet m_curLoopVnInvariantCache; + int m_loopVarInOutCount; + int m_loopVarCount; + int m_hoistedExprCount; + int m_loopVarFPCount; + int m_loopVarInOutFPCount; + int m_hoistedFPExprCount; + // Get the VN cache for current loop VNSet* GetHoistedInCurLoop(Compiler* comp) { @@ -6654,26 +6760,26 @@ class Compiler // by the loop "lnum" itself. bool optHoistLoopNest(unsigned lnum, LoopHoistContext* hoistCtxt); - // Do hoisting for a particular loop ("lnum" is an index into the optLoopTable.) - bool optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt); + // Do hoisting for a particular loop + bool optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* hoistCtxt); - // Hoist all expressions in "blocks" that are invariant in loop "loopNum" (an index into the optLoopTable) + // Hoist all expressions in "blocks" that are invariant in "loop" // outside of that loop. - void optHoistLoopBlocks(unsigned loopNum, ArrayStack* blocks, LoopHoistContext* hoistContext); + void optHoistLoopBlocks(FlowGraphNaturalLoop* loop, ArrayStack* blocks, LoopHoistContext* hoistContext); - // Return true if the tree looks profitable to hoist out of loop 'lnum'. - bool optIsProfitableToHoistTree(GenTree* tree, unsigned lnum); + // Return true if the tree looks profitable to hoist out of "loop" + bool optIsProfitableToHoistTree(GenTree* tree, FlowGraphNaturalLoop* loop, LoopHoistContext* hoistCtxt); - // Performs the hoisting 'tree' into the PreHeader for loop 'lnum' - void optHoistCandidate(GenTree* tree, BasicBlock* treeBb, unsigned lnum, LoopHoistContext* hoistCtxt); + // Performs the hoisting "tree" into the PreHeader for "loop" + void optHoistCandidate(GenTree* tree, BasicBlock* treeBb, FlowGraphNaturalLoop* loop, LoopHoistContext* hoistCtxt); // Note the new SSA uses in tree void optRecordSsaUses(GenTree* tree, BasicBlock* block); - // Returns true iff the ValueNum "vn" represents a value that is loop-invariant in "lnum". + // Returns true iff the ValueNum "vn" represents a value that is loop-invariant in "loop". // Constants and init values are always loop invariant. // VNPhi's connect VN's to the SSA definition, so we can know if the SSA def occurs in the loop. - bool optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* recordedVNs); + bool optVNIsLoopInvariant(ValueNum vn, FlowGraphNaturalLoop* loop, VNSet* recordedVNs); // If "blk" is the entry block of a natural loop, returns true and sets "*pLnum" to the index of the loop // in the loop table. @@ -6694,21 +6800,14 @@ class Compiler void optMarkLoopRemoved(unsigned loopNum); private: - // Requires "lnum" to be the index of an outermost loop in the loop table. Traverses the body of that loop, - // including all nested loops, and records the set of "side effects" of the loop: fields (object instance and - // static) written to, and SZ-array element type equivalence classes updated. - void optComputeLoopNestSideEffects(unsigned lnum); - - // Given a loop number 'lnum' mark it and any nested loops as having 'memoryHavoc' - void optRecordLoopNestsMemoryHavoc(unsigned lnum, MemoryKindSet memoryHavoc); + // Given a loop mark it and any nested loops as having 'memoryHavoc' + void optRecordLoopNestsMemoryHavoc(FlowGraphNaturalLoop* loop, MemoryKindSet memoryHavoc); // Add the side effects of "blk" (which is required to be within a loop) to all loops of which it is a part. - // Returns false if we encounter a block that is not marked as being inside a loop. - // - bool optComputeLoopSideEffectsOfBlock(BasicBlock* blk); + void optComputeLoopSideEffectsOfBlock(BasicBlock* blk, FlowGraphNaturalLoop* mostNestedLoop); - // Hoist the expression "expr" out of loop "lnum". - void optPerformHoistExpr(GenTree* expr, BasicBlock* exprBb, unsigned lnum); + // Hoist the expression "expr" out of "loop" + void optPerformHoistExpr(GenTree* expr, BasicBlock* exprBb, FlowGraphNaturalLoop* loop); public: PhaseStatus optOptimizeBools(); @@ -6723,6 +6822,8 @@ class Compiler PhaseStatus optFindLoopsPhase(); // Finds loops and records them in the loop table void optFindLoops(); + void optFindNewLoops(); + void optCrossCheckIterInfo(const NaturalLoopIterInfo& iterInfo, const LoopDsc& dsc); PhaseStatus optCloneLoops(); void optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* context); @@ -6742,12 +6843,6 @@ class Compiler CALLINT_ALL, // kills everything (normal method call) }; - enum class FieldKindForVN - { - SimpleStatic, - WithBaseAddr - }; - public: // A "LoopDsc" describes a ("natural") loop. We (currently) require the body of a loop to be a contiguous (in // bbNext order) sequence of basic blocks. (At times, we may require the blocks in a loop to be "properly numbered" @@ -6812,15 +6907,6 @@ class Compiler // arrays of that type are modified // in the loop. - // Adds the variable liveness information for 'blk' to 'this' LoopDsc - void AddVariableLiveness(Compiler* comp, BasicBlock* blk); - - inline void AddModifiedField(Compiler* comp, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind); - // This doesn't *always* take a class handle -- it can also take primitive types, encoded as class handles - // (shifted left, with a low-order bit set to distinguish.) - // Use the {Encode/Decode}ElemType methods to construct/destruct these. - inline void AddModifiedElemType(Compiler* comp, CORINFO_CLASS_HANDLE structHnd); - /* The following values are set only for iterator loops, i.e. has the flag LPFLG_ITER set */ GenTree* lpIterTree; // The "i = i const" tree @@ -7084,17 +7170,17 @@ class Compiler BlockToBlockMap* redirectMap, const RedirectBlockOption = RedirectBlockOption::DoNotChangePredLists); - // Marks the containsCall information to "lnum" and any parent loops. - void AddContainsCallAllContainingLoops(unsigned lnum); + // Marks the containsCall information to "loop" and any parent loops. + void AddContainsCallAllContainingLoops(FlowGraphNaturalLoop* loop); - // Adds the variable liveness information from 'blk' to "lnum" and any parent loops. - void AddVariableLivenessAllContainingLoops(unsigned lnum, BasicBlock* blk); + // Adds the variable liveness information from 'blk' to "loop" and any parent loops. + void AddVariableLivenessAllContainingLoops(FlowGraphNaturalLoop* loop, BasicBlock* blk); - // Adds "fldHnd" to the set of modified fields of "lnum" and any parent loops. - void AddModifiedFieldAllContainingLoops(unsigned lnum, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind); + // Adds "fldHnd" to the set of modified fields of "loop" and any parent loops. + void AddModifiedFieldAllContainingLoops(FlowGraphNaturalLoop* loop, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind); - // Adds "elemType" to the set of modified array element types of "lnum" and any parent loops. - void AddModifiedElemTypeAllContainingLoops(unsigned lnum, CORINFO_CLASS_HANDLE elemType); + // Adds "elemType" to the set of modified array element types of "loop" and any parent loops. + void AddModifiedElemTypeAllContainingLoops(FlowGraphNaturalLoop* loop, CORINFO_CLASS_HANDLE elemType); // Requires that "from" and "to" have the same "bbJumpKind" (perhaps because "to" is a clone // of "from".) Copies the jump destination from "from" to "to". @@ -12049,7 +12135,7 @@ class NewDomTreeVisitor { static_cast(this)->PreOrderVisit(block); - next = tree[block->bbPostorderNum].firstChild; + next = tree[block->bbNewPostorderNum].firstChild; if (next != nullptr) { @@ -12061,7 +12147,7 @@ class NewDomTreeVisitor { static_cast(this)->PostOrderVisit(block); - next = tree[block->bbPostorderNum].nextSibling; + next = tree[block->bbNewPostorderNum].nextSibling; if (next != nullptr) { @@ -12090,7 +12176,7 @@ class NewDomTreeVisitor // void WalkTree(const FlowGraphDominatorTree* domTree) { - WalkTree(domTree->m_tree); + WalkTree(domTree->m_domTree); } }; diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 2b3841d7c163d..a725ca79b899b 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -3624,26 +3624,6 @@ inline void Compiler::optAssertionRemove(AssertionIndex index) } } -inline void Compiler::LoopDsc::AddModifiedField(Compiler* comp, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind) -{ - if (lpFieldsModified == nullptr) - { - lpFieldsModified = - new (comp->getAllocatorLoopHoist()) Compiler::LoopDsc::FieldHandleSet(comp->getAllocatorLoopHoist()); - } - lpFieldsModified->Set(fldHnd, fieldKind, FieldHandleSet::Overwrite); -} - -inline void Compiler::LoopDsc::AddModifiedElemType(Compiler* comp, CORINFO_CLASS_HANDLE structHnd) -{ - if (lpArrayElemTypesModified == nullptr) - { - lpArrayElemTypesModified = - new (comp->getAllocatorLoopHoist()) Compiler::LoopDsc::ClassHandleSet(comp->getAllocatorLoopHoist()); - } - lpArrayElemTypesModified->Set(structHnd, true, ClassHandleSet::Overwrite); -} - inline void Compiler::LoopDsc::VERIFY_lpIterTree() const { #ifdef DEBUG @@ -4953,15 +4933,15 @@ inline bool Compiler::compCanHavePatchpoints(const char** reason) template BasicBlockVisit FlowGraphNaturalLoop::VisitLoopBlocksReversePostOrder(TFunc func) { - BitVecTraits traits(m_blocksSize, m_tree->GetCompiler()); + BitVecTraits traits(m_blocksSize, m_dfsTree->GetCompiler()); bool result = BitVecOps::VisitBits(&traits, m_blocks, [=](unsigned index) { // head block rpo index = PostOrderCount - 1 - headPreOrderIndex // loop block rpo index = head block rpoIndex + index // loop block po index = PostOrderCount - 1 - loop block rpo index // = headPreOrderIndex - index - unsigned poIndex = m_header->bbPostorderNum - index; - assert(poIndex < m_tree->GetPostOrderCount()); - return func(m_tree->GetPostOrder()[poIndex]) == BasicBlockVisit::Continue; + unsigned poIndex = m_header->bbNewPostorderNum - index; + assert(poIndex < m_dfsTree->GetPostOrderCount()); + return func(m_dfsTree->GetPostOrder()[poIndex]) == BasicBlockVisit::Continue; }); return result ? BasicBlockVisit::Continue : BasicBlockVisit::Abort; @@ -4985,11 +4965,11 @@ BasicBlockVisit FlowGraphNaturalLoop::VisitLoopBlocksReversePostOrder(TFunc func template BasicBlockVisit FlowGraphNaturalLoop::VisitLoopBlocksPostOrder(TFunc func) { - BitVecTraits traits(m_blocksSize, m_tree->GetCompiler()); + BitVecTraits traits(m_blocksSize, m_dfsTree->GetCompiler()); bool result = BitVecOps::VisitBitsReverse(&traits, m_blocks, [=](unsigned index) { - unsigned poIndex = m_header->bbPostorderNum - index; - assert(poIndex < m_tree->GetPostOrderCount()); - return func(m_tree->GetPostOrder()[poIndex]) == BasicBlockVisit::Continue; + unsigned poIndex = m_header->bbNewPostorderNum - index; + assert(poIndex < m_dfsTree->GetPostOrderCount()); + return func(m_dfsTree->GetPostOrder()[poIndex]) == BasicBlockVisit::Continue; }); return result ? BasicBlockVisit::Continue : BasicBlockVisit::Abort; diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 3be0af2fac06d..d701a9b4b4bec 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -65,8 +65,12 @@ void Compiler::fgInit() fgBBVarSetsInited = false; fgReturnCount = 0; - m_dfs = nullptr; - m_loops = nullptr; + m_dfsTree = nullptr; + m_loops = nullptr; + m_newToOldLoop = nullptr; + m_oldToNewLoop = nullptr; + m_loopSideEffects = nullptr; + m_blockToLoop = nullptr; // Initialize BlockSet data. fgCurBBEpoch = 0; diff --git a/src/coreclr/jit/fgprofilesynthesis.cpp b/src/coreclr/jit/fgprofilesynthesis.cpp index 63d3ba814728b..cd9d872d7ea17 100644 --- a/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/coreclr/jit/fgprofilesynthesis.cpp @@ -33,8 +33,8 @@ // void ProfileSynthesis::Run(ProfileSynthesisOption option) { - m_dfs = m_comp->fgComputeDfs(); - m_loops = FlowGraphNaturalLoops::Find(m_dfs); + m_dfsTree = m_comp->fgComputeDfs(); + m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); // Retain or compute edge likelihood information // @@ -721,7 +721,7 @@ void ProfileSynthesis::ComputeCyclicProbabilities(FlowGraphNaturalLoop* loop) } else { - FlowGraphNaturalLoop* const nestedLoop = m_loops->GetLoopFromHeader(block); + FlowGraphNaturalLoop* const nestedLoop = m_loops->GetLoopByHeader(block); if (nestedLoop != nullptr) { @@ -1000,9 +1000,9 @@ void ProfileSynthesis::ComputeBlockWeights() { JITDUMP("Computing block weights\n"); - for (unsigned i = m_dfs->GetPostOrderCount(); i != 0; i--) + for (unsigned i = m_dfsTree->GetPostOrderCount(); i != 0; i--) { - BasicBlock* block = m_dfs->GetPostOrder()[i - 1]; + BasicBlock* block = m_dfsTree->GetPostOrder()[i - 1]; ComputeBlockWeight(block); } } @@ -1015,7 +1015,7 @@ void ProfileSynthesis::ComputeBlockWeights() // void ProfileSynthesis::ComputeBlockWeight(BasicBlock* block) { - FlowGraphNaturalLoop* const loop = m_loops->GetLoopFromHeader(block); + FlowGraphNaturalLoop* const loop = m_loops->GetLoopByHeader(block); weight_t newWeight = block->bbWeight; const char* kind = ""; diff --git a/src/coreclr/jit/fgprofilesynthesis.h b/src/coreclr/jit/fgprofilesynthesis.h index ab82fffe5e37c..9297357049e8a 100644 --- a/src/coreclr/jit/fgprofilesynthesis.h +++ b/src/coreclr/jit/fgprofilesynthesis.h @@ -78,7 +78,7 @@ class ProfileSynthesis private: Compiler* const m_comp; - FlowGraphDfsTree* m_dfs; + FlowGraphDfsTree* m_dfsTree; FlowGraphNaturalLoops* m_loops; weight_t* m_cyclicProbabilities; unsigned m_improperLoopHeaders; diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index bff7879a69d71..ac387b24a6410 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -4011,7 +4011,7 @@ void Compiler::fgLclFldAssign(unsigned lclNum) // bool FlowGraphDfsTree::Contains(BasicBlock* block) const { - return (block->bbPostorderNum < m_postOrderCount) && (m_postOrder[block->bbPostorderNum] == block); + return (block->bbNewPostorderNum < m_postOrderCount) && (m_postOrder[block->bbNewPostorderNum] == block); } //------------------------------------------------------------------------ @@ -4033,7 +4033,7 @@ bool FlowGraphDfsTree::IsAncestor(BasicBlock* ancestor, BasicBlock* descendant) { assert(Contains(ancestor) && Contains(descendant)); return (ancestor->bbPreorderNum <= descendant->bbPreorderNum) && - (descendant->bbPostorderNum <= ancestor->bbPostorderNum); + (descendant->bbNewPostorderNum <= ancestor->bbNewPostorderNum); } //------------------------------------------------------------------------ @@ -4081,7 +4081,7 @@ FlowGraphDfsTree* Compiler::fgComputeDfs() { blocks.Pop(); postOrder[postOrderIndex] = block; - block->bbPostorderNum = postOrderIndex++; + block->bbNewPostorderNum = postOrderIndex++; } } @@ -4116,13 +4116,13 @@ FlowGraphDfsTree* Compiler::fgComputeDfs() // tree - The DFS tree // header - The loop header // -FlowGraphNaturalLoop::FlowGraphNaturalLoop(const FlowGraphDfsTree* tree, BasicBlock* header) - : m_tree(tree) +FlowGraphNaturalLoop::FlowGraphNaturalLoop(const FlowGraphDfsTree* dfsTree, BasicBlock* header) + : m_dfsTree(dfsTree) , m_header(header) , m_blocks(BitVecOps::UninitVal()) - , m_backEdges(tree->GetCompiler()->getAllocator(CMK_Loops)) - , m_entryEdges(tree->GetCompiler()->getAllocator(CMK_Loops)) - , m_exitEdges(tree->GetCompiler()->getAllocator(CMK_Loops)) + , m_backEdges(dfsTree->GetCompiler()->getAllocator(CMK_Loops)) + , m_entryEdges(dfsTree->GetCompiler()->getAllocator(CMK_Loops)) + , m_exitEdges(dfsTree->GetCompiler()->getAllocator(CMK_Loops)) { } @@ -4146,9 +4146,9 @@ FlowGraphNaturalLoop::FlowGraphNaturalLoop(const FlowGraphDfsTree* tree, BasicBl // unsigned FlowGraphNaturalLoop::LoopBlockBitVecIndex(BasicBlock* block) { - assert(m_tree->Contains(block)); + assert(m_dfsTree->Contains(block)); - unsigned index = m_header->bbPostorderNum - block->bbPostorderNum; + unsigned index = m_header->bbNewPostorderNum - block->bbNewPostorderNum; assert(index < m_blocksSize); return index; } @@ -4171,12 +4171,12 @@ unsigned FlowGraphNaturalLoop::LoopBlockBitVecIndex(BasicBlock* block) // bool FlowGraphNaturalLoop::TryGetLoopBlockBitVecIndex(BasicBlock* block, unsigned* pIndex) { - if (block->bbPostorderNum > m_header->bbPostorderNum) + if (block->bbNewPostorderNum > m_header->bbNewPostorderNum) { return false; } - unsigned index = m_header->bbPostorderNum - block->bbPostorderNum; + unsigned index = m_header->bbNewPostorderNum - block->bbNewPostorderNum; if (index >= m_blocksSize) { return false; @@ -4194,7 +4194,7 @@ bool FlowGraphNaturalLoop::TryGetLoopBlockBitVecIndex(BasicBlock* block, unsigne // BitVecTraits FlowGraphNaturalLoop::LoopBlockTraits() { - return BitVecTraits(m_blocksSize, m_tree->GetCompiler()); + return BitVecTraits(m_blocksSize, m_dfsTree->GetCompiler()); } //------------------------------------------------------------------------ @@ -4223,6 +4223,20 @@ bool FlowGraphNaturalLoop::ContainsBlock(BasicBlock* block) return BitVecOps::IsMember(&traits, m_blocks, index); } +//------------------------------------------------------------------------ +// ContainsLoop: Returns true if this loop contains the specified other loop. +// +// Parameters: +// childLoop - The potential candidate child loop +// +// Returns: +// True if the child loop is contained in this loop. +// +bool FlowGraphNaturalLoop::ContainsLoop(FlowGraphNaturalLoop* childLoop) +{ + return ContainsBlock(childLoop->GetHeader()); +} + //------------------------------------------------------------------------ // NumLoopBlocks: Get the number of blocks in the SCC of the loop. // @@ -4242,12 +4256,26 @@ unsigned FlowGraphNaturalLoop::NumLoopBlocks() // Parameters: // dfs - A DFS tree. // -FlowGraphNaturalLoops::FlowGraphNaturalLoops(const FlowGraphDfsTree* dfs) - : m_dfs(dfs), m_loops(m_dfs->GetCompiler()->getAllocator(CMK_Loops)) +FlowGraphNaturalLoops::FlowGraphNaturalLoops(const FlowGraphDfsTree* dfsTree) + : m_dfsTree(dfsTree), m_loops(m_dfsTree->GetCompiler()->getAllocator(CMK_Loops)) { } -// GetLoopFromHeader: See if a block is a loop header, and if so return the +// GetLoopByIndex: Get loop by a specified index. +// +// Parameters: +// index - Index of loop. Must be less than NumLoops() +// +// Returns: +// Loop with the specified index. +// +FlowGraphNaturalLoop* FlowGraphNaturalLoops::GetLoopByIndex(unsigned index) +{ + assert(index < m_loops.size()); + return m_loops[index]; +} + +// GetLoopByHeader: See if a block is a loop header, and if so return the // associated loop. // // Parameters: @@ -4256,7 +4284,7 @@ FlowGraphNaturalLoops::FlowGraphNaturalLoops(const FlowGraphDfsTree* dfs) // Returns: // Loop headed by block, or nullptr // -FlowGraphNaturalLoop* FlowGraphNaturalLoops::GetLoopFromHeader(BasicBlock* block) +FlowGraphNaturalLoop* FlowGraphNaturalLoops::GetLoopByHeader(BasicBlock* block) { // TODO-TP: This can use binary search based on post order number. for (FlowGraphNaturalLoop* loop : m_loops) @@ -4326,34 +4354,34 @@ bool FlowGraphNaturalLoops::IsLoopExitEdge(FlowEdge* edge) // constructed for the flow graph. // // Parameters: -// dfs - The DFS tree +// dfsTree - The DFS tree // // Returns: // Identified natural loops. // -FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) +FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfsTree) { - Compiler* comp = dfs->GetCompiler(); + Compiler* comp = dfsTree->GetCompiler(); comp->m_blockToEHPreds = nullptr; #ifdef DEBUG JITDUMP("Identifying loops in DFS tree with following reverse post order:\n"); - for (unsigned i = dfs->GetPostOrderCount(); i != 0; i--) + for (unsigned i = dfsTree->GetPostOrderCount(); i != 0; i--) { - unsigned rpoNum = dfs->GetPostOrderCount() - i; - BasicBlock* const block = dfs->GetPostOrder()[i - 1]; + unsigned rpoNum = dfsTree->GetPostOrderCount() - i; + BasicBlock* const block = dfsTree->GetPostOrder()[i - 1]; JITDUMP("%02u -> " FMT_BB "[%u, %u]\n", rpoNum + 1, block->bbNum, block->bbPreorderNum + 1, - block->bbPostorderNum + 1); + block->bbNewPostorderNum + 1); } #endif - FlowGraphNaturalLoops* loops = new (comp, CMK_Loops) FlowGraphNaturalLoops(dfs); + FlowGraphNaturalLoops* loops = new (comp, CMK_Loops) FlowGraphNaturalLoops(dfsTree); jitstd::list worklist(comp->getAllocator(CMK_Loops)); - for (unsigned i = dfs->GetPostOrderCount(); i != 0; i--) + for (unsigned i = dfsTree->GetPostOrderCount(); i != 0; i--) { - BasicBlock* const header = dfs->GetPostOrder()[i - 1]; + BasicBlock* const header = dfsTree->GetPostOrder()[i - 1]; // If a block is a DFS ancestor of one if its predecessors then the block is a loop header. // @@ -4362,11 +4390,11 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) for (FlowEdge* predEdge : header->PredEdges()) { BasicBlock* predBlock = predEdge->getSourceBlock(); - if (dfs->Contains(predBlock) && dfs->IsAncestor(header, predBlock)) + if (dfsTree->Contains(predBlock) && dfsTree->IsAncestor(header, predBlock)) { if (loop == nullptr) { - loop = new (comp, CMK_Loops) FlowGraphNaturalLoop(dfs, header); + loop = new (comp, CMK_Loops) FlowGraphNaturalLoop(dfsTree, header); JITDUMP("\n"); } @@ -4387,7 +4415,7 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) // worklist.clear(); - loop->m_blocksSize = loop->m_header->bbPostorderNum + 1; + loop->m_blocksSize = loop->m_header->bbNewPostorderNum + 1; BitVecTraits loopTraits = loop->LoopBlockTraits(); loop->m_blocks = BitVecOps::MakeEmpty(&loopTraits); @@ -4426,7 +4454,7 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) for (FlowEdge* const predEdge : loop->m_header->PredEdges()) { BasicBlock* predBlock = predEdge->getSourceBlock(); - if (dfs->Contains(predBlock) && !dfs->IsAncestor(header, predEdge->getSourceBlock())) + if (dfsTree->Contains(predBlock) && !dfsTree->IsAncestor(header, predEdge->getSourceBlock())) { JITDUMP(FMT_BB " -> " FMT_BB " is an entry edge\n", predEdge->getSourceBlock()->bbNum, loop->m_header->bbNum); @@ -4483,6 +4511,18 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) JITDUMP("Added loop " FMT_LP " with header " FMT_BB "\n", loop->GetIndex(), loop->GetHeader()->bbNum); } + // Now build sibling/child links by iterating loops in post order. This + // makes us end up with sibling links in reverse post order. + for (FlowGraphNaturalLoop* loop : loops->InPostOrder()) + { + if (loop->m_parent != nullptr) + { + loop->m_sibling = loop->m_parent->m_child; + loop->m_parent->m_child = loop; + } + } + +#ifdef DEBUG if (loops->m_loops.size() > 0) { JITDUMP("\nFound %zu loops\n", loops->m_loops.size()); @@ -4492,6 +4532,7 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) { JITDUMP("Rejected %u loop headers\n", loops->m_improperLoopHeaders); } +#endif return loops; } @@ -4510,8 +4551,8 @@ FlowGraphNaturalLoops* FlowGraphNaturalLoops::Find(const FlowGraphDfsTree* dfs) // bool FlowGraphNaturalLoops::FindNaturalLoopBlocks(FlowGraphNaturalLoop* loop, jitstd::list& worklist) { - const FlowGraphDfsTree* tree = loop->m_tree; - Compiler* comp = tree->GetCompiler(); + const FlowGraphDfsTree* dfsTree = loop->m_dfsTree; + Compiler* comp = dfsTree->GetCompiler(); BitVecTraits loopTraits = loop->LoopBlockTraits(); BitVecOps::AddElemD(&loopTraits, loop->m_blocks, 0); @@ -4544,14 +4585,14 @@ bool FlowGraphNaturalLoops::FindNaturalLoopBlocks(FlowGraphNaturalLoop* loop, ji { BasicBlock* const predBlock = predEdge->getSourceBlock(); - if (!tree->Contains(predBlock)) + if (!dfsTree->Contains(predBlock)) { continue; } // Head cannot dominate `predBlock` unless it is a DFS ancestor. // - if (!tree->IsAncestor(loop->GetHeader(), predBlock)) + if (!dfsTree->IsAncestor(loop->GetHeader(), predBlock)) { JITDUMP("Loop is not natural; witness " FMT_BB " -> " FMT_BB "\n", predBlock->bbNum, loopBlock->bbNum); return false; @@ -4619,7 +4660,7 @@ bool FlowGraphNaturalLoop::VisitDefs(TFunc func) } }; - VisitDefsVisitor visitor(m_tree->GetCompiler(), func); + VisitDefsVisitor visitor(m_dfsTree->GetCompiler(), func); BasicBlockVisit result = VisitLoopBlocks([&](BasicBlock* loopBlock) { for (Statement* stmt : loopBlock->Statements()) @@ -4688,7 +4729,7 @@ bool FlowGraphNaturalLoop::AnalyzeIteration(NaturalLoopIterInfo* info) { JITDUMP("Analyzing iteration for " FMT_LP " with header " FMT_BB "\n", m_index, m_header->bbNum); - const FlowGraphDfsTree* dfs = m_tree; + const FlowGraphDfsTree* dfs = m_dfsTree; Compiler* comp = dfs->GetCompiler(); assert((m_entryEdges.size() == 1) && "Expected preheader"); @@ -4995,7 +5036,7 @@ BasicBlock* FlowGraphNaturalLoop::GetLexicallyBottomMostBlock() // bool FlowGraphNaturalLoop::HasDef(unsigned lclNum) { - Compiler* comp = m_tree->GetCompiler(); + Compiler* comp = m_dfsTree->GetCompiler(); LclVarDsc* dsc = comp->lvaGetDesc(lclNum); assert(!comp->lvaVarAddrExposed(lclNum)); @@ -5237,7 +5278,7 @@ BasicBlock* FlowGraphDominatorTree::IntersectDom(BasicBlock* finger1, BasicBlock { return nullptr; } - while (finger1 != nullptr && finger1->bbPostorderNum < finger2->bbPostorderNum) + while (finger1 != nullptr && finger1->bbNewPostorderNum < finger2->bbNewPostorderNum) { finger1 = finger1->bbIDom; } @@ -5245,7 +5286,7 @@ BasicBlock* FlowGraphDominatorTree::IntersectDom(BasicBlock* finger1, BasicBlock { return nullptr; } - while (finger2 != nullptr && finger2->bbPostorderNum < finger1->bbPostorderNum) + while (finger2 != nullptr && finger2->bbNewPostorderNum < finger1->bbNewPostorderNum) { finger2 = finger2->bbIDom; } @@ -5275,7 +5316,7 @@ BasicBlock* FlowGraphDominatorTree::Intersect(BasicBlock* block1, BasicBlock* bl // bool FlowGraphDominatorTree::Dominates(BasicBlock* dominator, BasicBlock* dominated) { - assert(m_dfs->Contains(dominator) && m_dfs->Contains(dominated)); + assert(m_dfsTree->Contains(dominator) && m_dfsTree->Contains(dominated)); // What we want to ask here is basically if A is in the middle of the path // from B to the root (the entry node) in the dominator tree. Turns out @@ -5285,8 +5326,8 @@ bool FlowGraphDominatorTree::Dominates(BasicBlock* dominator, BasicBlock* domina // // where the equality holds when you ask if A dominates itself. // - return (m_preorderNum[dominator->bbPostorderNum] <= m_preorderNum[dominated->bbPostorderNum]) && - (m_postorderNum[dominator->bbPostorderNum] >= m_postorderNum[dominated->bbPostorderNum]); + return (m_preorderNum[dominator->bbNewPostorderNum] <= m_preorderNum[dominated->bbNewPostorderNum]) && + (m_postorderNum[dominator->bbNewPostorderNum] >= m_postorderNum[dominated->bbNewPostorderNum]); } //------------------------------------------------------------------------ @@ -5294,7 +5335,7 @@ bool FlowGraphDominatorTree::Dominates(BasicBlock* dominator, BasicBlock* domina // the DFS tree. // // Parameters: -// dfs - DFS tree. +// dfsTree - DFS tree. // // Returns: // Data structure representing dominator tree. Immediate dominators are @@ -5306,11 +5347,11 @@ bool FlowGraphDominatorTree::Dominates(BasicBlock* dominator, BasicBlock* domina // This might require creating a scratch root block in case the first block // has backedges or is in a try region. // -FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* dfs) +FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* dfsTree) { - Compiler* comp = dfs->GetCompiler(); - BasicBlock** postOrder = dfs->GetPostOrder(); - unsigned count = dfs->GetPostOrderCount(); + Compiler* comp = dfsTree->GetCompiler(); + BasicBlock** postOrder = dfsTree->GetPostOrder(); + unsigned count = dfsTree->GetPostOrderCount(); assert((comp->fgFirstBB->bbPreds == nullptr) && !comp->fgFirstBB->hasTryIndex()); assert(postOrder[count - 1] == comp->fgFirstBB); @@ -5334,12 +5375,12 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df for (FlowEdge* pred = comp->BlockDominancePreds(block); pred; pred = pred->getNextPredEdge()) { BasicBlock* domPred = pred->getSourceBlock(); - if (!dfs->Contains(domPred)) + if (!dfsTree->Contains(domPred)) { continue; // Unreachable pred } - if ((numIters <= 0) && (domPred->bbPostorderNum <= poNum)) + if ((numIters <= 0) && (domPred->bbNewPostorderNum <= poNum)) { continue; // Pred not yet visited } @@ -5375,10 +5416,10 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df { BasicBlock* block = postOrder[i]; BasicBlock* parent = block->bbIDom; - assert(dfs->Contains(block) && dfs->Contains(parent)); + assert(dfsTree->Contains(block) && dfsTree->Contains(parent)); - domTree[i].nextSibling = domTree[parent->bbPostorderNum].firstChild; - domTree[parent->bbPostorderNum].firstChild = block; + domTree[i].nextSibling = domTree[parent->bbNewPostorderNum].firstChild; + domTree[parent->bbNewPostorderNum].firstChild = block; } #ifdef DEBUG @@ -5395,7 +5436,7 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df printf(FMT_BB " :", postOrder[poNum]->bbNum); for (BasicBlock* child = domTree[poNum].firstChild; child != nullptr; - child = domTree[child->bbPostorderNum].nextSibling) + child = domTree[child->bbNewPostorderNum].nextSibling) { printf(" " FMT_BB, child->bbNum); } @@ -5421,12 +5462,12 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df void PreOrderVisit(BasicBlock* block) { - m_preorderNums[block->bbPostorderNum] = m_preNum++; + m_preorderNums[block->bbNewPostorderNum] = m_preNum++; } void PostOrderVisit(BasicBlock* block) { - m_postorderNums[block->bbPostorderNum] = m_postNum++; + m_postorderNums[block->bbNewPostorderNum] = m_postNum++; } }; @@ -5436,5 +5477,66 @@ FlowGraphDominatorTree* FlowGraphDominatorTree::Build(const FlowGraphDfsTree* df NumberDomTreeVisitor number(comp, preorderNums, postorderNums); number.WalkTree(domTree); - return new (comp, CMK_DominatorMemory) FlowGraphDominatorTree(dfs, domTree, preorderNums, postorderNums); + return new (comp, CMK_DominatorMemory) FlowGraphDominatorTree(dfsTree, domTree, preorderNums, postorderNums); +} + +//------------------------------------------------------------------------ +// BlockToNaturalLoopMap::GetLoop: Map a block back to its most nested +// containing loop. +// +// Parameters: +// block - The block +// +// Returns: +// Loop or nullptr if the block is not contained in any loop. +// +FlowGraphNaturalLoop* BlockToNaturalLoopMap::GetLoop(BasicBlock* block) +{ + const FlowGraphDfsTree* dfs = m_loops->GetDfsTree(); + if (!dfs->Contains(block)) + { + return nullptr; + } + + unsigned index = m_indices[block->bbNewPostorderNum]; + if (index == UINT_MAX) + { + return nullptr; + } + + return m_loops->GetLoopByIndex(index); +} + +//------------------------------------------------------------------------ +// BlockToNaturalLoopMap::Build: Build the map. +// +// Parameters: +// loops - Data structure describing loops +// +// Returns: +// The map. +// +BlockToNaturalLoopMap* BlockToNaturalLoopMap::Build(FlowGraphNaturalLoops* loops) +{ + const FlowGraphDfsTree* dfs = loops->GetDfsTree(); + Compiler* comp = dfs->GetCompiler(); + unsigned* indices = + dfs->GetPostOrderCount() == 0 ? nullptr : (new (comp, CMK_Loops) unsigned[dfs->GetPostOrderCount()]); + + for (unsigned i = 0; i < dfs->GetPostOrderCount(); i++) + { + indices[i] = UINT_MAX; + } + + // Now visit all loops in reverse post order, meaning that we see inner + // loops last and thus write their indices into the map last. + for (FlowGraphNaturalLoop* loop : loops->InReversePostOrder()) + { + loop->VisitLoopBlocks([=](BasicBlock* block) { + indices[block->bbNewPostorderNum] = loop->GetIndex(); + return BasicBlockVisit::Continue; + }); + } + + return new (comp, CMK_Loops) BlockToNaturalLoopMap(loops, indices); } diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index e2e643fea7752..70b4d6f138b22 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -1824,7 +1824,7 @@ bool Compiler::optIsLoopClonable(FlowGraphNaturalLoop* loop, LoopCloneContext* c // Loop canonicalization should have ensured that there is a unique preheader. assert(loop->EntryEdges().size() == 1); - BasicBlock* preheader = loop->EntryEdges()[0]->getSourceBlock(); + BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); // If the head and entry are in different EH regions, reject. if (!BasicBlock::sameEHRegion(preheader, loop->GetHeader())) @@ -2025,7 +2025,7 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex } assert(loop->EntryEdges().size() == 1); - BasicBlock* preheader = loop->EntryEdges()[0]->getSourceBlock(); + BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); // The ambient weight might be higher than we computed above. Be safe by // taking the max with the head block's weight. ambientWeight = max(ambientWeight, preheader->bbWeight); @@ -3116,6 +3116,7 @@ bool Compiler::optObtainLoopCloningOpts(LoopCloneContext* context) { if (loop->AnalyzeIteration(&iterInfo)) { + INDEBUG(optCrossCheckIterInfo(iterInfo, optLoopTable[i])); context->SetLoopIterInfo(loop->GetIndex(), new (this, CMK_LoopClone) NaturalLoopIterInfo(iterInfo)); } } @@ -3269,8 +3270,8 @@ PhaseStatus Compiler::optCloneLoops() // TODO: recompute the loop table, to include the slow loop path in the table? fgUpdateChangedFlowGraph(FlowGraphUpdates::COMPUTE_DOMS); - m_dfs = fgComputeDfs(); - m_loops = FlowGraphNaturalLoops::Find(m_dfs); + m_dfsTree = fgComputeDfs(); + optFindNewLoops(); } #ifdef DEBUG diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 6eb04320d3e6d..2a546d4d40cb5 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -4637,6 +4637,9 @@ PhaseStatus Compiler::optUnrollLoops() fgUpdateChangedFlowGraph(FlowGraphUpdates::COMPUTE_DOMS); } + m_dfsTree = fgComputeDfs(); + optFindNewLoops(); + DBEXEC(verbose, fgDispBasicBlocks()); } else @@ -5540,8 +5543,18 @@ PhaseStatus Compiler::optFindLoopsPhase() { optFindLoops(); - m_dfs = fgComputeDfs(); - m_loops = FlowGraphNaturalLoops::Find(m_dfs); + m_dfsTree = fgComputeDfs(); + optFindNewLoops(); + + return PhaseStatus::MODIFIED_EVERYTHING; +} + +//----------------------------------------------------------------------------- +// optFindNewLoops: Compute new loops and cross validate with old loop table. +// +void Compiler::optFindNewLoops() +{ + m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); m_newToOldLoop = m_loops->NumLoops() == 0 ? nullptr : (new (this, CMK_Loops) LoopDsc*[m_loops->NumLoops()]{}); m_oldToNewLoop = new (this, CMK_Loops) FlowGraphNaturalLoop*[BasicBlock::MAX_LOOP_NUM]{}; @@ -5561,34 +5574,28 @@ PhaseStatus Compiler::optFindLoopsPhase() m_oldToNewLoop[head->bbNatLoopNum] = loop; m_newToOldLoop[loop->GetIndex()] = dsc; } +} -#ifdef DEBUG - for (unsigned i = 0; i < optLoopCount; i++) +//----------------------------------------------------------------------------- +// optCrossCheckIterInfo: Validate that new IV analysis matches the old one. +// +// Parameters: +// iterInfo - New IV information +// dsc - Old loop structure containing IV analysis information +// +void Compiler::optCrossCheckIterInfo(const NaturalLoopIterInfo& iterInfo, const LoopDsc& dsc) +{ + assert(iterInfo.HasConstInit == ((dsc.lpFlags & LPFLG_CONST_INIT) != 0)); + assert(iterInfo.HasConstLimit == ((dsc.lpFlags & LPFLG_CONST_LIMIT) != 0)); + assert(iterInfo.HasSimdLimit == ((dsc.lpFlags & LPFLG_SIMD_LIMIT) != 0)); + assert(iterInfo.HasInvariantLocalLimit == ((dsc.lpFlags & LPFLG_VAR_LIMIT) != 0)); + assert(iterInfo.HasArrayLengthLimit == ((dsc.lpFlags & LPFLG_ARRLEN_LIMIT) != 0)); + if (iterInfo.HasConstInit) { - assert(m_oldToNewLoop[i] != nullptr); - LoopDsc* dsc = &optLoopTable[i]; - if ((dsc->lpFlags & LPFLG_ITER) == 0) - continue; - - NaturalLoopIterInfo iter; - bool analyzed = m_oldToNewLoop[i]->AnalyzeIteration(&iter); - assert(analyzed); - - assert(iter.HasConstInit == ((dsc->lpFlags & LPFLG_CONST_INIT) != 0)); - assert(iter.HasConstLimit == ((dsc->lpFlags & LPFLG_CONST_LIMIT) != 0)); - assert(iter.HasSimdLimit == ((dsc->lpFlags & LPFLG_SIMD_LIMIT) != 0)); - assert(iter.HasInvariantLocalLimit == ((dsc->lpFlags & LPFLG_VAR_LIMIT) != 0)); - assert(iter.HasArrayLengthLimit == ((dsc->lpFlags & LPFLG_ARRLEN_LIMIT) != 0)); - if (iter.HasConstInit) - { - assert(iter.ConstInitValue == dsc->lpConstInit); - assert(iter.InitBlock == dsc->lpInitBlock); - } - assert(iter.TestTree == dsc->lpTestTree); + assert(iterInfo.ConstInitValue == dsc.lpConstInit); + assert(iterInfo.InitBlock == dsc.lpInitBlock); } -#endif - - return PhaseStatus::MODIFIED_EVERYTHING; + assert(iterInfo.TestTree == dsc.lpTestTree); } /***************************************************************************** @@ -6374,21 +6381,22 @@ void Compiler::optRecordSsaUses(GenTree* tree, BasicBlock* block) // Arguments: // origExpr - tree to hoist // exprBb - block containing the tree -// lnum - loop that we're hoisting origExpr out of +// loop - loop that we're hoisting origExpr out of // -void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsigned lnum) +void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, FlowGraphNaturalLoop* loop) { assert(exprBb != nullptr); + assert(loop->EntryEdges().size() == 1); + BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); #ifdef DEBUG if (verbose) { printf("\nHoisting a copy of "); printTreeID(origExpr); printf(" " FMT_VN, origExpr->gtVNPair.GetLiberal()); - printf(" from " FMT_BB " into PreHeader " FMT_BB " for loop " FMT_LP " <" FMT_BB ".." FMT_BB ">:\n", - exprBb->bbNum, optLoopTable[lnum].lpHead->bbNum, lnum, optLoopTable[lnum].lpTop->bbNum, - optLoopTable[lnum].lpBottom->bbNum); + printf(" from " FMT_BB " into PreHeader " FMT_BB " for loop " FMT_LP " (head: " FMT_BB "):\n", exprBb->bbNum, + preheader->bbNum, loop->GetIndex(), loop->GetHeader()->bbNum); gtDispTree(origExpr); printf("\n"); } @@ -6411,27 +6419,21 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsign // The value of the expression isn't used. GenTree* hoist = gtUnusedValNode(hoistExpr); - /* Put the statement in the preheader */ - - INDEBUG(optLoopTable[lnum].lpValidatePreHeader()); - - BasicBlock* preHead = optLoopTable[lnum].lpHead; - // Scan the tree for any new SSA uses. // - optRecordSsaUses(hoist, preHead); + optRecordSsaUses(hoist, preheader); - preHead->CopyFlags(exprBb, BBF_COPY_PROPAGATE); + preheader->CopyFlags(exprBb, BBF_COPY_PROPAGATE); Statement* hoistStmt = gtNewStmt(hoist); // Simply append the statement at the end of the preHead's list. - Statement* firstStmt = preHead->firstStmt(); + Statement* firstStmt = preheader->firstStmt(); if (firstStmt != nullptr) { /* append after last statement */ - Statement* lastStmt = preHead->lastStmt(); + Statement* lastStmt = preheader->lastStmt(); assert(lastStmt->GetNextStmt() == nullptr); lastStmt->SetNextStmt(hoistStmt); @@ -6442,7 +6444,7 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsign { /* Empty pre-header - store the single statement in the block */ - preHead->bbStmtList = hoistStmt; + preheader->bbStmtList = hoistStmt; hoistStmt->SetPrevStmt(hoistStmt); } @@ -6451,7 +6453,7 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsign #ifdef DEBUG if (verbose) { - printf("This hoisted copy placed in PreHeader (" FMT_BB "):\n", preHead->bbNum); + printf("This hoisted copy placed in PreHeader (" FMT_BB "):\n", preheader->bbNum); gtDispTree(hoist); printf("\n"); } @@ -6468,13 +6470,7 @@ void Compiler::optPerformHoistExpr(GenTree* origExpr, BasicBlock* exprBb, unsign { // What is the depth of the loop "lnum"? - ssize_t depth = 0; - unsigned lnumIter = lnum; - while (optLoopTable[lnumIter].lpParent != BasicBlock::NOT_IN_LOOP) - { - depth++; - lnumIter = optLoopTable[lnumIter].lpParent; - } + ssize_t depth = optLoopDepth((unsigned)(m_newToOldLoop[loop->GetIndex()] - optLoopTable)); NodeToTestDataMap* testData = GetNodeTestData(); @@ -6586,6 +6582,10 @@ PhaseStatus Compiler::optHoistLoopCode() // Consider all the loop nests, in inner-to-outer order // + // TODO-Quirk: Switch this to postorder over the loops, instead of this + // loop tree based thing. It is not the exact same order, but it will still + // process child loops before parent loops. + bool modified = false; LoopHoistContext hoistCtxt(this); for (unsigned lnum = 0; lnum < optLoopCount; lnum++) @@ -6666,7 +6666,7 @@ bool Compiler::optHoistLoopNest(unsigned lnum, LoopHoistContext* hoistCtxt) } } - modified |= optHoistThisLoop(lnum, hoistCtxt); + modified |= optHoistThisLoop(m_oldToNewLoop[lnum], hoistCtxt); return modified; } @@ -6675,42 +6675,39 @@ bool Compiler::optHoistLoopNest(unsigned lnum, LoopHoistContext* hoistCtxt) // optHoistThisLoop: run loop hoisting for the indicated loop // // Arguments: -// lnum - loop to process +// loop - loop to process // hoistCtxt - context for the hoisting // // Returns: // true if any hoisting was done // -bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) +bool Compiler::optHoistThisLoop(FlowGraphNaturalLoop* loop, LoopHoistContext* hoistCtxt) { - LoopDsc* pLoopDsc = &optLoopTable[lnum]; - - /* If loop was removed continue */ - - if (pLoopDsc->lpIsRemoved()) - { - JITDUMP(" ... not hoisting " FMT_LP ": removed\n", lnum); - return false; - } - // Ensure the per-loop sets/tables are empty. hoistCtxt->m_curLoopVnInvariantCache.RemoveAll(); + const LoopSideEffects& sideEffs = m_loopSideEffects[loop->GetIndex()]; + #ifdef DEBUG if (verbose) { - printf("optHoistThisLoop for loop " FMT_LP " <" FMT_BB ".." FMT_BB ">:\n", lnum, pLoopDsc->lpTop->bbNum, - pLoopDsc->lpBottom->bbNum); - printf(" Loop body %s a call\n", (pLoopDsc->lpFlags & LPFLG_CONTAINS_CALL) ? "contains" : "does not contain"); - printf(" Loop has %s\n", (pLoopDsc->lpExitCnt == 1) ? "single exit" : "multiple exits"); + printf("optHoistThisLoop for loop " FMT_LP " (head: " FMT_BB "):\n", loop->GetIndex(), + loop->GetHeader()->bbNum); + printf(" Loop body %s a call\n", sideEffs.ContainsCall ? "contains" : "does not contain"); + printf(" Loop has %s\n", (loop->ExitEdges().size() == 1) ? "single exit" : "multiple exits"); + printf(" Blocks:\n"); + loop->VisitLoopBlocksReversePostOrder([](BasicBlock* bb) { + printf(" " FMT_BB "\n", bb->bbNum); + return BasicBlockVisit::Continue; + }); } #endif - VARSET_TP loopVars(VarSetOps::Intersection(this, pLoopDsc->lpVarInOut, pLoopDsc->lpVarUseDef)); + VARSET_TP loopVars(VarSetOps::Intersection(this, sideEffs.VarInOut, sideEffs.VarUseDef)); - pLoopDsc->lpVarInOutCount = VarSetOps::Count(this, pLoopDsc->lpVarInOut); - pLoopDsc->lpLoopVarCount = VarSetOps::Count(this, loopVars); - pLoopDsc->lpHoistedExprCount = 0; + hoistCtxt->m_loopVarInOutCount = VarSetOps::Count(this, sideEffs.VarInOut); + hoistCtxt->m_loopVarCount = VarSetOps::Count(this, loopVars); + hoistCtxt->m_hoistedExprCount = 0; #ifndef TARGET_64BIT @@ -6720,7 +6717,7 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) // the Counts such that each TYP_LONG variable counts twice. // VARSET_TP loopLongVars(VarSetOps::Intersection(this, loopVars, lvaLongVars)); - VARSET_TP inOutLongVars(VarSetOps::Intersection(this, pLoopDsc->lpVarInOut, lvaLongVars)); + VARSET_TP inOutLongVars(VarSetOps::Intersection(this, sideEffs.VarInOut, lvaLongVars)); #ifdef DEBUG if (verbose) @@ -6729,21 +6726,21 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) dumpConvertedVarSet(this, lvaLongVars); } #endif - pLoopDsc->lpLoopVarCount += VarSetOps::Count(this, loopLongVars); - pLoopDsc->lpVarInOutCount += VarSetOps::Count(this, inOutLongVars); + hoistCtxt->m_loopVarCount += VarSetOps::Count(this, loopLongVars); + hoistCtxt->m_loopVarInOutCount += VarSetOps::Count(this, inOutLongVars); } #endif // !TARGET_64BIT #ifdef DEBUG if (verbose) { - printf("\n USEDEF (%d)=", VarSetOps::Count(this, pLoopDsc->lpVarUseDef)); - dumpConvertedVarSet(this, pLoopDsc->lpVarUseDef); + printf("\n USEDEF (%d)=", VarSetOps::Count(this, sideEffs.VarUseDef)); + dumpConvertedVarSet(this, sideEffs.VarUseDef); - printf("\n INOUT (%d)=", pLoopDsc->lpVarInOutCount); - dumpConvertedVarSet(this, pLoopDsc->lpVarInOut); + printf("\n INOUT (%d)=", hoistCtxt->m_loopVarInOutCount); + dumpConvertedVarSet(this, sideEffs.VarInOut); - printf("\n LOOPVARS(%d)=", pLoopDsc->lpLoopVarCount); + printf("\n LOOPVARS(%d)=", hoistCtxt->m_loopVarCount); dumpConvertedVarSet(this, loopVars); printf("\n"); } @@ -6752,22 +6749,21 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) if (!VarSetOps::IsEmpty(this, lvaFloatVars)) { VARSET_TP loopFPVars(VarSetOps::Intersection(this, loopVars, lvaFloatVars)); - VARSET_TP inOutFPVars(VarSetOps::Intersection(this, pLoopDsc->lpVarInOut, lvaFloatVars)); - - pLoopDsc->lpLoopVarFPCount = VarSetOps::Count(this, loopFPVars); - pLoopDsc->lpVarInOutFPCount = VarSetOps::Count(this, inOutFPVars); - pLoopDsc->lpHoistedFPExprCount = 0; + VARSET_TP inOutFPVars(VarSetOps::Intersection(this, sideEffs.VarInOut, lvaFloatVars)); - pLoopDsc->lpLoopVarCount -= pLoopDsc->lpLoopVarFPCount; - pLoopDsc->lpVarInOutCount -= pLoopDsc->lpVarInOutFPCount; + hoistCtxt->m_loopVarFPCount = VarSetOps::Count(this, loopFPVars); + hoistCtxt->m_loopVarInOutFPCount = VarSetOps::Count(this, inOutFPVars); + hoistCtxt->m_hoistedFPExprCount = 0; + hoistCtxt->m_loopVarCount -= hoistCtxt->m_loopVarFPCount; + hoistCtxt->m_loopVarInOutCount -= hoistCtxt->m_loopVarInOutFPCount; #ifdef DEBUG if (verbose) { - printf(" INOUT-FP(%d)=", pLoopDsc->lpVarInOutFPCount); + printf(" INOUT-FP(%d)=", hoistCtxt->m_loopVarInOutFPCount); dumpConvertedVarSet(this, inOutFPVars); - printf("\n LOOPV-FP(%d)=", pLoopDsc->lpLoopVarFPCount); + printf("\n LOOPV-FP(%d)=", hoistCtxt->m_loopVarFPCount); dumpConvertedVarSet(this, loopFPVars); printf("\n"); @@ -6776,9 +6772,9 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) } else // lvaFloatVars is empty { - pLoopDsc->lpLoopVarFPCount = 0; - pLoopDsc->lpVarInOutFPCount = 0; - pLoopDsc->lpHoistedFPExprCount = 0; + hoistCtxt->m_loopVarFPCount = 0; + hoistCtxt->m_loopVarInOutFPCount = 0; + hoistCtxt->m_hoistedFPExprCount = 0; } // Find the set of definitely-executed blocks. @@ -6837,19 +6833,24 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) // convenient). But note that it is arbitrary because there is not guaranteed execution order amongst // the child loops. - for (BasicBlock::loopNumber childLoop = pLoopDsc->lpChild; // - childLoop != BasicBlock::NOT_IN_LOOP; // - childLoop = optLoopTable[childLoop].lpSibling) + // TODO-Quirk: Switch this order off the old loop table + LoopDsc* oldLoop = m_newToOldLoop[loop->GetIndex()]; + + for (BasicBlock::loopNumber childLoopNum = oldLoop->lpChild; childLoopNum != BasicBlock::NOT_IN_LOOP; + childLoopNum = optLoopTable[childLoopNum].lpSibling) { - if (optLoopTable[childLoop].lpIsRemoved()) + if (optLoopTable[childLoopNum].lpIsRemoved()) { continue; } - INDEBUG(optLoopTable[childLoop].lpValidatePreHeader()); - BasicBlock* childPreHead = optLoopTable[childLoop].lpHead; - if (pLoopDsc->lpExitCnt == 1) + + FlowGraphNaturalLoop* childLoop = m_oldToNewLoop[childLoopNum]; + + assert(childLoop->EntryEdges().size() == 1); + BasicBlock* childPreHead = childLoop->EntryEdge(0)->getSourceBlock(); + if (loop->ExitEdges().size() == 1) { - if (fgSsaDomTree->Dominates(childPreHead, pLoopDsc->lpExit)) + if (fgSsaDomTree->Dominates(childPreHead, loop->ExitEdges()[0]->getSourceBlock())) { // If the child loop pre-header dominates the exit, it will get added in the dominator tree // loop below. @@ -6859,7 +6860,7 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) else { // If the child loop pre-header is the loop entry for a multi-exit loop, it will get added below. - if (childPreHead == pLoopDsc->lpEntry) + if (childPreHead == loop->GetHeader()) { continue; } @@ -6868,24 +6869,24 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) defExec.Push(childPreHead); } - if (pLoopDsc->lpExitCnt == 1) + if (loop->ExitEdges().size() == 1) { - assert(pLoopDsc->lpExit != nullptr); + BasicBlock* exiting = loop->ExitEdges()[0]->getSourceBlock(); JITDUMP(" Considering hoisting in blocks that either dominate exit block " FMT_BB ", or pre-headers of nested loops, if any:\n", - pLoopDsc->lpExit->bbNum); + exiting->bbNum); // Push dominators, until we reach "entry" or exit the loop. - BasicBlock* cur = pLoopDsc->lpExit; - while ((cur != nullptr) && (cur != pLoopDsc->lpEntry)) + BasicBlock* cur = exiting; + while ((cur != nullptr) && (cur != loop->GetHeader())) { JITDUMP(" -- " FMT_BB " (dominate exit block)\n", cur->bbNum); - assert(pLoopDsc->lpContains(cur)); + assert(loop->ContainsBlock(cur)); defExec.Push(cur); cur = cur->bbIDom; } - noway_assert(cur == pLoopDsc->lpEntry); + noway_assert(cur == loop->GetHeader()); } else // More than one exit { @@ -6893,23 +6894,21 @@ bool Compiler::optHoistThisLoop(unsigned lnum, LoopHoistContext* hoistCtxt) // We could in the future do better. JITDUMP(" Considering hoisting in entry block " FMT_BB " because " FMT_LP " has more than one exit\n", - pLoopDsc->lpEntry->bbNum, lnum); + loop->GetHeader()->bbNum, loop->GetIndex()); } - JITDUMP(" -- " FMT_BB " (entry block)\n", pLoopDsc->lpEntry->bbNum); - defExec.Push(pLoopDsc->lpEntry); + JITDUMP(" -- " FMT_BB " (entry block)\n", loop->GetHeader()->bbNum); + defExec.Push(loop->GetHeader()); - optHoistLoopBlocks(lnum, &defExec, hoistCtxt); + optHoistLoopBlocks(loop, &defExec, hoistCtxt); - const unsigned numHoisted = pLoopDsc->lpHoistedFPExprCount + pLoopDsc->lpHoistedExprCount; + const unsigned numHoisted = hoistCtxt->m_hoistedFPExprCount + hoistCtxt->m_hoistedExprCount; return numHoisted > 0; } -bool Compiler::optIsProfitableToHoistTree(GenTree* tree, unsigned lnum) +bool Compiler::optIsProfitableToHoistTree(GenTree* tree, FlowGraphNaturalLoop* loop, LoopHoistContext* hoistCtxt) { - LoopDsc* pLoopDsc = &optLoopTable[lnum]; - - bool loopContainsCall = (pLoopDsc->lpFlags & LPFLG_CONTAINS_CALL) != 0; + bool loopContainsCall = m_loopSideEffects[loop->GetIndex()].ContainsCall; int availRegCount; int hoistedExprCount; @@ -6918,9 +6917,9 @@ bool Compiler::optIsProfitableToHoistTree(GenTree* tree, unsigned lnum) if (varTypeUsesIntReg(tree)) { - hoistedExprCount = pLoopDsc->lpHoistedExprCount; - loopVarCount = pLoopDsc->lpLoopVarCount; - varInOutCount = pLoopDsc->lpVarInOutCount; + hoistedExprCount = hoistCtxt->m_hoistedExprCount; + loopVarCount = hoistCtxt->m_loopVarCount; + varInOutCount = hoistCtxt->m_loopVarInOutCount; availRegCount = CNT_CALLEE_SAVED - 1; if (!loopContainsCall) @@ -6939,9 +6938,9 @@ bool Compiler::optIsProfitableToHoistTree(GenTree* tree, unsigned lnum) { assert(varTypeUsesFloatReg(tree)); - hoistedExprCount = pLoopDsc->lpHoistedFPExprCount; - loopVarCount = pLoopDsc->lpLoopVarFPCount; - varInOutCount = pLoopDsc->lpVarInOutFPCount; + hoistedExprCount = hoistCtxt->m_hoistedFPExprCount; + loopVarCount = hoistCtxt->m_loopVarFPCount; + varInOutCount = hoistCtxt->m_loopVarInOutFPCount; availRegCount = CNT_CALLEE_SAVED_FLOAT; if (!loopContainsCall) @@ -7023,22 +7022,12 @@ bool Compiler::optIsProfitableToHoistTree(GenTree* tree, unsigned lnum) // void Compiler::optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, ValueNum memoryVN) { - // If tree is not in a loop, we don't need to track its loop dependence. - // - unsigned const loopNum = block->bbNatLoopNum; - - assert(loopNum != BasicBlock::NOT_IN_LOOP); - // Find the loop associated with this memory VN. // - unsigned updateLoopNum = vnStore->LoopOfVN(memoryVN); + FlowGraphNaturalLoop* updateLoop = vnStore->LoopOfVN(memoryVN); - if (updateLoopNum >= BasicBlock::MAX_LOOP_NUM) + if (updateLoop == nullptr) { - // There should be only two special non-loop loop nums. - // - assert((updateLoopNum == BasicBlock::MAX_LOOP_NUM) || (updateLoopNum == BasicBlock::NOT_IN_LOOP)); - // memoryVN defined outside of any loop, we can ignore. // JITDUMP(" ==> Not updating loop memory dependence of [%06u], memory " FMT_VN " not defined in a loop\n", @@ -7046,36 +7035,26 @@ void Compiler::optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, V return; } - // If the loop was removed, then record the dependence in the nearest enclosing loop, if any. - // - while (optLoopTable[updateLoopNum].lpIsRemoved()) + // TODO-Quirk: Remove + if (m_newToOldLoop[updateLoop->GetIndex()] == nullptr) { - unsigned const updateParentLoopNum = optLoopTable[updateLoopNum].lpParent; - - if (updateParentLoopNum == BasicBlock::NOT_IN_LOOP) - { - // Memory VN was defined in a loop, but no longer. - // - JITDUMP(" ==> Not updating loop memory dependence of [%06u], memory " FMT_VN - " no longer defined in a loop\n", - dspTreeID(tree), memoryVN); - break; - } - - JITDUMP(" ==> " FMT_LP " removed, updating dependence to parent " FMT_LP "\n", updateLoopNum, - updateParentLoopNum); - - updateLoopNum = updateParentLoopNum; + return; } + assert(!m_newToOldLoop[updateLoop->GetIndex()]->lpIsRemoved()); + // If the update block is not the header of a loop containing // block, we can also ignore the update. // - if (!optLoopContains(updateLoopNum, loopNum)) + if (!updateLoop->ContainsBlock(block)) { +#ifdef DEBUG + FlowGraphNaturalLoop* blockLoop = m_blockToLoop->GetLoop(block); + JITDUMP(" ==> Not updating loop memory dependence of [%06u]/" FMT_LP ", memory " FMT_VN "/" FMT_LP - " is not defined in an enclosing loop\n", - dspTreeID(tree), loopNum, memoryVN, updateLoopNum); + " is not defined in a contained block\n", + dspTreeID(tree), blockLoop->GetIndex(), memoryVN, updateLoop->GetIndex()); +#endif return; } @@ -7088,17 +7067,19 @@ void Compiler::optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, V if (map->Lookup(tree, &mapBlock)) { - unsigned const mapLoopNum = mapBlock->bbNatLoopNum; - - // If the update loop contains the existing map loop, - // the existing map loop is more constraining. So no + // If the update loop contains the existing map block, + // the existing entry is more constraining. So no // update needed. // - if (optLoopContains(updateLoopNum, mapLoopNum)) + if (updateLoop->ContainsBlock(mapBlock)) { +#ifdef DEBUG + FlowGraphNaturalLoop* mapLoop = m_blockToLoop->GetLoop(mapBlock); + JITDUMP(" ==> Not updating loop memory dependence of [%06u]; alrady constrained to " FMT_LP " nested in " FMT_LP "\n", - dspTreeID(tree), mapLoopNum, updateLoopNum); + dspTreeID(tree), mapLoop->GetIndex(), updateLoop->GetIndex()); +#endif return; } } @@ -7106,8 +7087,9 @@ void Compiler::optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, V // MemoryVN now describes the most constraining loop memory dependence // we know of. Update the map. // - JITDUMP(" ==> Updating loop memory dependence of [%06u] to " FMT_LP "\n", dspTreeID(tree), updateLoopNum); - map->Set(tree, optLoopTable[updateLoopNum].lpEntry, NodeToLoopMemoryBlockMap::Overwrite); + JITDUMP(" ==> Updating loop memory dependence of [%06u] to " FMT_LP "\n", dspTreeID(tree), + updateLoop->GetIndex()); + map->Set(tree, updateLoop->GetHeader(), NodeToLoopMemoryBlockMap::Overwrite); } //------------------------------------------------------------------------ @@ -7129,11 +7111,62 @@ void Compiler::optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree) } } +//------------------------------------------------------------------------ +// AddVariableLiveness: Adds the variable liveness information for 'blk' +// +// Arguments: +// comp - Compiler instance +// blk - Block whose liveness is to be added +// +void LoopSideEffects::AddVariableLiveness(Compiler* comp, BasicBlock* blk) +{ + VarSetOps::UnionD(comp, VarInOut, blk->bbLiveIn); + VarSetOps::UnionD(comp, VarInOut, blk->bbLiveOut); + + VarSetOps::UnionD(comp, VarUseDef, blk->bbVarUse); + VarSetOps::UnionD(comp, VarUseDef, blk->bbVarDef); +} + +//------------------------------------------------------------------------ +// AddModifiedField: Record that a field is modified in the loop. +// +// Arguments: +// comp - Compiler instance +// fldHnd - Field handle being modified +// fieldKind - Kind of field +// +void LoopSideEffects::AddModifiedField(Compiler* comp, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind) +{ + if (FieldsModified == nullptr) + { + FieldsModified = new (comp->getAllocatorLoopHoist()) FieldHandleSet(comp->getAllocatorLoopHoist()); + } + FieldsModified->Set(fldHnd, fieldKind, FieldHandleSet::Overwrite); +} + +//------------------------------------------------------------------------ +// AddModifiedElemType: Record that an array with the specified element type is +// being modified. +// +// Arguments: +// comp - Compiler instance +// structHnd - Handle for struct. Can also be an encoding of a primitive +// handle, see {Encode/Decode}ElemType. +// +void LoopSideEffects::AddModifiedElemType(Compiler* comp, CORINFO_CLASS_HANDLE structHnd) +{ + if (ArrayElemTypesModified == nullptr) + { + ArrayElemTypesModified = new (comp->getAllocatorLoopHoist()) ClassHandleSet(comp->getAllocatorLoopHoist()); + } + ArrayElemTypesModified->Set(structHnd, true, ClassHandleSet::Overwrite); +} + //------------------------------------------------------------------------ // optHoistLoopBlocks: Hoist invariant expression out of the loop. // // Arguments: -// loopNum - The number of the loop +// loop - The loop // blocks - A stack of blocks belonging to the loop // hoistContext - The loop hoist context // @@ -7142,7 +7175,9 @@ void Compiler::optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree) // the loop, in the execution order, starting with the loop entry // block on top of the stack. // -void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blocks, LoopHoistContext* hoistContext) +void Compiler::optHoistLoopBlocks(FlowGraphNaturalLoop* loop, + ArrayStack* blocks, + LoopHoistContext* hoistContext) { class HoistVisitor : public GenTreeVisitor { @@ -7172,11 +7207,11 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo } }; - ArrayStack m_valueStack; - bool m_beforeSideEffect; - unsigned m_loopNum; - LoopHoistContext* m_hoistContext; - BasicBlock* m_currentBlock; + ArrayStack m_valueStack; + bool m_beforeSideEffect; + FlowGraphNaturalLoop* m_loop; + LoopHoistContext* m_hoistContext; + BasicBlock* m_currentBlock; bool IsNodeHoistable(GenTree* node) { @@ -7204,7 +7239,7 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo { ValueNum vn = tree->gtVNPair.GetLiberal(); bool vnIsInvariant = - m_compiler->optVNIsLoopInvariant(vn, m_loopNum, &m_hoistContext->m_curLoopVnInvariantCache); + m_compiler->optVNIsLoopInvariant(vn, m_loop, &m_hoistContext->m_curLoopVnInvariantCache); // Even though VN is invariant in the loop (say a constant) its value may depend on position // of tree, so for loop hoisting we must also check that any memory read by tree @@ -7269,7 +7304,7 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo ValueNum loopMemoryVN = m_compiler->GetMemoryPerSsaData(loopEntryBlock->bbMemorySsaNumIn[memoryKind]) ->m_vnPair.GetLiberal(); - if (!m_compiler->optVNIsLoopInvariant(loopMemoryVN, m_loopNum, + if (!m_compiler->optVNIsLoopInvariant(loopMemoryVN, m_loop, &m_hoistContext->m_curLoopVnInvariantCache)) { return false; @@ -7290,11 +7325,11 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo UseExecutionOrder = true, }; - HoistVisitor(Compiler* compiler, unsigned loopNum, LoopHoistContext* hoistContext) + HoistVisitor(Compiler* compiler, FlowGraphNaturalLoop* loop, LoopHoistContext* hoistContext) : GenTreeVisitor(compiler) , m_valueStack(compiler->getAllocator(CMK_LoopHoist)) , m_beforeSideEffect(true) - , m_loopNum(loopNum) + , m_loop(loop) , m_hoistContext(hoistContext) , m_currentBlock(nullptr) { @@ -7312,7 +7347,7 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo // hoist the top node? if (top.m_hoistable) { - m_compiler->optHoistCandidate(stmt->GetRootNode(), block, m_loopNum, m_hoistContext); + m_compiler->optHoistCandidate(stmt->GetRootNode(), block, m_loop, m_hoistContext); } else { @@ -7351,8 +7386,9 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo bool isInvariant = lclVar->HasSsaName(); // and the SSA definition must be outside the loop we're hoisting from ... isInvariant = isInvariant && - !m_compiler->optLoopTable[m_loopNum].lpContains( + !m_loop->ContainsBlock( m_compiler->lvaGetDesc(lclNum)->GetPerSsaData(lclVar->GetSsaNum())->GetBlock()); + // and the VN of the tree is considered invariant as well. // // TODO-CQ: This VN invariance check should not be necessary and in some cases it is conservative - it @@ -7661,7 +7697,7 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo if (IsHoistableOverExcepSibling(value.Node(), hasExcep)) { - m_compiler->optHoistCandidate(value.Node(), m_currentBlock, m_loopNum, m_hoistContext); + m_compiler->optHoistCandidate(value.Node(), m_currentBlock, m_loop, m_hoistContext); } // Don't hoist this tree again. @@ -7707,19 +7743,16 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo } }; - LoopDsc* loopDsc = &optLoopTable[loopNum]; - assert(blocks->Top() == loopDsc->lpEntry); - - HoistVisitor visitor(this, loopNum, hoistContext); + HoistVisitor visitor(this, loop, hoistContext); while (!blocks->Empty()) { BasicBlock* block = blocks->Pop(); weight_t blockWeight = block->getBBWeight(this); - JITDUMP("\n optHoistLoopBlocks " FMT_BB " (weight=%6s) of loop " FMT_LP " <" FMT_BB ".." FMT_BB ">\n", - block->bbNum, refCntWtd2str(blockWeight, /* padForDecimalPlaces */ true), loopNum, - loopDsc->lpTop->bbNum, loopDsc->lpBottom->bbNum); + JITDUMP("\n optHoistLoopBlocks " FMT_BB " (weight=%6s) of loop " FMT_LP " (head: " FMT_BB ")\n", + block->bbNum, refCntWtd2str(blockWeight, /* padForDecimalPlaces */ true), loop->GetIndex(), + loop->GetHeader()->bbNum); if (blockWeight < (BB_UNITY_WEIGHT / 10)) { @@ -7733,12 +7766,13 @@ void Compiler::optHoistLoopBlocks(unsigned loopNum, ArrayStack* blo hoistContext->ResetHoistedInCurLoop(); } -void Compiler::optHoistCandidate(GenTree* tree, BasicBlock* treeBb, unsigned lnum, LoopHoistContext* hoistCtxt) +void Compiler::optHoistCandidate(GenTree* tree, + BasicBlock* treeBb, + FlowGraphNaturalLoop* loop, + LoopHoistContext* hoistCtxt) { - assert(lnum != BasicBlock::NOT_IN_LOOP); - // It must pass the hoistable profitablity tests for this loop level - if (!optIsProfitableToHoistTree(tree, lnum)) + if (!optIsProfitableToHoistTree(tree, loop, hoistCtxt)) { JITDUMP(" ... not profitable to hoist\n"); return; @@ -7749,48 +7783,49 @@ void Compiler::optHoistCandidate(GenTree* tree, BasicBlock* treeBb, unsigned lnu // already hoisted this expression in the current loop, so don't hoist this expression. JITDUMP(" [%06u] ... already hoisted " FMT_VN " in " FMT_LP "\n ", dspTreeID(tree), - tree->gtVNPair.GetLiberal(), lnum); + tree->gtVNPair.GetLiberal(), loop->GetIndex()); return; } // We should already have a pre-header for the loop. - INDEBUG(optLoopTable[lnum].lpValidatePreHeader()); + assert(loop->EntryEdges().size() == 1); + BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); // If the block we're hoisting from and the pre-header are in different EH regions, don't hoist. // TODO: we could probably hoist things that won't raise exceptions, such as constants. - if (!BasicBlock::sameTryRegion(optLoopTable[lnum].lpHead, treeBb)) + if (!BasicBlock::sameTryRegion(preheader, treeBb)) { JITDUMP(" ... not hoisting in " FMT_LP ", eh region constraint (pre-header try index %d, candidate " FMT_BB " try index %d\n", - lnum, optLoopTable[lnum].lpHead->bbTryIndex, treeBb->bbNum, treeBb->bbTryIndex); + loop->GetIndex(), preheader->bbTryIndex, treeBb->bbNum, treeBb->bbTryIndex); return; } // Expression can be hoisted - optPerformHoistExpr(tree, treeBb, lnum); + optPerformHoistExpr(tree, treeBb, loop); // Increment lpHoistedExprCount or lpHoistedFPExprCount if (!varTypeIsFloating(tree->TypeGet())) { - optLoopTable[lnum].lpHoistedExprCount++; + hoistCtxt->m_hoistedExprCount++; #ifndef TARGET_64BIT // For our 32-bit targets Long types take two registers. if (varTypeIsLong(tree->TypeGet())) { - optLoopTable[lnum].lpHoistedExprCount++; + hoistCtxt->m_hoistedExprCount++; } #endif } else // Floating point expr hoisted { - optLoopTable[lnum].lpHoistedFPExprCount++; + hoistCtxt->m_hoistedFPExprCount++; } // Record the hoisted expression in hoistCtxt hoistCtxt->GetHoistedInCurLoop(this)->Set(tree->gtVNPair.GetLiberal(), true); } -bool Compiler::optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* loopVnInvariantCache) +bool Compiler::optVNIsLoopInvariant(ValueNum vn, FlowGraphNaturalLoop* loop, VNSet* loopVnInvariantCache) { // If it is not a VN, is not loop-invariant. if (vn == ValueNumStore::NoVN) @@ -7821,27 +7856,33 @@ bool Compiler::optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* loopVnInv unsigned lclNum = funcApp.m_args[0]; unsigned ssaNum = funcApp.m_args[1]; LclSsaVarDsc* ssaDef = lvaTable[lclNum].GetPerSsaData(ssaNum); - res = !optLoopContains(lnum, ssaDef->GetBlock()->bbNatLoopNum); + res = !loop->ContainsBlock(ssaDef->GetBlock()); } else if (funcApp.m_func == VNF_PhiMemoryDef) { BasicBlock* defnBlk = reinterpret_cast(vnStore->ConstantValue(funcApp.m_args[0])); - res = !optLoopContains(lnum, defnBlk->bbNatLoopNum); + res = !loop->ContainsBlock(defnBlk); } else if (funcApp.m_func == VNF_MemOpaque) { - const unsigned vnLoopNum = funcApp.m_args[0]; + const unsigned loopIndex = funcApp.m_args[0]; - // Check for the special "ambiguous" loop MemOpaque VN. + // Check for the special "ambiguous" loop index. // This is considered variant in every loop. // - if (vnLoopNum == BasicBlock::MAX_LOOP_NUM) + if (loopIndex == ValueNumStore::UnknownLoop) { res = false; } + else if (loopIndex == ValueNumStore::NoLoop) + { + res = true; + } else { - res = !optLoopContains(lnum, vnLoopNum); + FlowGraphNaturalLoop* otherLoop = m_loops->GetLoopByIndex(loopIndex); + assert(otherLoop != nullptr); + res = !loop->ContainsLoop(otherLoop); } } else @@ -7856,8 +7897,17 @@ bool Compiler::optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* loopVnInv if (i == 3) { - const unsigned vnLoopNum = funcApp.m_args[3]; - res = !optLoopContains(lnum, vnLoopNum); + const unsigned loopIndex = funcApp.m_args[3]; + assert((loopIndex == ValueNumStore::NoLoop) || (loopIndex < m_loops->NumLoops())); + if (loopIndex == ValueNumStore::NoLoop) + { + res = true; + } + else + { + FlowGraphNaturalLoop* otherLoop = m_loops->GetLoopByIndex(loopIndex); + res = !loop->ContainsLoop(otherLoop); + } break; } } @@ -7865,7 +7915,7 @@ bool Compiler::optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* loopVnInv // TODO-CQ: We need to either make sure that *all* VN functions // always take VN args, or else have a list of arg positions to exempt, as implicitly // constant. - if (!optVNIsLoopInvariant(funcApp.m_args[i], lnum, loopVnInvariantCache)) + if (!optVNIsLoopInvariant(funcApp.m_args[i], loop, loopVnInvariantCache)) { res = false; break; @@ -8352,27 +8402,54 @@ bool Compiler::optBlockIsLoopEntry(BasicBlock* blk, unsigned* pLnum) return false; } +LoopSideEffects::LoopSideEffects() : VarInOut(VarSetOps::UninitVal()), VarUseDef(VarSetOps::UninitVal()) +{ + for (MemoryKind mk : allMemoryKinds()) + { + HasMemoryHavoc[mk] = false; + } +} + void Compiler::optComputeLoopSideEffects() { - unsigned lnum; - for (lnum = 0; lnum < optLoopCount; lnum++) + m_loopSideEffects = + m_loops->NumLoops() == 0 ? nullptr : (new (this, CMK_LoopOpt) LoopSideEffects[m_loops->NumLoops()]); + + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) { - VarSetOps::AssignNoCopy(this, optLoopTable[lnum].lpVarInOut, VarSetOps::MakeEmpty(this)); - VarSetOps::AssignNoCopy(this, optLoopTable[lnum].lpVarUseDef, VarSetOps::MakeEmpty(this)); - optLoopTable[lnum].lpFlags &= ~LPFLG_CONTAINS_CALL; + m_loopSideEffects[loop->GetIndex()].VarInOut = VarSetOps::MakeEmpty(this); + m_loopSideEffects[loop->GetIndex()].VarUseDef = VarSetOps::MakeEmpty(this); } - for (lnum = 0; lnum < optLoopCount; lnum++) + BasicBlock** postOrder = m_dfsTree->GetPostOrder(); + unsigned postOrderCount = m_dfsTree->GetPostOrderCount(); + + // Iterate all blocks in loops. + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) { - if (optLoopTable[lnum].lpIsRemoved()) + if (loop->GetParent() != nullptr) { continue; } - if (optLoopTable[lnum].lpParent == BasicBlock::NOT_IN_LOOP) - { // Is outermost... - optComputeLoopNestSideEffects(lnum); - } + // The side effect code benefits from seeing things in RPO as it has some + // limited treatment assignments it has seen the value of. + loop->VisitLoopBlocksReversePostOrder([=](BasicBlock* loopBlock) { + FlowGraphNaturalLoop* loop = m_blockToLoop->GetLoop(loopBlock); + + // TODO-Quirk: Remove + while ((loop != nullptr) && (m_newToOldLoop[loop->GetIndex()] == nullptr)) + { + loop = loop->GetParent(); + } + + if (loop != nullptr) + { + optComputeLoopSideEffectsOfBlock(loopBlock, loop); + } + + return BasicBlockVisit::Continue; + }); } } @@ -8402,58 +8479,26 @@ void Compiler::optComputeInterestingVarSets() } } -void Compiler::optComputeLoopNestSideEffects(unsigned lnum) +void Compiler::optRecordLoopNestsMemoryHavoc(FlowGraphNaturalLoop* loop, MemoryKindSet memoryHavoc) { - JITDUMP("optComputeLoopNestSideEffects for " FMT_LP "\n", lnum); - assert(optLoopTable[lnum].lpParent == BasicBlock::NOT_IN_LOOP); // Requires: lnum is outermost. - for (BasicBlock* const bbInLoop : optLoopTable[lnum].LoopBlocks()) - { - if (!optComputeLoopSideEffectsOfBlock(bbInLoop)) - { - // When optComputeLoopSideEffectsOfBlock returns false, we encountered - // a block that was moved into the loop range (by fgReorderBlocks), - // but not marked correctly as being inside the loop. - // We conservatively mark this loop (and any outer loops) - // as having memory havoc side effects. - // - // Record that all loops containing this block have memory havoc effects. - // - optRecordLoopNestsMemoryHavoc(lnum, fullMemoryKindSet); - - // All done, no need to keep visiting more blocks - break; - } - } -} - -void Compiler::optRecordLoopNestsMemoryHavoc(unsigned lnum, MemoryKindSet memoryHavoc) -{ - // We should start out with 'lnum' set to a valid natural loop index - assert(lnum != BasicBlock::NOT_IN_LOOP); - - while (lnum != BasicBlock::NOT_IN_LOOP) + do { for (MemoryKind memoryKind : allMemoryKinds()) { if ((memoryHavoc & memoryKindSet(memoryKind)) != 0) { - optLoopTable[lnum].lpLoopHasMemoryHavoc[memoryKind] = true; + m_loopSideEffects[loop->GetIndex()].HasMemoryHavoc[memoryKind] = true; } } - // Move lnum to the next outtermost loop that we need to mark - lnum = optLoopTable[lnum].lpParent; - } + loop = loop->GetParent(); + } while (loop != nullptr); } -bool Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk) +void Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk, FlowGraphNaturalLoop* mostNestedLoop) { - unsigned mostNestedLoop = blk->bbNatLoopNum; - JITDUMP("optComputeLoopSideEffectsOfBlock " FMT_BB ", mostNestedLoop %d\n", blk->bbNum, mostNestedLoop); - if (mostNestedLoop == BasicBlock::NOT_IN_LOOP) - { - return false; - } + JITDUMP("optComputeLoopSideEffectsOfBlock " FMT_BB ", mostNestedLoop " FMT_LP "\n", blk->bbNum, + mostNestedLoop->GetIndex()); AddVariableLivenessAllContainingLoops(mostNestedLoop, blk); // MemoryKinds for which an in-loop call or store has arbitrary effects. @@ -8475,10 +8520,10 @@ bool Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk) AddContainsCallAllContainingLoops(mostNestedLoop); } - // If we just set LPFLG_CONTAINS_CALL or it was previously set - if (optLoopTable[mostNestedLoop].lpFlags & LPFLG_CONTAINS_CALL) + // If we just marked it as containing a call or it was previously set + if (m_loopSideEffects[mostNestedLoop->GetIndex()].ContainsCall) { - // We can early exit after both memoryHavoc and LPFLG_CONTAINS_CALL are both set to true. + // We can early exit after both memoryHavoc and ContainsCall are both set to true. break; } @@ -8688,74 +8733,76 @@ bool Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk) // Record that all loops containing this block have this kind of memoryHavoc effects. optRecordLoopNestsMemoryHavoc(mostNestedLoop, memoryHavoc); } - return true; } -// Marks the containsCall information to "lnum" and any parent loops. -void Compiler::AddContainsCallAllContainingLoops(unsigned lnum) +// TODO-Quirk: Remove +static bool HasOldChildLoop(Compiler* comp, FlowGraphNaturalLoop* loop) +{ + for (FlowGraphNaturalLoop* child = loop->GetChild(); child != nullptr; child = child->GetSibling()) + { + if (comp->m_newToOldLoop[child->GetIndex()] != nullptr) + return true; + + if (HasOldChildLoop(comp, child)) + return true; + } + + return false; +} + +// Marks the containsCall information to "loop" and any parent loops. +void Compiler::AddContainsCallAllContainingLoops(FlowGraphNaturalLoop* loop) { #if FEATURE_LOOP_ALIGN // If this is the inner most loop, reset the LOOP_ALIGN flag // because a loop having call will not likely to benefit from // alignment - if (optLoopTable[lnum].lpChild == BasicBlock::NOT_IN_LOOP) + if (!HasOldChildLoop(this, loop)) { - BasicBlock* top = optLoopTable[lnum].lpTop; + BasicBlock* top = loop->GetLexicallyTopMostBlock(); top->unmarkLoopAlign(this DEBUG_ARG("Loop with call")); } #endif - assert(0 <= lnum && lnum < optLoopCount); - while (lnum != BasicBlock::NOT_IN_LOOP) + do { - optLoopTable[lnum].lpFlags |= LPFLG_CONTAINS_CALL; - lnum = optLoopTable[lnum].lpParent; - } -} - -// Adds the variable liveness information for 'blk' to 'this' LoopDsc -void Compiler::LoopDsc::AddVariableLiveness(Compiler* comp, BasicBlock* blk) -{ - VarSetOps::UnionD(comp, this->lpVarInOut, blk->bbLiveIn); - VarSetOps::UnionD(comp, this->lpVarInOut, blk->bbLiveOut); - - VarSetOps::UnionD(comp, this->lpVarUseDef, blk->bbVarUse); - VarSetOps::UnionD(comp, this->lpVarUseDef, blk->bbVarDef); + m_loopSideEffects[loop->GetIndex()].ContainsCall = true; + loop = loop->GetParent(); + } while (loop != nullptr); } // Adds the variable liveness information for 'blk' to "lnum" and any parent loops. -void Compiler::AddVariableLivenessAllContainingLoops(unsigned lnum, BasicBlock* blk) +void Compiler::AddVariableLivenessAllContainingLoops(FlowGraphNaturalLoop* loop, BasicBlock* blk) { - assert(0 <= lnum && lnum < optLoopCount); - while (lnum != BasicBlock::NOT_IN_LOOP) + do { - optLoopTable[lnum].AddVariableLiveness(this, blk); - lnum = optLoopTable[lnum].lpParent; - } + m_loopSideEffects[loop->GetIndex()].AddVariableLiveness(this, blk); + loop = loop->GetParent(); + } while (loop != nullptr); } -// Adds "fldHnd" to the set of modified fields of "lnum" and any parent loops. -void Compiler::AddModifiedFieldAllContainingLoops(unsigned lnum, CORINFO_FIELD_HANDLE fldHnd, FieldKindForVN fieldKind) +// Adds "fldHnd" to the set of modified fields of "loop" and any parent loops. +void Compiler::AddModifiedFieldAllContainingLoops(FlowGraphNaturalLoop* loop, + CORINFO_FIELD_HANDLE fldHnd, + FieldKindForVN fieldKind) { - assert(0 <= lnum && lnum < optLoopCount); - while (lnum != BasicBlock::NOT_IN_LOOP) + do { - optLoopTable[lnum].AddModifiedField(this, fldHnd, fieldKind); - lnum = optLoopTable[lnum].lpParent; - } + m_loopSideEffects[loop->GetIndex()].AddModifiedField(this, fldHnd, fieldKind); + loop = loop->GetParent(); + } while (loop != nullptr); } -// Adds "elemType" to the set of modified array element types of "lnum" and any parent loops. -void Compiler::AddModifiedElemTypeAllContainingLoops(unsigned lnum, CORINFO_CLASS_HANDLE elemClsHnd) +// Adds "elemType" to the set of modified array element types of "loop" and any parent loops. +void Compiler::AddModifiedElemTypeAllContainingLoops(FlowGraphNaturalLoop* loop, CORINFO_CLASS_HANDLE elemClsHnd) { - assert(0 <= lnum && lnum < optLoopCount); - while (lnum != BasicBlock::NOT_IN_LOOP) + do { - optLoopTable[lnum].AddModifiedElemType(this, elemClsHnd); - lnum = optLoopTable[lnum].lpParent; - } + m_loopSideEffects[loop->GetIndex()].AddModifiedElemType(this, elemClsHnd); + loop = loop->GetParent(); + } while (loop != nullptr); } //------------------------------------------------------------------------------ diff --git a/src/coreclr/jit/redundantbranchopts.cpp b/src/coreclr/jit/redundantbranchopts.cpp index 285276728ff42..6647e8ebda470 100644 --- a/src/coreclr/jit/redundantbranchopts.cpp +++ b/src/coreclr/jit/redundantbranchopts.cpp @@ -818,7 +818,7 @@ bool Compiler::optJumpThreadCheck(BasicBlock* const block, BasicBlock* const dom { for (BasicBlock* const predBlock : block->PredBlocks()) { - if (m_dfs->Contains(predBlock) && !fgSsaDomTree->Dominates(domBlock, predBlock)) + if (m_dfsTree->Contains(predBlock) && !fgSsaDomTree->Dominates(domBlock, predBlock)) { JITDUMP("Dom " FMT_BB " is stale (does not dominate pred " FMT_BB "); no threading\n", domBlock->bbNum, predBlock->bbNum); diff --git a/src/coreclr/jit/ssabuilder.cpp b/src/coreclr/jit/ssabuilder.cpp index 4cb4dd3271aca..5eaeae1c7ac51 100644 --- a/src/coreclr/jit/ssabuilder.cpp +++ b/src/coreclr/jit/ssabuilder.cpp @@ -415,9 +415,9 @@ void SsaBuilder::InsertPhiFunctions() { JITDUMP("*************** In SsaBuilder::InsertPhiFunctions()\n"); - FlowGraphDfsTree* dfs = m_pCompiler->m_dfs; - BasicBlock** postOrder = dfs->GetPostOrder(); - unsigned count = dfs->GetPostOrderCount(); + FlowGraphDfsTree* dfsTree = m_pCompiler->m_dfsTree; + BasicBlock** postOrder = dfsTree->GetPostOrder(); + unsigned count = dfsTree->GetPostOrderCount(); // Compute dominance frontier. BlkToBlkVectorMap mapDF(m_allocator); @@ -1304,8 +1304,8 @@ void SsaBuilder::Build() m_visitedTraits = BitVecTraits(blockCount, m_pCompiler); m_visited = BitVecOps::MakeEmpty(&m_visitedTraits); - m_pCompiler->m_dfs = m_pCompiler->fgComputeDfs(); - m_pCompiler->fgSsaDomTree = FlowGraphDominatorTree::Build(m_pCompiler->m_dfs); + m_pCompiler->m_dfsTree = m_pCompiler->fgComputeDfs(); + m_pCompiler->fgSsaDomTree = FlowGraphDominatorTree::Build(m_pCompiler->m_dfsTree); EndPhase(PHASE_BUILD_SSA_DOMS); // Compute liveness on the graph. diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index e6ea3c49ea594..773f51f988808 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -2716,7 +2716,7 @@ ValueNum ValueNumStore::VNForFunc(var_types typ, VNFunc func, ValueNum arg0VN, V ValueNum ValueNumStore::VNForFunc( var_types typ, VNFunc func, ValueNum arg0VN, ValueNum arg1VN, ValueNum arg2VN, ValueNum arg3VN) { - assert(arg0VN != NoVN && arg1VN != NoVN && arg2VN != NoVN && arg3VN != NoVN); + assert(arg0VN != NoVN && arg1VN != NoVN && arg2VN != NoVN && ((arg3VN != NoVN) || func == VNF_MapStore)); // Function arguments carry no exceptions. assert(arg0VN == VNNormalValue(arg0VN)); @@ -2765,9 +2765,18 @@ ValueNum ValueNumStore::VNForMapStore(ValueNum map, ValueNum index, ValueNum val { assert(MapIsPrecise(map)); - BasicBlock* const bb = m_pComp->compCurBB; - BasicBlock::loopNumber const loopNum = bb->bbNatLoopNum; - ValueNum const result = VNForFunc(TypeOfVN(map), VNF_MapStore, map, index, value, loopNum); + BasicBlock* const bb = m_pComp->compCurBB; + FlowGraphNaturalLoop* bbLoop = m_pComp->m_blockToLoop->GetLoop(bb); + + // TODO-Quirk: Remove + while ((bbLoop != nullptr) && (m_pComp->m_newToOldLoop[bbLoop->GetIndex()] == nullptr)) + { + bbLoop = bbLoop->GetParent(); + } + + unsigned loopIndex = bbLoop == nullptr ? UINT_MAX : bbLoop->GetIndex(); + + ValueNum const result = VNForFunc(TypeOfVN(map), VNF_MapStore, map, index, value, loopIndex); #ifdef DEBUG if (m_pComp->verbose) @@ -2964,12 +2973,15 @@ ValueNum ValueNumStore::VNForMapSelectInner(ValueNumKind vnk, var_types type, Va // If the current tree is in a loop then record memory dependencies for // hoisting. Note that this function may be called by other phases than VN // (such as VN-based dead store removal). - if ((m_pComp->compCurBB != nullptr) && (m_pComp->compCurTree != nullptr) && - m_pComp->compCurBB->bbNatLoopNum != BasicBlock::NOT_IN_LOOP) + if ((m_pComp->compCurBB != nullptr) && (m_pComp->compCurTree != nullptr)) { - memoryDependencies.ForEach([this](ValueNum vn) { - m_pComp->optRecordLoopMemoryDependence(m_pComp->compCurTree, m_pComp->compCurBB, vn); - }); + FlowGraphNaturalLoop* loop = m_pComp->m_blockToLoop->GetLoop(m_pComp->compCurBB); + if (loop != nullptr) + { + memoryDependencies.ForEach([this](ValueNum vn) { + m_pComp->optRecordLoopMemoryDependence(m_pComp->compCurTree, m_pComp->compCurBB, vn); + }); + } } return result; @@ -5213,14 +5225,18 @@ ValueNum ValueNumStore::EvalUsingMathIdentity(var_types typ, VNFunc func, ValueN // ValueNum ValueNumStore::VNForExpr(BasicBlock* block, var_types type) { - BasicBlock::loopNumber loopNum; - if (block == nullptr) + unsigned loopIndex = ValueNumStore::UnknownLoop; + if (block != nullptr) { - loopNum = BasicBlock::MAX_LOOP_NUM; - } - else - { - loopNum = block->bbNatLoopNum; + FlowGraphNaturalLoop* loop = m_pComp->m_blockToLoop->GetLoop(block); + + // TODO-Quirk: Remove + while ((loop != nullptr) && (m_pComp->m_newToOldLoop[loop->GetIndex()] == nullptr)) + { + loop = loop->GetParent(); + } + + loopIndex = loop == nullptr ? ValueNumStore::NoLoop : loop->GetIndex(); } // VNForFunc(typ, func, vn) but bypasses looking in the cache @@ -5229,7 +5245,7 @@ ValueNum ValueNumStore::VNForExpr(BasicBlock* block, var_types type) unsigned const offsetWithinChunk = c->AllocVN(); VNDefFuncAppFlexible* fapp = c->PointerToFuncApp(offsetWithinChunk, 1); fapp->m_func = VNF_MemOpaque; - fapp->m_args[0] = loopNum; + fapp->m_args[0] = loopIndex; ValueNum resultVN = c->m_baseVN + offsetWithinChunk; return resultVN; @@ -5915,37 +5931,48 @@ var_types ValueNumStore::TypeOfVN(ValueNum vn) const //------------------------------------------------------------------------ // LoopOfVN: If the given value number is VNF_MemOpaque, VNF_MapStore, or -// VNF_MemoryPhiDef, return the loop number where the memory update occurs, -// otherwise returns MAX_LOOP_NUM. +// VNF_MemoryPhiDef, return the loop where the memory update occurs, +// otherwise returns nullptr // // Arguments: // vn - Value number to query // // Return Value: -// The memory loop number, which may be BasicBlock::NOT_IN_LOOP. -// Returns BasicBlock::MAX_LOOP_NUM if this VN is not a memory value number. +// The memory loop. // -BasicBlock::loopNumber ValueNumStore::LoopOfVN(ValueNum vn) +FlowGraphNaturalLoop* ValueNumStore::LoopOfVN(ValueNum vn) { VNFuncApp funcApp; if (GetVNFunc(vn, &funcApp)) { if (funcApp.m_func == VNF_MemOpaque) { - return (BasicBlock::loopNumber)funcApp.m_args[0]; + unsigned index = (unsigned)funcApp.m_args[0]; + if ((index == ValueNumStore::NoLoop) || (index == ValueNumStore::UnknownLoop)) + { + return nullptr; + } + + return m_pComp->m_loops->GetLoopByIndex(index); } else if (funcApp.m_func == VNF_MapStore) { - return (BasicBlock::loopNumber)funcApp.m_args[3]; + unsigned index = (unsigned)funcApp.m_args[3]; + if (index == ValueNumStore::NoLoop) + { + return nullptr; + } + + return m_pComp->m_loops->GetLoopByIndex(index); } else if (funcApp.m_func == VNF_PhiMemoryDef) { BasicBlock* const block = reinterpret_cast(ConstantValue(funcApp.m_args[0])); - return block->bbNatLoopNum; + return m_pComp->m_blockToLoop->GetLoop(block); } } - return BasicBlock::MAX_LOOP_NUM; + return nullptr; } bool ValueNumStore::IsVNConstant(ValueNum vn) @@ -9203,7 +9230,7 @@ void ValueNumStore::vnDumpMapStore(Compiler* comp, VNFuncApp* mapStore) printf(" := "); comp->vnPrint(newValVN, 0); printf("]"); - if (loopNum != BasicBlock::NOT_IN_LOOP) + if (loopNum != ValueNumStore::NoLoop) { printf("@" FMT_LP, loopNum); } @@ -9246,17 +9273,17 @@ void ValueNumStore::vnDumpMemOpaque(Compiler* comp, VNFuncApp* memOpaque) assert(memOpaque->m_func == VNF_MemOpaque); // Precondition. const unsigned loopNum = memOpaque->m_args[0]; - if (loopNum == BasicBlock::NOT_IN_LOOP) + if (loopNum == ValueNumStore::NoLoop) { printf("MemOpaque:NotInLoop"); } - else if (loopNum == BasicBlock::MAX_LOOP_NUM) + else if (loopNum == ValueNumStore::UnknownLoop) { printf("MemOpaque:Indeterminate"); } else { - printf("MemOpaque:L%02u", loopNum); + printf("MemOpaque:" FMT_LP, loopNum); } } @@ -9669,7 +9696,7 @@ class ValueNumberState // bool IsReachable(BasicBlock* bb) { - return m_comp->m_dfs->Contains(bb) && + return m_comp->m_dfsTree->Contains(bb) && !BitVecOps::IsMember(&m_blockTraits, m_provenUnreachableBlocks, bb->bbNum); } @@ -9765,6 +9792,7 @@ PhaseStatus Compiler::fgValueNumber() } } + m_blockToLoop = BlockToNaturalLoopMap::Build(m_loops); // Compute the side effects of loops. optComputeLoopSideEffects(); @@ -9850,8 +9878,8 @@ PhaseStatus Compiler::fgValueNumber() // SSA has already computed a post-order taking EH successors into account. // Visiting that in reverse will ensure we visit a block's predecessors // before itself whenever possible. - BasicBlock** postOrder = m_dfs->GetPostOrder(); - unsigned postOrderCount = m_dfs->GetPostOrderCount(); + BasicBlock** postOrder = m_dfsTree->GetPostOrder(); + unsigned postOrderCount = m_dfsTree->GetPostOrderCount(); for (unsigned i = postOrderCount; i != 0; i--) { BasicBlock* block = postOrder[i - 1]; @@ -10008,11 +10036,11 @@ void Compiler::fgValueNumberBlock(BasicBlock* blk) continue; } - unsigned loopNum; - ValueNum newMemoryVN; - if (optBlockIsLoopEntry(blk, &loopNum)) + ValueNum newMemoryVN; + FlowGraphNaturalLoop* loop = m_blockToLoop->GetLoop(blk); + if ((loop != nullptr) && (loop->GetHeader() == blk) && (m_newToOldLoop[loop->GetIndex()] != nullptr)) { - newMemoryVN = fgMemoryVNForLoopSideEffects(memoryKind, blk, loopNum); + newMemoryVN = fgMemoryVNForLoopSideEffects(memoryKind, blk, loop); } else { @@ -10133,40 +10161,28 @@ void Compiler::fgValueNumberBlock(BasicBlock* blk) compCurBB = nullptr; } -ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, - BasicBlock* entryBlock, - unsigned innermostLoopNum) +ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, + BasicBlock* entryBlock, + FlowGraphNaturalLoop* loop) { - // "loopNum" is the innermost loop for which "blk" is the entry; find the outermost one. - assert(innermostLoopNum != BasicBlock::NOT_IN_LOOP); - unsigned loopsInNest = innermostLoopNum; - unsigned loopNum = innermostLoopNum; - while (loopsInNest != BasicBlock::NOT_IN_LOOP) - { - if (optLoopTable[loopsInNest].lpEntry != entryBlock) - { - break; - } - loopNum = loopsInNest; - loopsInNest = optLoopTable[loopsInNest].lpParent; - } - #ifdef DEBUG if (verbose) { - printf("Computing %s state for block " FMT_BB ", entry block for loops %d to %d:\n", - memoryKindNames[memoryKind], entryBlock->bbNum, innermostLoopNum, loopNum); + printf("Computing %s state for block " FMT_BB ", entry block for loop " FMT_LP ":\n", + memoryKindNames[memoryKind], entryBlock->bbNum, loop->GetIndex()); } #endif // DEBUG + const LoopSideEffects& sideEffs = m_loopSideEffects[loop->GetIndex()]; + // If this loop has memory havoc effects, just use a new, unique VN. - if (optLoopTable[loopNum].lpLoopHasMemoryHavoc[memoryKind]) + if (sideEffs.HasMemoryHavoc[memoryKind]) { ValueNum res = vnStore->VNForExpr(entryBlock, TYP_HEAP); #ifdef DEBUG if (verbose) { - printf(" Loop %d has memory havoc effect; heap state is new unique $%x.\n", loopNum, res); + printf(" Loop " FMT_LP " has memory havoc effect; heap state is new unique $%x.\n", loop->GetIndex(), res); } #endif // DEBUG return res; @@ -10175,12 +10191,14 @@ ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, // Otherwise, find the predecessors of the entry block that are not in the loop. // If there is only one such, use its memory value as the "base." If more than one, // use a new unique VN. + // TODO-Cleanup: Ensure canonicalization creates loop preheaders properly for handlers + // and simplify this logic. BasicBlock* nonLoopPred = nullptr; bool multipleNonLoopPreds = false; for (FlowEdge* pred = BlockPredsWithEH(entryBlock); pred != nullptr; pred = pred->getNextPredEdge()) { BasicBlock* predBlock = pred->getSourceBlock(); - if (!optLoopTable[loopNum].lpContains(predBlock)) + if (!loop->ContainsBlock(predBlock)) { if (nonLoopPred == nullptr) { @@ -10229,11 +10247,10 @@ ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, if (memoryKind == GcHeap) { // First the fields/field maps. - Compiler::LoopDsc::FieldHandleSet* fieldsMod = optLoopTable[loopNum].lpFieldsModified; + FieldHandleSet* fieldsMod = sideEffs.FieldsModified; if (fieldsMod != nullptr) { - for (Compiler::LoopDsc::FieldHandleSet::Node* const ki : - Compiler::LoopDsc::FieldHandleSet::KeyValueIteration(fieldsMod)) + for (FieldHandleSet::Node* const ki : FieldHandleSet::KeyValueIteration(fieldsMod)) { CORINFO_FIELD_HANDLE fldHnd = ki->GetKey(); FieldKindForVN fieldKind = ki->GetValue(); @@ -10256,10 +10273,10 @@ ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, } } // Now do the array maps. - Compiler::LoopDsc::ClassHandleSet* elemTypesMod = optLoopTable[loopNum].lpArrayElemTypesModified; + ClassHandleSet* elemTypesMod = sideEffs.ArrayElemTypesModified; if (elemTypesMod != nullptr) { - for (const CORINFO_CLASS_HANDLE elemClsHnd : Compiler::LoopDsc::ClassHandleSet::KeyIteration(elemTypesMod)) + for (const CORINFO_CLASS_HANDLE elemClsHnd : ClassHandleSet::KeyIteration(elemTypesMod)) { #ifdef DEBUG if (verbose) @@ -10290,10 +10307,8 @@ ValueNum Compiler::fgMemoryVNForLoopSideEffects(MemoryKind memoryKind, // If there were any fields/elements modified, this should have been recorded as havoc // for ByrefExposed. assert(memoryKind == ByrefExposed); - assert((optLoopTable[loopNum].lpFieldsModified == nullptr) || - optLoopTable[loopNum].lpLoopHasMemoryHavoc[memoryKind]); - assert((optLoopTable[loopNum].lpArrayElemTypesModified == nullptr) || - optLoopTable[loopNum].lpLoopHasMemoryHavoc[memoryKind]); + assert((sideEffs.FieldsModified == nullptr) || sideEffs.HasMemoryHavoc[memoryKind]); + assert((sideEffs.ArrayElemTypesModified == nullptr) || sideEffs.HasMemoryHavoc[memoryKind]); } #ifdef DEBUG diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index e6ffb137584bb..f595d439ff37f 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -231,6 +231,12 @@ class ValueNumStore // A second special value, used to indicate that a function evaluation would cause infinite recursion. static const ValueNum RecursiveVN = UINT32_MAX - 1; + // Special value used to represent something that isn't in a loop for VN functions that take loop parameters. + static const unsigned NoLoop = UINT32_MAX; + // Special value used to represent something that may or may not be in a loop, so needs to be handled + // conservatively. + static const unsigned UnknownLoop = UINT32_MAX - 1; + // ================================================================================================== // VNMap - map from something to ValueNum, where something is typically a constant value or a VNFunc // This class has two purposes - to abstract the implementation and to validate the ValueNums @@ -875,8 +881,8 @@ class ValueNumStore // Returns TYP_UNKNOWN if the given value number has not been given a type. var_types TypeOfVN(ValueNum vn) const; - // Returns BasicBlock::MAX_LOOP_NUM if the given value number's loop nest is unknown or ill-defined. - BasicBlock::loopNumber LoopOfVN(ValueNum vn); + // Returns nullptr if the given value number is not dependent on memory defined in a loop. + class FlowGraphNaturalLoop* LoopOfVN(ValueNum vn); // Returns true iff the VN represents a constant. bool IsVNConstant(ValueNum vn);