Skip to content

Commit 733c2a7

Browse files
committed
JIT: revise how we track if nodes have been morphed
Allow remorphing. Require that all nodes be morphed. Ensure that no node is morphed more than a handful of times. Fixes dotnet#6218, dotnet#56962, dotnet#107542
1 parent 016d356 commit 733c2a7

7 files changed

Lines changed: 280 additions & 217 deletions

File tree

‎src/coreclr/jit/compiler.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5345,6 +5345,8 @@ class Compiler
53455345
bool fgGlobalMorph = false; // indicates if we are during the global morphing phase
53465346
// since fgMorphTree can be called from several places
53475347

5348+
bool fgGlobalRemorph = false; // true if we are intentionaly remorphing already morphed nodes
5349+
53485350
bool fgGlobalMorphDone = false;
53495351

53505352
bool impBoxTempInUse; // the temp below is valid and available
@@ -5446,6 +5448,10 @@ class Compiler
54465448
void fgMorphBlock(BasicBlock* block, MorphUnreachableInfo* unreachableInfo = nullptr);
54475449
void fgMorphStmts(BasicBlock* block);
54485450

5451+
#ifdef DEBUG
5452+
void fgPostGlobalMorphChecks();
5453+
#endif
5454+
54495455
void fgMergeBlockReturn(BasicBlock* block);
54505456

54515457
bool fgMorphBlockStmt(BasicBlock* block, Statement* stmt DEBUGARG(const char* msg), bool invalidateDFSTreeOnFGChange = true);
@@ -6788,7 +6794,7 @@ class Compiler
67886794
void fgKillDependentAssertionsSingle(unsigned lclNum DEBUGARG(GenTree* tree));
67896795
void fgKillDependentAssertions(unsigned lclNum DEBUGARG(GenTree* tree));
67906796
void fgMorphTreeDone(GenTree* tree);
6791-
void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone, bool isMorphedTree DEBUGARG(int morphNum = 0));
6797+
void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone DEBUGARG(int morphNum = 0));
67926798

67936799
Statement* fgMorphStmt;
67946800
unsigned fgBigOffsetMorphingTemps[TYP_COUNT];

‎src/coreclr/jit/compiler.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1354,6 +1354,7 @@ inline GenTree::GenTree(genTreeOps oper, var_types type DEBUGARG(bool largeNode)
13541354
gtLIRFlags = 0;
13551355
#ifdef DEBUG
13561356
gtDebugFlags = GTF_DEBUG_NONE;
1357+
gtMorphCount = 0;
13571358
#endif // DEBUG
13581359
gtCSEnum = NO_CSE;
13591360
ClearAssertion();

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 88 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -4903,7 +4903,6 @@ static void SetIndirectStoreEvalOrder(Compiler* comp, GenTreeIndir* store, bool*
49034903
* 1. GetCostEx() to the execution complexity estimate
49044904
* 2. GetCostSz() to the code size estimate
49054905
* 3. Sometimes sets GTF_ADDRMODE_NO_CSE on nodes in the tree.
4906-
* 4. DEBUG-only: clears GTF_DEBUG_NODE_MORPHED.
49074906
*/
49084907

49094908
#ifdef _PREFAST_
@@ -14388,10 +14387,7 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1438814387
// Helper function that creates a new IntCon node and morphs it, if required
1438914388
auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* {
1439014389
GenTreeIntCon* icon = gtNewIconNode(value);
14391-
if (fgGlobalMorph)
14392-
{
14393-
INDEBUG(icon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14394-
}
14390+
icon->SetMorphed(this);
1439514391
return icon;
1439614392
};
1439714393

@@ -14402,18 +14398,14 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1440214398
assert(varTypeIsUnsigned(castToType));
1440314399

1440414400
GenTreeCast* cast = gtNewCastNode(TYP_INT, op1, false, castToType);
14405-
if (fgGlobalMorph)
14406-
{
14407-
fgMorphTreeDone(cast);
14408-
}
14401+
cast->SetMorphed(this);
14402+
fgMorphTreeDone(cast);
1440914403

1441014404
if (type == TYP_LONG)
1441114405
{
1441214406
cast = gtNewCastNode(TYP_LONG, cast, true, TYP_LONG);
14413-
if (fgGlobalMorph)
14414-
{
14415-
fgMorphTreeDone(cast);
14416-
}
14407+
cast->SetMorphed(this);
14408+
fgMorphTreeDone(cast);
1441714409
}
1441814410

1441914411
return cast;
@@ -14680,14 +14672,7 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1468014672
DISPTREE(tree);
1468114673
JITDUMP("Transformed into:\n");
1468214674
DISPTREE(op);
14683-
14684-
if (fgGlobalMorph)
14685-
{
14686-
// We can sometimes produce a comma over the constant if the original op
14687-
// had a side effect, so just ensure we set the flag (which will be already
14688-
// set for the operands otherwise).
14689-
INDEBUG(op->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14690-
}
14675+
op->SetMorphed(this);
1469114676
return op;
1469214677
}
1469314678

@@ -14742,10 +14727,9 @@ GenTree* Compiler::gtFoldExprSpecialFloating(GenTree* tree)
1474214727
// Helper function that creates a new IntCon node and morphs it, if required
1474314728
auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* {
1474414729
GenTreeIntCon* icon = gtNewIconNode(value);
14745-
if (fgGlobalMorph)
14746-
{
14747-
INDEBUG(icon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14748-
}
14730+
14731+
icon->SetMorphed(this);
14732+
1474914733
return icon;
1475014734
};
1475114735

@@ -14907,14 +14891,8 @@ GenTree* Compiler::gtFoldExprSpecialFloating(GenTree* tree)
1490714891
DISPTREE(tree);
1490814892
JITDUMP("Transformed into:\n");
1490914893
DISPTREE(op);
14894+
op->SetMorphed(this);
1491014895

14911-
if (fgGlobalMorph)
14912-
{
14913-
// We can sometimes produce a comma over the constant if the original op
14914-
// had a side effect, so just ensure we set the flag (which will be already
14915-
// set for the operands otherwise).
14916-
INDEBUG(op->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14917-
}
1491814896
return op;
1491914897
}
1492014898

@@ -15435,6 +15413,11 @@ GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp)
1543515413
Statement* thisStoreStmt = thisOp->AsBox()->gtCopyStmtWhenInlinedBoxValue;
1543615414
thisStoreStmt->SetRootNode(thisStore);
1543715415
thisValOpt = gtNewLclvNode(thisTmp, type);
15416+
15417+
// If this is invoked during global morph we are adding code to a remote tree
15418+
// Despite this being a store, we can't meaningfully add assertions
15419+
//
15420+
thisStore->SetMorphed(this);
1543815421
}
1543915422

1544015423
if (flagVal->IsIntegralConst())
@@ -15452,6 +15435,11 @@ GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp)
1545215435
flagStoreStmt->SetRootNode(flagStore);
1545315436
flagValOpt = gtNewLclvNode(flagTmp, type);
1545415437
flagValOptCopy = gtNewLclvNode(flagTmp, type);
15438+
15439+
// If this is invoked during global morph we are adding code to a remote tree
15440+
// Despite this being a store, we can't meaningfully add assertions
15441+
//
15442+
flagStore->SetMorphed(this);
1545515443
}
1545615444

1545715445
// Turn the call into (thisValTmp & flagTmp) == flagTmp.
@@ -17436,16 +17424,7 @@ void Compiler::gtExtractSideEffList(GenTree* expr,
1743617424
}
1743717425

1743817426
GenTree* comma = m_compiler->gtNewOperNode(GT_COMMA, TYP_VOID, m_result, node);
17439-
17440-
#ifdef DEBUG
17441-
if (m_compiler->fgGlobalMorph)
17442-
{
17443-
// Either both should be morphed or neither should be.
17444-
assert((m_result->gtDebugFlags & GTF_DEBUG_NODE_MORPHED) ==
17445-
(node->gtDebugFlags & GTF_DEBUG_NODE_MORPHED));
17446-
comma->gtDebugFlags |= node->gtDebugFlags & GTF_DEBUG_NODE_MORPHED;
17447-
}
17448-
#endif
17427+
comma->SetMorphed(m_compiler);
1744917428

1745017429
// Both should have valuenumbers defined for both or for neither
1745117430
// one (unless we are remorphing, in which case a prior transform
@@ -30763,12 +30742,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3076330742
}
3076430743

3076530744
vecCon->gtSimdVal = simdVal;
30766-
30767-
if (fgGlobalMorph)
30768-
{
30769-
INDEBUG(vecCon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
30770-
}
30771-
30745+
vecCon->SetMorphed(this);
3077230746
fgUpdateConstTreeValueNumber(vecCon);
3077330747
return vecCon;
3077430748
}
@@ -30830,11 +30804,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3083030804
#endif // !TARGET_XARCH && !TARGET_ARM64
3083130805

3083230806
DEBUG_DESTROY_NODE(op, tree);
30833-
30834-
if (fgGlobalMorph)
30835-
{
30836-
INDEBUG(vectorNode->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
30837-
}
30807+
vectorNode->SetMorphed(this);
3083830808
return vectorNode;
3083930809
}
3084030810
}
@@ -32019,17 +31989,10 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3201931989

3202031990
if (resultNode != tree)
3202131991
{
32022-
if (fgGlobalMorph)
31992+
resultNode->SetMorphed(this);
31993+
if (resultNode->OperIs(GT_COMMA))
3202331994
{
32024-
// We can sometimes produce a comma over the constant if the original op
32025-
// had a side effect or even a new constant node, so just ensure we set
32026-
// the flag (which will be already set for the operands otherwise).
32027-
INDEBUG(resultNode->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
32028-
32029-
if (resultNode->OperIs(GT_COMMA))
32030-
{
32031-
INDEBUG(resultNode->AsOp()->gtGetOp2()->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
32032-
}
31995+
resultNode->AsOp()->gtGetOp2()->SetMorphed(this);
3203331996
}
3203431997

3203531998
if (resultNode->OperIsConst())
@@ -32156,3 +32119,65 @@ bool Compiler::gtCanSkipCovariantStoreCheck(GenTree* value, GenTree* array)
3215632119

3215732120
return false;
3215832121
}
32122+
32123+
#if defined(DEBUG)
32124+
//------------------------------------------------------------------------
32125+
// SetMorphed: mark a node as having been morphed
32126+
//
32127+
// Arguments:
32128+
// compiler - compiler instance
32129+
// doChildren - recursive mark child nodes
32130+
//
32131+
// Notes:
32132+
// Does nothing outside of global morph.
32133+
//
32134+
// Useful for morph post-order expansions / optimizations.
32135+
//
32136+
// Use care when invoking this on an assignment (or when doChildren is true,
32137+
// on trees containing assignments) as those usually will also require
32138+
// local assertion updates.
32139+
//
32140+
void GenTree::SetMorphed(Compiler* compiler, bool doChildren /* = false */)
32141+
{
32142+
if (!compiler->fgGlobalMorph)
32143+
{
32144+
return;
32145+
}
32146+
32147+
struct Visitor : GenTreeVisitor<Visitor>
32148+
{
32149+
enum
32150+
{
32151+
DoPostOrder = true,
32152+
};
32153+
32154+
Visitor(Compiler* comp)
32155+
: GenTreeVisitor(comp)
32156+
{
32157+
}
32158+
32159+
fgWalkResult PostOrderVisit(GenTree** use, GenTree* user)
32160+
{
32161+
GenTree* const node = *use;
32162+
if (!node->WasMorphed())
32163+
{
32164+
node->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED;
32165+
node->gtMorphCount++;
32166+
}
32167+
return Compiler::WALK_CONTINUE;
32168+
}
32169+
};
32170+
32171+
if (doChildren)
32172+
{
32173+
Visitor v(compiler);
32174+
GenTree* node = this;
32175+
v.WalkTree(&node, nullptr);
32176+
}
32177+
else if (!WasMorphed())
32178+
{
32179+
gtDebugFlags |= GTF_DEBUG_NODE_MORPHED;
32180+
gtMorphCount++;
32181+
}
32182+
}
32183+
#endif

‎src/coreclr/jit/gentree.h‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -602,21 +602,21 @@ inline GenTreeFlags& operator ^=(GenTreeFlags& a, GenTreeFlags b)
602602
//------------------------------------------------------------------------
603603
// GenTreeDebugFlags: a bitmask of debug-only flags for GenTree stored in gtDebugFlags
604604
//
605-
enum GenTreeDebugFlags : unsigned int
605+
enum GenTreeDebugFlags : unsigned short
606606
{
607-
GTF_DEBUG_NONE = 0x00000000, // No debug flags.
607+
GTF_DEBUG_NONE = 0x0000, // No debug flags.
608608

609-
GTF_DEBUG_NODE_MORPHED = 0x00000001, // the node has been morphed (in the global morphing phase)
610-
GTF_DEBUG_NODE_SMALL = 0x00000002,
611-
GTF_DEBUG_NODE_LARGE = 0x00000004,
612-
GTF_DEBUG_NODE_CG_PRODUCED = 0x00000008, // genProduceReg has been called on this node
613-
GTF_DEBUG_NODE_CG_CONSUMED = 0x00000010, // genConsumeReg has been called on this node
614-
GTF_DEBUG_NODE_LSRA_ADDED = 0x00000020, // This node was added by LSRA
609+
GTF_DEBUG_NODE_MORPHED = 0x0001, // the node has been morphed (in the global morphing phase)
610+
GTF_DEBUG_NODE_SMALL = 0x0002,
611+
GTF_DEBUG_NODE_LARGE = 0x0004,
612+
GTF_DEBUG_NODE_CG_PRODUCED = 0x0008, // genProduceReg has been called on this node
613+
GTF_DEBUG_NODE_CG_CONSUMED = 0x0010, // genConsumeReg has been called on this node
614+
GTF_DEBUG_NODE_LSRA_ADDED = 0x0020, // This node was added by LSRA
615615

616-
GTF_DEBUG_NODE_MASK = 0x0000003F, // These flags are all node (rather than operation) properties.
616+
GTF_DEBUG_NODE_MASK = 0x003F, // These flags are all node (rather than operation) properties.
617617

618-
GTF_DEBUG_VAR_CSE_REF = 0x00800000, // GT_LCL_VAR -- This is a CSE LCL_VAR node
619-
GTF_DEBUG_CAST_DONT_FOLD = 0x00400000, // GT_CAST -- Try to prevent this cast from being folded
618+
GTF_DEBUG_VAR_CSE_REF = 0x8000, // GT_LCL_VAR -- This is a CSE LCL_VAR node
619+
GTF_DEBUG_CAST_DONT_FOLD = 0x4000, // GT_CAST -- Try to prevent this cast from being folded
620620
};
621621

622622
inline constexpr GenTreeDebugFlags operator ~(GenTreeDebugFlags a)
@@ -972,7 +972,26 @@ struct GenTree
972972

973973
#if defined(DEBUG)
974974
GenTreeDebugFlags gtDebugFlags;
975-
#endif // defined(DEBUG)
975+
unsigned short gtMorphCount;
976+
void SetMorphed(Compiler* compiler, bool doChilren = false);
977+
978+
bool WasMorphed() const
979+
{
980+
return (gtDebugFlags & GTF_DEBUG_NODE_MORPHED) != 0;
981+
}
982+
983+
void ClearMorphed()
984+
{
985+
gtDebugFlags &= ~GTF_DEBUG_NODE_MORPHED;
986+
}
987+
#else
988+
void SetMorphed(Compiler*, bool)
989+
{
990+
}
991+
void ClearMorphed()
992+
{
993+
}
994+
#endif
976995

977996
ValueNumPair gtVNPair;
978997

0 commit comments

Comments
 (0)