Skip to content

Commit 4822cc0

Browse files
authored
JIT: revise how we track if nodes have been morphed (dotnet#110787)
Allow remorphing. Require that all nodes be morphed. Ensure that no node is morphed more than a handful of times. Fixes dotnet#6218, dotnet#56962, dotnet#107542
1 parent 3e5b44a commit 4822cc0

7 files changed

Lines changed: 293 additions & 222 deletions

File tree

‎src/coreclr/jit/compiler.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5438,6 +5438,10 @@ class Compiler
54385438
void fgMorphBlock(BasicBlock* block, MorphUnreachableInfo* unreachableInfo = nullptr);
54395439
void fgMorphStmts(BasicBlock* block);
54405440

5441+
#ifdef DEBUG
5442+
void fgPostGlobalMorphChecks();
5443+
#endif
5444+
54415445
void fgMergeBlockReturn(BasicBlock* block);
54425446

54435447
bool fgMorphBlockStmt(BasicBlock* block, Statement* stmt DEBUGARG(const char* msg), bool invalidateDFSTreeOnFGChange = true);
@@ -6785,7 +6789,7 @@ class Compiler
67856789
void fgKillDependentAssertionsSingle(unsigned lclNum DEBUGARG(GenTree* tree));
67866790
void fgKillDependentAssertions(unsigned lclNum DEBUGARG(GenTree* tree));
67876791
void fgMorphTreeDone(GenTree* tree);
6788-
void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone, bool isMorphedTree DEBUGARG(int morphNum = 0));
6792+
void fgMorphTreeDone(GenTree* tree, bool optAssertionPropDone DEBUGARG(int morphNum = 0));
67896793

67906794
Statement* fgMorphStmt;
67916795
unsigned fgBigOffsetMorphingTemps[TYP_COUNT];

‎src/coreclr/jit/compiler.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1354,6 +1354,7 @@ inline GenTree::GenTree(genTreeOps oper, var_types type DEBUGARG(bool largeNode)
13541354
gtLIRFlags = 0;
13551355
#ifdef DEBUG
13561356
gtDebugFlags = GTF_DEBUG_NONE;
1357+
gtMorphCount = 0;
13571358
#endif // DEBUG
13581359
gtCSEnum = NO_CSE;
13591360
ClearAssertion();

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 88 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -4903,7 +4903,6 @@ static void SetIndirectStoreEvalOrder(Compiler* comp, GenTreeIndir* store, bool*
49034903
* 1. GetCostEx() to the execution complexity estimate
49044904
* 2. GetCostSz() to the code size estimate
49054905
* 3. Sometimes sets GTF_ADDRMODE_NO_CSE on nodes in the tree.
4906-
* 4. DEBUG-only: clears GTF_DEBUG_NODE_MORPHED.
49074906
*/
49084907

49094908
#ifdef _PREFAST_
@@ -14388,10 +14387,7 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1438814387
// Helper function that creates a new IntCon node and morphs it, if required
1438914388
auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* {
1439014389
GenTreeIntCon* icon = gtNewIconNode(value);
14391-
if (fgGlobalMorph)
14392-
{
14393-
INDEBUG(icon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14394-
}
14390+
icon->SetMorphed(this);
1439514391
return icon;
1439614392
};
1439714393

@@ -14402,18 +14398,14 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1440214398
assert(varTypeIsUnsigned(castToType));
1440314399

1440414400
GenTreeCast* cast = gtNewCastNode(TYP_INT, op1, false, castToType);
14405-
if (fgGlobalMorph)
14406-
{
14407-
fgMorphTreeDone(cast);
14408-
}
14401+
cast->SetMorphed(this);
14402+
fgMorphTreeDone(cast);
1440914403

1441014404
if (type == TYP_LONG)
1441114405
{
1441214406
cast = gtNewCastNode(TYP_LONG, cast, true, TYP_LONG);
14413-
if (fgGlobalMorph)
14414-
{
14415-
fgMorphTreeDone(cast);
14416-
}
14407+
cast->SetMorphed(this);
14408+
fgMorphTreeDone(cast);
1441714409
}
1441814410

1441914411
return cast;
@@ -14680,14 +14672,7 @@ GenTree* Compiler::gtFoldExprSpecial(GenTree* tree)
1468014672
DISPTREE(tree);
1468114673
JITDUMP("Transformed into:\n");
1468214674
DISPTREE(op);
14683-
14684-
if (fgGlobalMorph)
14685-
{
14686-
// We can sometimes produce a comma over the constant if the original op
14687-
// had a side effect, so just ensure we set the flag (which will be already
14688-
// set for the operands otherwise).
14689-
INDEBUG(op->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14690-
}
14675+
op->SetMorphed(this);
1469114676
return op;
1469214677
}
1469314678

@@ -14742,10 +14727,9 @@ GenTree* Compiler::gtFoldExprSpecialFloating(GenTree* tree)
1474214727
// Helper function that creates a new IntCon node and morphs it, if required
1474314728
auto NewMorphedIntConNode = [&](int value) -> GenTreeIntCon* {
1474414729
GenTreeIntCon* icon = gtNewIconNode(value);
14745-
if (fgGlobalMorph)
14746-
{
14747-
INDEBUG(icon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14748-
}
14730+
14731+
icon->SetMorphed(this);
14732+
1474914733
return icon;
1475014734
};
1475114735

@@ -14907,14 +14891,8 @@ GenTree* Compiler::gtFoldExprSpecialFloating(GenTree* tree)
1490714891
DISPTREE(tree);
1490814892
JITDUMP("Transformed into:\n");
1490914893
DISPTREE(op);
14894+
op->SetMorphed(this);
1491014895

14911-
if (fgGlobalMorph)
14912-
{
14913-
// We can sometimes produce a comma over the constant if the original op
14914-
// had a side effect, so just ensure we set the flag (which will be already
14915-
// set for the operands otherwise).
14916-
INDEBUG(op->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
14917-
}
1491814896
return op;
1491914897
}
1492014898

@@ -15435,6 +15413,11 @@ GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp)
1543515413
Statement* thisStoreStmt = thisOp->AsBox()->gtCopyStmtWhenInlinedBoxValue;
1543615414
thisStoreStmt->SetRootNode(thisStore);
1543715415
thisValOpt = gtNewLclvNode(thisTmp, type);
15416+
15417+
// If this is invoked during global morph we are adding code to a remote tree
15418+
// Despite this being a store, we can't meaningfully add assertions
15419+
//
15420+
thisStore->SetMorphed(this);
1543815421
}
1543915422

1544015423
if (flagVal->IsIntegralConst())
@@ -15452,6 +15435,11 @@ GenTree* Compiler::gtOptimizeEnumHasFlag(GenTree* thisOp, GenTree* flagOp)
1545215435
flagStoreStmt->SetRootNode(flagStore);
1545315436
flagValOpt = gtNewLclvNode(flagTmp, type);
1545415437
flagValOptCopy = gtNewLclvNode(flagTmp, type);
15438+
15439+
// If this is invoked during global morph we are adding code to a remote tree
15440+
// Despite this being a store, we can't meaningfully add assertions
15441+
//
15442+
flagStore->SetMorphed(this);
1545515443
}
1545615444

1545715445
// Turn the call into (thisValTmp & flagTmp) == flagTmp.
@@ -17436,16 +17424,7 @@ void Compiler::gtExtractSideEffList(GenTree* expr,
1743617424
}
1743717425

1743817426
GenTree* comma = m_compiler->gtNewOperNode(GT_COMMA, TYP_VOID, m_result, node);
17439-
17440-
#ifdef DEBUG
17441-
if (m_compiler->fgGlobalMorph)
17442-
{
17443-
// Either both should be morphed or neither should be.
17444-
assert((m_result->gtDebugFlags & GTF_DEBUG_NODE_MORPHED) ==
17445-
(node->gtDebugFlags & GTF_DEBUG_NODE_MORPHED));
17446-
comma->gtDebugFlags |= node->gtDebugFlags & GTF_DEBUG_NODE_MORPHED;
17447-
}
17448-
#endif
17427+
comma->SetMorphed(m_compiler);
1744917428

1745017429
// Both should have valuenumbers defined for both or for neither
1745117430
// one (unless we are remorphing, in which case a prior transform
@@ -30762,12 +30741,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3076230741
}
3076330742

3076430743
vecCon->gtSimdVal = simdVal;
30765-
30766-
if (fgGlobalMorph)
30767-
{
30768-
INDEBUG(vecCon->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
30769-
}
30770-
30744+
vecCon->SetMorphed(this);
3077130745
fgUpdateConstTreeValueNumber(vecCon);
3077230746
return vecCon;
3077330747
}
@@ -30829,11 +30803,7 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3082930803
#endif // !TARGET_XARCH && !TARGET_ARM64
3083030804

3083130805
DEBUG_DESTROY_NODE(op, tree);
30832-
30833-
if (fgGlobalMorph)
30834-
{
30835-
INDEBUG(vectorNode->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
30836-
}
30806+
vectorNode->SetMorphed(this);
3083730807
return vectorNode;
3083830808
}
3083930809
}
@@ -32018,17 +31988,10 @@ GenTree* Compiler::gtFoldExprHWIntrinsic(GenTreeHWIntrinsic* tree)
3201831988

3201931989
if (resultNode != tree)
3202031990
{
32021-
if (fgGlobalMorph)
31991+
resultNode->SetMorphed(this);
31992+
if (resultNode->OperIs(GT_COMMA))
3202231993
{
32023-
// We can sometimes produce a comma over the constant if the original op
32024-
// had a side effect or even a new constant node, so just ensure we set
32025-
// the flag (which will be already set for the operands otherwise).
32026-
INDEBUG(resultNode->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
32027-
32028-
if (resultNode->OperIs(GT_COMMA))
32029-
{
32030-
INDEBUG(resultNode->AsOp()->gtGetOp2()->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED);
32031-
}
31994+
resultNode->AsOp()->gtGetOp2()->SetMorphed(this);
3203231995
}
3203331996

3203431997
if (resultNode->OperIsConst())
@@ -32155,3 +32118,65 @@ bool Compiler::gtCanSkipCovariantStoreCheck(GenTree* value, GenTree* array)
3215532118

3215632119
return false;
3215732120
}
32121+
32122+
#if defined(DEBUG)
32123+
//------------------------------------------------------------------------
32124+
// SetMorphed: mark a node as having been morphed
32125+
//
32126+
// Arguments:
32127+
// compiler - compiler instance
32128+
// doChildren - recursive mark child nodes
32129+
//
32130+
// Notes:
32131+
// Does nothing outside of global morph.
32132+
//
32133+
// Useful for morph post-order expansions / optimizations.
32134+
//
32135+
// Use care when invoking this on an assignment (or when doChildren is true,
32136+
// on trees containing assignments) as those usually will also require
32137+
// local assertion updates.
32138+
//
32139+
void GenTree::SetMorphed(Compiler* compiler, bool doChildren /* = false */)
32140+
{
32141+
if (!compiler->fgGlobalMorph)
32142+
{
32143+
return;
32144+
}
32145+
32146+
struct Visitor : GenTreeVisitor<Visitor>
32147+
{
32148+
enum
32149+
{
32150+
DoPostOrder = true,
32151+
};
32152+
32153+
Visitor(Compiler* comp)
32154+
: GenTreeVisitor(comp)
32155+
{
32156+
}
32157+
32158+
fgWalkResult PostOrderVisit(GenTree** use, GenTree* user)
32159+
{
32160+
GenTree* const node = *use;
32161+
if (!node->WasMorphed())
32162+
{
32163+
node->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED;
32164+
node->gtMorphCount++;
32165+
}
32166+
return Compiler::WALK_CONTINUE;
32167+
}
32168+
};
32169+
32170+
if (doChildren)
32171+
{
32172+
Visitor v(compiler);
32173+
GenTree* node = this;
32174+
v.WalkTree(&node, nullptr);
32175+
}
32176+
else if (!WasMorphed())
32177+
{
32178+
gtDebugFlags |= GTF_DEBUG_NODE_MORPHED;
32179+
gtMorphCount++;
32180+
}
32181+
}
32182+
#endif

‎src/coreclr/jit/gentree.h‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -602,21 +602,21 @@ inline GenTreeFlags& operator ^=(GenTreeFlags& a, GenTreeFlags b)
602602
//------------------------------------------------------------------------
603603
// GenTreeDebugFlags: a bitmask of debug-only flags for GenTree stored in gtDebugFlags
604604
//
605-
enum GenTreeDebugFlags : unsigned int
605+
enum GenTreeDebugFlags : unsigned short
606606
{
607-
GTF_DEBUG_NONE = 0x00000000, // No debug flags.
607+
GTF_DEBUG_NONE = 0x0000, // No debug flags.
608608

609-
GTF_DEBUG_NODE_MORPHED = 0x00000001, // the node has been morphed (in the global morphing phase)
610-
GTF_DEBUG_NODE_SMALL = 0x00000002,
611-
GTF_DEBUG_NODE_LARGE = 0x00000004,
612-
GTF_DEBUG_NODE_CG_PRODUCED = 0x00000008, // genProduceReg has been called on this node
613-
GTF_DEBUG_NODE_CG_CONSUMED = 0x00000010, // genConsumeReg has been called on this node
614-
GTF_DEBUG_NODE_LSRA_ADDED = 0x00000020, // This node was added by LSRA
609+
GTF_DEBUG_NODE_MORPHED = 0x0001, // the node has been morphed (in the global morphing phase)
610+
GTF_DEBUG_NODE_SMALL = 0x0002,
611+
GTF_DEBUG_NODE_LARGE = 0x0004,
612+
GTF_DEBUG_NODE_CG_PRODUCED = 0x0008, // genProduceReg has been called on this node
613+
GTF_DEBUG_NODE_CG_CONSUMED = 0x0010, // genConsumeReg has been called on this node
614+
GTF_DEBUG_NODE_LSRA_ADDED = 0x0020, // This node was added by LSRA
615615

616-
GTF_DEBUG_NODE_MASK = 0x0000003F, // These flags are all node (rather than operation) properties.
616+
GTF_DEBUG_NODE_MASK = 0x003F, // These flags are all node (rather than operation) properties.
617617

618-
GTF_DEBUG_VAR_CSE_REF = 0x00800000, // GT_LCL_VAR -- This is a CSE LCL_VAR node
619-
GTF_DEBUG_CAST_DONT_FOLD = 0x00400000, // GT_CAST -- Try to prevent this cast from being folded
618+
GTF_DEBUG_VAR_CSE_REF = 0x8000, // GT_LCL_VAR -- This is a CSE LCL_VAR node
619+
GTF_DEBUG_CAST_DONT_FOLD = 0x4000, // GT_CAST -- Try to prevent this cast from being folded
620620
};
621621

622622
inline constexpr GenTreeDebugFlags operator ~(GenTreeDebugFlags a)
@@ -972,7 +972,26 @@ struct GenTree
972972

973973
#if defined(DEBUG)
974974
GenTreeDebugFlags gtDebugFlags;
975-
#endif // defined(DEBUG)
975+
unsigned short gtMorphCount;
976+
void SetMorphed(Compiler* compiler, bool doChilren = false);
977+
978+
bool WasMorphed() const
979+
{
980+
return (gtDebugFlags & GTF_DEBUG_NODE_MORPHED) != 0;
981+
}
982+
983+
void ClearMorphed()
984+
{
985+
gtDebugFlags &= ~GTF_DEBUG_NODE_MORPHED;
986+
}
987+
#else
988+
void SetMorphed(Compiler* compiler, bool doChildren = false)
989+
{
990+
}
991+
void ClearMorphed()
992+
{
993+
}
994+
#endif
976995

977996
ValueNumPair gtVNPair;
978997

0 commit comments

Comments
 (0)