From 32afa8b03f47de2c55197ae22710ec1b0fc026fe Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 23 Sep 2026 16:16:04 +0100 Subject: [PATCH 1/3] JIT: add explicit multi-local stores for multi-register results --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenarmarch.cpp | 4 + src/coreclr/jit/codegencommon.cpp | 32 +++ src/coreclr/jit/codegenloongarch64.cpp | 4 + src/coreclr/jit/codegenriscv64.cpp | 4 + src/coreclr/jit/codegenxarch.cpp | 4 + src/coreclr/jit/compiler.hpp | 52 ++++- src/coreclr/jit/fgdiagnostic.cpp | 2 +- src/coreclr/jit/gentree.cpp | 99 ++++++++- src/coreclr/jit/gentree.h | 54 ++++- src/coreclr/jit/gtlist.h | 1 + src/coreclr/jit/gtstructs.h | 1 + src/coreclr/jit/lclmorph.cpp | 7 + src/coreclr/jit/lclvars.cpp | 22 ++ src/coreclr/jit/lir.cpp | 23 +- src/coreclr/jit/liveness.cpp | 29 +++ src/coreclr/jit/lower.cpp | 18 ++ src/coreclr/jit/lsra.h | 1 + src/coreclr/jit/lsraarm.cpp | 4 + src/coreclr/jit/lsraarm64.cpp | 4 + src/coreclr/jit/lsrabuild.cpp | 24 ++ src/coreclr/jit/lsraloongarch64.cpp | 4 + src/coreclr/jit/lsrariscv64.cpp | 4 + src/coreclr/jit/lsraxarch.cpp | 4 + src/coreclr/jit/optimizer.cpp | 26 +++ src/coreclr/jit/promotion.h | 1 + src/coreclr/jit/promotiondecomposition.cpp | 72 ++++++ src/coreclr/jit/sideeffects.cpp | 19 ++ src/coreclr/jit/sideeffects.h | 12 + src/coreclr/jit/ssabuilder.cpp | 3 +- src/coreclr/jit/treelifeupdater.cpp | 2 +- src/coreclr/jit/valuenum.cpp | 8 +- src/tests/JIT/opt/Structs/StoreLclVars.cs | 205 ++++++++++++++++++ src/tests/JIT/opt/Structs/StoreLclVars.csproj | 12 + 34 files changed, 743 insertions(+), 19 deletions(-) create mode 100644 src/tests/JIT/opt/Structs/StoreLclVars.cs create mode 100644 src/tests/JIT/opt/Structs/StoreLclVars.csproj diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 2020048e2acc3f..f2207fc571467a 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1329,6 +1329,7 @@ class CodeGen final : public CodeGenInterface void genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode); void genMultiRegStoreToLocal(GenTreeLclVar* lclNode); + void genStoreLclVars(GenTreeStoreLclVars* store); // Codegen for multi-register struct returns. bool isStructReturn(GenTree* treeNode); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 37cca461351f49..ec14a468295e14 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -280,6 +280,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForStoreLclFld(treeNode->AsLclFld()); break; + case GT_STORE_LCL_VARS: + genStoreLclVars(treeNode->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: genCodeForStoreLclVar(treeNode->AsLclVar()); break; diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 93659af0e67f23..8a15b49fe5288e 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8035,6 +8035,38 @@ void CodeGen::genJmpPlaceArgs(GenTree* jmp) } //---------------------------------------------------------------------------------- +// genStoreLclVars: Move each result to its independent register or stack home. +void CodeGen::genStoreLclVars(GenTreeStoreLclVars* store) +{ + for (unsigned i = 0; i < store->m_count; i++) + { + regNumber sourceReg = genConsumeReg(store->gtOp1, i); + GenTreeLclVar* destination = store->GetDestination(i); + LclVarDsc* varDsc = m_compiler->lvaGetDesc(destination); + regNumber targetReg = destination->GetRegNum(); + var_types type = varDsc->TypeGet(); + if (targetReg != REG_NA) + { + inst_Mov(type, targetReg, sourceReg, /* canSkip */ true); + } + if (((targetReg == REG_NA) || varDsc->IsAlwaysAliveInMemory()) && !destination->IsLastUse(0)) + { + GetEmitter()->emitIns_S_R(ins_StoreFromSrc(sourceReg, type), emitTypeSize(type), sourceReg, + destination->GetLclNum(), 0); + } + varDsc->SetRegNum(targetReg == REG_NA ? REG_STK : targetReg); + if (targetReg != REG_NA) + { + genProduceReg(destination); + } + else + { + genUpdateLife(destination); + } + } +} + +//------------------------------------------------------------------------ // genMultiRegStoreToLocal: store multi-reg value to a local // // Arguments: diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index 8ec2034dd91556..bf68b50fbed70e 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -4073,6 +4073,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForStoreLclFld(treeNode->AsLclFld()); break; + case GT_STORE_LCL_VARS: + genStoreLclVars(treeNode->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: genCodeForStoreLclVar(treeNode->AsLclVar()); break; diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index aa4abea743baca..08844756f86516 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -3919,6 +3919,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForStoreLclFld(treeNode->AsLclFld()); break; + case GT_STORE_LCL_VARS: + genStoreLclVars(treeNode->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: genCodeForStoreLclVar(treeNode->AsLclVar()); break; diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 68bf1de8804c21..5ef1353da51adf 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1997,6 +1997,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForStoreLclFld(treeNode->AsLclFld()); break; + case GT_STORE_LCL_VARS: + genStoreLclVars(treeNode->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: genCodeForStoreLclVar(treeNode->AsLclVar()); break; diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index db69f705d10497..62d70da60998a5 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4454,7 +4454,7 @@ GenTree::VisitResult GenTree::VisitOperandUses(TVisitor visitor) template struct LocalDefProvider { - bool HasMultiDefIndex() const + bool NeedsValueExtraction() const { return static_cast(this)->GetMultiDefIndex() != BAD_VAR_NUM; } @@ -4532,6 +4532,36 @@ struct StoreLclVarDef : LocalDefProvider } }; +struct StoreLclVarsDef : StoreLclVarDef +{ + // Each destination has a scalar SSA number, but its value is a slice of + // the multi-register source rather than the source's entire value. + unsigned m_valueOffset; + unsigned m_storeSize; + + StoreLclVarsDef(GenTreeStoreLclVars* store, unsigned index) + : StoreLclVarDef(store->GetDestination(index)) + , m_valueOffset(store->m_destinations[index].Offset) + , m_storeSize(store->m_size) + { + } + + bool NeedsValueExtraction() const + { + return true; + } + + ssize_t GetValueOffset(Compiler* compiler) const + { + return m_valueOffset; + } + + ValueSize GetStoreSize(Compiler* compiler) const + { + return ValueSize(m_storeSize); + } +}; + struct PromotedStoreLclVarDef : LocalDefProvider { GenTreeLclVarCommon* m_def; @@ -4818,7 +4848,7 @@ GenTree::VisitResult VisitPromotedRangeLocalDefs( // inline bool GenTree::IsEntireLocalDef(Compiler* comp, GenTreeLclVarCommon* def) { - if (OperIs(GT_STORE_LCL_VAR)) + if (OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_VARS)) { return true; } @@ -4921,6 +4951,15 @@ GenTree::VisitResult GenTree::VisitLocalDef( template GenTree::VisitResult GenTree::VisitLogicalLocalDefs(Compiler* comp, TVisitor visitor) { + if (OperIs(GT_STORE_LCL_VARS)) + { + GenTreeStoreLclVars* store = AsStoreLclVars(); + for (unsigned i = 0; i < store->m_count; i++) + { + RETURN_IF_ABORT(visitor(StoreLclVarsDef(store, i))); + } + return VisitResult::Continue; + } if (OperIs(GT_STORE_LCL_VAR)) { return VisitLocalDef(comp, AsLclVarCommon(), visitor); @@ -4978,6 +5017,15 @@ GenTree::VisitResult GenTree::VisitLogicalLocalDefs(Compiler* comp, TVisitor vis template GenTree::VisitResult GenTree::VisitPhysicalLocalDefNodes(Compiler* comp, TVisitor visitor) { + if (OperIs(GT_STORE_LCL_VARS)) + { + GenTreeStoreLclVars* store = AsStoreLclVars(); + for (unsigned i = 0; i < store->m_count; i++) + { + RETURN_IF_ABORT(visitor(store->GetDestination(i))); + } + return VisitResult::Continue; + } if (OperIs(GT_STORE_LCL_VAR)) { return visitor(AsLclVarCommon()); diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 2c34ef6cb05dc2..a3ce60bd41e812 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3847,7 +3847,7 @@ void Compiler::fgDebugCheckLinkedLocals() } } - if (node->IsCall()) + if (node->OperIs(GT_CALL, GT_STORE_LCL_VARS)) { auto linkDefs = [&](GenTree* def) { assert(ShouldLink(def)); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 1b5fba4c66966a..078a16dffb7000 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -295,6 +295,7 @@ void GenTree::InitNodeSize() #endif // FEATURE_SIMD static_assert(sizeof(GenTreeLclVarCommon) <= TREE_NODE_SZ_SMALL); static_assert(sizeof(GenTreeLclVar) <= TREE_NODE_SZ_SMALL); + static_assert(sizeof(GenTreeStoreLclVars) <= TREE_NODE_SZ_SMALL); static_assert(sizeof(GenTreeLclFld) <= TREE_NODE_SZ_SMALL); static_assert(sizeof(GenTreeCC) <= TREE_NODE_SZ_SMALL); static_assert(sizeof(GenTreeOpCC) <= TREE_NODE_SZ_SMALL); @@ -2901,6 +2902,25 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) // these should be included in the comparison. switch (oper) { + case GT_STORE_LCL_VARS: + { + GenTreeStoreLclVars* store1 = op1->AsStoreLclVars(); + GenTreeStoreLclVars* store2 = op2->AsStoreLclVars(); + if ((store1->m_count != store2->m_count) || (store1->m_size != store2->m_size)) + { + return false; + } + for (unsigned i = 0; i < store1->m_count; i++) + { + if (!GenTreeLclVarCommon::EqualsLocal(store1->GetDestination(i), store2->GetDestination(i)) || + (store1->m_destinations[i].Offset != store2->m_destinations[i].Offset)) + { + return false; + } + } + break; + } + case GT_STORE_LCL_FLD: if ((op1->AsLclFld()->GetLclOffs() != op2->AsLclFld()->GetLclOffs()) || (op1->AsLclFld()->GetLayout() != op2->AsLclFld()->GetLayout())) @@ -3197,6 +3217,17 @@ GenTree** GenTree::EffectiveUse(GenTree** use) if (tree->OperIsUnary()) { + if (tree->OperIs(GT_STORE_LCL_VARS)) + { + GenTreeStoreLclVars* store = tree->AsStoreLclVars(); + for (unsigned i = 0; i < store->m_count; i++) + { + if (store->GetDestination(i)->GetLclNum() == lclNum) + { + return true; + } + } + } if (tree->OperIsLocalStore() && (tree->AsLclVarCommon()->GetLclNum() == lclNum)) { return true; @@ -3520,6 +3551,17 @@ unsigned Compiler::gtHashValue(GenTree* tree) // these should be included in the hash code. switch (oper) { + case GT_STORE_LCL_VARS: + { + GenTreeStoreLclVars* store = tree->AsStoreLclVars(); + hash = genTreeHashAdd(hash, store->m_size); + for (unsigned i = 0; i < store->m_count; i++) + { + hash = genTreeHashAdd(hash, store->GetDestination(i)->GetLclNum()); + hash = genTreeHashAdd(hash, store->m_destinations[i].Offset); + } + break; + } case GT_STORE_LCL_VAR: hash = genTreeHashAdd(hash, tree->AsLclVar()->GetLclNum()); break; @@ -7145,6 +7187,11 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + costEx += tree->AsStoreLclVars()->m_count; + costSz += tree->AsStoreLclVars()->m_count; + break; + case GT_STORE_LCL_VAR: if (gtIsLikelyRegVar(tree)) { @@ -8010,19 +8057,19 @@ bool Compiler::gtMayHaveStoreInterference(GenTree* treeWithStores, GenTree* tree return WALK_SKIP_SUBTREES; } - if (node->OperIsLocalStore()) + if (node->OperIsLocalStore() || node->OperIs(GT_STORE_LCL_VARS)) { - // Check up to 8 stores before we bail with a conservative - // answer. Avoids quadratic behavior in case we have a large - // number of stores (e.g. created by physical promotion or by - // call args morphing). - if ((m_numStoresChecked >= 8) || - m_compiler->gtTreeHasLocalRead(m_readTree, node->AsLclVarCommon()->GetLclNum())) + auto interferes = [&](GenTreeLclVarCommon* def) { + if ((m_numStoresChecked++ >= 8) || m_compiler->gtTreeHasLocalRead(m_readTree, def->GetLclNum())) + { + return GenTree::VisitResult::Abort; + } + return GenTree::VisitResult::Continue; + }; + if (node->VisitPhysicalLocalDefNodes(m_compiler, interferes) == GenTree::VisitResult::Abort) { return WALK_ABORT; } - - m_numStoresChecked++; } return WALK_CONTINUE; @@ -8522,6 +8569,7 @@ bool GenTree::OperRequiresAsgFlag() const { switch (OperGet()) { + case GT_STORE_LCL_VARS: case GT_STORE_LCL_VAR: case GT_STORE_LCL_FLD: case GT_STOREIND: @@ -11308,6 +11356,19 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) switch (oper) { + case GT_STORE_LCL_VARS: + { + GenTreeStoreLclVars* store = tree->AsStoreLclVars(); + auto* destinations = new (this, CMK_ASTNode) GenTreeStoreLclVars::Destination[store->m_count]; + for (unsigned i = 0; i < store->m_count; i++) + { + destinations[i] = store->m_destinations[i]; + destinations[i].Node = gtCloneExpr(destinations[i].Node)->AsLclVar(); + } + copy = new (this, oper) GenTreeStoreLclVars(store->gtOp1, destinations, store->m_count, store->m_size); + break; + } + case GT_STORE_LCL_VAR: // Remember that the local node has been cloned. The flag will be set on 'copy' as well. tree->gtFlags |= GTF_VAR_MOREUSES; @@ -12080,6 +12141,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) return; // Standard unary operators + case GT_STORE_LCL_VARS: case GT_STORE_LCL_VAR: case GT_STORE_LCL_FLD: case GT_NOT: @@ -13258,6 +13320,25 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ } gtDispNodeName(tree); + if ((tree != nullptr) && tree->OperIs(GT_STORE_LCL_VARS)) + { + GenTreeStoreLclVars* store = tree->AsStoreLclVars(); + printf(" ["); + for (unsigned i = 0; i < store->m_count; i++) + { + GenTreeLclVar* def = store->GetDestination(i); + printf("%sV%02u", i == 0 ? "" : ", ", def->GetLclNum()); + if (def->HasSsaName()) + { + printf(".%u", def->GetSsaNum()); + } + if (def->IsLastUse(0)) + { + printf("(dead)"); + } + } + printf("]"); + } assert(tree == nullptr || tree->gtOper < GT_COUNT); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index a55b32004bd83a..487c5201727ce5 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -1801,7 +1801,7 @@ struct GenTree bool OperIsSsaDef() const { - return OperIsLocalStore() || OperIs(GT_CALL); + return OperIsLocalStore() || OperIs(GT_CALL, GT_STORE_LCL_VARS); } static bool OperIsHWIntrinsic(genTreeOps gtOper) @@ -4071,6 +4071,54 @@ struct GenTreeLclVar : public GenTreeLclVarCommon #endif }; +// A simultaneous definition of independent locals from a multi-register value. +// The definition nodes carry local, SSA, liveness and register information. They +// are metadata, not operands: the source is evaluated once, before any definition. +// Destinations are distinct, non-address-exposed scalar or SIMD locals. +struct GenTreeStoreLclVars : public GenTreeUnOp +{ + struct Destination + { + GenTreeLclVar* Node; + unsigned Offset; + }; + + Destination* m_destinations; + unsigned m_count; + unsigned m_size; + + GenTreeStoreLclVars(GenTree* source, Destination* destinations, unsigned count, unsigned size) + : GenTreeUnOp(GT_STORE_LCL_VARS, TYP_VOID, source) + , m_destinations(destinations) + , m_count(count) + , m_size(size) + { + assert(count > 1 && count <= MAX_MULTIREG_COUNT); + gtFlags |= GTF_ASG; +#ifdef DEBUG + for (unsigned i = 0; i < count; i++) + { + assert(destinations[i].Node->OperIs(GT_LCL_VAR)); + assert((destinations[i].Node->gtFlags & GTF_VAR_DEF) != 0); + for (unsigned j = 0; j < i; j++) + { + assert(destinations[i].Node->GetLclNum() != destinations[j].Node->GetLclNum()); + } + } +#endif + } + +#if DEBUGGABLE_GENTREE + GenTreeStoreLclVars() = default; +#endif + + GenTreeLclVar* GetDestination(unsigned index) const + { + assert(index < m_count); + return m_destinations[index].Node; + } +}; + // gtLclFld -- load/store/addr of local variable field struct GenTreeLclFld : public GenTreeLclVarCommon @@ -10121,6 +10169,10 @@ inline GenTree* GenTree::gtGetOp2IfPresent() const inline GenTree*& GenTree::Data() { assert(OperIsStore()); + if (OperIs(GT_STORE_LCL_VARS)) + { + return AsStoreLclVars()->gtOp1; + } return OperIsLocalStore() ? AsLclVarCommon()->Data() : AsIndir()->Data(); } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 622a9643e7db0b..66d801ed7e8df0 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -26,6 +26,7 @@ GTNODE(LCL_FLD , GenTreeLclFld ,0,0,GTK_LEAF) // field GTNODE(STORE_LCL_VAR , GenTreeLclVar ,0,1,GTK_UNOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // store to local variable GTNODE(STORE_LCL_FLD , GenTreeLclFld ,0,1,GTK_UNOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // store to a part of the variable GTNODE(LCL_ADDR , GenTreeLclFld ,0,0,GTK_LEAF) // local address +GTNODE(STORE_LCL_VARS , GenTreeStoreLclVars,0,1,GTK_UNOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // store register results to independent locals //----------------------------------------------------------------------------- // Leaf nodes (i.e. these nodes have no sub-operands): diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index bec1093d34d0b9..6e57df3fbcfd46 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -64,6 +64,7 @@ GTSTRUCT_1(MskCon , GT_CNS_MSK) #endif // FEATURE_MASKED_HW_INTRINSICS GTSTRUCT_N(LclVarCommon, GT_LCL_VAR, GT_LCL_FLD, GT_PHI_ARG, GT_STORE_LCL_VAR, GT_STORE_LCL_FLD, GT_LCL_ADDR) GTSTRUCT_2(LclVar , GT_LCL_VAR, GT_STORE_LCL_VAR) +GTSTRUCT_1(StoreLclVars, GT_STORE_LCL_VARS) GTSTRUCT_3(LclFld , GT_LCL_FLD, GT_STORE_LCL_FLD, GT_LCL_ADDR) GTSTRUCT_1(Cast , GT_CAST) GTSTRUCT_1(Box , GT_BOX) diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index b114ac05a48193..63eecaa36bf083 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -84,6 +84,13 @@ class LocalSequencer final : public GenTreeVisitor { SequenceCall(node->AsCall()); } + else if (node->OperIs(GT_STORE_LCL_VARS)) + { + node->VisitPhysicalLocalDefNodes(m_compiler, [=](GenTreeLclVarCommon* def) { + SequenceLocal(def); + return GenTree::VisitResult::Continue; + }); + } return fgWalkResult::WALK_CONTINUE; } diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 416259962376e5..804ff85fd69791 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -3196,6 +3196,18 @@ bool LclVarDsc::CanBeReplacedWithItsField(Compiler* comp) const void Compiler::lvaMarkLclRefs(GenTree* tree, BasicBlock* block, Statement* stmt) { + if (tree->OperIs(GT_STORE_LCL_VARS)) + { + tree->VisitPhysicalLocalDefNodes(this, [&](GenTreeLclVarCommon* def) { + lvaMarkLclRefs(def, block, stmt); + LclVarDsc* varDsc = lvaGetDesc(def); + varDsc->lvSingleDefRegCandidate = false; + varDsc->lvDisqualifySingleDefRegCandidate = true; + return GenTree::VisitResult::Continue; + }); + return; + } + const weight_t weight = block->getBBWeight(this); /* Is this a call to unmanaged code ? */ @@ -3680,6 +3692,16 @@ void Compiler::lvaComputePreciseRefCounts(bool isRecompute, bool setSlotNumbers) lvaGenericsContextInUse = true; } } + else if (node->OperIs(GT_STORE_LCL_VARS)) + { + node->VisitPhysicalLocalDefNodes(this, [&](GenTreeLclVarCommon* def) { + LclVarDsc* varDsc = lvaGetDesc(def); + bool zeroWeight = + varDsc->lvTracked && varDsc->IsLiveInOutOfHandler() && !varDsc->lvDoNotEnregister; + varDsc->incRefCnts(zeroWeight ? 0 : weight, this); + return GenTree::VisitResult::Continue; + }); + } } } else diff --git a/src/coreclr/jit/lir.cpp b/src/coreclr/jit/lir.cpp index da1a6ff5f5f41e..f4cabc6f4da0bb 100644 --- a/src/coreclr/jit/lir.cpp +++ b/src/coreclr/jit/lir.cpp @@ -1451,7 +1451,28 @@ class CheckLclVarSemanticsHelper PushLclVarRead(nodeInfo); } - if (nodeInfo.IsLclVarWrite()) + if (node->OperIs(GT_STORE_LCL_VARS)) + { + auto checkDef = [&](GenTreeLclVarCommon* def) { + SmallHashTable* reads; + if (unusedLclVarReads.TryGetValue(def->GetLclNum(), &reads)) + { + for (auto read : *reads) + { + JITDUMP("STORE_LCL_VARS [%06u] overlaps outstanding read [%06u] of V%02u\n", + Compiler::dspTreeID(node), Compiler::dspTreeID(read.Key()), def->GetLclNum()); + assert(!"Write to unaliased local overlaps outstanding read"); + return GenTree::VisitResult::Abort; + } + } + return GenTree::VisitResult::Continue; + }; + if (node->VisitPhysicalLocalDefNodes(compiler, checkDef) == GenTree::VisitResult::Abort) + { + return false; + } + } + else if (nodeInfo.IsLclVarWrite()) { // If this node is a lclVar write, it must not alias a lclVar with an outstanding read SmallHashTable* reads; diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 02af2ef176491d..1e211f8a54662b 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -727,6 +727,13 @@ void Liveness::PerNodeLocalVarLiveness(GenTree* tree) switch (tree->gtOper) { + case GT_STORE_LCL_VARS: + tree->VisitPhysicalLocalDefNodes(m_compiler, [=](GenTreeLclVarCommon* def) { + MarkUseDef(def); + return GenTree::VisitResult::Continue; + }); + break; + case GT_QMARK: case GT_COLON: // We never should encounter a GT_QMARK or GT_COLON node @@ -1613,6 +1620,13 @@ void Liveness::ComputeLife(VARSET_TP& life, varDsc = m_compiler->lvaGetDesc(partialDef); } } + else if (tree->OperIs(GT_STORE_LCL_VARS)) + { + tree->VisitPhysicalLocalDefNodes(m_compiler, [&](GenTreeLclVarCommon* def) { + ComputeLifeLocal(life, keepAliveVars, def); + return GenTree::VisitResult::Continue; + }); + } else if (tree->OperIsNonPhiLocal()) { isUse = (tree->gtFlags & GTF_VAR_USEASG) != 0; @@ -2254,6 +2268,21 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR bool isDeadStore; switch (node->OperGet()) { + case GT_STORE_LCL_VARS: + { + bool allDead = true; + node->VisitPhysicalLocalDefNodes(m_compiler, [&](GenTreeLclVarCommon* def) { + allDead &= ComputeLifeLocal(life, keepAliveVars, def); + return GenTree::VisitResult::Continue; + }); + if (allDead && TLiveness::EliminateDeadCode) + { + node->Data()->SetUnusedValue(); + blockRange.Remove(node); + } + break; + } + case GT_CALL: { GenTreeCall* const call = node->AsCall(); diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index edbd7c05e2e7a4..11ae8b25105930 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -626,6 +626,10 @@ GenTree* Lowering::LowerNode(GenTree* node) break; } + case GT_STORE_LCL_VARS: + // Each result already has the exact type of its destination. + break; + case GT_STORE_LCL_VAR: WidenSIMD12IfNecessary(node->AsLclVarCommon()); FALLTHROUGH; @@ -9186,6 +9190,20 @@ void Lowering::CheckNode(Compiler* compiler, GenTree* node) { switch (node->OperGet()) { + case GT_STORE_LCL_VARS: + { + GenTreeStoreLclVars* store = node->AsStoreLclVars(); + assert(store->gtOp1->IsMultiRegNode()); + assert(store->gtOp1->GetMultiRegCount(compiler) == store->m_count); + for (unsigned i = 0; i < store->m_count; i++) + { + LclVarDsc* varDsc = compiler->lvaGetDesc(store->GetDestination(i)); + assert(!varDsc->IsAddressExposed() && !varDsc->lvPromoted && (varDsc->TypeGet() != TYP_STRUCT)); + assert(genActualType(varDsc) == genActualType(store->gtOp1->GetRegTypeByIndex(i))); + } + break; + } + case GT_CALL: CheckCall(node->AsCall()); break; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 3541e1a2309349..32a470b0f55caf 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2015,6 +2015,7 @@ class LinearScan : public RegAllocInterface int BuildIntrinsic(GenTree* tree); void BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, LclVarDsc* varDsc, RefPosition* singleUseRef, int index); int BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc); + int BuildStoreLclVars(GenTreeStoreLclVars* store); int BuildStoreLoc(GenTreeLclVarCommon* tree); int BuildIndir(GenTreeIndir* indirTree); int BuildGCWriteBarrier(GenTree* tree); diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 8a99f1d37906f2..6a02e0992fd132 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -246,6 +246,10 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + srcCount = BuildStoreLclVars(tree->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) { diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7856d4302d8aa3..e041283d6d5688 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -675,6 +675,10 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + srcCount = BuildStoreLclVars(tree->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0df524b0a9abda..9521331dfd481c 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -4007,6 +4007,30 @@ void LinearScan::BuildStoreLocDef(GenTreeLclVarCommon* storeLoc, #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE } +//------------------------------------------------------------------------ +// BuildStoreLclVars: Consume each result before defining its independent local. +// As with promoted multi-register stores, separate locations let LSRA resolve +// conflicts between an earlier destination and a later source. +int LinearScan::BuildStoreLclVars(GenTreeStoreLclVars* store) +{ + assert(store->gtOp1->GetMultiRegCount(m_compiler) == store->m_count); + for (unsigned i = 0; i < store->m_count; i++) + { + GenTreeLclVar* destination = store->GetDestination(i); + LclVarDsc* varDsc = m_compiler->lvaGetDesc(destination); + RefPosition* use = BuildUse(store->gtOp1, RBM_NONE, i); + if (isCandidateVar(varDsc)) + { + BuildStoreLocDef(destination, varDsc, use, 0); + } + if (i + 1 < store->m_count) + { + currentLoc += 2; + } + } + return store->m_count; +} + //------------------------------------------------------------------------ // BuildMultiRegStoreLoc: Set register requirements for a store of a lclVar // diff --git a/src/coreclr/jit/lsraloongarch64.cpp b/src/coreclr/jit/lsraloongarch64.cpp index 5565c47033b8f0..a99cb4eba60db8 100644 --- a/src/coreclr/jit/lsraloongarch64.cpp +++ b/src/coreclr/jit/lsraloongarch64.cpp @@ -102,6 +102,10 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + srcCount = BuildStoreLclVars(tree->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) { diff --git a/src/coreclr/jit/lsrariscv64.cpp b/src/coreclr/jit/lsrariscv64.cpp index 2531ce69bf4605..bf3250d1d4be23 100644 --- a/src/coreclr/jit/lsrariscv64.cpp +++ b/src/coreclr/jit/lsrariscv64.cpp @@ -103,6 +103,10 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + srcCount = BuildStoreLclVars(tree->AsStoreLclVars()); + break; + case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) { diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 91adfbd6dc7ef0..d9789c57fb6902 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -112,6 +112,10 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_STORE_LCL_VARS: + srcCount = BuildStoreLclVars(tree->AsStoreLclVars()); + break; + case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 4f53dd503fb555..3c03e7c9bac580 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -5345,6 +5345,16 @@ void Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk, FlowGraphNatura // that the compiler creates. switch (oper) { + case GT_STORE_LCL_VARS: + tree->VisitPhysicalLocalDefNodes(this, [&](GenTreeLclVarCommon* def) { + if (lvaVarAddrExposed(def->GetLclNum())) + { + memoryHavoc |= memoryKindSet(ByrefExposed); + } + return GenTree::VisitResult::Continue; + }); + break; + case GT_STORE_LCL_VAR: case GT_STORE_LCL_FLD: { @@ -5824,6 +5834,22 @@ void Compiler::optRemoveRedundantZeroInits() switch (tree->gtOper) { + case GT_STORE_LCL_VARS: + tree->VisitLogicalLocalDefs(this, [&](const auto& def) { + unsigned lclNum = def.GetLclNum(); + unsigned count = 0; + refCounts.Lookup(lclNum, &count); + refCounts.Set(lclNum, count + 1, LclVarRefCounts::Overwrite); + if (lvaGetDesc(lclNum)->lvTracked) + { + count = 0; + defsInBlock.Lookup(lclNum, &count); + defsInBlock.Set(lclNum, count + 1, LclVarRefCounts::Overwrite); + } + return GenTree::VisitResult::Continue; + }); + break; + case GT_LCL_VAR: case GT_LCL_FLD: case GT_LCL_ADDR: diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index e82c0ae436ac9d..a909b268849a9c 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -309,6 +309,7 @@ class ReplaceVisitor : public GenTreeVisitor void MarkForReadBack(GenTreeLclVarCommon* lcl, unsigned size DEBUGARG(const char* reason)); void HandleStructStore(GenTree** use, GenTree* user); + bool TryStoreMultiRegValue(GenTree** use, Replacement* first, Replacement* end); bool OverlappingReplacements(GenTreeLclVarCommon* lcl, Replacement** firstReplacement, Replacement** endReplacement = nullptr); diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index c2c3eb28385c11..ea69ff323cb49d 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -1351,6 +1351,73 @@ const GenTree* Compiler::gtPeelFieldAddrs(const GenTree* addr) const return gtPeelFieldAddrs(const_cast(addr)); } +// TryStoreMultiRegValue: Replace a complete, exact register-to-field mapping. +// Packed fields and partial promotion continue through the existing readback path. +bool ReplaceVisitor::TryStoreMultiRegValue(GenTree** use, Replacement* first, Replacement* end) +{ + GenTree* store = *use; + if (!store->OperIsLocalStore()) + { + return false; + } + + GenTree* source = store->Data(); + if ((!source->IsCall() && !source->OperIsHWIntrinsic()) || !source->IsMultiRegNode()) + { + return false; + } +#ifdef SWIFT_SUPPORT + if (source->IsCall() && source->AsCall()->GetUnmanagedCallConv() == CorInfoCallConvExtension::Swift) + { + return false; + } +#endif + unsigned count = source->GetMultiRegCount(m_compiler); + if ((count <= 1) || (static_cast(end - first) != count)) + { + return false; + } + + unsigned size = store->AsLclVarCommon()->GetLayout(m_compiler)->GetSize(); + unsigned baseOffset = store->AsLclVarCommon()->GetLclOffs(); + unsigned offset = 0; + for (unsigned i = 0; i < count; i++) + { + var_types type = source->GetRegTypeByIndex(i); + if ((first[i].Offset != baseOffset + offset) || (genActualType(first[i].AccessType) != genActualType(type)) || + (genTypeSize(first[i].AccessType) != genTypeSize(type))) + { + return false; + } + if (source->IsCall() && source->AsCall()->GetReturnTypeDesc()->GetReturnFieldOffset(i) != offset) + { + return false; + } + offset += genTypeSize(type); + } + if (offset != size) + { + return false; + } + + auto* destinations = new (m_compiler, CMK_ASTNode) GenTreeStoreLclVars::Destination[count]; + for (unsigned i = 0; i < count; i++) + { + Replacement& rep = first[i]; + GenTreeLclVar* def = m_compiler->gtNewLclvNode(rep.LclNum, genActualType(rep.AccessType))->AsLclVar(); + def->gtFlags |= GTF_VAR_DEF; + destinations[i] = {def, rep.Offset - baseOffset}; + ClearNeedsReadBack(rep); + SetNeedsWriteBack(rep); + } + *use = new (m_compiler, GT_STORE_LCL_VARS) GenTreeStoreLclVars(source, destinations, count, size); + m_compiler->gtUpdateNodeSideEffects(*use); + m_madeChanges = true; + JITDUMP("Created STORE_LCL_VARS for %u register results\n", count); + return true; +} + +//------------------------------------------------------------------------ // HandleStructStore: // Handle a store that may be between struct locals with replacements. // @@ -1382,6 +1449,11 @@ void ReplaceVisitor::HandleStructStore(GenTree** use, GenTree* user) return; } + if (dstInvolvesReplacements && TryStoreMultiRegValue(use, dstFirstRep, dstEndRep)) + { + return; + } + JITDUMP("Processing block operation [%06u] that involves replacements\n", Compiler::dspTreeID(store)); if (src->OperIs(GT_LCL_VAR, GT_LCL_FLD, GT_BLK) || src->IsConstInitVal()) diff --git a/src/coreclr/jit/sideeffects.cpp b/src/coreclr/jit/sideeffects.cpp index 8a6c6dfbcf48c8..be9475c585a5aa 100644 --- a/src/coreclr/jit/sideeffects.cpp +++ b/src/coreclr/jit/sideeffects.cpp @@ -148,6 +148,19 @@ AliasSet::NodeInfo::NodeInfo(Compiler* compiler, GenTree* node) , m_lclNum(0) , m_lclOffs(0) { + if (node->OperIs(GT_STORE_LCL_VARS)) + { + m_flags = ALIAS_WRITES_LCL_VAR; + m_lclNum = BAD_VAR_NUM; + node->VisitPhysicalLocalDefNodes(compiler, [&](GenTreeLclVarCommon* def) { + if (compiler->lvaGetDesc(def)->IsAddressExposed()) + { + m_flags |= ALIAS_WRITES_ADDRESSABLE_LOCATION; + } + return GenTree::VisitResult::Continue; + }); + return; + } if (node->IsCall()) { // For calls having return buffer, update the local number that is written after this call. @@ -409,6 +422,12 @@ bool AliasSet::InterferesWith(const AliasSet& other) const // bool AliasSet::InterferesWith(const NodeInfo& other) const { + if (other.Node()->OperIs(GT_STORE_LCL_VARS)) + { + AliasSet otherSet; + otherSet.AddNode(other.TheCompiler(), other.Node()); + return InterferesWith(otherSet); + } // First check whether or not this set interferes with the lclVar uses associated with the given node. if (m_writesAddressableLocation || !m_lclVarWrites.IsEmpty()) { diff --git a/src/coreclr/jit/sideeffects.h b/src/coreclr/jit/sideeffects.h index 35ed1c26ca5f0b..95227a84c61712 100644 --- a/src/coreclr/jit/sideeffects.h +++ b/src/coreclr/jit/sideeffects.h @@ -111,6 +111,7 @@ class AliasSet final inline unsigned LclNum() const { assert(IsLclVarRead() || IsLclVarWrite()); + assert(!m_node->OperIs(GT_STORE_LCL_VARS)); return m_lclNum; } @@ -122,6 +123,17 @@ class AliasSet final inline bool WritesAnyLocation() const { + if (m_node->OperIs(GT_STORE_LCL_VARS)) + { + auto isVisible = [=](GenTreeLclVarCommon* def) { + LclVarDsc* varDsc = m_compiler->lvaGetDesc(def); + bool visible = + varDsc->IsAddressExposed() || + (varDsc->lvTracked ? varDsc->IsLiveInOutOfHandler() : m_compiler->compHndBBtabCount > 0); + return visible ? GenTree::VisitResult::Abort : GenTree::VisitResult::Continue; + }; + return m_node->VisitPhysicalLocalDefNodes(m_compiler, isVisible) == GenTree::VisitResult::Abort; + } if ((m_flags & ALIAS_WRITES_ADDRESSABLE_LOCATION) != 0) { return true; diff --git a/src/coreclr/jit/ssabuilder.cpp b/src/coreclr/jit/ssabuilder.cpp index 5d40a365bdd0ca..6bd64a3a63c8db 100644 --- a/src/coreclr/jit/ssabuilder.cpp +++ b/src/coreclr/jit/ssabuilder.cpp @@ -487,7 +487,8 @@ unsigned SsaBuilder::RenamePushDef(GenTree* defNode, BasicBlock* block, unsigned LclVarDsc* const varDsc = m_compiler->lvaGetDesc(lclNum); unsigned const ssaNum = - varDsc->lvPerSsaData.AllocSsaNum(m_allocator, block, !defNode->IsCall() ? defNode->AsLclVarCommon() : nullptr); + varDsc->lvPerSsaData.AllocSsaNum(m_allocator, block, + defNode->OperIsLocalStore() ? defNode->AsLclVarCommon() : nullptr); if (!isFullDef) { diff --git a/src/coreclr/jit/treelifeupdater.cpp b/src/coreclr/jit/treelifeupdater.cpp index bb472b6986239f..dcb14a970017fd 100644 --- a/src/coreclr/jit/treelifeupdater.cpp +++ b/src/coreclr/jit/treelifeupdater.cpp @@ -306,7 +306,7 @@ void TreeLifeUpdater::UpdateLife(GenTree* tree) { UpdateLifeVar(tree, tree->AsIndir()->Addr()->AsLclVarCommon()); } - else if (tree->IsCall()) + else if (tree->OperIs(GT_CALL, GT_STORE_LCL_VARS)) { auto visitDef = [=](GenTreeLclVarCommon* lcl) { UpdateLifeVar(tree, lcl); diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 9cf0a130006d4d..40edd3d33798d7 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -6691,7 +6691,7 @@ void Compiler::fgValueNumberLocalStore(GenTree* storeNode, const TDef& def, Valu unsigned defLclNum = def.GetLclNum(); LclVarDsc* defVarDsc = lvaGetDesc(defLclNum); ValueNumPair defValue = value; - if (def.HasMultiDefIndex()) + if (def.NeedsValueExtraction()) { var_types defValueType = TYP_STRUCT; if (def.IsEntire(this)) @@ -11524,7 +11524,7 @@ const uint8_t ValueNumStore::s_vnfOpAttribs[VNF_COUNT] = { static genTreeOps genTreeOpsIllegalAsVNFunc[] = {GT_IND, // When we do heap memory. GT_NULLCHECK, GT_QMARK, GT_COLON, GT_LOCKADD, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_LCLHEAP, GT_BOX, GT_XORR, GT_XAND, GT_STORE_LCL_VAR, - GT_STORE_LCL_FLD, GT_STOREIND, GT_STORE_BLK, + GT_STORE_LCL_FLD, GT_STORE_LCL_VARS, GT_STOREIND, GT_STORE_BLK, // These need special semantics: GT_COMMA, // == second argument (but with exception(s) from first). GT_ARR_ADDR, GT_BOUNDS_CHECK, @@ -12890,7 +12890,7 @@ void Compiler::fgValueNumberStore(GenTree* store) assert(valueVNPair.BothDefined()); // Is the type being stored different from the type computed by "value"? - if (value->TypeGet() != store->TypeGet()) + if ((value->TypeGet() != store->TypeGet()) && !store->OperIs(GT_STORE_LCL_VARS)) { if (store->OperIsInitBlkOp()) { @@ -12924,6 +12924,7 @@ void Compiler::fgValueNumberStore(GenTree* store) // Now, record the new VN for the store (performing the indicated "state update"). switch (store->OperGet()) { + case GT_STORE_LCL_VARS: case GT_STORE_LCL_VAR: case GT_STORE_LCL_FLD: { @@ -13721,6 +13722,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) // Some of the genTreeOps that aren't legal VNFuncs so they get special handling. switch (oper) { + case GT_STORE_LCL_VARS: case GT_STORE_LCL_VAR: case GT_STORE_LCL_FLD: case GT_STOREIND: diff --git a/src/tests/JIT/opt/Structs/StoreLclVars.cs b/src/tests/JIT/opt/Structs/StoreLclVars.cs new file mode 100644 index 00000000000000..2e971d5297d96d --- /dev/null +++ b/src/tests/JIT/opt/Structs/StoreLclVars.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +using Xunit; + +/// Checks independent local definitions from multi-register results. +public class StoreLclVars +{ + // Explicit layout prevents legacy promotion. On platforms with register + // struct returns, physical promotion can consume the two results directly. + [StructLayout(LayoutKind.Explicit)] + private struct Pair + { + [FieldOffset(0)] public long First; + [FieldOffset(8)] public long Second; + [FieldOffset(0)] public (ulong Quotient, ulong Remainder) Results; + } + + [StructLayout(LayoutKind.Explicit)] + private struct RefPair + { + [FieldOffset(0)] public object First; + [FieldOffset(0)] public object Alias; + [FieldOffset(8)] public object Second; + } + + [StructLayout(LayoutKind.Explicit)] + private struct VectorPair + { + [FieldOffset(0)] public Vector128 First; + [FieldOffset(16)] public Vector128 Second; + [FieldOffset(0)] public (Vector128, Vector128) Results; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe int VectorResults(byte* address) + { + VectorPair pair = new VectorPair { Results = AdvSimd.Arm64.LoadPairVector128(address) }; + return pair.First.GetElement(0) + pair.First.GetElement(15) + + pair.Second.GetElement(0) + pair.Second.GetElement(15); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static RefPair MakeRefs(object first, object second) => new RefPair { Alias = first, Second = second }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool CheckRefs(object first, object second) + { + RefPair pair = MakeRefs(first, second); + if (!ReferenceEquals(pair.First, first) || !ReferenceEquals(pair.Second, second)) + return false; + GC.Collect(); + return ReferenceEquals(pair.First, first) && ReferenceEquals(pair.Second, second); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Pair MakePair(long first, long second) => new Pair { First = first, Second = second }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Pair MakePairOrThrow(long first, long second, bool throws) + { + if (throws) + throw new InvalidOperationException(); + return MakePair(first, second); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long CallResults(long first, long second) + { + Pair pair = MakePair(first, second); + return pair.First * 17 + pair.Second + (pair.First ^ pair.Second); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long CallOverwrite(long first, long second) + { + Pair pair = MakePair(first, second); + pair.First = 5; + return pair.First * 17 + pair.Second; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Divide(ulong value, ulong divisor) + { + Pair result = new Pair { Results = X86Base.X64.DivRem(value, 0, divisor) }; + return (ulong)result.First * 17 + (ulong)result.Second; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong Remainder(ulong value, ulong divisor) + { + var result = X86Base.X64.DivRem(value, 0, divisor); + return result.Remainder; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong DiscardFirst(ulong value, ulong divisor) + { + Pair result = new Pair { Results = X86Base.X64.DivRem(value, 0, divisor) }; + return (ulong)(result.Second + (result.First ^ result.First)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong OverwriteResults(ulong value, ulong divisor) + { + Pair result = new Pair { Results = X86Base.X64.DivRem(value, 0, divisor) }; + ulong previousQuotient = (ulong)result.First; + result.Results = X86Base.X64.DivRem((ulong)result.Second + 100, 0, divisor); + return previousQuotient + (ulong)result.First * 17 + (ulong)result.Second; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong LoopResults(ulong value, ulong divisor, int count) + { + Pair result = new Pair { First = (long)value }; + for (int i = 0; i < count; i++) + result.Results = X86Base.X64.DivRem((ulong)result.First + (ulong)result.Second, 0, divisor); + return (ulong)result.First * 17 + (ulong)result.Second; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static ulong UnrolledResults(ulong value, ulong divisor) + { + Pair result = new Pair { First = (long)value }; + for (int i = 0; i < 4; i++) + result.Results = X86Base.X64.DivRem((ulong)result.First + (ulong)result.Second, 0, divisor); + return (ulong)result.First * 17 + (ulong)result.Second; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long DiscardCall(bool throws) + { + Pair result = MakePairOrThrow(100, 23, throws); + return (result.First ^ result.First) + (result.Second ^ result.Second); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static long ExceptionPreservesPrevious(bool throws) + { + Pair result = new Pair { First = 13, Second = 7 }; + try + { + result = MakePairOrThrow(100, 23, throws); + } + catch (InvalidOperationException) + { + return result.First * 17 + result.Second + (result.First ^ result.Second); + } + return result.First * 17 + result.Second + (result.First ^ result.Second); + } + + /// Checks both results, dead results, and exceptional definitions. + [Fact] + public static unsafe void TestEntryPoint() + { + Assert.Equal(1838, CallResults(100, 23)); + Assert.Equal(108, CallOverwrite(100, 23)); + Assert.Equal(238, ExceptionPreservesPrevious(true)); + Assert.Equal(1838, ExceptionPreservesPrevious(false)); + Assert.True(CheckRefs(new object(), new object())); + Assert.Equal(0, DiscardCall(false)); + Assert.Throws(() => DiscardCall(true)); + if (AdvSimd.Arm64.IsSupported) + { + byte* values = stackalloc byte[32]; + for (int i = 0; i < 32; i++) + values[i] = (byte)i; + Assert.Equal(62, VectorResults(values)); + } + if (X86Base.X64.IsSupported) + { + Assert.Equal(1700ul, LoopResults(100, 17, 0)); + ulong first = 123456789; + ulong second = 0; + for (int i = 0; i < 5; i++) + { + if (i == 4) + Assert.Equal(first * 17 + second, UnrolledResults(123456789, 17)); + ulong sum = first + second; + first = sum / 17; + second = sum % 17; + } + Assert.Equal(first * 17 + second, LoopResults(123456789, 17, 5)); + foreach (ulong value in new ulong[] { 0, 1, 17, 123456789, ulong.MaxValue }) + { + foreach (ulong divisor in new ulong[] { 1, 3, 17, 65537, ulong.MaxValue }) + { + ulong expected = unchecked(value / divisor * 17 + value % divisor); + Assert.Equal(expected, Divide(value, divisor)); + Assert.Equal(value % divisor, Remainder(value, divisor)); + Assert.Equal(value % divisor, DiscardFirst(value, divisor)); + ulong next = unchecked(value % divisor + 100); + Assert.Equal(unchecked(value / divisor + next / divisor * 17 + next % divisor), + OverwriteResults(value, divisor)); + } + } + } + } +} diff --git a/src/tests/JIT/opt/Structs/StoreLclVars.csproj b/src/tests/JIT/opt/Structs/StoreLclVars.csproj new file mode 100644 index 00000000000000..af02dee0b57b5d --- /dev/null +++ b/src/tests/JIT/opt/Structs/StoreLclVars.csproj @@ -0,0 +1,12 @@ + + + true + true + true + $(NoWarn);SYSLIB5004 + + + + + + From 48d1d3b0c0d34b6e09c485b8e9078024088d8079 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 23 Sep 2026 17:18:47 +0100 Subject: [PATCH 2/3] JIT: fix multi-local store async and side-effect integration --- src/coreclr/jit/asyncanalysis.cpp | 6 +- src/coreclr/jit/codegencommon.cpp | 20 ++---- src/coreclr/jit/compiler.cpp | 5 +- src/coreclr/jit/gentree.cpp | 4 ++ src/coreclr/jit/gentree.h | 1 + src/coreclr/jit/liveness.cpp | 1 + src/coreclr/jit/optimizer.cpp | 6 ++ src/coreclr/jit/promotiondecomposition.cpp | 6 ++ src/tests/JIT/opt/Structs/StoreLclVars.cs | 48 ++++++++++++-- .../async/store-lcl-vars/store-lcl-vars.cs | 64 +++++++++++++++++++ .../store-lcl-vars/store-lcl-vars.csproj | 5 ++ 11 files changed, 141 insertions(+), 25 deletions(-) create mode 100644 src/tests/async/store-lcl-vars/store-lcl-vars.cs create mode 100644 src/tests/async/store-lcl-vars/store-lcl-vars.csproj diff --git a/src/coreclr/jit/asyncanalysis.cpp b/src/coreclr/jit/asyncanalysis.cpp index 2a9a770be721ff..130642f87b1851 100644 --- a/src/coreclr/jit/asyncanalysis.cpp +++ b/src/coreclr/jit/asyncanalysis.cpp @@ -167,7 +167,7 @@ static void MarkMutatedVarDsc(Compiler* compiler, LclVarDsc* varDsc, VARSET_TP& // static void UpdateMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated) { - if (node->OperIsLocalStore()) + if (node->OperIsLocalStore() || node->OperIs(GT_STORE_LCL_VARS)) { // If this is a zero initialization then we do not need to consider it // mutated if we know the prolog will zero it anyway (otherwise we @@ -175,7 +175,7 @@ static void UpdateMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mut // We could improve this a bit by still skipping it but inserting // explicit zero init on resumption, but these cases seem to be rare // and that would require tracking additional information. - if (IsDefaultValue(node->AsLclVarCommon()->Data()) && + if (node->OperIsLocalStore() && IsDefaultValue(node->AsLclVarCommon()->Data()) && !compiler->fgVarNeedsExplicitZeroInit(node->AsLclVarCommon()->GetLclNum(), /* bbInALoop */ false, /* bbIsReturn */ false)) { @@ -350,7 +350,7 @@ void DefaultValueAnalysis::DumpMutatedVarsIn() // static void MarkMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated) { - if (node->IsCall()) + if (node->OperIs(GT_CALL, GT_STORE_LCL_VARS)) { auto visitDef = [&](const auto& def) { MarkMutatedVarDsc(compiler, compiler->lvaGetDesc(def.GetLclNum()), mutated); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 8a15b49fe5288e..a996bf01a76d22 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -8045,24 +8045,18 @@ void CodeGen::genStoreLclVars(GenTreeStoreLclVars* store) LclVarDsc* varDsc = m_compiler->lvaGetDesc(destination); regNumber targetReg = destination->GetRegNum(); var_types type = varDsc->TypeGet(); - if (targetReg != REG_NA) + if (targetReg == REG_NA) { - inst_Mov(type, targetReg, sourceReg, /* canSkip */ true); - } - if (((targetReg == REG_NA) || varDsc->IsAlwaysAliveInMemory()) && !destination->IsLastUse(0)) - { - GetEmitter()->emitIns_S_R(ins_StoreFromSrc(sourceReg, type), emitTypeSize(type), sourceReg, - destination->GetLclNum(), 0); - } - varDsc->SetRegNum(targetReg == REG_NA ? REG_STK : targetReg); - if (targetReg != REG_NA) - { - genProduceReg(destination); + // A dead register definition can still need a stack home visible to an EH handler. + unsigned lclNum = destination->GetLclNum(); + GetEmitter()->emitIns_S_R(ins_Store(type, m_compiler->isSIMDTypeLocalAligned(lclNum)), emitTypeSize(type), + sourceReg, lclNum, 0); } else { - genUpdateLife(destination); + inst_Mov(type, targetReg, sourceReg, /* canSkip */ true); } + genUpdateLifeStore(destination, targetReg, varDsc); } } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 4e6e06d1eca612..db32da118cb974 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -5480,7 +5480,8 @@ void Compiler::SplitTreesRandomly() int numTrees = 0; for (GenTree* tree : stmt->TreeList()) { - if (tree->OperIs(GT_JTRUE)) // Due to relop invariant + // Keep relops with JTRUE and multi-register sources with STORE_LCL_VARS. + if (tree->OperIs(GT_JTRUE, GT_STORE_LCL_VARS)) { continue; } @@ -5491,7 +5492,7 @@ void Compiler::SplitTreesRandomly() int splitTree = rng.Next(numTrees); for (GenTree* tree : stmt->TreeList()) { - if (tree->OperIs(GT_JTRUE)) + if (tree->OperIs(GT_JTRUE, GT_STORE_LCL_VARS)) continue; if (splitTree == 0) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 078a16dffb7000..c6108552d5c35a 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -8927,6 +8927,10 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const { switch (OperGet()) { + case GT_STORE_LCL_VARS: + // All destinations are non-address-exposed locals. Source effects are tracked separately. + return false; + case GT_LCL_VAR: case GT_LCL_FLD: case GT_STORE_LCL_VAR: diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 487c5201727ce5..ad7a8da4f91d38 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -4075,6 +4075,7 @@ struct GenTreeLclVar : public GenTreeLclVarCommon // The definition nodes carry local, SSA, liveness and register information. They // are metadata, not operands: the source is evaluated once, before any definition. // Destinations are distinct, non-address-exposed scalar or SIMD locals. +// This is not an atomic or volatile memory store. struct GenTreeStoreLclVars : public GenTreeUnOp { struct Destination diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 1e211f8a54662b..0984abfd58c429 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -2279,6 +2279,7 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR { node->Data()->SetUnusedValue(); blockRange.Remove(node); + m_compiler->fgStmtRemoved = true; } break; } diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 3c03e7c9bac580..48bed6844a9b5d 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -4845,6 +4845,12 @@ void Compiler::optHoistLoopBlocks(FlowGraphNaturalLoop* loop, { isGloballyVisibleStore = m_compiler->lvaGetDesc(tree->AsLclVarCommon())->IsAddressExposed(); } + else if (tree->OperIs(GT_STORE_LCL_VARS)) + { + // These destinations are non-address-exposed locals. The source is visited separately, + // including any calls, volatile accesses, or other ordering side effects. + isGloballyVisibleStore = false; + } else { isGloballyVisibleStore = true; diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index ea69ff323cb49d..9bb0bdff7f8b4f 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -1356,6 +1356,7 @@ const GenTree* Compiler::gtPeelFieldAddrs(const GenTree* addr) const bool ReplaceVisitor::TryStoreMultiRegValue(GenTree** use, Replacement* first, Replacement* end) { GenTree* store = *use; + // Volatile and other indirect stores must retain their memory semantics. if (!store->OperIsLocalStore()) { return false; @@ -1366,6 +1367,11 @@ bool ReplaceVisitor::TryStoreMultiRegValue(GenTree** use, Replacement* first, Re { return false; } + // Async lowering may replace the call result with a struct local. + if (source->IsCall() && source->AsCall()->IsAsync()) + { + return false; + } #ifdef SWIFT_SUPPORT if (source->IsCall() && source->AsCall()->GetUnmanagedCallConv() == CorInfoCallConvExtension::Swift) { diff --git a/src/tests/JIT/opt/Structs/StoreLclVars.cs b/src/tests/JIT/opt/Structs/StoreLclVars.cs index 2e971d5297d96d..a35761337d13cb 100644 --- a/src/tests/JIT/opt/Structs/StoreLclVars.cs +++ b/src/tests/JIT/opt/Structs/StoreLclVars.cs @@ -7,6 +7,7 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +using System.Threading; using Xunit; /// Checks independent local definitions from multi-register results. @@ -47,16 +48,30 @@ private static unsafe int VectorResults(byte* address) } [MethodImpl(MethodImplOptions.NoInlining)] - private static RefPair MakeRefs(object first, object second) => new RefPair { Alias = first, Second = second }; + private static unsafe int VolatileVectorResults(byte* address) + { + Volatile.Write(ref address[0], (byte)5); + VectorPair before = new VectorPair { Results = AdvSimd.Arm64.LoadPairVector128(address) }; + Volatile.Write(ref address[0], (byte)9); + int observed = Volatile.Read(ref address[0]); + VectorPair after = new VectorPair { Results = AdvSimd.Arm64.LoadPairVector128(address) }; + return before.First.GetElement(0) + before.First.GetElement(15) + + before.Second.GetElement(0) + before.Second.GetElement(15) + observed + + after.First.GetElement(0) + after.First.GetElement(15) + + after.Second.GetElement(0) + after.Second.GetElement(15); + } [MethodImpl(MethodImplOptions.NoInlining)] - private static bool CheckRefs(object first, object second) + private static RefPair MakeRefs() => new RefPair { Alias = new int[] { 17 }, Second = new int[] { 23 } }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool CheckRefs() { - RefPair pair = MakeRefs(first, second); - if (!ReferenceEquals(pair.First, first) || !ReferenceEquals(pair.Second, second)) - return false; + RefPair pair = MakeRefs(); + ((int[])pair.First)[0]++; + ((int[])pair.Second)[0]++; GC.Collect(); - return ReferenceEquals(pair.First, first) && ReferenceEquals(pair.Second, second); + return ((int[])pair.First)[0] == 18 && ((int[])pair.Second)[0] == 24; } [MethodImpl(MethodImplOptions.NoInlining)] @@ -155,6 +170,22 @@ private static long ExceptionPreservesPrevious(bool throws) return result.First * 17 + result.Second + (result.First ^ result.Second); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static long ExceptionPreservesIntermediate(bool throws) + { + Pair result = new Pair { First = 13, Second = 7 }; + try + { + result = MakePair(1, 2); + result = MakePairOrThrow(100, 23, throws); + } + catch (InvalidOperationException) + { + return result.First * 17 + result.Second + (result.First ^ result.Second); + } + return result.First * 17 + result.Second + (result.First ^ result.Second); + } + /// Checks both results, dead results, and exceptional definitions. [Fact] public static unsafe void TestEntryPoint() @@ -163,7 +194,9 @@ public static unsafe void TestEntryPoint() Assert.Equal(108, CallOverwrite(100, 23)); Assert.Equal(238, ExceptionPreservesPrevious(true)); Assert.Equal(1838, ExceptionPreservesPrevious(false)); - Assert.True(CheckRefs(new object(), new object())); + Assert.Equal(22, ExceptionPreservesIntermediate(true)); + Assert.Equal(1838, ExceptionPreservesIntermediate(false)); + Assert.True(CheckRefs()); Assert.Equal(0, DiscardCall(false)); Assert.Throws(() => DiscardCall(true)); if (AdvSimd.Arm64.IsSupported) @@ -172,6 +205,7 @@ public static unsafe void TestEntryPoint() for (int i = 0; i < 32; i++) values[i] = (byte)i; Assert.Equal(62, VectorResults(values)); + Assert.Equal(147, VolatileVectorResults(values)); } if (X86Base.X64.IsSupported) { diff --git a/src/tests/async/store-lcl-vars/store-lcl-vars.cs b/src/tests/async/store-lcl-vars/store-lcl-vars.cs new file mode 100644 index 00000000000000..28fbe37045d2b9 --- /dev/null +++ b/src/tests/async/store-lcl-vars/store-lcl-vars.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Xunit; + +/// Checks multi-register local definitions across runtime-async suspensions. +public class AsyncStoreLclVars +{ + [StructLayout(LayoutKind.Explicit)] + private struct Pair + { + [FieldOffset(0)] public long First; + [FieldOffset(8)] public long Second; + [FieldOffset(0)] public (long, long) Overlap; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Pair MakePair(long value) => new Pair { First = value, Second = value + 1 }; + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task MakePairAsync(long value) + { + await Task.Yield(); + return MakePair(value); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task CheckLocalDefinitions() + { + Pair pair = MakePair(3); + await Task.Yield(); + Assert.Equal(62, pair.First * 17 + pair.Second + (pair.First ^ pair.Second)); + + // Each resumption changes both fields before suspending again. Continuation + // reuse must save the new definitions rather than keeping the previous values. + for (int i = 4; i < 8; i++) + { + pair = MakePair(i); + await Task.Yield(); + Assert.Equal(i * 17 + i + 1 + (i ^ (i + 1)), + pair.First * 17 + pair.Second + (pair.First ^ pair.Second)); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static async Task CheckAsyncCall() + { + Pair pair = await MakePairAsync(3); + Assert.Equal(62, pair.First * 17 + pair.Second + (pair.First ^ pair.Second)); + await Task.Yield(); + Assert.Equal(62, pair.First * 17 + pair.Second + (pair.First ^ pair.Second)); + } + + /// Checks newly allocated and reused continuations and async struct returns. + [ConditionalFact(typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsMultithreadingSupported))] + public static void TestEntryPoint() + { + CheckAsyncCall().GetAwaiter().GetResult(); + CheckLocalDefinitions().GetAwaiter().GetResult(); + } +} diff --git a/src/tests/async/store-lcl-vars/store-lcl-vars.csproj b/src/tests/async/store-lcl-vars/store-lcl-vars.csproj new file mode 100644 index 00000000000000..197767e2c4e249 --- /dev/null +++ b/src/tests/async/store-lcl-vars/store-lcl-vars.csproj @@ -0,0 +1,5 @@ + + + + + From e0d8fa7fb6846348af920d73847462f120c7d24c Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 23 Sep 2026 18:17:48 +0100 Subject: [PATCH 3/3] JIT: guard small multi-register results and ensure optimized regression coverage --- src/coreclr/jit/promotiondecomposition.cpp | 4 +++- src/tests/JIT/opt/Structs/StoreLclVars.csproj | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/promotiondecomposition.cpp b/src/coreclr/jit/promotiondecomposition.cpp index 9bb0bdff7f8b4f..38dca0b41eb197 100644 --- a/src/coreclr/jit/promotiondecomposition.cpp +++ b/src/coreclr/jit/promotiondecomposition.cpp @@ -1390,7 +1390,9 @@ bool ReplaceVisitor::TryStoreMultiRegValue(GenTree** use, Replacement* first, Re for (unsigned i = 0; i < count; i++) { var_types type = source->GetRegTypeByIndex(i); - if ((first[i].Offset != baseOffset + offset) || (genActualType(first[i].AccessType) != genActualType(type)) || + // Small integer results may require normalization before defining the local. + if (varTypeIsSmall(type) || (first[i].Offset != baseOffset + offset) || + (genActualType(first[i].AccessType) != genActualType(type)) || (genTypeSize(first[i].AccessType) != genTypeSize(type))) { return false; diff --git a/src/tests/JIT/opt/Structs/StoreLclVars.csproj b/src/tests/JIT/opt/Structs/StoreLclVars.csproj index af02dee0b57b5d..fcf4afb1a24d10 100644 --- a/src/tests/JIT/opt/Structs/StoreLclVars.csproj +++ b/src/tests/JIT/opt/Structs/StoreLclVars.csproj @@ -7,6 +7,7 @@ +