JIT: add explicit multi-local stores for multi-register results

src/coreclr/jit/asyncanalysis.cpp

@@ -167,15 +167,15 @@ static void MarkMutatedVarDsc(Compiler* compiler, LclVarDsc* varDsc, VARSET_TP&
 //
 static void UpdateMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated)
 {
-    if (node->OperIsLocalStore())
+    if (node->OperIsLocalStore() || node->OperIs(GT_STORE_LCL_VARS))
     {
         // If this is a zero initialization then we do not need to consider it
         // mutated if we know the prolog will zero it anyway (otherwise we
         // could be skipping this explicit zero init on resumption).
         // We could improve this a bit by still skipping it but inserting
         // explicit zero init on resumption, but these cases seem to be rare
         // and that would require tracking additional information.
-        if (IsDefaultValue(node->AsLclVarCommon()->Data()) &&
+        if (node->OperIsLocalStore() && IsDefaultValue(node->AsLclVarCommon()->Data()) &&
             !compiler->fgVarNeedsExplicitZeroInit(node->AsLclVarCommon()->GetLclNum(), /* bbInALoop */ false,
                                                   /* bbIsReturn */ false))
         {
@@ -350,7 +350,7 @@ void DefaultValueAnalysis::DumpMutatedVarsIn()
 //
 static void MarkMutatedLocal(Compiler* compiler, GenTree* node, VARSET_TP& mutated)
 {
-    if (node->IsCall())
+    if (node->OperIs(GT_CALL, GT_STORE_LCL_VARS))
     {
         auto visitDef = [&](const auto& def) {
             MarkMutatedVarDsc(compiler, compiler->lvaGetDesc(def.GetLclNum()), mutated);

src/coreclr/jit/codegen.h

@@ -1329,6 +1329,7 @@ class CodeGen final : public CodeGenInterface
 
     void genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode);
     void genMultiRegStoreToLocal(GenTreeLclVar* lclNode);
+    void genStoreLclVars(GenTreeStoreLclVars* store);
 
     // Codegen for multi-register struct returns.
     bool isStructReturn(GenTree* treeNode);

src/coreclr/jit/codegenarmarch.cpp

@@ -280,6 +280,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
             genCodeForStoreLclFld(treeNode->AsLclFld());
             break;
 
+        case GT_STORE_LCL_VARS:
+            genStoreLclVars(treeNode->AsStoreLclVars());
+            break;
+
         case GT_STORE_LCL_VAR:
             genCodeForStoreLclVar(treeNode->AsLclVar());
             break;

src/coreclr/jit/codegencommon.cpp

@@ -8035,6 +8035,32 @@ void CodeGen::genJmpPlaceArgs(GenTree* jmp)
 }
 
 //----------------------------------------------------------------------------------
+// genStoreLclVars: Move each result to its independent register or stack home.
+void CodeGen::genStoreLclVars(GenTreeStoreLclVars* store)
+{
+    for (unsigned i = 0; i < store->m_count; i++)
+    {
+        regNumber      sourceReg   = genConsumeReg(store->gtOp1, i);
+        GenTreeLclVar* destination = store->GetDestination(i);
+        LclVarDsc*     varDsc      = m_compiler->lvaGetDesc(destination);
+        regNumber      targetReg   = destination->GetRegNum();
+        var_types      type        = varDsc->TypeGet();
+        if (targetReg == REG_NA)
+        {
+            // A dead register definition can still need a stack home visible to an EH handler.
+            unsigned lclNum = destination->GetLclNum();
+            GetEmitter()->emitIns_S_R(ins_Store(type, m_compiler->isSIMDTypeLocalAligned(lclNum)), emitTypeSize(type),
+                                      sourceReg, lclNum, 0);
+        }
+        else
+        {
+            inst_Mov(type, targetReg, sourceReg, /* canSkip */ true);
+        }
+        genUpdateLifeStore(destination, targetReg, varDsc);
+    }
+}
+
+//------------------------------------------------------------------------
 // genMultiRegStoreToLocal: store multi-reg value to a local
 //
 // Arguments:

src/coreclr/jit/codegenloongarch64.cpp

@@ -4073,6 +4073,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
             genCodeForStoreLclFld(treeNode->AsLclFld());
             break;
 
+        case GT_STORE_LCL_VARS:
+            genStoreLclVars(treeNode->AsStoreLclVars());
+            break;
+
         case GT_STORE_LCL_VAR:
             genCodeForStoreLclVar(treeNode->AsLclVar());
             break;

src/coreclr/jit/codegenriscv64.cpp

@@ -3919,6 +3919,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
             genCodeForStoreLclFld(treeNode->AsLclFld());
             break;
 
+        case GT_STORE_LCL_VARS:
+            genStoreLclVars(treeNode->AsStoreLclVars());
+            break;
+
         case GT_STORE_LCL_VAR:
             genCodeForStoreLclVar(treeNode->AsLclVar());
             break;

src/coreclr/jit/codegenxarch.cpp

@@ -1997,6 +1997,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
             genCodeForStoreLclFld(treeNode->AsLclFld());
             break;
 
+        case GT_STORE_LCL_VARS:
+            genStoreLclVars(treeNode->AsStoreLclVars());
+            break;
+
         case GT_STORE_LCL_VAR:
             genCodeForStoreLclVar(treeNode->AsLclVar());
             break;

src/coreclr/jit/compiler.cpp

@@ -5480,7 +5480,8 @@ void Compiler::SplitTreesRandomly()
             int numTrees = 0;
             for (GenTree* tree : stmt->TreeList())
             {
-                if (tree->OperIs(GT_JTRUE)) // Due to relop invariant
+                // Keep relops with JTRUE and multi-register sources with STORE_LCL_VARS.
+                if (tree->OperIs(GT_JTRUE, GT_STORE_LCL_VARS))
                 {
                     continue;
                 }
@@ -5491,7 +5492,7 @@ void Compiler::SplitTreesRandomly()
             int splitTree = rng.Next(numTrees);
             for (GenTree* tree : stmt->TreeList())
             {
-                if (tree->OperIs(GT_JTRUE))
+                if (tree->OperIs(GT_JTRUE, GT_STORE_LCL_VARS))
                     continue;
 
                 if (splitTree == 0)

src/coreclr/jit/compiler.hpp

@@ -4454,7 +4454,7 @@ GenTree::VisitResult GenTree::VisitOperandUses(TVisitor visitor)
 template <typename TDerived>
 struct LocalDefProvider
 {
-    bool HasMultiDefIndex() const
+    bool NeedsValueExtraction() const
     {
         return static_cast<const TDerived*>(this)->GetMultiDefIndex() != BAD_VAR_NUM;
     }
@@ -4532,6 +4532,36 @@ struct StoreLclVarDef : LocalDefProvider<StoreLclVarDef>
     }
 };
 
+struct StoreLclVarsDef : StoreLclVarDef
+{
+    // Each destination has a scalar SSA number, but its value is a slice of
+    // the multi-register source rather than the source's entire value.
+    unsigned m_valueOffset;
+    unsigned m_storeSize;
+
+    StoreLclVarsDef(GenTreeStoreLclVars* store, unsigned index)
+        : StoreLclVarDef(store->GetDestination(index))
+        , m_valueOffset(store->m_destinations[index].Offset)
+        , m_storeSize(store->m_size)
+    {
+    }
+
+    bool NeedsValueExtraction() const
+    {
+        return true;
+    }
+
+    ssize_t GetValueOffset(Compiler* compiler) const
+    {
+        return m_valueOffset;
+    }
+
+    ValueSize GetStoreSize(Compiler* compiler) const
+    {
+        return ValueSize(m_storeSize);
+    }
+};
+
 struct PromotedStoreLclVarDef : LocalDefProvider<PromotedStoreLclVarDef>
 {
     GenTreeLclVarCommon* m_def;
@@ -4818,7 +4848,7 @@ GenTree::VisitResult VisitPromotedRangeLocalDefs(
 //
 inline bool GenTree::IsEntireLocalDef(Compiler* comp, GenTreeLclVarCommon* def)
 {
-    if (OperIs(GT_STORE_LCL_VAR))
+    if (OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_VARS))
     {
         return true;
     }
@@ -4921,6 +4951,15 @@ GenTree::VisitResult GenTree::VisitLocalDef(
 template <typename TVisitor>
 GenTree::VisitResult GenTree::VisitLogicalLocalDefs(Compiler* comp, TVisitor visitor)
 {
+    if (OperIs(GT_STORE_LCL_VARS))
+    {
+        GenTreeStoreLclVars* store = AsStoreLclVars();
+        for (unsigned i = 0; i < store->m_count; i++)
+        {
+            RETURN_IF_ABORT(visitor(StoreLclVarsDef(store, i)));
+        }
+        return VisitResult::Continue;
+    }
     if (OperIs(GT_STORE_LCL_VAR))
     {
         return VisitLocalDef(comp, AsLclVarCommon(), visitor);
@@ -4978,6 +5017,15 @@ GenTree::VisitResult GenTree::VisitLogicalLocalDefs(Compiler* comp, TVisitor vis
 template <typename TVisitor>
 GenTree::VisitResult GenTree::VisitPhysicalLocalDefNodes(Compiler* comp, TVisitor visitor)
 {
+    if (OperIs(GT_STORE_LCL_VARS))
+    {
+        GenTreeStoreLclVars* store = AsStoreLclVars();
+        for (unsigned i = 0; i < store->m_count; i++)
+        {
+            RETURN_IF_ABORT(visitor(store->GetDestination(i)));
+        }
+        return VisitResult::Continue;
+    }
     if (OperIs(GT_STORE_LCL_VAR))
     {
         return visitor(AsLclVarCommon());

src/coreclr/jit/fgdiagnostic.cpp

@@ -3847,7 +3847,7 @@ void Compiler::fgDebugCheckLinkedLocals()
                 }
             }
 
-            if (node->IsCall())
+            if (node->OperIs(GT_CALL, GT_STORE_LCL_VARS))
             {
                 auto linkDefs = [&](GenTree* def) {
                     assert(ShouldLink(def));