[wasm] [spmi] Collect SPMI contexts from libraries via crossgen2

eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml

@@ -19,6 +19,8 @@ parameters:
   collectionName: ''
   targetOS: ''
   targetArch: ''
+  maxPartitionSizeMB: 25
+  crossgen2Composite: ''
 
 jobs:
 - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml
@@ -34,7 +36,7 @@ jobs:
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true
     continueOnError: ${{ parameters.continueOnError }}
-    collectionType: $ {{ parameters.collectionType }}
+    collectionType: ${{ parameters.collectionType }}
     collectionName: ${{ parameters.collectionName }}
 
     ${{ if ne(parameters.displayName, '') }}:
@@ -77,6 +79,15 @@ jobs:
       - name: TargetArchForCollection
         value: ${{ parameters.archType }}
 
+    # Cross-target collections can share collectionName/collectionType with a host-target
+    # collection on the same osGroup/archType/buildConfig, so published artifact names need
+    # the target to stay unique.
+    - name: CollectionTargetSuffix
+      ${{ if ne(parameters.targetArch, '') }}:
+        value: '_${{ parameters.targetOS }}_${{ parameters.targetArch }}'
+      ${{ else }}:
+        value: ''
+
     - template: /eng/pipelines/coreclr/templates/jit-python-variables.yml
       parameters:
         osGroup: ${{ parameters.osGroup }}
@@ -109,25 +120,20 @@ jobs:
         value: '$(Build.SourcesDirectory)/payload'
 
     - ${{ if eq(parameters.collectionName, 'libraries') }}:
-      - name: InputDirectory
-        value: '$(Core_Root_Dir)'
-    - ${{ if eq(parameters.collectionName, 'corelib') }}:
-      # For the corelib crossgen2 collection, InputDirectory holds the directory
-      # containing the single System.Private.CoreLib.dll that crossgen2 will compile.
-      # For wasm cross-target collections, source the wasm-built corelib (which has
-      # the correct TARGET_BROWSER / 32-bit conditional IL) instead of the host
-      # release Core_Root corelib. The wasm corelib build emits the managed assembly
-      # under an 'IL' subdirectory of the configuration output.
+      # Crossgen2 collections targeting browser/wasm compile the browser-wasm CoreCLR
+      # runtime pack rather than the host Core_Root. InputDirectory is the runtime pack
+      # RID directory; superpmi_collect_setup.py flattens lib/<tfm>/*.dll together with
+      # native/System.Private.CoreLib.dll into the assembly set to collect over.
       - ${{ if eq(parameters.targetArch, 'wasm') }}:
         - ${{ if eq(parameters.osGroup, 'windows') }}:
           - name: InputDirectory
-            value: '$(Build.SourcesDirectory)\artifacts\bin\coreclr\browser.wasm.Release\IL'
+            value: '$(Build.SourcesDirectory)\artifacts\bin\microsoft.netcore.app.runtime.browser-wasm\Release\runtimes\browser-wasm'
         - ${{ if ne(parameters.osGroup, 'windows') }}:
           - name: InputDirectory
-            value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/browser.wasm.Release/IL'
+            value: '$(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm/Release/runtimes/browser-wasm'
       - ${{ if ne(parameters.targetArch, 'wasm') }}:
         - name: InputDirectory
-          value: '$(Release_Core_Root_Dir)'
+          value: '$(Core_Root_Dir)'
     - ${{ if eq(parameters.collectionName, 'benchmarks') }}:
       - name: InputDirectory
         value: '$(Core_Root_Dir)'
@@ -167,7 +173,7 @@ jobs:
       displayName: Enable python venv
       condition: always()
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -target_os $(TargetOSForCollection) -target_arch $(TargetArchForCollection) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -target_os $(TargetOSForCollection) -target_arch $(TargetArchForCollection) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size ${{ parameters.maxPartitionSizeMB }} # size in MB
       displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }}
 
     # Create required directories for merged mch collection and superpmi logs
@@ -186,7 +192,7 @@ jobs:
     - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml
       parameters:
         HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
-        HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)'
+        HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)$(CollectionTargetSuffix)/$(Architecture)'
         HelixAccessToken: $(HelixApiAccessToken)
         HelixTargetQueues: $(Queue)
         HelixPreCommands: $(HelixPreCommand)
@@ -206,6 +212,8 @@ jobs:
         InputArtifacts: '$(InputArtifacts)'
         CollectionType: '$(CollectionType)'
         CollectionName: '$(CollectionName)'
+        Crossgen2ReferenceDirName: '$(Crossgen2ReferenceDirName)'
+        Crossgen2Composite: '${{ parameters.crossgen2Composite }}'
         continueOnError: true # Run the future step i.e. merge-mch step even if this step fails.
 
     # Always run merge step even if collection of some partition fails so we can store collection
@@ -223,7 +231,7 @@ jobs:
         archiveType: $(archiveType)
         tarCompression: $(tarCompression)
         archiveExtension: $(archiveExtension)
-        artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
         displayName: ${{ format('Upload artifacts SuperPMI {0}-{1} collection', parameters.collectionName, parameters.collectionType) }}
 
     - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
@@ -260,14 +268,14 @@ jobs:
       displayName: Publish SuperPMI logs
       inputs:
         targetPath: $(SpmiLogsLocation)
-        artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        artifactName: 'SuperPMI_Logs_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
       condition: always()
       continueOnError: true
 
     - task: PublishPipelineArtifact@1
       displayName: Publish SuperPMI build logs
       inputs:
         targetPath: $(Build.SourcesDirectory)/artifacts/log
-        artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
+        artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)$(CollectionTargetSuffix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)'
       condition: always()
       continueOnError: true

eng/pipelines/coreclr/templates/superpmi-collect-job.yml

@@ -13,6 +13,8 @@ parameters:
   collectionName: ''
   targetOS: ''
   targetArch: ''
+  maxPartitionSizeMB: 25
+  crossgen2Composite: ''
 
 ### SuperPMI job
 ### Each collection job depends on a corresponding build job with the same buildConfig and archType.
@@ -22,7 +24,12 @@ jobs:
   parameters:
     # Compute job name from template parameters
     jobName: ${{ format('superpmi_collect_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-    displayName: ${{ format('SuperPMI collect {4} {5} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType) }}
+    ${{ if eq(parameters.targetArch, '') }}:
+      displayName: ${{ format('SuperPMI collect {4} {5} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType) }}
+    ${{ else }}:
+      # Cross-target collections share the host osGroup/archType with a same-named host
+      # collection, so include the target in the display name to tell them apart.
+      displayName: ${{ format('SuperPMI collect {4} {5} {6} {7} {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.collectionName, parameters.collectionType, parameters.targetOS, parameters.targetArch) }}
     pool: ${{ parameters.pool }}
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -35,15 +42,17 @@ jobs:
     collectionName: ${{ parameters.collectionName }}
     targetOS: ${{ parameters.targetOS }}
     targetArch: ${{ parameters.targetArch }}
+    maxPartitionSizeMB: ${{ parameters.maxPartitionSizeMB }}
+    crossgen2Composite: ${{ parameters.crossgen2Composite }}
     # Test job depends on the corresponding build job
     dependsOn:
      - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_'
      - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}:
         - 'coreclr_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
-     # For the corelib crossgen2 collection targeting browser/wasm we also need a
-     # browser/wasm-built System.Private.CoreLib.dll, produced by the browser_wasm_win
-     # Release build.
-     - ${{ if and(eq(parameters.collectionName, 'corelib'), eq(parameters.targetArch, 'wasm')) }}:
+     # Crossgen2 collections targeting browser/wasm compile the browser/wasm managed
+     # assembly set, which comes from the browser-wasm CoreCLR runtime pack produced by
+     # the browser_wasm_win Release build.
+     - ${{ if eq(parameters.targetArch, 'wasm') }}:
         - 'build_browser_wasm_windows_Release_'
 
     variables:
@@ -78,16 +87,16 @@ jobs:
         cleanUnpackFolder: false
         overwriteExistingFiles: true
 
-    # For the corelib crossgen2 collection targeting browser/wasm, also download the
-    # browser/wasm-built runtime artifacts. The wasm-built System.Private.CoreLib.dll
-    # under artifacts/bin/coreclr/browser.wasm.Release/ is what crossgen2 will compile.
-    - ${{ if and(eq(parameters.collectionName, 'corelib'), eq(parameters.targetArch, 'wasm')) }}:
+    # For crossgen2 collections targeting browser/wasm, also download the browser-wasm
+    # CoreCLR runtime pack. Its runtimes/browser-wasm/lib/<tfm>/*.dll and
+    # runtimes/browser-wasm/native/System.Private.CoreLib.dll are what crossgen2 compiles.
+    - ${{ if eq(parameters.targetArch, 'wasm') }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
           unpackFolder: $(Build.SourcesDirectory)/artifacts/bin
-          artifactFileName: 'BuildArtifacts_browser_wasm_Release$(archiveExtension)'
-          artifactName: BuildArtifacts_browser_wasm_Release
-          displayName: 'browser/wasm Release runtime build artifacts'
+          artifactFileName: 'WasmRuntimePack_browser_wasm_Release$(archiveExtension)'
+          artifactName: WasmRuntimePack_browser_wasm_Release
+          displayName: 'browser/wasm Release runtime pack'
           cleanUnpackFolder: false
           overwriteExistingFiles: true
 

eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml

@@ -19,7 +19,7 @@ extends:
             testGroup: outerloop
             # Build clr.wasmjit on x64 hosts so the checked wasm cross-targeting JIT
             # (clrjit_universal_wasm_x64.dll) is present in the checked Core_Root for
-            # crossgen2-corelib-wasm SPMI collection. clr.alljits does not include it.
+            # crossgen2-libraries-wasm SPMI collection. clr.alljits does not include it.
             buildArgs: -s clr+libs+libs.tests+clr.wasmjit -rc $(_BuildConfig) -c Release /p:ArchiveTests=true
             timeoutInMinutes: 120
             postBuildSteps:
@@ -169,28 +169,32 @@ extends:
                   testGroup: outerloop
             disableComponentGovernance: true # No shipping artifacts produced by this pipeline
 
-      # Release build of System.Private.CoreLib.dll targeting browser/wasm, hosted on
-      # Windows x64. Produces artifacts/bin/coreclr/browser.wasm.Release/System.Private.CoreLib.dll,
-      # which is the assembly the corelib_wasm SPMI collection runs crossgen2 over.
-      # We only need the managed CoreLib here, so build the minimal clr.corelib subset.
+      # Release build of the browser/wasm CoreCLR runtime pack, hosted on Windows x64.
+      # Produces artifacts/bin/microsoft.netcore.app.runtime.browser-wasm/Release, whose
+      # runtimes/browser-wasm/lib/<tfm>/*.dll and runtimes/browser-wasm/native/System.Private.CoreLib.dll
+      # are the managed assemblies the wasm SPMI collection runs crossgen2 over.
+      # The runtime pack layout comes from the packs.product subset, which harvests native
+      # runtime files, so the full clr+libs+packs set has to be built here.
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
           jobTemplate: /eng/pipelines/common/global-build-job.yml
           buildConfig: Release
           platforms:
           - browser_wasm_win
           jobParameters:
-            buildArgs: -s clr.corelib -c Release
-            timeoutInMinutes: 120
+            buildArgs: -s clr+libs+packs -c Release
+            timeoutInMinutes: 180
             postBuildSteps:
+              # Only the runtime pack is needed downstream, so avoid uploading the whole
+              # (much larger) artifacts/bin tree that clr+libs+packs produces.
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:
-                  rootFolder: $(Build.SourcesDirectory)/artifacts/bin
-                  includeRootFolder: false
+                  rootFolder: $(Build.SourcesDirectory)/artifacts/bin/microsoft.netcore.app.runtime.browser-wasm
+                  includeRootFolder: true
                   archiveType: $(archiveType)
                   archiveExtension: $(archiveExtension)
                   tarCompression: $(tarCompression)
-                  artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)
+                  artifactName: WasmRuntimePack_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig)
             disableComponentGovernance: true # No shipping artifacts produced by this pipeline
 
       - template: /eng/pipelines/common/platform-matrix.yml
@@ -330,7 +334,8 @@ extends:
             collectionType: crossgen2
             collectionName: libraries
 
-      # Crossgen2 of release System.Private.CoreLib.dll targeting browser/wasm.
+      # Crossgen2 of the release browser/wasm managed assembly set (the browser-wasm
+      # CoreCLR runtime pack: all shared framework libraries plus System.Private.CoreLib.dll).
       # Runs crossgen2 on a windows_x64 host with the wasm cross-targeting JIT
       # (clrjit_universal_wasm_x64.dll) under superpmi-shim-collector, producing an MCH
       # uploaded under <jit_ee_version>/browser/x64/ (the wasm mch_arch override in
@@ -350,9 +355,23 @@ extends:
             testGroup: outerloop
             liveLibrariesBuildConfig: Release
             collectionType: crossgen2
-            collectionName: corelib
+            collectionName: libraries
             targetOS: browser
             targetArch: wasm
+            # Compile in composite mode so the whole framework is one version bubble and
+            # generic instantiations over types from other assemblies (most importantly
+            # CoreLib) get compiled instead of being skipped as cross-bubble. Measured on
+            # this collection: 82,870 -> 109,684 contexts (+32%).
+            crossgen2Composite: true
+            # Keep the collection partitioned. Each work item roots eligible methods from
+            # its partition's positional input assemblies. The rest of the framework is
+            # included as -u unrooted composite inputs in the same compilation and version
+            # bubble: not independently rooted, but able to contribute compiled methods
+            # and generic instantiations as dependencies. The redundant work between
+            # partitions is deduplicated by the merge step (177,230 contexts collected,
+            # 109,684 kept). That costs CPU rather than coverage, and keeps a failed
+            # partition from losing the whole collection. Work items ran 1-2 minutes each.
+            maxPartitionSizeMB: 5
 
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:

eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml

@@ -24,6 +24,8 @@ parameters:
   InputArtifacts: ''
   CollectionType: ''
   CollectionName: ''
+  Crossgen2ReferenceDirName: ''         # optional -- name of the crossgen2 reference assembly folder in the correlation payload (cross-target crossgen2 collections)
+  Crossgen2Composite: ''                # optional -- 'true' to run crossgen2 in composite mode (whole reference set in the version bubble)
   ProjectFile: ''
   targetOS: ''                           # optional -- override target OS for cross-target collections (e.g. browser/wasm). Defaults to osGroup.
   targetArch: ''                         # optional -- override target architecture. Defaults to archType.
@@ -51,6 +53,8 @@ steps:
       InputArtifacts: ${{ parameters.InputArtifacts }}
       CollectionType: ${{ parameters.CollectionType }}
       CollectionName: ${{ parameters.CollectionName }}
+      Crossgen2ReferenceDirName: ${{ parameters.Crossgen2ReferenceDirName }}
+      Crossgen2Composite: ${{ parameters.Crossgen2Composite }}
       HelixSource: ${{ parameters.HelixSource }}
       HelixType: ${{ parameters.HelixType }}
       HelixBuild: ${{ parameters.HelixBuild }}

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* 406b139f-b910-4579-a9a8-91eb153ab024 */
-    0x406b139f,
-    0xb910,
-    0x4579,
-    {0xa9, 0xa8, 0x91, 0xeb, 0x15, 0x3a, 0xb0, 0x24}
+constexpr GUID JITEEVersionIdentifier = { /* d9636771-eb8f-4e21-8a6f-d97f21e658f7 */
+    0xd9636771,
+    0xeb8f,
+    0x4e21,
+    {0x8a, 0x6f, 0xd9, 0x7f, 0x21, 0xe6, 0x58, 0xf7}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H