Stabilize fork-sensitive nightly tests and AutoSP coverage

.github/workflows/aws-torch-latest-full.yml

@@ -29,10 +29,13 @@ on:
           - '2.9.1-cu126'
           - '2.10.0-cu126'
           - '2.11.0-cu126'
+      # Used only when a manual workflow_dispatch explicitly selects
+      # transformers_source=pypi. Scheduled nightlies and default manual runs
+      # install Transformers from Git and ignore this value.
       transformers_version:
         description: Hugging Face Transformers PyPI package version to install
         required: false
-        default: '4.50.0'
+        default: '4.51.3'
         type: string
       transformers_source:
         description: Hugging Face Transformers source for manual runs
@@ -101,7 +104,8 @@ jobs:
     env:
       DEFAULT_TORCH_PRESET: '2.10.0-cu126'
       DEFAULT_TRANSFORMERS_SOURCE: 'git'
-      DEFAULT_TRANSFORMERS_VERSION: '4.50.0'
+      # Manual PyPI fallback only; scheduled and default manual runs use Git.
+      DEFAULT_TRANSFORMERS_VERSION: '4.51.3'
       DEFAULT_TRANSFORMERS_REF: 'main'
       CUTLASS_PATH: /opt/cutlass
       # Disable reuse_dist_env to prevent pool worker cleanup hangs in full test runs
@@ -220,8 +224,13 @@ jobs:
           echo "Resolved test expectations: torch=$torch_test_version cuda=$cuda_test_version"
           echo "Resolved PyTorch index: $pytorch_index_url"
           echo "Resolved Transformers source: $transformers_source"
-          echo "Resolved Transformers version: $transformers_version"
-          echo "Resolved Transformers ref: $transformers_ref"
+          if [ "$transformers_source" = 'git' ]; then
+            echo "Requested Transformers PyPI version: N/A"
+            echo "Requested Transformers git ref: $transformers_ref"
+          else
+            echo "Requested Transformers PyPI version: $transformers_version"
+            echo "Requested Transformers git ref: N/A"
+          fi
 
       - name: Install CUTLASS
         run: |
@@ -242,6 +251,7 @@ jobs:
           case "$TRANSFORMERS_SOURCE" in
             'pypi')
               pip install "transformers==$TRANSFORMERS_VERSION"
+              echo "TRANSFORMERS_RESOLVED_REF=N/A" >> "$GITHUB_ENV"
               ;;
             'git')
               git clone --filter=blob:none https://github.com/huggingface/transformers /tmp/transformers
@@ -275,9 +285,15 @@ jobs:
           echo "PyTorch index URL: $PYTORCH_INDEX_URL"
           echo "Expected test versions: torch=$TORCH_TEST_VERSION cuda=$CUDA_TEST_VERSION"
           echo "Transformers source: $TRANSFORMERS_SOURCE"
-          echo "Transformers version: $TRANSFORMERS_VERSION"
-          echo "Transformers ref: $TRANSFORMERS_REF"
-          echo "Transformers resolved ref: ${TRANSFORMERS_RESOLVED_REF:-}"
+          if [ "$TRANSFORMERS_SOURCE" = 'git' ]; then
+            echo "Requested Transformers PyPI version: N/A"
+            echo "Requested Transformers git ref: $TRANSFORMERS_REF"
+            echo "Resolved Transformers git SHA: $TRANSFORMERS_RESOLVED_REF"
+          else
+            echo "Requested Transformers PyPI version: $TRANSFORMERS_VERSION"
+            echo "Requested Transformers git ref: N/A"
+            echo "Resolved Transformers git SHA: N/A"
+          fi
           echo ""
           echo "=== GPU Information ==="
           nvidia-smi
@@ -341,7 +357,32 @@ jobs:
               exit 1
               ;;
           esac
-          python -c "import transformers; print('transformers:', transformers.__version__, transformers)"
+          installed_version="$(python -c 'import transformers; print(transformers.__version__)')"
+          if [ "$TRANSFORMERS_SOURCE" = 'git' ]; then
+            requested_version='N/A'
+            requested_ref="$TRANSFORMERS_REF"
+            resolved_ref="$TRANSFORMERS_RESOLVED_REF"
+          else
+            requested_version="$TRANSFORMERS_VERSION"
+            requested_ref='N/A'
+            resolved_ref='N/A'
+          fi
+          {
+            echo "Transformers source: $TRANSFORMERS_SOURCE"
+            echo "Requested Transformers PyPI version: $requested_version"
+            echo "Requested Transformers git ref: $requested_ref"
+            echo "Resolved Transformers git SHA: $resolved_ref"
+            echo "Imported Transformers version: $installed_version"
+          } | tee /dev/stderr
+          {
+            echo "### Transformers provenance"
+            echo ""
+            echo "- Source: \`$TRANSFORMERS_SOURCE\`"
+            echo "- Requested PyPI version: \`$requested_version\`"
+            echo "- Requested git ref: \`$requested_ref\`"
+            echo "- Resolved git SHA: \`$resolved_ref\`"
+            echo "- Imported version: \`$installed_version\`"
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Python environment
         run: |

.github/workflows/cpu-torch-latest.yml

@@ -13,10 +13,13 @@ on:
           - '2.8.0-cpu'
           - '2.9.1-cpu'
           - '2.10.0-cpu'
+      # Used only when a manual workflow_dispatch explicitly selects
+      # transformers_source=pypi. Scheduled nightlies and default manual runs
+      # install Transformers from Git and ignore this value.
       transformers_version:
         description: Hugging Face Transformers PyPI package version to install
         required: false
-        default: '4.50.0'
+        default: '4.51.3'
         type: string
       transformers_source:
         description: Hugging Face Transformers source for manual runs
@@ -80,7 +83,8 @@ jobs:
     env:
       DEFAULT_TORCH_PRESET: '2.10.0-cpu'
       DEFAULT_TRANSFORMERS_SOURCE: 'git'
-      DEFAULT_TRANSFORMERS_VERSION: '4.50.0'
+      # Manual PyPI fallback only; scheduled and default manual runs use Git.
+      DEFAULT_TRANSFORMERS_VERSION: '4.51.3'
       DEFAULT_TRANSFORMERS_REF: 'main'
 
     steps:
@@ -175,8 +179,13 @@ jobs:
           echo "Resolved install tuple: torch==$torch_install_version torchvision==$torchvision_install_version"
           echo "Resolved test expectation: torch=$torch_test_version"
           echo "Resolved Transformers source: $transformers_source"
-          echo "Resolved Transformers version: $transformers_version"
-          echo "Resolved Transformers ref: $transformers_ref"
+          if [ "$transformers_source" = 'git' ]; then
+            echo "Requested Transformers PyPI version: N/A"
+            echo "Requested Transformers git ref: $transformers_ref"
+          else
+            echo "Requested Transformers PyPI version: $transformers_version"
+            echo "Requested Transformers git ref: N/A"
+          fi
 
       - name: Install PyTorch
         run: |
@@ -192,6 +201,7 @@ jobs:
           case "$TRANSFORMERS_SOURCE" in
             'pypi')
               pip install "transformers==$TRANSFORMERS_VERSION"
+              echo "TRANSFORMERS_RESOLVED_REF=N/A" >> "$GITHUB_ENV"
               ;;
             'git')
               git clone --filter=blob:none https://github.com/huggingface/transformers /tmp/transformers
@@ -229,7 +239,32 @@ jobs:
               exit 1
               ;;
           esac
-          python -c "import transformers; print('transformers:', transformers.__version__, transformers)"
+          installed_version="$(python -c 'import transformers; print(transformers.__version__)')"
+          if [ "$TRANSFORMERS_SOURCE" = 'git' ]; then
+            requested_version='N/A'
+            requested_ref="$TRANSFORMERS_REF"
+            resolved_ref="$TRANSFORMERS_RESOLVED_REF"
+          else
+            requested_version="$TRANSFORMERS_VERSION"
+            requested_ref='N/A'
+            resolved_ref='N/A'
+          fi
+          {
+            echo "Transformers source: $TRANSFORMERS_SOURCE"
+            echo "Requested Transformers PyPI version: $requested_version"
+            echo "Requested Transformers git ref: $requested_ref"
+            echo "Resolved Transformers git SHA: $resolved_ref"
+            echo "Imported Transformers version: $installed_version"
+          } | tee /dev/stderr
+          {
+            echo "### Transformers provenance"
+            echo ""
+            echo "- Source: \`$TRANSFORMERS_SOURCE\`"
+            echo "- Requested PyPI version: \`$requested_version\`"
+            echo "- Requested git ref: \`$requested_ref\`"
+            echo "- Resolved git SHA: \`$resolved_ref\`"
+            echo "- Imported version: \`$installed_version\`"
+          } >> "$GITHUB_STEP_SUMMARY"
 
       - name: Python environment
         run: |

tests/unit/profiling/flops_profiler/test_flops_profiler.py

@@ -123,6 +123,7 @@ def test_flops_profiler_in_inference(self):
         assert params == 61706
 
 
+@pytest.mark.sequential
 def test_print_model_profile_with_none_dp_world_size(capsys):
     # Regression test for https://github.com/deepspeedai/DeepSpeed/issues/7483
     # Under sequence parallelism (Ulysses) the engine reports dp_world_size as None, which used to

tests/unit/runtime/zero/test_zero_tensor_fragment.py

@@ -98,7 +98,14 @@ class TestTensorFragmentGet(DistributedTest):
     @pytest.mark.parametrize('dtype', [torch.bfloat16, torch.float16, torch.float32])
     @pytest.mark.parametrize('api_type', ['local', 'full'])
     @pytest.mark.parametrize('zero_stage', [1, 2, 3])
-    @pytest.mark.parametrize('offload_device', [OffloadDeviceEnum.none, OffloadDeviceEnum.cpu, OffloadDeviceEnum.nvme])
+    @pytest.mark.parametrize(
+        'offload_device',
+        [
+            OffloadDeviceEnum.none,
+            OffloadDeviceEnum.cpu,
+            pytest.param(OffloadDeviceEnum.nvme, marks=pytest.mark.sequential),
+        ],
+    )
     def test_zero_fragments(self, tmpdir, dtype, api_type, zero_stage, offload_device, frozen_weights):
         if not dtype in get_accelerator().supported_dtypes():
             pytest.skip(f"{get_accelerator()._name} does not support {dtype} data type")

tests/unit/v1/compile/test_compile_autosp.py

@@ -26,6 +26,18 @@
 _SP_SIZE = 2
 
 
+def _create_sdpa_graph(seq_len):
+    graph = Graph()
+    inputs = []
+    for name in ("query", "key", "value"):
+        node = graph.placeholder(name)
+        node.meta["example_value"] = torch.empty(1, 2, seq_len, 8)
+        inputs.append(node)
+    sdpa = graph.call_function(F.scaled_dot_product_attention, args=tuple(inputs))
+    graph.output(sdpa)
+    return GraphModule({}, graph)
+
+
 class TestAutoSPCompile(DistributedTest):
     world_size = 4
     non_daemonic_procs = True
@@ -79,12 +91,13 @@ def test(self, zero_stage, dtype, sp_size):
 
 class TestSDPANodesCompile:
 
-    @pytest.mark.sequential
     @pytest.mark.parametrize('seq_len', [64, 128, 256])
     def test(self, seq_len):
         from deepspeed.compile.util import get_sdpa_nodes
 
-        gm, _ = create_gm_nodes(seq_len=seq_len)
+        # This DeepSpeed 0.19.3 patch was historically validated with Transformers 4.51.3, 5.13.0,
+        # and main@e9c987a3eceb1cd130f30cb5e1f1fac542e4fcfe (5.15.0.dev0); future revisions are not guaranteed.
+        gm = _create_sdpa_graph(seq_len)
         sdpa_nodes = get_sdpa_nodes(gm)
 
         assert len(sdpa_nodes) >= 1, f"Expected at least 1 SDPA node, got {len(sdpa_nodes)}"