CI: trim per-PR simulator scene coverage

.claude/skills/testing/SKILL.md

@@ -27,7 +27,7 @@ and will report failures that CI never sees:
 
 | Marker | Tests | CI behavior |
 | ------ | ----- | ----------- |
-| `@pytest.mark.manual` / `CASES[*]["manual"]` | Standalone pytest tests / individual scene-test cases; optionally scoped to a platform list | Per-PR: excluded by default on the selected platforms; `daily.yml`: full sweep with `--manual include` |
+| `@pytest.mark.manual` / `CASES[*]["manual"]` | Standalone pytest tests / individual scene-test cases; optionally scoped to a platform list | Per-PR main sweep: excluded by default on the selected platforms; dedicated DFX steps: included; `daily.yml`: full sweep with `--manual include` |
 | `@pytest.mark.sdma` | a2a3: `sdma_async_completion_demo`, `prefetch_async_demo`; a5: `sdma_async_completion_demo` | a2a3: the dedicated SDMA step; a5: included in the non-pod sweep |
 
 The a2a3 SDMA demos provision 48 device-only STARS streams, which makes an

.github/workflows/_st-sim-a2a3.yml

@@ -111,34 +111,35 @@ jobs:
       - name: dep_gen smoke (a2a3)
         if: inputs.include_dfx_smokes
         run: |
-          .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py \
+          .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/ \
             --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-dep-gen
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen
 
       - name: dep_gen smoke (a2a3 host_build_graph)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py \
             --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-dep-gen
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen
 
       - name: chip_swimlane smoke (a2a3)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/ \
             --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-chip-swimlane --enable-dep-gen
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \
+            --enable-chip-swimlane --enable-dep-gen
 
       - name: PMU smoke (a2a3)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py \
             --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-pmu 2
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-pmu 2
 
       - name: args_dump smoke (a2a3)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py \
             --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --dump-args
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --dump-args

.github/workflows/_st-sim-a5.yml

@@ -111,27 +111,28 @@ jobs:
       - name: dep_gen smoke (a5)
         if: inputs.include_dfx_smokes
         run: |
-          .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py \
+          .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/ \
             --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-dep-gen
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen
 
       - name: chip_swimlane smoke (a5)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/ \
             --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-chip-swimlane --enable-dep-gen
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \
+            --enable-chip-swimlane --enable-dep-gen
 
       - name: PMU smoke (a5)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py \
             --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --enable-pmu 2
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-pmu 2
 
       - name: args_dump smoke (a5)
         if: inputs.include_dfx_smokes
         run: |
           .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py \
             --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \
-            --require-pto-isa --dump-args
+            --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --dump-args

docs/ci.md

@@ -106,9 +106,14 @@ scene-test corpus with `--manual include` once per day and supports manual
 re-runs through `workflow_dispatch`. Simulation runs on Ubuntu and macOS for
 both architectures; onboard runs on the A2/A3 and A5 self-hosted pools. The
 same DFX smoke steps used by Per-PR run once in each Daily platform job, and
-the A2/A3 Pod corpus runs through the existing two-machine workflow. Per-PR
-scene-test jobs keep the default `--manual exclude`, so moving a case to Daily
-does not require a second workflow exclusion list.
+the A2/A3 Pod corpus runs through the existing two-machine workflow. The main
+Per-PR scene-test steps keep the default `--manual exclude`, so moving an
+ordinary case to Daily does not require a second workflow exclusion list.
+Dedicated DFX steps instead use `include` for the normal Per-PR and Daily modes
+because they own the full corpus under their target paths; a `manual_mode` of
+`only` remains `only` in those steps. Marking a DFX case manual therefore
+removes its duplicate execution from the main step without removing its
+dedicated Per-PR coverage.
 
 Use `"manual": True` on an individual `SceneTestCase.CASES` entry and
 `@pytest.mark.manual` on a standalone pytest test. The reusable scene-test
@@ -117,8 +122,10 @@ Daily caller passes `include`.
 
 For platform-specific pruning, the same `manual` value accepts a platform list,
 for example `"manual": ["a2a3sim", "a5sim"]`; standalone tests use
-`@pytest.mark.manual(["a2a3sim", "a5sim"])`. This removes only the Sim execution
-from Per-PR while retaining onboard coverage.
+`@pytest.mark.manual(["a2a3sim", "a5sim"])`. This removes only the listed Sim
+executions from Per-PR. It retains same-architecture onboard coverage only when
+the case also declares that onboard platform; otherwise that architecture's
+path becomes Daily-only and must be called out in the PR's coverage analysis.
 
 ### Nightly sanitizer sweep
 

docs/testing.md

@@ -731,16 +731,21 @@ still respect the manual filter — to run a manual case by name, pass
 The separate `daily.yml` workflow runs the full corpus with `--manual include`
 once per day on A2/A3 and A5, simulation and onboard. Mark a whole standalone
 pytest test with `@pytest.mark.manual`; mark only one case in a `SceneTestCase`
-by setting `"manual": True` on that `CASES` entry. Per-PR excludes those tests,
-while Daily runs them together with the regular corpus. The A2/A3 Pod cases run
-in the same Daily workflow through their existing two-machine job.
+by setting `"manual": True` on that `CASES` entry. The main Per-PR scene sweep
+excludes those tests, while Daily runs them together with the regular corpus.
+Dedicated DFX steps are the exception: they use `--manual include` in normal
+Per-PR and Daily jobs, so marking a case under their target path removes only
+its duplicate main-sweep execution. A caller selecting `--manual only` keeps
+that mode in the DFX steps. The A2/A3 Pod cases run in the same Daily workflow
+through their existing two-machine job.
 
 To move only selected platforms, pass the platform list to the same marker:
 use `@pytest.mark.manual(["a2a3sim", "a5sim"])` (or the equivalent
 `@pytest.mark.manual(platforms=["a2a3sim", "a5sim"])`) for a standalone test,
 or `"manual": ["a2a3sim", "a5sim"]` for a scene-test case. Do not mix the two
-standalone marker forms. The onboard execution then remains in the default
-Per-PR sweep.
+standalone marker forms. An onboard execution remains in the default Per-PR
+sweep only when that case also declares the corresponding onboard platform;
+otherwise the selected architecture becomes Daily-only.
 
 ### Sharing an Example Between examples/ and tests/st/
 

examples/a2a3/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py

@@ -54,6 +54,7 @@ class TestBenchmarkBgemmHostBuildGraph(SceneTestCase):
         {
             "name": "Case0",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {"matmul_add_task_num": 500, "incore_data_size": 128, "incore_loop": 4, "grid_k": 2},
         },
     ]

examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py

@@ -113,6 +113,7 @@ class TestPagedAttentionManualScope(SceneTestCase):
         {
             "name": "CaseSmall1",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {
                 "batch": 1,
                 "num_heads": 16,

examples/a5/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py

@@ -52,6 +52,7 @@ class TestBenchmarkBgemmHostBuildGraph(SceneTestCase):
         {
             "name": "Case0",
             "platforms": ["a5sim", "a5"],
+            "manual": ["a5sim"],
             "params": {"matmul_add_task_num": 500, "incore_data_size": 128, "incore_loop": 4, "grid_k": 2},
         },
     ]

examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py

@@ -66,6 +66,7 @@ class TestPagedAttentionManualScope(SceneTestCase):
         {
             "name": "SmallCase1",
             "platforms": ["a5sim", "a5"],
+            "manual": ["a5sim"],
             "params": {
                 "batch": 1,
                 "num_heads": 16,

examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/test_paged_attention_unroll_manual_scope.py

@@ -110,6 +110,7 @@ class TestPagedAttentionUnrollManualScope(SceneTestCase):
         {
             "name": "SmallCase1",
             "platforms": ["a5sim", "a5"],
+            "manual": ["a5sim"],
             "params": {
                 "batch": 1,
                 "num_heads": 16,

examples/workers/l2/per_task_runtime_env/test_per_task_runtime_env.py

@@ -14,6 +14,7 @@
 
 
 @pytest.mark.platforms(["a2a3sim", "a2a3"])
+@pytest.mark.manual(["a2a3sim"])
 @pytest.mark.runtime("tensormap_and_ringbuffer")
 @pytest.mark.device_count(1)
 def test_per_task_runtime_env(st_platform, st_device_ids):

examples/workers/l3/allreduce/test_allreduce.py

@@ -22,6 +22,7 @@
 # declares all four platforms — so widening this is likely safe, but nobody has
 # exercised *this* demo there yet.
 @pytest.mark.platforms(["a2a3sim", "a2a3"])
+@pytest.mark.manual(["a2a3sim"])
 @pytest.mark.runtime("tensormap_and_ringbuffer")
 @pytest.mark.device_count(2)
 def test_allreduce(st_platform, st_device_ids):

examples/workers/l3/domain_rank_map/test_domain_rank_map.py

@@ -14,6 +14,7 @@
 
 
 @pytest.mark.platforms(["a2a3sim", "a2a3", "a5sim"])
+@pytest.mark.manual(["a2a3sim", "a5sim"])
 @pytest.mark.runtime("tensormap_and_ringbuffer")
 @pytest.mark.device_count(3)
 def test_domain_rank_map(st_platform, st_device_ids, capsys):

examples/workers/l3/multi_chip_dispatch/test_multi_chip_dispatch.py

@@ -14,6 +14,7 @@
 
 
 @pytest.mark.platforms(["a2a3sim", "a2a3", "a5sim", "a5"])
+@pytest.mark.manual(["a2a3sim", "a5sim"])
 @pytest.mark.runtime("tensormap_and_ringbuffer")
 @pytest.mark.device_count(2)
 def test_multi_chip_dispatch(st_platform, st_device_ids):

simpler_setup/scene_test.py

@@ -1673,9 +1673,13 @@ def _effective_enable_dep_gen(request, *, warn: bool = False) -> bool:
 
     def test_run(self, st_platform, st_worker, request):
         """Auto test method — runs matching cases for the current platform."""
-        raw_selectors = request.config.getoption("--case", default=None) or []
-        selectors = [_parse_case_selector(v) for v in raw_selectors]
+        matched = self._matching_cases(st_platform, request)
         manual_mode = request.config.getoption("--manual", default="exclude")
+        if not matched:
+            import pytest  # noqa: PLC0415
+
+            pytest.skip(f"No cases matched {type(self).__name__} (platform={st_platform}, manual={manual_mode})")
+
         rounds = request.config.getoption("--rounds", default=1)
         skip_golden = request.config.getoption("--skip-golden", default=False)
         enable_chip_swimlane = request.config.getoption("--enable-chip-swimlane", default=0)
@@ -1698,7 +1702,6 @@ def test_run(self, st_platform, st_worker, request):
                 logger.warning("scope_stats disabled: --rounds > 1")
                 enable_scope_stats = False
 
-        cls_name = type(self).__name__
         callable_obj = self.build_callable(st_platform)
         sub_handles = getattr(type(self), "_st_sub_handles", {})
         # For L3, use registered chip handles instead of raw ChipCallable
@@ -1707,24 +1710,6 @@ def test_run(self, st_platform, st_worker, request):
         if self._st_level == 3 and chip_handles:
             callable_obj = {**chip_handles}
 
-        matched = []
-        for case in self.CASES:
-            if st_platform not in case["platforms"]:
-                continue
-            if not _match_selectors(cls_name, case["name"], selectors):
-                continue
-            is_manual = is_manual_for_platform(case.get("manual"), st_platform)
-            if manual_mode == "exclude" and is_manual:
-                continue
-            if manual_mode == "only" and not is_manual:
-                continue
-            matched.append(case)
-
-        if not matched:
-            import pytest  # noqa: PLC0415
-
-            pytest.skip(f"No cases matched {cls_name} (platform={st_platform}, manual={manual_mode})")
-
         run_class_cases(
             st_worker,
             self,
@@ -1741,6 +1726,26 @@ def test_run(self, st_platform, st_worker, request):
             enable_swimlane_overhead=enable_swimlane_overhead,
         )
 
+    def _matching_cases(self, st_platform, request):
+        """Return cases selected by the platform, case, and manual filters."""
+        raw_selectors = request.config.getoption("--case", default=None) or []
+        selectors = [_parse_case_selector(v) for v in raw_selectors]
+        manual_mode = request.config.getoption("--manual", default="exclude")
+        cls_name = type(self).__name__
+        matched = []
+        for case in self.CASES:
+            if st_platform not in case["platforms"]:
+                continue
+            if not _match_selectors(cls_name, case["name"], selectors):
+                continue
+            is_manual = is_manual_for_platform(case.get("manual"), st_platform)
+            if manual_mode == "exclude" and is_manual:
+                continue
+            if manual_mode == "only" and not is_manual:
+                continue
+            matched.append(case)
+        return matched
+
     # ------------------------------------------------------------------
     # Standalone entry point
     # ------------------------------------------------------------------

tests/st/a2a3/host_build_graph/available_aicore_counts/test_available_aicore_counts.py

@@ -86,6 +86,7 @@ class TestAvailableAicoreCounts(SceneTestCase):
         {
             "name": "Default",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {},
         },
     ]

tests/st/a2a3/host_build_graph/batch_paged_attention/test_batch_paged_attention.py

@@ -100,6 +100,7 @@ class TestBatchPagedAttentionHostBuildGraph(SceneTestCase):
         {
             "name": "CaseSmall1",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {
                 "batch": 1,
                 "num_heads": 16,

tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py

@@ -124,6 +124,7 @@ class TestDepGenHostBuildGraph(SceneTestCase):
         {
             "name": "default",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {},
         },
     ]
@@ -148,9 +149,8 @@ def test_run(self, st_platform, st_worker, request):
         super().test_run(st_platform, st_worker, request)
         if not self._effective_enable_dep_gen(request):
             return
-        for case in self.CASES:
-            if st_platform in case.get("platforms", []):
-                self._post_validate(case, run_marker)
+        for case in self._matching_cases(st_platform, request):
+            self._post_validate(case, run_marker)
 
     def _post_validate(self, case, run_marker):
         """Assert deps.json holds the 6 edges of example_orchestration.cpp."""
@@ -236,6 +236,7 @@ class TestDepGenHostBuildGraphEdgeSources(SceneTestCase):
         {
             "name": "gate_open",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "config": {"aicpu_thread_num": 2},
             "params": {"case": 2},
         },
@@ -260,9 +261,8 @@ def test_run(self, st_platform, st_worker, request):
         super().test_run(st_platform, st_worker, request)
         if not self._effective_enable_dep_gen(request):
             return
-        for case in self.CASES:
-            if st_platform in case.get("platforms", []):
-                self._post_validate(case, run_marker)
+        for case in self._matching_cases(st_platform, request):
+            self._post_validate(case, run_marker)
 
     def _post_validate(self, case, run_marker):
         deps = _load_deps("TestDepGenHostBuildGraphEdgeSources", case["name"], run_marker)

tests/st/a2a3/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py

@@ -52,6 +52,7 @@ class TestGraphExecutionAicAivHostBuildGraph(SceneTestCase):
         {
             "name": "record_then_replay_aic_aiv",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "config": {"aicpu_thread_num": 4},
             "params": {},
         },

tests/st/a2a3/host_build_graph/prepared_callable/test_prepared_callable.py

@@ -90,6 +90,7 @@ class TestPreparedCallableHbg(SceneTestCase):
         {
             "name": "prepare_run_twice",
             "platforms": _PLATFORMS,
+            "manual": ["a2a3sim"],
             "params": {"a": 2.0, "b": 3.0},
         },
     ]

tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py

@@ -87,6 +87,7 @@ class TestArgsDump(SceneTestCase):
         {
             "name": "default",
             "platforms": ["a2a3sim", "a2a3"],
+            "manual": ["a2a3sim"],
             "params": {},
         },
     ]