From 1b2ef52c89271b487d822d73155181a91ba1c45e Mon Sep 17 00:00:00 2001 From: majin0824 Date: Sun, 16 Aug 2026 21:06:46 -0700 Subject: [PATCH] CI: trim per-PR simulator scene coverage Move simulator-only lifecycle, error, stress, scale, and redundant coverage to the manual set retained by daily CI. Keep representative paths and dedicated DFX smoke validation in per-PR CI while leaving onboard selection unchanged. Run dep-gen smoke targets over their complete directories and make DFX post-validation reuse the same platform, case, and manual filters as scene execution. Preserve the full HBG invalid-input matrix, validate retained fatal-code case names, and document the dedicated-DFX and platform-fallback semantics. --- .claude/skills/testing/SKILL.md | 2 +- .github/workflows/_st-sim-a2a3.yml | 13 ++--- .github/workflows/_st-sim-a5.yml | 11 +++-- docs/ci.md | 17 +++++-- docs/testing.md | 15 ++++-- .../benchmark_bgemm/test_benchmark_bgemm.py | 1 + .../test_paged_attention_manual_scope.py | 1 + .../benchmark_bgemm/test_benchmark_bgemm.py | 1 + .../test_paged_attention_manual_scope.py | 1 + ...est_paged_attention_unroll_manual_scope.py | 1 + .../test_per_task_runtime_env.py | 1 + .../workers/l3/allreduce/test_allreduce.py | 1 + .../domain_rank_map/test_domain_rank_map.py | 1 + .../test_multi_chip_dispatch.py | 1 + simpler_setup/scene_test.py | 47 ++++++++++--------- .../test_available_aicore_counts.py | 1 + .../test_batch_paged_attention.py | 1 + .../dfx/dep_gen/test_dep_gen.py | 12 ++--- .../test_graph_execution_aic_aiv.py | 1 + .../test_prepared_callable.py | 1 + .../dfx/args_dump/test_args_dump.py | 1 + .../dfx/chip_swimlane/test_chip_swimlane.py | 11 +++-- .../chip_swimlane/test_chip_swimlane_mixed.py | 14 +++--- .../test_sync_start_drain_phases.py | 9 +++- .../test_sync_start_early_local_owner.py | 3 ++ .../dfx/dep_gen/test_dep_gen.py | 11 +++-- .../dfx/dep_gen/test_dep_gen_chain.py | 8 ++-- .../dfx/pmu/test_pmu.py | 6 +-- .../dfx/scope_stats/test_scope_stats.py | 5 +- .../dummy_task/test_dummy_task.py | 1 + .../dynamic_register/test_dynamic_register.py | 4 ++ .../test_multi_round_paged_attention.py | 1 + .../test_predicated_dispatch.py | 2 + .../test_spmd_multiblock_aiv.py | 1 + .../spmd_starvation/test_spmd_starvation.py | 1 + .../test_spmd_sync_start_aiv.py | 1 + .../test_spmd_sync_start_stress.py | 1 + .../graph_execution/test_graph_execution.py | 1 + .../test_graph_execution_aic_aiv.py | 1 + .../test_prepared_callable.py | 1 + .../test_alternating_matmul_add.py | 1 + .../dfx/args_dump/test_args_dump.py | 1 + .../dfx/chip_swimlane/test_chip_swimlane.py | 17 +++---- .../chip_swimlane/test_chip_swimlane_mixed.py | 22 ++++----- .../test_sync_start_drain_phases.py | 9 +++- .../test_sync_start_early_local_owner.py | 3 ++ .../dfx/dep_gen/test_dep_gen.py | 11 +++-- .../dfx/dep_gen/test_dep_gen_chain.py | 8 ++-- .../dfx/pmu/test_pmu.py | 6 +-- .../dfx/scope_stats/test_scope_stats.py | 5 +- .../dummy_task/test_dummy_task.py | 1 + .../dynamic_register/test_dynamic_register.py | 3 ++ .../test_multi_round_paged_attention.py | 1 + .../test_paged_attention_unroll_4dims.py | 1 + .../test_spmd_multiblock_aiv.py | 1 + .../spmd_starvation/test_spmd_starvation.py | 1 + .../test_spmd_sync_start_aiv.py | 1 + .../test_spmd_sync_start_stress.py | 1 + .../test_host_build_graph_wide_dispatch.py | 1 + .../test_runtime_fatal_codes.py | 11 ++++- .../collectives/all_to_all/test_all_to_all.py | 1 + .../collectives/allgather/test_allgather.py | 1 + .../collectives/allreduce/test_allreduce.py | 1 + .../collectives/broadcast/test_broadcast.py | 1 + .../test_group_reservation.py | 1 + .../reduce_scatter/test_reduce_scatter.py | 1 + tests/ut/py/test_manual_selection.py | 28 ++++++++++- 67 files changed, 240 insertions(+), 112 deletions(-) diff --git a/.claude/skills/testing/SKILL.md b/.claude/skills/testing/SKILL.md index b8d982df2b..f6bf554f51 100644 --- a/.claude/skills/testing/SKILL.md +++ b/.claude/skills/testing/SKILL.md @@ -27,7 +27,7 @@ and will report failures that CI never sees: | Marker | Tests | CI behavior | | ------ | ----- | ----------- | -| `@pytest.mark.manual` / `CASES[*]["manual"]` | Standalone pytest tests / individual scene-test cases; optionally scoped to a platform list | Per-PR: excluded by default on the selected platforms; `daily.yml`: full sweep with `--manual include` | +| `@pytest.mark.manual` / `CASES[*]["manual"]` | Standalone pytest tests / individual scene-test cases; optionally scoped to a platform list | Per-PR main sweep: excluded by default on the selected platforms; dedicated DFX steps: included; `daily.yml`: full sweep with `--manual include` | | `@pytest.mark.sdma` | a2a3: `sdma_async_completion_demo`, `prefetch_async_demo`; a5: `sdma_async_completion_demo` | a2a3: the dedicated SDMA step; a5: included in the non-pod sweep | The a2a3 SDMA demos provision 48 device-only STARS streams, which makes an diff --git a/.github/workflows/_st-sim-a2a3.yml b/.github/workflows/_st-sim-a2a3.yml index a8eafdb3f7..2bffbf195f 100644 --- a/.github/workflows/_st-sim-a2a3.yml +++ b/.github/workflows/_st-sim-a2a3.yml @@ -111,34 +111,35 @@ jobs: - name: dep_gen smoke (a2a3) if: inputs.include_dfx_smokes run: | - .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py \ + .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/ \ --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-dep-gen + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen - name: dep_gen smoke (a2a3 host_build_graph) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py \ --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-dep-gen + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen - name: chip_swimlane smoke (a2a3) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/ \ --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-chip-swimlane --enable-dep-gen + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \ + --enable-chip-swimlane --enable-dep-gen - name: PMU smoke (a2a3) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py \ --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-pmu 2 + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-pmu 2 - name: args_dump smoke (a2a3) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py \ --platform a2a3sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --dump-args + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --dump-args diff --git a/.github/workflows/_st-sim-a5.yml b/.github/workflows/_st-sim-a5.yml index 8449149ce9..5a7c87c558 100644 --- a/.github/workflows/_st-sim-a5.yml +++ b/.github/workflows/_st-sim-a5.yml @@ -111,27 +111,28 @@ jobs: - name: dep_gen smoke (a5) if: inputs.include_dfx_smokes run: | - .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py \ + .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/ \ --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-dep-gen + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-dep-gen - name: chip_swimlane smoke (a5) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/ \ --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-chip-swimlane --enable-dep-gen + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" \ + --enable-chip-swimlane --enable-dep-gen - name: PMU smoke (a5) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py \ --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --enable-pmu 2 + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --enable-pmu 2 - name: args_dump smoke (a5) if: inputs.include_dfx_smokes run: | .venv/bin/python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py \ --platform a5sim --device 0-15 -p no:xdist --pto-session-timeout 600 \ - --require-pto-isa --dump-args + --require-pto-isa --manual "${{ inputs.manual_mode == 'only' && 'only' || 'include' }}" --dump-args diff --git a/docs/ci.md b/docs/ci.md index ed6ac9016f..78372a0ea3 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -106,9 +106,14 @@ scene-test corpus with `--manual include` once per day and supports manual re-runs through `workflow_dispatch`. Simulation runs on Ubuntu and macOS for both architectures; onboard runs on the A2/A3 and A5 self-hosted pools. The same DFX smoke steps used by Per-PR run once in each Daily platform job, and -the A2/A3 Pod corpus runs through the existing two-machine workflow. Per-PR -scene-test jobs keep the default `--manual exclude`, so moving a case to Daily -does not require a second workflow exclusion list. +the A2/A3 Pod corpus runs through the existing two-machine workflow. The main +Per-PR scene-test steps keep the default `--manual exclude`, so moving an +ordinary case to Daily does not require a second workflow exclusion list. +Dedicated DFX steps instead use `include` for the normal Per-PR and Daily modes +because they own the full corpus under their target paths; a `manual_mode` of +`only` remains `only` in those steps. Marking a DFX case manual therefore +removes its duplicate execution from the main step without removing its +dedicated Per-PR coverage. Use `"manual": True` on an individual `SceneTestCase.CASES` entry and `@pytest.mark.manual` on a standalone pytest test. The reusable scene-test @@ -117,8 +122,10 @@ Daily caller passes `include`. For platform-specific pruning, the same `manual` value accepts a platform list, for example `"manual": ["a2a3sim", "a5sim"]`; standalone tests use -`@pytest.mark.manual(["a2a3sim", "a5sim"])`. This removes only the Sim execution -from Per-PR while retaining onboard coverage. +`@pytest.mark.manual(["a2a3sim", "a5sim"])`. This removes only the listed Sim +executions from Per-PR. It retains same-architecture onboard coverage only when +the case also declares that onboard platform; otherwise that architecture's +path becomes Daily-only and must be called out in the PR's coverage analysis. ### Nightly sanitizer sweep diff --git a/docs/testing.md b/docs/testing.md index c5809b1680..612d0f131e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -731,16 +731,21 @@ still respect the manual filter — to run a manual case by name, pass The separate `daily.yml` workflow runs the full corpus with `--manual include` once per day on A2/A3 and A5, simulation and onboard. Mark a whole standalone pytest test with `@pytest.mark.manual`; mark only one case in a `SceneTestCase` -by setting `"manual": True` on that `CASES` entry. Per-PR excludes those tests, -while Daily runs them together with the regular corpus. The A2/A3 Pod cases run -in the same Daily workflow through their existing two-machine job. +by setting `"manual": True` on that `CASES` entry. The main Per-PR scene sweep +excludes those tests, while Daily runs them together with the regular corpus. +Dedicated DFX steps are the exception: they use `--manual include` in normal +Per-PR and Daily jobs, so marking a case under their target path removes only +its duplicate main-sweep execution. A caller selecting `--manual only` keeps +that mode in the DFX steps. The A2/A3 Pod cases run in the same Daily workflow +through their existing two-machine job. To move only selected platforms, pass the platform list to the same marker: use `@pytest.mark.manual(["a2a3sim", "a5sim"])` (or the equivalent `@pytest.mark.manual(platforms=["a2a3sim", "a5sim"])`) for a standalone test, or `"manual": ["a2a3sim", "a5sim"]` for a scene-test case. Do not mix the two -standalone marker forms. The onboard execution then remains in the default -Per-PR sweep. +standalone marker forms. An onboard execution remains in the default Per-PR +sweep only when that case also declares the corresponding onboard platform; +otherwise the selected architecture becomes Daily-only. ### Sharing an Example Between examples/ and tests/st/ diff --git a/examples/a2a3/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py b/examples/a2a3/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py index d4eb836865..d0d67f9638 100644 --- a/examples/a2a3/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py +++ b/examples/a2a3/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py @@ -54,6 +54,7 @@ class TestBenchmarkBgemmHostBuildGraph(SceneTestCase): { "name": "Case0", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {"matmul_add_task_num": 500, "incore_data_size": 128, "incore_loop": 4, "grid_k": 2}, }, ] diff --git a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py index eaf7ab4e5a..afc866288c 100644 --- a/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py +++ b/examples/a2a3/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py @@ -113,6 +113,7 @@ class TestPagedAttentionManualScope(SceneTestCase): { "name": "CaseSmall1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/examples/a5/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py b/examples/a5/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py index e99fae5cf0..03834c9aee 100644 --- a/examples/a5/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py +++ b/examples/a5/host_build_graph/benchmark_bgemm/test_benchmark_bgemm.py @@ -52,6 +52,7 @@ class TestBenchmarkBgemmHostBuildGraph(SceneTestCase): { "name": "Case0", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {"matmul_add_task_num": 500, "incore_data_size": 128, "incore_loop": 4, "grid_k": 2}, }, ] diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py b/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py index c096edf63b..382152d2e8 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_manual_scope/test_paged_attention_manual_scope.py @@ -66,6 +66,7 @@ class TestPagedAttentionManualScope(SceneTestCase): { "name": "SmallCase1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/test_paged_attention_unroll_manual_scope.py b/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/test_paged_attention_unroll_manual_scope.py index 11907a2453..fbb4a9aea0 100644 --- a/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/test_paged_attention_unroll_manual_scope.py +++ b/examples/a5/tensormap_and_ringbuffer/paged_attention_unroll_manual_scope/test_paged_attention_unroll_manual_scope.py @@ -110,6 +110,7 @@ class TestPagedAttentionUnrollManualScope(SceneTestCase): { "name": "SmallCase1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/examples/workers/l2/per_task_runtime_env/test_per_task_runtime_env.py b/examples/workers/l2/per_task_runtime_env/test_per_task_runtime_env.py index d827db0f6f..98ab0dde9c 100644 --- a/examples/workers/l2/per_task_runtime_env/test_per_task_runtime_env.py +++ b/examples/workers/l2/per_task_runtime_env/test_per_task_runtime_env.py @@ -14,6 +14,7 @@ @pytest.mark.platforms(["a2a3sim", "a2a3"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(1) def test_per_task_runtime_env(st_platform, st_device_ids): diff --git a/examples/workers/l3/allreduce/test_allreduce.py b/examples/workers/l3/allreduce/test_allreduce.py index 02172dd75b..2760982951 100644 --- a/examples/workers/l3/allreduce/test_allreduce.py +++ b/examples/workers/l3/allreduce/test_allreduce.py @@ -22,6 +22,7 @@ # declares all four platforms — so widening this is likely safe, but nobody has # exercised *this* demo there yet. @pytest.mark.platforms(["a2a3sim", "a2a3"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(2) def test_allreduce(st_platform, st_device_ids): diff --git a/examples/workers/l3/domain_rank_map/test_domain_rank_map.py b/examples/workers/l3/domain_rank_map/test_domain_rank_map.py index 41bf2ec1e8..bd2605bd93 100644 --- a/examples/workers/l3/domain_rank_map/test_domain_rank_map.py +++ b/examples/workers/l3/domain_rank_map/test_domain_rank_map.py @@ -14,6 +14,7 @@ @pytest.mark.platforms(["a2a3sim", "a2a3", "a5sim"]) +@pytest.mark.manual(["a2a3sim", "a5sim"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(3) def test_domain_rank_map(st_platform, st_device_ids, capsys): diff --git a/examples/workers/l3/multi_chip_dispatch/test_multi_chip_dispatch.py b/examples/workers/l3/multi_chip_dispatch/test_multi_chip_dispatch.py index a7bec22f0c..61e43e585f 100644 --- a/examples/workers/l3/multi_chip_dispatch/test_multi_chip_dispatch.py +++ b/examples/workers/l3/multi_chip_dispatch/test_multi_chip_dispatch.py @@ -14,6 +14,7 @@ @pytest.mark.platforms(["a2a3sim", "a2a3", "a5sim", "a5"]) +@pytest.mark.manual(["a2a3sim", "a5sim"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(2) def test_multi_chip_dispatch(st_platform, st_device_ids): diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index 2a2f7f9562..028494e582 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -1673,9 +1673,13 @@ def _effective_enable_dep_gen(request, *, warn: bool = False) -> bool: def test_run(self, st_platform, st_worker, request): """Auto test method — runs matching cases for the current platform.""" - raw_selectors = request.config.getoption("--case", default=None) or [] - selectors = [_parse_case_selector(v) for v in raw_selectors] + matched = self._matching_cases(st_platform, request) manual_mode = request.config.getoption("--manual", default="exclude") + if not matched: + import pytest # noqa: PLC0415 + + pytest.skip(f"No cases matched {type(self).__name__} (platform={st_platform}, manual={manual_mode})") + rounds = request.config.getoption("--rounds", default=1) skip_golden = request.config.getoption("--skip-golden", default=False) enable_chip_swimlane = request.config.getoption("--enable-chip-swimlane", default=0) @@ -1698,7 +1702,6 @@ def test_run(self, st_platform, st_worker, request): logger.warning("scope_stats disabled: --rounds > 1") enable_scope_stats = False - cls_name = type(self).__name__ callable_obj = self.build_callable(st_platform) sub_handles = getattr(type(self), "_st_sub_handles", {}) # For L3, use registered chip handles instead of raw ChipCallable @@ -1707,24 +1710,6 @@ def test_run(self, st_platform, st_worker, request): if self._st_level == 3 and chip_handles: callable_obj = {**chip_handles} - matched = [] - for case in self.CASES: - if st_platform not in case["platforms"]: - continue - if not _match_selectors(cls_name, case["name"], selectors): - continue - is_manual = is_manual_for_platform(case.get("manual"), st_platform) - if manual_mode == "exclude" and is_manual: - continue - if manual_mode == "only" and not is_manual: - continue - matched.append(case) - - if not matched: - import pytest # noqa: PLC0415 - - pytest.skip(f"No cases matched {cls_name} (platform={st_platform}, manual={manual_mode})") - run_class_cases( st_worker, self, @@ -1741,6 +1726,26 @@ def test_run(self, st_platform, st_worker, request): enable_swimlane_overhead=enable_swimlane_overhead, ) + def _matching_cases(self, st_platform, request): + """Return cases selected by the platform, case, and manual filters.""" + raw_selectors = request.config.getoption("--case", default=None) or [] + selectors = [_parse_case_selector(v) for v in raw_selectors] + manual_mode = request.config.getoption("--manual", default="exclude") + cls_name = type(self).__name__ + matched = [] + for case in self.CASES: + if st_platform not in case["platforms"]: + continue + if not _match_selectors(cls_name, case["name"], selectors): + continue + is_manual = is_manual_for_platform(case.get("manual"), st_platform) + if manual_mode == "exclude" and is_manual: + continue + if manual_mode == "only" and not is_manual: + continue + matched.append(case) + return matched + # ------------------------------------------------------------------ # Standalone entry point # ------------------------------------------------------------------ diff --git a/tests/st/a2a3/host_build_graph/available_aicore_counts/test_available_aicore_counts.py b/tests/st/a2a3/host_build_graph/available_aicore_counts/test_available_aicore_counts.py index abd217613b..6c901c0003 100644 --- a/tests/st/a2a3/host_build_graph/available_aicore_counts/test_available_aicore_counts.py +++ b/tests/st/a2a3/host_build_graph/available_aicore_counts/test_available_aicore_counts.py @@ -86,6 +86,7 @@ class TestAvailableAicoreCounts(SceneTestCase): { "name": "Default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] diff --git a/tests/st/a2a3/host_build_graph/batch_paged_attention/test_batch_paged_attention.py b/tests/st/a2a3/host_build_graph/batch_paged_attention/test_batch_paged_attention.py index 68ccd092e3..353088dbe0 100644 --- a/tests/st/a2a3/host_build_graph/batch_paged_attention/test_batch_paged_attention.py +++ b/tests/st/a2a3/host_build_graph/batch_paged_attention/test_batch_paged_attention.py @@ -100,6 +100,7 @@ class TestBatchPagedAttentionHostBuildGraph(SceneTestCase): { "name": "CaseSmall1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py b/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py index 3288370874..1af2cb3b81 100644 --- a/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a2a3/host_build_graph/dfx/dep_gen/test_dep_gen.py @@ -124,6 +124,7 @@ class TestDepGenHostBuildGraph(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] @@ -148,9 +149,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): """Assert deps.json holds the 6 edges of example_orchestration.cpp.""" @@ -236,6 +236,7 @@ class TestDepGenHostBuildGraphEdgeSources(SceneTestCase): { "name": "gate_open", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"case": 2}, }, @@ -260,9 +261,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): deps = _load_deps("TestDepGenHostBuildGraphEdgeSources", case["name"], run_marker) diff --git a/tests/st/a2a3/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py b/tests/st/a2a3/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py index 3e42943d1e..88d1462d54 100644 --- a/tests/st/a2a3/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py +++ b/tests/st/a2a3/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py @@ -52,6 +52,7 @@ class TestGraphExecutionAicAivHostBuildGraph(SceneTestCase): { "name": "record_then_replay_aic_aiv", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 4}, "params": {}, }, diff --git a/tests/st/a2a3/host_build_graph/prepared_callable/test_prepared_callable.py b/tests/st/a2a3/host_build_graph/prepared_callable/test_prepared_callable.py index f70ac746cb..4757b97cb5 100644 --- a/tests/st/a2a3/host_build_graph/prepared_callable/test_prepared_callable.py +++ b/tests/st/a2a3/host_build_graph/prepared_callable/test_prepared_callable.py @@ -90,6 +90,7 @@ class TestPreparedCallableHbg(SceneTestCase): { "name": "prepare_run_twice", "platforms": _PLATFORMS, + "manual": ["a2a3sim"], "params": {"a": 2.0, "b": 3.0}, }, ] diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py index 9a43f80a74..3f6bace352 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py @@ -87,6 +87,7 @@ class TestArgsDump(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 71b851902e..20631585c8 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -76,11 +76,13 @@ class TestChipSwimlane(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, { "name": "aicpu_threads_2", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {}, }, @@ -104,11 +106,10 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-chip-swimlane", default=0): return - for case in self.CASES: - if st_platform in case["platforms"]: - validate_perf_artifact( - f"TestChipSwimlane_{case['name']}", since=run_marker, expected_task_count=_EXPECTED_TASK_COUNT - ) + for case in self._matching_cases(st_platform, request): + validate_perf_artifact( + f"TestChipSwimlane_{case['name']}", since=run_marker, expected_task_count=_EXPECTED_TASK_COUNT + ) if __name__ == "__main__": diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py index c7fd4e1a52..2fe295b06d 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py @@ -88,6 +88,7 @@ class TestChipSwimlaneMixed(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] @@ -137,13 +138,12 @@ def test_run(self, st_platform, st_worker, request): run_marker = int(time.time()) # floor to whole seconds: safe if outputs/ ever lands on a coarse-mtime fs super().test_run(st_platform, st_worker, request) if request.config.getoption("--enable-chip-swimlane", default=False): - for case in self.CASES: - if st_platform in case["platforms"]: - # Rely on the differential gate (Pop / Fanout / Fanin) — - # the chain produces 3 MIX task_ids × 2 subtask rows = 6 - # perf rows and 2 deps.json edges, so the dedup branch in - # the oracle has an arithmetically observable effect. - validate_perf_artifact(f"TestChipSwimlaneMixed_{case['name']}", since=run_marker) + for case in self._matching_cases(st_platform, request): + # Rely on the differential gate (Pop / Fanout / Fanin) — + # the chain produces 3 MIX task_ids × 2 subtask rows = 6 + # perf rows and 2 deps.json edges, so the dedup branch in + # the oracle has an arithmetically observable effect. + validate_perf_artifact(f"TestChipSwimlaneMixed_{case['name']}", since=run_marker) # Full-dump modes give the func_id array its regression barrier on the # cooperative-mix path (single-kernel coverage lives in test_args_dump). if int(request.config.getoption("--dump-args", default=0)) >= 2: diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py index 0899c7620c..d3e50c7357 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py @@ -47,7 +47,14 @@ class TestSyncStartDrainPhases(SceneTestCase): ], } - CASES = [{"name": "global_aiv", "platforms": ["a2a3sim", "a2a3"], "params": {}}] + CASES = [ + { + "name": "global_aiv", + "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], + "params": {}, + } + ] def generate_args(self, params): return TaskArgsBuilder( diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py index 54ebbcd9d9..ca507c1122 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py @@ -82,6 +82,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "single_scheduler_idle", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], # One orchestrator plus one scheduler. That scheduler owns all AIV # cores, so the eight-block consumer fits its local idle capacity. "config": {"aicpu_thread_num": 2}, @@ -90,6 +91,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "three_schedulers_idle", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], # One orchestrator plus three schedulers. Whichever scheduler pops # the four-block cohort owns enough AIV cores to stage it locally, # so no non-owner scheduler should participate in a global drain. @@ -99,6 +101,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "single_scheduler_pending_only", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], # Every blocker reports started before an ACK-sweep fence. A second # scheduler-loop fence completes during their measured one-second # hold, proving the consumer used pending rather than idle slots. diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py index 600ec2da6a..1a1b81b13f 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py @@ -88,6 +88,7 @@ class TestDepGen(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] @@ -117,15 +118,15 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): """Assert deps.json for this invocation contains the 6 edges documented in example_orchestration.cpp. Reached only with dep_gen effectively on - and the case's platform matching, so the run must have produced an - output dir; a missing one is a capture regression, not a skip. + and the case selected by the active filters, so the run must have + produced an output dir; a missing one is a capture regression, not a + skip. """ case_name = case["name"] safe_label = _sanitize_for_filename(f"TestDepGen_{case_name}") diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py index b38acabbcc..90dfc4ac94 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py @@ -87,6 +87,7 @@ class TestDepGenChain(SceneTestCase): { "name": "n_64_no_chain", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 64}, }, @@ -99,12 +100,14 @@ class TestDepGenChain(SceneTestCase): { "name": "n_200_single_overflow", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 200}, }, { "name": "n_391_two_overflow", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 391}, }, @@ -136,9 +139,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): """Verify every explicit dep edge survived the writer → replay round-trip. diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py index 0996540669..4c23dcd4a9 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py @@ -69,6 +69,7 @@ class TestPmu(SceneTestCase): { "name": "default", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, }, ] @@ -91,9 +92,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-pmu", default=0): return - for case in self.CASES: - if st_platform in case["platforms"]: - self._validate_pmu_artifact(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._validate_pmu_artifact(case, run_marker) def _validate_pmu_artifact(self, case, run_marker): safe_label = _sanitize_for_filename(f"TestPmu_{case['name']}") diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py index 26281e8e27..28950a38c5 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py @@ -116,9 +116,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-scope-stats", default=False): return - for case in self.CASES: - if st_platform in case["platforms"]: - self._validate_scope_stats_artifact(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._validate_scope_stats_artifact(case, run_marker) def _validate_scope_stats_artifact(self, case, run_marker): safe_label = _sanitize_for_filename(f"TestScopeStats_{case['name']}") diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py index d7d3b6ba80..6b436de75e 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py @@ -101,6 +101,7 @@ class TestDummyTask(SceneTestCase): # Correctness is still just the copy. "name": "DenseFanoutFanin", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"case": 4}, }, diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py b/tests/st/a2a3/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py index 85952ae514..0698413c87 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py @@ -192,6 +192,7 @@ def orch_post(o, _args, _cfg): @pytest.mark.platforms(["a2a3sim"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.device_count(2) @pytest.mark.runtime(_RUNTIME) def test_prepare_new_identity_after_start_parallel_broadcast(st_platform, st_device_ids): @@ -247,6 +248,7 @@ def orch_post(o, _a, _c): @pytest.mark.platforms(["a2a3sim"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_prepare_capacity_overflow_post_start(st_platform, st_device_ids): @@ -297,6 +299,7 @@ def orch_pre(o, _a, _c): @pytest.mark.platforms(["a2a3sim"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_duplicate_prepare_same_hashid_survives_one_unregister(st_platform, st_device_ids): @@ -366,6 +369,7 @@ def orch_two(o, _args, _cfg): @pytest.mark.platforms(["a2a3sim"]) +@pytest.mark.manual(["a2a3sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_unregister_last_handle_allows_reprepare_same_hashid(st_platform, st_device_ids): diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py b/tests/st/a2a3/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py index be706b7105..52d8b989bb 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py @@ -69,6 +69,7 @@ class TestMultiRoundPagedAttention(SceneTestCase): { "name": "Case1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/predicated_dispatch/test_predicated_dispatch.py b/tests/st/a2a3/tensormap_and_ringbuffer/predicated_dispatch/test_predicated_dispatch.py index 64cfa512ea..43075379cc 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/predicated_dispatch/test_predicated_dispatch.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/predicated_dispatch/test_predicated_dispatch.py @@ -89,12 +89,14 @@ class TestPredicatedDispatch(SceneTestCase): { "name": "PredicateFalseSkips", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"case": 1}, }, { "name": "PredicateTrueDispatches", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "config": {"aicpu_thread_num": 2}, "params": {"case": 2}, }, diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py index 56cb57b832..a3c192f0bf 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py @@ -50,6 +50,7 @@ class TestSpmdMultiblockAiv(SceneTestCase): { "name": "Case1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, } ] diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py index fa1c37fb76..2a65333470 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py @@ -85,6 +85,7 @@ class TestSpmdStarvation(SceneTestCase): { "name": "Case1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, } ] diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py index 69f4758b6e..4242558201 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py @@ -54,6 +54,7 @@ class TestSpmdSyncStartAiv(SceneTestCase): { "name": "Case1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, } ] diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py index c9291e6dc9..7947a35d70 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py @@ -100,6 +100,7 @@ class TestSpmdSyncStartStress(SceneTestCase): { "name": "Case1", "platforms": ["a2a3sim", "a2a3"], + "manual": ["a2a3sim"], "params": {}, } ] diff --git a/tests/st/a5/host_build_graph/graph_execution/test_graph_execution.py b/tests/st/a5/host_build_graph/graph_execution/test_graph_execution.py index dbef5d2b07..7f93d4ef9a 100644 --- a/tests/st/a5/host_build_graph/graph_execution/test_graph_execution.py +++ b/tests/st/a5/host_build_graph/graph_execution/test_graph_execution.py @@ -52,6 +52,7 @@ class TestGraphExecutionHostBuildGraphA5(SceneTestCase): { "name": "record_then_replay_1d", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 4, "block_dim": 3}, "params": {"shape": (128 * 128,)}, }, diff --git a/tests/st/a5/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py b/tests/st/a5/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py index 979060cb95..cd1cdae58c 100644 --- a/tests/st/a5/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py +++ b/tests/st/a5/host_build_graph/graph_execution/test_graph_execution_aic_aiv.py @@ -46,6 +46,7 @@ class TestGraphExecutionAicAivHostBuildGraphA5(SceneTestCase): { "name": "record_then_replay_aic_aiv", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 4}, "params": {}, }, diff --git a/tests/st/a5/host_build_graph/prepared_callable/test_prepared_callable.py b/tests/st/a5/host_build_graph/prepared_callable/test_prepared_callable.py index 02e8a8b20a..1566e2b931 100644 --- a/tests/st/a5/host_build_graph/prepared_callable/test_prepared_callable.py +++ b/tests/st/a5/host_build_graph/prepared_callable/test_prepared_callable.py @@ -90,6 +90,7 @@ class TestPreparedCallableHbgA5(SceneTestCase): { "name": "prepare_run_twice", "platforms": _PLATFORMS, + "manual": ["a5sim"], "params": {"a": 2.0, "b": 3.0}, }, ] diff --git a/tests/st/a5/tensormap_and_ringbuffer/alternating_matmul_add/test_alternating_matmul_add.py b/tests/st/a5/tensormap_and_ringbuffer/alternating_matmul_add/test_alternating_matmul_add.py index e01224dba1..6eeea8c3f8 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/alternating_matmul_add/test_alternating_matmul_add.py +++ b/tests/st/a5/tensormap_and_ringbuffer/alternating_matmul_add/test_alternating_matmul_add.py @@ -54,6 +54,7 @@ class TestAlternatingMatmulAdd(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {"batch": 1, "M": 1, "N": 1, "matmul_batch": 1, "add_batch": 1}, }, { diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py index e3dee9215c..47d5b29fff 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py @@ -87,6 +87,7 @@ class TestArgsDump(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, }, ] diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py index 9ef545741e..5f3b366307 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane.py @@ -76,12 +76,14 @@ class TestChipSwimlane(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, "required_sched_phases": ("release",), }, { "name": "aicpu_threads_2", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 2}, "params": {}, "required_sched_phases": ("release",), @@ -106,14 +108,13 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-chip-swimlane", default=0): return - for case in self.CASES: - if st_platform in case["platforms"]: - validate_perf_artifact( - f"TestChipSwimlane_{case['name']}", - since=run_marker, - expected_task_count=_EXPECTED_TASK_COUNT, - required_sched_phases=case["required_sched_phases"], - ) + for case in self._matching_cases(st_platform, request): + validate_perf_artifact( + f"TestChipSwimlane_{case['name']}", + since=run_marker, + expected_task_count=_EXPECTED_TASK_COUNT, + required_sched_phases=case["required_sched_phases"], + ) if __name__ == "__main__": diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py index b1ab1846d0..0078476142 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_chip_swimlane_mixed.py @@ -88,6 +88,7 @@ class TestChipSwimlaneMixed(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, }, ] @@ -137,17 +138,16 @@ def test_run(self, st_platform, st_worker, request): run_marker = int(time.time()) # floor to whole seconds: safe if outputs/ ever lands on a coarse-mtime fs super().test_run(st_platform, st_worker, request) if request.config.getoption("--enable-chip-swimlane", default=False): - for case in self.CASES: - if st_platform in case["platforms"]: - # Rely on the differential gate (Pop / Fanout / Fanin) — - # the chain produces 3 MIX task_ids × 2 subtask rows = 6 - # perf rows and 2 deps.json edges, so the dedup branch in - # the oracle has an arithmetically observable effect. - validate_perf_artifact( - f"TestChipSwimlaneMixed_{case['name']}", - since=run_marker, - expected_complete_finishes=6, - ) + for case in self._matching_cases(st_platform, request): + # Rely on the differential gate (Pop / Fanout / Fanin) — + # the chain produces 3 MIX task_ids × 2 subtask rows = 6 + # perf rows and 2 deps.json edges, so the dedup branch in + # the oracle has an arithmetically observable effect. + validate_perf_artifact( + f"TestChipSwimlaneMixed_{case['name']}", + since=run_marker, + expected_complete_finishes=6, + ) # Full-dump modes give the func_id array its regression barrier on the # cooperative-mix path (single-kernel coverage lives in test_args_dump). if int(request.config.getoption("--dump-args", default=0)) >= 2: diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py index c4945e7a2b..605b064682 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_drain_phases.py @@ -46,7 +46,14 @@ class TestSyncStartDrainPhases(SceneTestCase): ], } - CASES = [{"name": "global_aiv", "platforms": ["a5sim", "a5"], "params": {}}] + CASES = [ + { + "name": "global_aiv", + "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], + "params": {}, + } + ] def generate_args(self, params): return TaskArgsBuilder( diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py index b9757714be..643900b3de 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/chip_swimlane/test_sync_start_early_local_owner.py @@ -77,6 +77,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "single_scheduler_idle", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], # One orchestrator plus one scheduler. The scheduler owns every AIV # core, so this eight-block cohort fits its local idle capacity. "config": {"aicpu_thread_num": 2}, @@ -85,6 +86,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "three_schedulers_idle", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], # One orchestrator plus three schedulers. The popping scheduler can # stage all four blocks without stopping either peer. "config": {"aicpu_thread_num": 4}, @@ -93,6 +95,7 @@ class TestSyncStartEarlyLocalOwner(SceneTestCase): { "name": "single_scheduler_pending_only", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], # Every blocker reports started before an ACK-sweep fence. A second # scheduler-loop fence completes during their measured one-second # hold, proving the consumer used pending rather than idle slots. diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py index 0f6e28fa98..f4054228a2 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py @@ -88,6 +88,7 @@ class TestDepGen(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, }, ] @@ -117,15 +118,15 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): """Assert deps.json for this invocation contains the 6 edges documented in example_orchestration.cpp. Reached only with dep_gen effectively on - and the case's platform matching, so the run must have produced an - output dir; a missing one is a capture regression, not a skip. + and the case selected by the active filters, so the run must have + produced an output dir; a missing one is a capture regression, not a + skip. """ case_name = case["name"] safe_label = _sanitize_for_filename(f"TestDepGen_{case_name}") diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py index f67559052c..fcce0a8f71 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen_chain.py @@ -83,6 +83,7 @@ class TestDepGenChain(SceneTestCase): { "name": "n_64_no_chain", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 64}, }, @@ -95,12 +96,14 @@ class TestDepGenChain(SceneTestCase): { "name": "n_200_single_overflow", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 200}, }, { "name": "n_391_two_overflow", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 2}, "params": {"n": 391}, }, @@ -132,9 +135,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not self._effective_enable_dep_gen(request): return - for case in self.CASES: - if st_platform in case.get("platforms", []): - self._post_validate(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._post_validate(case, run_marker) def _post_validate(self, case, run_marker): """Verify every explicit dep edge survived the writer → replay round-trip. diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py index 378d6dd3d6..9f1c3c0646 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py @@ -69,6 +69,7 @@ class TestPmu(SceneTestCase): { "name": "default", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, }, ] @@ -91,9 +92,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-pmu", default=0): return - for case in self.CASES: - if st_platform in case["platforms"]: - self._validate_pmu_artifact(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._validate_pmu_artifact(case, run_marker) def _validate_pmu_artifact(self, case, run_marker): safe_label = _sanitize_for_filename(f"TestPmu_{case['name']}") diff --git a/tests/st/a5/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py b/tests/st/a5/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py index 887062b6ec..6df05d9600 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dfx/scope_stats/test_scope_stats.py @@ -116,9 +116,8 @@ def test_run(self, st_platform, st_worker, request): super().test_run(st_platform, st_worker, request) if not request.config.getoption("--enable-scope-stats", default=False): return - for case in self.CASES: - if st_platform in case["platforms"]: - self._validate_scope_stats_artifact(case, run_marker) + for case in self._matching_cases(st_platform, request): + self._validate_scope_stats_artifact(case, run_marker) def _validate_scope_stats_artifact(self, case, run_marker): safe_label = _sanitize_for_filename(f"TestScopeStats_{case['name']}") diff --git a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py index cfd001fdb0..4bb5fe35bf 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dummy_task/test_dummy_task.py @@ -97,6 +97,7 @@ class TestDummyTask(SceneTestCase): # Correctness is still just the copy. "name": "DenseFanoutFanin", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "config": {"aicpu_thread_num": 2}, "params": {"case": 4}, }, diff --git a/tests/st/a5/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py b/tests/st/a5/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py index 11f58c3f5d..e0c025c39c 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py +++ b/tests/st/a5/tensormap_and_ringbuffer/dynamic_register/test_dynamic_register.py @@ -192,6 +192,7 @@ def orch_post(o, _args, _cfg): @pytest.mark.platforms(["a5sim"]) +@pytest.mark.manual(["a5sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_prepare_capacity_overflow_post_start(st_platform, st_device_ids): @@ -242,6 +243,7 @@ def orch_pre(o, _a, _c): @pytest.mark.platforms(["a5sim"]) +@pytest.mark.manual(["a5sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_duplicate_prepare_same_hashid_survives_one_unregister(st_platform, st_device_ids): @@ -311,6 +313,7 @@ def orch_two(o, _args, _cfg): @pytest.mark.platforms(["a5sim"]) +@pytest.mark.manual(["a5sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(_RUNTIME) def test_unregister_last_handle_allows_reprepare_same_hashid(st_platform, st_device_ids): diff --git a/tests/st/a5/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py b/tests/st/a5/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py index c93f87ab8b..6a25c4268c 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py +++ b/tests/st/a5/tensormap_and_ringbuffer/multi_round_paged_attention/test_multi_round_paged_attention.py @@ -69,6 +69,7 @@ class TestMultiRoundPagedAttention(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": { "batch": 1, "num_heads": 16, diff --git a/tests/st/a5/tensormap_and_ringbuffer/paged_attention_unroll_4dims/test_paged_attention_unroll_4dims.py b/tests/st/a5/tensormap_and_ringbuffer/paged_attention_unroll_4dims/test_paged_attention_unroll_4dims.py index 3f63fbedb4..75bc37edc8 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/paged_attention_unroll_4dims/test_paged_attention_unroll_4dims.py +++ b/tests/st/a5/tensormap_and_ringbuffer/paged_attention_unroll_4dims/test_paged_attention_unroll_4dims.py @@ -67,6 +67,7 @@ class TestPagedAttentionUnroll4dims(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": { "batch": 256, "num_heads": 16, diff --git a/tests/st/a5/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py b/tests/st/a5/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py index cd0e55bbd9..9d8e67e154 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py +++ b/tests/st/a5/tensormap_and_ringbuffer/spmd_multiblock_aiv/test_spmd_multiblock_aiv.py @@ -43,6 +43,7 @@ class TestSpmdMultiblockAiv(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, } ] diff --git a/tests/st/a5/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py b/tests/st/a5/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py index 062a56a4bd..ac21220fba 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py +++ b/tests/st/a5/tensormap_and_ringbuffer/spmd_starvation/test_spmd_starvation.py @@ -85,6 +85,7 @@ class TestSpmdStarvation(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, } ] diff --git a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py index 4a20886b22..15fcaae308 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py +++ b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_aiv/test_spmd_sync_start_aiv.py @@ -52,6 +52,7 @@ class TestSpmdSyncStartAiv(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, } ] diff --git a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py index 53295f5243..2444d1b357 100644 --- a/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py +++ b/tests/st/a5/tensormap_and_ringbuffer/spmd_sync_start_stress/test_spmd_sync_start_stress.py @@ -100,6 +100,7 @@ class TestSpmdSyncStartStress(SceneTestCase): { "name": "Case1", "platforms": ["a5sim", "a5"], + "manual": ["a5sim"], "params": {}, } ] diff --git a/tests/st/host_build_graph_wide_dispatch/test_host_build_graph_wide_dispatch.py b/tests/st/host_build_graph_wide_dispatch/test_host_build_graph_wide_dispatch.py index 031658c3bb..cab739effb 100644 --- a/tests/st/host_build_graph_wide_dispatch/test_host_build_graph_wide_dispatch.py +++ b/tests/st/host_build_graph_wide_dispatch/test_host_build_graph_wide_dispatch.py @@ -88,6 +88,7 @@ class TestHostBuildGraphWideDispatch(SceneTestCase): { "name": "TwoThreads", "platforms": ["a2a3sim", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"aicpu_thread_num": 2}, "params": {}, } diff --git a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py index 9fafc44963..e088f23e3d 100644 --- a/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py +++ b/tests/st/runtime_fatal_codes/test_runtime_fatal_codes.py @@ -218,6 +218,15 @@ # tensor_wait_timeout case. } +_PER_PR_SIM_CASES = {"async_completion_invalid", "explicit_fatal"} +assert _PER_PR_SIM_CASES <= CASES.keys(), f"unknown Per-PR Sim cases: {_PER_PR_SIM_CASES - CASES.keys()}" +_SIM_CASE_PARAMS = [ + case_name + if case_name in _PER_PR_SIM_CASES + else pytest.param(case_name, marks=pytest.mark.manual(["a2a3sim", "a5sim"]), id=case_name) + for case_name in CASES +] + def _assert_annotated(log: str, case: dict) -> None: """The failure line must be followed by what the code *means*, not just the number. @@ -288,7 +297,7 @@ def _make_worker(platform: str, device_id: int, case_name: str, monkeypatch): @pytest.mark.platforms(["a5sim", "a2a3sim"]) @pytest.mark.device_count(1) @pytest.mark.runtime(RUNTIME) -@pytest.mark.parametrize("case_name", list(CASES)) +@pytest.mark.parametrize("case_name", _SIM_CASE_PARAMS) def test_fatal_code_surfaces_on_sim(st_platform, st_device_ids, case_name, monkeypatch, capfd): """sim: the runtime status (``code -N``) reaches the host directly.""" configure_logging("error") diff --git a/tests/st/worker/collectives/all_to_all/test_all_to_all.py b/tests/st/worker/collectives/all_to_all/test_all_to_all.py index ed35b27a9b..b17cd8b309 100644 --- a/tests/st/worker/collectives/all_to_all/test_all_to_all.py +++ b/tests/st/worker/collectives/all_to_all/test_all_to_all.py @@ -92,6 +92,7 @@ class TestAllToAllP2(SceneTestCase): { "name": "p2", "platforms": ["a2a3sim", "a2a3", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 2}, "params": {"nranks": 2}, } diff --git a/tests/st/worker/collectives/allgather/test_allgather.py b/tests/st/worker/collectives/allgather/test_allgather.py index f37f03a15b..d68ad176ce 100644 --- a/tests/st/worker/collectives/allgather/test_allgather.py +++ b/tests/st/worker/collectives/allgather/test_allgather.py @@ -90,6 +90,7 @@ class TestAllgatherP2(SceneTestCase): { "name": "p2", "platforms": ["a2a3sim", "a2a3", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 2}, "params": {"nranks": 2}, } diff --git a/tests/st/worker/collectives/allreduce/test_allreduce.py b/tests/st/worker/collectives/allreduce/test_allreduce.py index 4ac8bd0055..093874c37f 100644 --- a/tests/st/worker/collectives/allreduce/test_allreduce.py +++ b/tests/st/worker/collectives/allreduce/test_allreduce.py @@ -141,6 +141,7 @@ class TestAllreduceRingP2(SceneTestCase): { "name": "ring", "platforms": ["a2a3sim", "a2a3", "a5sim", "a5"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 2}, "params": {"nranks": 2, "mode_id": 2}, } diff --git a/tests/st/worker/collectives/broadcast/test_broadcast.py b/tests/st/worker/collectives/broadcast/test_broadcast.py index 42150f0409..a045f03043 100644 --- a/tests/st/worker/collectives/broadcast/test_broadcast.py +++ b/tests/st/worker/collectives/broadcast/test_broadcast.py @@ -96,6 +96,7 @@ class TestBroadcastP2(SceneTestCase): { "name": "p2", "platforms": ["a2a3sim", "a2a3", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 2}, "params": {"nranks": 2}, } diff --git a/tests/st/worker/collectives/group_reservation/test_group_reservation.py b/tests/st/worker/collectives/group_reservation/test_group_reservation.py index fc4d642c5a..6bef37a268 100644 --- a/tests/st/worker/collectives/group_reservation/test_group_reservation.py +++ b/tests/st/worker/collectives/group_reservation/test_group_reservation.py @@ -119,6 +119,7 @@ class TestConsecutiveGroupReservation(SceneTestCase): { "name": "overlapping_targets", "platforms": ["a2a3sim", "a2a3", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 3}, "params": {}, } diff --git a/tests/st/worker/collectives/reduce_scatter/test_reduce_scatter.py b/tests/st/worker/collectives/reduce_scatter/test_reduce_scatter.py index 5ee9d8698a..a433afc19f 100644 --- a/tests/st/worker/collectives/reduce_scatter/test_reduce_scatter.py +++ b/tests/st/worker/collectives/reduce_scatter/test_reduce_scatter.py @@ -90,6 +90,7 @@ class TestReduceScatterP2(SceneTestCase): { "name": "p2", "platforms": ["a2a3sim", "a2a3", "a5sim"], + "manual": ["a2a3sim", "a5sim"], "config": {"device_count": 2}, "params": {"nranks": 2}, } diff --git a/tests/ut/py/test_manual_selection.py b/tests/ut/py/test_manual_selection.py index 2541885f8f..8ef0f72c02 100644 --- a/tests/ut/py/test_manual_selection.py +++ b/tests/ut/py/test_manual_selection.py @@ -11,10 +11,11 @@ import importlib.util from pathlib import Path +from types import SimpleNamespace import pytest -from simpler_setup.scene_test import is_manual_for_platform +from simpler_setup.scene_test import SceneTestCase, is_manual_for_platform _ROOT = Path(__file__).resolve().parents[3] _SPEC = importlib.util.spec_from_file_location("_root_conftest_for_manual_selection_tests", _ROOT / "conftest.py") @@ -67,6 +68,14 @@ def getoption(self, name, default=None): return self.options.get(name, self.options.get(name.lstrip("-"), default)) +class _SelectionScene(SceneTestCase): + CASES = [ + {"name": "ordinary", "platforms": ["a2a3sim", "a5sim"]}, + {"name": "sim_manual", "platforms": ["a2a3sim", "a5sim"], "manual": ["a2a3sim"]}, + {"name": "other_platform", "platforms": ["a5sim"]}, + ] + + @pytest.mark.parametrize( ("manual", "platform", "expected"), [ @@ -156,3 +165,20 @@ def test_collection_filters_platform_scoped_manual_tests(platform, manual_mode, root_conftest.pytest_collection_modifyitems(None, config, items) assert [item.nodeid for item in items] == expected + + +@pytest.mark.parametrize( + ("manual_mode", "selectors", "expected"), + [ + ("exclude", [], ["ordinary"]), + ("include", [], ["ordinary", "sim_manual"]), + ("only", [], ["sim_manual"]), + ("include", ["_SelectionScene::sim_manual"], ["sim_manual"]), + ], +) +def test_scene_case_selection_matches_run_filters(manual_mode, selectors, expected): + request = SimpleNamespace(config=_FakeConfig(manual=manual_mode, case=selectors)) + + matched = _SelectionScene()._matching_cases("a2a3sim", request) + + assert [case["name"] for case in matched] == expected