From 9bb13bfd04cce4454c76dc8f2ce9883e3e076283 Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:52:45 +0800 Subject: [PATCH 1/2] Update: unify pod selection on scene-test levels - Add SceneTestLevel and scene_level for shared level metadata - Switch pod wrappers, CI, and docs to use level 4 selection - Add unit coverage for level filtering and pod log gating --- .github/workflows/_st-npu-a2a3.yml | 10 +- .github/workflows/_st-npu-a5.yml | 4 +- .github/workflows/_st-pod.yml | 2 +- conftest.py | 116 ++++++++--- docs/ci.md | 20 +- docs/testing.md | 18 +- .../a2a3-507899-aicpu-shared-so-fault.md | 2 +- docs/user/reference/cli.md | 5 +- examples/README.md | 2 +- examples/workers/README.md | 10 +- .../test_compute_then_tload_mixed_l3.py | 4 +- .../test_global_tload_mixed_l3.py | 4 +- .../test_vector_add_mixed_l3.py | 4 +- simpler_setup/__init__.py | 13 +- simpler_setup/scene_test.py | 31 ++- .../l4_pod/test_global_tload_mixed_l3_pod.py | 3 +- tests/ut/py/test_scene_level_selection.py | 187 ++++++++++++++++++ 17 files changed, 367 insertions(+), 68 deletions(-) create mode 100644 tests/ut/py/test_scene_level_selection.py diff --git a/.github/workflows/_st-npu-a2a3.yml b/.github/workflows/_st-npu-a2a3.yml index 69f8eb1cfc..aad8821222 100644 --- a/.github/workflows/_st-npu-a2a3.yml +++ b/.github/workflows/_st-npu-a2a3.yml @@ -70,7 +70,7 @@ jobs: source /usr/local/Ascend/cann/set_env.sh source .venv/bin/activate python -m simpler_setup.tools.scene_test_compile examples tests/st \ - --platform a2a3 --require-pto-isa --compile-workers 8 -q + --platform a2a3 --exclude-level 4 --require-pto-isa --compile-workers 8 -q - name: Run pytest scene tests (a2a3) if: inputs.a2a3_sdma_mode == 'marker' @@ -78,10 +78,10 @@ jobs: source /usr/local/Ascend/cann/set_env.sh source .venv/bin/activate if [ "$(uname -m)" = "x86_64" ]; then - python -m pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200 + python -m pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200 else task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \ - --run "python -m pytest examples tests/st -m 'not sdma and not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200" + --run "python -m pytest examples tests/st -m 'not sdma' --platform a2a3 --exclude-level 4 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 1200" fi - name: Run pytest scene tests (a2a3 legacy SDMA paths) @@ -91,10 +91,10 @@ jobs: source .venv/bin/activate SDMA_IGNORE="--ignore=examples/a2a3/tensormap_and_ringbuffer/prefetch_async_demo --ignore=examples/a2a3/tensormap_and_ringbuffer/sdma_async_completion_demo" if [ "$(uname -m)" = "x86_64" ]; then - python -m pytest examples tests/st $SDMA_IGNORE -m "not pod" --platform a2a3 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600 + python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --exclude-level 4 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 600 else task-submit --timeout 1800 --max-time 1800 --device auto --device-num "$DEVICE_NUM" \ - --run "python -m pytest examples tests/st $SDMA_IGNORE -m 'not pod' --platform a2a3 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600" + --run "python -m pytest examples tests/st $SDMA_IGNORE --platform a2a3 --exclude-level 4 --device \$TASK_DEVICE -v --require-pto-isa --pto-session-timeout 600" fi - name: SDMA pytest (a2a3) diff --git a/.github/workflows/_st-npu-a5.yml b/.github/workflows/_st-npu-a5.yml index 400016965f..93ebccec03 100644 --- a/.github/workflows/_st-npu-a5.yml +++ b/.github/workflows/_st-npu-a5.yml @@ -57,10 +57,10 @@ jobs: run: | source .venv/bin/activate python -m simpler_setup.tools.scene_test_compile examples tests/st \ - --platform a5 --require-pto-isa --compile-workers 8 -q + --platform a5 --exclude-level 4 --require-pto-isa --compile-workers 8 -q DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))") task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" \ - --run "python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa -m 'not pod' --pto-session-timeout 1200" + --run "python -m pytest examples tests/st --platform a5 --exclude-level 4 --device ${DEVICE_RANGE} -v --require-pto-isa --pto-session-timeout 1200" - name: DFX smokes (a5) if: inputs.include_dfx_smokes diff --git a/.github/workflows/_st-pod.yml b/.github/workflows/_st-pod.yml index 6527bbd771..be62fc6466 100644 --- a/.github/workflows/_st-pod.yml +++ b/.github/workflows/_st-pod.yml @@ -181,7 +181,7 @@ jobs: with: pytest-args: >- examples tests/st - -m pod + --level 4 --platform ${{ inputs.platform }} --device ${{ env.POD_LOCAL_DEVICES }} --max-parallel 1 diff --git a/conftest.py b/conftest.py index 4995ba5332..a1450eb451 100644 --- a/conftest.py +++ b/conftest.py @@ -53,12 +53,13 @@ from simpler_setup import parallel_scheduler as _ps # noqa: E402 from simpler_setup.log_config import DEFAULT_LOG_LEVEL, configure_logging # noqa: E402 from simpler_setup.pto_isa import ensure_pto_isa_root # noqa: E402 -from simpler_setup.scene_test import clear_compile_cache # noqa: E402 +from simpler_setup.scene_test import SceneTestLevel, clear_compile_cache # noqa: E402 # Exit code used when the session watchdog fires. Matches the GNU `timeout` # convention so shell wrappers (e.g. CI) can distinguish timeout from other # failures. TIMEOUT_EXIT_CODE = 124 +_SCENE_LEVEL_CHOICES = [int(level) for level in SceneTestLevel] def _parse_device_range(s: str) -> list[int]: @@ -71,6 +72,25 @@ def _parse_device_range(s: str) -> list[int]: return _ps.device_range_to_list(s) +def _normalize_cli_scene_level(level: int | None) -> SceneTestLevel | None: + if level is None: + return None + return SceneTestLevel(level) + + +def _item_scene_level(item) -> SceneTestLevel | None: + cls = getattr(item, "cls", None) + if cls is not None: + level = getattr(cls, "_st_level", None) + if level is not None: + return SceneTestLevel(level) + function = getattr(item, "function", None) + level = getattr(function, "_st_level", None) + if level is not None: + return SceneTestLevel(level) + return None + + class DevicePool: """Device allocator for pytest fixtures. @@ -125,8 +145,16 @@ def pytest_addoption(parser): action="store", type=int, default=None, - choices=[2, 3], - help="Only run tests for this SceneTestCase level (2 or 3); default: all levels", + choices=_SCENE_LEVEL_CHOICES, + help="Only run tests for this scene-test level (2, 3, or 4); default: all levels", + ) + parser.addoption( + "--exclude-level", + action="store", + type=int, + default=None, + choices=_SCENE_LEVEL_CHOICES, + help="Exclude tests carrying this scene-test level (2, 3, or 4)", ) parser.addoption( "--max-parallel", @@ -432,12 +460,18 @@ def _configure_sanitizer(config): ) +def _validate_level_filters(config) -> None: + if config.getoption("--level", default=None) is not None and ( + config.getoption("--exclude-level", default=None) is not None + ): + raise pytest.UsageError("--level and --exclude-level cannot be used together") + + def pytest_configure(config): """Register custom markers and apply global config.""" config.addinivalue_line("markers", "platforms(list): supported platforms for standalone ST functions") config.addinivalue_line("markers", "requires_hardware: test needs Ascend toolchain and real device") config.addinivalue_line("markers", "device_count(n): number of NPU devices needed") - config.addinivalue_line("markers", "pod: test needs the pod runner and its peer machine") config.addinivalue_line( "markers", "pod_remote_device_count(n): number of remote NPU devices needed on the peer machine", @@ -463,6 +497,7 @@ def pytest_configure(config): "filtering so non-@scene_test tests only run under their matching runtime", ) + _validate_level_filters(config) _configure_sanitizer(config) # Configure logging unconditionally (not only when --log-level is passed) so @@ -527,7 +562,7 @@ def pytest_configure(config): def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 - """Filter ST tests by --platform / --runtime / --level; order L3 before L2. + """Filter ST tests by --platform / --runtime / level axes; order L3 before L2. Static filter mismatches (wrong level, wrong runtime, wrong platform) are **deselected** rather than marked ``pytest.skip`` so they don't @@ -543,7 +578,8 @@ def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 """ platform = config.getoption("--platform") runtime_filter = config.getoption("--runtime") - level_filter = config.getoption("--level") + level_filter = _normalize_cli_scene_level(config.getoption("--level")) + exclude_level_filter = _normalize_cli_scene_level(config.getoption("--exclude-level")) keep: list = [] deselected: list = [] @@ -557,13 +593,7 @@ def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 cls = getattr(item, "cls", None) - # Under --level, non-SceneTestCase items don't participate in - # level-based dispatch at all. Resource phase collects them - # separately in the parent; in a level-filtered child they're - # simply not this phase's concern. - if level_filter is not None and cls is None: - deselected.append(item) - continue + item_level = _item_scene_level(item) if cls is not None and hasattr(cls, "CASES") and isinstance(cls.CASES, list): # SceneTestCase class item. @@ -578,7 +608,10 @@ def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 if runtime_filter and getattr(cls, "_st_runtime", None) != runtime_filter: deselected.append(item) continue - if level_filter is not None and getattr(cls, "_st_level", None) != level_filter: + if level_filter is not None and item_level != level_filter: + deselected.append(item) + continue + if exclude_level_filter is not None and item_level == exclude_level_filter: deselected.append(item) continue keep.append(item) @@ -605,6 +638,13 @@ def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 deselected.append(item) continue + if level_filter is not None and item_level != level_filter: + deselected.append(item) + continue + if exclude_level_filter is not None and item_level == exclude_level_filter: + deselected.append(item) + continue + keep.append(item) if deselected: @@ -614,8 +654,7 @@ def pytest_collection_modifyitems(session, config, items): # noqa: PLR0912 # Sort: L3 tests first (they fork child processes that inherit main process CANN state, # so they must run before L2 tests pollute the CANN context). def sort_key(item): - cls = getattr(item, "cls", None) - level = getattr(cls, "_st_level", 0) if cls else 0 + level = _item_scene_level(item) or 0 # SDMA last, for the same class of reason L3 goes first: provisioning # the workspace leaves 48 STARS streams in the device's fault domain, # so every fault-injection case must have already run on a device that @@ -634,7 +673,7 @@ def sort_key(item): l3_items = [ i for i in items - if getattr(getattr(i, "cls", None), "_st_level", None) == 3 + if _item_scene_level(i) == SceneTestLevel.HOST and not any(m.name == "skip" for m in i.iter_markers()) ] if l3_items: @@ -760,10 +799,28 @@ def _collect_resource_jobs(items, platform): return jobs -def _base_pytest_argv(session): +def _strip_value_options(args, options): + stripped = [] + skip_next = False + for arg in args: + if skip_next: + skip_next = False + continue + text = str(arg) + if text in options: + skip_next = True + continue + if any(text.startswith(f"{option}=") for option in options): + continue + stripped.append(text) + return stripped + + +def _base_pytest_argv(session, *, strip_options=()): """Inherit the user's original pytest invocation args.""" base = [sys.executable, "-m", "pytest"] - for arg in session.config.invocation_params.args: + args = _strip_value_options(session.config.invocation_params.args, set(strip_options)) + for arg in args: base.append(str(arg)) return base @@ -849,7 +906,7 @@ def _dispatch_test_phases(session, resource_specs): # noqa: PLR0912 platform = cfg.getoption("--platform") max_parallel = _resolve_max_parallel(cfg, platform or "", device_ids) - base_args = _base_pytest_argv(session) + base_args = _base_pytest_argv(session, strip_options=("--exclude-level",)) cwd = session.config.invocation_params.dir # ----- Phase 1: Resource (L3 classes + standalone resource functions) ----- @@ -1025,16 +1082,19 @@ def _on_done(res): def pytest_runtestloop(session): """Dispatch Resource + L2 phases unless caller is already in child mode. - Child mode (both --runtime and --level set, or --collect-only) skips the - dispatcher and falls through to pytest's default runtestloop. + Child mode (runtime-filtered runs, L2/L3 level-filtered runs, or + --collect-only) skips the dispatcher and falls through to pytest's + default runtestloop. Level 4 pod selection is not child mode; it still + uses the Resource dispatcher. """ runtime_filter = session.config.getoption("--runtime") level_filter = session.config.getoption("--level") - # Child mode: if the caller filters by runtime or level, it wants direct - # control — don't re-enter the multi-phase dispatcher (which would cause - # nested dispatch, device pool exhaustion, and timeout). - if runtime_filter is not None or level_filter is not None: + # Child mode: if the caller filters by runtime, or by the SceneTestCase + # levels the dispatcher itself uses for children, it wants direct control. + if runtime_filter is not None: + return + if _normalize_cli_scene_level(level_filter) in (SceneTestLevel.CHIP, SceneTestLevel.HOST): return # User explicitly asked for collect-only / scoped-run — don't orchestrate. @@ -1299,8 +1359,8 @@ def st_pod_remote_device_ids(request, st_pod_peer): @pytest.fixture() def st_pod_logs(request, monkeypatch): """Per-test parent log directory for pod scene tests.""" - if request.node.get_closest_marker("pod") is None: - pytest.fail("st_pod_logs requires @pytest.mark.pod") + if _item_scene_level(request.node) != SceneTestLevel.POD: + pytest.fail("st_pod_logs requires SceneTestLevel.POD") run_dir = os.environ.get("RUN_DIR") if not run_dir: if not os.environ.get("POD_REMOTE_ENDPOINT") or not os.environ.get("POD_REMOTE_DEVICES"): diff --git a/docs/ci.md b/docs/ci.md index 4edf5a8786..b5d4df7880 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -58,10 +58,10 @@ PullRequest | `st-sim-a2a3` | `ubuntu-latest`, `macos-latest` | `pytest examples tests/st --platform a2a3sim` | | `st-sim-a5` | `ubuntu-latest`, `macos-latest` | `pytest examples tests/st --platform a5sim` | | `ut-a2a3` | a2a3 self-hosted | `pytest tests/ut --platform a2a3` + `ctest -L "^requires_hardware(_a2a3)?$" --resource-spec-file ...` + build `tools/cann-examples/query` and run `query version` (no device) + build `tools/cann-examples/aicpu-device-query` and `tools/cann-examples/aicpu-kernel-launch` (host + cross-compiled device SO, link smoke only) | -| `st-onboard-a2a3` | a2a3 self-hosted | `pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device ...`, then a separate `-m sdma` step, then adaptive-parallel DFX feature smokes | +| `st-onboard-a2a3` | a2a3 self-hosted | `pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device ...`, then a separate `-m sdma` step, then adaptive-parallel DFX feature smokes | | `ut-a5` | a5 self-hosted | `pytest tests/ut --platform a5` + `ctest -L "^requires_hardware(_a5)?$"` + build `tools/cann-examples/query` and run `query version` (no device) + build `tools/cann-examples/aicpu-device-query` and `tools/cann-examples/aicpu-kernel-launch` (link smoke only) | -| `st-onboard-a5` | a5 self-hosted | `pytest examples tests/st -m "not pod" --platform a5 --device ...`, including SDMA tests, then adaptive-parallel DFX feature smokes | -| `st-pod-onboard-a2a3` | a pair of `a2a3pod` machines | `pytest examples tests/st -m pod --platform a2a3 --device ... --max-parallel 1`, one L3 daemon on the peer | +| `st-onboard-a5` | a5 self-hosted | `pytest examples tests/st --platform a5 --exclude-level 4 --device ...`, including SDMA tests, then adaptive-parallel DFX feature smokes | +| `st-pod-onboard-a2a3` | a pair of `a2a3pod` machines | `pytest examples tests/st --level 4 --platform a2a3 --device ... --max-parallel 1`, one L3 daemon on the peer | ### Multi-machine pod jobs @@ -78,13 +78,15 @@ Its body splits by what is per-run and what is per-pytest session: | Action | Called | What it does | | ------ | ------ | ------------ | | `pod-stage` | once | rsync this run's tree onto the peer and build it there | -| `pod-run-pytest` | once | start the peer's L3 daemon, set the pod pytest environment, run `pytest examples tests/st -m pod`, stop the daemon and pull its logs | +| `pod-run-pytest` | once | start the peer's L3 daemon, set the pod pytest environment, run `pytest examples tests/st --level 4`, stop the daemon and pull its logs | | `pod-teardown` | once, `if: always()` | remove the run's tree from the peer | Staging and the peer-side build are the job's whole cost, and every pod test runs against that same tree and venv, so they happen once. The pytest command -owns selection: adding an L4 pod example means adding a `test_*.py` wrapper with -`@pytest.mark.pod`, not editing `_st-pod.yml`. +owns selection through the scene-test level axis: adding an L4 pod example +means adding a `test_*.py` wrapper with `@scene_level(SceneTestLevel.POD)`, not +editing `_st-pod.yml`. `pod_remote_device_count` stays as the peer-resource +declaration; `pod` is the runner topology, not a pytest selection marker. Pod logs go to `output/pod-ci--/pytest/` and the whole directory is uploaded as one artifact. Parent-side `ASCEND_PROCESS_LOG_PATH` is split per @@ -132,11 +134,11 @@ benefit — device bin-packing for L3, xdist fanout for L2, and a shared ```bash # Recommended CI invocation — a2a3 deselects SDMA and pod tests, as the job does, # and runs SDMA as a second pass afterwards -pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 -x +pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device 4-7 -x pytest examples tests/st -m sdma --platform a2a3 --device 4-5 -x # A5 runners run the non-pod corpus, including SDMA tests -pytest examples tests/st -m "not pod" --platform a5 --device 0-7 -x +pytest examples tests/st --platform a5 --exclude-level 4 --device 0-7 -x ``` `-x` (`--exitfirst`) is appropriate for CI, where aborting on first @@ -188,7 +190,7 @@ not need `--max-parallel` manually. The arch flags subtract `NON_CODE` before deciding, so a non-code-only change already makes both `false`. An arch-gated job therefore needs no separate non-code check. See [`.claude/rules/ci-change-detection.md`](../.claude/rules/ci-change-detection.md) for the invariants these gates must keep. -- **SDMA tests run as their own step inside `st-onboard-a2a3`.** The ordinary sweep deselects them with `-m "not sdma and not pod"` and a later step runs `-m sdma`. Ordering is what the two SDMA paths share: the SDMA step is always second, so no fault-injection case can land on a device that has already provisioned SDMA. Device acquisition differs by host arch — on aarch64 the SDMA step takes its own `task-submit --device auto --device-num 2`, so the two steps are disjoint in devices as well; on x86_64 there is no `task-submit` and both steps use the same `${DEVICE_RANGE}`, leaving ordering as the only separation. Provisioning the SDMA workspace creates device-only STARS streams that live in the device fault domain, so an AICore fault on a device that has provisioned SDMA costs minutes instead of milliseconds — the sweep's `aicore_op_timeout` fault injection must therefore never share a device with them ([#1425](https://github.com/hw-native-sys/simpler/issues/1425)). Selection is by marker on both sides, so the two cannot drift apart; the split can be dropped once #1425 is fixed. Pod tests are selected by `-m pod` in `st-pod-onboard-a2a3` and explicitly excluded from ordinary onboard ST lanes. +- **SDMA tests run as their own step inside `st-onboard-a2a3`.** The ordinary sweep deselects them with `-m "not sdma"` and `--exclude-level 4`, and a later step runs `-m sdma`. Ordering is what the two SDMA paths share: the SDMA step is always second, so no fault-injection case can land on a device that has already provisioned SDMA. Device acquisition differs by host arch — on aarch64 the SDMA step takes its own `task-submit --device auto --device-num 2`, so the two steps are disjoint in devices as well; on x86_64 there is no `task-submit` and both steps use the same `${DEVICE_RANGE}`, leaving ordering as the only separation. Provisioning the SDMA workspace creates device-only STARS streams that live in the device fault domain, so an AICore fault on a device that has provisioned SDMA costs minutes instead of milliseconds — the sweep's `aicore_op_timeout` fault injection must therefore never share a device with them ([#1425](https://github.com/hw-native-sys/simpler/issues/1425)). Selection for SDMA remains by marker on both sides, so the two cannot drift apart; the split can be dropped once #1425 is fixed. Pod tests are selected by `--level 4` in `st-pod-onboard-a2a3` and explicitly excluded from ordinary onboard ST lanes. ### CPU emergency lane (`ci-self-cpu.yml`) and the `/run-cpu` button diff --git a/docs/testing.md b/docs/testing.md index 77ca2c76ca..d3b4340b01 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,12 +46,12 @@ ctest --test-dir tests/ut/cpp/build -L "^requires_hardware(_a2a3)?$" --output-on # Scene tests (pytest, @scene_test classes) pytest examples tests/st # all sim platforms (auto-parametrized) pytest examples tests/st --platform a2a3sim # specific sim -pytest examples tests/st -m "not sdma and not pod" --platform a2a3 # hardware -pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 # hardware with device pool +pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 # hardware +pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device 4-7 # hardware with device pool # Compile the selected hardware batch without creating a Worker or using an NPU python -m simpler_setup.tools.scene_test_compile examples tests/st \ - -m "not sdma and not pod" --platform a2a3 --require-pto-isa --compile-workers 8 + -m "not sdma" --platform a2a3 --exclude-level 4 --require-pto-isa --compile-workers 8 # SDMA cases run separately, as they do in CI: they are quarantined by # @pytest.mark.sdma so no fault-injection case shares a device with a @@ -59,7 +59,7 @@ python -m simpler_setup.tools.scene_test_compile examples tests/st \ pytest examples tests/st -m sdma --platform a2a3 --device 4-5 # A5 runs the non-pod corpus, including SDMA tests, on both host architectures -pytest examples tests/st -m "not pod" --platform a5 --device 0-7 +pytest examples tests/st --platform a5 --exclude-level 4 --device 0-7 # Single scene test (standalone) python examples/a2a3/tensormap_and_ringbuffer/vector_example/test_vector_example.py -p a2a3sim @@ -111,8 +111,9 @@ A class that fails to compile is reported and skipped rather than aborting the pass, so the run that follows still recompiles it and reports the error against the case that owns it. -Pass the same paths, `-m`, `--platform`, `--runtime`, and `--level` selections -to warm-up and execution. A normal pytest or standalone run loads matching +Pass the same paths, `-m`, `--platform`, `--runtime`, `--level`, and +`--exclude-level` selections to warm-up and execution. A normal pytest or +standalone run loads matching artifacts from the persistent cache; changed orchestration, incore, or included source content, compiler identity, fixed compilation flags, compilation logic, or compilation schema produces a new cache entry automatically. Entries unused @@ -153,7 +154,8 @@ python test_xxx.py -p a2a3sim --log-level debug # verbose C++ l | `--device IDS` | `-d` | `0` | Single id (`0`), range (`0-7`), or list (`0,2,5`). Sets the device-id pool for L3 cases and the available slots for L2 fanout. | | `--max-parallel N` | | `auto` | Max in-flight subprocesses (make-style). `auto` = `min(nproc, len(--device))` on sim, `len(--device)` on hardware. Decouples device-id pool size from parallelism; use to throttle sim on a CPU-constrained runner. | | `--runtime NAME` | | (all) | Restrict to one runtime (also used internally as the child-mode marker) | -| `--level {2,3}` | | (all) | Restrict to one SceneTestCase level (also the child-mode marker) | +| `--level {2,3,4}` | | (all) | Restrict to one scene-test level. Level 4 selects pod wrappers. | +| `--exclude-level {2,3,4}` | | (none) | Exclude tests explicitly carrying that scene-test level. Ordinary onboard lanes use `--exclude-level 4`. | | `--case SEL` | | (all) | Case selector, repeatable: `Foo`, `ClassA::Foo`, `ClassA::` | | `--manual` | | `exclude` | `exclude`/`include`/`only` for manual cases | | `--skip-golden` | | false | Skip golden comparison (for benchmarking) | @@ -664,7 +666,7 @@ pytest examples tests/st --platform a2a3sim python test_my_kernel.py -p a2a3sim # On hardware (SDMA cases quarantined by marker; run them with -m sdma) -pytest examples tests/st -m "not sdma and not pod" --platform a2a3 +pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 ``` Key fields: diff --git a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md index c143d7017e..ebad23fa3f 100644 --- a/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md +++ b/docs/troubleshooting/a2a3-507899-aicpu-shared-so-fault.md @@ -63,7 +63,7 @@ exception you must surface the CANN device slog, which is otherwise hidden: ```bash ASCEND_SLOG_PRINT_TO_STDOUT=1 ASCEND_GLOBAL_LOG_LEVEL=1 \ - python -m pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device -v \ + python -m pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device -v \ --pto-session-timeout 600 ``` diff --git a/docs/user/reference/cli.md b/docs/user/reference/cli.md index 1460f6937a..59feab4a09 100644 --- a/docs/user/reference/cli.md +++ b/docs/user/reference/cli.md @@ -6,7 +6,7 @@ How you select what runs, turn diagnostics on, and read the artifacts back. ```bash pytest examples tests/st --platform a2a3sim # simulation, no device -pytest examples tests/st -m "not sdma and not pod" --platform a2a3 --device 4-7 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) +pytest examples tests/st -m "not sdma" --platform a2a3 --exclude-level 4 --device 4-7 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) python examples/my_example/test_my_example.py -p a2a3sim # standalone, no pytest ``` @@ -17,7 +17,8 @@ python examples/my_example/test_my_example.py -p a2a3sim # standalone, no pyte | `--platform` | Target platform: `a2a3sim`, `a2a3`, `a5sim`, `a5`. Required | | `--device` | Device id or range, e.g. `0` or `4-7`. Default `0` | | `--runtime` | Restrict to one runtime (`tensormap_and_ringbuffer`, `host_build_graph`) | -| `--level` | Restrict to one level, e.g. `--level 2` | +| `--level` | Restrict to one level, e.g. `--level 2`; pod wrappers use `--level 4` | +| `--exclude-level` | Exclude tests explicitly carrying one level, e.g. ordinary onboard lanes use `--exclude-level 4` | | `--case` | Case selector, repeatable: `Foo`, `ClassA::Foo`, or `ClassA::` for a whole class | | `--manual` | Manual-case handling: `exclude` (default), `include`, `only` | | `--rounds` | Run each case N times. Default `1`; use this so first-run effects do not dominate a measurement | diff --git a/examples/README.md b/examples/README.md index 842d083c62..a3abc1e189 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,7 +17,7 @@ on both simulators: ```bash pytest examples --platform a2a3sim -pytest examples -m "not sdma and not pod" --platform a2a3 --device 0-1 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) +pytest examples -m "not sdma" --platform a2a3 --exclude-level 4 --device 0-1 # hardware (SDMA/pod quarantined; pod runs in the two-machine job) ``` A single example: diff --git a/examples/workers/README.md b/examples/workers/README.md index f811dae79c..1509655208 100644 --- a/examples/workers/README.md +++ b/examples/workers/README.md @@ -68,7 +68,7 @@ l4// kernels/aiv/*.cpp kernels/orchestration/*.cpp main.py # entry point: argparse + main() delegating to run() - test_.py # @pytest.mark.pod wrapper collected by pod CI + test_.py # @scene_level(SceneTestLevel.POD) wrapper collected by pod CI run_parent.sh # maps environment variables onto main.py's flags ``` @@ -105,9 +105,11 @@ Anything else — platform, runtime — defaults inside `run_parent.sh`. ### Running it in CI The `st-pod-onboard-a2a3` job runs L4 examples across a pair of a2a3 machines. -The job runs one `pytest examples tests/st -m pod` sweep, so adding yours means -adding a `test_*.py` wrapper carrying `@pytest.mark.pod`. Do not edit -`_st-pod.yml`. The wiring and the log artifact are described in +The job runs one `pytest examples tests/st --level 4` sweep, so adding yours +means adding a `test_*.py` wrapper carrying +`@scene_level(SceneTestLevel.POD)`. Keep `pod_remote_device_count` when the peer +side needs more than one remote device; it declares remote resource demand, +not selection. Do not edit `_st-pod.yml`. The wiring and the log artifact are described in [`docs/ci.md`](../../docs/ci.md#multi-machine-pod-jobs). ## Prerequisites diff --git a/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py b/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py index 91bc43f3f6..5b9089e611 100644 --- a/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py +++ b/examples/workers/l4/compute_then_tload_mixed_l3/test_compute_then_tload_mixed_l3.py @@ -10,6 +10,8 @@ import pytest +from simpler_setup import SceneTestLevel, scene_level + from .main import run @@ -17,7 +19,7 @@ def _device_spec(device_ids) -> str: return ",".join(str(device_id) for device_id in device_ids) -@pytest.mark.pod +@scene_level(SceneTestLevel.POD) @pytest.mark.platforms(["a2a3"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(1) diff --git a/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py b/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py index dbcf6fada3..89ffd896ac 100644 --- a/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py +++ b/examples/workers/l4/global_tload_mixed_l3/test_global_tload_mixed_l3.py @@ -10,6 +10,8 @@ import pytest +from simpler_setup import SceneTestLevel, scene_level + from .main import run @@ -17,7 +19,7 @@ def _device_spec(device_ids) -> str: return ",".join(str(device_id) for device_id in device_ids) -@pytest.mark.pod +@scene_level(SceneTestLevel.POD) @pytest.mark.platforms(["a2a3"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(1) diff --git a/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py b/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py index a41aa2c150..cb55ab4b3f 100644 --- a/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py +++ b/examples/workers/l4/vector_add_mixed_l3/test_vector_add_mixed_l3.py @@ -10,6 +10,8 @@ import pytest +from simpler_setup import SceneTestLevel, scene_level + from .main import run @@ -17,7 +19,7 @@ def _device_spec(device_ids) -> str: return ",".join(str(device_id) for device_id in device_ids) -@pytest.mark.pod +@scene_level(SceneTestLevel.POD) @pytest.mark.platforms(["a2a3"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(2) diff --git a/simpler_setup/__init__.py b/simpler_setup/__init__.py index 4cd0b8bd8c..b6792c60f0 100644 --- a/simpler_setup/__init__.py +++ b/simpler_setup/__init__.py @@ -13,7 +13,16 @@ from .platform_info import parse_platform from .pto_isa import ensure_pto_isa_root from .runtime_builder import RuntimeBuilder -from .scene_test import CallableNamespace, Scalar, SceneTestCase, TaskArgsBuilder, TensorArg, scene_test +from .scene_test import ( + CallableNamespace, + Scalar, + SceneTestCase, + SceneTestLevel, + TaskArgsBuilder, + TensorArg, + scene_level, + scene_test, +) from .torch_interop import make_chip_tensor_arg, torch_dtype_to_datatype __all__ = [ @@ -22,12 +31,14 @@ "RuntimeBuilder", "Scalar", "SceneTestCase", + "SceneTestLevel", "TensorArg", "TaskArgsBuilder", "ensure_pto_isa_root", "extract_text_section", "make_chip_tensor_arg", "parse_platform", + "scene_level", "scene_test", "torch_dtype_to_datatype", ] diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index d7bc492519..7553880d9b 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -29,6 +29,7 @@ import platform as host_platform import sys from contextlib import contextmanager +from enum import IntEnum from pathlib import Path from typing import Any, NamedTuple @@ -141,6 +142,31 @@ def _golden_thread_cap(): # --------------------------------------------------------------------------- +class SceneTestLevel(IntEnum): + CHIP = 2 + HOST = 3 + POD = 4 + + +def _normalize_scene_level(level: int | SceneTestLevel) -> SceneTestLevel: + try: + return SceneTestLevel(level) + except ValueError as e: + supported = ", ".join(str(int(member)) for member in SceneTestLevel) + raise ValueError(f"Unsupported scene test level {level!r}; expected one of: {supported}") from e + + +def scene_level(level: int | SceneTestLevel): + """Decorator attaching scene-test level metadata to a class or function.""" + normalized = _normalize_scene_level(level) + + def decorator(obj): + obj._st_level = normalized + return obj + + return decorator + + class TensorArg(NamedTuple): """Named torch.Tensor argument spec.""" @@ -1203,14 +1229,15 @@ def compile_callable(): # --------------------------------------------------------------------------- -def scene_test(level: int, runtime: str): +def scene_test(level: int | SceneTestLevel, runtime: str): """Decorator marking a SceneTestCase with level and runtime. Platforms are declared per-case in CASES, not here. """ + level_decorator = scene_level(level) def decorator(cls): - cls._st_level = level + level_decorator(cls) cls._st_runtime = runtime cls_dir = Path(inspect.getfile(cls)).parent if hasattr(cls, "CALLABLE"): diff --git a/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py b/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py index 9911a78fbd..9d66553415 100644 --- a/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py +++ b/tests/st/a2a3/tensormap_and_ringbuffer/l4_pod/test_global_tload_mixed_l3_pod.py @@ -15,13 +15,14 @@ import pytest from examples.workers.l4.global_tload_mixed_l3.main import run +from simpler_setup import SceneTestLevel, scene_level def _device_spec(device_ids) -> str: return ",".join(str(device_id) for device_id in device_ids) -@pytest.mark.pod +@scene_level(SceneTestLevel.POD) @pytest.mark.platforms(["a2a3"]) @pytest.mark.runtime("tensormap_and_ringbuffer") @pytest.mark.device_count(1) diff --git a/tests/ut/py/test_scene_level_selection.py b/tests/ut/py/test_scene_level_selection.py new file mode 100644 index 0000000000..6ceb4f8521 --- /dev/null +++ b/tests/ut/py/test_scene_level_selection.py @@ -0,0 +1,187 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +import pytest +from _pytest.outcomes import Failed + +from simpler_setup import SceneTestLevel, scene_level, scene_test + +_ROOT = Path(__file__).resolve().parents[3] +_SPEC = importlib.util.spec_from_file_location("_root_conftest_for_scene_level_tests", _ROOT / "conftest.py") +assert _SPEC is not None and _SPEC.loader is not None +root_conftest = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(root_conftest) + + +class _FakeMarker: + def __init__(self, name, *args): + self.name = name + self.args = args + + +class _FakeItem: + def __init__(self, nodeid, *, cls=None, function=None, markers=()): + self.nodeid = nodeid + self.cls = cls + self.function = function + self._markers = list(markers) + + def iter_markers(self, name=None): + return (marker for marker in self._markers if name is None or marker.name == name) + + def get_closest_marker(self, name): + for marker in self._markers: + if marker.name == name: + return marker + return None + + def add_marker(self, marker): + self._markers.append(marker) + + +class _FakeHook: + def __init__(self): + self.deselected = [] + + def pytest_deselected(self, items): + self.deselected.extend(items) + + +class _FakeConfig: + def __init__(self, **options): + self.options = options + self.hook = _FakeHook() + + def getoption(self, name, default=None): + return self.options.get(name, self.options.get(name.lstrip("-"), default)) + + +def test_scene_level_normalizes_int_and_enum(): + @scene_level(4) + def pod_fn(): + return None + + @scene_level(SceneTestLevel.CHIP) + def chip_fn(): + return None + + assert pod_fn._st_level is SceneTestLevel.POD + assert chip_fn._st_level is SceneTestLevel.CHIP + + +def test_scene_test_accepts_int_levels(): + @scene_test(level=2, runtime="tensormap_and_ringbuffer") + class L2: + CALLABLE = {} + CASES = [] + + assert L2._st_level is SceneTestLevel.CHIP + assert L2._st_runtime == "tensormap_and_ringbuffer" + + +def test_invalid_scene_level_rejected(): + with pytest.raises(ValueError): + scene_level(5) + + with pytest.raises(ValueError): + scene_test(level=5, runtime="tensormap_and_ringbuffer") + + +def test_level4_filter_keeps_only_explicit_level4_functions(): + def plain_fn(): + return None + + @scene_level(SceneTestLevel.POD) + def pod_fn(): + return None + + items = [ + _FakeItem("tests::plain", function=plain_fn), + _FakeItem("tests::pod", function=pod_fn), + ] + config = _FakeConfig( + platform="a2a3", + level=4, + **{"exclude-level": None, "runtime": None, "enable-chip-swimlane": 0}, + ) + + root_conftest.pytest_collection_modifyitems(None, config, items) + + assert [item.nodeid for item in items] == ["tests::pod"] + + +def test_exclude_level4_keeps_unlevelled_functions(): + def plain_fn(): + return None + + @scene_level(SceneTestLevel.POD) + def pod_fn(): + return None + + items = [ + _FakeItem("tests::plain", function=plain_fn), + _FakeItem("tests::pod", function=pod_fn), + ] + config = _FakeConfig( + platform="a2a3", + **{"level": None, "exclude-level": 4, "runtime": None, "enable-chip-swimlane": 0}, + ) + + root_conftest.pytest_collection_modifyitems(None, config, items) + + assert [item.nodeid for item in items] == ["tests::plain"] + + +def test_sorting_uses_function_level_metadata(): + @scene_level(SceneTestLevel.HOST) + def host_fn(): + return None + + class L2: + _st_level = SceneTestLevel.CHIP + _st_runtime = "tensormap_and_ringbuffer" + CASES = [{"platforms": ["a2a3"]}] + + items = [ + _FakeItem("tests::l2", cls=L2), + _FakeItem("tests::host", function=host_fn), + ] + config = _FakeConfig( + platform="a2a3", + level=None, + **{"exclude-level": None, "runtime": None, "enable-chip-swimlane": 0}, + ) + + root_conftest.pytest_collection_modifyitems(None, config, items) + + assert [item.nodeid for item in items] == ["tests::host", "tests::l2"] + + +def test_level_filters_are_mutually_exclusive(): + config = _FakeConfig(level=4, **{"exclude-level": 4}) + + with pytest.raises(pytest.UsageError, match="cannot be used together"): + root_conftest._validate_level_filters(config) + + +def test_pod_logs_requires_pod_level(monkeypatch): + @scene_level(SceneTestLevel.CHIP) + def chip_fn(): + return None + + request = SimpleNamespace(node=_FakeItem("tests::chip", function=chip_fn)) + + with pytest.raises(Failed, match="SceneTestLevel\\.POD"): + root_conftest.st_pod_logs.__wrapped__(request, monkeypatch) From dbdb63f1eeccc1d8bf8f60b35f4f3fac8866f37e Mon Sep 17 00:00:00 2001 From: ccyywwen <75376396+ccyywwen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:45:42 +0800 Subject: [PATCH 2/2] Fix: satisfy scene-level pre-commit checks --- conftest.py | 3 +-- tests/ut/py/test_scene_level_selection.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index a1450eb451..c73fe84b11 100644 --- a/conftest.py +++ b/conftest.py @@ -673,8 +673,7 @@ def sort_key(item): l3_items = [ i for i in items - if _item_scene_level(i) == SceneTestLevel.HOST - and not any(m.name == "skip" for m in i.iter_markers()) + if _item_scene_level(i) == SceneTestLevel.HOST and not any(m.name == "skip" for m in i.iter_markers()) ] if l3_items: sample = ", ".join(sorted({i.nodeid for i in l3_items})[:3]) diff --git a/tests/ut/py/test_scene_level_selection.py b/tests/ut/py/test_scene_level_selection.py index 6ceb4f8521..940f5e4338 100644 --- a/tests/ut/py/test_scene_level_selection.py +++ b/tests/ut/py/test_scene_level_selection.py @@ -2,7 +2,7 @@ # This program is free software, you can redistribute it and/or modify it under the terms and conditions of # CANN Open Software License Agreement Version 2.0 (the "License"). # Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # -----------------------------------------------------------------------------------------------------------