test(conformance): retire 13 body-less scenarios, pin README (#749)

CONTRIBUTING.md

@@ -331,7 +331,10 @@ exemptions are published in the README as credibility. When adding an adapter,
 wire every scenario; if one genuinely can't pass for a structural reason, add a
 `KNOWN_LIMITATIONS` entry (not "not supported yet"). Never add a
 framework-specific scenario — a shared-suite invariant must apply to all
-frameworks or it doesn't belong there.
+frameworks or it doesn't belong there. A scenario is added together with the
+tests that run it: `SCENARIO_BODIES` in the same file maps each name to its
+tests, and CI fails on a name with no body or a README exemption row that the
+code does not assert (#749).
 
 ### Adding a framework adapter: the integration checklist
 

python/tests/conformance/suite.py

@@ -9,33 +9,41 @@
 (#223/#244). The customer-facing pytest plugin (``donkey_kit.conformance``, #191)
 is the shipped deliverable this internal matrix backstops.
 
-The scenario bodies are wired against captured contract fixtures (BG §1.5) and
-the local gateway (BG §1.4) as their gating features land (#444). This module
-fixes the scenario list and the exemption table now so the kit exists before
-the second adapter is built (working instruction #5).
+Every name below has an executable body (#749). ``SCENARIO_BODIES`` maps each
+one to the tests that run it, and ``test_transport_exemptions.py`` checks that
+the two lists match and that every listed test exists, so a name cannot outlive
+its body. Most bodies are in the adapter contract suite
+(``test_adapter_contract.py``, #742), which drives each real framework against a
+loopback gateway. ``attribution_headers_present`` asserts the consumer-auth
+``client_id`` / ``client_secret`` headers. The ``client_id`` is the verified
+per-agent attribution unit. The application and business-group header names
+are still UNVERIFIED placeholders (docs/verified-apis.md §3).
+
+Thirteen names were **retired, not implemented** (#749): ``single_tool_call``,
+``multi_tool_multi_server``, ``tool_filtering``, ``governed_filter_excludes``,
+``governance_resolve_drift``, ``governance_target_switch``,
+``descriptor_auto_stable``, ``publication_verify_drift``,
+``publication_idempotent``, ``descriptor_matches_framework``,
+``auto_vs_live_agree``, ``dynamic_tools_detected`` and ``asset_type_detection``.
+Each one needs a per-adapter surface that still raises ``_verify.blocked(...)``
+under the verification discipline. Binding MCP tools into a framework is
+blocked in ``tools/session.py``. Exchange discovery is blocked in
+``registry/exchange.py``, publication in ``registry/publication.py`` and live
+introspection in ``registry/introspect.py``. No adapter can run a scenario
+against a surface that always refuses. The pure logic beneath them
+(``ToolFilter``, collision prefixes, governance criteria, content digests) is
+unit-tested in ``tests/unit/test_pure_logic.py``. A retired name comes back
+only with a body, once its surface is verified and built.
 """
 
 from __future__ import annotations
 
 CONFORMANCE_SCENARIOS = [
     "simple_completion",
     "streaming_completion",
-    "single_tool_call",
-    "multi_tool_multi_server",
-    "tool_filtering",
     "policy_violation_terminal",
     "attribution_headers_present",
     "correlation_id_propagated",
-    "governed_filter_excludes",
-    "governance_resolve_drift",
-    "governance_target_switch",
-    "descriptor_auto_stable",
-    "publication_verify_drift",
-    "publication_idempotent",
-    "descriptor_matches_framework",
-    "auto_vs_live_agree",
-    "dynamic_tools_detected",
-    "asset_type_detection",
     # After a governed 200, donkey.last_call carries the gateway's own identity
     # (request_id / api_instance_id / environment_id) for the call just made
     # (BG §1.1, #362). Mirrors correlation_id_propagated: the adapters that route
@@ -57,6 +65,36 @@
     "typed_refusal_bridged",
 ]
 
+# Where each scenario runs (#749), as pytest node ids relative to ``python/``.
+# test_transport_exemptions.py keeps the keys equal to CONFORMANCE_SCENARIOS and
+# checks that every node id names a test function that exists.
+_CONTRACT = "tests/conformance/test_adapter_contract.py"
+SCENARIO_BODIES: dict[str, tuple[str, ...]] = {
+    "simple_completion": (
+        f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation",
+        f"{_CONTRACT}::test_sync_call",
+    ),
+    "streaming_completion": (f"{_CONTRACT}::test_streamed_call",),
+    "policy_violation_terminal": (f"{_CONTRACT}::test_refusal_is_sent_once_and_typed",),
+    "attribution_headers_present": (
+        f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation",
+    ),
+    "correlation_id_propagated": (
+        f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation",
+    ),
+    "gateway_identity_observed": (
+        f"{_CONTRACT}::test_last_call_matches_the_declared_metadata",
+        "tests/conformance/test_transport_exemptions.py::test_exemption_matches_observes_last_call_flag",
+    ),
+    "jwt_token_refreshed": (
+        "tests/unit/test_adapter_jwt_mode.py::test_async_call_carries_the_jwt",
+        "tests/unit/test_adapter_jwt_mode.py::test_crewai_refuses_jwt_mode_before_sending",
+    ),
+    "typed_refusal_bridged": (
+        "tests/conformance/test_typed_refusal_bridge.py::test_refusal_reaches_user_code_typed",
+    ),
+}
+
 # Documented, ASSERTED exemptions — published in the README (the conformance kit). A framework
 # that cannot satisfy a scenario records WHY here rather than skipping silently.
 # Every adapter but CrewAI sends through our shared httpx client (#740), so

python/tests/conformance/test_readme_exemptions_table.py

@@ -0,0 +1,101 @@
+"""The README "Conformance exemptions" table is tested against the code (#749).
+
+The table appears twice: in the repo ``README.md`` (GitHub) and in
+``python/README.md`` (the PyPI description). Both are checked.
+
+Each row names a framework and a scenario that the code records as an asserted
+exemption. Most rows come from ``KNOWN_LIMITATIONS`` in
+``tests/conformance/suite.py``, the internal matrix. The one row on a scenario
+the internal matrix lacks comes from ``examples/crewai/main.py``, whose
+``KNOWN_LIMITATIONS`` the public plugin reads. Both directions are checked: a
+row with no exemption in code fails, and so does an internal exemption that the
+README does not publish.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+from suite import KNOWN_LIMITATIONS
+
+_PYTHON_ROOT = Path(__file__).resolve().parents[2]
+_READMES = [_PYTHON_ROOT.parent / "README.md", _PYTHON_ROOT / "README.md"]
+_IDS = ["repo", "pypi"]
+
+# Table label -> scenario name. Every label names an internal matrix scenario
+# except "budget refusal not retried", which names a public plugin scenario.
+_SCENARIOS = {
+    "correlation ID propagated": "correlation_id_propagated",
+    "gateway identity observed": "gateway_identity_observed",
+    "JWT refreshed per send": "jwt_token_refreshed",
+    "budget refusal not retried": "retries_token_budget",
+    "typed refusal bridged": "typed_refusal_bridged",
+}
+_PUBLIC_ONLY = {"retries_token_budget"}
+
+# Table label -> the ADAPTERS keys the row names.
+_FRAMEWORKS = {
+    "CrewAI": ("crewai",),
+    "ADK `model()`, CrewAI": ("adk", "crewai"),
+}
+
+
+def _published(readme: Path) -> set[tuple[str, str]]:
+    """The ``(framework, scenario)`` pairs the README table publishes."""
+    text = readme.read_text()
+    start = text.index("## Conformance exemptions")
+    end = text.find("\n## ", start + 1)
+    section = text[start : end if end != -1 else len(text)]
+    rows = [line for line in section.splitlines() if line.startswith("| ")][2:]
+    assert rows, f"no data rows under '## Conformance exemptions' in {readme}"
+    pairs: set[tuple[str, str]] = set()
+    for row in rows:
+        cells = [cell.strip() for cell in row.strip().strip("|").split("|")]
+        assert len(cells) == 3, f"expected 3 cells, got {cells!r}"
+        framework, scenario = cells[0], cells[1]
+        assert scenario in _SCENARIOS, f"README scenario {scenario!r} is not in _SCENARIOS"
+        assert framework in _FRAMEWORKS, f"README framework {framework!r} is not in _FRAMEWORKS"
+        pairs.update((fw, _SCENARIOS[scenario]) for fw in _FRAMEWORKS[framework])
+    return pairs
+
+
+def _crewai_example_limits() -> dict[str, str]:
+    """``KNOWN_LIMITATIONS`` from ``examples/crewai/main.py``, loaded by path.
+
+    Plain ``pytest`` (unlike ``python -m pytest``) does not put ``python/`` on
+    ``sys.path``, so ``examples`` is not importable as a package.
+    """
+    path = _PYTHON_ROOT / "examples" / "crewai" / "main.py"
+    spec = importlib.util.spec_from_file_location("_crewai_example_main", path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    limits: dict[str, str] = module.KNOWN_LIMITATIONS
+    return limits
+
+
+@pytest.mark.parametrize("readme", _READMES, ids=_IDS)
+def test_every_readme_row_is_an_exemption_in_code(readme: Path) -> None:
+    example = _crewai_example_limits()
+    for framework, scenario in _published(readme):
+        if scenario in _PUBLIC_ONLY:
+            assert framework == "crewai", (framework, scenario)
+            assert scenario in example, f"examples/crewai/main.py does not exempt {scenario!r}"
+        else:
+            assert scenario in KNOWN_LIMITATIONS.get(framework, {}), (
+                f"README exempts {framework!r} from {scenario!r}, but "
+                f"tests/conformance/suite.py's KNOWN_LIMITATIONS does not"
+            )
+
+
+@pytest.mark.parametrize("readme", _READMES, ids=_IDS)
+def test_every_internal_exemption_is_in_the_readme(readme: Path) -> None:
+    asserted = {
+        (framework, scenario)
+        for framework, limits in KNOWN_LIMITATIONS.items()
+        for scenario in limits
+    }
+    missing = asserted - _published(readme)
+    assert not missing, f"KNOWN_LIMITATIONS entries missing from the README table: {missing}"

python/tests/conformance/test_transport_exemptions.py

@@ -17,6 +17,7 @@
 3. The adapters that record no exemption really are handed the shared client:
    MS Agent Framework and ADK ``model()`` get an OpenAI client that sends
    through it; LlamaIndex and ADK ``gemini()`` get its view (#740).
+4. Every scenario names the tests that run it, and those tests exist (#749).
 
 Reading ``capabilities()`` off each adapter class imports only the adapter
 modules, which import their framework lazily inside methods — so this needs no
@@ -26,20 +27,23 @@
 
 from __future__ import annotations
 
+import ast
 import importlib
+from pathlib import Path
 
 import pytest
 
 # Sibling data module: tests/conformance/ is not a package, so pytest's prepend
 # import mode puts this directory on sys.path and ``suite`` resolves to it.
-from suite import CONFORMANCE_SCENARIOS, KNOWN_LIMITATIONS
+from suite import CONFORMANCE_SCENARIOS, KNOWN_LIMITATIONS, SCENARIO_BODIES
 
 from donkey_kit.core.config import DonkeyConfig
 from donkey_kit.core.transport import build_http_client
 from donkey_kit.integrations import ADAPTERS
 from donkey_kit.integrations._base import Adapter
 from donkey_kit.llm import client as llm_client
 
+_PYTHON_ROOT = Path(__file__).resolve().parents[2]
 _GATEWAY_SCENARIO = "gateway_identity_observed"
 _CORRELATION_SCENARIO = "correlation_id_propagated"
 _JWT_SCENARIO = "jwt_token_refreshed"
@@ -65,6 +69,30 @@ def test_scenario_is_registered() -> None:
     )
 
 
+def _test_functions(path: Path) -> set[str]:
+    tree = ast.parse(path.read_text())
+    return {
+        node.name
+        for node in tree.body
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
+        and node.name.startswith("test_")
+    }
+
+
+def test_every_scenario_has_an_executable_body() -> None:
+    # #749: a scenario name with no test behind it overstates what is verified.
+    # Each name maps to the tests that run it, and each of those tests exists.
+    assert len(CONFORMANCE_SCENARIOS) == len(set(CONFORMANCE_SCENARIOS))
+    assert set(SCENARIO_BODIES) == set(CONFORMANCE_SCENARIOS)
+    for scenario, node_ids in SCENARIO_BODIES.items():
+        assert node_ids, f"{scenario!r} lists no test"
+        for node_id in node_ids:
+            path, _, name = node_id.partition("::")
+            file = _PYTHON_ROOT / path
+            assert file.is_file(), f"{scenario!r}: {path} does not exist"
+            assert name in _test_functions(file), f"{scenario!r}: {node_id} does not exist"
+
+
 def test_every_known_limitation_names_a_real_scenario() -> None:
     scenarios = set(CONFORMANCE_SCENARIOS)
     for adapter, limits in KNOWN_LIMITATIONS.items():