diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 297c3231..56761404 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -331,7 +331,10 @@ exemptions are published in the README as credibility. When adding an adapter, wire every scenario; if one genuinely can't pass for a structural reason, add a `KNOWN_LIMITATIONS` entry (not "not supported yet"). Never add a framework-specific scenario — a shared-suite invariant must apply to all -frameworks or it doesn't belong there. +frameworks or it doesn't belong there. A scenario is added together with the +tests that run it: `SCENARIO_BODIES` in the same file maps each name to its +tests, and CI fails on a name with no body or a README exemption row that the +code does not assert (#749). ### Adding a framework adapter: the integration checklist diff --git a/python/tests/conformance/suite.py b/python/tests/conformance/suite.py index 73b16f22..50351d5a 100644 --- a/python/tests/conformance/suite.py +++ b/python/tests/conformance/suite.py @@ -9,10 +9,31 @@ (#223/#244). The customer-facing pytest plugin (``donkey_kit.conformance``, #191) is the shipped deliverable this internal matrix backstops. -The scenario bodies are wired against captured contract fixtures (BG §1.5) and -the local gateway (BG §1.4) as their gating features land (#444). This module -fixes the scenario list and the exemption table now so the kit exists before -the second adapter is built (working instruction #5). +Every name below has an executable body (#749). ``SCENARIO_BODIES`` maps each +one to the tests that run it, and ``test_transport_exemptions.py`` checks that +the two lists match and that every listed test exists, so a name cannot outlive +its body. Most bodies are in the adapter contract suite +(``test_adapter_contract.py``, #742), which drives each real framework against a +loopback gateway. ``attribution_headers_present`` asserts the consumer-auth +``client_id`` / ``client_secret`` headers. The ``client_id`` is the verified +per-agent attribution unit. The application and business-group header names +are still UNVERIFIED placeholders (docs/verified-apis.md §3). + +Thirteen names were **retired, not implemented** (#749): ``single_tool_call``, +``multi_tool_multi_server``, ``tool_filtering``, ``governed_filter_excludes``, +``governance_resolve_drift``, ``governance_target_switch``, +``descriptor_auto_stable``, ``publication_verify_drift``, +``publication_idempotent``, ``descriptor_matches_framework``, +``auto_vs_live_agree``, ``dynamic_tools_detected`` and ``asset_type_detection``. +Each one needs a per-adapter surface that still raises ``_verify.blocked(...)`` +under the verification discipline. Binding MCP tools into a framework is +blocked in ``tools/session.py``. Exchange discovery is blocked in +``registry/exchange.py``, publication in ``registry/publication.py`` and live +introspection in ``registry/introspect.py``. No adapter can run a scenario +against a surface that always refuses. The pure logic beneath them +(``ToolFilter``, collision prefixes, governance criteria, content digests) is +unit-tested in ``tests/unit/test_pure_logic.py``. A retired name comes back +only with a body, once its surface is verified and built. """ from __future__ import annotations @@ -20,22 +41,9 @@ CONFORMANCE_SCENARIOS = [ "simple_completion", "streaming_completion", - "single_tool_call", - "multi_tool_multi_server", - "tool_filtering", "policy_violation_terminal", "attribution_headers_present", "correlation_id_propagated", - "governed_filter_excludes", - "governance_resolve_drift", - "governance_target_switch", - "descriptor_auto_stable", - "publication_verify_drift", - "publication_idempotent", - "descriptor_matches_framework", - "auto_vs_live_agree", - "dynamic_tools_detected", - "asset_type_detection", # After a governed 200, donkey.last_call carries the gateway's own identity # (request_id / api_instance_id / environment_id) for the call just made # (BG §1.1, #362). Mirrors correlation_id_propagated: the adapters that route @@ -57,6 +65,36 @@ "typed_refusal_bridged", ] +# Where each scenario runs (#749), as pytest node ids relative to ``python/``. +# test_transport_exemptions.py keeps the keys equal to CONFORMANCE_SCENARIOS and +# checks that every node id names a test function that exists. +_CONTRACT = "tests/conformance/test_adapter_contract.py" +SCENARIO_BODIES: dict[str, tuple[str, ...]] = { + "simple_completion": ( + f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation", + f"{_CONTRACT}::test_sync_call", + ), + "streaming_completion": (f"{_CONTRACT}::test_streamed_call",), + "policy_violation_terminal": (f"{_CONTRACT}::test_refusal_is_sent_once_and_typed",), + "attribution_headers_present": ( + f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation", + ), + "correlation_id_propagated": ( + f"{_CONTRACT}::test_call_sends_governed_headers_and_correlation", + ), + "gateway_identity_observed": ( + f"{_CONTRACT}::test_last_call_matches_the_declared_metadata", + "tests/conformance/test_transport_exemptions.py::test_exemption_matches_observes_last_call_flag", + ), + "jwt_token_refreshed": ( + "tests/unit/test_adapter_jwt_mode.py::test_async_call_carries_the_jwt", + "tests/unit/test_adapter_jwt_mode.py::test_crewai_refuses_jwt_mode_before_sending", + ), + "typed_refusal_bridged": ( + "tests/conformance/test_typed_refusal_bridge.py::test_refusal_reaches_user_code_typed", + ), +} + # Documented, ASSERTED exemptions — published in the README (the conformance kit). A framework # that cannot satisfy a scenario records WHY here rather than skipping silently. # Every adapter but CrewAI sends through our shared httpx client (#740), so diff --git a/python/tests/conformance/test_readme_exemptions_table.py b/python/tests/conformance/test_readme_exemptions_table.py new file mode 100644 index 00000000..207d1616 --- /dev/null +++ b/python/tests/conformance/test_readme_exemptions_table.py @@ -0,0 +1,101 @@ +"""The README "Conformance exemptions" table is tested against the code (#749). + +The table appears twice: in the repo ``README.md`` (GitHub) and in +``python/README.md`` (the PyPI description). Both are checked. + +Each row names a framework and a scenario that the code records as an asserted +exemption. Most rows come from ``KNOWN_LIMITATIONS`` in +``tests/conformance/suite.py``, the internal matrix. The one row on a scenario +the internal matrix lacks comes from ``examples/crewai/main.py``, whose +``KNOWN_LIMITATIONS`` the public plugin reads. Both directions are checked: a +row with no exemption in code fails, and so does an internal exemption that the +README does not publish. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest +from suite import KNOWN_LIMITATIONS + +_PYTHON_ROOT = Path(__file__).resolve().parents[2] +_READMES = [_PYTHON_ROOT.parent / "README.md", _PYTHON_ROOT / "README.md"] +_IDS = ["repo", "pypi"] + +# Table label -> scenario name. Every label names an internal matrix scenario +# except "budget refusal not retried", which names a public plugin scenario. +_SCENARIOS = { + "correlation ID propagated": "correlation_id_propagated", + "gateway identity observed": "gateway_identity_observed", + "JWT refreshed per send": "jwt_token_refreshed", + "budget refusal not retried": "retries_token_budget", + "typed refusal bridged": "typed_refusal_bridged", +} +_PUBLIC_ONLY = {"retries_token_budget"} + +# Table label -> the ADAPTERS keys the row names. +_FRAMEWORKS = { + "CrewAI": ("crewai",), + "ADK `model()`, CrewAI": ("adk", "crewai"), +} + + +def _published(readme: Path) -> set[tuple[str, str]]: + """The ``(framework, scenario)`` pairs the README table publishes.""" + text = readme.read_text() + start = text.index("## Conformance exemptions") + end = text.find("\n## ", start + 1) + section = text[start : end if end != -1 else len(text)] + rows = [line for line in section.splitlines() if line.startswith("| ")][2:] + assert rows, f"no data rows under '## Conformance exemptions' in {readme}" + pairs: set[tuple[str, str]] = set() + for row in rows: + cells = [cell.strip() for cell in row.strip().strip("|").split("|")] + assert len(cells) == 3, f"expected 3 cells, got {cells!r}" + framework, scenario = cells[0], cells[1] + assert scenario in _SCENARIOS, f"README scenario {scenario!r} is not in _SCENARIOS" + assert framework in _FRAMEWORKS, f"README framework {framework!r} is not in _FRAMEWORKS" + pairs.update((fw, _SCENARIOS[scenario]) for fw in _FRAMEWORKS[framework]) + return pairs + + +def _crewai_example_limits() -> dict[str, str]: + """``KNOWN_LIMITATIONS`` from ``examples/crewai/main.py``, loaded by path. + + Plain ``pytest`` (unlike ``python -m pytest``) does not put ``python/`` on + ``sys.path``, so ``examples`` is not importable as a package. + """ + path = _PYTHON_ROOT / "examples" / "crewai" / "main.py" + spec = importlib.util.spec_from_file_location("_crewai_example_main", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + limits: dict[str, str] = module.KNOWN_LIMITATIONS + return limits + + +@pytest.mark.parametrize("readme", _READMES, ids=_IDS) +def test_every_readme_row_is_an_exemption_in_code(readme: Path) -> None: + example = _crewai_example_limits() + for framework, scenario in _published(readme): + if scenario in _PUBLIC_ONLY: + assert framework == "crewai", (framework, scenario) + assert scenario in example, f"examples/crewai/main.py does not exempt {scenario!r}" + else: + assert scenario in KNOWN_LIMITATIONS.get(framework, {}), ( + f"README exempts {framework!r} from {scenario!r}, but " + f"tests/conformance/suite.py's KNOWN_LIMITATIONS does not" + ) + + +@pytest.mark.parametrize("readme", _READMES, ids=_IDS) +def test_every_internal_exemption_is_in_the_readme(readme: Path) -> None: + asserted = { + (framework, scenario) + for framework, limits in KNOWN_LIMITATIONS.items() + for scenario in limits + } + missing = asserted - _published(readme) + assert not missing, f"KNOWN_LIMITATIONS entries missing from the README table: {missing}" diff --git a/python/tests/conformance/test_transport_exemptions.py b/python/tests/conformance/test_transport_exemptions.py index 8638e714..9524072c 100644 --- a/python/tests/conformance/test_transport_exemptions.py +++ b/python/tests/conformance/test_transport_exemptions.py @@ -17,6 +17,7 @@ 3. The adapters that record no exemption really are handed the shared client: MS Agent Framework and ADK ``model()`` get an OpenAI client that sends through it; LlamaIndex and ADK ``gemini()`` get its view (#740). +4. Every scenario names the tests that run it, and those tests exist (#749). Reading ``capabilities()`` off each adapter class imports only the adapter modules, which import their framework lazily inside methods — so this needs no @@ -26,13 +27,15 @@ from __future__ import annotations +import ast import importlib +from pathlib import Path import pytest # Sibling data module: tests/conformance/ is not a package, so pytest's prepend # import mode puts this directory on sys.path and ``suite`` resolves to it. -from suite import CONFORMANCE_SCENARIOS, KNOWN_LIMITATIONS +from suite import CONFORMANCE_SCENARIOS, KNOWN_LIMITATIONS, SCENARIO_BODIES from donkey_kit.core.config import DonkeyConfig from donkey_kit.core.transport import build_http_client @@ -40,6 +43,7 @@ from donkey_kit.integrations._base import Adapter from donkey_kit.llm import client as llm_client +_PYTHON_ROOT = Path(__file__).resolve().parents[2] _GATEWAY_SCENARIO = "gateway_identity_observed" _CORRELATION_SCENARIO = "correlation_id_propagated" _JWT_SCENARIO = "jwt_token_refreshed" @@ -65,6 +69,30 @@ def test_scenario_is_registered() -> None: ) +def _test_functions(path: Path) -> set[str]: + tree = ast.parse(path.read_text()) + return { + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + } + + +def test_every_scenario_has_an_executable_body() -> None: + # #749: a scenario name with no test behind it overstates what is verified. + # Each name maps to the tests that run it, and each of those tests exists. + assert len(CONFORMANCE_SCENARIOS) == len(set(CONFORMANCE_SCENARIOS)) + assert set(SCENARIO_BODIES) == set(CONFORMANCE_SCENARIOS) + for scenario, node_ids in SCENARIO_BODIES.items(): + assert node_ids, f"{scenario!r} lists no test" + for node_id in node_ids: + path, _, name = node_id.partition("::") + file = _PYTHON_ROOT / path + assert file.is_file(), f"{scenario!r}: {path} does not exist" + assert name in _test_functions(file), f"{scenario!r}: {node_id} does not exist" + + def test_every_known_limitation_names_a_real_scenario() -> None: scenarios = set(CONFORMANCE_SCENARIOS) for adapter, limits in KNOWN_LIMITATIONS.items():