diff --git a/scripts/ci/contextual_orchestrator_review_launcher.py b/scripts/ci/contextual_orchestrator_review_launcher.py index 02bc07c28a..2e56809639 100644 --- a/scripts/ci/contextual_orchestrator_review_launcher.py +++ b/scripts/ci/contextual_orchestrator_review_launcher.py @@ -29,6 +29,8 @@ from pathlib import Path from typing import Any +from scripts.ci.contextual_orchestrator_review_policy import FREE_POOL_CREDENTIAL_NAMES + # The vendored server's generic 64 KiB default is intentionally conservative. # This loopback, bearer-authenticated review sidecar accepts OpenAI's image-input @@ -689,18 +691,29 @@ def _with_discovery_counts( from whatever narrower row set it was given, would otherwise contradict that field's documented "among *all* discovered free routes" contract. """ + free_rows = [row for row in rows if row.get("cost_evidence") == "free"] + free_pool_rows = [ + row + for row in free_rows + if isinstance(row.get("credential_key"), str) + and row["credential_key"] in FREE_POOL_CREDENTIAL_NAMES + ] enriched = dict(report) enriched.update( { "total_routes": len(rows), - "total_free_routes": sum(row.get("cost_evidence") == "free" for row in rows), + "total_free_routes": len(free_rows), "total_priced_routes": sum(row.get("cost_evidence") == "priced" for row in rows), "total_unknown_routes": sum(row.get("cost_evidence") == "unknown" for row in rows), "free_account_diversity": len( + {provider_account(str(row["provider"])) for row in free_rows} + ), + "free_pool_admitted_routes": len(free_pool_rows), + "free_pool_excluded_source_count": len(free_rows) - len(free_pool_rows), + "free_pool_account_diversity": len( { provider_account(str(row["provider"])) - for row in rows - if row.get("cost_evidence") == "free" + for row in free_pool_rows } ), } diff --git a/scripts/ci/contextual_orchestrator_review_policy.py b/scripts/ci/contextual_orchestrator_review_policy.py index 1c8a170144..53e66cfa36 100644 --- a/scripts/ci/contextual_orchestrator_review_policy.py +++ b/scripts/ci/contextual_orchestrator_review_policy.py @@ -1,9 +1,11 @@ """Build governed contextual-orchestrator review catalogs from discovery evidence. -``orchestrator/free`` remains strictly zero-priced. ``orchestrator/auto`` is -free-first and then uses fully price-attested routes. Models without a complete -price vector remain visible in audit counts but are never admitted to CI review. -Partial, malformed, or contradictory price vectors fail closed. +``orchestrator/free`` remains strictly zero-priced and admits only provider +accounts explicitly authorized for that pool. ``orchestrator/auto`` may retain +other globally discovered providers, including OpenAI, when their independent +policy permits them. Models without a complete price vector remain visible in +audit counts but are never admitted to CI review. Partial, malformed, or +contradictory price vectors fail closed. """ from __future__ import annotations @@ -30,6 +32,20 @@ DEFAULT_CATALOG_LIMIT = 12 DEFAULT_ACCOUNT_CAP = 4 +FREE_POOL_CREDENTIAL_NAMES = frozenset( + { + "BYTEZ_API_KEY", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENROUTER_API_KEY", + } +) +"""Credential sources authorized to contribute to ``orchestrator/free``. + +``OPENAI_API_KEY`` is intentionally absent. It may still be present, registered, +and globally discovered; only candidate admission to the free pool is denied. +""" + COST_FREE = "free" COST_PRICED = "priced" COST_UNKNOWN = "unknown" @@ -147,6 +163,18 @@ def parse_discovery_report(report: Mapping[str, Any]) -> list[dict[str, Any]]: f"model {provider}/{model} lacks an explicit is_free marker" ) + expected_credential_key = PROVIDER_CREDENTIAL_NAMES[provider] + supplied_credential_key = row.get("credential_key") + credential_key = ( + expected_credential_key + if supplied_credential_key is None + else supplied_credential_key + ) + if credential_key != expected_credential_key: + raise PolicyError( + f"model {provider}/{model} credential source does not match provider evidence" + ) + is_free = is_free_route(row.get("is_free")) route = f"{provider}/{model}" cost_evidence, prompt_price, completion_price, currency_code = ( @@ -170,8 +198,7 @@ def parse_discovery_report(report: Mapping[str, Any]) -> list[dict[str, Any]]: "completion_price_per_1k": completion_price, "currency_code": currency_code, "base_url": row.get("base_url") or PROVIDER_BASE_URLS[provider], - "credential_key": row.get("credential_key") - or PROVIDER_CREDENTIAL_NAMES[provider], + "credential_key": credential_key, "auth_scheme": row.get("auth_scheme") or PROVIDER_AUTH_SCHEMES[provider], } @@ -189,6 +216,15 @@ def _cost_evidence(row: Mapping[str, Any]) -> str: return COST_FREE if row.get("is_free") is True else COST_UNKNOWN +def _free_pool_source_admitted(row: Mapping[str, Any]) -> bool: + """Return whether a normalized row has an authorized free-pool source.""" + credential_key = row.get("credential_key") + return ( + isinstance(credential_key, str) + and credential_key in FREE_POOL_CREDENTIAL_NAMES + ) + + def build_zdr_prioritized_catalog( rows: Iterable[Mapping[str, Any]], *, @@ -200,28 +236,26 @@ def build_zdr_prioritized_catalog( ) -> dict[str, Any]: """Select a free-first, ZDR-aware, credential-account-diverse catalog. - The returned report's ``free_account_diversity`` counts the distinct - credential accounts among *all* discovered free routes, independent of - ``pool`` or the per-account selection cap. Vendor identity is not model - equivalence; only an explicit contextual-orchestrator ``model_group`` may - share routing evidence across routes. - - This counts routes discovery reports as free, not routes runtime - preflight has confirmed are actually serving requests: a value of two or - more is evidence that one account failure cannot immediately empty the free - catalog, not proof that either account is presently reachable. A caller - needing readiness, not just discovery-time diversity, must combine this - with the runtime preflight report the sidecar already produces. + ``orchestrator/free`` first applies a source-identity invariant: only rows + whose credential source is in :data:`FREE_POOL_CREDENTIAL_NAMES` are free + candidates. This is independent from global credential discovery, so an + OpenAI model may remain visible to audit or ``orchestrator/auto`` while + contributing zero free-pool candidates. + + Existing discovery-wide counters keep their historical meaning so runtime + enrichment cannot silently rewrite the contract. Additional + ``free_pool_*`` fields expose the narrower admitted subset explicitly. """ if pool not in {"free", "auto"}: raise PolicyError(f"unsupported review pool {pool!r}") all_rows = list(rows) all_free_rows = [row for row in all_rows if _cost_evidence(row) == COST_FREE] + free_pool_rows = [row for row in all_free_rows if _free_pool_source_admitted(row)] all_priced_rows = [row for row in all_rows if _cost_evidence(row) == COST_PRICED] all_unknown_rows = [row for row in all_rows if _cost_evidence(row) == COST_UNKNOWN] candidate_rows = ( - all_free_rows if pool == "free" else [*all_free_rows, *all_priced_rows] + free_pool_rows if pool == "free" else [*all_free_rows, *all_priced_rows] ) eligible_rows = [ row @@ -304,6 +338,9 @@ def build_zdr_prioritized_catalog( free_account_diversity = len( {provider_account(str(row["provider"])) for row in all_free_rows} ) + free_pool_account_diversity = len( + {provider_account(str(row["provider"])) for row in free_pool_rows} + ) selected_evidence = [_cost_evidence(row) for row in picked] return { @@ -312,9 +349,12 @@ def build_zdr_prioritized_catalog( "pool": f"orchestrator/{pool}", "total_routes": len(all_rows), "total_free_routes": len(all_free_rows), + "free_account_diversity": free_account_diversity, + "free_pool_admitted_routes": len(free_pool_rows), + "free_pool_excluded_source_count": len(all_free_rows) - len(free_pool_rows), + "free_pool_account_diversity": free_pool_account_diversity, "total_priced_routes": len(all_priced_rows), "total_unknown_routes": len(all_unknown_rows), - "free_account_diversity": free_account_diversity, "zdr_required": require_zdr, "selected_count": len(catalog_rows), "free_selected_count": selected_evidence.count(COST_FREE), @@ -437,4 +477,4 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": # pragma: no cover - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file diff --git a/tests/test_contextual_orchestrator_free_credential_admission.py b/tests/test_contextual_orchestrator_free_credential_admission.py new file mode 100644 index 0000000000..b9ae6c207b --- /dev/null +++ b/tests/test_contextual_orchestrator_free_credential_admission.py @@ -0,0 +1,89 @@ +"""Free-pool credential-source admission contracts for the review catalog.""" + +from __future__ import annotations + +import pytest + +from scripts.ci.contextual_orchestrator_review_policy import ( + PolicyError, + build_zdr_prioritized_catalog, + parse_discovery_report, +) +from scripts.ci.zdr_policy import PROVIDER_CREDENTIAL_NAMES + + +def _zero_cost_row(provider: str) -> dict[str, object]: + """Return one complete zero-cost discovery row with authentic source identity.""" + return { + "provider": provider, + "model": f"{provider}-review-model", + "agent_id": f"{provider}_review_model", + "is_free": True, + "prompt_price_per_1k": 0.0, + "completion_price_per_1k": 0.0, + "currency_code": "USD", + "credential_key": PROVIDER_CREDENTIAL_NAMES[provider], + } + + +def test_free_pool_excludes_openai_while_global_discovery_keeps_all_five() -> None: + """All providers stay discoverable, but OpenAI contributes no free candidate.""" + providers = ( + "bytez", + "nvidia_nim", + "nvidia_nim_sub", + "openrouter", + "openai", + ) + rows = parse_discovery_report({"models": [_zero_cost_row(provider) for provider in providers]}) + + result = build_zdr_prioritized_catalog(rows, pool="free", limit=12, account_cap=12) + + selected = result["report"]["selected"] + assert {entry["provider"] for entry in selected} == { + "bytez", + "nvidia_nim", + "nvidia_nim_sub", + "openrouter", + } + assert all(agent["credential_key"] != "OPENAI_API_KEY" for agent in result["agents"]) + # Discovery-wide counters keep their established meaning; narrower pool + # admission gets separate fields so runtime enrichment cannot relabel them. + assert result["report"]["total_free_routes"] == 5 + assert result["report"]["free_account_diversity"] == 5 + assert result["report"]["free_pool_admitted_routes"] == 4 + assert result["report"]["free_pool_excluded_source_count"] == 1 + assert result["report"]["free_pool_account_diversity"] == 4 + + +def test_auto_pool_may_retain_globally_discovered_openai() -> None: + """The free-source rule does not delete OpenAI from non-free/global routing.""" + rows = parse_discovery_report( + {"models": [_zero_cost_row("openrouter"), _zero_cost_row("openai")]} + ) + + result = build_zdr_prioritized_catalog(rows, pool="auto", limit=12, account_cap=12) + + assert {entry["provider"] for entry in result["report"]["selected"]} == { + "openrouter", + "openai", + } + assert result["report"]["free_pool_admitted_routes"] == 1 + assert result["report"]["free_pool_excluded_source_count"] == 1 + + +def test_openai_only_zero_cost_discovery_fails_closed_for_free_pool() -> None: + """A globally discovered OpenAI route cannot become the sole free fallback.""" + rows = parse_discovery_report({"models": [_zero_cost_row("openai")]}) + + with pytest.raises(PolicyError, match="orchestrator/free would fail closed"): + build_zdr_prioritized_catalog(rows, pool="free", limit=12, account_cap=12) + + +def test_provider_credential_source_mismatch_is_rejected() -> None: + """A row cannot spoof an eligible provider while using the OpenAI credential.""" + row = _zero_cost_row("openrouter") + row["credential_key"] = "OPENAI_API_KEY" + + with pytest.raises(PolicyError, match="credential source does not match provider evidence"): + parse_discovery_report({"models": [row]}) diff --git a/tests/test_contextual_orchestrator_free_pool_enrichment.py b/tests/test_contextual_orchestrator_free_pool_enrichment.py new file mode 100644 index 0000000000..d28dbbc98d --- /dev/null +++ b/tests/test_contextual_orchestrator_free_pool_enrichment.py @@ -0,0 +1,42 @@ +"""Runtime report enrichment contracts for free-pool source admission.""" + +from scripts.ci.contextual_orchestrator_review_launcher import _with_discovery_counts + + +def test_discovery_enrichment_recomputes_free_pool_counts_from_full_rows() -> None: + """Priced fallback reports retain full discovered authorized free capacity.""" + stage_report = { + "free_pool_admitted_routes": 0, + "free_pool_excluded_source_count": 0, + "free_pool_account_diversity": 0, + } + rows = [ + { + "provider": "openrouter", + "credential_key": "OPENROUTER_API_KEY", + "cost_evidence": "free", + }, + { + "provider": "openai", + "credential_key": "OPENAI_API_KEY", + "cost_evidence": "free", + }, + { + "provider": "openai", + "credential_key": "OPENAI_API_KEY", + "cost_evidence": "priced", + }, + ] + + enriched = _with_discovery_counts( + stage_report, + rows, + provider_account=lambda provider: provider, + ) + + assert enriched["total_routes"] == 3 + assert enriched["total_free_routes"] == 2 + assert enriched["free_account_diversity"] == 2 + assert enriched["free_pool_admitted_routes"] == 1 + assert enriched["free_pool_excluded_source_count"] == 1 + assert enriched["free_pool_account_diversity"] == 1