feat(skeleton): surface gateway routing/fallback, never double-retry it (#309)

docs/verified-apis.md

@@ -88,7 +88,7 @@ request against the deployed gateway.
 | Ingress base URL the SDK targets | `llm/client.py` | VERIFIED (LIVE) | `https://<ingress-gw-host>/<instance-path>/` — e.g. `https://agent-network-ingress-gw-zovwbn.jeg62f.usa-e2.cloudhub.io/openai-sdk/`. **No `/v1` at the ingress**; OpenAI path segment appended directly (`/responses`, and OpenAI-native routes) | 2026-08-28 | live probe |
 | Endpoint/API surface | `llm/client.py` | VERIFIED (LIVE) | OpenAI **Responses API** works (`POST /openai-sdk/responses`, body `{model, input}`). Upstream registered as `https://api.openai.com/v1/`; `proxyUri http://0.0.0.0:8081/openai-sdk` | 2026-08-28 | `api:describe`, live probe |
 | Auth: header name / model | `core/transport.py`, `core/config.py` | VERIFIED (LIVE) | **`client_id` + `client_secret` request headers** (NOT bearer) — enforced by `client-id-enforcement` 1.3.3. A consumer **credential pair**, mapped to an Anypoint client application | 2026-08-28 | live probe + `policy:list` |
-| Model routing | `llm/client.py` | VERIFIED (LIVE) | `model-based-routing` 1.0.3 reads `model` from body → provider. Response headers `x-llm-proxy-routing-type: ModelBased`, `x-llm-proxy-llm-provider`, `x-llm-proxy-llm-model` | 2026-08-28 | live probe |
+| Model routing | `llm/client.py`; consumed by `core/lastcall.py` + `core/transport.py` (#309) | VERIFIED (LIVE) | `model-based-routing` 1.0.3 reads `model` from body → provider. Response headers `x-llm-proxy-routing-type: ModelBased`, `x-llm-proxy-routing-fallback: false`, `x-llm-proxy-llm-provider: openai`, `x-llm-proxy-llm-model: gpt-5.1`, `x-llm-proxy-model-based-routing-success: Request successfully matched. …`. As of #309 the SDK surfaces these at `donkey.last_call` (`served_provider`/`served_model`/`routing_type`/`fallback`, `substituted` when served≠requested), emits them as `gen_ai.response.model` / `donkey.routing.type` / `donkey.routing.fallback` span attrs, and **does not retry** a `503` a fallback header already marked failed-over (§309, must not fight #183) | 2026-08-28 | live probe (`responses.success.headers.txt`) |
 | Token accounting (cost attribution) | `llm/*`, telemetry | VERIFIED (LIVE) | response `usage: {input_tokens, output_tokens, total_tokens, input_tokens_details, output_tokens_details}` — the detail sub-objects carry `cached_tokens` / `cache_write_tokens` (under `input_tokens_details`) and `reasoning_tokens` (under `output_tokens_details`), surfaced per-call at `donkey.last_call` and on the span (#307, see §3); also upstream `x-ratelimit-*` headers passed through (Go-style **duration-string** resets, e.g. `0s`/`12ms` — distinct from the gateway's own `x-llm-proxy-ratelimit`/`x-token-*` window, see §4) | 2026-08-28 | `responses.success.body.json` |
 | Request fields passed through | `llm/client.py` | VERIFIED (LIVE) | OpenAI body passed through verbatim; response returned verbatim (model `gpt-5.1`→`gpt-5.1-2025-11-13`, full Responses object) | 2026-08-28 | live probe |
 | Streaming support | `llm/client.py` | VERIFIED (LIVE) | `"stream": true` → `200`, `content-type: text/event-stream`, chunked SSE (`event: response.created` / `response.in_progress` / …); same `x-llm-proxy-*` headers | 2026-08-28 | live probe (`responses.stream.*`) |

python/src/donkey_kit/__init__.py

@@ -28,6 +28,7 @@
     ContentSafetyBlocked,
     DonkeyError,
     GovernanceDrift,
+    ModelSubstituted,
     PIIDetected,
     PolicyViolation,
     PromptInjectionBlocked,
@@ -74,6 +75,7 @@
     "Governance",
     "GovernanceCriteria",
     "GovernanceDrift",
+    "ModelSubstituted",
     "PIIDetected",
     "PolicyBinding",
     "PolicyPortability",

python/src/donkey_kit/core/__init__.py

@@ -16,6 +16,7 @@
     ContentSafetyBlocked,
     DonkeyError,
     GovernanceDrift,
+    ModelSubstituted,
     PIIDetected,
     PolicyViolation,
     PromptInjectionBlocked,
@@ -57,6 +58,7 @@
     "DonkeyConfig",
     "DonkeyError",
     "GovernanceDrift",
+    "ModelSubstituted",
     "PIIDetected",
     "PolicyViolation",
     "PromptInjectionBlocked",

python/src/donkey_kit/core/config.py

@@ -28,6 +28,14 @@
 
 Region = Literal["us", "eu", "ca", "jp"]
 
+# What to do when the gateway serves a different model than the one requested —
+# a routing fallback substituted the model (§3, #309). ``"off"`` (default): the
+# substitution is surfaced passively on ``donkey.last_call.substituted`` and the
+# span, but the call succeeds. ``"raise"``: the transport raises
+# :class:`~donkey_kit.core.errors.ModelSubstituted`, for callers who need model
+# determinism (e.g. an evaluation whose results are only comparable per-model).
+OnModelSubstitution = Literal["off", "raise"]
+
 _TOML_NAME = ".donkey-kit.toml"
 
 
@@ -85,6 +93,10 @@ class DonkeyConfig:
     # very content the platform just masked to whatever OTLP collector is wired
     # up (#306, BG §1.6). Opting in is the developer assuming that obligation.
     telemetry_capture_content: bool = False
+    # What to do when the gateway serves a different model than requested (§3,
+    # #309). Default "off" — the substitution is surfaced passively on
+    # ``donkey.last_call``; "raise" opts into a hard ``ModelSubstituted`` error.
+    on_model_substitution: OnModelSubstitution = "off"
 
     # ----------------------------------------------------------------- factory
     @classmethod
@@ -148,6 +160,9 @@ def pick(env: str, key: str, default: object) -> object:
             telemetry_capture_content=_as_bool(
                 pick("DONKEY_TELEMETRY_CAPTURE_CONTENT", "telemetry_capture_content", False)
             ),
+            on_model_substitution=_as_substitution(
+                pick("DONKEY_ON_MODEL_SUBSTITUTION", "on_model_substitution", "off")
+            ),
         )
 
     # --------------------------------------------------------------- derived
@@ -242,6 +257,19 @@ def _as_bool(v: object) -> bool:
     return str(v).strip().lower() in ("1", "true", "yes", "on")
 
 
+def _as_substitution(v: object) -> OnModelSubstitution:
+    """Coerce and validate ``on_model_substitution`` (§3, #309). An unknown value
+    is a config mistake worth reporting up front — a silent fall-back to ``"off"``
+    would leave a caller who typed ``"error"`` believing they had opted into
+    strictness. Validated at resolve time, like ``region``."""
+    token = str(v).strip().lower()
+    if token not in ("off", "raise"):
+        raise ConfigError(
+            f"Unknown on_model_substitution {v!r}. Expected 'off' or 'raise'."
+        )
+    return cast(OnModelSubstitution, token)
+
+
 def _load_toml() -> dict[str, object]:
     """Read ``[donkey]`` table from ``.donkey-kit.toml`` in cwd or
     ``$XDG_CONFIG_HOME``. Missing file is fine; malformed file raises."""

python/src/donkey_kit/core/errors.py

@@ -136,6 +136,37 @@ def __init__(
         self.reset_at = reset_at
 
 
+class ModelSubstituted(DonkeyError):
+    """The gateway served a *different* model than the one requested — a routing
+    fallback substituted the model, and ``on_model_substitution="raise"`` opted
+    the caller into treating that as an error (§3, #309).
+
+    Deliberately NOT a :class:`PolicyViolation`: the request was neither refused
+    nor failed — it succeeded, just against a model the developer did not choose.
+    It is a client-side determinism signal (like :class:`BudgetReserveReached`),
+    off by default; a caller who has not opted in gets the substitution surfaced
+    passively on ``donkey.last_call.substituted`` instead. Carries the response,
+    so a handler can still read the served completion if it decides to accept it.
+
+    ``request_id`` (the gateway's own id) is inherited from :class:`DonkeyError`
+    and populated from the response, so a substitution can be quoted in a ticket
+    on the same terms as any other gateway event."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        requested_model: str,
+        served_model: str,
+        served_provider: str | None = None,
+        **kw: Any,
+    ) -> None:
+        super().__init__(message, **kw)
+        self.requested_model = requested_model
+        self.served_model = served_model
+        self.served_provider = served_provider
+
+
 class PromptInjectionBlocked(PolicyViolation):
     policy = "prompt-injection-protection"
 

python/src/donkey_kit/core/lastcall.py

@@ -77,6 +77,19 @@
 REQUEST_ID_HEADER = "x-request-id"
 DECORATOR_OPERATION_HEADER = "x-envoy-decorator-operation"
 
+# VERIFIED (LIVE, docs/verified-apis.md §3 "Gateway identity on response",
+# 2026-08-28, ``responses.success.headers.txt``). The gateway states what it did
+# with the request: which provider/model actually served it, whether that was a
+# routing FALLBACK (the "Enhanced Resilience for Intelligent Routing" failover),
+# and the routing strategy. All four are consumed here on the success path
+# (#309) — a substitution the developer did not choose is otherwise invisible.
+# ``LLM_PROVIDER_HEADER`` is also the SOLE source of ``gen_ai.system`` on the
+# span, so ``core/transport.py`` imports it from here (one definition, §0.3).
+ROUTING_TYPE_HEADER = "x-llm-proxy-routing-type"
+ROUTING_FALLBACK_HEADER = "x-llm-proxy-routing-fallback"
+LLM_PROVIDER_HEADER = "x-llm-proxy-llm-provider"
+LLM_MODEL_HEADER = "x-llm-proxy-llm-model"
+
 # ``api-instance-21133858.3e6ce455-e3e8-4402-b830-9fcf07d9207b.svc`` → instance
 # ``21133858`` + environment ``3e6ce455-…`` (a UUID; it carries dashes but no
 # dots, so a plain three-way split on ``.`` is unambiguous). An unrecognised
@@ -115,6 +128,42 @@ def _parse_decorator_operation(raw: str | None) -> tuple[str | None, str | None]
     return m["instance"], m["env"]
 
 
+def _parse_fallback(raw: str | None) -> bool | None:
+    """The routing-fallback flag from ``x-llm-proxy-routing-fallback``, or ``None``
+    when the header is absent (a non-proxy / simulated response) or carries an
+    unrecognised value. VERIFIED values are the literal strings ``"true"`` /
+    ``"false"``; anything else is treated as unknown (``None``) rather than
+    guessed, so a bare ``None`` never masquerades as a definitive ``False``
+    (§0.3)."""
+    if raw is None:
+        return None
+    token = raw.strip().lower()
+    if token == "true":
+        return True
+    if token == "false":
+        return False
+    return None
+
+
+def routing_fallback(response: httpx.Response) -> bool | None:
+    """The tri-state routing-fallback flag for a response: ``True`` / ``False``
+    when the gateway stated it, ``None`` when the header is absent (non-proxy /
+    simulated) or unrecognised. This is the value that lands on
+    :attr:`LastCall.fallback` and the span, where ``False`` is a meaningful
+    observation distinct from ``None``."""
+    return _parse_fallback(response.headers.get(ROUTING_FALLBACK_HEADER))
+
+
+def is_fallback(response: httpx.Response) -> bool:
+    """True iff the gateway definitively marked this response as a routing
+    FALLBACK. Used by the transport's retry predicate: a response the gateway
+    already failed over must not be additionally retried by the SDK, or an outage
+    the gateway handled gets a second recovery layer stacked on top (#309, #183).
+    An absent or unrecognised header is *not* a fallback, so this definitive-
+    ``True``-only reading is safe to gate a retry decision on."""
+    return routing_fallback(response) is True
+
+
 # --- per-call usage token counts (#307, BG §1.3) ----------------------------
 # LIVE-VERIFIED (docs/verified-apis.md §3, ``responses.success.body.json``,
 # 2026-08-28): the gateway returns token counts in the response BODY's ``usage``
@@ -249,6 +298,22 @@ class LastCall:
     observed_at: datetime | None = None
     #: For UNAVAILABLE, the adapter surface(s) that cannot observe; else ``None``.
     surface: str | None = None
+    # --- gateway routing & fallback (§3, #309) -----------------------------
+    #: The model the caller ASKED for (from the request body). The reference
+    #: point for :attr:`substituted`; ``None`` when the request carried none.
+    requested_model: str | None = None
+    #: The provider the gateway actually routed to (``x-llm-proxy-llm-provider``).
+    served_provider: str | None = None
+    #: The model the gateway actually served (``x-llm-proxy-llm-model``). After a
+    #: routing fallback this may differ from :attr:`requested_model`.
+    served_model: str | None = None
+    #: The routing strategy the gateway applied (``x-llm-proxy-routing-type``,
+    #: e.g. ``"ModelBased"``).
+    routing_type: str | None = None
+    #: Whether the gateway performed a routing FALLBACK. ``None`` when the header
+    #: is absent (non-proxy / simulated response) — distinct from a definitive
+    #: ``False`` ("no fallback occurred").
+    fallback: bool | None = None
     # --- per-call usage token counts (#307), from the response BODY's ``usage``.
     # ``None`` (never ``0``) when unobserved or absent; on a stream they land once
     # the terminal SSE event is scanned (:func:`observe_usage`), not at record time.
@@ -270,6 +335,19 @@ def observed(self) -> bool:
         """True iff a governed response actually populated this record."""
         return self.status is LastCallStatus.OBSERVED
 
+    @property
+    def substituted(self) -> bool:
+        """True iff the gateway served a *different* model than the one requested
+        — a silent model substitution the developer's cost model, token
+        assumptions and evaluation are otherwise blind to (#309). Requires both
+        the requested and served model to be known; a missing either side is not
+        a substitution claim."""
+        return (
+            self.requested_model is not None
+            and self.served_model is not None
+            and self.requested_model != self.served_model
+        )
+
     @property
     def available(self) -> bool:
         """False only when the current surface structurally cannot be observed
@@ -278,13 +356,21 @@ def available(self) -> bool:
         return self.status is not LastCallStatus.UNAVAILABLE
 
     @classmethod
-    def from_response(cls, response: httpx.Response, *, now: datetime | None = None) -> LastCall:
+    def from_response(
+        cls,
+        response: httpx.Response,
+        *,
+        requested_model: str | None = None,
+        now: datetime | None = None,
+    ) -> LastCall:
         """Build an ``OBSERVED`` record from a governed response's §3 headers.
 
         Always ``OBSERVED`` — the SDK saw a response — even if the gateway
         carried no identity headers (each field then stays ``None``): "we saw the
         response, it said nothing" is a true and different statement from "we
-        never saw a response". ``now`` is injectable for tests."""
+        never saw a response". ``requested_model`` is the model the caller sent
+        (the transport reads it from the request body); it is the reference point
+        for :attr:`substituted`. ``now`` is injectable for tests."""
         api_instance_id, environment_id = _parse_decorator_operation(
             response.headers.get(DECORATOR_OPERATION_HEADER)
         )
@@ -295,6 +381,11 @@ def from_response(cls, response: httpx.Response, *, now: datetime | None = None)
             api_instance_id=api_instance_id,
             environment_id=environment_id,
             observed_at=now if now is not None else _utcnow(),
+            requested_model=requested_model,
+            served_provider=response.headers.get(LLM_PROVIDER_HEADER),
+            served_model=response.headers.get(LLM_MODEL_HEADER),
+            routing_type=response.headers.get(ROUTING_TYPE_HEADER),
+            fallback=_parse_fallback(response.headers.get(ROUTING_FALLBACK_HEADER)),
             input_tokens=usage["input_tokens"],
             output_tokens=usage["output_tokens"],
             total_tokens=usage["total_tokens"],
@@ -331,12 +422,18 @@ def current_last_call() -> LastCall | None:
     return _last_call.get()
 
 
-def observe_last_call(response: httpx.Response, *, now: datetime | None = None) -> LastCall:
-    """Record the gateway identity from a governed model response into this
-    context (§2.3, #362). Called from the transport's ``_on_response`` beside
-    :meth:`Budget.observe`. Returns the record it set, for tests. ``now`` is
-    injectable; production uses the wall clock (UTC)."""
-    record = LastCall.from_response(response, now=now)
+def observe_last_call(
+    response: httpx.Response,
+    *,
+    requested_model: str | None = None,
+    now: datetime | None = None,
+) -> LastCall:
+    """Record the gateway identity, routing and fallback from a governed model
+    response into this context (§2.3, #362/#309). Called from the transport's
+    ``_on_response`` beside :meth:`Budget.observe`, which passes the requested
+    model it already parsed from the request body. Returns the record it set, for
+    tests. ``now`` is injectable; production uses the wall clock (UTC)."""
+    record = LastCall.from_response(response, requested_model=requested_model, now=now)
     _last_call.set(record)
     return record
 

python/src/donkey_kit/core/telemetry.py

@@ -59,6 +59,12 @@
 # gen_ai.* — pinned to GEN_AI_SEMCONV_VERSION.
 GEN_AI_SYSTEM = "gen_ai.system"
 GEN_AI_REQUEST_MODEL = "gen_ai.request.model"
+# The model the gateway ACTUALLY served (``x-llm-proxy-llm-model``). Distinct
+# from ``gen_ai.request.model``: after a routing fallback the two differ, and the
+# semconv's ``gen_ai.response.model`` is exactly "the model that generated the
+# response" (#309). When latency spikes, seeing request≠response model on the
+# span is the single fastest read that a failover happened.
+GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"
 GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
 GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
 # Cached-input and reasoning-output token counts (#307). The GenAI semconv at
@@ -91,6 +97,13 @@
 DONKEY_COST_PROJECT = "donkey.cost.project"
 DONKEY_COST_ENV = "donkey.cost.env"
 DONKEY_COST_ENDUSER = "donkey.cost.enduser.id"
+# Gateway routing & resilience (§3, #309). The served provider already lands on
+# ``gen_ai.system`` and the served model on ``gen_ai.response.model``; these two
+# carry the gateway-specific routing facts the semconv has no key for. Emitted
+# even when ``fallback`` is ``False`` — "we routed normally" is a signal an
+# operator wants on every span, not just the failover ones.
+DONKEY_ROUTING_TYPE = "donkey.routing.type"
+DONKEY_ROUTING_FALLBACK = "donkey.routing.fallback"
 # Per-call usage token counts the semconv has no pinned key for (#307). Stable
 # public API, same as the other donkey.* keys — renaming one is a breaking change.
 DONKEY_USAGE_CACHED_TOKENS = "donkey.usage.cached_tokens"
@@ -119,6 +132,7 @@
     {
         GEN_AI_SYSTEM,
         GEN_AI_REQUEST_MODEL,
+        GEN_AI_RESPONSE_MODEL,
         GEN_AI_USAGE_INPUT_TOKENS,
         GEN_AI_USAGE_OUTPUT_TOKENS,
         DONKEY_USAGE_CACHED_TOKENS,
@@ -132,6 +146,8 @@
         DONKEY_COST_PROJECT,
         DONKEY_COST_ENV,
         DONKEY_COST_ENDUSER,
+        DONKEY_ROUTING_TYPE,
+        DONKEY_ROUTING_FALLBACK,
     }
 )
 
@@ -337,6 +353,9 @@ def build_genai_attributes(
     *,
     system: str | None = None,
     request_model: str | None = None,
+    response_model: str | None = None,
+    routing_type: str | None = None,
+    fallback: bool | None = None,
     input_tokens: int | None = None,
     output_tokens: int | None = None,
     cached_tokens: int | None = None,
@@ -372,6 +391,14 @@ def build_genai_attributes(
         attrs[GEN_AI_SYSTEM] = system
     if request_model is not None:
         attrs[GEN_AI_REQUEST_MODEL] = request_model
+    if response_model is not None:
+        attrs[GEN_AI_RESPONSE_MODEL] = response_model
+    if routing_type is not None:
+        attrs[DONKEY_ROUTING_TYPE] = routing_type
+    # ``fallback`` is emitted even when ``False`` — "no fallback" is a real,
+    # useful observation; only an absent header (``None``) is dropped (#309).
+    if fallback is not None:
+        attrs[DONKEY_ROUTING_FALLBACK] = fallback
     if input_tokens is not None:
         attrs[GEN_AI_USAGE_INPUT_TOKENS] = input_tokens
     if output_tokens is not None: