From 8958543b270798138811e3936eaafe6df204b092 Mon Sep 17 00:00:00 2001 From: Randy Date: Thu, 24 Sep 2026 18:06:34 +0000 Subject: [PATCH] Add supervision.decision-policy responsibility (decision-policy core, no MCP) Ports the decision-policy core from PR #14 onto the responsibility architecture: tighten-only abstention policy, ForemanModel.decide(), FOREMAN_DECIDED audit events, and a first-class DecisionPolicyResponsibility registered alongside the builtins with a packaged TOML definition. Drops all MCP stdio server code: no foreman mcp, no mcp package, no docs/mcp.md. The decision path is a single async seam, resolve_decision(), kept lifecycle-agnostic (no hard-coded BEFORE_TOOL); the hook point remains an open maintainer question. Abstention routes to the human via ESCALATE and never terminates the worker. --- .env.example | 4 + README.md | 6 + src/foreman/cli.py | 55 +++-- src/foreman/config.py | 29 +++ src/foreman/decision_policy.py | 79 +++++++ src/foreman/foreman/base.py | 11 +- src/foreman/foreman/jev.py | 199 +++++++++++++++++- src/foreman/foreman/simulation.py | 33 ++- src/foreman/models/__init__.py | 4 + src/foreman/models/decision.py | 60 ++++++ src/foreman/models/events.py | 1 + src/foreman/persistence.py | 30 ++- src/foreman/responsibilities/__init__.py | 10 + src/foreman/responsibilities/builtin.py | 14 ++ src/foreman/responsibilities/decision.py | 144 +++++++++++++ .../supervision.decision-policy.toml | 10 + tests/test_cli.py | 76 +++++++ tests/test_decision_policy.py | 126 +++++++++++ tests/test_decision_responsibility.py | 170 +++++++++++++++ tests/test_hooks.py | 1 + tests/test_persistence.py | 40 ++++ tests/test_responsibility_config.py | 3 +- 22 files changed, 1077 insertions(+), 28 deletions(-) create mode 100644 src/foreman/decision_policy.py create mode 100644 src/foreman/models/decision.py create mode 100644 src/foreman/responsibilities/decision.py create mode 100644 src/foreman/responsibilities/definitions/supervision.decision-policy.toml create mode 100644 tests/test_cli.py create mode 100644 tests/test_decision_policy.py create mode 100644 tests/test_decision_responsibility.py diff --git a/.env.example b/.env.example index 07a0b9d..84eee25 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,7 @@ TYPESAFE_API_KEY= # FOREMAN_STEERING_ENABLED=true # FOREMAN_MAX_STEERS_PER_WORKER=1 # FOREMAN_STEERING_GRACE_SECONDS=30 + +# Decision-policy responsibility configuration (baseline safety floor) +# FOREMAN_DECISION_THRESHOLD=0.70 +# FOREMAN_ALWAYS_ABSTAIN=destructive,credentials diff --git a/README.md b/README.md index 835ed3a..73b6b50 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,12 @@ exec backend. Live steering into an active turn is only available with the Codex backend; other backends degrade to stop/retry. See [Worker backends](docs/workers.md) for the `Worker` protocol and how to add your own. +### Decision policy + +The `supervision.decision-policy` responsibility lets `foreman serve` gate worker decisions +automatically: bounded decision models, a tighten-only abstention policy, and an audit trail +of every autonomous decision (`FOREMAN_DECIDED` events, rendered by `foreman inspect`). + ## What Foreman watches Each observation is compact and bounded. It contains: diff --git a/src/foreman/cli.py b/src/foreman/cli.py index 435b8b3..52ac6b1 100644 --- a/src/foreman/cli.py +++ b/src/foreman/cli.py @@ -4,7 +4,7 @@ import json import os import sys -from datetime import UTC +from datetime import UTC, datetime from pathlib import Path from typing import Annotated @@ -413,26 +413,42 @@ def inspect_run( store = RunStore(repo) try: - state = store.load_state(run_id) events = store.load_events(run_id) except PersistenceError as error: console.print(str(error)) raise typer.Exit(code=2) from error - finished = state.finished_at or state.updated_at - duration = max(0.0, (finished - state.started_at).total_seconds()) - console.print(f"[bold]FOREMAN RUN: {state.run_id}[/bold]") - console.print("Job:") - console.print(state.job) - console.print(f"Duration: {duration_label(duration)}") - console.print(f"Workers: {len(state.workers)}") - console.print(f"Evaluations: {len(state.result_history)}") - if state.active_responsibility_ids: - console.print(f"Responsibilities: {', '.join(state.active_responsibility_ids)}") - console.print(f"Result: {state.status.value}") + try: + state = store.load_state(run_id) + except PersistenceError: + state = None + + if state is not None: + finished = state.finished_at or state.updated_at + duration = max(0.0, (finished - state.started_at).total_seconds()) + console.print(f"[bold]FOREMAN RUN: {state.run_id}[/bold]") + console.print("Job:") + console.print(state.job) + console.print(f"Duration: {duration_label(duration)}") + console.print(f"Workers: {len(state.workers)}") + console.print(f"Evaluations: {len(state.result_history)}") + if state.active_responsibility_ids: + console.print(f"Responsibilities: {', '.join(state.active_responsibility_ids)}") + console.print(f"Result: {state.status.value}") + started_at = state.started_at + else: + started_at = events[0].timestamp if events else datetime.now(UTC) + duration = ( + max(0.0, (events[-1].timestamp - started_at).total_seconds()) + if len(events) > 1 + else 0.0 + ) + console.print(f"[bold]FOREMAN RUN: {run_id}[/bold]") + console.print(f"Duration: {duration_label(duration)}") + console.print(f"Events: {len(events)}") for event in events: - offset = (event.timestamp - state.started_at).total_seconds() + offset = (event.timestamp - started_at).total_seconds() prefix = elapsed_label(offset) payload = event.payload if event.event_type is EventType.FACTORY_STARTED: @@ -479,6 +495,17 @@ def inspect_run( console.print(f" {label:<16} {float(assessment[key]):.2f}") elif event.event_type is EventType.FOREMAN_INTERVENED: console.print(f" {payload.get('action')}: {payload.get('reason', '')}") + elif event.event_type is EventType.FOREMAN_DECIDED: + status = "answered" if payload.get("answered") else "abstained" + console.print(f"{prefix} Foreman decided: {status}") + if payload.get("question"): + q = str(payload.get("question")).replace("\n", " ") + console.print(f" question: {q[:80] + '...' if len(q) > 80 else q}") + console.print(f" choice: {payload.get('choice')}") + console.print(f" confidence: {float(payload.get('confidence', 0.0)):.2f}") + if payload.get("rationale"): + r = str(payload.get("rationale")).replace("\n", " ") + console.print(f" rationale: {r[:100] + '...' if len(r) > 100 else r}") elif event.event_type in {EventType.WORKER_STEERED, EventType.WORKER_STEER_FAILED}: label = "steered" if event.event_type is EventType.WORKER_STEERED else "steer failed" console.print(f"{prefix} {payload.get('worker_id')} {label}") diff --git a/src/foreman/config.py b/src/foreman/config.py index ecb84a3..8fbf9ee 100644 --- a/src/foreman/config.py +++ b/src/foreman/config.py @@ -6,6 +6,8 @@ from pydantic import BaseModel, ConfigDict, Field +from foreman.models import AbstainCategory + def _environment_bool(value: str) -> bool: normalized = value.strip().lower() @@ -16,6 +18,24 @@ def _environment_bool(value: str) -> bool: raise ValueError(f"invalid boolean environment value: {value!r}") +def _environment_abstain_categories(value: str) -> list[AbstainCategory]: + """Parse a comma-separated FOREMAN_ALWAYS_ABSTAIN value.""" + + categories: list[AbstainCategory] = [] + for raw in value.split(","): + name = raw.strip().lower() + if not name: + continue + try: + categories.append(AbstainCategory(name)) + except ValueError as error: + valid = ", ".join(category.value for category in AbstainCategory) + raise ValueError( + f"invalid abstain category {raw!r}; expected one of: {valid}" + ) from error + return categories + + class FactoryConfig(BaseModel): model_config = ConfigDict(extra="forbid") @@ -35,6 +55,13 @@ class FactoryConfig(BaseModel): steering_enabled: bool = True max_steers_per_worker: int = Field(default=1, ge=0) steering_grace_seconds: float = Field(default=30.0, ge=0.0) + decision_threshold: float = Field(default=0.70, ge=0.0, le=1.0) + always_abstain: list[AbstainCategory] = Field( + default_factory=lambda: [ + AbstainCategory.DESTRUCTIVE, + AbstainCategory.CREDENTIALS, + ] + ) diff_limit: int = Field(default=20_000, ge=100) output_limit: int = Field(default=12_000, ge=100) @@ -63,6 +90,8 @@ def from_environment(cls) -> FactoryConfig: int, ), "FOREMAN_CODEX_BACKEND": ("codex_backend", str), + "FOREMAN_DECISION_THRESHOLD": ("decision_threshold", float), + "FOREMAN_ALWAYS_ABSTAIN": ("always_abstain", _environment_abstain_categories), "FOREMAN_STEERING_ENABLED": ( "steering_enabled", _environment_bool, diff --git a/src/foreman/decision_policy.py b/src/foreman/decision_policy.py new file mode 100644 index 0000000..9707d56 --- /dev/null +++ b/src/foreman/decision_policy.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from collections.abc import Collection, Sequence +from dataclasses import dataclass + +from foreman.config import FactoryConfig +from foreman.models import AbstainCategory, Decision, DecisionRequest + + +@dataclass(frozen=True, slots=True) +class EffectivePolicy: + """The merged safety policy for one decision request. + + The caller's request can only tighten the baseline: the effective + threshold is the maximum of the two, and the effective denylist is the + union of both. A caller can never loosen foreman's safety floor. + """ + + threshold: float + denylist: frozenset[AbstainCategory] + + +@dataclass(frozen=True, slots=True) +class PolicyOutcome: + answered: bool + choice: str | None + rationale: str + + +def effective_policy(config: FactoryConfig, request: DecisionRequest) -> EffectivePolicy: + """Merge the configured baseline with one request, tighten-only.""" + + requested = request.min_confidence if request.min_confidence is not None else 0.0 + return EffectivePolicy( + threshold=max(config.decision_threshold, requested), + denylist=frozenset(config.always_abstain) | frozenset(request.extra_abstain_categories), + ) + + +def apply_policy( + decision: Decision, + *, + threshold: float, + denylist: Collection[AbstainCategory], + options: Sequence[str], +) -> PolicyOutcome: + """Decide whether the model's verdict may be returned as an answer. + + Anything short of a confident, valid, denylist-clean verdict becomes an + abstention: the question goes back to the human, and nothing is + terminated or escalated. + """ + + problems: list[str] = [] + if decision.abstained: + problems.append("the foreman model abstained") + if decision.confidence < threshold: + problems.append( + f"confidence {decision.confidence:.2f} is below the {threshold:.2f} threshold" + ) + if decision.choice is None or decision.choice not in options: + problems.append("the choice is not one of the offered options") + denied = set(decision.classification) & set(denylist) + if denied: + names = ", ".join(sorted(category.value for category in denied)) + problems.append(f"the question is classified as {names}, which must abstain") + if problems: + return PolicyOutcome( + answered=False, + choice=None, + rationale=( + "Abstained: " + "; ".join(problems) + f". Model rationale: {decision.rationale}" + ), + ) + return PolicyOutcome( + answered=True, + choice=decision.choice, + rationale=decision.rationale, + ) diff --git a/src/foreman/foreman/base.py b/src/foreman/foreman/base.py index 1472aa2..6957f7d 100644 --- a/src/foreman/foreman/base.py +++ b/src/foreman/foreman/base.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from typing import Protocol -from foreman.models import ForemanResult +from foreman.models import Decision, DecisionRequest, ForemanResult from foreman.observation import FactoryObservation from foreman.responsibilities import Check @@ -19,4 +19,13 @@ async def assess( checks: Sequence[Check], ) -> ForemanResult: ... + async def decide(self, request: DecisionRequest) -> Decision: + """Answer one multiple-choice question or abstain. + + The returned Decision is the model's raw judgment; the decision + policy applies the configured confidence threshold and abstain + denylist before any answer reaches the caller. + """ + ... + async def close(self) -> None: ... diff --git a/src/foreman/foreman/jev.py b/src/foreman/foreman/jev.py index 4e84b45..9101090 100644 --- a/src/foreman/foreman/jev.py +++ b/src/foreman/foreman/jev.py @@ -6,7 +6,7 @@ from typing import Any from foreman.foreman.base import ForemanModelError -from foreman.models import ForemanResult +from foreman.models import AbstainCategory, Decision, DecisionRequest, ForemanResult from foreman.observation import FactoryObservation from foreman.responsibilities import Check @@ -56,6 +56,135 @@ def parse_jev_response(response: Any, checks: Sequence[Check]) -> ForemanResult: return normalize_check_results(values, checks) +ABSTAIN_CATEGORY_QUESTIONS: dict[AbstainCategory, str] = { + AbstainCategory.DESTRUCTIVE: ( + "Would answering this question, or acting on the answer, delete or corrupt " + "data or state (removing files, dropping databases, force-pushing, tearing " + "down infrastructure)?" + ), + AbstainCategory.IRREVERSIBLE: ( + "Would answering this question, or acting on the answer, do something that " + "cannot be taken back (publishing a release, sending notifications or " + "email, merging)?" + ), + AbstainCategory.EXTERNAL: ( + "Would answering this question, or acting on the answer, cause side effects " + "beyond the local machine (mutating remote APIs, deployments, webhooks, " + "anything cost-incurring, exfiltration of code or data to third parties)?" + ), + AbstainCategory.CREDENTIALS: ( + "Would answering this question, or acting on the answer, touch " + "authentication material (reading or using tokens and secrets, granting " + "permissions, signing artifacts)?" + ), +} + +SHOULD_ABSTAIN_INSTRUCTIONS = ( + "Should this question be left to the human instead of answered autonomously? " + "Answer yes when the question is ambiguous, the options are all poor, the " + "stakes are high, or the right answer depends on judgment only the human has." +) + +_SHOULD_ABSTAIN_NAME = "should_abstain" +_CLASSIFICATION_THRESHOLD = 0.5 + + +def _decision_option_name(index: int) -> str: + return f"option_{index}" + + +def _decision_risk_name(category: AbstainCategory) -> str: + return f"risk_{category.value}" + + +def _extract_noul_score(response: Any, name: str) -> float: + """Read one Noul probability from an SDK response or a test double.""" + + raw: Any = None + nouls = getattr(response, "nouls", None) + if nouls is not None: + answer = nouls.get(name) + if answer is not None: + raw = getattr(answer, "noul", None) + elif isinstance(response, Mapping): + answers = response.get("answers", response) + if isinstance(answers, Mapping): + answer = answers.get(name) + if isinstance(answer, Mapping): + raw = answer.get("noul") + elif answer is not None: + raw = answer + if isinstance(raw, bool) or not isinstance(raw, (int, float)): + raise ForemanModelError(f"Jev decision answer {name!r} is not numeric") + value = float(raw) + if not isfinite(value): + raise ForemanModelError(f"Jev decision answer {name!r} is not finite") + return min(1.0, max(0.0, value)) + + +def build_decision( + *, + options: list[str], + option_scores: list[float], + risk_scores: dict[AbstainCategory, float], + abstain_score: float, +) -> Decision: + """Translate raw Noul scores into a Decision. + + The model classifies the question, picks the best-scoring option, and may + abstain outright. The decision policy still applies the configured + confidence threshold and denylist afterwards. + """ + + classification = [ + category + for category in AbstainCategory + if risk_scores[category] >= _CLASSIFICATION_THRESHOLD + ] + best_index = max(range(len(options)), key=lambda index: option_scores[index]) + scored = ", ".join( + f"{option!r}={score:.2f}" for option, score in zip(options, option_scores, strict=True) + ) + if abstain_score >= _CLASSIFICATION_THRESHOLD: + return Decision( + choice=None, + confidence=abstain_score, + rationale=( + f"abstained (should_abstain={abstain_score:.2f}); best option " + f"{options[best_index]!r} scored {option_scores[best_index]:.2f}; " + f"options: {scored}" + ), + classification=classification, + abstained=True, + ) + return Decision( + choice=options[best_index], + confidence=option_scores[best_index], + rationale=( + f"chose {options[best_index]!r} " + f"(score {option_scores[best_index]:.2f}); options: {scored}" + ), + classification=classification, + abstained=False, + ) + + +def _noul_type(client: Any | None) -> Any: + """Return the SDK Noul class, or a test-double stand-in when injected.""" + + try: + from typesafe_sdk import Noul + except ImportError as error: + if client is None: + raise ForemanModelError("typesafe-sdk is not installed") from error + + class Noul: # type: ignore[no-redef] + def __init__(self, *, instructions: str) -> None: + self.instructions = instructions + + return Noul + + class JevForemanModel: """Adapter for TypeSafe's official asynchronous Python SDK.""" @@ -94,15 +223,7 @@ async def assess( observation: FactoryObservation, checks: Sequence[Check], ) -> ForemanResult: - try: - from typesafe_sdk import Noul - except ImportError as error: - if self._client is None: - raise ForemanModelError("typesafe-sdk is not installed") from error - - class Noul: # type: ignore[no-redef] - def __init__(self, *, instructions: str) -> None: - self.instructions = instructions + Noul = _noul_type(self._client) # One API call evaluates every active responsibility's checks in parallel. questions = {check.key: Noul(instructions=check.instructions) for check in checks} @@ -128,6 +249,64 @@ def __init__(self, *, instructions: str) -> None: message = f"Jev assessment failed: {type(error).__name__}: {error}" raise ForemanModelError(message) from error + async def decide(self, request: DecisionRequest) -> Decision: + """Classify the question, pick the best option, or abstain. + + One API call scores every option, every abstain category, and an + explicit should-abstain question in parallel. The decision policy + applies the configured confidence threshold and denylist afterwards. + """ + + Noul = _noul_type(self._client) + questions = { + _decision_option_name(index): Noul( + instructions=( + f"Is {option!r} the best answer to the question? Consider the " + "question, the other options, and the context." + ) + ) + for index, option in enumerate(request.options) + } + for category, instructions in ABSTAIN_CATEGORY_QUESTIONS.items(): + questions[_decision_risk_name(category)] = Noul(instructions=instructions) + questions[_SHOULD_ABSTAIN_NAME] = Noul(instructions=SHOULD_ABSTAIN_INSTRUCTIONS) + + client = self._client or self._make_client() + if self._client is None: + self._client = client + try: + response = await asyncio.wait_for( + client.system_one( + state=request.model_dump(mode="json"), + questions=questions, + model=self.model, + timeout=self.timeout_seconds, + ), + timeout=self.timeout_seconds + 0.5, + ) + option_scores = [ + _extract_noul_score(response, _decision_option_name(index)) + for index in range(len(request.options)) + ] + risk_scores = { + category: _extract_noul_score(response, _decision_risk_name(category)) + for category in AbstainCategory + } + abstain_score = _extract_noul_score(response, _SHOULD_ABSTAIN_NAME) + return build_decision( + options=request.options, + option_scores=option_scores, + risk_scores=risk_scores, + abstain_score=abstain_score, + ) + except TimeoutError as error: + raise ForemanModelError("Jev decision timed out") from error + except ForemanModelError: + raise + except Exception as error: + message = f"Jev decision failed: {type(error).__name__}: {error}" + raise ForemanModelError(message) from error + async def close(self) -> None: if self._owns_client and self._client is not None: close = getattr(self._client, "aclose", None) diff --git a/src/foreman/foreman/simulation.py b/src/foreman/foreman/simulation.py index 8cb8469..ac743da 100644 --- a/src/foreman/foreman/simulation.py +++ b/src/foreman/foreman/simulation.py @@ -3,7 +3,7 @@ import asyncio from collections.abc import Sequence -from foreman.models import FactoryAssessment, ForemanResult +from foreman.models import Decision, DecisionRequest, FactoryAssessment, ForemanResult from foreman.observation import FactoryObservation from foreman.responsibilities import Check @@ -67,6 +67,7 @@ def __init__( self, assessments: Sequence[ForemanResult | FactoryAssessment] | None = None, *, + decisions: Sequence[Decision] | None = None, delay_seconds: float = 0.0, repeat_last: bool = True, ) -> None: @@ -76,10 +77,13 @@ def __init__( ] if not self.results: raise ValueError("at least one fake result is required") + self.decisions = list(decisions or []) self.delay_seconds = delay_seconds self.repeat_last = repeat_last self.calls: list[FactoryObservation] = [] + self.decision_calls: list[DecisionRequest] = [] self._index = 0 + self._decision_index = 0 async def assess( self, observation: FactoryObservation, checks: Sequence[Check] @@ -98,3 +102,30 @@ async def assess( async def close(self) -> None: return None + + async def decide(self, request: DecisionRequest) -> Decision: + """Return a queued decision, or a deterministic default. + + Without queued decisions the fake answers with the first option at + high confidence, so offline tests exercise the answered path unless + they say otherwise. + """ + + self.decision_calls.append(request) + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + if self.decisions: + if self._decision_index >= len(self.decisions): + if not self.repeat_last: + raise RuntimeError("fake decision sequence exhausted") + return self.decisions[-1].model_copy(deep=True) + decision = self.decisions[self._decision_index].model_copy(deep=True) + self._decision_index += 1 + return decision + return Decision( + choice=request.options[0], + confidence=0.95, + rationale="deterministic fake decision", + classification=[], + abstained=False, + ) diff --git a/src/foreman/models/__init__.py b/src/foreman/models/__init__.py index 3366c87..8ec46cc 100644 --- a/src/foreman/models/__init__.py +++ b/src/foreman/models/__init__.py @@ -1,10 +1,14 @@ from foreman.models.assessment import FactoryAssessment +from foreman.models.decision import AbstainCategory, Decision, DecisionRequest from foreman.models.events import Directive, EventType, FactoryEvent, Intervention, InterventionType from foreman.models.result import ForemanResult from foreman.models.state import FactoryState, FactoryStatus, VerificationResult from foreman.models.worker import WorkerRecord, WorkerStatus, WorkerType __all__ = [ + "AbstainCategory", + "Decision", + "DecisionRequest", "EventType", "Directive", "FactoryAssessment", diff --git a/src/foreman/models/decision.py b/src/foreman/models/decision.py new file mode 100644 index 0000000..3169eba --- /dev/null +++ b/src/foreman/models/decision.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class AbstainCategory(StrEnum): + """Question categories the foreman must never answer on its own. + + These form the safety floor for autonomous decisions: a question + classified into a denylisted category is handed back to the human + instead of being answered, no matter how confident the model is. + """ + + DESTRUCTIVE = "destructive" + """Deletes or corrupts data or state: removing files, dropping databases, + force-pushing, tearing down infrastructure.""" + + IRREVERSIBLE = "irreversible" + """Cannot be taken back once done, even if nothing is destroyed: + publishing a release, sending notifications or email, merging.""" + + EXTERNAL = "external" + """Side effects beyond the local machine: mutating remote APIs, + deployments, webhooks, anything cost-incurring, and exfiltration of + code or data to third parties.""" + + CREDENTIALS = "credentials" + """Touches authentication material: reading or using tokens and secrets, + granting permissions, signing artifacts.""" + + +class DecisionRequest(BaseModel): + """A single question handed to the foreman for an autonomous decision.""" + + model_config = ConfigDict(extra="forbid") + + question: str = Field(min_length=1, max_length=10_000) + options: list[str] = Field(min_length=1, max_length=20) + context: str = Field(default="", max_length=20_000) + risk_hint: str = Field(default="", max_length=2_000) + extra_abstain_categories: list[AbstainCategory] = Field(default_factory=list) + min_confidence: float | None = Field(default=None, ge=0.0, le=1.0) + + +class Decision(BaseModel): + """The foreman model's raw verdict on a DecisionRequest. + + This is the model's judgment only; the decision policy applies the + configured threshold and denylist before an answer is returned. + """ + + model_config = ConfigDict(extra="forbid") + + choice: str | None = Field(default=None, max_length=5_000) + confidence: float = Field(ge=0.0, le=1.0) + rationale: str = Field(min_length=1, max_length=5_000) + classification: list[AbstainCategory] = Field(default_factory=list) + abstained: bool = False diff --git a/src/foreman/models/events.py b/src/foreman/models/events.py index 7d7f750..d1d29e9 100644 --- a/src/foreman/models/events.py +++ b/src/foreman/models/events.py @@ -24,6 +24,7 @@ class EventType(StrEnum): FOREMAN_OBSERVED = "FOREMAN_OBSERVED" FOREMAN_ASSESSED = "FOREMAN_ASSESSED" FOREMAN_INTERVENED = "FOREMAN_INTERVENED" + FOREMAN_DECIDED = "FOREMAN_DECIDED" VERIFICATION_STARTED = "VERIFICATION_STARTED" VERIFICATION_COMPLETED = "VERIFICATION_COMPLETED" FACTORY_FINISHED = "FACTORY_FINISHED" diff --git a/src/foreman/persistence.py b/src/foreman/persistence.py index 7fd30f4..7d137f2 100644 --- a/src/foreman/persistence.py +++ b/src/foreman/persistence.py @@ -8,7 +8,7 @@ from pydantic import ValidationError -from foreman.models import FactoryAssessment, FactoryEvent, FactoryState +from foreman.models import EventType, FactoryAssessment, FactoryEvent, FactoryState class PersistenceError(RuntimeError): @@ -151,6 +151,34 @@ def list_states(self) -> Iterable[FactoryState]: continue return states + def list_run_ids(self) -> list[str]: + if not self.runs_dir.exists(): + return [] + paths = sorted( + [ + path + for path in self.runs_dir.iterdir() + if path.is_dir() and not path.name.startswith(".") + ], + key=lambda item: (item.stat().st_mtime, item.name), + reverse=True, + ) + return [path.name for path in paths] + + def load_decision_events(self, run_id: str | None = None) -> list[FactoryEvent]: + """Load FOREMAN_DECIDED events for one run or across all runs in this store.""" + target_ids = [run_id] if run_id is not None else self.list_run_ids() + events: list[FactoryEvent] = [] + for rid in target_ids: + try: + run_events = self.load_events(rid, tolerate_malformed=True) + except PersistenceError: + continue + for event in run_events: + if event.event_type is EventType.FOREMAN_DECIDED: + events.append(event) + return sorted(events, key=lambda event: event.timestamp) + def recent_event_dicts(self, run_id: str, limit: int) -> list[dict[str, object]]: events = self.load_events(run_id, tolerate_malformed=True)[-limit:] return [json.loads(event.model_dump_json()) for event in events] diff --git a/src/foreman/responsibilities/__init__.py b/src/foreman/responsibilities/__init__.py index 2031f8a..e3d127a 100644 --- a/src/foreman/responsibilities/__init__.py +++ b/src/foreman/responsibilities/__init__.py @@ -6,6 +6,7 @@ ) from foreman.responsibilities.builtin import ( COMPLETION, + DECISION_POLICY, DOCUMENTATION, HUMAN_ESCALATION, REPOSITORY_INSTRUCTIONS, @@ -27,9 +28,16 @@ load_responsibility_configs, responsibility_config_dir, ) +from foreman.responsibilities.decision import ( + DECISION_REQUIRED, + DecisionPolicyResponsibility, + DecisionResolution, +) __all__ = [ "COMPLETION", + "DECISION_POLICY", + "DECISION_REQUIRED", "DOCUMENTATION", "HUMAN_ESCALATION", "REPOSITORY_INSTRUCTIONS", @@ -38,6 +46,8 @@ "Check", "CheckFileConfig", "CompletionResponsibility", + "DecisionPolicyResponsibility", + "DecisionResolution", "DocumentationResponsibility", "HumanEscalationResponsibility", "RepositoryInstructionsResponsibility", diff --git a/src/foreman/responsibilities/builtin.py b/src/foreman/responsibilities/builtin.py index b1ae129..c07cdf9 100644 --- a/src/foreman/responsibilities/builtin.py +++ b/src/foreman/responsibilities/builtin.py @@ -16,6 +16,7 @@ REPOSITORY_INSTRUCTIONS = "repository.instructions" HUMAN_ESCALATION = "core.human-escalation" DOCUMENTATION = "quality.documentation" +DECISION_POLICY = "supervision.decision-policy" @dataclass(slots=True, kw_only=True) @@ -334,6 +335,10 @@ def directives(self, state: FactoryState, result: ForemanResult) -> list[Directi HUMAN_ESCALATION: {}, REPOSITORY_INSTRUCTIONS: {}, DOCUMENTATION: {}, + DECISION_POLICY: { + "decision_threshold": "decision_threshold", + "always_abstain": "always_abstain", + }, WORKER_HEALTH: { "steering_enabled": "steering_enabled", "max_steers_per_worker": "max_steers_per_worker", @@ -397,6 +402,10 @@ def builtin_registry( all_checks = tuple( check for configured_checks in checks.values() for check in configured_checks ) + # Imported here to avoid a module-level cycle: decision.py builds on the + # builtin responsibility base, and builtin_registry is its only consumer. + from foreman.responsibilities.decision import DecisionPolicyResponsibility + responsibilities = [ HumanEscalationResponsibility( _responsibility_config(config, HUMAN_ESCALATION, configured.get(HUMAN_ESCALATION, {})) @@ -433,5 +442,10 @@ def builtin_registry( ) .configured_checks(checks.get(VERIFICATION, ())) .configured_minimums(all_checks), + DecisionPolicyResponsibility( + _responsibility_config(config, DECISION_POLICY, configured.get(DECISION_POLICY, {})) + ) + .configured_checks(checks.get(DECISION_POLICY, ())) + .configured_minimums(all_checks), ] return ResponsibilityRegistry(responsibilities, routes=routes) diff --git a/src/foreman/responsibilities/decision.py b/src/foreman/responsibilities/decision.py new file mode 100644 index 0000000..33ee98b --- /dev/null +++ b/src/foreman/responsibilities/decision.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar + +from foreman.config import FactoryConfig +from foreman.decision_policy import PolicyOutcome, apply_policy, effective_policy +from foreman.models import ( + Decision, + DecisionRequest, + Directive, + EventType, + FactoryEvent, + FactoryState, + ForemanResult, + InterventionType, +) +from foreman.responsibilities.base import ResponsibilityRoute +from foreman.responsibilities.builtin import ( + DECISION_POLICY, + _CheckConfiguredResponsibility, +) + +if TYPE_CHECKING: + from foreman.foreman.base import ForemanModel + from foreman.persistence import RunStore + +DECISION_REQUIRED = "decision-required" + +# Abstaining routes the question back to the human. It never terminates the +# worker, so the only directive this responsibility may propose is ESCALATE. +_ABSTAIN_PRIORITY = 950 + + +@dataclass(frozen=True, slots=True) +class DecisionResolution: + """The outcome of one trip through the decision path. + + `directive` is an ESCALATE proposal when the policy abstained (the human + must answer), and None when the model answered within policy. It is never + a terminating directive: abstention routes to the human, it never stops + the worker. + """ + + outcome: PolicyOutcome + decision: Decision + directive: Directive | None + + +@dataclass(slots=True) +class DecisionPolicyResponsibility(_CheckConfiguredResponsibility): + """Serve-side decision gating as a first-class responsibility. + + The synchronous ``directives()`` protocol method stays a no-op: the + decision path needs the foreman model, which only arrives with the + request at the lifecycle-hook seam. The hook seam is a single call site, + :meth:`resolve_decision`, which runs model verdict -> tighten-only + policy gate -> FOREMAN_DECIDED audit, and returns an ESCALATE directive + when the policy abstains. + + The ``decision-required`` check still participates in every Jev + assessment, so ``result.check("supervision.decision-policy", + "decision-required")`` tells serve whether a decision is on the table. + """ + + config: FactoryConfig + id: str = DECISION_POLICY + required_check_ids: ClassVar[frozenset[str]] = frozenset({DECISION_REQUIRED}) + required_minimum_keys: ClassVar[frozenset[str]] = frozenset( + {f"{DECISION_POLICY}__{DECISION_REQUIRED}"} + ) + + def route(self) -> ResponsibilityRoute: + return ResponsibilityRoute(always=True) + + def decision_armed(self, result: ForemanResult) -> bool: + """True when the assessment says a worker decision needs gating.""" + + return result.probability(self.id, DECISION_REQUIRED) >= self.minimum( + self.id, DECISION_REQUIRED + ) + + async def resolve_decision( + self, + state: FactoryState, + model: ForemanModel, + request: DecisionRequest, + *, + store: RunStore | None = None, + run_id: str = "", + ) -> DecisionResolution: + """The single call site for the serve decision path. + + Runs the model verdict through the tighten-only policy, appends the + FOREMAN_DECIDED audit event, and proposes an ESCALATE directive on + abstain so the question routes to the human. Answered decisions + carry no directive. Never terminates the worker. + """ + + decision = await model.decide(request) + policy = effective_policy(self.config, request) + outcome = apply_policy( + decision, + threshold=policy.threshold, + denylist=policy.denylist, + options=request.options, + ) + if store is not None and run_id: + store.append_event( + FactoryEvent( + run_id=run_id, + event_type=EventType.FOREMAN_DECIDED, + payload={ + "question": request.question, + "options": request.options, + "classification": [category.value for category in decision.classification], + "choice": decision.choice, + "confidence": decision.confidence, + "answered": outcome.answered, + "rationale": outcome.rationale, + "effective_threshold": policy.threshold, + "effective_denylist": sorted( + category.value for category in policy.denylist + ), + }, + ) + ) + directive = None + if not outcome.answered: + directive = Directive( + action=InterventionType.ESCALATE, + reason=f"decision abstained, routing to human: {request.question[:120]}", + assessment_iteration=max(1, state.iteration), + responsibility_id=self.id, + priority=_ABSTAIN_PRIORITY, + confidence=decision.confidence, + ) + return DecisionResolution(outcome=outcome, decision=decision, directive=directive) + + def directives(self, state: FactoryState, result: ForemanResult) -> Sequence[Directive]: + # The decision path is armed asynchronously through resolve_decision() + # from the lifecycle-hook seam; nothing is proposed synchronously here. + return [] diff --git a/src/foreman/responsibilities/definitions/supervision.decision-policy.toml b/src/foreman/responsibilities/definitions/supervision.decision-policy.toml new file mode 100644 index 0000000..e88d3f8 --- /dev/null +++ b/src/foreman/responsibilities/definitions/supervision.decision-policy.toml @@ -0,0 +1,10 @@ +enabled = true +always = true + +[checks.decision-required] +instructions = """ +Is a worker decision required before proceeding — a multiple-choice question +where the foreman could answer autonomously within policy, or must defer to +the human? +""" +min_threshold = 0.70 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..4cc6b63 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from pathlib import Path + +from typer.testing import CliRunner + +from foreman.cli import app +from foreman.models import EventType, FactoryEvent, FactoryState +from foreman.persistence import RunStore + +runner = CliRunner() + + +def test_inspect_factory_run_with_state(tmp_path: Path, state: FactoryState) -> None: + store = RunStore(tmp_path) + store.initialize(state) + event = FactoryEvent( + run_id=state.run_id, + event_type=EventType.FACTORY_STARTED, + payload={}, + ) + store.append_event(event) + + result = runner.invoke(app, ["inspect", state.run_id, "--repo", str(tmp_path)]) + assert result.exit_code == 0 + assert f"FOREMAN RUN: {state.run_id}" in result.output + assert "Factory started" in result.output + + +def test_inspect_decided_run_without_state(tmp_path: Path) -> None: + store = RunStore(tmp_path) + event = FactoryEvent( + run_id="serve-session-123", + event_type=EventType.FOREMAN_DECIDED, + payload={ + "question": "Use join or subquery?", + "answered": True, + "choice": "join", + "confidence": 0.88, + "rationale": "chose join", + }, + ) + store.append_event(event) + + result = runner.invoke(app, ["inspect", "serve-session-123", "--repo", str(tmp_path)]) + assert result.exit_code == 0 + assert "FOREMAN RUN: serve-session-123" in result.output + assert "Foreman decided: answered" in result.output + assert "choice: join" in result.output + assert "confidence: 0.88" in result.output + + +def test_inspect_abstained_decision(tmp_path: Path) -> None: + store = RunStore(tmp_path) + event = FactoryEvent( + run_id="serve-session-124", + event_type=EventType.FOREMAN_DECIDED, + payload={ + "question": "Should we force push to main?", + "answered": False, + "choice": None, + "confidence": 0.50, + "rationale": "Abstained: classified as destructive, which must abstain.", + }, + ) + store.append_event(event) + + result = runner.invoke(app, ["inspect", "serve-session-124", "--repo", str(tmp_path)]) + assert result.exit_code == 0 + assert "Foreman decided: abstained" in result.output + + +def test_inspect_nonexistent_run(tmp_path: Path) -> None: + result = runner.invoke(app, ["inspect", "does-not-exist", "--repo", str(tmp_path)]) + assert result.exit_code == 2 + assert "not found" in result.output diff --git a/tests/test_decision_policy.py b/tests/test_decision_policy.py new file mode 100644 index 0000000..053b276 --- /dev/null +++ b/tests/test_decision_policy.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from foreman.config import FactoryConfig +from foreman.decision_policy import EffectivePolicy, apply_policy, effective_policy +from foreman.models import AbstainCategory, Decision, DecisionRequest + + +def _request(**overrides) -> DecisionRequest: + fields: dict = {"question": "Which approach?", "options": ["a", "b"]} + fields.update(overrides) + return DecisionRequest(**fields) + + +def _decision(**overrides) -> Decision: + fields: dict = { + "choice": "a", + "confidence": 0.9, + "rationale": "a is best", + "classification": [], + "abstained": False, + } + fields.update(overrides) + return Decision(**fields) + + +def _config(**overrides) -> FactoryConfig: + fields: dict = { + "decision_threshold": 0.8, + "always_abstain": [AbstainCategory.DESTRUCTIVE], + } + fields.update(overrides) + return FactoryConfig(**fields) + + +def test_effective_policy_defaults_to_baseline(): + policy = effective_policy(_config(), _request()) + assert policy == EffectivePolicy( + threshold=0.8, denylist=frozenset({AbstainCategory.DESTRUCTIVE}) + ) + + +def test_effective_policy_request_can_raise_threshold_but_not_lower_it(): + raised = effective_policy(_config(), _request(min_confidence=0.95)) + assert raised.threshold == 0.95 + lowered = effective_policy(_config(), _request(min_confidence=0.2)) + assert lowered.threshold == 0.8 + + +def test_effective_policy_denylist_is_union_and_never_shrinks(): + policy = effective_policy( + _config(), _request(extra_abstain_categories=[AbstainCategory.EXTERNAL]) + ) + assert policy.denylist == frozenset( + {AbstainCategory.DESTRUCTIVE, AbstainCategory.EXTERNAL} + ) + # No way for the caller to remove a baseline category. + tight = effective_policy(_config(always_abstain=list(AbstainCategory)), _request()) + assert tight.denylist == frozenset(AbstainCategory) + + +def test_apply_policy_answers_confident_clean_decision(): + outcome = apply_policy( + _decision(), + threshold=0.8, + denylist={AbstainCategory.DESTRUCTIVE}, + options=["a", "b"], + ) + assert outcome.answered is True + assert outcome.choice == "a" + assert outcome.rationale == "a is best" + + +def test_apply_policy_abstains_when_model_abstained(): + outcome = apply_policy( + _decision(abstained=True, choice=None, confidence=0.99), + threshold=0.8, + denylist=set(), + options=["a", "b"], + ) + assert outcome.answered is False + assert outcome.choice is None + assert "abstained" in outcome.rationale + + +def test_apply_policy_abstains_below_threshold(): + outcome = apply_policy( + _decision(confidence=0.79), + threshold=0.8, + denylist=set(), + options=["a", "b"], + ) + assert outcome.answered is False + assert "0.79" in outcome.rationale + + +def test_apply_policy_rejects_choice_not_in_options(): + for bad in (None, "c"): + outcome = apply_policy( + _decision(choice=bad), + threshold=0.8, + denylist=set(), + options=["a", "b"], + ) + assert outcome.answered is False + assert "not one of the offered options" in outcome.rationale + + +def test_apply_policy_abstains_on_denylisted_classification(): + outcome = apply_policy( + _decision(classification=[AbstainCategory.CREDENTIALS], confidence=0.99), + threshold=0.8, + denylist={AbstainCategory.CREDENTIALS}, + options=["a", "b"], + ) + assert outcome.answered is False + assert "credentials" in outcome.rationale + + +def test_apply_policy_ignores_classification_not_denylisted(): + outcome = apply_policy( + _decision(classification=[AbstainCategory.EXTERNAL]), + threshold=0.8, + denylist={AbstainCategory.DESTRUCTIVE}, + options=["a", "b"], + ) + assert outcome.answered is True diff --git a/tests/test_decision_responsibility.py b/tests/test_decision_responsibility.py new file mode 100644 index 0000000..facec6b --- /dev/null +++ b/tests/test_decision_responsibility.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from foreman.config import FactoryConfig +from foreman.foreman import FakeForemanModel +from foreman.models import ( + AbstainCategory, + Decision, + DecisionRequest, + EventType, + FactoryState, + ForemanResult, + InterventionType, +) +from foreman.persistence import RunStore +from foreman.responsibilities import ( + DECISION_POLICY, + DecisionPolicyResponsibility, + builtin_registry, +) + + +def _responsibility(config: FactoryConfig | None = None) -> DecisionPolicyResponsibility: + config = config or FactoryConfig() + registry = builtin_registry(config) + found = next(r for r in registry.responsibilities if r.id == DECISION_POLICY) + assert isinstance(found, DecisionPolicyResponsibility) + return found + + +def _request(**overrides) -> DecisionRequest: + payload = { + "question": "Use join or subquery?", + "options": ["join", "subquery"], + } + payload.update(overrides) + return DecisionRequest(**payload) + + +def _result(score: float) -> ForemanResult: + return ForemanResult(checks={DECISION_POLICY: {"decision-required": score}}) + + +def test_registered_in_builtin_registry() -> None: + registry = builtin_registry(FactoryConfig()) + ids = [responsibility.id for responsibility in registry.responsibilities] + assert DECISION_POLICY == "supervision.decision-policy" + assert DECISION_POLICY in ids + + check = next(check for check in registry.checks() if check.responsibility_id == DECISION_POLICY) + assert check.check_id == "decision-required" + assert check.min_threshold == pytest.approx(0.70) + assert registry.route_for(DECISION_POLICY).always is True + + +def test_directives_is_synchronous_noop(state: FactoryState) -> None: + responsibility = _responsibility() + assert responsibility.directives(state, _result(0.99)) == [] + + +def test_decision_armed_follows_check_threshold(state: FactoryState) -> None: + responsibility = _responsibility() + assert responsibility.decision_armed(_result(0.70)) is True + assert responsibility.decision_armed(_result(0.69)) is False + + +async def test_resolve_decision_answered_writes_audit(state: FactoryState, tmp_path: Path) -> None: + responsibility = _responsibility() + model = FakeForemanModel() + store = RunStore(tmp_path) + + resolution = await responsibility.resolve_decision( + state, model, _request(), store=store, run_id="run-1" + ) + + assert resolution.outcome.answered is True + assert resolution.outcome.choice == "join" + assert resolution.directive is None + assert model.decision_calls and model.decision_calls[0].question == "Use join or subquery?" + + events = store.load_decision_events(run_id="run-1") + assert len(events) == 1 + event = events[0] + assert event.event_type is EventType.FOREMAN_DECIDED + assert event.payload["question"] == "Use join or subquery?" + assert event.payload["answered"] is True + assert event.payload["choice"] == "join" + assert event.payload["effective_threshold"] == pytest.approx(0.70) + assert event.payload["effective_denylist"] == ["credentials", "destructive"] + + +async def test_resolve_decision_tighten_only(state: FactoryState) -> None: + # A request asking for a looser floor than the baseline cannot lower it. + responsibility = _responsibility() + model = FakeForemanModel() + request = _request(min_confidence=0.10) + + resolution = await responsibility.resolve_decision(state, model, request) + + assert resolution.outcome.answered is True + assert resolution.outcome.choice == "join" + + +@pytest.mark.parametrize( + "decision", + [ + # The model abstains outright. + Decision( + choice=None, + confidence=0.99, + rationale="ambiguous", + classification=[], + abstained=True, + ), + # Below the 0.70 baseline threshold. + Decision( + choice="join", + confidence=0.40, + rationale="unsure", + classification=[], + abstained=False, + ), + # Denylisted category at high confidence. + Decision( + choice="join", + confidence=0.99, + rationale="risky", + classification=[AbstainCategory.DESTRUCTIVE], + abstained=False, + ), + # Choice outside the offered options. + Decision( + choice="truncate", + confidence=0.99, + rationale="off-menu", + classification=[], + abstained=False, + ), + ], + ids=["model-abstained", "low-confidence", "denylisted", "invalid-choice"], +) +async def test_abstain_never_terminates( + state: FactoryState, tmp_path: Path, decision: Decision +) -> None: + """Abstention routes the question to the human; it never stops the worker.""" + + responsibility = _responsibility() + model = FakeForemanModel(decisions=[decision]) + store = RunStore(tmp_path) + + resolution = await responsibility.resolve_decision( + state, model, _request(), store=store, run_id="run-1" + ) + + assert resolution.outcome.answered is False + assert resolution.directive is not None + assert resolution.directive.action is InterventionType.ESCALATE + assert resolution.directive.responsibility_id == DECISION_POLICY + assert resolution.directive.action not in { + InterventionType.STOP_WORKER, + InterventionType.FINISH, + InterventionType.RETRY_WORKER, + } + + events = store.load_decision_events(run_id="run-1") + assert len(events) == 1 + assert events[0].payload["answered"] is False diff --git a/tests/test_hooks.py b/tests/test_hooks.py index f7499d4..d81aeb9 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -79,6 +79,7 @@ async def assess(self, observation, checks): "agents_md_drift": 0.0, "needs_human": 0.0, "documentation_sufficient": 0.2, + "decision-required": 0.0, } defaults.update(self.scores) grouped = {} diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 3c57424..1b1e71b 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -173,3 +173,43 @@ def test_initialize_locally_excludes_runtime_directory(state, tmp_path) -> None: ).stdout assert ".foreman" not in status assert "/.foreman/" in (tmp_path / ".git" / "info" / "exclude").read_text() + + +def test_list_run_ids_and_load_decision_events(tmp_path) -> None: + store = RunStore(tmp_path) + assert store.list_run_ids() == [] + assert store.load_decision_events() == [] + + # Create two runs: one factory run and one serve session (no state.json) + event1 = FactoryEvent( + run_id="run-1", + event_type=EventType.FOREMAN_DECIDED, + payload={"question": "Q1", "answered": True, "choice": "A", "confidence": 0.85}, + ) + event2 = FactoryEvent( + run_id="run-2", + event_type=EventType.FACTORY_STARTED, + payload={}, + ) + event3 = FactoryEvent( + run_id="run-2", + event_type=EventType.FOREMAN_DECIDED, + payload={"question": "Q2", "answered": False, "choice": None, "confidence": 0.60}, + ) + + store.append_event(event1) + store.append_event(event2) + store.append_event(event3) + + run_ids = store.list_run_ids() + assert set(run_ids) == {"run-1", "run-2"} + + # All decisions across runs + decisions = store.load_decision_events() + assert len(decisions) == 2 + assert {d.payload["question"] for d in decisions} == {"Q1", "Q2"} + + # Decisions for specific run + run1_decisions = store.load_decision_events(run_id="run-1") + assert len(run1_decisions) == 1 + assert run1_decisions[0].payload["question"] == "Q1" diff --git a/tests/test_responsibility_config.py b/tests/test_responsibility_config.py index c13614b..2e9348e 100644 --- a/tests/test_responsibility_config.py +++ b/tests/test_responsibility_config.py @@ -22,7 +22,7 @@ def write_config(tmp_path, responsibility_id: str, contents: str) -> None: def test_packaged_config_uses_all_builtin_defaults() -> None: registry = configured_registry(FactoryConfig()) - assert len(registry.responsibilities) == 6 + assert len(registry.responsibilities) == 7 assert registry.global_ids() == tuple( item.id for item in registry.responsibilities if item.id != "quality.documentation" ) @@ -33,6 +33,7 @@ def test_packaged_config_uses_all_builtin_defaults() -> None: "core.worker-health.toml", "quality.documentation.toml", "repository.instructions.toml", + "supervision.decision-policy.toml", ] completion = next(item for item in registry.responsibilities if item.id == "core.completion") assert [check.check_id for check in completion.checks()] == [