From c6615811ac07909d5211219d282b4a6d1fe15afa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 10 Oct 2026 22:51:09 +0900 Subject: [PATCH 1/4] feat(review): route public reviews through LiteLLM auto Use the centrally configured LiteLLM auto model for public Noema and OpenCode reviews while retaining the attested ZDR sidecar for private targets. Fail closed when visibility or gateway configuration is unavailable. Signed-off-by: Seongho Bae Commit-Message-Assisted-by: Codex (via Orca) Co-Authored-By: Codex --- .github/workflows/noema-review.yml | 34 +- .../workflows/opencode-review-dispatch.yml | 121 +++--- AGENTS.md | 24 +- ...ntextual-orchestrator-vendored-free-zdr.md | 24 +- ...-noema-review-model-job-timeout-removal.md | 3 +- .../noema-model-output-repair-boundary.md | 4 +- docs/doctoring/noema-orchestrator-free-zdr.md | 31 +- docs/product-goal-directive.md | 2 +- ...al_orchestrator_review_sidecar_contract.py | 43 ++- ...st_noema_orchestrator_workflow_contract.py | 106 ++++- tests/test_opencode_agent_contract.py | 364 +++++------------- .../test_required_workflow_queue_contract.py | 14 +- 12 files changed, 386 insertions(+), 384 deletions(-) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 08ea600538..62374ec40d 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -855,7 +855,7 @@ jobs: - name: Provision contextual-orchestrator review sidecar id: noema_sidecar - if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' + if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' && steps.target_visibility.outputs.require_zdr == 'true' env: SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} @@ -917,6 +917,9 @@ jobs: NOEMA_REVIEW_ACTOR: ${{ steps.noema_github_app_token.outputs['app-slug'] && format('{0}[bot]', steps.noema_github_app_token.outputs['app-slug']) || '' }} NOEMA_REVIEW_INSTALLATION_ID: ${{ steps.noema_github_app_token.outputs['installation-id'] }} NOEMA_TRANSPORT_RETRY_ATTEMPT: ${{ toJSON(github.event.client_payload.transport_retry_attempt) }} + REQUIRE_ZDR: ${{ steps.target_visibility.outputs.require_zdr }} + LLM_GATEWAY_API_KEY: ${{ steps.target_visibility.outputs.require_zdr == 'false' && secrets.LLM_GATEWAY_API_KEY || '' }} + LLM_GATEWAY_MODEL: ${{ vars.LLM_GATEWAY_MODEL }} run: | set -euo pipefail if [ -z "${PR_NUMBER:-}" ]; then @@ -928,15 +931,30 @@ jobs: echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot prepare a verdict." exit 1 fi - if [ -z "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ] || [ -z "${CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE:-}" ]; then - echo "::error::contextual-orchestrator review sidecar must be provisioned before Noema LLM review." + if [ "$REQUIRE_ZDR" = true ]; then + if [ -z "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ] || [ -z "${CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE:-}" ]; then + echo "::error::contextual-orchestrator ZDR sidecar must be provisioned before private Noema review." + exit 1 + fi + source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" + export NOEMA_LLM_MODEL="orchestrator/free" + export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN}" + export NOEMA_LLM_VIA_ORCHESTRATOR=1 + elif [ "$REQUIRE_ZDR" = false ]; then + if [ "${LLM_GATEWAY_MODEL:-}" != auto ] || [ -z "${LLM_GATEWAY_API_KEY:-}" ]; then + echo "::error::Public Noema review requires the configured auto model and LiteLLM key." + exit 1 + fi + printf '::add-mask::%s\n' "$LLM_GATEWAY_API_KEY" + export NOEMA_LLM_API_URL="https://litellm.poinnetworks.net/v1/chat/completions" + export NOEMA_LLM_MODEL="$LLM_GATEWAY_MODEL" + export NOEMA_LLM_API_KEY="$LLM_GATEWAY_API_KEY" + export NOEMA_LLM_VIA_ORCHESTRATOR=0 + else + echo "::error::Noema review refuses unknown target visibility." exit 1 fi - source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" - export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" - export NOEMA_LLM_MODEL="orchestrator/free" - export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN}" - export NOEMA_LLM_VIA_ORCHESTRATOR=1 verdict_file="${RUNNER_TEMP}/noema-verdict-envelope.json" rm -f "$verdict_file" python3 "$GITHUB_WORKSPACE/.github/actions/noema-review/two_phase.py" --repo "$TARGET_REPOSITORY" --pr-number "$PR_NUMBER" --expected-head "$EXPECTED_HEAD_SHA" --prepare-verdict-file "$verdict_file" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 09bbf8181a..7698728df3 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -2656,6 +2656,7 @@ jobs: update-environment: false - name: Provision contextual-orchestrator review sidecar + if: needs.validate-pr-metadata.outputs.is_private == 'true' env: SIDECAR_PYTHON: ${{ steps.sidecar_python.outputs.python-path }} BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} @@ -3757,6 +3758,9 @@ jobs: OPENCODE_FAILED_CHECK_EVIDENCE_FILE: ${{ runner.temp }}/opencode-failed-check-evidence.md OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-pr-head + EXPECTED_IS_PRIVATE: ${{ needs.validate-pr-metadata.outputs.is_private }} + LLM_GATEWAY_API_KEY: ${{ needs.validate-pr-metadata.outputs.is_private == 'false' && secrets.LLM_GATEWAY_API_KEY || '' }} + LLM_GATEWAY_MODEL: ${{ vars.LLM_GATEWAY_MODEL || '' }} run: | set -euo pipefail mkdir -p "$OPENCODE_REVIEW_WORKDIR" @@ -4099,11 +4103,44 @@ jobs: cp "$GITHUB_WORKSPACE/ci-review-prompt.md" "${OPENCODE_REVIEW_WORKDIR}/ci-review-prompt.md" cp "$GITHUB_WORKSPACE/code-reviewer-prompt.md" "${OPENCODE_REVIEW_WORKDIR}/code-reviewer-prompt.md" - jq -n '{ + if [ "$EXPECTED_IS_PRIVATE" = true ]; then + review_model="contextual-orchestrator/orchestrator/free" + model_provider="contextual-orchestrator" + model_id="orchestrator/free" + provider_name="Contextual Orchestrator" + base_url="{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1" + api_key="{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" + model_name="Orchestrator Free (ZDR-first zero-cost pool)" + elif [ "$EXPECTED_IS_PRIVATE" = false ]; then + if [ "$LLM_GATEWAY_MODEL" != auto ] || [ -z "$LLM_GATEWAY_API_KEY" ]; then + echo '::error::Public OpenCode review requires LLM_GATEWAY_MODEL=auto and LLM_GATEWAY_API_KEY.' + exit 1 + fi + printf '::add-mask::%s\n' "$LLM_GATEWAY_API_KEY" + review_model="litellm-poinnetworks/${LLM_GATEWAY_MODEL}" + model_provider="litellm-poinnetworks" + model_id="$LLM_GATEWAY_MODEL" + provider_name="PoinNetworks LiteLLM" + base_url="https://litellm.poinnetworks.net/v1" + api_key='{env:LLM_GATEWAY_API_KEY}' + model_name="Auto" + else + echo '::error::OpenCode review refuses unknown target visibility.' + exit 1 + fi + + jq -n \ + --arg review_model "$review_model" \ + --arg model_provider "$model_provider" \ + --arg model_id "$model_id" \ + --arg provider_name "$provider_name" \ + --arg base_url "$base_url" \ + --arg api_key "$api_key" \ + --arg model_name "$model_name" '{ "$schema": "https://opencode.ai/config.json", - "model": "contextual-orchestrator/orchestrator/free", - "small_model": "contextual-orchestrator/orchestrator/free", - "enabled_providers": ["contextual-orchestrator"], + "model": $review_model, + "small_model": $review_model, + "enabled_providers": [$model_provider], "lsp": false, "mcp": {}, "permission": { @@ -4179,49 +4216,29 @@ jobs: } } }, - "provider": {} - }' >"${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" - - gateway_config="${OPENCODE_REVIEW_WORKDIR}/opencode.gateway.jsonc" - jq ' - .model = "contextual-orchestrator/orchestrator/free" | - .small_model = "contextual-orchestrator/orchestrator/free" | - .enabled_providers = ["contextual-orchestrator"] | - .provider = { - "contextual-orchestrator": { + "provider": { + ($model_provider): { "npm": "@ai-sdk/openai-compatible", - "name": "Contextual Orchestrator", - "options": { - "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1", - "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" - }, + "name": $provider_name, + "options": {"baseURL": $base_url, "apiKey": $api_key}, "models": { - "orchestrator/free": { - "name": "Orchestrator Free (ZDR-first zero-cost pool)", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 200000, - "output": 32768 - } - } + ($model_id): ( + {"name": $model_name, "tool_call": true} + + (if $model_provider == "contextual-orchestrator" then { + "reasoning": true, + "options": {"reasoningEffort": "high"}, + "variants": {"high": {"reasoningEffort": "high"}}, + "limit": {"context": 200000, "output": 32768} + } else {} end) + ) } } } - ' "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" >"$gateway_config" - mv -- "$gateway_config" "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" + }' >"${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" - if ! grep -Fq 'contextual-orchestrator/orchestrator/free' "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" \ - || ! grep -Fq 'CONTEXTUAL_ORCHESTRATOR_BASE_URL' "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc"; then - echo '::error::Generated isolated opencode.jsonc is missing the contextual-orchestrator gateway route.' + if ! grep -Fq "\"$review_model\"" "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" \ + || ! grep -Fq "\"$model_provider\"" "${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc"; then + echo '::error::Generated isolated opencode.jsonc is missing its visibility-scoped model route.' exit 1 fi printf 'Prepared isolated OpenCode review workspace: %s\n' "$OPENCODE_REVIEW_WORKDIR" @@ -4234,7 +4251,8 @@ jobs: SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" - OPENCODE_MODEL_CANDIDATES: "contextual-orchestrator/orchestrator/free" + OPENCODE_MODEL_CANDIDATES: ${{ needs.validate-pr-metadata.outputs.is_private == 'true' && 'contextual-orchestrator/orchestrator/free' || format('litellm-poinnetworks/{0}', vars.LLM_GATEWAY_MODEL) }} + LLM_GATEWAY_API_KEY: ${{ needs.validate-pr-metadata.outputs.is_private == 'false' && secrets.LLM_GATEWAY_API_KEY || '' }} # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. @@ -4272,7 +4290,14 @@ jobs: RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail - source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + if [ "$OPENCODE_MODEL_CANDIDATES" = contextual-orchestrator/orchestrator/free ]; then + source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + elif [ "$OPENCODE_MODEL_CANDIDATES" = litellm-poinnetworks/auto ] && [ -n "$LLM_GATEWAY_API_KEY" ]; then + printf '::add-mask::%s\n' "$LLM_GATEWAY_API_KEY" + else + echo '::error::OpenCode model route or credential is unavailable.' + exit 1 + fi set +e bash "$GITHUB_WORKSPACE/scripts/ci/run_opencode_review_model_pool.sh" pool_status=$? @@ -4852,7 +4877,8 @@ jobs: COVERAGE_EVIDENCE_SUMMARY: ${{ needs.coverage-evidence.outputs.coverage_summary || 'Coverage evidence job did not run or did not publish coverage evidence.' }} OPENCODE_REVIEW_WORKDIR: ${{ runner.temp }}/opencode-review-project OPENCODE_SOURCE_WORKDIR: ${{ runner.temp }}/opencode-pr-head - MODEL: contextual-orchestrator/orchestrator/free + MODEL: ${{ needs.validate-pr-metadata.outputs.is_private == 'true' && 'contextual-orchestrator/orchestrator/free' || format('litellm-poinnetworks/{0}', vars.LLM_GATEWAY_MODEL) }} + LLM_GATEWAY_API_KEY: ${{ needs.validate-pr-metadata.outputs.is_private == 'false' && secrets.LLM_GATEWAY_API_KEY || '' }} USE_GITHUB_TOKEN: "true" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" @@ -6214,8 +6240,11 @@ jobs: printf 'Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair; using collected current-head failed-check logs/SARIF fallback so the publish step stays bounded.\n' >&2 return 1 fi - source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" - if [ -z "${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" ]; then + if [ "$MODEL" = contextual-orchestrator/orchestrator/free ]; then + source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh" + elif [ "$MODEL" = litellm-poinnetworks/auto ] && [ -n "$LLM_GATEWAY_API_KEY" ]; then + printf '::add-mask::%s\n' "$LLM_GATEWAY_API_KEY" + else return 1 fi if ! python3 "$GITHUB_WORKSPACE/scripts/ci/assert_opencode_reasoning_effort.py" \ diff --git a/AGENTS.md b/AGENTS.md index 0972af51c5..c8aeb651ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,21 +15,15 @@ Organization edge runtimes use Cloudflare Pingora. Do not add or preserve active Semgrep hosted scans bind one job-level `SEMGREP_IMAGE` digest for log evidence, manifest inspection, and `docker run`. See [`docs/doctoring/semgrep-image-digest-single-source.md`](docs/doctoring/semgrep-image-digest-single-source.md). OpenCode may repair only trusted `path:line` bindings on LLM probes that already carry an independent proof and source-line digest. See [`docs/doctoring/opencode-llm-review-publication.md`](docs/doctoring/opencode-llm-review-publication.md). -Central review routes through the vendored **contextual-orchestrator** gateway -sidecar (`scripts/ci/contextual_orchestrator_review_sidecar.sh`). The five -provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, -`NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) enter its KV -as bootstrap transport in the same process that discovers models and serves; -OpenCode, Noema, and Strix all use the fail-closed zero-cost pool -`orchestrator/free`. Strix was switched onto `orchestrator/free` on -2026-08-30, superseding the prior `orchestrator/auto` (provider-diverse, -non-free-admitting) default; private targets still require ZDR-compliant -routes under [`scripts/ci/zdr_policy.py`](scripts/ci/zdr_policy.py). That -switch was made by an autonomous agent session, not per any owner decision — -see [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md)'s -2026-08-30 amendment and its 2026-08-31 correction, which retracts an earlier -false claim of explicit owner direction and records the resulting -availability risk as open and unreviewed, not accepted. +Central Noema and OpenCode reviews use the configured PoinNetworks LiteLLM +`auto` route for public targets (`LLM_GATEWAY_MODEL` and the secret +`LLM_GATEWAY_API_KEY`). Private/internal targets use the vendored +**contextual-orchestrator** sidecar with `orchestrator/free` and an attested +ZDR-only catalog. Unknown visibility fails closed without selecting either +route. Strix uses that sidecar's `orchestrator/free` pool. The sidecar's five +provider secrets enter its in-process KV only as bootstrap transport. See +[`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md) +and [`docs/doctoring/noema-orchestrator-free-zdr.md`](docs/doctoring/noema-orchestrator-free-zdr.md). Sidecar diagnostics may retain only a server-generated `request_id` matching exactly 32 lowercase hexadecimal characters, plus the producer's explicit `-` or `` marker where that event contract permits it. Keep free-form diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 7543f736e8..4f9f0c9f22 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -1,11 +1,11 @@ # ADR-0003: Vendored contextual-orchestrator review sidecar with governed gateway pools -- Status: accepted, amended 2026-09-02 (see amendment history below — Strix - now uses `orchestrator/free`, not the `orchestrator/auto` this header - originally recorded) +- Status: accepted, amended 2026-10-10 (public Noema/OpenCode reviews use the + configured LiteLLM `auto` route; private/internal reviews retain ZDR-only + `orchestrator/free`) - Date: 2026-08-27 - Scope: ContextualWisdomLab/.github central review pipelines (OpenCode autofix/dispatch + shared `opencode.jsonc` default + required Noema + Strix review) -- Decision: Route every central CI review write/model execution that touches contracts in this repository through the **vendored** `contextual-orchestrator` gateway, served as a per-runner sidecar. OpenCode, Noema, and (as of the 2026-08-30 amendment) Strix all use the fail-closed zero-cost virtual model id `orchestrator/free`. **Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets.** +- Decision: Public Noema/OpenCode reviews use the configured PoinNetworks LiteLLM `auto` model with the centrally stored `LLM_GATEWAY_API_KEY`. Private/internal reviews use the **vendored** `contextual-orchestrator` sidecar and only an attested ZDR route from `orchestrator/free`. Unknown visibility fails closed. Strix continues to use the sidecar's `orchestrator/free` pool. **ZDR-compliant routes remain mandatory for private targets.** - Ownership: `.github` owns control-plane evidence; `ContextualWisdomLab/contextual-orchestrator` owns the gateway. The 2026-08-18 org decision (recorded in `ContextualWisdomLab/contextual-orchestrator` AGENTS.md) already migrated OpenCode/Noema/Strix to the orchestrator backend; this ADR is the org-repo (provider-config) half of that decision. - Figma File ID: N/A (no customer UI). @@ -134,6 +134,13 @@ all five, and auto-optimize routing by cost. - Workers need egress to the five provider model-list hosts and, when reachable, `https://openrouter.ai/api/v1/endpoints/zdr`; the feed failure path is graceful (static table). +- Public Noema/OpenCode review calls use the existing centrally stored + `LLM_GATEWAY_API_KEY` only in the model-request step and require + `LLM_GATEWAY_MODEL=auto`. The provider credential is never placed in a + generated config or available to PR-head build/test processes. +- Private/internal review content stays on the attested ZDR sidecar. If target + visibility cannot be established, neither route is selected and review fails + closed. - A new central dep (vendored repo pinned to a SHA) must be reviewed when the orchestrator upgrades; the pin is centralized in one script and one contract test. @@ -315,3 +322,12 @@ all five, and auto-optimize routing by cost. vulnerabilities in the same audit. The earlier byte-identical-lock claim describes the superseded proposal, not this amended target. No review completion or runtime provider success is inferred from the lock audit. + +- **2026-10-10 amendment: public-target automatic model routing.** Noema and + OpenCode select the configured PoinNetworks LiteLLM `auto` alias for public + repository targets, using the already stored `LLM_GATEWAY_API_KEY` only in + trusted model-request steps. Private/internal targets remain on the + attested-ZDR `orchestrator/free` sidecar. Both workflows fail closed when + visibility is unknown; `LLM_GATEWAY_MODEL` must equal `auto`, and no + provider key is copied into generated configuration or PR-head test jobs. + Strix and the local OpenCode default are unchanged. diff --git a/docs/doctoring/autofix-and-noema-review-model-job-timeout-removal.md b/docs/doctoring/autofix-and-noema-review-model-job-timeout-removal.md index 3e80cfb35b..2fe75e4819 100644 --- a/docs/doctoring/autofix-and-noema-review-model-job-timeout-removal.md +++ b/docs/doctoring/autofix-and-noema-review-model-job-timeout-removal.md @@ -26,7 +26,8 @@ jobs the model call itself runs synchronously, in-job: `opencode run` for base-merge conflict resolution, later in the same job). - `noema-review`'s "Prepare Noema model verdict" step runs `python3 .github/actions/noema-review/two_phase.py ...`, which itself - calls the model (`NOEMA_LLM_API_URL`, `NOEMA_LLM_MODEL=orchestrator/free`) + calls the model (`NOEMA_LLM_API_URL`, using `orchestrator/free` for + private/internal targets or configured `auto` for public targets) and blocks until it returns. A job-level `timeout-minutes` on either job does not merely bound "how long diff --git a/docs/doctoring/noema-model-output-repair-boundary.md b/docs/doctoring/noema-model-output-repair-boundary.md index 88635d01a2..8105200763 100644 --- a/docs/doctoring/noema-model-output-repair-boundary.md +++ b/docs/doctoring/noema-model-output-repair-boundary.md @@ -21,7 +21,7 @@ Three `timeout --kill-after=20 900` commands remain in `opencode-review-dispatch 1. Model-produced JSON, envelope, schema, and semantic-contract failures remain fail-closed and are not consumer-source findings. 2. Every provider attempt reports a phase such as connecting, reading, decoding, or validating, its elapsed duration, a stable failure category, and the served model if known. Provider status classes such as 413, 429, 500, and 502 are retained as categories without copying provider secrets or raw model output. 3. The triggering pull-request head is checked before model work and again before publication. A push to the same PR makes the old head obsolete; the old run must not publish a verdict or spend a second repair call. -4. All model traffic for required review remains on contextual-orchestrator `orchestrator/free` and is subject to its discovery, capability, failover, and privacy policy. +4. Private/internal model traffic for required review remains on contextual-orchestrator `orchestrator/free` and is subject to its discovery, capability, failover, and ZDR policy. Public Noema/OpenCode targets use the configured LiteLLM `auto` route and centrally stored API key. 5. A workflow shell timeout is evidence about that shell command only. It must never be used as evidence that the gateway or provider ended inference. ## Verification @@ -38,4 +38,4 @@ Python Software Foundation. (2026). urllib.request — Extensible library for op ## Actionable diagnostic boundary -Corrective prompts, when implemented by the gateway, may use the deterministic class of a malformed verdict but do not need arbitrary model-produced values. Trusted structural validator messages remain available after secret scrubbing. Unsupported decision values and unknown model-output text are represented by stable diagnostics, and raw model exceptions are not retained as public causes. \ No newline at end of file +Corrective prompts, when implemented by the gateway, may use the deterministic class of a malformed verdict but do not need arbitrary model-produced values. Trusted structural validator messages remain available after secret scrubbing. Unsupported decision values and unknown model-output text are represented by stable diagnostics, and raw model exceptions are not retained as public causes. diff --git a/docs/doctoring/noema-orchestrator-free-zdr.md b/docs/doctoring/noema-orchestrator-free-zdr.md index ec1add661a..89748ba523 100644 --- a/docs/doctoring/noema-orchestrator-free-zdr.md +++ b/docs/doctoring/noema-orchestrator-free-zdr.md @@ -1,16 +1,16 @@ -# Doctoring record: Required reviews through the vendored orchestrator sidecar +# Doctoring record: ZDR sidecar for private reviews, configured gateway for public reviews - **Date:** 2026-08-28 -- **Subject:** Required Noema, OpenCode dispatch, and Strix reviews no longer - select direct provider model routes. They use the same vendored - `contextual-orchestrator` sidecar (`orchestrator/free`, ZDR-first - auto-discovery). +- **Subject:** Private/internal Noema and OpenCode dispatch plus Strix use the + vendored `contextual-orchestrator` sidecar (`orchestrator/free`, ZDR-first + auto-discovery). Public Noema/OpenCode targets use the configured PoinNetworks + LiteLLM `auto` route, per the 2026-10-10 amendment below. - **Decision record:** [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](../adr/0003-contextual-orchestrator-vendored-free-zdr.md) - **Related:** [`docs/doctoring/contextual-orchestrator-vendored-sidecar.md`](contextual-orchestrator-vendored-sidecar.md) ## What changed -`.github/workflows/noema-review.yml` provisions +For private/internal targets, `.github/workflows/noema-review.yml` provisions `scripts/ci/contextual_orchestrator_review_sidecar.sh` with the five provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) and points `NOEMA_LLM_API_URL` at the @@ -20,6 +20,11 @@ bearer. The public-repo `integrate.api.nvidia.com` / `nvidia/nemotron-3-ultra-550b-a55b` hardcode is deleted. There is no sequential OpenAI or Azure fallback hop. +For public targets, Noema uses the existing `LLM_GATEWAY_MODEL=auto` variable +and `LLM_GATEWAY_API_KEY` secret with the LiteLLM base URL. OpenCode's generated +configuration has the same visibility split. The key is available only to +trusted model-call steps and is never copied into configuration. + `scripts/ci/noema_review_gate.py` `call_llm` still rejects `localhost` and arbitrary private, link-local, multicast, and unspecified targets. It allows only `127.0.0.1` / `::1` when that origin matches the exact configured @@ -27,9 +32,8 @@ only `127.0.0.1` / `::1` when that origin matches the exact configured marker is metadata only and never widens the allowlist. Noema reviewer identity is unchanged: `NOEMA_REVIEW_TOKEN` / GitHub App / -OIDC. Review mutation is still not `github.token`. Required OpenCode and Strix -use the same gateway model and do not add direct-provider fallback paths. The -hourly-review-repair roster is untouched. +OIDC. Review mutation is still not `github.token`. Strix continues to use the +ZDR sidecar route. The hourly-review-repair roster is untouched. ## Verification contract @@ -82,3 +86,12 @@ is not Noema review evidence. For GitHub App credentials, reviewer identity is bound to the pinned token mint action's app slug and numeric installation ID. PAT and OIDC credentials continue to resolve their actor through GitHub's authenticated API. + +## 2026-10-10 public-target gateway routing + +The central repository already held `LLM_GATEWAY_MODEL=auto` and the +`LLM_GATEWAY_API_KEY` secret, but Noema and OpenCode did not use them. Public +repository reviews now use `https://litellm.poinnetworks.net` with that model +and key. Private/internal reviews still use the attested-ZDR sidecar; unknown +visibility and a missing public gateway configuration fail closed. The API +key value is read only by trusted review jobs and is never printed or committed. diff --git a/docs/product-goal-directive.md b/docs/product-goal-directive.md index c76c4226e4..b858942238 100644 --- a/docs/product-goal-directive.md +++ b/docs/product-goal-directive.md @@ -64,7 +64,7 @@ Per this file's own conflict policy above: this note is the resolution, and `doc > LLM·오케스트레이션·Embedding LLM이 필요한 테스트는 contextual-orchestrator 기반 OpenCode Agent로 만든다. contextual-orchestrator는 GitHub Secrets의 BYTEZ_API_KEY, NVIDIA_NIM_API_KEY, NVIDIA_NIM_API_KEY_SUB, OPENROUTER_API_KEY, OPENAI_API_KEY를 모두 써 auto model discovery로 최적 모형을 제공한다. embedding·responses·completions, audio, video, image, ommi-modal 등 가용 모델을 폭넓게 지원한다. 가능하면 반입해 쓰고 발견한 해당 저장소 문제도 함께 수정한다. LLM 사용 소프트웨어와 contextual-orchestrator는 Fugu·Conductor·TRINITY 연구를 근거로 단일 모델 라우팅과 심층 다중 Agent 오케스트레이션 사이의 계산량을 배분한다. 워크플로 단계, 재귀 깊이, 작업 분해, 접근 목록으로 test-time compute를 조절하고 역할별 reasoning effort를 다르게 하며 추론 수준 ablation을 수행한다. 속도는 핵심 고려사항이 아니며 정확성을 우선한다. 중앙 OpenCode, Strix, Noema는 모델당 두 시간 이상 걸릴 수 있음을 수용한다. LLM Chat model은 chat completion API와 responses API를 모두 지원하고 json_object와 json_schema를 모두 처리한다. Embedding은 문단·구문·DOM·송수신자 등 의미 단위를 식별해 chunking한다. 본문에 base64 이미지가 있으면 텍스트 인식, 객체 인식, 태그 설명, 이미지 별도 검색 방법을 연구 근거와 함께 DB 설계에 넣고 원래 삽입 위치를 보존해 그림 맥락까지 검색·표현한다. GitHub Actions scheduler는 contextual-orchestrator 기반 OpenCode Agent로 전환한다. COPILOT_GITHUB_TOKEN은 쓰지 않고 기존 리뷰 Agent 키 체계를 유지한다. -**Note (flagged by CodeRabbit on this PR, 2026-08-30):** section 8's quoted text describes `contextual-orchestrator`'s general product capability — broad model/modality support and all-five-secret auto model discovery as a *design principle for the orchestrator itself*. It does not specify, and must not be read as overriding, which pool each CI consumer routes through: that is governed exclusively by `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md` and its doctoring records — `OpenCode` and `Noema` use the fail-closed, ZDR-prioritized `orchestrator/free` pool; only `Strix` security analysis uses the provider-diverse `orchestrator/auto` pool; private/internal review targets require an attested ZDR-only catalog and never fall back to a non-ZDR provider. Do not loosen any CI consumer's pool or credential scope on the strength of this section's general wording alone. +**Note (2026-10-10 routing update):** this section's five-secret model-discovery guidance remains the `contextual-orchestrator` design contract and Strix route. Central Noema and OpenCode reviews now send public-repository review content to the explicitly configured LiteLLM `auto` model using `LLM_GATEWAY_API_KEY`. Private/internal targets use only the attested ZDR sidecar pool `orchestrator/free`; unknown visibility fails closed without selecting either route. See `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md` and `docs/doctoring/noema-orchestrator-free-zdr.md`. **Note (2026-08-30, superseded by the merged pin flip — see the correction below):** an earlier draft of this note said Strix stayed on `orchestrator/auto` pending `free_family_diversity` reaching `>= 2`. That is no longer true and must not be read as current: `.github/workflows/strix.yml` now hardcodes `STRIX_MODEL`/`CONTEXTUAL_ORCHESTRATOR_POOL` to `orchestrator/free` and fails closed on any other value. This note originally went on to say that ADR-0003's 2026-08-30 amendment "records the owner's decision to accept the residual single-outage-domain risk immediately rather than wait for the evidence-gated threshold this note originally described" — that framing was false, as ADR-0003's own 2026-08-31 correction now records: no owner reviewed or accepted this switch or its risk. `free_account_diversity` (`scripts/ci/contextual_orchestrator_review_policy.py`; renamed from `free_family_diversity` once every KV credential became an independent discovery account rather than being grouped into a vendor "family", see #1468) remains useful as ongoing monitoring evidence for that open, unreviewed risk, not as a gate blocking the pin. diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 16b85c5c78..b9a9c69f93 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -1,12 +1,9 @@ """Contract tests for the vendored contextual-orchestrator review sidecar. -These static contracts pin the org policy: every central CI review path routes -through the vendored ``contextual-orchestrator`` gateway, all five provider -secrets (``BYTEZ_API_KEY``, ``NVIDIA_NIM_API_KEY``, ``NVIDIA_NIM_API_KEY_SUB``, -``OPENROUTER_API_KEY``, ``OPENAI_API_KEY``) enter its process-local KV as -bootstrap transport, models are auto-discovered, and the ``orchestrator/free`` -fail-closed zero-cost pool (prioritized by the ZDR policy in -``scripts/ci/zdr_policy.py``) is the review model. +These contracts pin the org policy: private CI reviews use the vendored +``contextual-orchestrator`` ZDR route, while public reviews use the central +LiteLLM ``auto`` model. Provider secrets bootstrap the private sidecar's +process-local credential store; ``orchestrator/free`` remains its model. """ from __future__ import annotations @@ -526,14 +523,24 @@ def test_sidecar_surfaces_nonfatal_discovery_warnings_on_a_successful_startup() assert "wait_for_sidecar_sanitizers" not in text[healthz_confirmed:] -def test_noema_review_workflow_provisions_sidecar_with_all_five_secrets() -> None: - """Required Noema review uses the gateway; the public NIM hardcode is gone.""" +def test_noema_review_workflow_provisions_private_sidecar_with_five_secrets() -> None: + """Only private Noema reviews provision the ZDR sidecar and its bootstrap keys.""" workflow = _read(NOEMA_WORKFLOW) + sidecar_step = workflow.split( + " - name: Provision contextual-orchestrator review sidecar\n", 1 + )[1].split("\n - name:", 1)[0] + sidecar_guard = ( + "if: env.PR_NUMBER != '' && steps.live_draft.outputs.live_draft != 'true' " + "&& steps.target_visibility.outputs.require_zdr == 'true'" + ) + assert sidecar_guard in sidecar_step assert "contextual_orchestrator_review_sidecar.sh" in workflow for secret in FIVE_SECRETS: assert f"{secret}: ${{{{ secrets.{secret} }}}}" in workflow assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow assert "NOEMA_LLM_VIA_ORCHESTRATOR=1" in workflow + assert "Public Noema review requires the configured auto model and LiteLLM key." in workflow + assert "https://litellm.poinnetworks.net/v1/chat/completions" in workflow assert "${CONTEXTUAL_ORCHESTRATOR_BASE_URL%/}/v1/chat/completions" in workflow assert "${CONTEXTUAL_ORCHESTRATOR_TOKEN}" in workflow assert "https://integrate.api.nvidia.com" not in workflow @@ -559,17 +566,23 @@ def test_noema_private_targets_require_zdr_only_sidecar_routing() -> None: def test_required_opencode_dispatch_uses_the_gateway_for_model_pool_and_diagnosis() -> None: - """The privileged Required OpenCode path has no direct-provider model route.""" + """Private reviews use the ZDR sidecar; public reviews use LiteLLM auto.""" workflow = _read(OPENCODE_DISPATCH_WORKFLOW) assert "Provision contextual-orchestrator review sidecar" in workflow assert workflow.index("Validate pull request head repository trust") < workflow.index( "Provision contextual-orchestrator review sidecar" ) - assert 'OPENCODE_MODEL_CANDIDATES: "contextual-orchestrator/orchestrator/free"' in workflow - assert 'MODEL: contextual-orchestrator/orchestrator/free' in workflow - assert '.enabled_providers = ["contextual-orchestrator"]' in workflow - assert '.model = "contextual-orchestrator/orchestrator/free"' in workflow - assert 'CONTEXTUAL_ORCHESTRATOR_TOKEN:-' in workflow + assert "if: needs.validate-pr-metadata.outputs.is_private == 'true'" in workflow + assert ( + "OPENCODE_MODEL_CANDIDATES: ${{ needs.validate-pr-metadata.outputs.is_private" + in workflow + ) + assert "MODEL: ${{ needs.validate-pr-metadata.outputs.is_private" in workflow + assert "litellm-poinnetworks/{0}" in workflow + assert "LLM_GATEWAY_MODEL=auto" in workflow + assert "https://litellm.poinnetworks.net/v1" in workflow + assert 'api_key="{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}"' in workflow + assert 'source "$GITHUB_WORKSPACE/scripts/ci/load_contextual_orchestrator_token.sh"' in workflow assert 'STRIX_GITHUB_MODELS_TOKEN:-' not in workflow assert 'MODEL: github-models/' not in workflow diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 8de7551814..63bdf2acaa 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -163,8 +163,8 @@ def test_noema_close_cleanup_selects_only_the_closed_pr_across_shared_display_ti assert "/actions/runs/105/cancel" not in calls -def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: - """Require reviewer credentials and the sidecar; the public NIM hardcode is gone.""" +def test_noema_review_credentials_and_model_routing_fail_closed() -> None: + """Keep private reviews on the sidecar and require auto LiteLLM for public reviews.""" workflow = workflow_text("noema-review.yml") assert "fail_unavailable()" in workflow @@ -221,9 +221,16 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert " contents: read" in review_job assert "python3 -m scripts.ci.noema_review_gate" not in workflow assert ( - "contextual-orchestrator review sidecar must be provisioned before Noema LLM review." + "contextual-orchestrator ZDR sidecar must be provisioned before private Noema review." in workflow ) + assert "Public Noema review requires the configured auto model and LiteLLM key." in workflow + assert "Noema review refuses unknown target visibility." in workflow + assert "LLM_GATEWAY_MODEL: ${{ vars.LLM_GATEWAY_MODEL }}" in prepare + assert 'LLM_GATEWAY_API_KEY: ${{ steps.target_visibility.outputs.require_zdr == \'false\' && secrets.LLM_GATEWAY_API_KEY || \'\' }}' in prepare + assert 'export NOEMA_LLM_MODEL="$LLM_GATEWAY_MODEL"' in prepare + assert 'export NOEMA_LLM_API_URL="https://litellm.poinnetworks.net/v1/chat/completions"' in prepare + assert 'export NOEMA_LLM_VIA_ORCHESTRATOR=0' in prepare assert "mark_unconfigured()" not in workflow assert "review skipped until Noema is deployed" not in workflow assert "Noema app token is unavailable; review skipped." not in workflow @@ -231,6 +238,99 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "secrets: inherit" not in workflow +def test_noema_prepare_routes_visibility_and_fails_closed(tmp_path: Path) -> None: + """Execute the workflow's exact Prepare step without calling a model endpoint.""" + bash = shutil.which("bash") or "/bin/bash" + workflow = workflow_text("noema-review.yml") + script = textwrap.dedent( + workflow_step(workflow, "Prepare Noema model verdict").split(" run: |\n", 1)[1] + ) + python = shutil.which("python3") + assert python is not None + + def run_case(name: str, visibility: str, model: str, key: str): + case = tmp_path / name + workspace = case / "workspace" + bin_dir = case / "bin" + runner_temp = case / "runner-temp" + (workspace / "scripts/ci").mkdir(parents=True) + bin_dir.mkdir(parents=True) + runner_temp.mkdir() + capture = case / "prepared-env.bin" + github_output = case / "github-output" + token_loader = workspace / "scripts/ci/load_contextual_orchestrator_token.sh" + token_loader.write_text( + "export CONTEXTUAL_ORCHESTRATOR_TOKEN=synthetic-private-token\n", + encoding="utf-8", + ) + fake_python = bin_dir / "python3" + fake_python.write_text( + f"#!{python}\n" + "import json, os, sys\n" + "names = ('NOEMA_LLM_API_URL', 'NOEMA_LLM_MODEL', 'NOEMA_LLM_API_KEY', 'NOEMA_LLM_VIA_ORCHESTRATOR')\n" + "with open(os.environ['NOEMA_CAPTURE'], 'w', encoding='utf-8') as f:\n" + " json.dump({'env': {name: os.environ.get(name) for name in names}, 'argv': sys.argv[1:]}, f)\n", + encoding="utf-8", + ) + fake_python.chmod(0o755) + result = subprocess.run( # noqa: S603, S607 + [bash, "-c", script], + env={ + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}", + "GITHUB_WORKSPACE": str(workspace), + "RUNNER_TEMP": str(runner_temp), + "GITHUB_OUTPUT": str(github_output), + "NOEMA_CAPTURE": str(capture), + "PR_NUMBER": "7", + "GH_TOKEN": "synthetic-review-token", + "TARGET_REPOSITORY": "ContextualWisdomLab/example", + "EXPECTED_HEAD_SHA": "a" * 40, + "REQUIRE_ZDR": visibility, + "CONTEXTUAL_ORCHESTRATOR_BASE_URL": "https://private-sidecar.example", + "CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE": str(case / "sidecar-token"), + "LLM_GATEWAY_MODEL": model, + "LLM_GATEWAY_API_KEY": key, + }, + capture_output=True, + text=True, + check=False, + ) + return result, json.loads(capture.read_text(encoding="utf-8")) if capture.exists() else None + + public_key = "synthetic-public-key" + public, public_capture = run_case("public", "false", "auto", public_key) + assert public.returncode == 0 + assert public_capture["env"] == { + "NOEMA_LLM_API_URL": "https://litellm.poinnetworks.net/v1/chat/completions", + "NOEMA_LLM_MODEL": "auto", + "NOEMA_LLM_API_KEY": public_key, + "NOEMA_LLM_VIA_ORCHESTRATOR": "0", + } + assert "--prepare-verdict-file" in public_capture["argv"] + assert f"::add-mask::{public_key}" in public.stdout + + private, private_capture = run_case("private", "true", "unused", "") + assert private.returncode == 0 + assert private_capture["env"] == { + "NOEMA_LLM_API_URL": "https://private-sidecar.example/v1/chat/completions", + "NOEMA_LLM_MODEL": "orchestrator/free", + "NOEMA_LLM_API_KEY": "synthetic-private-token", + "NOEMA_LLM_VIA_ORCHESTRATOR": "1", + } + assert "synthetic-private-token" not in private.stdout + + for name, visibility, model, key, error in ( + ("unknown", "unknown", "auto", public_key, "refuses unknown target visibility"), + ("keymissing", "false", "auto", "", "requires the configured auto model and LiteLLM key"), + ("modelnotauto", "false", "gpt-4.1", public_key, "requires the configured auto model and LiteLLM key"), + ): + result, capture = run_case(name, visibility, model, key) + assert result.returncode != 0 + assert capture is None + assert error in result.stdout + + def test_noema_continuation_dispatch_uses_central_handler_and_live_identity(tmp_path: Path) -> None: """Central continuation preserves target identity and rejects stale or fork heads.""" script = textwrap.dedent( diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 8a80d65461..f0dd70f464 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -9,6 +9,7 @@ import pytest from scripts.ci.assert_opencode_reasoning_effort import strip_jsonc_comments +from tests.test_required_workflow_queue_contract import workflow_step, workflow_text def load_opencode_jsonc() -> dict: @@ -89,284 +90,99 @@ def test_code_reviewer_subagent_contract_is_configured(): ) -def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): - """Guard every review-pool candidate against silent reasoning-effort drift.""" - config = load_opencode_jsonc() - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") - github_models = config["provider"]["github-models"]["models"] - candidates_match = re.search(r'OPENCODE_MODEL_CANDIDATES: "([^"]+)"', workflow) - - assert candidates_match is not None - conditional_public_candidate = ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " - "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 " - "nvidia-nim/nvidia/nemotron-3-super-120b-a12b " - "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b " - "nvidia-nim/meta/llama-3.3-70b-instruct " - "nvidia-nim/deepseek-ai/deepseek-v4-pro " - "nvidia-nim/mistralai/codestral-22b-instruct-v0.1 " - "opencode-free/nemotron-3-ultra-free " - "opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free " - "opencode-free/laguna-s-2.1-free " - "opencode-free/ling-3.0-flash-free " - "opencode-free/big-pickle " - "opencode-free/mimo-v2.5-free " - "opencode-free/hy3-free " - "opencode-free/minimax-m3-free " - "opencode-free/glm-5-free " - "opencode-free/kimi-k2.5-free " - "opencode-free/qwen3.6-plus-free ' || '' }}" - ) - candidates_text = candidates_match.group(1) - if candidates_text == "contextual-orchestrator/orchestrator/free": - assert 'OPENCODE_MODEL_CANDIDATES: "contextual-orchestrator/orchestrator/free"' in workflow - assert 'MODEL: contextual-orchestrator/orchestrator/free' in workflow - assert '.enabled_providers = ["contextual-orchestrator"]' in workflow - return - assert candidates_text.startswith(conditional_public_candidate) - candidates = [ - "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", - "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1", - "nvidia-nim/nvidia/nemotron-3-super-120b-a12b", - "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b", - "nvidia-nim/meta/llama-3.3-70b-instruct", - "nvidia-nim/deepseek-ai/deepseek-v4-pro", - "nvidia-nim/mistralai/codestral-22b-instruct-v0.1", - "opencode-free/nemotron-3-ultra-free", - "opencode-free/deepseek-v4-flash-free", - "opencode-free/north-mini-code-free", - "opencode-free/laguna-s-2.1-free", - "opencode-free/ling-3.0-flash-free", - "opencode-free/big-pickle", - "opencode-free/mimo-v2.5-free", - "opencode-free/hy3-free", - "opencode-free/minimax-m3-free", - "opencode-free/glm-5-free", - "opencode-free/kimi-k2.5-free", - "opencode-free/qwen3.6-plus-free", - *candidates_text.removeprefix(conditional_public_candidate).split(), - ] - candidate_pairs = [candidate.split("/", 1) for candidate in candidates] - direct_openai_models = [ - model_name for provider, model_name in candidate_pairs if provider == "openai" - ] - zen_models = [ - model_name for provider, model_name in candidate_pairs if provider == "opencode" - ] - openrouter_models = [ - model_name for provider, model_name in candidate_pairs if provider == "openrouter" - ] - github_candidate_models = [ - model_name - for provider, model_name in candidate_pairs - if provider == "github-models" - ] +def _opencode_config_route_script() -> str: + """Extract the live workflow's routing and jq config block for execution.""" + workflow = workflow_text("opencode-review-dispatch.yml") + step = workflow_step(workflow, "Prepare isolated OpenCode review workspace") + script = textwrap.dedent(step.split(" run: |\n", 1)[1]) + start = script.index('if [ "$EXPECTED_IS_PRIVATE" = true ]; then') + end = script.index("printf 'Prepared isolated OpenCode review workspace:", start) + return script[start:end] - assert candidate_pairs - assert all( - not candidate.startswith("nvidia-nim/") - for candidate in candidates_text.removeprefix(conditional_public_candidate).split() - ) - assert candidate_pairs == [ - ["nvidia-nim", "nvidia/llama-3.3-nemotron-super-49b-v1.5"], - ["nvidia-nim", "nvidia/llama-3.1-nemotron-ultra-253b-v1"], - ["nvidia-nim", "nvidia/nemotron-3-super-120b-a12b"], - ["nvidia-nim", "nvidia/nemotron-3-ultra-550b-a55b"], - ["nvidia-nim", "meta/llama-3.3-70b-instruct"], - ["nvidia-nim", "deepseek-ai/deepseek-v4-pro"], - ["nvidia-nim", "mistralai/codestral-22b-instruct-v0.1"], - ["opencode-free", "nemotron-3-ultra-free"], - ["opencode-free", "deepseek-v4-flash-free"], - ["opencode-free", "north-mini-code-free"], - ["opencode-free", "laguna-s-2.1-free"], - ["opencode-free", "ling-3.0-flash-free"], - ["opencode-free", "big-pickle"], - ["opencode-free", "mimo-v2.5-free"], - ["opencode-free", "hy3-free"], - ["opencode-free", "minimax-m3-free"], - ["opencode-free", "glm-5-free"], - ["opencode-free", "kimi-k2.5-free"], - ["opencode-free", "qwen3.6-plus-free"], - ["opencode", "gpt-5.6-terra"], - ["github-models", "deepseek/deepseek-v3-0324"], - ["openai", "gpt-5.4"], - ["openrouter", "deepseek/deepseek-v3.2"], - ["openrouter", "qwen/qwen3-coder"], - ["github-models", "openai/gpt-4.1"], - ["github-models", "openai/gpt-5"], - ["github-models", "openai/gpt-5-chat"], - ["github-models", "openai/o3"], - ["github-models", "deepseek/deepseek-r1-0528"], - ["github-models", "deepseek/deepseek-r1"], - ] - assert zen_models == ["gpt-5.6-terra"] - assert direct_openai_models == ["gpt-5.4"] - assert openrouter_models == [ - "deepseek/deepseek-v3.2", - "qwen/qwen3-coder", - ] - assert set(github_candidate_models).issubset(set(github_models)) - assert '"context": 256000' in workflow - assert '"output": 64000' in workflow - generated_config_match = re.search( - r"jq -n '(\{.*?\})' >\"\$\{OPENCODE_REVIEW_WORKDIR\}/opencode\.jsonc\"", - workflow, - re.DOTALL, - ) - assert generated_config_match is not None - generated_config = json.loads(generated_config_match.group(1)) - nvidia_provider = generated_config["provider"]["nvidia-nim"] - assert nvidia_provider["options"] == { - "baseURL": "https://integrate.api.nvidia.com/v1", - "apiKey": "{env:NVIDIA_API_KEY}", - } - assert nvidia_provider["models"]["nvidia/nemotron-3-ultra-550b-a55b"][ - "limit" - ] == {"context": 131072, "output": 8192} - scoped_provider_binding = ( - "NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" - ) - jobs_text = workflow[workflow.index("\njobs:\n") + len("\njobs:\n") :] - job_headers = list( - re.finditer(r"^ ([A-Za-z0-9_-]+):\n", jobs_text, re.MULTILINE) - ) - job_blocks = { - match.group(1): jobs_text[ - match.start() : ( - job_headers[index + 1].start() - if index + 1 < len(job_headers) - else len(jobs_text) - ) - ] - for index, match in enumerate(job_headers) - } - privileged_review_job = job_blocks["opencode-review-target"] - assert privileged_review_job.count(scoped_provider_binding) == 2 - assert ( - privileged_review_job.count( - "NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" - ) - == 2 +def _run_opencode_config_route( + tmp_path: Path, *, name: str, private: str, model: str, key: str +): + workdir = tmp_path / name / "review" + workdir.mkdir(parents=True) + result = subprocess.run( # noqa: S603, S607 + [shutil.which("bash") or "/bin/bash", "-c", _opencode_config_route_script()], + env={ + **os.environ, + "OPENCODE_REVIEW_WORKDIR": str(workdir), + "EXPECTED_IS_PRIVATE": private, + "LLM_GATEWAY_MODEL": model, + "LLM_GATEWAY_API_KEY": key, + }, + capture_output=True, + text=True, + check=False, ) - for job_name, job_block in job_blocks.items(): - if job_name == "opencode-review-target": - continue - assert "secrets.NVIDIA_NIM_API_KEY" not in job_block, job_name - assert "secrets.NVIDIA_API_KEY" not in job_block, job_name - assert "secrets.NVIDIA_NIM_API_KEY || secrets.NVIDIA_API_KEY" not in workflow - free_models = generated_config["provider"]["opencode-free"]["models"] - paid_zen_models = generated_config["provider"]["opencode"]["models"] - assert set(free_models) == { - "nemotron-3-ultra-free", - "deepseek-v4-flash-free", - "north-mini-code-free", - "laguna-s-2.1-free", - "ling-3.0-flash-free", - "big-pickle", - "mimo-v2.5-free", - "hy3-free", - "minimax-m3-free", - "glm-5-free", - "kimi-k2.5-free", - "qwen3.6-plus-free", - } - assert set(paid_zen_models) == {"gpt-5.6-terra"} - terra_model = paid_zen_models["gpt-5.6-terra"] - assert terra_model["tool_call"] is True - assert terra_model["reasoning"] is True - assert terra_model["options"]["reasoningEffort"] == "high" - assert terra_model["variants"]["high"]["reasoningEffort"] == "high" - assert terra_model["limit"] == {"context": 1000000, "output": 128000} - nemotron_model = free_models["nemotron-3-ultra-free"] - deepseek_model = free_models["deepseek-v4-flash-free"] - north_model = free_models["north-mini-code-free"] - assert nemotron_model["tool_call"] is True - assert nemotron_model["limit"] == {"context": 1000000, "output": 128000} - assert "response_format" not in nemotron_model.get("options", {}) - assert deepseek_model["tool_call"] is True - assert deepseek_model["limit"] == {"context": 200000, "output": 128000} - assert "response_format" not in deepseek_model.get("options", {}) - assert north_model["tool_call"] is True - assert "response_format" not in north_model["options"] - assert free_models["laguna-s-2.1-free"]["limit"] == { - "context": 256000, - "output": 32000, - } - assert free_models["ling-3.0-flash-free"]["limit"] == { - "context": 262144, - "output": 32768, - } - assert free_models["big-pickle"]["limit"] == { - "context": 200000, - "output": 32000, - } - assert free_models["mimo-v2.5-free"]["limit"] == { - "context": 200000, - "output": 32000, + config_path = workdir / "opencode.jsonc" + return result, json.loads(config_path.read_text(encoding="utf-8")) if config_path.exists() else None + + +def test_opencode_model_route_is_visibility_scoped_and_fails_closed(tmp_path: Path): + """Execute the jq route block for public, private, unknown, and invalid config.""" + assert shutil.which("jq") is not None + + public_key = "synthetic-public-key" + public, public_config = _run_opencode_config_route( + tmp_path, name="public", private="false", model="auto", key=public_key + ) + assert public.returncode == 0 + assert public_config["model"] == "litellm-poinnetworks/auto" + assert public_config["small_model"] == "litellm-poinnetworks/auto" + assert public_config["enabled_providers"] == ["litellm-poinnetworks"] + public_provider = public_config["provider"]["litellm-poinnetworks"] + assert public_provider["name"] == "PoinNetworks LiteLLM" + assert public_provider["options"] == { + "baseURL": "https://litellm.poinnetworks.net/v1", + "apiKey": "{env:LLM_GATEWAY_API_KEY}", } - for model_name, model_config in free_models.items(): - # Every free-pool candidate must declare tool_call support: the reviewer - # drives CodeGraph/web-search tooling, so a non-tool_call model in the - # pool cannot produce a structured review and would burn its failover - # slot before yielding. Guard the whole pool, not just a hand-picked few. - assert model_config["tool_call"] is True, model_name - if model_config.get("reasoning") is True: - assert model_config["options"]["reasoningEffort"] == "high", model_name - assert model_config["variants"]["high"]["reasoningEffort"] == "high", ( - model_name - ) - assert github_candidate_models == [ - "deepseek/deepseek-v3-0324", - "openai/gpt-4.1", - "openai/gpt-5", - "openai/gpt-5-chat", - "openai/o3", - "deepseek/deepseek-r1-0528", - "deepseek/deepseek-r1", - ] - banned_review_candidates = { - "gpt-5-nano", - "openai/gpt-5-nano", - "openai/o3-mini", + assert public_provider["models"]["auto"] == {"name": "Auto", "tool_call": True} + assert public_key not in json.dumps(public_config) + assert f"::add-mask::{public_key}" in public.stdout + + private, private_config = _run_opencode_config_route( + tmp_path, name="private", private="true", model="unused", key="" + ) + assert private.returncode == 0 + assert private_config["model"] == "contextual-orchestrator/orchestrator/free" + assert private_config["small_model"] == private_config["model"] + assert private_config["enabled_providers"] == ["contextual-orchestrator"] + private_provider = private_config["provider"]["contextual-orchestrator"] + assert private_provider["options"] == { + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}/v1", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}", } - assert banned_review_candidates.isdisjoint( - set(direct_openai_models) | set(openrouter_models) | set(github_candidate_models) - ) - assert '"opencode": {' in workflow - assert '"apiKey": "{env:OPENCODE_API_KEY}"' in workflow - assert "OPENCODE_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }}" in workflow - assert '"openai": {' in workflow - assert '"apiKey": "{env:OPENAI_API_KEY}"' in workflow - assert '"openrouter": {' in workflow - assert '"apiKey": "{env:OPENROUTER_API_KEY}"' in workflow - for model_name in direct_openai_models + openrouter_models + github_candidate_models: - assert f'"{model_name}": {{' in workflow - - def is_reasoning_capable(model_name: str) -> bool: - return ( - model_name.startswith("gpt-5") - or model_name.startswith("openai/gpt-5") - or model_name.startswith("openai/o3") - or model_name.startswith("openai/o4") - or model_name.startswith("deepseek/deepseek-r1") + private_model = private_provider["models"]["orchestrator/free"] + assert private_model["reasoning"] is True + assert private_model["options"]["reasoningEffort"] == "high" + assert private_model["variants"]["high"]["reasoningEffort"] == "high" + assert private_model["limit"] == {"context": 200000, "output": 32768} + + for name, visibility, model, key, error in ( + ("unknown", "unknown", "auto", public_key, "refuses unknown target visibility"), + ("keymissing", "false", "auto", "", "requires LLM_GATEWAY_MODEL=auto and LLM_GATEWAY_API_KEY"), + ("modelnotauto", "false", "gpt-4.1", public_key, "requires LLM_GATEWAY_MODEL=auto and LLM_GATEWAY_API_KEY"), + ): + result, config = _run_opencode_config_route( + tmp_path, name=name, private=visibility, model=model, key=key ) + assert result.returncode != 0 + assert config is None + assert error in result.stdout - for model_name in github_candidate_models: - model_config = github_models[model_name] - if is_reasoning_capable(model_name): - assert model_config["reasoning"] is True, model_name - assert model_config["options"]["reasoningEffort"] == "high", model_name - assert model_config["variants"]["high"]["reasoningEffort"] == "high", ( - model_name - ) - else: - assert model_config.get("reasoning") is not True, model_name - assert "reasoningEffort" not in model_config.get("options", {}), model_name - assert "variants" not in model_config, model_name +def test_opencode_model_pool_uses_visibility_scoped_model_names(): + """The reviewer and diagnostic jobs select the same private or auto route.""" + workflow = workflow_text("opencode-review-dispatch.yml") + assert "needs.validate-pr-metadata.outputs.is_private == 'true'" in workflow + assert "litellm-poinnetworks/{0}" in workflow + assert "LLM_GATEWAY_MODEL" in workflow + assert "LLM_GATEWAY_API_KEY" in workflow + assert 'MODEL: ${{ needs.validate-pr-metadata.outputs.is_private' in workflow def test_model_pool_cannot_synthesize_approval_after_provider_exhaustion(): """Provider exhaustion must remain exhausted without a command-only reviewer.""" @@ -2076,7 +1892,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair" in workflow ) - assert 'OPENCODE_MODEL_CANDIDATES: "contextual-orchestrator/orchestrator/free"' in workflow + assert "OPENCODE_MODEL_CANDIDATES: ${{ needs.validate-pr-metadata.outputs.is_private" in workflow + assert "litellm-poinnetworks/{0}" in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "180"' in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow @@ -2113,7 +1930,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'gh api -X GET "repos/${GH_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate' not in publish_step ) - assert "MODEL: contextual-orchestrator/orchestrator/free" in publish_step + assert "MODEL: ${{ needs.validate-pr-metadata.outputs.is_private" in workflow assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "120"' not in publish_step assert "${OPENCODE_RUN_TIMEOUT_SECONDS:-120}s" not in publish_step assert ( @@ -2192,7 +2009,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert ( 'OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-5-nano"' not in workflow ) - assert 'OPENCODE_MODEL_CANDIDATES: "contextual-orchestrator/orchestrator/free"' in workflow + assert "OPENCODE_MODEL_CANDIDATES: ${{ needs.validate-pr-metadata.outputs.is_private" in workflow + assert "litellm-poinnetworks/{0}" in workflow assert "${{ runner.temp }}/opencode-review-model-pool.md" in workflow assert re.search( r'check-runs" \\\n\s+-f per_page=100 \\\n\s+--paginate \\\n\s+--slurp \|\n\s+jq -r "\$jq_filter"', diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index a24b3c7d0c..efb26f9af3 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1234,8 +1234,8 @@ def test_noema_triggers_preserve_standalone_pull_request_review() -> None: assert '[ "${live_head_sha,,}" != "${EXPECTED_HEAD_SHA,,}" ]' in workflow -def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() -> None: - """Require explicit reviewer credentials and the trusted orchestrator sidecar.""" +def test_noema_review_credentials_and_visibility_model_routes_fail_closed() -> None: + """Require review credentials and route private/public targets by visibility.""" workflow = workflow_text("noema-review.yml") assert "fail_unavailable()" in workflow @@ -1277,10 +1277,9 @@ def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() - assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow assert "contextual_orchestrator_review_sidecar.sh" in workflow assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow - assert ( - "contextual-orchestrator review sidecar must be provisioned before Noema LLM review." - in workflow - ) + assert "contextual-orchestrator ZDR sidecar must be provisioned before private Noema review." in workflow + assert "Public Noema review requires the configured auto model and LiteLLM key." in workflow + assert "Noema review refuses unknown target visibility." in workflow assert "BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }}" in workflow assert "NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow assert "NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}" in workflow @@ -1346,6 +1345,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( **os.environ, "PR_NUMBER": "1", "GH_TOKEN": "synthetic-review-token", + "REQUIRE_ZDR": "true", } for key in ( "CONTEXTUAL_ORCHESTRATOR_BASE_URL", @@ -1366,7 +1366,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( check=False, ) assert noema.returncode == 1 - assert "sidecar must be provisioned before Noema LLM review" in noema.stdout + assert "ZDR sidecar must be provisioned before private Noema review" in noema.stdout def test_noema_workflow_run_without_pull_request_skips_before_token_exchange() -> None: From 5ff59e40dd2cc6b7cdb316b89fd67a4d67268b9d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 11 Oct 2026 01:20:34 +0900 Subject: [PATCH 2/4] fix(review): reject malformed Noema visibility metadata --- .github/workflows/noema-review.yml | 2 +- ...st_noema_orchestrator_workflow_contract.py | 76 ++++++++++++++++- tests/test_opencode_agent_contract.py | 82 +++++++++++++++++++ 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 62374ec40d..e3260cb03c 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -779,7 +779,7 @@ jobs: visibility="" for target_visibility_attempt in 1 2 3 4 5 6; do if visibility="$( - gh api "/repos/${TARGET_REPOSITORY}" --jq '.visibility // (if .private then "private" else "public" end)' + gh api "/repos/${TARGET_REPOSITORY}" --jq '.visibility // (if .private == true then "private" elif .private == false then "public" else "" end)' )"; then break fi diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 63bdf2acaa..0fb9ce0b52 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -10,6 +10,8 @@ import textwrap from pathlib import Path +import pytest + from tests.test_required_workflow_queue_contract import workflow_step, workflow_text @@ -724,6 +726,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> **os.environ, "PR_NUMBER": "1", "GH_TOKEN": "synthetic-review-token", + "REQUIRE_ZDR": "true", } for key in ( "CONTEXTUAL_ORCHESTRATOR_BASE_URL", @@ -740,7 +743,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> check=False, ) assert noema.returncode == 1 - assert "sidecar must be provisioned before Noema LLM review" in noema.stdout + assert "sidecar must be provisioned before private Noema review" in noema.stdout def test_cancel_closed_pr_runs_has_a_bounded_runtime() -> None: @@ -825,3 +828,74 @@ def test_noema_review_retains_sanitized_sidecar_evidence_after_any_outcome() -> ) assert prepare < upload < refresh assert workflow.count("actions/upload-artifact@") == 1 + + +@pytest.mark.parametrize( + "metadata,expected", + [ + pytest.param({"visibility": "public"}, "false", id="public"), + pytest.param({"visibility": "private"}, "true", id="private"), + pytest.param({"visibility": "internal"}, "true", id="internal"), + pytest.param({"private": True}, "true", id="legacy-true"), + pytest.param({"private": False}, "false", id="legacy-false"), + pytest.param({}, None, id="missing"), + pytest.param({"private": None}, None, id="null"), + pytest.param({"private": "false"}, None, id="string"), + pytest.param({"visibility": "unknown"}, None, id="unknown"), + pytest.param({"private": 0}, None, id="zero"), + pytest.param({"private": 1}, None, id="one"), + pytest.param({"private": []}, None, id="array"), + pytest.param({"private": {}}, None, id="object"), + pytest.param({"visibility": None, "private": False}, "false", id="null-visibility-boolean"), + pytest.param({"visibility": "unknown", "private": False}, None, id="unknown-overrides-fallback"), + ], +) +def test_noema_visibility_requires_explicit_visibility_or_boolean_fallback( + tmp_path: Path, metadata: dict, expected: str | None +) -> None: + """Execute the actual gh/jq visibility shell; malformed metadata must not route.""" + assert shutil.which("jq") is not None + script = textwrap.dedent( + workflow_step( + workflow_text("noema-review.yml"), + "Resolve Noema target repository visibility", + ).split(" run: |\n", 1)[1] + ) + fake_gh = tmp_path / "gh" + fake_gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'test "$1" = api\ntest "$2" = /repos/ContextualWisdomLab/example\n' + 'test "$3" = --jq\ntest "$#" = 4\n' + 'printf called >> "$FAKE_GH_CALLS"\n' + 'printf "%s" "$FAKE_METADATA" | jq -r "$4"\n', + encoding="utf-8", + ) + fake_gh.chmod(0o755) + output = tmp_path / "github-output" + calls = tmp_path / "gh-calls" + result = subprocess.run( + [shutil.which("bash") or "/bin/bash", "-c", script], + env={ + "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", + "HOME": str(tmp_path), + "TMPDIR": str(tmp_path), + "GITHUB_OUTPUT": str(output), + "GH_TOKEN": "synthetic-review-token", + "TARGET_REPOSITORY": "ContextualWisdomLab/example", + "FAKE_METADATA": json.dumps(metadata), + "FAKE_GH_CALLS": str(calls), + }, + capture_output=True, + text=True, + check=False, + timeout=5, + ) + assert calls.read_text(encoding="utf-8") == "called" + assert "synthetic-review-token" not in result.stdout + result.stderr + if expected is None: + assert result.returncode != 0, (metadata, result.stdout) + assert not output.exists() + assert "visibility is missing or unsupported" in result.stdout + else: + assert result.returncode == 0, result.stderr + assert output.read_text(encoding="utf-8") == f"require_zdr={expected}\n" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index f0dd70f464..27edd15a4c 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -3048,3 +3048,85 @@ def test_r_package_load_deferral_requires_current_head_r_cmd_check(): assert ( "if (!is.na(pkg) && !requireNamespace(pkg, quietly = TRUE))" not in workflow ) + + +@pytest.mark.parametrize( + "repository_metadata,expected", + [ + pytest.param({"visibility": "public"}, "false", id="public"), + pytest.param({"visibility": "private"}, "true", id="private"), + pytest.param({"visibility": "internal"}, "true", id="internal"), + pytest.param({}, None, id="missing"), + pytest.param({"visibility": None}, None, id="null"), + pytest.param({"visibility": "unknown"}, None, id="unknown"), + pytest.param({"visibility": 0}, None, id="number"), + pytest.param({"visibility": []}, None, id="array"), + pytest.param({"visibility": {}}, None, id="object"), + pytest.param({"private": True}, None, id="no-boolean-fallback-true"), + pytest.param({"private": False}, None, id="no-boolean-fallback-false"), + ], +) +def test_opencode_live_visibility_metadata_fails_closed( + tmp_path: Path, repository_metadata: dict, expected: str | None +) -> None: + """Execute the sibling metadata admission shell with real jq and no network.""" + assert shutil.which("jq") is not None + repository = "ContextualWisdomLab/example" + metadata = { + "state": "open", + "base": {"repo": {"full_name": repository, **repository_metadata}, "ref": "main", "sha": "a" * 40}, + "head": {"repo": {"full_name": repository}, "ref": "feature", "sha": "b" * 40}, + } + script = textwrap.dedent( + workflow_step( + workflow_text("opencode-review-dispatch.yml"), + "Bind workflow inputs to live organization pull request metadata", + ).split(" run: |\n", 1)[1] + ) + fake_gh = tmp_path / "gh" + fake_gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'test "$1" = api\ntest "$2" = repos/ContextualWisdomLab/example/pulls/7\n' + 'test "$#" = 2\nprintf called >> "$FAKE_GH_CALLS"\n' + 'printf "%s" "$FAKE_METADATA"\n', + encoding="utf-8", + ) + fake_gh.chmod(0o755) + output = tmp_path / "github-output" + calls = tmp_path / "gh-calls" + result = subprocess.run( + [shutil.which("bash") or "/bin/bash", "-c", script], + env={ + "PATH": f"{tmp_path}{os.pathsep}{os.environ.get('PATH', '')}", + "HOME": str(tmp_path), + "TMPDIR": str(tmp_path), + "GITHUB_OUTPUT": str(output), + "GH_TOKEN": "synthetic-review-token", + "EVENT_NAME": "repository_dispatch", + "DISPATCH_ACTOR": "synthetic-bot", + "DISPATCH_SENDER": "synthetic-bot", + "ALLOWED_DISPATCH_ACTOR": "synthetic-bot", + "ALLOWED_DISPATCH_TARGETS": repository, + "TARGET_REPOSITORY": repository, + "PR_NUMBER": "7", + "SUPPLIED_BASE_REF": "main", + "SUPPLIED_BASE_SHA": "a" * 40, + "SUPPLIED_HEAD_REF": "feature", + "SUPPLIED_HEAD_SHA": "b" * 40, + "FAKE_METADATA": json.dumps(metadata), + "FAKE_GH_CALLS": str(calls), + }, + capture_output=True, + text=True, + check=False, + timeout=5, + ) + assert calls.read_text(encoding="utf-8") == "called" + assert "synthetic-review-token" not in result.stdout + result.stderr + if expected is None: + assert result.returncode != 0 + assert not output.exists() + else: + assert result.returncode == 0, result.stderr + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert values["is_private"] == expected From 7d293e75ce578b51d9dba73f4c971237d8c9dd15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 11 Oct 2026 22:47:27 +0900 Subject: [PATCH 3/4] test(review): bind LiteLLM auto workflow to its exact blob Refresh the immutable reviewer workflow contract after public auto routing changes; retain fail-closed exact source binding rather than weakening the checks. Co-Authored-By: Claude Code --- tests/test_pr_review_autofix_nvidia_nim_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 1bfc13292e..8584239d89 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "09bbf8181a443f7a5630ec91ca958440c5dcfc68" +REVIEW_DISPATCH_BLOB_SHA = "7698728df31f2f35d7e4ef37a3a116a2b30e8bcb" def _workflow_text(path: Path) -> str: From 7f5ba4aa44347b7994eda6965983a9d4b2fb7a2a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 11 Oct 2026 23:39:28 +0900 Subject: [PATCH 4/4] test(review): preserve privacy and routing contracts for auto integration Keep live Draft gating and both /v1 routes asserted, restore explicit unchanged private ZDR and Strix zero-cost documentation, and freeze aging clock fixtures without weakening freshness thresholds. Full suite: 5190 passed, 7 skipped, 40 subtests passed. Co-Authored-By: Claude Code --- AGENTS.md | 2 +- ...ntextual-orchestrator-vendored-free-zdr.md | 2 +- tests/test_audit_org_codeql_coverage.py | 16 ++++++++++++++-- ...st_noema_draft_admission_before_sidecar.py | 7 ++++++- .../test_review_failure_taxonomy_contract.py | 19 +++++++++---------- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c8aeb651ee..458913b6f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ Central Noema and OpenCode reviews use the configured PoinNetworks LiteLLM `LLM_GATEWAY_API_KEY`). Private/internal targets use the vendored **contextual-orchestrator** sidecar with `orchestrator/free` and an attested ZDR-only catalog. Unknown visibility fails closed without selecting either -route. Strix uses that sidecar's `orchestrator/free` pool. The sidecar's five +route. Strix uses that sidecar's zero-cost `orchestrator/free` pool. The sidecar's five provider secrets enter its in-process KV only as bootstrap transport. See [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md) and [`docs/doctoring/noema-orchestrator-free-zdr.md`](docs/doctoring/noema-orchestrator-free-zdr.md). diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 4f9f0c9f22..159281ae10 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -5,7 +5,7 @@ `orchestrator/free`) - Date: 2026-08-27 - Scope: ContextualWisdomLab/.github central review pipelines (OpenCode autofix/dispatch + shared `opencode.jsonc` default + required Noema + Strix review) -- Decision: Public Noema/OpenCode reviews use the configured PoinNetworks LiteLLM `auto` model with the centrally stored `LLM_GATEWAY_API_KEY`. Private/internal reviews use the **vendored** `contextual-orchestrator` sidecar and only an attested ZDR route from `orchestrator/free`. Unknown visibility fails closed. Strix continues to use the sidecar's `orchestrator/free` pool. **ZDR-compliant routes remain mandatory for private targets.** +- Decision: Public Noema/OpenCode reviews use the configured PoinNetworks LiteLLM `auto` model with the centrally stored `LLM_GATEWAY_API_KEY`. Private/internal reviews use the **vendored** `contextual-orchestrator` sidecar and only an attested ZDR route from `orchestrator/free`. Unknown visibility fails closed. Strix continues to use the sidecar's `orchestrator/free` pool. **Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets.** - Ownership: `.github` owns control-plane evidence; `ContextualWisdomLab/contextual-orchestrator` owns the gateway. The 2026-08-18 org decision (recorded in `ContextualWisdomLab/contextual-orchestrator` AGENTS.md) already migrated OpenCode/Noema/Strix to the orchestrator backend; this ADR is the org-repo (provider-config) half of that decision. - Figma File ID: N/A (no customer UI). diff --git a/tests/test_audit_org_codeql_coverage.py b/tests/test_audit_org_codeql_coverage.py index 0ffc8fa749..18be9e46aa 100644 --- a/tests/test_audit_org_codeql_coverage.py +++ b/tests/test_audit_org_codeql_coverage.py @@ -262,7 +262,13 @@ def test_null_latest_analysis_is_not_coverage() -> None: ] -def test_audit_codeql_coverage_defaults_now_to_current_time() -> None: +def test_audit_codeql_coverage_defaults_now_to_current_time(monkeypatch) -> None: + class Clock(datetime): + @classmethod + def now(cls, tz=None): + return NOW.astimezone(tz) + + monkeypatch.setattr(audit, "datetime", Clock) repositories = [covered_by_recent_analysis("DefaultNowRepo", days_ago=0)] assert audit.audit_codeql_coverage(repositories) == [] @@ -299,7 +305,13 @@ def test_main_fail_path_reports_gaps_from_stdin(monkeypatch, capsys) -> None: assert "FAIL: 1 repositories have no CodeQL coverage" in captured.err -def test_main_pass_path_reports_from_file_arg(tmp_path, capsys) -> None: +def test_main_pass_path_reports_from_file_arg(tmp_path, capsys, monkeypatch) -> None: + class Clock(datetime): + @classmethod + def now(cls, tz=None): + return NOW.astimezone(tz) + + monkeypatch.setattr(audit, "datetime", Clock) payload_path = tmp_path / "repositories.json" payload_path.write_text( json.dumps([covered_by_default_setup("ELUNVERA"), covered_by_recent_analysis("Orgmetra")]), diff --git a/tests/test_noema_draft_admission_before_sidecar.py b/tests/test_noema_draft_admission_before_sidecar.py index 715f2be495..e38c068980 100644 --- a/tests/test_noema_draft_admission_before_sidecar.py +++ b/tests/test_noema_draft_admission_before_sidecar.py @@ -84,7 +84,12 @@ def test_model_heavy_steps_are_gated_on_the_live_draft_output() -> None: workflow = workflow_text("noema-review.yml") for name in GATED_MODEL_STEPS: step = workflow_step(workflow, name) - assert f"if: env.PR_NUMBER != '' && {DRAFT_GATE}\n" in step, name + visibility_gate = ( + " && steps.target_visibility.outputs.require_zdr == 'true'" + if name == "Provision contextual-orchestrator review sidecar" + else "" + ) + assert f"if: env.PR_NUMBER != '' && {DRAFT_GATE}{visibility_gate}\n" in step, name def test_downstream_publication_treats_unset_prepare_outputs_as_skipped() -> None: diff --git a/tests/test_review_failure_taxonomy_contract.py b/tests/test_review_failure_taxonomy_contract.py index bc2b599f96..75ac605d84 100644 --- a/tests/test_review_failure_taxonomy_contract.py +++ b/tests/test_review_failure_taxonomy_contract.py @@ -66,18 +66,17 @@ def test_autofix_opencode_provider_targets_the_served_gateway_path() -> None: def test_dispatch_opencode_provider_targets_the_served_gateway_path() -> None: - """The dispatch gateway overlay must carry the same /v1 prefix. - - The dispatch workflow writes a provider-free base config and then layers - the gateway provider on with a second ``jq`` filter, so the assertion is - on every ``baseURL`` the workflow binds for that provider. - """ + """Both visibility-scoped providers retain their served /v1 prefix.""" workflow = _read(OPENCODE_DISPATCH_WORKFLOW) - bound = re.findall( - r'"baseURL": "(\{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL\}[^"]*)"', workflow + route_block = workflow.split(' if [ "$EXPECTED_IS_PRIVATE" = true ]; then', 1)[1] + private_route, public_route = route_block.split( + ' elif [ "$EXPECTED_IS_PRIVATE" = false ]; then', 1 ) - assert bound, "the dispatch workflow binds no gateway baseURL" - assert set(bound) == {EXPECTED_BASE_URL} + public_route = public_route.split(" else\n", 1)[0] + assert f'base_url="{EXPECTED_BASE_URL}"' in private_route + assert 'base_url="https://litellm.poinnetworks.net/v1"' in public_route + assert '--arg base_url "$base_url"' in route_block + assert '"options": {"baseURL": $base_url, "apiKey": $api_key}' in route_block def test_repository_opencode_config_targets_the_served_gateway_path() -> None: