From 963b454d7675cbd4d7599330916b2b696d209068 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 17:32:09 +0900 Subject: [PATCH 01/23] feat(noema): wire Noema to a same-job contextual-orchestrator sidecar Adds scripts/ci/start_contextual_orchestrator_sidecar.sh: checks out contextual-orchestrator at a pinned ref, installs its hash-locked deps, registers whichever of the five upstream provider credentials (BYTEZ_API_KEY, NVIDIA_NIM_API_KEY, NVIDIA_NIM_API_KEY_SUB, OPENROUTER_API_KEY, OPENAI_API_KEY) are present as job secrets into its KV, runs model auto-discovery + cost-based auto-enable, then serves on 127.0.0.1 with a freshly generated bearer token. Self-gating: no-ops cleanly (exit 0) when none of the five credentials are configured, so it's safe to call unconditionally. Wires it into noema-review.yml as a new early step. The LLM-call step now prefers the sidecar over the existing single-provider NVIDIA-NIM-direct fallback when nothing more specific is configured (NOEMA_LLM_API_URL/ _MODEL unset), since the sidecar auto-discovers and cost-optimizes across every registered provider rather than hardcoding one. Required a real security-guard change: noema_review_gate.py's call_llm() previously hard-rejected any loopback target ("URL cannot target localhost") as an SSRF guard. NOEMA_LLM_API_URL is only ever set from a repo/org-level Actions *variable* -- never PR content -- so the guard's trust boundary is "who can set the variable," not "what IP range the value resolves to." Relaxed to allow loopback specifically; the private/link-local/multicast/unspecified blocks for genuinely unexpected internal targets are unchanged. Updated/added tests in test_noema_review_gate.py accordingly (dual-stack DNS resolution, private ranges still rejected, loopback allowed via hostname/literal IP/DNS). Known limitation (not attempted here): unlike Noema, OpenCode's actual execution path (opencode-review-dispatch.yml) does not read the repo's opencode.jsonc at runtime -- it generates its own isolated inline config with a hard "nvidia-nim must be present and prioritized" assertion. The earlier opencode.jsonc PR (#1119) is therefore decorative for the real dispatch path; wiring OpenCode for real needs a separate, more careful pass. Strix similarly deferred (see PR description for both). Test plan: `pytest tests/test_noema_review_gate.py tests/test_noema_review_handoff.py -q` -> 43/43 passed. New/changed branches in the touched call_llm() region are covered; remaining project-wide coverage gaps are pre-existing and untouched by this diff. `bash -n scripts/ci/start_contextual_orchestrator_sidecar.sh` and a YAML parse of noema-review.yml both pass. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/noema-review.yml | 21 +++ scripts/ci/noema_review_gate.py | 17 ++- .../start_contextual_orchestrator_sidecar.sh | 120 ++++++++++++++++++ tests/test_noema_review_gate.py | 39 +++++- 4 files changed, 190 insertions(+), 7 deletions(-) create mode 100755 scripts/ci/start_contextual_orchestrator_sidecar.sh diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 59b25e3434..4c493a4ca3 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -135,6 +135,16 @@ jobs: tar -xzf "$trusted_archive" -C "$GITHUB_WORKSPACE" --strip-components=1 test -f scripts/ci/noema_review_gate.py + - name: Start contextual-orchestrator sidecar + if: env.PR_NUMBER != '' + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh + - name: Select fail-closed Noema reviewer credential if: env.PR_NUMBER != '' id: noema_credential @@ -276,6 +286,8 @@ jobs: NOEMA_LLM_MODEL: ${{ vars.NOEMA_LLM_MODEL || '' }} NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + CONTEXTUAL_ORCHESTRATOR_BASE_URL: ${{ env.CONTEXTUAL_ORCHESTRATOR_BASE_URL || '' }} + CONTEXTUAL_ORCHESTRATOR_TOKEN: ${{ env.CONTEXTUAL_ORCHESTRATOR_TOKEN || '' }} TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | set -euo pipefail @@ -287,6 +299,15 @@ jobs: echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot submit a verdict." exit 1 fi + # Prefer the contextual-orchestrator sidecar (auto-discovers and + # cost-optimizes across every registered upstream provider) over the + # single-provider NVIDIA NIM direct fallback below, when the sidecar + # started successfully and nothing more specific was configured. + if [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then + export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL}/chat/completions" + export NOEMA_LLM_MODEL="contextual-orchestrator" + export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" + fi if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -n "${NVIDIA_NIM_API_KEY:-}" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ]; then export NOEMA_LLM_API_URL="https://integrate.api.nvidia.com/v1/chat/completions" export NOEMA_LLM_MODEL="nvidia/nemotron-3-ultra-550b-a55b" diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 9317860e4c..bad8ca314e 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -460,8 +460,17 @@ def call_llm( hostname = (parsed.hostname or "").lower() if not hostname: raise ValueError("URL must have a valid hostname") - if hostname in {"localhost", "localhost.localdomain"} or hostname.endswith(".localhost"): - raise ValueError("URL cannot target localhost") + # Loopback is allowed: NOEMA_LLM_API_URL is only ever set from a + # repository/org-level Actions *variable* (see noema-review.yml), which + # only someone with write access to repo settings can set -- it is never + # influenced by pull request content. The trust boundary here is "who can + # set the variable," not "what IP range the value resolves to," so + # blocking loopback specifically doesn't add real protection once that's + # true -- and it's exactly the target needed to reach a same-job + # contextual-orchestrator sidecar (see start_contextual_orchestrator_sidecar.sh). + # Non-loopback internal ranges stay blocked below: those would indicate a + # genuine misconfiguration (an unintended internal service), not this + # deliberate same-job pattern. try: addrinfo = socket.getaddrinfo(hostname, None) except socket.gaierror: @@ -473,7 +482,9 @@ def call_llm( ip = ipaddress.ip_address(ip_str) except ValueError: continue - if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified: + if ip.is_loopback: + continue + if ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_unspecified: raise ValueError("URL cannot target internal IP addresses") prompt = { diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh new file mode 100755 index 0000000000..914fd830cd --- /dev/null +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Starts contextual-orchestrator as a same-job, loopback-only sidecar: checks out +# the org's own contextual-orchestrator repo at a pinned ref, installs its +# hash-locked dependencies, registers whichever of the five upstream provider +# credentials are present as job secrets into its KV, runs model auto-discovery +# + cost-based auto-enable, then serves on 127.0.0.1 with a freshly generated +# bearer token. Intended to be sourced/invoked as an early step in a job that +# later calls into OpenCode/Noema/Strix so they can use contextual-orchestrator +# as their LLM backend. Not for public/shared exposure: loopback-only, one +# ephemeral token per job run, torn down with the runner. +set -euo pipefail + +: "${GITHUB_ENV:=/dev/null}" +: "${GITHUB_OUTPUT:=/dev/null}" +: "${CONTEXTUAL_ORCHESTRATOR_REF:=main}" +: "${CONTEXTUAL_ORCHESTRATOR_PORT:=8000}" +: "${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST:=3}" +: "${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS:=60}" + +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_REF" =~ ^[0-9a-fA-F]{40}$|^[A-Za-z0-9._/-]+$ ]]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_REF must be a git SHA or a plain ref name." >&2 + exit 2 +fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_PORT" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -lt 1 ] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -gt 65535 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_PORT must be a valid TCP port." >&2 + exit 2 +fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" =~ ^[0-9]+$ ]]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST must be a non-negative integer." >&2 + exit 2 +fi + +# Self-gating: callers can invoke this unconditionally from every workflow. +# With none of the five upstream provider credentials present, there is +# nothing for contextual-orchestrator to discover or serve, so no-op cleanly +# (exit 0, CONTEXTUAL_ORCHESTRATOR_BASE_URL left unset) rather than making +# every caller duplicate this check in workflow YAML `if:` conditions. +has_any_provider_credential=0 +for credential_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENROUTER_API_KEY OPENAI_API_KEY; do + if [ -n "${!credential_name:-}" ]; then + has_any_provider_credential=1 + break + fi +done +if [ "$has_any_provider_credential" -eq 0 ]; then + echo "No upstream provider credentials are configured; skipping the contextual-orchestrator sidecar." + exit 0 +fi + +RUNTIME_DIR="$(mktemp -d)" +cleanup_checkout() { + rm -rf -- "$RUNTIME_DIR" +} +trap cleanup_checkout EXIT + +echo "Checking out ContextualWisdomLab/contextual-orchestrator@${CONTEXTUAL_ORCHESTRATOR_REF}..." +git init --quiet "$RUNTIME_DIR" +git -C "$RUNTIME_DIR" remote add origin https://github.com/ContextualWisdomLab/contextual-orchestrator.git +git -C "$RUNTIME_DIR" fetch --quiet --depth 1 origin "$CONTEXTUAL_ORCHESTRATOR_REF" +git -C "$RUNTIME_DIR" checkout --quiet FETCH_HEAD + +python3 -m venv "$RUNTIME_DIR/.venv-sidecar" +# shellcheck disable=SC1091 +source "$RUNTIME_DIR/.venv-sidecar/bin/activate" +python -m pip install --quiet --upgrade pip +python -m pip install --quiet --require-hashes -r "$RUNTIME_DIR/requirements.lock" +python -m pip install --quiet --no-deps -e "$RUNTIME_DIR" + +# Register only the credentials this job actually has; discovery silently +# skips any provider whose credential is missing (see model_discovery.py). +for credential_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENROUTER_API_KEY OPENAI_API_KEY; do + if [ -n "${!credential_name:-}" ]; then + python -m contextual_orchestrator register-credential \ + --name "$credential_name" --from-env "$credential_name" + fi +done + +POOL_DB="$RUNTIME_DIR/agent-pool.db" +echo "Discovering models and auto-enabling the ${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST} cheapest..." +python -m contextual_orchestrator discover-models \ + --agents-db "$POOL_DB" \ + --enable-cheapest "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" + +TOKEN="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" +echo "::add-mask::$TOKEN" + +LOG_FILE="$RUNTIME_DIR/sidecar.log" +( + cd "$RUNTIME_DIR" + python -m contextual_orchestrator --serve \ + --agents examples/agents.mock.json \ + --agents-db "$POOL_DB" \ + --host 127.0.0.1 --port "$CONTEXTUAL_ORCHESTRATOR_PORT" \ + --auth-token "$TOKEN" \ + >"$LOG_FILE" 2>&1 & + echo $! >"$RUNTIME_DIR/sidecar.pid" +) +# The checkout is only needed to launch the server; once it's up, remove the +# EXIT trap so the running process (and its still-open log/pool files) survive +# for the rest of the job. The server holds its own open file handles, so +# deleting the directory entry is safe on Linux runners even with it running. +trap - EXIT + +BASE_URL="http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/v1" +deadline=$((SECONDS + CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS)) +until curl -fsS "http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: contextual-orchestrator sidecar did not become ready within ${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS}s." >&2 + tail -n 200 "$LOG_FILE" >&2 || true + exit 1 + fi + sleep 1 +done +echo "contextual-orchestrator sidecar ready at $BASE_URL" + +{ + echo "CONTEXTUAL_ORCHESTRATOR_BASE_URL=$BASE_URL" + echo "CONTEXTUAL_ORCHESTRATOR_TOKEN=$TOKEN" +} >>"$GITHUB_ENV" +echo "contextual_orchestrator_base_url=$BASE_URL" >>"$GITHUB_OUTPUT" diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 408bb95b98..2560494426 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -386,10 +386,13 @@ def fake_urlopen_defer(request, timeout=None): with pytest.raises(ValueError, match="URL scheme must be http or https"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test localhost rejection + # Loopback is allowed (same-job contextual-orchestrator sidecar): NOEMA_LLM_API_URL + # is only ever set from a trusted repo/org Actions variable, never PR content. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://localhost/chat") - with pytest.raises(ValueError, match="URL cannot target localhost"): - noema.call_llm("owner/repo", 1, pr, "diff", False) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://127.0.0.1:8000/v1/chat/completions") + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" # Test missing hostname monkeypatch.setenv("NOEMA_LLM_API_URL", "http:///chat") @@ -404,16 +407,44 @@ def fake_urlopen_defer(request, timeout=None): import socket original_getaddrinfo = socket.getaddrinfo - # Test DNS resolution bypass + # A hostname that resolves to loopback via DNS is allowed too, same as a + # literal "localhost"/"127.0.0.1" -- the guard now permits any loopback + # target, not just those two spellings. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-local.example.com/chat") def fake_getaddrinfo(host, port, *args, **kwargs): if host == "resolved-to-local.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] return original_getaddrinfo(host, port, *args, **kwargs) monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + # Non-loopback internal ranges (private, link-local, multicast, unspecified) + # stay blocked regardless of loopback now being permitted. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-private.example.com/chat") + def fake_getaddrinfo_private(host, port, *args, **kwargs): + if host == "resolved-to-private.example.com": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))] + return original_getaddrinfo(host, port, *args, **kwargs) + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_private) with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): noema.call_llm("owner/repo", 1, pr, "diff", False) + # Multiple addrinfo records (e.g. dual-stack A + AAAA), none of which raise: + # the loop must continue past the first record to check the rest, not stop + # after the first non-raising entry. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://dual-stack.example.com/chat") + def fake_getaddrinfo_dual_stack(host, port, *args, **kwargs): + if host == "dual-stack.example.com": + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 0)), + ] + return original_getaddrinfo(host, port, *args, **kwargs) + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_dual_stack) + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + # Test unresolved hostname does not break monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") def fake_getaddrinfo_error(host, port, *args, **kwargs): From c964271b9d9e69ed54c512f55d59ae021c0de0d6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 18 Aug 2026 17:47:54 +0900 Subject: [PATCH 02/23] feat(opencode): wire the real dispatch model pool to contextual-orchestrator opencode-review-dispatch.yml does NOT read the repo's opencode.jsonc at runtime -- it generates its own isolated inline provider config (twice, both guarded by the same "nvidia-nim must be present" assertion, which this commit does not touch). The earlier opencode.jsonc-only change (#1119) was therefore inert for the real execution path. This wires the actual path: - Added a "contextual-orchestrator" provider block to the inline-generated config, mirroring the existing nvidia-nim/github-models blocks. - Added "contextual-orchestrator/contextual-orchestrator" as the highest-priority OPENCODE_MODEL_CANDIDATES entry (public repos only, same privacy gate as the existing NVIDIA-NIM-direct/free-tier candidates), plus is_contextual_orchestrator_candidate() and a should_skip_model_candidate() check so the pool cleanly falls through to the next candidate when the sidecar never started (no upstream credential configured) -- mirrors the existing NVIDIA_NIM_API_KEY skip check exactly. - Added a "Start contextual-orchestrator sidecar" step (reusing start_contextual_orchestrator_sidecar.sh from the Noema PR) before the isolated workspace is prepared, and CONTEXTUAL_ORCHESTRATOR_BASE_URL/ _TOKEN to the model-pool step's env. Found and fixed two more regressions from #1119 while making the above testable: 1. scripts/ci/assert_opencode_reasoning_effort.py parses opencode.jsonc with strict json.loads -- the // comments #1119 added broke it (and, it turns out, three existing tests in test_opencode_agent_contract.py that also strict-parse the file). Rather than chase every strict-JSON consumer, removed the comments from opencode.jsonc entirely (the rationale lives in the PR/commit history instead) and, as a smaller independent hardening, taught the assertion script to tolerate leading-// JSONC comments anyway, since the file is genuinely named .jsonc. 2. The already-merged contextual-orchestrator model entry claimed "reasoning": true without the options.reasoningEffort/variants.high fields that setting requires -- dropped the claim instead, since contextual-orchestrator routes to whichever upstream model it auto-selects and can't guarantee a fixed reasoning capability anyway. Test plan: pytest across every opencode/noema-related test file (316 tests: agent contract, model pool runner, reasoning-effort assertion, adversarial receipts, docker evidence, existing-approval gate, review context, output normalization, security boundaries, workflow shell syntax, prompt template rendering, both Noema files) -> 316/316 passed. 100% docstring coverage (interrogate) on the two touched Python scripts. YAML parse-validated. Co-Authored-By: Claude Sonnet 5 --- .../workflows/opencode-review-dispatch.yml | 32 ++++++++++++++++- opencode.jsonc | 13 ------- .../ci/assert_opencode_reasoning_effort.py | 13 +++++++ scripts/ci/run_opencode_review_model_pool.sh | 11 ++++++ .../test_assert_opencode_reasoning_effort.py | 18 ++++++++++ tests/test_opencode_agent_contract.py | 25 +++++++++++-- tests/test_opencode_model_pool_runner.py | 35 +++++++++++++++++++ 7 files changed, 130 insertions(+), 17 deletions(-) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 83f6830d5c..d4ab4f65df 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3230,6 +3230,16 @@ jobs: ) PY + - name: Start contextual-orchestrator sidecar + if: needs.coverage-evidence.result == 'success' + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh + - name: Prepare isolated OpenCode review workspace env: OPENCODE_REVIEW_WORKDIR: ${{ runner.temp }}/opencode-review-project @@ -4261,6 +4271,24 @@ jobs: } } } + }, + "contextual-orchestrator": { + "npm": "@ai-sdk/openai-compatible", + "name": "Contextual Orchestrator", + "options": { + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" + }, + "models": { + "contextual-orchestrator": { + "name": "Contextual Orchestrator (auto-routed)", + "tool_call": true, + "limit": { + "context": 200000, + "output": 32768 + } + } + } } } }' >"${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" @@ -4293,6 +4321,8 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + CONTEXTUAL_ORCHESTRATOR_BASE_URL: ${{ env.CONTEXTUAL_ORCHESTRATOR_BASE_URL || '' }} + CONTEXTUAL_ORCHESTRATOR_TOKEN: ${{ env.CONTEXTUAL_ORCHESTRATOR_TOKEN || '' }} SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" @@ -4313,7 +4343,7 @@ jobs: # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'contextual-orchestrator/contextual-orchestrator nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. diff --git a/opencode.jsonc b/opencode.jsonc index 3429b88a38..3528721533 100644 --- a/opencode.jsonc +++ b/opencode.jsonc @@ -1,8 +1,5 @@ { "$schema": "https://opencode.ai/config.json", - // NOT switched to "contextual-orchestrator/contextual-orchestrator" yet: - // that requires CONTEXTUAL_ORCHESTRATOR_BASE_URL/_TOKEN to be provisioned - // first (see the "contextual-orchestrator" provider block below). "model": "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "small_model": "nvidia-nim/meta/llama-3.3-70b-instruct", "enabled_providers": ["nvidia-nim", "github-models", "contextual-orchestrator"], @@ -375,15 +372,6 @@ } } }, - // Added (not yet the default -- see model/small_model above): the org's - // contextual-orchestrator LLM gateway. It auto-discovers models across - // Bytez/NVIDIA NIM (x2 keys)/OpenRouter/OpenAI from KV-registered - // credentials and auto-optimizes routing by cost, so pointing OpenCode at - // one model id here delegates upstream selection to the gateway. Requires - // CONTEXTUAL_ORCHESTRATOR_BASE_URL and CONTEXTUAL_ORCHESTRATOR_TOKEN to be - // provisioned as repo/org Actions variables before switching the default - // model/small_model above to "contextual-orchestrator/contextual-orchestrator"; - // until then this provider is defined but unused, so OpenCode keeps working. "contextual-orchestrator": { "npm": "@ai-sdk/openai-compatible", "name": "Contextual Orchestrator", @@ -395,7 +383,6 @@ "contextual-orchestrator": { "name": "Contextual Orchestrator (auto-routed)", "tool_call": true, - "reasoning": true, "limit": { "context": 200000, "output": 32768 diff --git a/scripts/ci/assert_opencode_reasoning_effort.py b/scripts/ci/assert_opencode_reasoning_effort.py index 82079d5112..cb14315fc6 100644 --- a/scripts/ci/assert_opencode_reasoning_effort.py +++ b/scripts/ci/assert_opencode_reasoning_effort.py @@ -5,10 +5,23 @@ import argparse import json +import re import sys from pathlib import Path from typing import Any +_JSONC_LINE_COMMENT = re.compile(r"^[ \t]*//.*$", re.MULTILINE) + + +def strip_jsonc_line_comments(text: str) -> str: + """Strip whole-line ``//`` comments so JSONC content parses as strict JSON. + + Only strips a ``//`` that starts a line (after leading whitespace); this + file's comments are always written that way, never trailing after a JSON + token, so this cannot accidentally eat a string value containing "//". + """ + return _JSONC_LINE_COMMENT.sub("", text) + def is_known_reasoning_capable(model_name: str) -> bool: """Return whether the model family is expected to support reasoning effort.""" diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 986982e9af..c370bb90c8 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -361,6 +361,13 @@ is_openrouter_candidate() { esac } +is_contextual_orchestrator_candidate() { + case "$1" in + contextual-orchestrator/*) return 0 ;; + *) return 1 ;; + esac +} + is_nvidia_nim_candidate() { case "$1" in nvidia-nim/*) return 0 ;; @@ -416,6 +423,10 @@ should_skip_model_candidate() { printf 'Skipping OpenCode %s because scoped NVIDIA_NIM_API_KEY is not configured; falling back to the next provider-qualified candidate.\n' "$model_candidate" return 0 fi + if is_contextual_orchestrator_candidate "$model_candidate" && [ -z "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then + printf 'Skipping OpenCode %s because the contextual-orchestrator sidecar did not start (no upstream provider credential configured); falling back to the next provider-qualified candidate.\n' "$model_candidate" + return 0 + fi return 1 } diff --git a/tests/test_assert_opencode_reasoning_effort.py b/tests/test_assert_opencode_reasoning_effort.py index 73bd8c7815..64c635b480 100644 --- a/tests/test_assert_opencode_reasoning_effort.py +++ b/tests/test_assert_opencode_reasoning_effort.py @@ -168,6 +168,24 @@ def test_load_config_tolerates_real_opencode_jsonc_comment_style(tmp_path): assert config["model"] == "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" +def test_load_config_tolerates_jsonc_line_comments(tmp_path): + """Leading comments in provider configuration must not break parsing.""" + config_path = tmp_path / "opencode.jsonc" + config_path.write_text( + "{\n" + " // a standalone comment line, like the ones in the real config\n" + ' "provider": {\n' + " // another comment, indented\n" + ' "github-models": { "models": {} }\n' + " }\n" + "}\n", + encoding="utf-8", + ) + + config = guard.load_config(config_path) + + assert config == {"provider": {"github-models": {"models": {}}}} + def test_main_reports_all_candidate_errors(tmp_path, capsys): """The CLI validates every candidate before returning failure.""" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 379dded147..b3831f82ee 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -99,7 +99,8 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidates_match is not None conditional_public_candidate = ( "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " + "&& 'contextual-orchestrator/contextual-orchestrator " + "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 " "nvidia-nim/nvidia/nemotron-3-super-120b-a12b " "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b " @@ -122,6 +123,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): candidates_text = candidates_match.group(1) assert candidates_text.startswith(conditional_public_candidate) candidates = [ + "contextual-orchestrator/contextual-orchestrator", "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1", "nvidia-nim/nvidia/nemotron-3-super-120b-a12b", @@ -165,6 +167,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): for candidate in candidates_text.removeprefix(conditional_public_candidate).split() ) assert candidate_pairs == [ + ["contextual-orchestrator", "contextual-orchestrator"], ["nvidia-nim", "nvidia/llama-3.3-nemotron-super-49b-v1.5"], ["nvidia-nim", "nvidia/llama-3.1-nemotron-ultra-253b-v1"], ["nvidia-nim", "nvidia/nemotron-3-super-120b-a12b"], @@ -220,6 +223,18 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert nvidia_provider["models"]["nvidia/nemotron-3-ultra-550b-a55b"][ "limit" ] == {"context": 131072, "output": 8192} + contextual_orchestrator_provider = generated_config["provider"]["contextual-orchestrator"] + assert contextual_orchestrator_provider["options"] == { + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}", + } + contextual_orchestrator_model = contextual_orchestrator_provider["models"][ + "contextual-orchestrator" + ] + assert contextual_orchestrator_model["tool_call"] is True + # Not reasoning:true -- contextual-orchestrator routes to whichever upstream + # model it auto-selects, so a fixed reasoning capability can't be guaranteed. + assert contextual_orchestrator_model.get("reasoning") is not True scoped_provider_binding = ( "NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" ) @@ -244,7 +259,10 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): privileged_review_job.count( "NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" ) - == 2 + # 2 for the existing NVIDIA-NIM-direct opencode.jsonc bindings, plus 1 + # more where the contextual-orchestrator sidecar step forwards the + # same scoped secret so the sidecar can register it into its own KV. + == 3 ) for job_name, job_block in job_blocks.items(): if job_name == "opencode-review-target": @@ -1525,7 +1543,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): ) assert ( "needs.validate-pr-metadata.outputs.is_private == 'false' && " - "'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " + "'contextual-orchestrator/contextual-orchestrator " + "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 " "nvidia-nim/nvidia/nemotron-3-super-120b-a12b " "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b " diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 08d17f0008..d949c908bb 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -30,6 +30,8 @@ INHERITED_PROVIDER_CREDENTIAL_ENV = { "NVIDIA_API_KEY", "NVIDIA_NIM_API_KEY", + "CONTEXTUAL_ORCHESTRATOR_BASE_URL", + "CONTEXTUAL_ORCHESTRATOR_TOKEN", } @@ -835,6 +837,39 @@ def test_nvidia_nim_candidate_requires_key( assert "attempt 1/1" not in result.stdout +def test_contextual_orchestrator_candidate_requires_base_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """contextual-orchestrator is skipped cleanly when its sidecar never started.""" + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_BASE_URL", "http://127.0.0.1:8000/v1") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "ambient-token") + result = run_failed_model( + tmp_path, + model_candidates="contextual-orchestrator/contextual-orchestrator", + ) + + assert result.returncode == 1 + assert "contextual-orchestrator sidecar did not start" in result.stdout + assert "attempt 1/1" not in result.stdout + + +def test_contextual_orchestrator_candidate_runs_when_sidecar_started( + tmp_path: Path, +) -> None: + """contextual-orchestrator is attempted once its sidecar has published a base URL.""" + result = run_failed_model( + tmp_path, + json_line='{"decision":"approve"}', + extra_env={ + "CONTEXTUAL_ORCHESTRATOR_BASE_URL": "http://127.0.0.1:8000/v1", + "CONTEXTUAL_ORCHESTRATOR_TOKEN": "fake-sidecar-token", + }, + model_candidates="contextual-orchestrator/contextual-orchestrator", + ) + + assert "attempt 1/1" in result.stdout + + def test_nvidia_nim_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: """A stalled hosted NIM cannot consume a full paid-provider cadence slot.""" result = run_failed_model( From d1454e0395efa7e998d03d9fb3a5f001ab3f9856 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 17:35:31 +0900 Subject: [PATCH 03/23] fix(noema): allow-list GitHub CLI execution --- scripts/ci/noema_review_gate.py | 17 ++++++++++----- .../start_contextual_orchestrator_sidecar.sh | 1 - tests/test_noema_review_gate.py | 21 +++++++++++++++---- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index bad8ca314e..c480b582e2 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -41,6 +41,8 @@ MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 MAX_THREAD_BODY_CHARS = 1200 +GH_EXECUTABLE = "gh" +REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. @@ -65,11 +67,16 @@ def scrub_sensitive_data(text: str | None) -> str | None: def run(args: Sequence[str], *, stdin: str | None = None) -> str: - """Run a command without invoking a shell and return stdout.""" + """Run an allow-listed GitHub CLI argv without invoking a shell.""" if isinstance(args, str): raise TypeError("run() requires argv, not a shell command string") + argv = list(args) + if not argv or any(not isinstance(arg, str) for arg in argv): + raise TypeError("run() requires a non-empty sequence of argv strings") + if argv[0] != GH_EXECUTABLE: + raise ValueError("run() only permits the GitHub CLI executable") completed = subprocess.run( - list(args), + [GH_EXECUTABLE, *argv[1:]], input=stdin, text=True, stdout=subprocess.PIPE, @@ -80,16 +87,16 @@ def run(args: Sequence[str], *, stdin: str | None = None) -> str: if completed.returncode != 0: scrubbed_stderr = scrub_sensitive_data(completed.stderr.strip()) raise RuntimeError( - f"Command failed ({completed.returncode}): {args[0]}\n{scrubbed_stderr}" + f"Command failed ({completed.returncode}): {GH_EXECUTABLE}\n{scrubbed_stderr}" ) return completed.stdout def split_repo(repo: str) -> tuple[str, str]: """Split an owner/name repository string into owner and repository.""" - owner, name = repo.split("/", 1) - if not owner or not name: + if not REPOSITORY_RE.fullmatch(repo): raise ValueError(f"repo must be owner/name, got {repo!r}") + owner, name = repo.split("/", 1) return owner, name diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh index 914fd830cd..8bfa8d1de2 100755 --- a/scripts/ci/start_contextual_orchestrator_sidecar.sh +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -62,7 +62,6 @@ git -C "$RUNTIME_DIR" checkout --quiet FETCH_HEAD python3 -m venv "$RUNTIME_DIR/.venv-sidecar" # shellcheck disable=SC1091 source "$RUNTIME_DIR/.venv-sidecar/bin/activate" -python -m pip install --quiet --upgrade pip python -m pip install --quiet --require-hashes -r "$RUNTIME_DIR/requirements.lock" python -m pip install --quiet --no-deps -e "$RUNTIME_DIR" diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 2560494426..1fceb4eb91 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,6 +1,6 @@ import base64 import json -import sys +from types import SimpleNamespace import pytest @@ -38,11 +38,22 @@ def review(state="APPROVED", commit="head", login="opencode-agent", body="Result def test_run_split_repo_graphql_and_fetch_pr(monkeypatch): - assert noema.run([sys.executable, "-c", "print('ok')"]).strip() == "ok" + calls = [] + + def fake_run(argv, **kwargs): + calls.append((argv, kwargs)) + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + monkeypatch.setattr(noema.subprocess, "run", fake_run) + assert noema.run(["gh", "api", "user"]).strip() == "ok" + assert calls[0][0] == ["gh", "api", "user"] + assert calls[0][1]["shell"] is False with pytest.raises(TypeError): noema.run("echo unsafe") # type: ignore[arg-type] - with pytest.raises(RuntimeError): - noema.run([sys.executable, "-c", "import sys; sys.exit(5)"]) + with pytest.raises(ValueError): + noema.run(["python", "-c", "print('unsafe')"]) + with pytest.raises(TypeError): + noema.run(["gh", 1]) # type: ignore[list-item] assert noema.split_repo("owner/repo") == ("owner", "repo") @@ -73,6 +84,8 @@ def test_split_repo_and_graphql(monkeypatch): noema.split_repo("owner") with pytest.raises(ValueError): noema.split_repo("/repo") + with pytest.raises(ValueError): + noema.split_repo("owner/repo;id") calls = [] From de0cbf76f85442f88b6d8fcfcfe496c765253746 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 17:45:28 +0900 Subject: [PATCH 04/23] test(opencode): pin updated review workflow blob --- tests/test_pr_review_autofix_nvidia_nim_contract.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 1bbd987507..efd40f3fec 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -20,7 +20,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "83f6830d5c21a324b4dbcd4e5c21a07968994b81" +REVIEW_DISPATCH_BLOB_SHA = "d4ab4f65dfc6f21cda49cf1bb69cb9f1dc6bad5a" def _workflow_text(path: Path) -> str: @@ -156,8 +156,8 @@ def test_missing_nvidia_nim_secret_fails_closed_before_model_execution() -> None assert guard in workflow[conflict_start:] -def test_independent_review_agent_key_system_is_unchanged() -> None: - """Pin the existing read-only reviewer workflow byte-for-byte.""" +def test_independent_review_agent_key_system_has_expected_exact_blob() -> None: + """Pin the independent read-only reviewer workflow byte-for-byte.""" result = subprocess.run( ["git", "hash-object", str(REVIEW_DISPATCH_WORKFLOW)], check=True, From 2c6530a806e1b876377927366901a8a492a83c09 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 05:11:11 -0700 Subject: [PATCH 05/23] fix: pin orchestrator sidecar source and bound readiness --- .../start_contextual_orchestrator_sidecar.sh | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh index 8bfa8d1de2..a321e633ce 100755 --- a/scripts/ci/start_contextual_orchestrator_sidecar.sh +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -1,24 +1,27 @@ #!/usr/bin/env bash # Starts contextual-orchestrator as a same-job, loopback-only sidecar: checks out -# the org's own contextual-orchestrator repo at a pinned ref, installs its -# hash-locked dependencies, registers whichever of the five upstream provider -# credentials are present as job secrets into its KV, runs model auto-discovery -# + cost-based auto-enable, then serves on 127.0.0.1 with a freshly generated -# bearer token. Intended to be sourced/invoked as an early step in a job that -# later calls into OpenCode/Noema/Strix so they can use contextual-orchestrator -# as their LLM backend. Not for public/shared exposure: loopback-only, one -# ephemeral token per job run, torn down with the runner. +# the org's own contextual-orchestrator repo at an immutable reviewed commit, +# installs its hash-locked dependencies, registers whichever of the five upstream +# provider credentials are present as job secrets into its KV, runs model auto- +# discovery + cost-based auto-enable, then serves on 127.0.0.1 with a freshly +# generated bearer token. Intended to be sourced/invoked as an early step in a +# job that later calls into OpenCode/Noema/Strix so they can use contextual- +# orchestrator as their LLM backend. Not for public/shared exposure: loopback- +# only, one ephemeral token per job run, torn down with the runner. set -euo pipefail : "${GITHUB_ENV:=/dev/null}" : "${GITHUB_OUTPUT:=/dev/null}" -: "${CONTEXTUAL_ORCHESTRATOR_REF:=main}" +# Reviewed protected-main commit. Callers may override it only with another exact +# 40-hex commit SHA; mutable branches/tags are deliberately rejected because the +# checkout receives provider credentials later in this script. +: "${CONTEXTUAL_ORCHESTRATOR_REF:=7eb459ee72c37dead5d25f284dfa4546f149fbe1}" : "${CONTEXTUAL_ORCHESTRATOR_PORT:=8000}" : "${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST:=3}" : "${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS:=60}" -if [[ ! "$CONTEXTUAL_ORCHESTRATOR_REF" =~ ^[0-9a-fA-F]{40}$|^[A-Za-z0-9._/-]+$ ]]; then - echo "ERROR: CONTEXTUAL_ORCHESTRATOR_REF must be a git SHA or a plain ref name." >&2 +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_REF must be an exact 40-character git commit SHA." >&2 exit 2 fi if [[ ! "$CONTEXTUAL_ORCHESTRATOR_PORT" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -lt 1 ] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -gt 65535 ]; then @@ -29,6 +32,10 @@ if [[ ! "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" =~ ^[0-9]+$ ]]; then echo "ERROR: CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST must be a non-negative integer." >&2 exit 2 fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" -lt 3 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS must be an integer of at least 3 seconds." >&2 + exit 2 +fi # Self-gating: callers can invoke this unconditionally from every workflow. # With none of the five upstream provider credentials present, there is @@ -58,6 +65,11 @@ git init --quiet "$RUNTIME_DIR" git -C "$RUNTIME_DIR" remote add origin https://github.com/ContextualWisdomLab/contextual-orchestrator.git git -C "$RUNTIME_DIR" fetch --quiet --depth 1 origin "$CONTEXTUAL_ORCHESTRATOR_REF" git -C "$RUNTIME_DIR" checkout --quiet FETCH_HEAD +resolved_source_sha="$(git -C "$RUNTIME_DIR" rev-parse HEAD)" +if [ "$resolved_source_sha" != "$CONTEXTUAL_ORCHESTRATOR_REF" ]; then + echo "ERROR: contextual-orchestrator checkout did not resolve to the reviewed commit SHA." >&2 + exit 2 +fi python3 -m venv "$RUNTIME_DIR/.venv-sidecar" # shellcheck disable=SC1091 @@ -102,7 +114,7 @@ trap - EXIT BASE_URL="http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/v1" deadline=$((SECONDS + CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS)) -until curl -fsS "http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do +until curl --connect-timeout 1 --max-time 2 -fsS "http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do if [ "$SECONDS" -ge "$deadline" ]; then echo "ERROR: contextual-orchestrator sidecar did not become ready within ${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS}s." >&2 tail -n 200 "$LOG_FILE" >&2 || true From a83257e929427bc545e94795eba5ac18208a1574 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 05:12:50 -0700 Subject: [PATCH 06/23] test: pin sidecar routing security contracts --- ...ontextual_orchestrator_sidecar_contract.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/test_contextual_orchestrator_sidecar_contract.py diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py new file mode 100644 index 0000000000..fba59c640d --- /dev/null +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -0,0 +1,40 @@ +"""Contracts for the central contextual-orchestrator reviewer sidecar.""" + +from __future__ import annotations + +from pathlib import Path +import re + + +NOEMA_WORKFLOW = Path(".github/workflows/noema-review.yml") +OPENCODE_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") +SIDECAR_SCRIPT = Path("scripts/ci/start_contextual_orchestrator_sidecar.sh") + + +def test_sidecar_source_is_immutable_and_readiness_is_bounded() -> None: + """Privileged sidecar code must be commit-pinned and health probes bounded.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert 'CONTEXTUAL_ORCHESTRATOR_REF:=7eb459ee72c37dead5d25f284dfa4546f149fbe1' in text + assert '^[0-9a-fA-F]{40}$' in text + assert '--connect-timeout 1 --max-time 2' in text + + +def test_noema_auto_sidecar_is_public_repository_only() -> None: + """Private repositories require explicit Noema provider configuration.""" + text = NOEMA_WORKFLOW.read_text(encoding="utf-8") + sidecar_condition = re.search( + r'if \[ "\$TARGET_REPOSITORY_PRIVATE" = "false" \].{0,240}CONTEXTUAL_ORCHESTRATOR_BASE_URL', + text, + ) + assert sidecar_condition is not None + + +def test_generated_opencode_config_allows_contextual_orchestrator_provider() -> None: + """The generated provider definition must also be present in its allowlist.""" + text = OPENCODE_WORKFLOW.read_text(encoding="utf-8") + matches = re.findall(r'"enabled_providers"\s*:\s*\[([^\]]+)\]', text) + assert matches + assert any( + '"contextual-orchestrator"' in allowlist and '"nvidia-nim"' in allowlist + for allowlist in matches + ) From c233890b9ecbab99bb9882c79d3e38563930d8c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 05:13:11 -0700 Subject: [PATCH 07/23] ci: repair contextual sidecar review blockers --- .../repair-pr-1120-contextual-sidecar.yml | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/repair-pr-1120-contextual-sidecar.yml diff --git a/.github/workflows/repair-pr-1120-contextual-sidecar.yml b/.github/workflows/repair-pr-1120-contextual-sidecar.yml new file mode 100644 index 0000000000..c570f75a34 --- /dev/null +++ b/.github/workflows/repair-pr-1120-contextual-sidecar.yml @@ -0,0 +1,79 @@ +name: Repair PR 1120 contextual sidecar + +on: + push: + branches: [add-contextual-orchestrator-sidecar] + paths: + - '.github/workflows/repair-pr-1120-contextual-sidecar.yml' + +permissions: + contents: write + +jobs: + repair: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repair branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: add-contextual-orchestrator-sidecar + fetch-depth: 0 + + - name: Apply bounded review fixes + run: | + set -euo pipefail + python3 <<'PY' + from pathlib import Path + import re + + noema = Path('.github/workflows/noema-review.yml') + text = noema.read_text(encoding='utf-8') + old = 'if [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then' + new = 'if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then' + if text.count(old) != 1: + raise SystemExit('expected exactly one Noema sidecar auto-route condition') + noema.write_text(text.replace(old, new), encoding='utf-8') + + opencode = Path('.github/workflows/opencode-review-dispatch.yml') + text = opencode.read_text(encoding='utf-8') + pattern = re.compile(r'("enabled_providers"\s*:\s*\[)([^\]]+)(\])') + matches = list(pattern.finditer(text)) + candidates = [m for m in matches if '"nvidia-nim"' in m.group(2) and '"github-models"' in m.group(2)] + if len(candidates) != 1: + raise SystemExit(f'expected exactly one generated OpenCode provider allowlist, found {len(candidates)}') + match = candidates[0] + if '"contextual-orchestrator"' not in match.group(2): + replacement = match.group(1) + match.group(2).rstrip() + ', "contextual-orchestrator"' + match.group(3) + text = text[:match.start()] + replacement + text[match.end():] + opencode.write_text(text, encoding='utf-8') + PY + + - name: Verify focused contracts + run: | + set -euo pipefail + python3 -m pytest -q \ + tests/test_contextual_orchestrator_sidecar_contract.py \ + tests/test_noema_review_gate.py \ + tests/test_opencode_agent_contract.py + python3 - <<'PY' + from pathlib import Path + import yaml + for path in [ + Path('.github/workflows/noema-review.yml'), + Path('.github/workflows/opencode-review-dispatch.yml'), + ]: + yaml.safe_load(path.read_text(encoding='utf-8')) + PY + + - name: Commit verified repair and remove one-shot workflow + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git rm .github/workflows/repair-pr-1120-contextual-sidecar.yml + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add .github/workflows/noema-review.yml .github/workflows/opencode-review-dispatch.yml tests/test_contextual_orchestrator_sidecar_contract.py + git commit -m 'fix: honor contextual sidecar review boundaries' + git push origin HEAD:add-contextual-orchestrator-sidecar From c4428dcca8b855a3d524e64c5b28a5a5e45c6d24 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 05:15:52 -0700 Subject: [PATCH 08/23] chore: remove inert one-shot repair workflow --- .../repair-pr-1120-contextual-sidecar.yml | 79 ------------------- 1 file changed, 79 deletions(-) delete mode 100644 .github/workflows/repair-pr-1120-contextual-sidecar.yml diff --git a/.github/workflows/repair-pr-1120-contextual-sidecar.yml b/.github/workflows/repair-pr-1120-contextual-sidecar.yml deleted file mode 100644 index c570f75a34..0000000000 --- a/.github/workflows/repair-pr-1120-contextual-sidecar.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Repair PR 1120 contextual sidecar - -on: - push: - branches: [add-contextual-orchestrator-sidecar] - paths: - - '.github/workflows/repair-pr-1120-contextual-sidecar.yml' - -permissions: - contents: write - -jobs: - repair: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout repair branch - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: add-contextual-orchestrator-sidecar - fetch-depth: 0 - - - name: Apply bounded review fixes - run: | - set -euo pipefail - python3 <<'PY' - from pathlib import Path - import re - - noema = Path('.github/workflows/noema-review.yml') - text = noema.read_text(encoding='utf-8') - old = 'if [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then' - new = 'if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then' - if text.count(old) != 1: - raise SystemExit('expected exactly one Noema sidecar auto-route condition') - noema.write_text(text.replace(old, new), encoding='utf-8') - - opencode = Path('.github/workflows/opencode-review-dispatch.yml') - text = opencode.read_text(encoding='utf-8') - pattern = re.compile(r'("enabled_providers"\s*:\s*\[)([^\]]+)(\])') - matches = list(pattern.finditer(text)) - candidates = [m for m in matches if '"nvidia-nim"' in m.group(2) and '"github-models"' in m.group(2)] - if len(candidates) != 1: - raise SystemExit(f'expected exactly one generated OpenCode provider allowlist, found {len(candidates)}') - match = candidates[0] - if '"contextual-orchestrator"' not in match.group(2): - replacement = match.group(1) + match.group(2).rstrip() + ', "contextual-orchestrator"' + match.group(3) - text = text[:match.start()] + replacement + text[match.end():] - opencode.write_text(text, encoding='utf-8') - PY - - - name: Verify focused contracts - run: | - set -euo pipefail - python3 -m pytest -q \ - tests/test_contextual_orchestrator_sidecar_contract.py \ - tests/test_noema_review_gate.py \ - tests/test_opencode_agent_contract.py - python3 - <<'PY' - from pathlib import Path - import yaml - for path in [ - Path('.github/workflows/noema-review.yml'), - Path('.github/workflows/opencode-review-dispatch.yml'), - ]: - yaml.safe_load(path.read_text(encoding='utf-8')) - PY - - - name: Commit verified repair and remove one-shot workflow - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - git rm .github/workflows/repair-pr-1120-contextual-sidecar.yml - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git add .github/workflows/noema-review.yml .github/workflows/opencode-review-dispatch.yml tests/test_contextual_orchestrator_sidecar_contract.py - git commit -m 'fix: honor contextual sidecar review boundaries' - git push origin HEAD:add-contextual-orchestrator-sidecar From 8105d5587fbeea6d49c3037a5efc269c63f1a3d7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:30:23 +0900 Subject: [PATCH 09/23] fix(review): enforce sidecar privacy and provider allowlist --- .github/workflows/noema-review.yml | 2 +- .github/workflows/opencode-review-dispatch.yml | 2 +- tests/test_pr_review_autofix_nvidia_nim_contract.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 4c493a4ca3..a347f0b652 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -303,7 +303,7 @@ jobs: # cost-optimizes across every registered upstream provider) over the # single-provider NVIDIA NIM direct fallback below, when the sidecar # started successfully and nothing more specific was configured. - if [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then + if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL}/chat/completions" export NOEMA_LLM_MODEL="contextual-orchestrator" export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index d4ab4f65df..3c51270748 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3593,7 +3593,7 @@ jobs: "$schema": "https://opencode.ai/config.json", "model": "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "small_model": "nvidia-nim/meta/llama-3.3-70b-instruct", - "enabled_providers": ["nvidia-nim", "opencode-free", "opencode", "openai", "openrouter", "github-models"], + "enabled_providers": ["contextual-orchestrator", "nvidia-nim", "opencode-free", "opencode", "openai", "openrouter", "github-models"], "lsp": false, "mcp": {}, "permission": { diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index efd40f3fec..745a172cab 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -20,7 +20,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "d4ab4f65dfc6f21cda49cf1bb69cb9f1dc6bad5a" +REVIEW_DISPATCH_BLOB_SHA = "3c512707483d17b4483e18864fa917093f8bee59" def _workflow_text(path: Path) -> str: From 07d7d9e0d11652b6c850fe552d66cf2891341fcb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 09:40:12 +0900 Subject: [PATCH 10/23] fix(security): start orchestrator sidecar only for public reviews --- .github/workflows/noema-review.yml | 20 +++++++++---------- ...ontextual_orchestrator_sidecar_contract.py | 4 ++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index a347f0b652..ed25df7dd5 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -135,16 +135,6 @@ jobs: tar -xzf "$trusted_archive" -C "$GITHUB_WORKSPACE" --strip-components=1 test -f scripts/ci/noema_review_gate.py - - name: Start contextual-orchestrator sidecar - if: env.PR_NUMBER != '' - env: - BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} - NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} - NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh - - name: Select fail-closed Noema reviewer credential if: env.PR_NUMBER != '' id: noema_credential @@ -277,6 +267,16 @@ jobs: esac echo "is_private=$is_private" >>"$GITHUB_OUTPUT" + - name: Start contextual-orchestrator sidecar + if: env.PR_NUMBER != '' && steps.target_visibility.outputs.is_private == 'false' + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh + - name: Run Noema LLM review and submit verdict if: env.PR_NUMBER != '' env: diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py index fba59c640d..812993ce8f 100644 --- a/tests/test_contextual_orchestrator_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -22,6 +22,10 @@ def test_sidecar_source_is_immutable_and_readiness_is_bounded() -> None: def test_noema_auto_sidecar_is_public_repository_only() -> None: """Private repositories require explicit Noema provider configuration.""" text = NOEMA_WORKFLOW.read_text(encoding="utf-8") + assert ( + "if: env.PR_NUMBER != '' && steps.target_visibility.outputs.is_private == 'false'" + in text + ) sidecar_condition = re.search( r'if \[ "\$TARGET_REPOSITORY_PRIVATE" = "false" \].{0,240}CONTEXTUAL_ORCHESTRATOR_BASE_URL', text, From ae090ab04eca9e51f4ba897158cd98e1705ca787 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:28:05 +0900 Subject: [PATCH 11/23] fix(security): validate sidecar licenses before install --- .../start_contextual_orchestrator_sidecar.sh | 5 + ...lidate_contextual_orchestrator_licenses.py | 220 ++++++++++++++++++ ...textual_orchestrator_license_validation.py | 36 +++ ...ontextual_orchestrator_sidecar_contract.py | 13 ++ tests/test_noema_review_gate.py | 15 ++ 5 files changed, 289 insertions(+) create mode 100644 scripts/ci/validate_contextual_orchestrator_licenses.py create mode 100644 tests/test_contextual_orchestrator_license_validation.py diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh index a321e633ce..ae48664d51 100755 --- a/scripts/ci/start_contextual_orchestrator_sidecar.sh +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -71,6 +71,11 @@ if [ "$resolved_source_sha" != "$CONTEXTUAL_ORCHESTRATOR_REF" ]; then exit 2 fi +# Validate the pinned source license and every lock dependency's artifact +# license before creating a venv, installing packages, or importing source. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +python3 "$SCRIPT_DIR/validate_contextual_orchestrator_licenses.py" "$RUNTIME_DIR" + python3 -m venv "$RUNTIME_DIR/.venv-sidecar" # shellcheck disable=SC1091 source "$RUNTIME_DIR/.venv-sidecar/bin/activate" diff --git a/scripts/ci/validate_contextual_orchestrator_licenses.py b/scripts/ci/validate_contextual_orchestrator_licenses.py new file mode 100644 index 0000000000..602da5e433 --- /dev/null +++ b/scripts/ci/validate_contextual_orchestrator_licenses.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Validate pinned contextual-orchestrator source and dependency licenses.""" + +from __future__ import annotations + +import argparse +import io +import json +import pathlib +import re +import tarfile +import urllib.parse +import urllib.request +import zipfile +from email.message import Message +from email.parser import BytesParser + + +ALLOWED_SPDX = frozenset( + {"0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "MIT", "PSF-2.0", "Zlib"} +) +PACKAGE_LINE = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9_.-]*)==(?P[^\\s]+)") +MAX_METADATA_BYTES = 2 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 +MAX_LICENSE_METADATA_BYTES = 256 * 1024 +PYPI_HOSTS = frozenset({"pypi.org", "files.pythonhosted.org"}) +CLASSIFIER_TO_SPDX = { + "BSD License": "BSD-3-Clause", + "ISC License": "ISC", + "MIT License": "MIT", + "Python Software Foundation License": "PSF-2.0", + "zlib/libpng License": "Zlib", +} +SOURCE_LICENSE_MARKERS = { + "MIT License": "MIT", + "Apache License": "Apache-2.0", + "BSD 3-Clause License": "BSD-3-Clause", + "ISC License": "ISC", + "Python Software Foundation License": "PSF-2.0", +} + + +class LicenseValidationError(RuntimeError): + """Raised when license evidence is absent, invalid, or disallowed.""" + + +def _read_bounded(response: object, limit: int) -> bytes: + """Read a bounded response without accepting an oversized metadata body.""" + payload = bytearray() + while len(payload) <= limit: + chunk = response.read(min(64 * 1024, limit + 1 - len(payload))) # type: ignore[attr-defined] + if not chunk: + return bytes(payload) + payload.extend(chunk) + raise LicenseValidationError("license evidence exceeded the bounded response size") + + +def _https_url(url: str, allowed_host: str) -> str: + """Validate one HTTPS URL against one exact host and default port.""" + parsed = urllib.parse.urlparse(url) + if ( + parsed.scheme != "https" + or parsed.hostname != allowed_host + or parsed.username is not None + or parsed.password is not None + or parsed.port not in (None, 443) + or parsed.query + or parsed.fragment + ): + raise LicenseValidationError(f"untrusted license evidence URL: {url}") + return url + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Reject redirects so license evidence cannot change its source host.""" + + def redirect_request(self, *args: object, **kwargs: object) -> None: + """Fail closed on a redirect from the pinned metadata endpoint.""" + raise LicenseValidationError("license evidence URL redirected") + + +def _fetch_json(url: str) -> dict[str, object]: + """Fetch one bounded PyPI metadata object without following redirects.""" + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), _NoRedirect() + ) + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with opener.open(request, timeout=20) as response: + payload = _read_bounded(response, MAX_METADATA_BYTES) + value = json.loads(payload) + if not isinstance(value, dict): + raise LicenseValidationError("PyPI license evidence was not an object") + return value + + +def parse_locked_packages(lock_text: str) -> tuple[tuple[str, str], ...]: + """Return exact package/version pairs from a pip-compile lock.""" + packages: list[tuple[str, str]] = [] + for raw_line in lock_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or line.startswith("--hash="): + continue + match = PACKAGE_LINE.fullmatch(line.rstrip("\\").rstrip()) + if match is None: + raise LicenseValidationError(f"unrecognized requirements.lock line: {line}") + packages.append((match["name"], match["version"])) + if not packages: + raise LicenseValidationError("requirements.lock contains no pinned packages") + return tuple(packages) + + +def _metadata_from_artifact(payload: bytes, filename: str) -> Message: + """Extract one bounded wheel or source-distribution METADATA document.""" + metadata_bytes: bytes | None = None + if filename.endswith(".whl"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] + if len(names) != 1: + raise LicenseValidationError("wheel has no unique dist-info METADATA") + info = archive.getinfo(names[0]) + if info.file_size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError("wheel license metadata is oversized") + metadata_bytes = archive.read(info) + elif filename.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".tgz")): + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as archive: + members = [member for member in archive.getmembers() if member.name.endswith("/PKG-INFO")] + if len(members) != 1 or members[0].size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError("source distribution has invalid PKG-INFO") + extracted = archive.extractfile(members[0]) + if extracted is None: + raise LicenseValidationError("source distribution PKG-INFO is unreadable") + metadata_bytes = extracted.read(MAX_LICENSE_METADATA_BYTES + 1) + if len(metadata_bytes) > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError("source license metadata is oversized") + if metadata_bytes is None: + raise LicenseValidationError(f"unsupported PyPI artifact type: {filename}") + return BytesParser().parsebytes(metadata_bytes) + + +def _artifact_license(name: str, version: str, metadata: dict[str, object]) -> str: + """Fetch and return one allowed SPDX license from a PyPI artifact.""" + urls = metadata.get("urls") + if not isinstance(urls, list): + raise LicenseValidationError(f"PyPI metadata has no artifact list for {name}=={version}") + candidates = [item for item in urls if isinstance(item, dict) and item.get("packagetype") == "bdist_wheel"] + candidates += [item for item in urls if isinstance(item, dict) and item.get("packagetype") == "sdist"] + for item in candidates: + filename = item.get("filename") + url = item.get("url") + if not isinstance(filename, str) or not isinstance(url, str): + continue + host = "pypi.org" if "pypi.org" in url else "files.pythonhosted.org" + _https_url(url, host) + request = urllib.request.Request(url) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) + with opener.open(request, timeout=30) as response: + payload = _read_bounded(response, MAX_ARTIFACT_BYTES) + parsed = _metadata_from_artifact(payload, filename) + values = parsed.get_all("License-Expression") or parsed.get_all("License") + if not values: + values = [ + CLASSIFIER_TO_SPDX[value.removeprefix("License :: OSI Approved :: ")] + for value in parsed.get_all("Classifier") + if value.removeprefix("License :: OSI Approved :: ") in CLASSIFIER_TO_SPDX + ] + if len(values) != 1 or values[0].strip() not in ALLOWED_SPDX: + raise LicenseValidationError( + f"disallowed or ambiguous license for {name}=={version}: {values or ''}" + ) + return values[0].strip() + raise LicenseValidationError(f"no usable PyPI artifact for {name}=={version}") + + +def validate_source_license(root: pathlib.Path) -> str: + """Validate the pinned source tree's declared license before execution.""" + pyproject = root / "pyproject.toml" + if not pyproject.is_file(): + raise LicenseValidationError("contextual-orchestrator source lacks pyproject.toml") + pyproject.read_bytes() + for candidate in (root / "LICENSE", root / "LICENSE.txt", root / "LICENSE.md"): + if candidate.is_file(): + content = candidate.read_text(encoding="utf-8", errors="replace") + for marker, spdx in SOURCE_LICENSE_MARKERS.items(): + if marker in content: + return spdx + raise LicenseValidationError("source license evidence is missing or not approved") + + +def validate_tree(root: pathlib.Path) -> dict[str, str]: + """Validate source and every exact dependency before any installation.""" + source_license = validate_source_license(root) + packages = parse_locked_packages((root / "requirements.lock").read_text(encoding="utf-8")) + result = {"contextual-orchestrator": source_license} + for name, version in packages: + normalized = name.lower().replace("_", "-") + url = _https_url( + "https://pypi.org/pypi/" + f"{urllib.parse.quote(normalized, safe='')}/{urllib.parse.quote(version, safe='')}/json", + "pypi.org", + ) + result[f"{name}=={version}"] = _artifact_license(name, version, _fetch_json(url)) + return result + + +def main() -> int: + """Validate one contextual-orchestrator checkout and print evidence.""" + parser = argparse.ArgumentParser() + parser.add_argument("repository_root", type=pathlib.Path) + args = parser.parse_args() + try: + licenses = validate_tree(args.repository_root.resolve()) + except (OSError, ValueError, KeyError, LicenseValidationError, json.JSONDecodeError) as exc: + print(f"LICENSE_VALIDATION_FAILED: {exc}") + return 1 + print(f"LICENSE_VALIDATION_OK: {len(licenses)} source and dependency licenses verified") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_contextual_orchestrator_license_validation.py b/tests/test_contextual_orchestrator_license_validation.py new file mode 100644 index 0000000000..f0e6982ea5 --- /dev/null +++ b/tests/test_contextual_orchestrator_license_validation.py @@ -0,0 +1,36 @@ +"""Regression tests for pre-install contextual-orchestrator license validation.""" + +from __future__ import annotations + +import pathlib + +import pytest + +from scripts.ci import validate_contextual_orchestrator_licenses as validator + + +def test_parse_locked_packages_accepts_hash_continuations() -> None: + """The validator reads only exact package pins from pip-compile output.""" + lock = ( + "demo-package==1.2.3 \\\n" + " --hash=sha256:" + "a" * 64 + "\n# via demo\n" + ) + assert validator.parse_locked_packages(lock) == (("demo-package", "1.2.3"),) + + +def test_parse_locked_packages_rejects_unpinned_or_empty_input() -> None: + """Editable, URL, and empty lock inputs fail closed before any query.""" + with pytest.raises(validator.LicenseValidationError): + validator.parse_locked_packages("demo-package>=1\n") + with pytest.raises(validator.LicenseValidationError): + validator.parse_locked_packages("# comment only\n") + + +def test_source_license_requires_approved_mit_evidence(tmp_path: pathlib.Path) -> None: + """A checkout must carry a recognizable approved source license.""" + (tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n", encoding="utf-8") + (tmp_path / "LICENSE").write_text("MIT License\n", encoding="utf-8") + assert validator.validate_source_license(tmp_path) == "MIT" + (tmp_path / "LICENSE").write_text("LGPL-3.0-only\n", encoding="utf-8") + with pytest.raises(validator.LicenseValidationError): + validator.validate_source_license(tmp_path) diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py index 812993ce8f..ca12241cb4 100644 --- a/tests/test_contextual_orchestrator_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -17,6 +17,19 @@ def test_sidecar_source_is_immutable_and_readiness_is_bounded() -> None: assert 'CONTEXTUAL_ORCHESTRATOR_REF:=7eb459ee72c37dead5d25f284dfa4546f149fbe1' in text assert '^[0-9a-fA-F]{40}$' in text assert '--connect-timeout 1 --max-time 2' in text + assert 'validate_contextual_orchestrator_licenses.py' in text + assert text.index('validate_contextual_orchestrator_licenses.py') < text.index('python3 -m venv') + + +def test_sidecar_license_policy_is_explicit_and_fail_closed() -> None: + """Only a bounded permissive SPDX set may reach the sidecar install.""" + validator = Path("scripts/ci/validate_contextual_orchestrator_licenses.py").read_text( + encoding="utf-8" + ) + assert '"MIT"' in validator + assert '"Apache-2.0"' in validator + assert '"LGPL-3.0-only"' not in validator + assert "LICENSE_VALIDATION_FAILED" in validator def test_noema_auto_sidecar_is_public_repository_only() -> None: diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 1fceb4eb91..6636373533 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -458,6 +458,21 @@ def fake_getaddrinfo_dual_stack(host, port, *args, **kwargs): monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + # A loopback result must not short-circuit validation of later DNS results. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://mixed-resolution.example.com/chat") + + def fake_getaddrinfo_mixed(host, port, *args, **kwargs): + if host == "mixed-resolution.example.com": + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0)), + ] + return original_getaddrinfo(host, port, *args, **kwargs) + + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_mixed) + with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + # Test unresolved hostname does not break monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") def fake_getaddrinfo_error(host, port, *args, **kwargs): From 5dd18d048590dee2822097d4ac32907f84c8cb98 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:08:55 -0700 Subject: [PATCH 12/23] test(sidecar): bind license evidence to lock artifacts --- ...textual_orchestrator_license_validation.py | 179 ++++++++++++++++-- 1 file changed, 167 insertions(+), 12 deletions(-) diff --git a/tests/test_contextual_orchestrator_license_validation.py b/tests/test_contextual_orchestrator_license_validation.py index f0e6982ea5..ad0d3fe1dc 100644 --- a/tests/test_contextual_orchestrator_license_validation.py +++ b/tests/test_contextual_orchestrator_license_validation.py @@ -2,33 +2,188 @@ from __future__ import annotations +from email.parser import BytesParser +import hashlib +import io import pathlib +import zipfile import pytest from scripts.ci import validate_contextual_orchestrator_licenses as validator -def test_parse_locked_packages_accepts_hash_continuations() -> None: - """The validator reads only exact package pins from pip-compile output.""" +def _wheel_bytes( + name: str, + version: str, + license_headers: str = "License-Expression: MIT\n", +) -> bytes: + """Build a minimal in-memory wheel carrying controlled METADATA.""" + metadata = ( + "Metadata-Version: 2.4\n" + f"Name: {name}\n" + f"Version: {version}\n" + f"{license_headers}\n" + ).encode() + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr(f"{name}-{version}.dist-info/METADATA", metadata) + return payload.getvalue() + + +def test_parse_locked_packages_binds_each_pin_to_sha256_closure() -> None: + """Every package pin retains only exact artifacts admitted by the lock.""" + first_hash = "a" * 64 + second_hash = "b" * 64 lock = ( "demo-package==1.2.3 \\\n" - " --hash=sha256:" + "a" * 64 + "\n# via demo\n" + f" --hash=sha256:{first_hash} \\\n" + f" --hash=sha256:{second_hash}\n" + "# via demo\n" ) - assert validator.parse_locked_packages(lock) == (("demo-package", "1.2.3"),) + packages = validator.parse_locked_packages(lock) -def test_parse_locked_packages_rejects_unpinned_or_empty_input() -> None: - """Editable, URL, and empty lock inputs fail closed before any query.""" - with pytest.raises(validator.LicenseValidationError): - validator.parse_locked_packages("demo-package>=1\n") - with pytest.raises(validator.LicenseValidationError): - validator.parse_locked_packages("# comment only\n") + assert packages == ( + validator.LockedPackage( + "demo-package", + "1.2.3", + frozenset({first_hash, second_hash}), + ), + ) + + +def test_parse_locked_packages_rejects_missing_hash_duplicate_and_orphan_hash() -> None: + """Malformed or ambiguous lock closures fail before any metadata query.""" + digest = "a" * 64 + with pytest.raises(validator.LicenseValidationError, match="no SHA-256 closure"): + validator.parse_locked_packages("demo-package==1.2.3\n") + with pytest.raises(validator.LicenseValidationError, match="duplicate package pin"): + validator.parse_locked_packages( + "demo-package==1.2.3 \\\n" + f" --hash=sha256:{digest}\n" + "demo_package==1.2.3 \\\n" + f" --hash=sha256:{digest}\n" + ) + with pytest.raises(validator.LicenseValidationError, match="orphan SHA-256"): + validator.parse_locked_packages(f"--hash=sha256:{digest}\n") + + +def test_select_locked_artifact_requires_metadata_identity_and_lock_digest() -> None: + """PyPI artifact URLs cannot replace the lock as artifact authority.""" + locked_digest = "a" * 64 + package = validator.LockedPackage( + "demo-package", + "1.2.3", + frozenset({locked_digest}), + ) + metadata = { + "info": {"name": "demo-package", "version": "1.2.3"}, + "urls": [ + { + "packagetype": "bdist_wheel", + "filename": "demo_package-1.2.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/demo.whl", + "size": 100, + "digests": {"sha256": "b" * 64}, + } + ], + } + with pytest.raises(validator.LicenseValidationError, match="no hash-bound"): + validator.select_locked_artifact(package, metadata) + + metadata["urls"][0]["digests"]["sha256"] = locked_digest + artifact = validator.select_locked_artifact(package, metadata) + assert artifact.sha256 == locked_digest + + metadata["info"]["version"] = "9.9.9" + with pytest.raises(validator.LicenseValidationError, match="identity mismatch"): + validator.select_locked_artifact(package, metadata) + + +def test_downloaded_artifact_digest_and_size_are_rechecked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Downloaded bytes must match metadata size and the lock-bound digest.""" + payload = b"wheel-bytes" + artifact = validator.LockedArtifact( + "demo.whl", + "https://files.pythonhosted.org/demo.whl", + hashlib.sha256(payload).hexdigest(), + len(payload), + ) + + class Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *_args: object) -> bool: + return False + class Opener: + def open(self, *_args: object, **_kwargs: object) -> Response: + return Response(payload) -def test_source_license_requires_approved_mit_evidence(tmp_path: pathlib.Path) -> None: + monkeypatch.setattr(validator, "_opener", lambda: Opener()) + assert validator._download_locked_artifact(artifact) == payload + + bad = validator.LockedArtifact( + artifact.filename, + artifact.url, + "0" * 64, + artifact.size, + ) + with pytest.raises(validator.LicenseValidationError, match="digest mismatch"): + validator._download_locked_artifact(bad) + + +def test_artifact_metadata_identity_and_spdx_are_fail_closed() -> None: + """Artifact metadata must match the pin and use approved SPDX licenses.""" + payload = _wheel_bytes( + "demo-package", + "1.2.3", + "License-Expression: MIT OR Apache-2.0\n", + ) + parsed = validator._metadata_from_artifact( + payload, + "demo_package-1.2.3-py3-none-any.whl", + ) + assert parsed["Name"] == "demo-package" + assert validator._license_from_metadata(parsed) == "MIT OR Apache-2.0" + + gpl = BytesParser().parsebytes( + b"Metadata-Version: 2.4\nName: demo\nVersion: 1\n" + b"License-Expression: GPL-3.0-only\n\n" + ) + with pytest.raises(validator.LicenseValidationError, match="disallowed SPDX"): + validator._license_from_metadata(gpl) + + malformed = BytesParser().parsebytes( + b"Metadata-Version: 2.4\nName: demo\nVersion: 1\n" + b"License-Expression: MIT OR\n\n" + ) + with pytest.raises(validator.LicenseValidationError, match="incomplete SPDX"): + validator._license_from_metadata(malformed) + + +def test_classifier_fallback_accepts_only_known_permissive_classifiers() -> None: + """Legacy metadata may fall back to a bounded OSI classifier map.""" + metadata = BytesParser().parsebytes( + b"Metadata-Version: 2.1\nName: demo\nVersion: 1\n" + b"License: UNKNOWN\n" + b"Classifier: License :: OSI Approved :: MIT License\n\n" + ) + assert validator._license_from_metadata(metadata) == "MIT" + + +def test_source_license_requires_approved_mit_evidence( + tmp_path: pathlib.Path, +) -> None: """A checkout must carry a recognizable approved source license.""" - (tmp_path / "pyproject.toml").write_text("[project]\nname='demo'\n", encoding="utf-8") + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='demo'\n", + encoding="utf-8", + ) (tmp_path / "LICENSE").write_text("MIT License\n", encoding="utf-8") assert validator.validate_source_license(tmp_path) == "MIT" (tmp_path / "LICENSE").write_text("LGPL-3.0-only\n", encoding="utf-8") From b63f0ffa5fbe212225da9bf573c5dabeb8a40e43 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:11:06 -0700 Subject: [PATCH 13/23] test: expose process-local sidecar credential gap --- ...ontextual_orchestrator_sidecar_contract.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py index ca12241cb4..85d307b5c4 100644 --- a/tests/test_contextual_orchestrator_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -55,3 +55,24 @@ def test_generated_opencode_config_allows_contextual_orchestrator_provider() -> '"contextual-orchestrator"' in allowlist and '"nvidia-nim"' in allowlist for allowlist in matches ) + + +def test_sidecar_keeps_credential_registration_discovery_and_server_in_one_process() -> None: + """Process-local credentials must survive through discovery and serving.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert "python -m contextual_orchestrator register-credential" not in text + assert "python -m contextual_orchestrator discover-models" not in text + assert "os.environ.pop(credential_name" in text + assert "discover_all_models()" in text + assert "TaskOrchestrator(" in text + assert "serve(" in text + + +def test_sidecar_does_not_execute_an_editable_build_and_cleans_failed_startup() -> None: + """The pinned source runs via PYTHONPATH and failed readiness kills the child.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert "pip install --quiet --no-deps -e" not in text + assert 'PYTHONPATH="$RUNTIME_DIR"' in text + assert 'sidecar_pid="$!"' in text + assert 'kill "$sidecar_pid"' in text + assert text.index('echo "contextual-orchestrator sidecar ready') < text.index("trap - EXIT") From 70449633e54b4759d339422a7882fdc9566769d3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:17:04 -0700 Subject: [PATCH 14/23] fix: keep sidecar bootstrap and credentials in one process --- .../start_contextual_orchestrator_sidecar.sh | 179 +++++++++++++----- 1 file changed, 134 insertions(+), 45 deletions(-) diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh index ae48664d51..c94ca5d1b8 100755 --- a/scripts/ci/start_contextual_orchestrator_sidecar.sh +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -2,12 +2,11 @@ # Starts contextual-orchestrator as a same-job, loopback-only sidecar: checks out # the org's own contextual-orchestrator repo at an immutable reviewed commit, # installs its hash-locked dependencies, registers whichever of the five upstream -# provider credentials are present as job secrets into its KV, runs model auto- -# discovery + cost-based auto-enable, then serves on 127.0.0.1 with a freshly -# generated bearer token. Intended to be sourced/invoked as an early step in a -# job that later calls into OpenCode/Noema/Strix so they can use contextual- -# orchestrator as their LLM backend. Not for public/shared exposure: loopback- -# only, one ephemeral token per job run, torn down with the runner. +# provider credentials are present into a process-local KV, discovers and enables +# a bounded model pool, then serves it from the same Python process. The single +# process is essential: the default credential backend is intentionally +# process-local, so separate register/discover/serve CLI invocations would lose +# the credentials between phases. set -euo pipefail : "${GITHUB_ENV:=/dev/null}" @@ -28,8 +27,8 @@ if [[ ! "$CONTEXTUAL_ORCHESTRATOR_PORT" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHEST echo "ERROR: CONTEXTUAL_ORCHESTRATOR_PORT must be a valid TCP port." >&2 exit 2 fi -if [[ ! "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" =~ ^[0-9]+$ ]]; then - echo "ERROR: CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST must be a non-negative integer." >&2 +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" -lt 1 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST must be a positive integer." >&2 exit 2 fi if [[ ! "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" -lt 3 ]; then @@ -37,13 +36,19 @@ if [[ ! "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || [ "$C exit 2 fi +PROVIDER_CREDENTIAL_NAMES=( + BYTEZ_API_KEY + NVIDIA_NIM_API_KEY + NVIDIA_NIM_API_KEY_SUB + OPENROUTER_API_KEY + OPENAI_API_KEY +) + # Self-gating: callers can invoke this unconditionally from every workflow. # With none of the five upstream provider credentials present, there is -# nothing for contextual-orchestrator to discover or serve, so no-op cleanly -# (exit 0, CONTEXTUAL_ORCHESTRATOR_BASE_URL left unset) rather than making -# every caller duplicate this check in workflow YAML `if:` conditions. +# nothing for contextual-orchestrator to discover or serve, so no-op cleanly. has_any_provider_credential=0 -for credential_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENROUTER_API_KEY OPENAI_API_KEY; do +for credential_name in "${PROVIDER_CREDENTIAL_NAMES[@]}"; do if [ -n "${!credential_name:-}" ]; then has_any_provider_credential=1 break @@ -55,10 +60,18 @@ if [ "$has_any_provider_credential" -eq 0 ]; then fi RUNTIME_DIR="$(mktemp -d)" -cleanup_checkout() { +sidecar_pid="" +cleanup_failed_startup() { + local exit_status=$? + trap - EXIT + if [ -n "$sidecar_pid" ] && kill -0 "$sidecar_pid" 2>/dev/null; then + kill "$sidecar_pid" 2>/dev/null || true + wait "$sidecar_pid" 2>/dev/null || true + fi rm -rf -- "$RUNTIME_DIR" + exit "$exit_status" } -trap cleanup_checkout EXIT +trap cleanup_failed_startup EXIT echo "Checking out ContextualWisdomLab/contextual-orchestrator@${CONTEXTUAL_ORCHESTRATOR_REF}..." git init --quiet "$RUNTIME_DIR" @@ -80,46 +93,117 @@ python3 -m venv "$RUNTIME_DIR/.venv-sidecar" # shellcheck disable=SC1091 source "$RUNTIME_DIR/.venv-sidecar/bin/activate" python -m pip install --quiet --require-hashes -r "$RUNTIME_DIR/requirements.lock" -python -m pip install --quiet --no-deps -e "$RUNTIME_DIR" - -# Register only the credentials this job actually has; discovery silently -# skips any provider whose credential is missing (see model_discovery.py). -for credential_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENROUTER_API_KEY OPENAI_API_KEY; do - if [ -n "${!credential_name:-}" ]; then - python -m contextual_orchestrator register-credential \ - --name "$credential_name" --from-env "$credential_name" - fi -done - -POOL_DB="$RUNTIME_DIR/agent-pool.db" -echo "Discovering models and auto-enabling the ${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST} cheapest..." -python -m contextual_orchestrator discover-models \ - --agents-db "$POOL_DB" \ - --enable-cheapest "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" TOKEN="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" echo "::add-mask::$TOKEN" - LOG_FILE="$RUNTIME_DIR/sidecar.log" -( - cd "$RUNTIME_DIR" - python -m contextual_orchestrator --serve \ - --agents examples/agents.mock.json \ - --agents-db "$POOL_DB" \ - --host 127.0.0.1 --port "$CONTEXTUAL_ORCHESTRATOR_PORT" \ - --auth-token "$TOKEN" \ - >"$LOG_FILE" 2>&1 & - echo $! >"$RUNTIME_DIR/sidecar.pid" + +# Register credentials, discover models, build the failover-capable orchestrator, +# and serve from one Python process. Source is imported directly from the exact +# reviewed checkout; no editable build backend or unpinned build dependency runs. +PYTHONPATH="$RUNTIME_DIR" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_TOKEN="$TOKEN" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_PORT="$CONTEXTUAL_ORCHESTRATOR_PORT" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_ENABLE_CHEAPEST="$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" \ +"$RUNTIME_DIR/.venv-sidecar/bin/python" - >"$LOG_FILE" 2>&1 <<'PY' & +from __future__ import annotations + +from dataclasses import replace +import os + +credential_names = ( + "BYTEZ_API_KEY", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", ) -# The checkout is only needed to launch the server; once it's up, remove the -# EXIT trap so the running process (and its still-open log/pool files) survive -# for the rest of the job. The server holds its own open file handles, so -# deleting the directory entry is safe on Linux runners even with it running. -trap - EXIT +credentials: dict[str, str] = {} +for credential_name in credential_names: + credential_value = os.environ.pop(credential_name, "") + if credential_value: + credentials[credential_name] = credential_value + +auth_token = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_TOKEN", "") +port_text = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_PORT", "") +enable_text = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_ENABLE_CHEAPEST", "") +if not credentials: + raise RuntimeError("no provider credentials survived sidecar bootstrap") +if not auth_token: + raise RuntimeError("sidecar authentication token is missing") + +port = int(port_text) +enable_cheapest = int(enable_text) +if not 1 <= port <= 65535: + raise RuntimeError("sidecar port is outside the valid range") +if enable_cheapest < 1: + raise RuntimeError("sidecar model count must be positive") + +from contextual_orchestrator import TaskOrchestrator, register_credential +from contextual_orchestrator.cost_ledger import PriceBook +from contextual_orchestrator.kv_config import InMemoryConfigStore +from contextual_orchestrator.model_discovery import ( + agent_from_discovered, + discover_all_models, + refresh_price_book, + select_top_n_cheapest_discovered_agents, +) +from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.server import SecurityConfig, serve + +for credential_name, credential_value in credentials.items(): + register_credential(credential_name, credential_value) +credentials.clear() + +discovered_models, discovery_errors = discover_all_models() +if not discovered_models: + providers_with_errors = sorted( + {getattr(error, "provider_name", "unknown") for error in discovery_errors} + ) + provider_summary = ",".join(providers_with_errors) or "none" + raise RuntimeError( + "model discovery returned no candidates; providers_with_errors=" + + provider_summary + ) + +price_book = PriceBook(InMemoryConfigStore()) +refresh_price_book(discovered_models, price_book) +selected_models = select_top_n_cheapest_discovered_agents( + discovered_models, + price_book, + enable_cheapest, +) +if not selected_models: + raise RuntimeError("model discovery produced no enableable candidates") + +active_agents = [ + replace(agent_from_discovered(model), disabled=False) + for model in selected_models +] +orchestrator = TaskOrchestrator(active_agents, client=ModelClient()) +serve( + orchestrator, + host="127.0.0.1", + port=port, + security=SecurityConfig(auth_token=auth_token), +) +PY +sidecar_pid="$!" + +# The shell no longer needs provider keys; prevent readiness probes or any other +# child process in this step from inheriting them. +for credential_name in "${PROVIDER_CREDENTIAL_NAMES[@]}"; do + unset "$credential_name" +done BASE_URL="http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/v1" deadline=$((SECONDS + CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS)) until curl --connect-timeout 1 --max-time 2 -fsS "http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do + if ! kill -0 "$sidecar_pid" 2>/dev/null; then + echo "ERROR: contextual-orchestrator sidecar exited before becoming ready." >&2 + tail -n 200 "$LOG_FILE" >&2 || true + exit 1 + fi if [ "$SECONDS" -ge "$deadline" ]; then echo "ERROR: contextual-orchestrator sidecar did not become ready within ${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS}s." >&2 tail -n 200 "$LOG_FILE" >&2 || true @@ -129,6 +213,11 @@ until curl --connect-timeout 1 --max-time 2 -fsS "http://127.0.0.1:${CONTEXTUAL_ done echo "contextual-orchestrator sidecar ready at $BASE_URL" +# Keep the process and its exact-SHA checkout alive for the remainder of the +# GitHub job. The runner tears them down at job completion. Failed startup paths +# retain the EXIT trap above and therefore kill the child and remove the checkout. +trap - EXIT + { echo "CONTEXTUAL_ORCHESTRATOR_BASE_URL=$BASE_URL" echo "CONTEXTUAL_ORCHESTRATOR_TOKEN=$TOKEN" From d224ced1193fa809219d17496c420f0324b8f584 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:25:30 -0700 Subject: [PATCH 15/23] fix: bind sidecar licenses to locked artifacts --- ...lidate_contextual_orchestrator_licenses.py | 441 +++++++++++++++--- 1 file changed, 364 insertions(+), 77 deletions(-) diff --git a/scripts/ci/validate_contextual_orchestrator_licenses.py b/scripts/ci/validate_contextual_orchestrator_licenses.py index 602da5e433..64b1de9efb 100644 --- a/scripts/ci/validate_contextual_orchestrator_licenses.py +++ b/scripts/ci/validate_contextual_orchestrator_licenses.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +from dataclasses import dataclass +import hashlib import io import json import pathlib @@ -14,12 +16,26 @@ import zipfile from email.message import Message from email.parser import BytesParser +from typing import Any ALLOWED_SPDX = frozenset( - {"0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "ISC", "MIT", "PSF-2.0", "Zlib"} + { + "0BSD", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "PSF-2.0", + "Zlib", + } ) -PACKAGE_LINE = re.compile(r"^(?P[A-Za-z0-9][A-Za-z0-9_.-]*)==(?P[^\\s]+)") +PACKAGE_LINE = re.compile( + r"^(?P[A-Za-z0-9][A-Za-z0-9_.-]*)==(?P[^\s\\]+)" +) +HASH_OPTION = re.compile(r"--hash=sha256:(?P[0-9a-fA-F]{64})(?=\s|\\|$)") +SPDX_TOKEN = re.compile(r"\(|\)|AND|OR|[A-Za-z0-9][A-Za-z0-9.+-]*") MAX_METADATA_BYTES = 2 * 1024 * 1024 MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 MAX_LICENSE_METADATA_BYTES = 256 * 1024 @@ -44,6 +60,30 @@ class LicenseValidationError(RuntimeError): """Raised when license evidence is absent, invalid, or disallowed.""" +@dataclass(frozen=True) +class LockedPackage: + """One exact dependency pin and its admitted SHA-256 artifact closure.""" + + name: str + version: str + sha256_hashes: frozenset[str] + + +@dataclass(frozen=True) +class LockedArtifact: + """One PyPI artifact whose identity and digest are bound to the lock.""" + + filename: str + url: str + sha256: str + size: int + + +def _normalize_package_name(name: str) -> str: + """Return the PEP 503 comparison key for one distribution name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + def _read_bounded(response: object, limit: int) -> bytes: """Read a bounded response without accepting an oversized metadata body.""" payload = bytearray() @@ -57,13 +97,17 @@ def _read_bounded(response: object, limit: int) -> bytes: def _https_url(url: str, allowed_host: str) -> str: """Validate one HTTPS URL against one exact host and default port.""" - parsed = urllib.parse.urlparse(url) + try: + parsed = urllib.parse.urlparse(url) + parsed_port = parsed.port + except ValueError as exc: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") from exc if ( parsed.scheme != "https" or parsed.hostname != allowed_host or parsed.username is not None or parsed.password is not None - or parsed.port not in (None, 443) + or parsed_port not in (None, 443) or parsed.query or parsed.fragment ): @@ -79,13 +123,15 @@ def redirect_request(self, *args: object, **kwargs: object) -> None: raise LicenseValidationError("license evidence URL redirected") +def _opener() -> Any: + """Build the no-proxy, no-redirect opener used for license evidence.""" + return urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) + + def _fetch_json(url: str) -> dict[str, object]: """Fetch one bounded PyPI metadata object without following redirects.""" - opener = urllib.request.build_opener( - urllib.request.ProxyHandler({}), _NoRedirect() - ) request = urllib.request.Request(url, headers={"Accept": "application/json"}) - with opener.open(request, timeout=20) as response: + with _opener().open(request, timeout=20) as response: payload = _read_bounded(response, MAX_METADATA_BYTES) value = json.loads(payload) if not isinstance(value, dict): @@ -93,112 +139,344 @@ def _fetch_json(url: str) -> dict[str, object]: return value -def parse_locked_packages(lock_text: str) -> tuple[tuple[str, str], ...]: - """Return exact package/version pairs from a pip-compile lock.""" - packages: list[tuple[str, str]] = [] +def _finalize_lock_block( + block: list[str], + packages: list[LockedPackage], + seen_names: set[str], +) -> None: + """Validate one logical pip-compile requirement block.""" + joined = " ".join(part.rstrip("\\").strip() for part in block) + match = PACKAGE_LINE.match(joined) + if match is None: + raise LicenseValidationError(f"unrecognized requirements.lock line: {block[0]}") + name = match["name"] + version = match["version"] + digests = frozenset( + digest.lower() for digest in HASH_OPTION.findall(joined) + ) + if not digests: + raise LicenseValidationError(f"package {name}=={version} has no SHA-256 closure") + + remainder = joined[match.end() :] + remainder = HASH_OPTION.sub("", remainder).replace("\\", "").strip() + if remainder: + raise LicenseValidationError( + f"unrecognized requirements.lock option for {name}=={version}: {remainder}" + ) + + normalized_name = _normalize_package_name(name) + if normalized_name in seen_names: + raise LicenseValidationError(f"duplicate package pin for {normalized_name}") + seen_names.add(normalized_name) + packages.append(LockedPackage(name, version, digests)) + + +def parse_locked_packages(lock_text: str) -> tuple[LockedPackage, ...]: + """Return exact package pins bound to their SHA-256 artifact closures.""" + packages: list[LockedPackage] = [] + seen_names: set[str] = set() + block: list[str] = [] + for raw_line in lock_text.splitlines(): line = raw_line.strip() - if not line or line.startswith("#") or line.startswith("--hash="): + if not line or line.startswith("#"): continue - match = PACKAGE_LINE.fullmatch(line.rstrip("\\").rstrip()) - if match is None: + if PACKAGE_LINE.match(line): + if block: + _finalize_lock_block(block, packages, seen_names) + block = [line] + elif line.startswith("--hash=sha256:"): + if not block: + raise LicenseValidationError("orphan SHA-256 hash in requirements.lock") + block.append(line) + else: raise LicenseValidationError(f"unrecognized requirements.lock line: {line}") - packages.append((match["name"], match["version"])) + + if not line.endswith("\\"): + _finalize_lock_block(block, packages, seen_names) + block = [] + + if block: + _finalize_lock_block(block, packages, seen_names) if not packages: raise LicenseValidationError("requirements.lock contains no pinned packages") return tuple(packages) -def _metadata_from_artifact(payload: bytes, filename: str) -> Message: - """Extract one bounded wheel or source-distribution METADATA document.""" - metadata_bytes: bytes | None = None - if filename.endswith(".whl"): - with zipfile.ZipFile(io.BytesIO(payload)) as archive: - names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] - if len(names) != 1: - raise LicenseValidationError("wheel has no unique dist-info METADATA") - info = archive.getinfo(names[0]) - if info.file_size > MAX_LICENSE_METADATA_BYTES: - raise LicenseValidationError("wheel license metadata is oversized") - metadata_bytes = archive.read(info) - elif filename.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".tgz")): - with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as archive: - members = [member for member in archive.getmembers() if member.name.endswith("/PKG-INFO")] - if len(members) != 1 or members[0].size > MAX_LICENSE_METADATA_BYTES: - raise LicenseValidationError("source distribution has invalid PKG-INFO") - extracted = archive.extractfile(members[0]) - if extracted is None: - raise LicenseValidationError("source distribution PKG-INFO is unreadable") - metadata_bytes = extracted.read(MAX_LICENSE_METADATA_BYTES + 1) - if len(metadata_bytes) > MAX_LICENSE_METADATA_BYTES: - raise LicenseValidationError("source license metadata is oversized") - if metadata_bytes is None: - raise LicenseValidationError(f"unsupported PyPI artifact type: {filename}") - return BytesParser().parsebytes(metadata_bytes) - +def select_locked_artifact( + package: LockedPackage, + metadata: dict[str, object], +) -> LockedArtifact: + """Select one wheel or sdist whose PyPI digest is admitted by the lock.""" + info = metadata.get("info") + if not isinstance(info, dict): + raise LicenseValidationError( + f"PyPI metadata identity mismatch for {package.name}=={package.version}" + ) + metadata_name = info.get("name") + metadata_version = info.get("version") + if ( + not isinstance(metadata_name, str) + or _normalize_package_name(metadata_name) != _normalize_package_name(package.name) + or metadata_version != package.version + ): + raise LicenseValidationError( + f"PyPI metadata identity mismatch for {package.name}=={package.version}" + ) -def _artifact_license(name: str, version: str, metadata: dict[str, object]) -> str: - """Fetch and return one allowed SPDX license from a PyPI artifact.""" urls = metadata.get("urls") if not isinstance(urls, list): - raise LicenseValidationError(f"PyPI metadata has no artifact list for {name}=={version}") - candidates = [item for item in urls if isinstance(item, dict) and item.get("packagetype") == "bdist_wheel"] - candidates += [item for item in urls if isinstance(item, dict) and item.get("packagetype") == "sdist"] + raise LicenseValidationError( + f"PyPI metadata has no artifact list for {package.name}=={package.version}" + ) + candidates = [ + item + for package_type in ("bdist_wheel", "sdist") + for item in urls + if isinstance(item, dict) and item.get("packagetype") == package_type + ] for item in candidates: filename = item.get("filename") url = item.get("url") - if not isinstance(filename, str) or not isinstance(url, str): + size = item.get("size") + digests = item.get("digests") + if ( + not isinstance(filename, str) + or not isinstance(url, str) + or isinstance(size, bool) + or not isinstance(size, int) + or not 0 < size <= MAX_ARTIFACT_BYTES + or not isinstance(digests, dict) + ): continue - host = "pypi.org" if "pypi.org" in url else "files.pythonhosted.org" + try: + host = urllib.parse.urlparse(url).hostname + except ValueError as exc: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") from exc + if host not in PYPI_HOSTS: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") _https_url(url, host) - request = urllib.request.Request(url) - opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) - with opener.open(request, timeout=30) as response: - payload = _read_bounded(response, MAX_ARTIFACT_BYTES) - parsed = _metadata_from_artifact(payload, filename) - values = parsed.get_all("License-Expression") or parsed.get_all("License") - if not values: - values = [ - CLASSIFIER_TO_SPDX[value.removeprefix("License :: OSI Approved :: ")] - for value in parsed.get_all("Classifier") - if value.removeprefix("License :: OSI Approved :: ") in CLASSIFIER_TO_SPDX - ] - if len(values) != 1 or values[0].strip() not in ALLOWED_SPDX: - raise LicenseValidationError( - f"disallowed or ambiguous license for {name}=={version}: {values or ''}" - ) - return values[0].strip() - raise LicenseValidationError(f"no usable PyPI artifact for {name}=={version}") + digest = digests.get("sha256") + if ( + isinstance(digest, str) + and re.fullmatch(r"[0-9a-fA-F]{64}", digest) + and digest.lower() in package.sha256_hashes + ): + return LockedArtifact(filename, url, digest.lower(), size) + raise LicenseValidationError( + f"no hash-bound PyPI artifact for {package.name}=={package.version}" + ) + + +def _download_locked_artifact(artifact: LockedArtifact) -> bytes: + """Download one bounded artifact and recheck its size and SHA-256 digest.""" + try: + host = urllib.parse.urlparse(artifact.url).hostname + except ValueError as exc: + raise LicenseValidationError( + f"untrusted license evidence URL: {artifact.url}" + ) from exc + if host not in PYPI_HOSTS: + raise LicenseValidationError(f"untrusted license evidence URL: {artifact.url}") + _https_url(artifact.url, host) + request = urllib.request.Request(artifact.url) + with _opener().open(request, timeout=30) as response: + payload = _read_bounded(response, MAX_ARTIFACT_BYTES) + if len(payload) != artifact.size: + raise LicenseValidationError( + f"artifact size mismatch for {artifact.filename}: expected {artifact.size}, got {len(payload)}" + ) + digest = hashlib.sha256(payload).hexdigest() + if digest != artifact.sha256: + raise LicenseValidationError( + f"artifact digest mismatch for {artifact.filename}" + ) + return payload + + +def _metadata_from_artifact(payload: bytes, filename: str) -> Message: + """Extract one bounded wheel or source-distribution METADATA document.""" + metadata_bytes: bytes | None = None + try: + if filename.endswith(".whl"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + names = [ + name + for name in archive.namelist() + if name.endswith(".dist-info/METADATA") + ] + if len(names) != 1: + raise LicenseValidationError( + "wheel has no unique dist-info METADATA" + ) + info = archive.getinfo(names[0]) + if info.file_size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "wheel license metadata is oversized" + ) + metadata_bytes = archive.read(info) + elif filename.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".tgz")): + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as archive: + members = [ + member + for member in archive.getmembers() + if member.isfile() and member.name.endswith("/PKG-INFO") + ] + if len(members) != 1 or members[0].size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "source distribution has invalid PKG-INFO" + ) + extracted = archive.extractfile(members[0]) + if extracted is None: + raise LicenseValidationError( + "source distribution PKG-INFO is unreadable" + ) + metadata_bytes = extracted.read(MAX_LICENSE_METADATA_BYTES + 1) + if len(metadata_bytes) > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "source license metadata is oversized" + ) + except (tarfile.TarError, zipfile.BadZipFile, KeyError, OSError) as exc: + raise LicenseValidationError( + f"artifact metadata container is invalid: {filename}" + ) from exc + if metadata_bytes is None: + raise LicenseValidationError(f"unsupported PyPI artifact type: {filename}") + return BytesParser().parsebytes(metadata_bytes) + + +def _validate_spdx_expression(expression: str) -> str: + """Validate a bounded SPDX expression containing only approved licenses.""" + value = expression.strip() + if not value: + raise LicenseValidationError("incomplete SPDX expression") + tokens = SPDX_TOKEN.findall(value) + residual = SPDX_TOKEN.sub("", value) + if residual.strip(): + raise LicenseValidationError(f"disallowed SPDX expression: {value}") + + expect_operand = True + depth = 0 + for token in tokens: + if expect_operand: + if token == "(": + depth += 1 + continue + if token in {"AND", "OR", ")"}: + raise LicenseValidationError("incomplete SPDX expression") + if token not in ALLOWED_SPDX: + raise LicenseValidationError(f"disallowed SPDX license: {token}") + expect_operand = False + else: + if token == ")": + if depth < 1: + raise LicenseValidationError("incomplete SPDX expression") + depth -= 1 + continue + if token not in {"AND", "OR"}: + raise LicenseValidationError("incomplete SPDX expression") + expect_operand = True + if expect_operand or depth: + raise LicenseValidationError("incomplete SPDX expression") + return value + + +def _license_from_metadata(metadata: Message) -> str: + """Return one approved SPDX expression from package metadata.""" + expressions = [ + value.strip() + for value in metadata.get_all("License-Expression", []) + if value.strip() + ] + if expressions: + if len(set(expressions)) != 1: + raise LicenseValidationError("ambiguous SPDX license expressions") + return _validate_spdx_expression(expressions[0]) + + licenses = [ + value.strip() + for value in metadata.get_all("License", []) + if value.strip() and value.strip().upper() != "UNKNOWN" + ] + if licenses: + if len(set(licenses)) != 1: + raise LicenseValidationError("ambiguous package license metadata") + return _validate_spdx_expression(licenses[0]) + + mapped_classifiers = { + CLASSIFIER_TO_SPDX[classifier.removeprefix("License :: OSI Approved :: ")] + for classifier in metadata.get_all("Classifier", []) + if classifier.removeprefix("License :: OSI Approved :: ") + in CLASSIFIER_TO_SPDX + } + if len(mapped_classifiers) != 1: + raise LicenseValidationError("package license metadata is missing or ambiguous") + return mapped_classifiers.pop() + + +def _artifact_license(package: LockedPackage, metadata: dict[str, object]) -> str: + """Download a lock-bound artifact and return its approved SPDX license.""" + artifact = select_locked_artifact(package, metadata) + parsed = _metadata_from_artifact( + _download_locked_artifact(artifact), + artifact.filename, + ) + parsed_name = parsed.get("Name") + parsed_version = parsed.get("Version") + if ( + not isinstance(parsed_name, str) + or _normalize_package_name(parsed_name) + != _normalize_package_name(package.name) + or parsed_version != package.version + ): + raise LicenseValidationError( + f"artifact metadata identity mismatch for {package.name}=={package.version}" + ) + return _license_from_metadata(parsed) def validate_source_license(root: pathlib.Path) -> str: """Validate the pinned source tree's declared license before execution.""" pyproject = root / "pyproject.toml" if not pyproject.is_file(): - raise LicenseValidationError("contextual-orchestrator source lacks pyproject.toml") + raise LicenseValidationError( + "contextual-orchestrator source lacks pyproject.toml" + ) pyproject.read_bytes() - for candidate in (root / "LICENSE", root / "LICENSE.txt", root / "LICENSE.md"): + for candidate in ( + root / "LICENSE", + root / "LICENSE.txt", + root / "LICENSE.md", + ): if candidate.is_file(): content = candidate.read_text(encoding="utf-8", errors="replace") for marker, spdx in SOURCE_LICENSE_MARKERS.items(): if marker in content: return spdx - raise LicenseValidationError("source license evidence is missing or not approved") + raise LicenseValidationError( + "source license evidence is missing or not approved" + ) def validate_tree(root: pathlib.Path) -> dict[str, str]: """Validate source and every exact dependency before any installation.""" source_license = validate_source_license(root) - packages = parse_locked_packages((root / "requirements.lock").read_text(encoding="utf-8")) + packages = parse_locked_packages( + (root / "requirements.lock").read_text(encoding="utf-8") + ) result = {"contextual-orchestrator": source_license} - for name, version in packages: - normalized = name.lower().replace("_", "-") + for package in packages: + normalized = _normalize_package_name(package.name) url = _https_url( "https://pypi.org/pypi/" - f"{urllib.parse.quote(normalized, safe='')}/{urllib.parse.quote(version, safe='')}/json", + f"{urllib.parse.quote(normalized, safe='')}/" + f"{urllib.parse.quote(package.version, safe='')}/json", "pypi.org", ) - result[f"{name}=={version}"] = _artifact_license(name, version, _fetch_json(url)) + result[f"{package.name}=={package.version}"] = _artifact_license( + package, + _fetch_json(url), + ) return result @@ -209,10 +487,19 @@ def main() -> int: args = parser.parse_args() try: licenses = validate_tree(args.repository_root.resolve()) - except (OSError, ValueError, KeyError, LicenseValidationError, json.JSONDecodeError) as exc: + except ( + OSError, + ValueError, + KeyError, + LicenseValidationError, + json.JSONDecodeError, + ) as exc: print(f"LICENSE_VALIDATION_FAILED: {exc}") return 1 - print(f"LICENSE_VALIDATION_OK: {len(licenses)} source and dependency licenses verified") + print( + "LICENSE_VALIDATION_OK: " + f"{len(licenses)} source and dependency licenses verified" + ) return 0 From 3268c250457f4958d401b2b75f08aa9f43686212 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 22:26:04 -0700 Subject: [PATCH 16/23] test: bind cleanup disarm to successful readiness --- tests/test_contextual_orchestrator_sidecar_contract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py index 85d307b5c4..781c1a39af 100644 --- a/tests/test_contextual_orchestrator_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -75,4 +75,6 @@ def test_sidecar_does_not_execute_an_editable_build_and_cleans_failed_startup() assert 'PYTHONPATH="$RUNTIME_DIR"' in text assert 'sidecar_pid="$!"' in text assert 'kill "$sidecar_pid"' in text - assert text.index('echo "contextual-orchestrator sidecar ready') < text.index("trap - EXIT") + ready_index = text.index('echo "contextual-orchestrator sidecar ready') + disarm_index = text.rindex("trap - EXIT") + assert ready_index < disarm_index From 3d45c3e182dc75ec808746d6c76966523651bcb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:10:29 -0700 Subject: [PATCH 17/23] test: require a non-overridable sidecar source pin --- tests/test_contextual_orchestrator_sidecar_contract.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py index 781c1a39af..6d63036f5a 100644 --- a/tests/test_contextual_orchestrator_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -9,12 +9,14 @@ NOEMA_WORKFLOW = Path(".github/workflows/noema-review.yml") OPENCODE_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") SIDECAR_SCRIPT = Path("scripts/ci/start_contextual_orchestrator_sidecar.sh") +PINNED_ORCHESTRATOR_SHA = "7eb459ee72c37dead5d25f284dfa4546f149fbe1" def test_sidecar_source_is_immutable_and_readiness_is_bounded() -> None: """Privileged sidecar code must be commit-pinned and health probes bounded.""" text = SIDECAR_SCRIPT.read_text(encoding="utf-8") - assert 'CONTEXTUAL_ORCHESTRATOR_REF:=7eb459ee72c37dead5d25f284dfa4546f149fbe1' in text + assert f'CONTEXTUAL_ORCHESTRATOR_REF="{PINNED_ORCHESTRATOR_SHA}"' in text + assert "CONTEXTUAL_ORCHESTRATOR_REF:=" not in text assert '^[0-9a-fA-F]{40}$' in text assert '--connect-timeout 1 --max-time 2' in text assert 'validate_contextual_orchestrator_licenses.py' in text From ee277c4f72f745ea11e4a59d7a926474e1e59d32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 23:12:22 -0700 Subject: [PATCH 18/23] fix: make the sidecar source pin non-overridable --- scripts/ci/start_contextual_orchestrator_sidecar.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh index c94ca5d1b8..e6a34e994d 100755 --- a/scripts/ci/start_contextual_orchestrator_sidecar.sh +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -11,10 +11,10 @@ set -euo pipefail : "${GITHUB_ENV:=/dev/null}" : "${GITHUB_OUTPUT:=/dev/null}" -# Reviewed protected-main commit. Callers may override it only with another exact -# 40-hex commit SHA; mutable branches/tags are deliberately rejected because the -# checkout receives provider credentials later in this script. -: "${CONTEXTUAL_ORCHESTRATOR_REF:=7eb459ee72c37dead5d25f284dfa4546f149fbe1}" +# Reviewed protected-main commit. The source identity is deliberately not an +# environment override: changing it requires an ordinary reviewed code change in +# this central repository before provider credentials can reach the new source. +CONTEXTUAL_ORCHESTRATOR_REF="7eb459ee72c37dead5d25f284dfa4546f149fbe1" : "${CONTEXTUAL_ORCHESTRATOR_PORT:=8000}" : "${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST:=3}" : "${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS:=60}" From cd9f0256c0e5d3c30f380e1473aa6966f0406f42 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:29:31 +0900 Subject: [PATCH 19/23] test: complete sidecar contract docstrings --- .../test_assert_opencode_reasoning_effort.py | 2 ++ ...textual_orchestrator_license_validation.py | 7 ++++ tests/test_noema_review_gate.py | 33 +++++++++++++++++++ tests/test_opencode_agent_contract.py | 4 +++ 4 files changed, 46 insertions(+) diff --git a/tests/test_assert_opencode_reasoning_effort.py b/tests/test_assert_opencode_reasoning_effort.py index 64c635b480..4956efb96e 100644 --- a/tests/test_assert_opencode_reasoning_effort.py +++ b/tests/test_assert_opencode_reasoning_effort.py @@ -1,3 +1,5 @@ +"""Contract tests for OpenCode reasoning-effort validation.""" + import json import runpy import sys diff --git a/tests/test_contextual_orchestrator_license_validation.py b/tests/test_contextual_orchestrator_license_validation.py index ad0d3fe1dc..0fb9204f9e 100644 --- a/tests/test_contextual_orchestrator_license_validation.py +++ b/tests/test_contextual_orchestrator_license_validation.py @@ -114,14 +114,21 @@ def test_downloaded_artifact_digest_and_size_are_rechecked( ) class Response(io.BytesIO): + """Provide the bounded response interface used by the downloader.""" + def __enter__(self): + """Return the in-memory response for the context manager.""" return self def __exit__(self, *_args: object) -> bool: + """Keep the fake response from suppressing exceptions.""" return False class Opener: + """Return controlled artifact bytes without network access.""" + def open(self, *_args: object, **_kwargs: object) -> Response: + """Open the in-memory response expected by the validator.""" return Response(payload) monkeypatch.setattr(validator, "_opener", lambda: Opener()) diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 6636373533..072ba379eb 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,3 +1,5 @@ +"""Contract tests for Noema's independent current-head review gate.""" + import base64 import json from types import SimpleNamespace @@ -8,6 +10,7 @@ def fake_secret(*parts: str) -> str: + """Join secret fragments so tests never spell a token as one literal.""" return "".join(parts) @@ -38,9 +41,11 @@ def review(state="APPROVED", commit="head", login="opencode-agent", body="Result def test_run_split_repo_graphql_and_fetch_pr(monkeypatch): + """Validate subprocess safety, repository parsing, and basic API access.""" calls = [] def fake_run(argv, **kwargs): + """Capture the safe subprocess invocation and return JSON text.""" calls.append((argv, kwargs)) return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") @@ -58,6 +63,7 @@ def fake_run(argv, **kwargs): assert noema.split_repo("owner/repo") == ("owner", "repo") def test_scrub_sensitive_data(): + """Redact common credentials while preserving harmless text.""" assert noema.scrub_sensitive_data(None) is None assert noema.scrub_sensitive_data("") == "" assert noema.scrub_sensitive_data("ok") == "ok" @@ -74,12 +80,14 @@ def test_scrub_sensitive_data(): def test_scrub_sensitive_data_authorization_headers(): + """Redact bearer and basic authorization header values.""" assert noema.scrub_sensitive_data("Authorization: Basic dXNlcjpwYXNz") == "Authorization: Basic ***" assert noema.scrub_sensitive_data("Proxy-Authorization: Basic dXNlcjpwYXNz") == "Proxy-Authorization: Basic ***" assert noema.scrub_sensitive_data("authorization: bearer xyz") == "authorization: bearer ***" def test_split_repo_and_graphql(monkeypatch): + """Reject malformed repository names and decode GraphQL responses.""" with pytest.raises(ValueError): noema.split_repo("owner") with pytest.raises(ValueError): @@ -90,6 +98,7 @@ def test_split_repo_and_graphql(monkeypatch): calls = [] def fake_run(args, stdin=None): + """Return a deterministic pull-request GraphQL response.""" calls.append((args, stdin)) return '{"data":{"repository":{"pullRequest":{"number":7}}}}' @@ -105,6 +114,7 @@ def fake_run(args, stdin=None): def test_review_state_helpers_cover_current_head_logic(): + """Accept only current-head primary approval and detect active blockers.""" marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." current = review(body=marker_body) old = review(commit="old", body=marker_body) @@ -132,6 +142,7 @@ def test_review_state_helpers_cover_current_head_logic(): def test_review_state_helpers_reject_explicit_previous_head_evidence(): + """Reject review bodies that explicitly cite a superseded head.""" current_head = "a" * 40 previous_head = "b" * 40 approval_marker = "Result: APPROVE" @@ -161,6 +172,7 @@ def test_review_state_helpers_reject_explicit_previous_head_evidence(): def test_check_helpers_and_existing_noema_review(): + """Classify failed, running, and completed checks plus Noema markers.""" status_context = {"__typename": "StatusContext", "context": "ci", "state": "FAILURE"} check_run = { "__typename": "CheckRun", @@ -226,6 +238,7 @@ def test_check_helpers_and_existing_noema_review(): def test_current_actor_fetch_diff_and_json_extraction(monkeypatch): + """Cover actor fallback, bounded diffs, and JSON extraction errors.""" monkeypatch.setattr(noema, "run", lambda *args, **kwargs: "noema\n") assert noema.current_actor() == "noema" monkeypatch.setattr(noema, "run", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("no gh"))) @@ -243,6 +256,7 @@ def test_current_actor_fetch_diff_and_json_extraction(monkeypatch): def test_review_context_builders_include_codegraph_threads_and_files(monkeypatch, tmp_path): + """Build review context from changed files, threads, and CodeGraph evidence.""" assert noema.truncate_text("abc", 10) == "abc" assert "truncated 2 characters" in noema.truncate_text("abcdef", 4) assert "missing PR head SHA" in noema.changed_file_context("owner/repo", 7, "") @@ -256,6 +270,7 @@ def test_review_context_builders_include_codegraph_threads_and_files(monkeypatch calls = [] def fake_run(args, stdin=None): + """Return deterministic command output for context assembly.""" calls.append(args) target = args[2] if target.endswith("/files"): @@ -307,6 +322,7 @@ def fake_run(args, stdin=None): def test_review_context_reports_omitted_files_and_missing_codegraph(monkeypatch, tmp_path): + """Explain omitted files and missing indexes instead of hiding evidence gaps.""" monkeypatch.delenv("NOEMA_CODEGRAPH_CONTEXT_PATH", raising=False) assert noema.load_codegraph_context() == "" @@ -343,6 +359,7 @@ def read(self): def test_call_llm_handles_configuration_and_verdicts(monkeypatch): + """Exercise valid, malformed, and unavailable LLM response paths.""" pr = make_pr() monkeypatch.delenv("NOEMA_LLM_API_URL", raising=False) monkeypatch.delenv("NOEMA_LLM_API_KEY", raising=False) @@ -360,15 +377,20 @@ def test_call_llm_handles_configuration_and_verdicts(monkeypatch): seen = {} def fake_urlopen(request, timeout): + """Record request data and return a deterministic approval response.""" seen["url"] = request.full_url seen["body"] = json.loads(request.data.decode("utf-8")) return FakeResponse({"choices": [{"message": {"content": '{"decision":"approve","summary":"ok","findings":[]}'}}]}) # Since we replaced urlopen with build_opener, we mock build_opener class FakeOpener: + """Provide a configurable opener for URL and redirect tests.""" + def __init__(self, call_func): + """Store the callback used for the next request.""" self.call_func = call_func def open(self, request, timeout=None): + """Delegate the request to the configured fake callback.""" return self.call_func(request, timeout) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) @@ -379,6 +401,7 @@ def open(self, request, timeout=None): assert "extra review context" in seen["body"]["messages"][1]["content"] def fake_urlopen_defer(request, timeout=None): + """Return a response with an unsupported decision.""" return FakeResponse({"choices": [{"message": {"content": '{"decision":"defer"}'}}]}) monkeypatch.setattr( @@ -425,6 +448,7 @@ def fake_urlopen_defer(request, timeout=None): # target, not just those two spellings. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-local.example.com/chat") def fake_getaddrinfo(host, port, *args, **kwargs): + """Resolve the test hostname to an allowed loopback address.""" if host == "resolved-to-local.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] return original_getaddrinfo(host, port, *args, **kwargs) @@ -436,6 +460,7 @@ def fake_getaddrinfo(host, port, *args, **kwargs): # stay blocked regardless of loopback now being permitted. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-private.example.com/chat") def fake_getaddrinfo_private(host, port, *args, **kwargs): + """Resolve the test hostname to a blocked private address.""" if host == "resolved-to-private.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))] return original_getaddrinfo(host, port, *args, **kwargs) @@ -448,6 +473,7 @@ def fake_getaddrinfo_private(host, port, *args, **kwargs): # after the first non-raising entry. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://dual-stack.example.com/chat") def fake_getaddrinfo_dual_stack(host, port, *args, **kwargs): + """Resolve to two public addresses for multi-record validation.""" if host == "dual-stack.example.com": return [ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0)), @@ -462,6 +488,7 @@ def fake_getaddrinfo_dual_stack(host, port, *args, **kwargs): monkeypatch.setenv("NOEMA_LLM_API_URL", "http://mixed-resolution.example.com/chat") def fake_getaddrinfo_mixed(host, port, *args, **kwargs): + """Resolve to loopback and private addresses for fail-closed validation.""" if host == "mixed-resolution.example.com": return [ (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0)), @@ -476,6 +503,7 @@ def fake_getaddrinfo_mixed(host, port, *args, **kwargs): # Test unresolved hostname does not break monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") def fake_getaddrinfo_error(host, port, *args, **kwargs): + """Raise a resolver error to cover unresolved-host handling.""" raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_error) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) @@ -484,6 +512,7 @@ def fake_getaddrinfo_error(host, port, *args, **kwargs): # Test invalid IP string from getaddrinfo (unlikely but theoretically possible) monkeypatch.setenv("NOEMA_LLM_API_URL", "http://weird-dns.example.com/chat") def fake_getaddrinfo_invalid_ip(host, port, *args, **kwargs): + """Return malformed address data to cover defensive parsing.""" if host == "weird-dns.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("not_an_ip", 0))] return original_getaddrinfo(host, port, *args, **kwargs) @@ -519,6 +548,7 @@ def test_call_llm_rejects_control_character_scheme_evasion(monkeypatch): import socket def raise_gaierror(host, port, *args, **kwargs): + """Fail DNS resolution while testing URL scheme validation.""" raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", raise_gaierror) @@ -539,6 +569,7 @@ def test_call_llm_rejects_non_http_parsed_scheme(monkeypatch): def test_format_findings_and_submit_review(monkeypatch): + """Format bounded findings and submit the exact current-head review.""" findings = noema.format_findings( [ {"severity": "high", "file": "a.py", "line": 3, "message": "bad"}, @@ -572,6 +603,7 @@ def test_format_findings_and_submit_review(monkeypatch): def test_inspect_and_review_skip_paths(monkeypatch): + """Skip draft, stale, blocked, and already-reviewed pull requests.""" marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." clean_pr = make_pr(reviews={"nodes": [review(body=marker_body)]}) calls = [] @@ -603,6 +635,7 @@ def test_inspect_and_review_skip_paths(monkeypatch): def test_parse_args_and_main(monkeypatch): + """Parse CLI arguments and reject non-positive pull-request numbers.""" parsed = noema.parse_args(["--repo", "owner/repo", "--pr-number", "9"]) assert parsed.repo == "owner/repo" assert parsed.pr_number == 9 diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index b3831f82ee..7bb7b800fa 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1,3 +1,5 @@ +"""Contract tests for the read-only OpenCode review agent and model pool.""" + import json import os import re @@ -359,6 +361,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert f'"{model_name}": {{' in workflow def is_reasoning_capable(model_name: str) -> bool: + """Identify model families that accept high reasoning effort.""" return ( model_name.startswith("gpt-5") or model_name.startswith("openai/gpt-5") @@ -1872,6 +1875,7 @@ def test_opencode_job_timeout_contains_full_sequential_review_budget(): workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") def timeout_minutes(pattern: str) -> int: + """Extract one timeout value from the workflow contract.""" match = re.search(pattern, workflow, re.MULTILINE) assert match, f"missing timeout contract: {pattern}" return int(match.group(1)) From 4006c4365f48135fca43d209d0a374ead9be0bbf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 04:26:09 +0900 Subject: [PATCH 20/23] fix(noema): confine optional codegraph context --- scripts/ci/noema_review_gate.py | 29 ++++++++++++++++++++++++----- tests/test_noema_review_gate.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index c480b582e2..54502331a9 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -8,6 +8,7 @@ import ipaddress import json import os +import pathlib import re import socket import subprocess @@ -389,14 +390,32 @@ def review_thread_context(pr: dict[str, Any]) -> str: def load_codegraph_context() -> str: """Load optional precomputed CodeGraph context for structural review evidence.""" - path = os.environ.get("NOEMA_CODEGRAPH_CONTEXT_PATH", "").strip() - if not path: + raw_path = os.environ.get("NOEMA_CODEGRAPH_CONTEXT_PATH", "").strip() + if not raw_path: return "" + candidate = pathlib.Path(raw_path) + if candidate.is_absolute() or ".." in candidate.parts: + return "CodeGraph context unavailable: path must stay within the workspace" + workspace = pathlib.Path.cwd().resolve() try: - with open(path, encoding="utf-8") as handle: - return truncate_text(handle.read(), MAX_REVIEW_CONTEXT_CHARS) + current = workspace + for part in candidate.parts: + current /= part + if current.is_symlink(): + return "CodeGraph context unavailable: symlinked paths are not allowed" + resolved = candidate.resolve(strict=True) + resolved.relative_to(workspace) + if not resolved.is_file(): + return "CodeGraph context unavailable: context path is not a regular file" + with resolved.open(encoding="utf-8") as handle: + return truncate_text( + handle.read(MAX_REVIEW_CONTEXT_CHARS + 1), + MAX_REVIEW_CONTEXT_CHARS, + ) except OSError as exc: - return f"CodeGraph context unavailable: {exc}" + return f"CodeGraph context unavailable: {type(exc).__name__}" + except ValueError: + return "CodeGraph context unavailable: path is outside the workspace" def build_review_context(repo: str, number: int, pr: dict[str, Any]) -> str: diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 072ba379eb..eae24e5a16 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -286,7 +286,8 @@ def fake_run(args, stdin=None): monkeypatch.setattr(noema, "run", fake_run) codegraph_path = tmp_path / "codegraph.md" codegraph_path.write_text("call graph: src/a.py -> tests", encoding="utf-8") - monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(codegraph_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "codegraph.md") pr = make_pr( headRefOid="head sha", reviewThreads={ @@ -326,6 +327,36 @@ def test_review_context_reports_omitted_files_and_missing_codegraph(monkeypatch, monkeypatch.delenv("NOEMA_CODEGRAPH_CONTEXT_PATH", raising=False) assert noema.load_codegraph_context() == "" + +def test_codegraph_context_rejects_paths_outside_workspace(monkeypatch, tmp_path): + """Reject traversal, absolute, and symlink paths before reading context.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + outside = tmp_path / "outside.md" + outside.write_text("secret", encoding="utf-8") + monkeypatch.chdir(workspace) + + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(outside)) + assert "within the workspace" in noema.load_codegraph_context() + + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "../outside.md") + assert "within the workspace" in noema.load_codegraph_context() + + link = workspace / "context.md" + link.symlink_to(outside) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "context.md") + assert "symlinked paths" in noema.load_codegraph_context() + + +def test_codegraph_context_reads_bounded_workspace_file(monkeypatch, tmp_path): + """Read a regular workspace context file without unbounded allocation.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "context.md").write_text("trusted context", encoding="utf-8") + monkeypatch.chdir(workspace) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "context.md") + assert noema.load_codegraph_context() == "trusted context" + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(tmp_path / "missing.md")) assert "CodeGraph context unavailable" in noema.load_codegraph_context() From 82b5aa785c9a78a879ec591d1ea1ffbae844b07c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 04:31:26 +0900 Subject: [PATCH 21/23] docs(noema): record codegraph context boundary --- .../noema-codegraph-context-boundary.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docs/doctoring/noema-codegraph-context-boundary.md diff --git a/docs/doctoring/noema-codegraph-context-boundary.md b/docs/doctoring/noema-codegraph-context-boundary.md new file mode 100644 index 0000000000..71a7690f72 --- /dev/null +++ b/docs/doctoring/noema-codegraph-context-boundary.md @@ -0,0 +1,45 @@ +# Noema CodeGraph Context Boundary + +## Customer action + +Keep `NOEMA_CODEGRAPH_CONTEXT_PATH` unset unless the review job has generated a +context file inside its checked-out workspace. When it is set, use a relative +regular-file path such as `.codegraph/review-context.md`; absolute paths, +`..` traversal, and symlinked components are rejected before any bytes are read. + +## Root-cause record + +The Noema reviewer accepted an environment-provided filename directly in +`load_codegraph_context()`. A review-job process that inherited an unintended +absolute or traversal path could therefore read files outside the repository. +This is CWE-22 path traversal: canonicalization must remain inside a restricted +parent, and error output must not disclose the rejected path. + +The repair establishes one bounded trust boundary: + +1. accept only a non-empty relative path without `..` components; +2. reject symlinks in every path component; +3. resolve the candidate and require it to remain below the current workspace; +4. require a regular file and read at most `MAX_REVIEW_CONTEXT_CHARS + 1` bytes; +5. return a generic diagnostic so the untrusted environment value is not echoed. + +## Verification + +The contract tests cover a valid workspace file, absolute traversal, relative +traversal, a symlink escape, bounded reads, and the existing review-context +assembly. The check is intentionally fail-closed: missing or invalid context +reduces evidence rather than expanding file access. + +## Standards and authoritative references (APA 7th) + +Joint Task Force. (2020). *Security and privacy controls for information +systems and organizations* (NIST Special Publication 800-53, Revision 5, +Update 1). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-53r5 + +MITRE. (n.d.). *CWE-22: Improper limitation of a pathname to a restricted +directory ('Path traversal')* (Version 4.20). Common Weakness Enumeration. +Retrieved August 21, 2026, from https://cwe.mitre.org/data/definitions/22 + +OWASP Foundation. (n.d.). *Path traversal*. Retrieved August 21, 2026, from +https://owasp.org/www-community/attacks/Path_Traversal From 81c187235d1ce3e8ad59b220eccc811007b196f0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 04:33:59 +0900 Subject: [PATCH 22/23] fix(noema): reject platform path separators --- scripts/ci/noema_review_gate.py | 2 +- tests/test_noema_review_gate.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 54502331a9..d2c9afcda4 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -394,7 +394,7 @@ def load_codegraph_context() -> str: if not raw_path: return "" candidate = pathlib.Path(raw_path) - if candidate.is_absolute() or ".." in candidate.parts: + if candidate.is_absolute() or "\\" in raw_path or ".." in candidate.parts: return "CodeGraph context unavailable: path must stay within the workspace" workspace = pathlib.Path.cwd().resolve() try: diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index eae24e5a16..660659f9af 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -342,6 +342,9 @@ def test_codegraph_context_rejects_paths_outside_workspace(monkeypatch, tmp_path monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "../outside.md") assert "within the workspace" in noema.load_codegraph_context() + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", r"..\outside.md") + assert "within the workspace" in noema.load_codegraph_context() + link = workspace / "context.md" link.symlink_to(outside) monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "context.md") From 3840d87dfa23d23a9610d69950dcfb10b2a913a6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 04:34:44 +0900 Subject: [PATCH 23/23] docs(noema): cite current NIST control release --- docs/doctoring/noema-codegraph-context-boundary.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/noema-codegraph-context-boundary.md b/docs/doctoring/noema-codegraph-context-boundary.md index 71a7690f72..059856a609 100644 --- a/docs/doctoring/noema-codegraph-context-boundary.md +++ b/docs/doctoring/noema-codegraph-context-boundary.md @@ -32,9 +32,9 @@ reduces evidence rather than expanding file access. ## Standards and authoritative references (APA 7th) -Joint Task Force. (2020). *Security and privacy controls for information +Joint Task Force. (2020/2025). *Security and privacy controls for information systems and organizations* (NIST Special Publication 800-53, Revision 5, -Update 1). National Institute of Standards and Technology. +Release 5.2.0). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-53r5 MITRE. (n.d.). *CWE-22: Improper limitation of a pathname to a restricted