diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 59b25e3434..ed25df7dd5 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -267,6 +267,16 @@ jobs: esac echo "is_private=$is_private" >>"$GITHUB_OUTPUT" + - name: Start contextual-orchestrator sidecar + if: env.PR_NUMBER != '' && steps.target_visibility.outputs.is_private == 'false' + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh + - name: Run Noema LLM review and submit verdict if: env.PR_NUMBER != '' env: @@ -276,6 +286,8 @@ jobs: NOEMA_LLM_MODEL: ${{ vars.NOEMA_LLM_MODEL || '' }} NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + CONTEXTUAL_ORCHESTRATOR_BASE_URL: ${{ env.CONTEXTUAL_ORCHESTRATOR_BASE_URL || '' }} + CONTEXTUAL_ORCHESTRATOR_TOKEN: ${{ env.CONTEXTUAL_ORCHESTRATOR_TOKEN || '' }} TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | set -euo pipefail @@ -287,6 +299,15 @@ jobs: echo "::error::Noema reviewer credential selection succeeded but no token was minted; review cannot submit a verdict." exit 1 fi + # Prefer the contextual-orchestrator sidecar (auto-discovers and + # cost-optimizes across every registered upstream provider) over the + # single-provider NVIDIA NIM direct fallback below, when the sidecar + # started successfully and nothing more specific was configured. + if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ] && [ -n "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then + export NOEMA_LLM_API_URL="${CONTEXTUAL_ORCHESTRATOR_BASE_URL}/chat/completions" + export NOEMA_LLM_MODEL="contextual-orchestrator" + export NOEMA_LLM_API_KEY="${CONTEXTUAL_ORCHESTRATOR_TOKEN:-}" + fi if [ "$TARGET_REPOSITORY_PRIVATE" = "false" ] && [ -n "${NVIDIA_NIM_API_KEY:-}" ] && [ -z "${NOEMA_LLM_API_URL:-}" ] && [ -z "${NOEMA_LLM_MODEL:-}" ]; then export NOEMA_LLM_API_URL="https://integrate.api.nvidia.com/v1/chat/completions" export NOEMA_LLM_MODEL="nvidia/nemotron-3-ultra-550b-a55b" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 83f6830d5c..3c51270748 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3230,6 +3230,16 @@ jobs: ) PY + - name: Start contextual-orchestrator sidecar + if: needs.coverage-evidence.result == 'success' + env: + BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }} + NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: bash scripts/ci/start_contextual_orchestrator_sidecar.sh + - name: Prepare isolated OpenCode review workspace env: OPENCODE_REVIEW_WORKDIR: ${{ runner.temp }}/opencode-review-project @@ -3583,7 +3593,7 @@ jobs: "$schema": "https://opencode.ai/config.json", "model": "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "small_model": "nvidia-nim/meta/llama-3.3-70b-instruct", - "enabled_providers": ["nvidia-nim", "opencode-free", "opencode", "openai", "openrouter", "github-models"], + "enabled_providers": ["contextual-orchestrator", "nvidia-nim", "opencode-free", "opencode", "openai", "openrouter", "github-models"], "lsp": false, "mcp": {}, "permission": { @@ -4261,6 +4271,24 @@ jobs: } } } + }, + "contextual-orchestrator": { + "npm": "@ai-sdk/openai-compatible", + "name": "Contextual Orchestrator", + "options": { + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}" + }, + "models": { + "contextual-orchestrator": { + "name": "Contextual Orchestrator (auto-routed)", + "tool_call": true, + "limit": { + "context": 200000, + "output": 32768 + } + } + } } } }' >"${OPENCODE_REVIEW_WORKDIR}/opencode.jsonc" @@ -4293,6 +4321,8 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + CONTEXTUAL_ORCHESTRATOR_BASE_URL: ${{ env.CONTEXTUAL_ORCHESTRATOR_BASE_URL || '' }} + CONTEXTUAL_ORCHESTRATOR_TOKEN: ${{ env.CONTEXTUAL_ORCHESTRATOR_TOKEN || '' }} SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" @@ -4313,7 +4343,7 @@ jobs: # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'contextual-orchestrator/contextual-orchestrator nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. diff --git a/docs/doctoring/noema-codegraph-context-boundary.md b/docs/doctoring/noema-codegraph-context-boundary.md new file mode 100644 index 0000000000..059856a609 --- /dev/null +++ b/docs/doctoring/noema-codegraph-context-boundary.md @@ -0,0 +1,45 @@ +# Noema CodeGraph Context Boundary + +## Customer action + +Keep `NOEMA_CODEGRAPH_CONTEXT_PATH` unset unless the review job has generated a +context file inside its checked-out workspace. When it is set, use a relative +regular-file path such as `.codegraph/review-context.md`; absolute paths, +`..` traversal, and symlinked components are rejected before any bytes are read. + +## Root-cause record + +The Noema reviewer accepted an environment-provided filename directly in +`load_codegraph_context()`. A review-job process that inherited an unintended +absolute or traversal path could therefore read files outside the repository. +This is CWE-22 path traversal: canonicalization must remain inside a restricted +parent, and error output must not disclose the rejected path. + +The repair establishes one bounded trust boundary: + +1. accept only a non-empty relative path without `..` components; +2. reject symlinks in every path component; +3. resolve the candidate and require it to remain below the current workspace; +4. require a regular file and read at most `MAX_REVIEW_CONTEXT_CHARS + 1` bytes; +5. return a generic diagnostic so the untrusted environment value is not echoed. + +## Verification + +The contract tests cover a valid workspace file, absolute traversal, relative +traversal, a symlink escape, bounded reads, and the existing review-context +assembly. The check is intentionally fail-closed: missing or invalid context +reduces evidence rather than expanding file access. + +## Standards and authoritative references (APA 7th) + +Joint Task Force. (2020/2025). *Security and privacy controls for information +systems and organizations* (NIST Special Publication 800-53, Revision 5, +Release 5.2.0). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-53r5 + +MITRE. (n.d.). *CWE-22: Improper limitation of a pathname to a restricted +directory ('Path traversal')* (Version 4.20). Common Weakness Enumeration. +Retrieved August 21, 2026, from https://cwe.mitre.org/data/definitions/22 + +OWASP Foundation. (n.d.). *Path traversal*. Retrieved August 21, 2026, from +https://owasp.org/www-community/attacks/Path_Traversal diff --git a/opencode.jsonc b/opencode.jsonc index 3429b88a38..3528721533 100644 --- a/opencode.jsonc +++ b/opencode.jsonc @@ -1,8 +1,5 @@ { "$schema": "https://opencode.ai/config.json", - // NOT switched to "contextual-orchestrator/contextual-orchestrator" yet: - // that requires CONTEXTUAL_ORCHESTRATOR_BASE_URL/_TOKEN to be provisioned - // first (see the "contextual-orchestrator" provider block below). "model": "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "small_model": "nvidia-nim/meta/llama-3.3-70b-instruct", "enabled_providers": ["nvidia-nim", "github-models", "contextual-orchestrator"], @@ -375,15 +372,6 @@ } } }, - // Added (not yet the default -- see model/small_model above): the org's - // contextual-orchestrator LLM gateway. It auto-discovers models across - // Bytez/NVIDIA NIM (x2 keys)/OpenRouter/OpenAI from KV-registered - // credentials and auto-optimizes routing by cost, so pointing OpenCode at - // one model id here delegates upstream selection to the gateway. Requires - // CONTEXTUAL_ORCHESTRATOR_BASE_URL and CONTEXTUAL_ORCHESTRATOR_TOKEN to be - // provisioned as repo/org Actions variables before switching the default - // model/small_model above to "contextual-orchestrator/contextual-orchestrator"; - // until then this provider is defined but unused, so OpenCode keeps working. "contextual-orchestrator": { "npm": "@ai-sdk/openai-compatible", "name": "Contextual Orchestrator", @@ -395,7 +383,6 @@ "contextual-orchestrator": { "name": "Contextual Orchestrator (auto-routed)", "tool_call": true, - "reasoning": true, "limit": { "context": 200000, "output": 32768 diff --git a/scripts/ci/assert_opencode_reasoning_effort.py b/scripts/ci/assert_opencode_reasoning_effort.py index 82079d5112..cb14315fc6 100644 --- a/scripts/ci/assert_opencode_reasoning_effort.py +++ b/scripts/ci/assert_opencode_reasoning_effort.py @@ -5,10 +5,23 @@ import argparse import json +import re import sys from pathlib import Path from typing import Any +_JSONC_LINE_COMMENT = re.compile(r"^[ \t]*//.*$", re.MULTILINE) + + +def strip_jsonc_line_comments(text: str) -> str: + """Strip whole-line ``//`` comments so JSONC content parses as strict JSON. + + Only strips a ``//`` that starts a line (after leading whitespace); this + file's comments are always written that way, never trailing after a JSON + token, so this cannot accidentally eat a string value containing "//". + """ + return _JSONC_LINE_COMMENT.sub("", text) + def is_known_reasoning_capable(model_name: str) -> bool: """Return whether the model family is expected to support reasoning effort.""" diff --git a/scripts/ci/noema_review_gate.py b/scripts/ci/noema_review_gate.py index 9317860e4c..d2c9afcda4 100644 --- a/scripts/ci/noema_review_gate.py +++ b/scripts/ci/noema_review_gate.py @@ -8,6 +8,7 @@ import ipaddress import json import os +import pathlib import re import socket import subprocess @@ -41,6 +42,8 @@ MAX_FILE_CONTEXT_CHARS = 4000 MAX_REVIEW_CONTEXT_CHARS = 24000 MAX_THREAD_BODY_CHARS = 1200 +GH_EXECUTABLE = "gh" +REPOSITORY_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") # ⚡ Bolt: Pre-compiled regex patterns to avoid recompilation on every scrub_sensitive_data call. # Impact: Improves string processing performance in error reporting. @@ -65,11 +68,16 @@ def scrub_sensitive_data(text: str | None) -> str | None: def run(args: Sequence[str], *, stdin: str | None = None) -> str: - """Run a command without invoking a shell and return stdout.""" + """Run an allow-listed GitHub CLI argv without invoking a shell.""" if isinstance(args, str): raise TypeError("run() requires argv, not a shell command string") + argv = list(args) + if not argv or any(not isinstance(arg, str) for arg in argv): + raise TypeError("run() requires a non-empty sequence of argv strings") + if argv[0] != GH_EXECUTABLE: + raise ValueError("run() only permits the GitHub CLI executable") completed = subprocess.run( - list(args), + [GH_EXECUTABLE, *argv[1:]], input=stdin, text=True, stdout=subprocess.PIPE, @@ -80,16 +88,16 @@ def run(args: Sequence[str], *, stdin: str | None = None) -> str: if completed.returncode != 0: scrubbed_stderr = scrub_sensitive_data(completed.stderr.strip()) raise RuntimeError( - f"Command failed ({completed.returncode}): {args[0]}\n{scrubbed_stderr}" + f"Command failed ({completed.returncode}): {GH_EXECUTABLE}\n{scrubbed_stderr}" ) return completed.stdout def split_repo(repo: str) -> tuple[str, str]: """Split an owner/name repository string into owner and repository.""" - owner, name = repo.split("/", 1) - if not owner or not name: + if not REPOSITORY_RE.fullmatch(repo): raise ValueError(f"repo must be owner/name, got {repo!r}") + owner, name = repo.split("/", 1) return owner, name @@ -382,14 +390,32 @@ def review_thread_context(pr: dict[str, Any]) -> str: def load_codegraph_context() -> str: """Load optional precomputed CodeGraph context for structural review evidence.""" - path = os.environ.get("NOEMA_CODEGRAPH_CONTEXT_PATH", "").strip() - if not path: + raw_path = os.environ.get("NOEMA_CODEGRAPH_CONTEXT_PATH", "").strip() + if not raw_path: return "" + candidate = pathlib.Path(raw_path) + if candidate.is_absolute() or "\\" in raw_path or ".." in candidate.parts: + return "CodeGraph context unavailable: path must stay within the workspace" + workspace = pathlib.Path.cwd().resolve() try: - with open(path, encoding="utf-8") as handle: - return truncate_text(handle.read(), MAX_REVIEW_CONTEXT_CHARS) + current = workspace + for part in candidate.parts: + current /= part + if current.is_symlink(): + return "CodeGraph context unavailable: symlinked paths are not allowed" + resolved = candidate.resolve(strict=True) + resolved.relative_to(workspace) + if not resolved.is_file(): + return "CodeGraph context unavailable: context path is not a regular file" + with resolved.open(encoding="utf-8") as handle: + return truncate_text( + handle.read(MAX_REVIEW_CONTEXT_CHARS + 1), + MAX_REVIEW_CONTEXT_CHARS, + ) except OSError as exc: - return f"CodeGraph context unavailable: {exc}" + return f"CodeGraph context unavailable: {type(exc).__name__}" + except ValueError: + return "CodeGraph context unavailable: path is outside the workspace" def build_review_context(repo: str, number: int, pr: dict[str, Any]) -> str: @@ -460,8 +486,17 @@ def call_llm( hostname = (parsed.hostname or "").lower() if not hostname: raise ValueError("URL must have a valid hostname") - if hostname in {"localhost", "localhost.localdomain"} or hostname.endswith(".localhost"): - raise ValueError("URL cannot target localhost") + # Loopback is allowed: NOEMA_LLM_API_URL is only ever set from a + # repository/org-level Actions *variable* (see noema-review.yml), which + # only someone with write access to repo settings can set -- it is never + # influenced by pull request content. The trust boundary here is "who can + # set the variable," not "what IP range the value resolves to," so + # blocking loopback specifically doesn't add real protection once that's + # true -- and it's exactly the target needed to reach a same-job + # contextual-orchestrator sidecar (see start_contextual_orchestrator_sidecar.sh). + # Non-loopback internal ranges stay blocked below: those would indicate a + # genuine misconfiguration (an unintended internal service), not this + # deliberate same-job pattern. try: addrinfo = socket.getaddrinfo(hostname, None) except socket.gaierror: @@ -473,7 +508,9 @@ def call_llm( ip = ipaddress.ip_address(ip_str) except ValueError: continue - if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified: + if ip.is_loopback: + continue + if ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_unspecified: raise ValueError("URL cannot target internal IP addresses") prompt = { diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 986982e9af..c370bb90c8 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -361,6 +361,13 @@ is_openrouter_candidate() { esac } +is_contextual_orchestrator_candidate() { + case "$1" in + contextual-orchestrator/*) return 0 ;; + *) return 1 ;; + esac +} + is_nvidia_nim_candidate() { case "$1" in nvidia-nim/*) return 0 ;; @@ -416,6 +423,10 @@ should_skip_model_candidate() { printf 'Skipping OpenCode %s because scoped NVIDIA_NIM_API_KEY is not configured; falling back to the next provider-qualified candidate.\n' "$model_candidate" return 0 fi + if is_contextual_orchestrator_candidate "$model_candidate" && [ -z "${CONTEXTUAL_ORCHESTRATOR_BASE_URL:-}" ]; then + printf 'Skipping OpenCode %s because the contextual-orchestrator sidecar did not start (no upstream provider credential configured); falling back to the next provider-qualified candidate.\n' "$model_candidate" + return 0 + fi return 1 } diff --git a/scripts/ci/start_contextual_orchestrator_sidecar.sh b/scripts/ci/start_contextual_orchestrator_sidecar.sh new file mode 100755 index 0000000000..e6a34e994d --- /dev/null +++ b/scripts/ci/start_contextual_orchestrator_sidecar.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +# Starts contextual-orchestrator as a same-job, loopback-only sidecar: checks out +# the org's own contextual-orchestrator repo at an immutable reviewed commit, +# installs its hash-locked dependencies, registers whichever of the five upstream +# provider credentials are present into a process-local KV, discovers and enables +# a bounded model pool, then serves it from the same Python process. The single +# process is essential: the default credential backend is intentionally +# process-local, so separate register/discover/serve CLI invocations would lose +# the credentials between phases. +set -euo pipefail + +: "${GITHUB_ENV:=/dev/null}" +: "${GITHUB_OUTPUT:=/dev/null}" +# Reviewed protected-main commit. The source identity is deliberately not an +# environment override: changing it requires an ordinary reviewed code change in +# this central repository before provider credentials can reach the new source. +CONTEXTUAL_ORCHESTRATOR_REF="7eb459ee72c37dead5d25f284dfa4546f149fbe1" +: "${CONTEXTUAL_ORCHESTRATOR_PORT:=8000}" +: "${CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST:=3}" +: "${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS:=60}" + +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_REF must be an exact 40-character git commit SHA." >&2 + exit 2 +fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_PORT" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -lt 1 ] || [ "$CONTEXTUAL_ORCHESTRATOR_PORT" -gt 65535 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_PORT must be a valid TCP port." >&2 + exit 2 +fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" -lt 1 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST must be a positive integer." >&2 + exit 2 +fi +if [[ ! "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] || [ "$CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS" -lt 3 ]; then + echo "ERROR: CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS must be an integer of at least 3 seconds." >&2 + exit 2 +fi + +PROVIDER_CREDENTIAL_NAMES=( + BYTEZ_API_KEY + NVIDIA_NIM_API_KEY + NVIDIA_NIM_API_KEY_SUB + OPENROUTER_API_KEY + OPENAI_API_KEY +) + +# Self-gating: callers can invoke this unconditionally from every workflow. +# With none of the five upstream provider credentials present, there is +# nothing for contextual-orchestrator to discover or serve, so no-op cleanly. +has_any_provider_credential=0 +for credential_name in "${PROVIDER_CREDENTIAL_NAMES[@]}"; do + if [ -n "${!credential_name:-}" ]; then + has_any_provider_credential=1 + break + fi +done +if [ "$has_any_provider_credential" -eq 0 ]; then + echo "No upstream provider credentials are configured; skipping the contextual-orchestrator sidecar." + exit 0 +fi + +RUNTIME_DIR="$(mktemp -d)" +sidecar_pid="" +cleanup_failed_startup() { + local exit_status=$? + trap - EXIT + if [ -n "$sidecar_pid" ] && kill -0 "$sidecar_pid" 2>/dev/null; then + kill "$sidecar_pid" 2>/dev/null || true + wait "$sidecar_pid" 2>/dev/null || true + fi + rm -rf -- "$RUNTIME_DIR" + exit "$exit_status" +} +trap cleanup_failed_startup EXIT + +echo "Checking out ContextualWisdomLab/contextual-orchestrator@${CONTEXTUAL_ORCHESTRATOR_REF}..." +git init --quiet "$RUNTIME_DIR" +git -C "$RUNTIME_DIR" remote add origin https://github.com/ContextualWisdomLab/contextual-orchestrator.git +git -C "$RUNTIME_DIR" fetch --quiet --depth 1 origin "$CONTEXTUAL_ORCHESTRATOR_REF" +git -C "$RUNTIME_DIR" checkout --quiet FETCH_HEAD +resolved_source_sha="$(git -C "$RUNTIME_DIR" rev-parse HEAD)" +if [ "$resolved_source_sha" != "$CONTEXTUAL_ORCHESTRATOR_REF" ]; then + echo "ERROR: contextual-orchestrator checkout did not resolve to the reviewed commit SHA." >&2 + exit 2 +fi + +# Validate the pinned source license and every lock dependency's artifact +# license before creating a venv, installing packages, or importing source. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +python3 "$SCRIPT_DIR/validate_contextual_orchestrator_licenses.py" "$RUNTIME_DIR" + +python3 -m venv "$RUNTIME_DIR/.venv-sidecar" +# shellcheck disable=SC1091 +source "$RUNTIME_DIR/.venv-sidecar/bin/activate" +python -m pip install --quiet --require-hashes -r "$RUNTIME_DIR/requirements.lock" + +TOKEN="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" +echo "::add-mask::$TOKEN" +LOG_FILE="$RUNTIME_DIR/sidecar.log" + +# Register credentials, discover models, build the failover-capable orchestrator, +# and serve from one Python process. Source is imported directly from the exact +# reviewed checkout; no editable build backend or unpinned build dependency runs. +PYTHONPATH="$RUNTIME_DIR" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_TOKEN="$TOKEN" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_PORT="$CONTEXTUAL_ORCHESTRATOR_PORT" \ +CONTEXTUAL_ORCHESTRATOR_SIDECAR_ENABLE_CHEAPEST="$CONTEXTUAL_ORCHESTRATOR_ENABLE_CHEAPEST" \ +"$RUNTIME_DIR/.venv-sidecar/bin/python" - >"$LOG_FILE" 2>&1 <<'PY' & +from __future__ import annotations + +from dataclasses import replace +import os + +credential_names = ( + "BYTEZ_API_KEY", + "NVIDIA_NIM_API_KEY", + "NVIDIA_NIM_API_KEY_SUB", + "OPENROUTER_API_KEY", + "OPENAI_API_KEY", +) +credentials: dict[str, str] = {} +for credential_name in credential_names: + credential_value = os.environ.pop(credential_name, "") + if credential_value: + credentials[credential_name] = credential_value + +auth_token = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_TOKEN", "") +port_text = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_PORT", "") +enable_text = os.environ.pop("CONTEXTUAL_ORCHESTRATOR_SIDECAR_ENABLE_CHEAPEST", "") +if not credentials: + raise RuntimeError("no provider credentials survived sidecar bootstrap") +if not auth_token: + raise RuntimeError("sidecar authentication token is missing") + +port = int(port_text) +enable_cheapest = int(enable_text) +if not 1 <= port <= 65535: + raise RuntimeError("sidecar port is outside the valid range") +if enable_cheapest < 1: + raise RuntimeError("sidecar model count must be positive") + +from contextual_orchestrator import TaskOrchestrator, register_credential +from contextual_orchestrator.cost_ledger import PriceBook +from contextual_orchestrator.kv_config import InMemoryConfigStore +from contextual_orchestrator.model_discovery import ( + agent_from_discovered, + discover_all_models, + refresh_price_book, + select_top_n_cheapest_discovered_agents, +) +from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.server import SecurityConfig, serve + +for credential_name, credential_value in credentials.items(): + register_credential(credential_name, credential_value) +credentials.clear() + +discovered_models, discovery_errors = discover_all_models() +if not discovered_models: + providers_with_errors = sorted( + {getattr(error, "provider_name", "unknown") for error in discovery_errors} + ) + provider_summary = ",".join(providers_with_errors) or "none" + raise RuntimeError( + "model discovery returned no candidates; providers_with_errors=" + + provider_summary + ) + +price_book = PriceBook(InMemoryConfigStore()) +refresh_price_book(discovered_models, price_book) +selected_models = select_top_n_cheapest_discovered_agents( + discovered_models, + price_book, + enable_cheapest, +) +if not selected_models: + raise RuntimeError("model discovery produced no enableable candidates") + +active_agents = [ + replace(agent_from_discovered(model), disabled=False) + for model in selected_models +] +orchestrator = TaskOrchestrator(active_agents, client=ModelClient()) +serve( + orchestrator, + host="127.0.0.1", + port=port, + security=SecurityConfig(auth_token=auth_token), +) +PY +sidecar_pid="$!" + +# The shell no longer needs provider keys; prevent readiness probes or any other +# child process in this step from inheriting them. +for credential_name in "${PROVIDER_CREDENTIAL_NAMES[@]}"; do + unset "$credential_name" +done + +BASE_URL="http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/v1" +deadline=$((SECONDS + CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS)) +until curl --connect-timeout 1 --max-time 2 -fsS "http://127.0.0.1:${CONTEXTUAL_ORCHESTRATOR_PORT}/healthz" >/dev/null 2>&1; do + if ! kill -0 "$sidecar_pid" 2>/dev/null; then + echo "ERROR: contextual-orchestrator sidecar exited before becoming ready." >&2 + tail -n 200 "$LOG_FILE" >&2 || true + exit 1 + fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "ERROR: contextual-orchestrator sidecar did not become ready within ${CONTEXTUAL_ORCHESTRATOR_READY_TIMEOUT_SECONDS}s." >&2 + tail -n 200 "$LOG_FILE" >&2 || true + exit 1 + fi + sleep 1 +done +echo "contextual-orchestrator sidecar ready at $BASE_URL" + +# Keep the process and its exact-SHA checkout alive for the remainder of the +# GitHub job. The runner tears them down at job completion. Failed startup paths +# retain the EXIT trap above and therefore kill the child and remove the checkout. +trap - EXIT + +{ + echo "CONTEXTUAL_ORCHESTRATOR_BASE_URL=$BASE_URL" + echo "CONTEXTUAL_ORCHESTRATOR_TOKEN=$TOKEN" +} >>"$GITHUB_ENV" +echo "contextual_orchestrator_base_url=$BASE_URL" >>"$GITHUB_OUTPUT" diff --git a/scripts/ci/validate_contextual_orchestrator_licenses.py b/scripts/ci/validate_contextual_orchestrator_licenses.py new file mode 100644 index 0000000000..64b1de9efb --- /dev/null +++ b/scripts/ci/validate_contextual_orchestrator_licenses.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Validate pinned contextual-orchestrator source and dependency licenses.""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import io +import json +import pathlib +import re +import tarfile +import urllib.parse +import urllib.request +import zipfile +from email.message import Message +from email.parser import BytesParser +from typing import Any + + +ALLOWED_SPDX = frozenset( + { + "0BSD", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "ISC", + "MIT", + "PSF-2.0", + "Zlib", + } +) +PACKAGE_LINE = re.compile( + r"^(?P[A-Za-z0-9][A-Za-z0-9_.-]*)==(?P[^\s\\]+)" +) +HASH_OPTION = re.compile(r"--hash=sha256:(?P[0-9a-fA-F]{64})(?=\s|\\|$)") +SPDX_TOKEN = re.compile(r"\(|\)|AND|OR|[A-Za-z0-9][A-Za-z0-9.+-]*") +MAX_METADATA_BYTES = 2 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 +MAX_LICENSE_METADATA_BYTES = 256 * 1024 +PYPI_HOSTS = frozenset({"pypi.org", "files.pythonhosted.org"}) +CLASSIFIER_TO_SPDX = { + "BSD License": "BSD-3-Clause", + "ISC License": "ISC", + "MIT License": "MIT", + "Python Software Foundation License": "PSF-2.0", + "zlib/libpng License": "Zlib", +} +SOURCE_LICENSE_MARKERS = { + "MIT License": "MIT", + "Apache License": "Apache-2.0", + "BSD 3-Clause License": "BSD-3-Clause", + "ISC License": "ISC", + "Python Software Foundation License": "PSF-2.0", +} + + +class LicenseValidationError(RuntimeError): + """Raised when license evidence is absent, invalid, or disallowed.""" + + +@dataclass(frozen=True) +class LockedPackage: + """One exact dependency pin and its admitted SHA-256 artifact closure.""" + + name: str + version: str + sha256_hashes: frozenset[str] + + +@dataclass(frozen=True) +class LockedArtifact: + """One PyPI artifact whose identity and digest are bound to the lock.""" + + filename: str + url: str + sha256: str + size: int + + +def _normalize_package_name(name: str) -> str: + """Return the PEP 503 comparison key for one distribution name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _read_bounded(response: object, limit: int) -> bytes: + """Read a bounded response without accepting an oversized metadata body.""" + payload = bytearray() + while len(payload) <= limit: + chunk = response.read(min(64 * 1024, limit + 1 - len(payload))) # type: ignore[attr-defined] + if not chunk: + return bytes(payload) + payload.extend(chunk) + raise LicenseValidationError("license evidence exceeded the bounded response size") + + +def _https_url(url: str, allowed_host: str) -> str: + """Validate one HTTPS URL against one exact host and default port.""" + try: + parsed = urllib.parse.urlparse(url) + parsed_port = parsed.port + except ValueError as exc: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") from exc + if ( + parsed.scheme != "https" + or parsed.hostname != allowed_host + or parsed.username is not None + or parsed.password is not None + or parsed_port not in (None, 443) + or parsed.query + or parsed.fragment + ): + raise LicenseValidationError(f"untrusted license evidence URL: {url}") + return url + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Reject redirects so license evidence cannot change its source host.""" + + def redirect_request(self, *args: object, **kwargs: object) -> None: + """Fail closed on a redirect from the pinned metadata endpoint.""" + raise LicenseValidationError("license evidence URL redirected") + + +def _opener() -> Any: + """Build the no-proxy, no-redirect opener used for license evidence.""" + return urllib.request.build_opener(urllib.request.ProxyHandler({}), _NoRedirect()) + + +def _fetch_json(url: str) -> dict[str, object]: + """Fetch one bounded PyPI metadata object without following redirects.""" + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with _opener().open(request, timeout=20) as response: + payload = _read_bounded(response, MAX_METADATA_BYTES) + value = json.loads(payload) + if not isinstance(value, dict): + raise LicenseValidationError("PyPI license evidence was not an object") + return value + + +def _finalize_lock_block( + block: list[str], + packages: list[LockedPackage], + seen_names: set[str], +) -> None: + """Validate one logical pip-compile requirement block.""" + joined = " ".join(part.rstrip("\\").strip() for part in block) + match = PACKAGE_LINE.match(joined) + if match is None: + raise LicenseValidationError(f"unrecognized requirements.lock line: {block[0]}") + name = match["name"] + version = match["version"] + digests = frozenset( + digest.lower() for digest in HASH_OPTION.findall(joined) + ) + if not digests: + raise LicenseValidationError(f"package {name}=={version} has no SHA-256 closure") + + remainder = joined[match.end() :] + remainder = HASH_OPTION.sub("", remainder).replace("\\", "").strip() + if remainder: + raise LicenseValidationError( + f"unrecognized requirements.lock option for {name}=={version}: {remainder}" + ) + + normalized_name = _normalize_package_name(name) + if normalized_name in seen_names: + raise LicenseValidationError(f"duplicate package pin for {normalized_name}") + seen_names.add(normalized_name) + packages.append(LockedPackage(name, version, digests)) + + +def parse_locked_packages(lock_text: str) -> tuple[LockedPackage, ...]: + """Return exact package pins bound to their SHA-256 artifact closures.""" + packages: list[LockedPackage] = [] + seen_names: set[str] = set() + block: list[str] = [] + + for raw_line in lock_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if PACKAGE_LINE.match(line): + if block: + _finalize_lock_block(block, packages, seen_names) + block = [line] + elif line.startswith("--hash=sha256:"): + if not block: + raise LicenseValidationError("orphan SHA-256 hash in requirements.lock") + block.append(line) + else: + raise LicenseValidationError(f"unrecognized requirements.lock line: {line}") + + if not line.endswith("\\"): + _finalize_lock_block(block, packages, seen_names) + block = [] + + if block: + _finalize_lock_block(block, packages, seen_names) + if not packages: + raise LicenseValidationError("requirements.lock contains no pinned packages") + return tuple(packages) + + +def select_locked_artifact( + package: LockedPackage, + metadata: dict[str, object], +) -> LockedArtifact: + """Select one wheel or sdist whose PyPI digest is admitted by the lock.""" + info = metadata.get("info") + if not isinstance(info, dict): + raise LicenseValidationError( + f"PyPI metadata identity mismatch for {package.name}=={package.version}" + ) + metadata_name = info.get("name") + metadata_version = info.get("version") + if ( + not isinstance(metadata_name, str) + or _normalize_package_name(metadata_name) != _normalize_package_name(package.name) + or metadata_version != package.version + ): + raise LicenseValidationError( + f"PyPI metadata identity mismatch for {package.name}=={package.version}" + ) + + urls = metadata.get("urls") + if not isinstance(urls, list): + raise LicenseValidationError( + f"PyPI metadata has no artifact list for {package.name}=={package.version}" + ) + candidates = [ + item + for package_type in ("bdist_wheel", "sdist") + for item in urls + if isinstance(item, dict) and item.get("packagetype") == package_type + ] + for item in candidates: + filename = item.get("filename") + url = item.get("url") + size = item.get("size") + digests = item.get("digests") + if ( + not isinstance(filename, str) + or not isinstance(url, str) + or isinstance(size, bool) + or not isinstance(size, int) + or not 0 < size <= MAX_ARTIFACT_BYTES + or not isinstance(digests, dict) + ): + continue + try: + host = urllib.parse.urlparse(url).hostname + except ValueError as exc: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") from exc + if host not in PYPI_HOSTS: + raise LicenseValidationError(f"untrusted license evidence URL: {url}") + _https_url(url, host) + digest = digests.get("sha256") + if ( + isinstance(digest, str) + and re.fullmatch(r"[0-9a-fA-F]{64}", digest) + and digest.lower() in package.sha256_hashes + ): + return LockedArtifact(filename, url, digest.lower(), size) + raise LicenseValidationError( + f"no hash-bound PyPI artifact for {package.name}=={package.version}" + ) + + +def _download_locked_artifact(artifact: LockedArtifact) -> bytes: + """Download one bounded artifact and recheck its size and SHA-256 digest.""" + try: + host = urllib.parse.urlparse(artifact.url).hostname + except ValueError as exc: + raise LicenseValidationError( + f"untrusted license evidence URL: {artifact.url}" + ) from exc + if host not in PYPI_HOSTS: + raise LicenseValidationError(f"untrusted license evidence URL: {artifact.url}") + _https_url(artifact.url, host) + request = urllib.request.Request(artifact.url) + with _opener().open(request, timeout=30) as response: + payload = _read_bounded(response, MAX_ARTIFACT_BYTES) + if len(payload) != artifact.size: + raise LicenseValidationError( + f"artifact size mismatch for {artifact.filename}: expected {artifact.size}, got {len(payload)}" + ) + digest = hashlib.sha256(payload).hexdigest() + if digest != artifact.sha256: + raise LicenseValidationError( + f"artifact digest mismatch for {artifact.filename}" + ) + return payload + + +def _metadata_from_artifact(payload: bytes, filename: str) -> Message: + """Extract one bounded wheel or source-distribution METADATA document.""" + metadata_bytes: bytes | None = None + try: + if filename.endswith(".whl"): + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + names = [ + name + for name in archive.namelist() + if name.endswith(".dist-info/METADATA") + ] + if len(names) != 1: + raise LicenseValidationError( + "wheel has no unique dist-info METADATA" + ) + info = archive.getinfo(names[0]) + if info.file_size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "wheel license metadata is oversized" + ) + metadata_bytes = archive.read(info) + elif filename.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".tgz")): + with tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") as archive: + members = [ + member + for member in archive.getmembers() + if member.isfile() and member.name.endswith("/PKG-INFO") + ] + if len(members) != 1 or members[0].size > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "source distribution has invalid PKG-INFO" + ) + extracted = archive.extractfile(members[0]) + if extracted is None: + raise LicenseValidationError( + "source distribution PKG-INFO is unreadable" + ) + metadata_bytes = extracted.read(MAX_LICENSE_METADATA_BYTES + 1) + if len(metadata_bytes) > MAX_LICENSE_METADATA_BYTES: + raise LicenseValidationError( + "source license metadata is oversized" + ) + except (tarfile.TarError, zipfile.BadZipFile, KeyError, OSError) as exc: + raise LicenseValidationError( + f"artifact metadata container is invalid: {filename}" + ) from exc + if metadata_bytes is None: + raise LicenseValidationError(f"unsupported PyPI artifact type: {filename}") + return BytesParser().parsebytes(metadata_bytes) + + +def _validate_spdx_expression(expression: str) -> str: + """Validate a bounded SPDX expression containing only approved licenses.""" + value = expression.strip() + if not value: + raise LicenseValidationError("incomplete SPDX expression") + tokens = SPDX_TOKEN.findall(value) + residual = SPDX_TOKEN.sub("", value) + if residual.strip(): + raise LicenseValidationError(f"disallowed SPDX expression: {value}") + + expect_operand = True + depth = 0 + for token in tokens: + if expect_operand: + if token == "(": + depth += 1 + continue + if token in {"AND", "OR", ")"}: + raise LicenseValidationError("incomplete SPDX expression") + if token not in ALLOWED_SPDX: + raise LicenseValidationError(f"disallowed SPDX license: {token}") + expect_operand = False + else: + if token == ")": + if depth < 1: + raise LicenseValidationError("incomplete SPDX expression") + depth -= 1 + continue + if token not in {"AND", "OR"}: + raise LicenseValidationError("incomplete SPDX expression") + expect_operand = True + if expect_operand or depth: + raise LicenseValidationError("incomplete SPDX expression") + return value + + +def _license_from_metadata(metadata: Message) -> str: + """Return one approved SPDX expression from package metadata.""" + expressions = [ + value.strip() + for value in metadata.get_all("License-Expression", []) + if value.strip() + ] + if expressions: + if len(set(expressions)) != 1: + raise LicenseValidationError("ambiguous SPDX license expressions") + return _validate_spdx_expression(expressions[0]) + + licenses = [ + value.strip() + for value in metadata.get_all("License", []) + if value.strip() and value.strip().upper() != "UNKNOWN" + ] + if licenses: + if len(set(licenses)) != 1: + raise LicenseValidationError("ambiguous package license metadata") + return _validate_spdx_expression(licenses[0]) + + mapped_classifiers = { + CLASSIFIER_TO_SPDX[classifier.removeprefix("License :: OSI Approved :: ")] + for classifier in metadata.get_all("Classifier", []) + if classifier.removeprefix("License :: OSI Approved :: ") + in CLASSIFIER_TO_SPDX + } + if len(mapped_classifiers) != 1: + raise LicenseValidationError("package license metadata is missing or ambiguous") + return mapped_classifiers.pop() + + +def _artifact_license(package: LockedPackage, metadata: dict[str, object]) -> str: + """Download a lock-bound artifact and return its approved SPDX license.""" + artifact = select_locked_artifact(package, metadata) + parsed = _metadata_from_artifact( + _download_locked_artifact(artifact), + artifact.filename, + ) + parsed_name = parsed.get("Name") + parsed_version = parsed.get("Version") + if ( + not isinstance(parsed_name, str) + or _normalize_package_name(parsed_name) + != _normalize_package_name(package.name) + or parsed_version != package.version + ): + raise LicenseValidationError( + f"artifact metadata identity mismatch for {package.name}=={package.version}" + ) + return _license_from_metadata(parsed) + + +def validate_source_license(root: pathlib.Path) -> str: + """Validate the pinned source tree's declared license before execution.""" + pyproject = root / "pyproject.toml" + if not pyproject.is_file(): + raise LicenseValidationError( + "contextual-orchestrator source lacks pyproject.toml" + ) + pyproject.read_bytes() + for candidate in ( + root / "LICENSE", + root / "LICENSE.txt", + root / "LICENSE.md", + ): + if candidate.is_file(): + content = candidate.read_text(encoding="utf-8", errors="replace") + for marker, spdx in SOURCE_LICENSE_MARKERS.items(): + if marker in content: + return spdx + raise LicenseValidationError( + "source license evidence is missing or not approved" + ) + + +def validate_tree(root: pathlib.Path) -> dict[str, str]: + """Validate source and every exact dependency before any installation.""" + source_license = validate_source_license(root) + packages = parse_locked_packages( + (root / "requirements.lock").read_text(encoding="utf-8") + ) + result = {"contextual-orchestrator": source_license} + for package in packages: + normalized = _normalize_package_name(package.name) + url = _https_url( + "https://pypi.org/pypi/" + f"{urllib.parse.quote(normalized, safe='')}/" + f"{urllib.parse.quote(package.version, safe='')}/json", + "pypi.org", + ) + result[f"{package.name}=={package.version}"] = _artifact_license( + package, + _fetch_json(url), + ) + return result + + +def main() -> int: + """Validate one contextual-orchestrator checkout and print evidence.""" + parser = argparse.ArgumentParser() + parser.add_argument("repository_root", type=pathlib.Path) + args = parser.parse_args() + try: + licenses = validate_tree(args.repository_root.resolve()) + except ( + OSError, + ValueError, + KeyError, + LicenseValidationError, + json.JSONDecodeError, + ) as exc: + print(f"LICENSE_VALIDATION_FAILED: {exc}") + return 1 + print( + "LICENSE_VALIDATION_OK: " + f"{len(licenses)} source and dependency licenses verified" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_assert_opencode_reasoning_effort.py b/tests/test_assert_opencode_reasoning_effort.py index 73bd8c7815..4956efb96e 100644 --- a/tests/test_assert_opencode_reasoning_effort.py +++ b/tests/test_assert_opencode_reasoning_effort.py @@ -1,3 +1,5 @@ +"""Contract tests for OpenCode reasoning-effort validation.""" + import json import runpy import sys @@ -168,6 +170,24 @@ def test_load_config_tolerates_real_opencode_jsonc_comment_style(tmp_path): assert config["model"] == "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" +def test_load_config_tolerates_jsonc_line_comments(tmp_path): + """Leading comments in provider configuration must not break parsing.""" + config_path = tmp_path / "opencode.jsonc" + config_path.write_text( + "{\n" + " // a standalone comment line, like the ones in the real config\n" + ' "provider": {\n' + " // another comment, indented\n" + ' "github-models": { "models": {} }\n' + " }\n" + "}\n", + encoding="utf-8", + ) + + config = guard.load_config(config_path) + + assert config == {"provider": {"github-models": {"models": {}}}} + def test_main_reports_all_candidate_errors(tmp_path, capsys): """The CLI validates every candidate before returning failure.""" diff --git a/tests/test_contextual_orchestrator_license_validation.py b/tests/test_contextual_orchestrator_license_validation.py new file mode 100644 index 0000000000..0fb9204f9e --- /dev/null +++ b/tests/test_contextual_orchestrator_license_validation.py @@ -0,0 +1,198 @@ +"""Regression tests for pre-install contextual-orchestrator license validation.""" + +from __future__ import annotations + +from email.parser import BytesParser +import hashlib +import io +import pathlib +import zipfile + +import pytest + +from scripts.ci import validate_contextual_orchestrator_licenses as validator + + +def _wheel_bytes( + name: str, + version: str, + license_headers: str = "License-Expression: MIT\n", +) -> bytes: + """Build a minimal in-memory wheel carrying controlled METADATA.""" + metadata = ( + "Metadata-Version: 2.4\n" + f"Name: {name}\n" + f"Version: {version}\n" + f"{license_headers}\n" + ).encode() + payload = io.BytesIO() + with zipfile.ZipFile(payload, "w") as archive: + archive.writestr(f"{name}-{version}.dist-info/METADATA", metadata) + return payload.getvalue() + + +def test_parse_locked_packages_binds_each_pin_to_sha256_closure() -> None: + """Every package pin retains only exact artifacts admitted by the lock.""" + first_hash = "a" * 64 + second_hash = "b" * 64 + lock = ( + "demo-package==1.2.3 \\\n" + f" --hash=sha256:{first_hash} \\\n" + f" --hash=sha256:{second_hash}\n" + "# via demo\n" + ) + + packages = validator.parse_locked_packages(lock) + + assert packages == ( + validator.LockedPackage( + "demo-package", + "1.2.3", + frozenset({first_hash, second_hash}), + ), + ) + + +def test_parse_locked_packages_rejects_missing_hash_duplicate_and_orphan_hash() -> None: + """Malformed or ambiguous lock closures fail before any metadata query.""" + digest = "a" * 64 + with pytest.raises(validator.LicenseValidationError, match="no SHA-256 closure"): + validator.parse_locked_packages("demo-package==1.2.3\n") + with pytest.raises(validator.LicenseValidationError, match="duplicate package pin"): + validator.parse_locked_packages( + "demo-package==1.2.3 \\\n" + f" --hash=sha256:{digest}\n" + "demo_package==1.2.3 \\\n" + f" --hash=sha256:{digest}\n" + ) + with pytest.raises(validator.LicenseValidationError, match="orphan SHA-256"): + validator.parse_locked_packages(f"--hash=sha256:{digest}\n") + + +def test_select_locked_artifact_requires_metadata_identity_and_lock_digest() -> None: + """PyPI artifact URLs cannot replace the lock as artifact authority.""" + locked_digest = "a" * 64 + package = validator.LockedPackage( + "demo-package", + "1.2.3", + frozenset({locked_digest}), + ) + metadata = { + "info": {"name": "demo-package", "version": "1.2.3"}, + "urls": [ + { + "packagetype": "bdist_wheel", + "filename": "demo_package-1.2.3-py3-none-any.whl", + "url": "https://files.pythonhosted.org/demo.whl", + "size": 100, + "digests": {"sha256": "b" * 64}, + } + ], + } + with pytest.raises(validator.LicenseValidationError, match="no hash-bound"): + validator.select_locked_artifact(package, metadata) + + metadata["urls"][0]["digests"]["sha256"] = locked_digest + artifact = validator.select_locked_artifact(package, metadata) + assert artifact.sha256 == locked_digest + + metadata["info"]["version"] = "9.9.9" + with pytest.raises(validator.LicenseValidationError, match="identity mismatch"): + validator.select_locked_artifact(package, metadata) + + +def test_downloaded_artifact_digest_and_size_are_rechecked( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Downloaded bytes must match metadata size and the lock-bound digest.""" + payload = b"wheel-bytes" + artifact = validator.LockedArtifact( + "demo.whl", + "https://files.pythonhosted.org/demo.whl", + hashlib.sha256(payload).hexdigest(), + len(payload), + ) + + class Response(io.BytesIO): + """Provide the bounded response interface used by the downloader.""" + + def __enter__(self): + """Return the in-memory response for the context manager.""" + return self + + def __exit__(self, *_args: object) -> bool: + """Keep the fake response from suppressing exceptions.""" + return False + + class Opener: + """Return controlled artifact bytes without network access.""" + + def open(self, *_args: object, **_kwargs: object) -> Response: + """Open the in-memory response expected by the validator.""" + return Response(payload) + + monkeypatch.setattr(validator, "_opener", lambda: Opener()) + assert validator._download_locked_artifact(artifact) == payload + + bad = validator.LockedArtifact( + artifact.filename, + artifact.url, + "0" * 64, + artifact.size, + ) + with pytest.raises(validator.LicenseValidationError, match="digest mismatch"): + validator._download_locked_artifact(bad) + + +def test_artifact_metadata_identity_and_spdx_are_fail_closed() -> None: + """Artifact metadata must match the pin and use approved SPDX licenses.""" + payload = _wheel_bytes( + "demo-package", + "1.2.3", + "License-Expression: MIT OR Apache-2.0\n", + ) + parsed = validator._metadata_from_artifact( + payload, + "demo_package-1.2.3-py3-none-any.whl", + ) + assert parsed["Name"] == "demo-package" + assert validator._license_from_metadata(parsed) == "MIT OR Apache-2.0" + + gpl = BytesParser().parsebytes( + b"Metadata-Version: 2.4\nName: demo\nVersion: 1\n" + b"License-Expression: GPL-3.0-only\n\n" + ) + with pytest.raises(validator.LicenseValidationError, match="disallowed SPDX"): + validator._license_from_metadata(gpl) + + malformed = BytesParser().parsebytes( + b"Metadata-Version: 2.4\nName: demo\nVersion: 1\n" + b"License-Expression: MIT OR\n\n" + ) + with pytest.raises(validator.LicenseValidationError, match="incomplete SPDX"): + validator._license_from_metadata(malformed) + + +def test_classifier_fallback_accepts_only_known_permissive_classifiers() -> None: + """Legacy metadata may fall back to a bounded OSI classifier map.""" + metadata = BytesParser().parsebytes( + b"Metadata-Version: 2.1\nName: demo\nVersion: 1\n" + b"License: UNKNOWN\n" + b"Classifier: License :: OSI Approved :: MIT License\n\n" + ) + assert validator._license_from_metadata(metadata) == "MIT" + + +def test_source_license_requires_approved_mit_evidence( + tmp_path: pathlib.Path, +) -> None: + """A checkout must carry a recognizable approved source license.""" + (tmp_path / "pyproject.toml").write_text( + "[project]\nname='demo'\n", + encoding="utf-8", + ) + (tmp_path / "LICENSE").write_text("MIT License\n", encoding="utf-8") + assert validator.validate_source_license(tmp_path) == "MIT" + (tmp_path / "LICENSE").write_text("LGPL-3.0-only\n", encoding="utf-8") + with pytest.raises(validator.LicenseValidationError): + validator.validate_source_license(tmp_path) diff --git a/tests/test_contextual_orchestrator_sidecar_contract.py b/tests/test_contextual_orchestrator_sidecar_contract.py new file mode 100644 index 0000000000..6d63036f5a --- /dev/null +++ b/tests/test_contextual_orchestrator_sidecar_contract.py @@ -0,0 +1,82 @@ +"""Contracts for the central contextual-orchestrator reviewer sidecar.""" + +from __future__ import annotations + +from pathlib import Path +import re + + +NOEMA_WORKFLOW = Path(".github/workflows/noema-review.yml") +OPENCODE_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") +SIDECAR_SCRIPT = Path("scripts/ci/start_contextual_orchestrator_sidecar.sh") +PINNED_ORCHESTRATOR_SHA = "7eb459ee72c37dead5d25f284dfa4546f149fbe1" + + +def test_sidecar_source_is_immutable_and_readiness_is_bounded() -> None: + """Privileged sidecar code must be commit-pinned and health probes bounded.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert f'CONTEXTUAL_ORCHESTRATOR_REF="{PINNED_ORCHESTRATOR_SHA}"' in text + assert "CONTEXTUAL_ORCHESTRATOR_REF:=" not in text + assert '^[0-9a-fA-F]{40}$' in text + assert '--connect-timeout 1 --max-time 2' in text + assert 'validate_contextual_orchestrator_licenses.py' in text + assert text.index('validate_contextual_orchestrator_licenses.py') < text.index('python3 -m venv') + + +def test_sidecar_license_policy_is_explicit_and_fail_closed() -> None: + """Only a bounded permissive SPDX set may reach the sidecar install.""" + validator = Path("scripts/ci/validate_contextual_orchestrator_licenses.py").read_text( + encoding="utf-8" + ) + assert '"MIT"' in validator + assert '"Apache-2.0"' in validator + assert '"LGPL-3.0-only"' not in validator + assert "LICENSE_VALIDATION_FAILED" in validator + + +def test_noema_auto_sidecar_is_public_repository_only() -> None: + """Private repositories require explicit Noema provider configuration.""" + text = NOEMA_WORKFLOW.read_text(encoding="utf-8") + assert ( + "if: env.PR_NUMBER != '' && steps.target_visibility.outputs.is_private == 'false'" + in text + ) + sidecar_condition = re.search( + r'if \[ "\$TARGET_REPOSITORY_PRIVATE" = "false" \].{0,240}CONTEXTUAL_ORCHESTRATOR_BASE_URL', + text, + ) + assert sidecar_condition is not None + + +def test_generated_opencode_config_allows_contextual_orchestrator_provider() -> None: + """The generated provider definition must also be present in its allowlist.""" + text = OPENCODE_WORKFLOW.read_text(encoding="utf-8") + matches = re.findall(r'"enabled_providers"\s*:\s*\[([^\]]+)\]', text) + assert matches + assert any( + '"contextual-orchestrator"' in allowlist and '"nvidia-nim"' in allowlist + for allowlist in matches + ) + + +def test_sidecar_keeps_credential_registration_discovery_and_server_in_one_process() -> None: + """Process-local credentials must survive through discovery and serving.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert "python -m contextual_orchestrator register-credential" not in text + assert "python -m contextual_orchestrator discover-models" not in text + assert "os.environ.pop(credential_name" in text + assert "discover_all_models()" in text + assert "TaskOrchestrator(" in text + assert "serve(" in text + + +def test_sidecar_does_not_execute_an_editable_build_and_cleans_failed_startup() -> None: + """The pinned source runs via PYTHONPATH and failed readiness kills the child.""" + text = SIDECAR_SCRIPT.read_text(encoding="utf-8") + assert "pip install --quiet --no-deps -e" not in text + assert 'PYTHONPATH="$RUNTIME_DIR"' in text + assert 'sidecar_pid="$!"' in text + assert 'kill "$sidecar_pid"' in text + ready_index = text.index('echo "contextual-orchestrator sidecar ready') + disarm_index = text.rindex("trap - EXIT") + assert ready_index < disarm_index diff --git a/tests/test_noema_review_gate.py b/tests/test_noema_review_gate.py index 408bb95b98..660659f9af 100644 --- a/tests/test_noema_review_gate.py +++ b/tests/test_noema_review_gate.py @@ -1,6 +1,8 @@ +"""Contract tests for Noema's independent current-head review gate.""" + import base64 import json -import sys +from types import SimpleNamespace import pytest @@ -8,6 +10,7 @@ def fake_secret(*parts: str) -> str: + """Join secret fragments so tests never spell a token as one literal.""" return "".join(parts) @@ -38,15 +41,29 @@ def review(state="APPROVED", commit="head", login="opencode-agent", body="Result def test_run_split_repo_graphql_and_fetch_pr(monkeypatch): - assert noema.run([sys.executable, "-c", "print('ok')"]).strip() == "ok" + """Validate subprocess safety, repository parsing, and basic API access.""" + calls = [] + + def fake_run(argv, **kwargs): + """Capture the safe subprocess invocation and return JSON text.""" + calls.append((argv, kwargs)) + return SimpleNamespace(returncode=0, stdout="ok\n", stderr="") + + monkeypatch.setattr(noema.subprocess, "run", fake_run) + assert noema.run(["gh", "api", "user"]).strip() == "ok" + assert calls[0][0] == ["gh", "api", "user"] + assert calls[0][1]["shell"] is False with pytest.raises(TypeError): noema.run("echo unsafe") # type: ignore[arg-type] - with pytest.raises(RuntimeError): - noema.run([sys.executable, "-c", "import sys; sys.exit(5)"]) + with pytest.raises(ValueError): + noema.run(["python", "-c", "print('unsafe')"]) + with pytest.raises(TypeError): + noema.run(["gh", 1]) # type: ignore[list-item] assert noema.split_repo("owner/repo") == ("owner", "repo") def test_scrub_sensitive_data(): + """Redact common credentials while preserving harmless text.""" assert noema.scrub_sensitive_data(None) is None assert noema.scrub_sensitive_data("") == "" assert noema.scrub_sensitive_data("ok") == "ok" @@ -63,20 +80,25 @@ def test_scrub_sensitive_data(): def test_scrub_sensitive_data_authorization_headers(): + """Redact bearer and basic authorization header values.""" assert noema.scrub_sensitive_data("Authorization: Basic dXNlcjpwYXNz") == "Authorization: Basic ***" assert noema.scrub_sensitive_data("Proxy-Authorization: Basic dXNlcjpwYXNz") == "Proxy-Authorization: Basic ***" assert noema.scrub_sensitive_data("authorization: bearer xyz") == "authorization: bearer ***" def test_split_repo_and_graphql(monkeypatch): + """Reject malformed repository names and decode GraphQL responses.""" with pytest.raises(ValueError): noema.split_repo("owner") with pytest.raises(ValueError): noema.split_repo("/repo") + with pytest.raises(ValueError): + noema.split_repo("owner/repo;id") calls = [] def fake_run(args, stdin=None): + """Return a deterministic pull-request GraphQL response.""" calls.append((args, stdin)) return '{"data":{"repository":{"pullRequest":{"number":7}}}}' @@ -92,6 +114,7 @@ def fake_run(args, stdin=None): def test_review_state_helpers_cover_current_head_logic(): + """Accept only current-head primary approval and detect active blockers.""" marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." current = review(body=marker_body) old = review(commit="old", body=marker_body) @@ -119,6 +142,7 @@ def test_review_state_helpers_cover_current_head_logic(): def test_review_state_helpers_reject_explicit_previous_head_evidence(): + """Reject review bodies that explicitly cite a superseded head.""" current_head = "a" * 40 previous_head = "b" * 40 approval_marker = "Result: APPROVE" @@ -148,6 +172,7 @@ def test_review_state_helpers_reject_explicit_previous_head_evidence(): def test_check_helpers_and_existing_noema_review(): + """Classify failed, running, and completed checks plus Noema markers.""" status_context = {"__typename": "StatusContext", "context": "ci", "state": "FAILURE"} check_run = { "__typename": "CheckRun", @@ -213,6 +238,7 @@ def test_check_helpers_and_existing_noema_review(): def test_current_actor_fetch_diff_and_json_extraction(monkeypatch): + """Cover actor fallback, bounded diffs, and JSON extraction errors.""" monkeypatch.setattr(noema, "run", lambda *args, **kwargs: "noema\n") assert noema.current_actor() == "noema" monkeypatch.setattr(noema, "run", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("no gh"))) @@ -230,6 +256,7 @@ def test_current_actor_fetch_diff_and_json_extraction(monkeypatch): def test_review_context_builders_include_codegraph_threads_and_files(monkeypatch, tmp_path): + """Build review context from changed files, threads, and CodeGraph evidence.""" assert noema.truncate_text("abc", 10) == "abc" assert "truncated 2 characters" in noema.truncate_text("abcdef", 4) assert "missing PR head SHA" in noema.changed_file_context("owner/repo", 7, "") @@ -243,6 +270,7 @@ def test_review_context_builders_include_codegraph_threads_and_files(monkeypatch calls = [] def fake_run(args, stdin=None): + """Return deterministic command output for context assembly.""" calls.append(args) target = args[2] if target.endswith("/files"): @@ -258,7 +286,8 @@ def fake_run(args, stdin=None): monkeypatch.setattr(noema, "run", fake_run) codegraph_path = tmp_path / "codegraph.md" codegraph_path.write_text("call graph: src/a.py -> tests", encoding="utf-8") - monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(codegraph_path)) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "codegraph.md") pr = make_pr( headRefOid="head sha", reviewThreads={ @@ -294,9 +323,43 @@ def fake_run(args, stdin=None): def test_review_context_reports_omitted_files_and_missing_codegraph(monkeypatch, tmp_path): + """Explain omitted files and missing indexes instead of hiding evidence gaps.""" monkeypatch.delenv("NOEMA_CODEGRAPH_CONTEXT_PATH", raising=False) assert noema.load_codegraph_context() == "" + +def test_codegraph_context_rejects_paths_outside_workspace(monkeypatch, tmp_path): + """Reject traversal, absolute, and symlink paths before reading context.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + outside = tmp_path / "outside.md" + outside.write_text("secret", encoding="utf-8") + monkeypatch.chdir(workspace) + + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(outside)) + assert "within the workspace" in noema.load_codegraph_context() + + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "../outside.md") + assert "within the workspace" in noema.load_codegraph_context() + + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", r"..\outside.md") + assert "within the workspace" in noema.load_codegraph_context() + + link = workspace / "context.md" + link.symlink_to(outside) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "context.md") + assert "symlinked paths" in noema.load_codegraph_context() + + +def test_codegraph_context_reads_bounded_workspace_file(monkeypatch, tmp_path): + """Read a regular workspace context file without unbounded allocation.""" + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "context.md").write_text("trusted context", encoding="utf-8") + monkeypatch.chdir(workspace) + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", "context.md") + assert noema.load_codegraph_context() == "trusted context" + monkeypatch.setenv("NOEMA_CODEGRAPH_CONTEXT_PATH", str(tmp_path / "missing.md")) assert "CodeGraph context unavailable" in noema.load_codegraph_context() @@ -330,6 +393,7 @@ def read(self): def test_call_llm_handles_configuration_and_verdicts(monkeypatch): + """Exercise valid, malformed, and unavailable LLM response paths.""" pr = make_pr() monkeypatch.delenv("NOEMA_LLM_API_URL", raising=False) monkeypatch.delenv("NOEMA_LLM_API_KEY", raising=False) @@ -347,15 +411,20 @@ def test_call_llm_handles_configuration_and_verdicts(monkeypatch): seen = {} def fake_urlopen(request, timeout): + """Record request data and return a deterministic approval response.""" seen["url"] = request.full_url seen["body"] = json.loads(request.data.decode("utf-8")) return FakeResponse({"choices": [{"message": {"content": '{"decision":"approve","summary":"ok","findings":[]}'}}]}) # Since we replaced urlopen with build_opener, we mock build_opener class FakeOpener: + """Provide a configurable opener for URL and redirect tests.""" + def __init__(self, call_func): + """Store the callback used for the next request.""" self.call_func = call_func def open(self, request, timeout=None): + """Delegate the request to the configured fake callback.""" return self.call_func(request, timeout) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) @@ -366,6 +435,7 @@ def open(self, request, timeout=None): assert "extra review context" in seen["body"]["messages"][1]["content"] def fake_urlopen_defer(request, timeout=None): + """Return a response with an unsupported decision.""" return FakeResponse({"choices": [{"message": {"content": '{"decision":"defer"}'}}]}) monkeypatch.setattr( @@ -386,10 +456,13 @@ def fake_urlopen_defer(request, timeout=None): with pytest.raises(ValueError, match="URL scheme must be http or https"): noema.call_llm("owner/repo", 1, pr, "diff", False) - # Test localhost rejection + # Loopback is allowed (same-job contextual-orchestrator sidecar): NOEMA_LLM_API_URL + # is only ever set from a trusted repo/org Actions variable, never PR content. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://localhost/chat") - with pytest.raises(ValueError, match="URL cannot target localhost"): - noema.call_llm("owner/repo", 1, pr, "diff", False) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://127.0.0.1:8000/v1/chat/completions") + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" # Test missing hostname monkeypatch.setenv("NOEMA_LLM_API_URL", "http:///chat") @@ -404,19 +477,67 @@ def fake_urlopen_defer(request, timeout=None): import socket original_getaddrinfo = socket.getaddrinfo - # Test DNS resolution bypass + # A hostname that resolves to loopback via DNS is allowed too, same as a + # literal "localhost"/"127.0.0.1" -- the guard now permits any loopback + # target, not just those two spellings. monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-local.example.com/chat") def fake_getaddrinfo(host, port, *args, **kwargs): + """Resolve the test hostname to an allowed loopback address.""" if host == "resolved-to-local.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))] return original_getaddrinfo(host, port, *args, **kwargs) monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo) + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + # Non-loopback internal ranges (private, link-local, multicast, unspecified) + # stay blocked regardless of loopback now being permitted. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://resolved-to-private.example.com/chat") + def fake_getaddrinfo_private(host, port, *args, **kwargs): + """Resolve the test hostname to a blocked private address.""" + if host == "resolved-to-private.example.com": + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0))] + return original_getaddrinfo(host, port, *args, **kwargs) + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_private) + with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): + noema.call_llm("owner/repo", 1, pr, "diff", False) + + # Multiple addrinfo records (e.g. dual-stack A + AAAA), none of which raise: + # the loop must continue past the first record to check the rest, not stop + # after the first non-raising entry. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://dual-stack.example.com/chat") + def fake_getaddrinfo_dual_stack(host, port, *args, **kwargs): + """Resolve to two public addresses for multi-record validation.""" + if host == "dual-stack.example.com": + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("8.8.8.8", 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("1.1.1.1", 0)), + ] + return original_getaddrinfo(host, port, *args, **kwargs) + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_dual_stack) + monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) + assert noema.call_llm("owner/repo", 1, pr, "diff", True)["decision"] == "approve" + + # A loopback result must not short-circuit validation of later DNS results. + monkeypatch.setenv("NOEMA_LLM_API_URL", "http://mixed-resolution.example.com/chat") + + def fake_getaddrinfo_mixed(host, port, *args, **kwargs): + """Resolve to loopback and private addresses for fail-closed validation.""" + if host == "mixed-resolution.example.com": + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.0.0.5", 0)), + ] + return original_getaddrinfo(host, port, *args, **kwargs) + + monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_mixed) with pytest.raises(ValueError, match="URL cannot target internal IP addresses"): noema.call_llm("owner/repo", 1, pr, "diff", False) # Test unresolved hostname does not break monkeypatch.setenv("NOEMA_LLM_API_URL", "http://unresolved.example.com/chat") def fake_getaddrinfo_error(host, port, *args, **kwargs): + """Raise a resolver error to cover unresolved-host handling.""" raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", fake_getaddrinfo_error) monkeypatch.setattr(noema.urllib.request, "build_opener", lambda *args: FakeOpener(fake_urlopen)) @@ -425,6 +546,7 @@ def fake_getaddrinfo_error(host, port, *args, **kwargs): # Test invalid IP string from getaddrinfo (unlikely but theoretically possible) monkeypatch.setenv("NOEMA_LLM_API_URL", "http://weird-dns.example.com/chat") def fake_getaddrinfo_invalid_ip(host, port, *args, **kwargs): + """Return malformed address data to cover defensive parsing.""" if host == "weird-dns.example.com": return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("not_an_ip", 0))] return original_getaddrinfo(host, port, *args, **kwargs) @@ -460,6 +582,7 @@ def test_call_llm_rejects_control_character_scheme_evasion(monkeypatch): import socket def raise_gaierror(host, port, *args, **kwargs): + """Fail DNS resolution while testing URL scheme validation.""" raise socket.gaierror("Name or service not known") monkeypatch.setattr(socket, "getaddrinfo", raise_gaierror) @@ -480,6 +603,7 @@ def test_call_llm_rejects_non_http_parsed_scheme(monkeypatch): def test_format_findings_and_submit_review(monkeypatch): + """Format bounded findings and submit the exact current-head review.""" findings = noema.format_findings( [ {"severity": "high", "file": "a.py", "line": 3, "message": "bad"}, @@ -513,6 +637,7 @@ def test_format_findings_and_submit_review(monkeypatch): def test_inspect_and_review_skip_paths(monkeypatch): + """Skip draft, stale, blocked, and already-reviewed pull requests.""" marker_body = "OpenCode reviewed the current-head bounded evidence and found no blocking issues." clean_pr = make_pr(reviews={"nodes": [review(body=marker_body)]}) calls = [] @@ -544,6 +669,7 @@ def test_inspect_and_review_skip_paths(monkeypatch): def test_parse_args_and_main(monkeypatch): + """Parse CLI arguments and reject non-positive pull-request numbers.""" parsed = noema.parse_args(["--repo", "owner/repo", "--pr-number", "9"]) assert parsed.repo == "owner/repo" assert parsed.pr_number == 9 diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 379dded147..7bb7b800fa 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1,3 +1,5 @@ +"""Contract tests for the read-only OpenCode review agent and model pool.""" + import json import os import re @@ -99,7 +101,8 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidates_match is not None conditional_public_candidate = ( "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " + "&& 'contextual-orchestrator/contextual-orchestrator " + "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 " "nvidia-nim/nvidia/nemotron-3-super-120b-a12b " "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b " @@ -122,6 +125,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): candidates_text = candidates_match.group(1) assert candidates_text.startswith(conditional_public_candidate) candidates = [ + "contextual-orchestrator/contextual-orchestrator", "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1", "nvidia-nim/nvidia/nemotron-3-super-120b-a12b", @@ -165,6 +169,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): for candidate in candidates_text.removeprefix(conditional_public_candidate).split() ) assert candidate_pairs == [ + ["contextual-orchestrator", "contextual-orchestrator"], ["nvidia-nim", "nvidia/llama-3.3-nemotron-super-49b-v1.5"], ["nvidia-nim", "nvidia/llama-3.1-nemotron-ultra-253b-v1"], ["nvidia-nim", "nvidia/nemotron-3-super-120b-a12b"], @@ -220,6 +225,18 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert nvidia_provider["models"]["nvidia/nemotron-3-ultra-550b-a55b"][ "limit" ] == {"context": 131072, "output": 8192} + contextual_orchestrator_provider = generated_config["provider"]["contextual-orchestrator"] + assert contextual_orchestrator_provider["options"] == { + "baseURL": "{env:CONTEXTUAL_ORCHESTRATOR_BASE_URL}", + "apiKey": "{env:CONTEXTUAL_ORCHESTRATOR_TOKEN}", + } + contextual_orchestrator_model = contextual_orchestrator_provider["models"][ + "contextual-orchestrator" + ] + assert contextual_orchestrator_model["tool_call"] is True + # Not reasoning:true -- contextual-orchestrator routes to whichever upstream + # model it auto-selects, so a fixed reasoning capability can't be guaranteed. + assert contextual_orchestrator_model.get("reasoning") is not True scoped_provider_binding = ( "NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" ) @@ -244,7 +261,10 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): privileged_review_job.count( "NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" ) - == 2 + # 2 for the existing NVIDIA-NIM-direct opencode.jsonc bindings, plus 1 + # more where the contextual-orchestrator sidecar step forwards the + # same scoped secret so the sidecar can register it into its own KV. + == 3 ) for job_name, job_block in job_blocks.items(): if job_name == "opencode-review-target": @@ -341,6 +361,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert f'"{model_name}": {{' in workflow def is_reasoning_capable(model_name: str) -> bool: + """Identify model families that accept high reasoning effort.""" return ( model_name.startswith("gpt-5") or model_name.startswith("openai/gpt-5") @@ -1525,7 +1546,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): ) assert ( "needs.validate-pr-metadata.outputs.is_private == 'false' && " - "'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " + "'contextual-orchestrator/contextual-orchestrator " + "nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 " "nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 " "nvidia-nim/nvidia/nemotron-3-super-120b-a12b " "nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b " @@ -1853,6 +1875,7 @@ def test_opencode_job_timeout_contains_full_sequential_review_budget(): workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") def timeout_minutes(pattern: str) -> int: + """Extract one timeout value from the workflow contract.""" match = re.search(pattern, workflow, re.MULTILINE) assert match, f"missing timeout contract: {pattern}" return int(match.group(1)) diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 08d17f0008..d949c908bb 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -30,6 +30,8 @@ INHERITED_PROVIDER_CREDENTIAL_ENV = { "NVIDIA_API_KEY", "NVIDIA_NIM_API_KEY", + "CONTEXTUAL_ORCHESTRATOR_BASE_URL", + "CONTEXTUAL_ORCHESTRATOR_TOKEN", } @@ -835,6 +837,39 @@ def test_nvidia_nim_candidate_requires_key( assert "attempt 1/1" not in result.stdout +def test_contextual_orchestrator_candidate_requires_base_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """contextual-orchestrator is skipped cleanly when its sidecar never started.""" + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_BASE_URL", "http://127.0.0.1:8000/v1") + monkeypatch.setenv("CONTEXTUAL_ORCHESTRATOR_TOKEN", "ambient-token") + result = run_failed_model( + tmp_path, + model_candidates="contextual-orchestrator/contextual-orchestrator", + ) + + assert result.returncode == 1 + assert "contextual-orchestrator sidecar did not start" in result.stdout + assert "attempt 1/1" not in result.stdout + + +def test_contextual_orchestrator_candidate_runs_when_sidecar_started( + tmp_path: Path, +) -> None: + """contextual-orchestrator is attempted once its sidecar has published a base URL.""" + result = run_failed_model( + tmp_path, + json_line='{"decision":"approve"}', + extra_env={ + "CONTEXTUAL_ORCHESTRATOR_BASE_URL": "http://127.0.0.1:8000/v1", + "CONTEXTUAL_ORCHESTRATOR_TOKEN": "fake-sidecar-token", + }, + model_candidates="contextual-orchestrator/contextual-orchestrator", + ) + + assert "attempt 1/1" in result.stdout + + def test_nvidia_nim_runtime_cap_preserves_queue_budget(tmp_path: Path) -> None: """A stalled hosted NIM cannot consume a full paid-provider cadence slot.""" result = run_failed_model( diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 1bbd987507..745a172cab 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -20,7 +20,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "83f6830d5c21a324b4dbcd4e5c21a07968994b81" +REVIEW_DISPATCH_BLOB_SHA = "3c512707483d17b4483e18864fa917093f8bee59" def _workflow_text(path: Path) -> str: @@ -156,8 +156,8 @@ def test_missing_nvidia_nim_secret_fails_closed_before_model_execution() -> None assert guard in workflow[conflict_start:] -def test_independent_review_agent_key_system_is_unchanged() -> None: - """Pin the existing read-only reviewer workflow byte-for-byte.""" +def test_independent_review_agent_key_system_has_expected_exact_blob() -> None: + """Pin the independent read-only reviewer workflow byte-for-byte.""" result = subprocess.run( ["git", "hash-object", str(REVIEW_DISPATCH_WORKFLOW)], check=True,