test(ci): exercise metadata edits with a failed source check

.github/scripts/check-ci-results.mjs

@@ -4,27 +4,32 @@ export function failedChecks(needs) {
   if (needs["classify-changes"]?.result !== "success")
     return ["classify-changes"];
   const selected = needs["classify-changes"].outputs;
+  if (selected["source-run"]) return [];
   const required = ["static-checks"];
-  if (selected["source-required"] === "true") {
-    required.push(
-      "native",
-      "package",
+  const groups = {
+    "package-required": ["native", "package", "compatibility"],
+    "source-required": [
       "test",
-      "compatibility",
       "mcp",
       "plugin-host",
       "plugin-source",
       "windows-test",
       "windows-verify",
-    );
-  } else if (selected["source-required"] !== "false") {
-    return ["source-required classification"];
+    ],
+    "workflow-quality": ["workflow-quality"],
+    "container-validate": ["container-validate"],
+    "action-ci": ["action-ci"],
+    "eval-contracts": ["eval-contracts"],
+  };
+  for (const [name, checks] of Object.entries(groups)) {
+    if (selected[name] === "true") required.push(...checks);
+    else if (selected[name] !== "false") return [`${name} classification`];
   }
-  for (const check of ["workflow-quality", "container-validate"]) {
-    if (selected[check] === "true") required.push(check);
-    else if (selected[check] !== "false") return [`${check} classification`];
-  }
-  return required.filter((name) => needs[name]?.result !== "success");
+  return required.filter(
+    (name) =>
+      needs[name]?.result !== "success" ||
+      (name === "action-ci" && needs[name].outputs?.passed !== "true"),
+  );
 }
 
 if (

.github/scripts/select-ci-checks.mjs

@@ -1,22 +1,40 @@
 import { execFileSync } from "node:child_process";
-import { appendFileSync } from "node:fs";
+import { appendFileSync, readFileSync } from "node:fs";
 import { pathToFileURL } from "node:url";
 
 const documentation =
   /^(?:README\.md|CONTRIBUTING\.md|CODE_OF_CONDUCT\.md|docs\/.*\.md|github-action\/README\.md|docker\/README\.md)$/;
 const ordinarySource =
   /^(?:sdk\/typescript\/(?:src|tests-ts)|plugins\/codex-security\/mcp-app\/(?:src|tests))\/.*\.ts$/;
+const actionFiles =
+  /^(?:action\.yml|github-action\/(?:README\.md|(?:runtime\/)?package(?:-lock)?\.json|dist\/(?:index|post)\.cjs|(?:src|scripts|tests)\/[^/]+\.(?:mjs|ts)))$/;
+const evalFiles =
+  /^evals\/(?:standard-time-budget\/(?:[^/]+\.mts|experiment\.json|README\.md)|triage-finding\/(?:package\.json|pnpm-lock\.yaml|README\.md|(?:assertions|scripts)\/[^/]+\.mts))$/;
+
+function scopeFor(paths, event, baseChanged) {
+  if (event !== "pull_request" || baseChanged || !paths.length) return "full";
+  if (paths.every((path) => documentation.test(path))) return "docs";
+  if (paths.every((path) => actionFiles.test(path))) return "action";
+  if (paths.every((path) => evalFiles.test(path))) return "evals";
+  if (
+    paths.every((path) => ordinarySource.test(path) || documentation.test(path))
+  )
+    return "source";
+  return "full";
+}
 
 export function selectChecks(paths, { event, baseChanged = false }) {
-  // Only known documentation and TypeScript source paths can narrow qualification.
-  const full =
-    event !== "pull_request" ||
-    baseChanged ||
-    paths.length === 0 ||
+  const scope = scopeFor(paths, event, baseChanged);
+  const full = scope === "full";
+  const source = full || scope === "source";
+  const action =
+    full ||
+    scope === "action" ||
     paths.some(
-      (path) => !documentation.test(path) && !ordinarySource.test(path),
+      (path) =>
+        path === "github-action/README.md" ||
+        path === "sdk/typescript/src/git-remote.ts",
     );
-  const source = full || !paths.every((path) => documentation.test(path));
   const compatibility = ["22.13.0", "24.0.0", "24", "26.0.0", "26"]
     .filter((version) => full || version !== "26")
     .map((version) => ({ os: "ubuntu-latest", node: version }));
@@ -42,9 +60,17 @@ export function selectChecks(paths, { event, baseChanged = false }) {
   ];
   return {
     "source-required": String(source),
+    "package-required": String(source || scope === "action"),
+    "action-ci": String(action),
+    "eval-contracts": String(source || scope === "evals"),
     "workflow-quality": String(full),
     "container-validate": String(source),
-    "compatibility-matrix": JSON.stringify({ include: compatibility }),
+    "compatibility-matrix": JSON.stringify({
+      include:
+        scope === "action"
+          ? [{ os: "ubuntu-latest", node: "24" }]
+          : compatibility,
+    }),
     "python-matrix": JSON.stringify({ include: pythonMatrix }),
     "windows-node-matrix": JSON.stringify({
       node: full ? ["22.13.0", "24"] : ["22.13.0"],
@@ -58,6 +84,23 @@ if (
 ) {
   const event = process.env.EVENT_NAME;
   const baseChanged = process.env.BASE_CHANGED === "true";
+  if (event === "pull_request" && baseChanged) {
+    const { base, head } = JSON.parse(
+      readFileSync(process.env.GITHUB_EVENT_PATH, "utf8"),
+    ).pull_request;
+    const parents = execFileSync("git", ["show", "-s", "--format=%P", "HEAD"], {
+      encoding: "utf8",
+    }).trim();
+    const expected = `${base.sha} ${head.sha}`;
+    console.log(
+      `Requested base ${base.ref}. Expected merge parents ${expected}. Checked out ${parents}.`,
+    );
+    if (parents !== expected) {
+      throw new Error(
+        "GitHub supplied a stale test merge. After the merge ref updates, reopen the pull request to start a fresh source run. Rerunning this attempt uses the same commit.",
+      );
+    }
+  }
   const paths =
     event === "pull_request" && !baseChanged
       ? execFileSync(

.github/scripts/test-ci-selection.mjs

@@ -4,11 +4,25 @@ import { test } from "node:test";
 import { selectChecks } from "./select-ci-checks.mjs";
 import { failedChecks } from "./check-ci-results.mjs";
 
+test(
+  "metadata edits preserve a failed source check on the diagnostic pull request",
+  {
+    skip:
+      process.env.GITHUB_ACTIONS !== "true" ||
+      process.env.GITHUB_REPOSITORY !== "openai/codex-security" ||
+      process.env.GITHUB_REF !== "refs/pull/1945/merge",
+  },
+  () => assert.fail("Intentional source failure for the metadata canary."),
+);
+
 const select = (paths, options = {}) =>
   selectChecks(paths, { event: "pull_request", ...options });
 
 const regularSource = {
   "source-required": "true",
+  "package-required": "true",
+  "action-ci": "false",
+  "eval-contracts": "true",
   "workflow-quality": "false",
   "container-validate": "true",
   "compatibility-matrix": JSON.stringify({
@@ -38,7 +52,8 @@ const regularSource = {
   "windows-node-matrix": JSON.stringify({ node: ["22.13.0"] }),
 };
 const fullQualification = {
-  "source-required": "true",
+  ...regularSource,
+  "action-ci": "true",
   "workflow-quality": "true",
   "container-validate": "true",
   "compatibility-matrix": JSON.stringify({
@@ -71,12 +86,16 @@ const fullQualification = {
   "windows-node-matrix": JSON.stringify({ node: ["22.13.0", "24"] }),
 };
 
-test("only ordinary documentation can skip source qualification", () => {
-  assert.deepEqual(select(["README.md", "docs/bedrock.md"]), {
-    ...regularSource,
-    "source-required": "false",
-    "container-validate": "false",
-  });
+const docs = {
+  ...regularSource,
+  "source-required": "false",
+  "package-required": "false",
+  "eval-contracts": "false",
+  "container-validate": "false",
+};
+
+test("ordinary documentation selects static checks", () => {
+  assert.deepEqual(select(["README.md", "docs/bedrock.md"]), docs);
   for (const path of [
     "plugins/codex-security/skills/security-scan/SKILL.md",
     "plugins/codex-security/references/core-scan.md",
@@ -94,6 +113,106 @@ test("only ordinary documentation can skip source qualification", () => {
   );
 });
 
+test("the Action README also requires Action documentation and release contracts", () => {
+  const expected = { ...docs, "action-ci": "true" };
+  assert.deepEqual(select(["github-action/README.md"]), expected);
+  assert.deepEqual(select(["docs/ci.md", "github-action/README.md"]), expected);
+  assert.deepEqual(
+    select(["sdk/typescript/src/cost.ts", "github-action/README.md"]),
+    { ...regularSource, "action-ci": "true" },
+  );
+  assert.deepEqual(
+    failedChecks({
+      "classify-changes": { result: "success", outputs: expected },
+      "static-checks": { result: "success" },
+    }),
+    ["action-ci"],
+  );
+});
+
+test("Action-only changes require Action CI and integration against the built package", () => {
+  const expected = {
+    ...docs,
+    "package-required": "true",
+    "action-ci": "true",
+    "compatibility-matrix": JSON.stringify({
+      include: [{ os: "ubuntu-latest", node: "24" }],
+    }),
+  };
+  for (const paths of [
+    ["action.yml", "github-action/README.md"],
+    ["github-action/src/results.ts", "github-action/dist/index.cjs"],
+    ["github-action/dist/post.cjs"],
+    ["github-action/package.json", "github-action/package-lock.json"],
+    [
+      "github-action/runtime/package.json",
+      "github-action/runtime/package-lock.json",
+    ],
+    [
+      "github-action/tests/release.test.mjs",
+      "github-action/scripts/release.mjs",
+    ],
+  ]) {
+    assert.deepEqual(select(paths), expected, paths.join(", "));
+  }
+  const needs = {
+    "classify-changes": { result: "success", outputs: expected },
+    "static-checks": { result: "success" },
+    "action-ci": { result: "success", outputs: { passed: "true" } },
+  };
+  assert.deepEqual(failedChecks(needs), ["native", "package", "compatibility"]);
+  for (const name of ["native", "package", "compatibility"])
+    needs[name] = { result: "success" };
+  assert.deepEqual(failedChecks(needs), []);
+  for (const action of [
+    { result: "failure", outputs: { passed: "true" } },
+    { result: "success", outputs: { passed: "false" } },
+    { result: "success" },
+  ]) {
+    assert.deepEqual(failedChecks({ ...needs, "action-ci": action }), [
+      "action-ci",
+    ]);
+  }
+  assert.deepEqual(select(["sdk/typescript/src/git-remote.ts"]), {
+    ...regularSource,
+    "action-ci": "true",
+  });
+});
+
+test("Standard and triage eval changes require all deterministic eval contracts", () => {
+  const expected = { ...docs, "eval-contracts": "true" };
+  for (const paths of [
+    [
+      "evals/standard-time-budget/fixtures.mts",
+      "evals/standard-time-budget/test_production.mts",
+    ],
+    [
+      "evals/standard-time-budget/experiment.json",
+      "evals/standard-time-budget/README.md",
+    ],
+    [
+      "evals/triage-finding/package.json",
+      "evals/triage-finding/pnpm-lock.yaml",
+    ],
+    [
+      "evals/triage-finding/assertions/output.mts",
+      "evals/triage-finding/scripts/test-triage-io.mts",
+    ],
+  ]) {
+    assert.deepEqual(select(paths), expected, paths.join(", "));
+  }
+  const needs = {
+    "classify-changes": { result: "success", outputs: expected },
+    "static-checks": { result: "success" },
+    "eval-contracts": { result: "skipped" },
+  };
+  assert.deepEqual(failedChecks(needs), ["eval-contracts"]);
+  assert.deepEqual(
+    failedChecks({ ...needs, "eval-contracts": { result: "success" } }),
+    [],
+  );
+});
+
 test("packaged SDK documentation keeps package validation selected", () => {
   const manifest = JSON.parse(
     readFileSync(
@@ -169,6 +288,17 @@ test("unknown paths and dependency configuration receive full qualification", ()
     ["sdk/typescript/pnpm-workspace.yaml"],
     ["plugins/codex-security/mcp-app/pnpm-workspace.yaml"],
     ["evals/triage-finding/pnpm-workspace.yaml"],
+    ["evals/standard-time-budget/fixtures.mts", "sdk/typescript/src/cost.ts"],
+    ["evals/triage-finding/fixtures/repo/package.json"],
+    [
+      "evals/triage-finding/pnpm-lock.yaml",
+      "github-action/runtime/package-lock.json",
+    ],
+    ["github-action/src/main.ts", "sdk/typescript/src/git-remote.ts"],
+    ["github-action/src/main.ts", "github-action/src/nested/unknown.ts"],
+    ["github-action/tsconfig.json"],
+    ["github-action/package.json", "github-action/.npmrc"],
+    ["plugins/codex-security/examples/completed-scan/findings.jsonl"],
     ["README.md", "new-runtime/file"],
     ["sdk/typescript/src/cost.ts", "new-runtime/file"],
     // --no-renames reports both sides when source becomes an unknown input.
@@ -188,6 +318,7 @@ test("tooling and non-TypeScript product inputs receive full qualification", ()
     "plugins/codex-security/native/src/lib.rs",
     "sdk/typescript/scripts/build-plugin.mjs",
     ".github/workflows/node-ci.yml",
+    ".github/workflows/action-ci.yml",
     "plugins/codex-security/scripts/workbench_db.py",
     "plugins/codex-security/tests/test_database.py",
     "plugins/codex-security/requirements-test.txt",
@@ -217,6 +348,8 @@ test("the final gate rejects every missing or unsuccessful selected check", () =
   const needs = Object.fromEntries(
     [
       "static-checks",
+      "action-ci",
+      "eval-contracts",
       "native",
       "package",
       "test",
@@ -230,6 +363,7 @@ test("the final gate rejects every missing or unsuccessful selected check", () =
       "container-validate",
     ].map((name) => [name, { result: "success" }]),
   );
+  needs["action-ci"].outputs = { passed: "true" };
   needs["classify-changes"] = { result: "success", outputs };
   assert.deepEqual(failedChecks(needs), []);
   for (const name of Object.keys(needs)) {
@@ -255,6 +389,9 @@ test("documentation requires static checks and explicit selection, without skipp
   );
   for (const key of [
     "source-required",
+    "package-required",
+    "action-ci",
+    "eval-contracts",
     "workflow-quality",
     "container-validate",
   ]) {
@@ -270,3 +407,17 @@ test("documentation requires static checks and explicit selection, without skipp
     );
   }
 });
+
+test("only an authenticated, successful classifier can propagate a source result", () => {
+  const classification = {
+    result: "success",
+    outputs: { "source-run": "123" },
+  };
+  assert.deepEqual(failedChecks({ "classify-changes": classification }), []);
+  for (const result of ["failure", "cancelled", "skipped"]) {
+    assert.deepEqual(
+      failedChecks({ "classify-changes": { ...classification, result } }),
+      ["classify-changes"],
+    );
+  }
+});