Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions sdk/typescript/docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2780,6 +2780,11 @@ for `patch`. Omit it or use `--verification-mode standard` for the default check
Results include evidence and `fixed`, `still_vulnerable`, or `inconclusive` status.
`--json` returns structured output. Exit codes are `0` when all findings are
fixed, `1` when any remain vulnerable, and `2` for inconclusive/failed checks.
Authentication and execution failures print diagnostics on stderr and return
`status: "failed"`, `code: "VERIFICATION_FAILED"`, and `message` with `--json`
or `--format jsonl`. With JSON/JSONL `--full-output`, failures use an `ok: false`
error envelope. Other formats retain stderr-only failures. Interruptions retain
exit codes `130` and `143` without a result.

## Command discovery and integrations

Expand Down
39 changes: 32 additions & 7 deletions sdk/typescript/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5818,7 +5818,9 @@ export async function main(
},
],
output: z.record(z.string(), z.unknown()).optional(),
async run({ format, options }) {
async run({ format, options, error: incurError }) {
const diagnostics = captureOutput();
let failureReported = false;
try {
if (options.host)
throw new CodexSecurityError("Use verify-fix --host alone.");
Expand Down Expand Up @@ -5906,13 +5908,23 @@ export async function main(
identifiers.length,
);
progress.startVerification();
const verificationErrors = new NodeWritable({
write(chunk, _encoding, callback) {
diagnostics.stream.write(chunk);
progress.stop();
void writeCliOutput(errorOutput, chunk).then(
() => callback(),
callback,
);
},
});
try {
exitCode = await runSkill(
"verify-fix",
selected === undefined ? [...positionals, ...imports] : [],
options,
response.stream,
errorOutput,
verificationErrors,
dependencies,
{
directory: repository,
Expand All @@ -5927,9 +5939,15 @@ export async function main(
},
);
if (exitCode !== 0) {
if (exitCode !== 130 && exitCode !== 143) exitCode = 2;
frameworkExit = exitCode;
return undefined;
if (exitCode === 130 || exitCode === 143) {
frameworkExit = exitCode;
return undefined;
}
failureReported = diagnostics.text().trim().length > 0;
throw new CodexSecurityError(
diagnostics.text().trim() ||
`Codex verification failed with exit code ${exitCode}.`,
);
}

results = parseFindingVerifications(response.text(), identifiers);
Expand Down Expand Up @@ -5960,8 +5978,15 @@ export async function main(
return undefined;
} catch (error) {
exitCode = 2;
errorOutput.write(`codex-security: ${diagnosticValue(error)}\n`);
throw error;
const message = errorMessage(error);
if (!failureReported)
errorOutput.write(`codex-security: ${diagnosticValue(message)}\n`);
return finishScan(
{ exitCode, error: message },
format,
incurError,
"VERIFICATION_FAILED",
);
}
},
})
Expand Down
114 changes: 108 additions & 6 deletions sdk/typescript/tests-ts/cli-verify-fix.test.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,20 @@
import { requestMetadata } from "./support/request-metadata.js";
import { describe, expect, test, mock } from "bun:test";
import { afterEach, describe, expect, test, mock } from "bun:test";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { Writable } from "node:stream";
import { stripVTControlCharacters } from "node:util";
import { main } from "../src/cli.js";
import { main, runCodexSkillCommand } from "../src/cli.js";
import { VERSION } from "../src/version.js";
import type { Finding, JsonObject } from "../src/index.js";
import type { LinearClientFactory } from "../src/linear.js";
import { capture, dependencies, fakeResult } from "./cli-fixtures.js";
import { PLUGIN_ROOT } from "./plugin-root.js";
import { createCliTest, captureCli } from "./support/cli-run.js";
import { createApiTestFixtures } from "./support/temporary-directories.js";

const { temporaryDirectory, cleanup } = createApiTestFixtures("verify-fix-");
afterEach(cleanup);

async function linearIssue(identifier: string) {
return {
Expand Down Expand Up @@ -430,18 +434,113 @@ describe("read-only finding verification", () => {
["--format", "yaml", "--full-output"],
["--token-count"],
]) {
const stdout = captureCli(main, "stdout");
const { stdout, stderr, runCli } = createCliTest(main);
expect(
await stdout.run(
await runCli(
["verify-fix", "Synthetic finding", ...flags],
dependencies({ onCodex: () => codexStatus }),
),
).toBe(expectedStatus);
expect(stdout.text()).toBe("");
const message = `Codex verification failed with exit code ${codexStatus}.`;
if (expectedStatus === 2) {
expect(stderr.text()).toContain(`codex-security: ${message}\n`);
expect(stderr.text().split(message)).toHaveLength(2);
} else expect(stderr.text()).not.toContain(message);
if (expectedStatus === 2 && flags[0] === "--json") {
const failure = JSON.parse(stdout.text());
if (flags.includes("--full-output")) {
expect(failure.ok).toBe(false);
expect(failure.error).toMatchObject({
code: "VERIFICATION_FAILED",
message,
});
} else {
expect(failure).toMatchObject({
status: "failed",
code: "VERIFICATION_FAILED",
message,
});
}
} else expect(stdout.text()).toBe("");
}
},
);

test.each(["exit", "throw"] as const)(
"reports ChatGPT authentication diagnostics after a Codex %s failure",
async (transport) => {
const message =
"401 Unauthorized: stored ChatGPT credentials expired. Run codex-security login --auth chatgpt.";
for (const flags of [
[],
["--json"],
["--json", "--full-output"],
["--format", "jsonl"],
]) {
const { stdout, stderr, runCli } = createCliTest(main);
expect(
await runCli(
["verify-fix", "Synthetic finding", "--auth", "chatgpt", ...flags],
dependencies({
onCodex: (_args, output) => {
if (transport === "throw") throw new Error(message);
output!.stderr.write(`${message}\n`);
return 1;
},
}),
),
).toBe(2);
if (flags.length === 0) expect(stdout.text()).toBe("");
else {
const failure = JSON.parse(stdout.text());
const details = flags.includes("--full-output")
? failure.error
: failure;
expect(details).toMatchObject({
code: "VERIFICATION_FAILED",
message,
});
expect(failure.results).toBeUndefined();
}
expect(stderr.text()).toContain(message);
expect(stderr.text().split(message)).toHaveLength(2);
}
},
);

test("reports an authentication protocol error from a successfully exited Codex child", async () => {
const directory = await temporaryDirectory();
const message = "401 Unauthorized: synthetic ChatGPT token expired";
const source = `
const lines = require("node:readline").createInterface({ input: process.stdin });
lines.once("line", () => process.stdout.write(JSON.stringify({
id: 1, error: { code: -32001, message: ${JSON.stringify(message)} }
}) + "\\n"));
`;
const { stdout, stderr, runCli } = createCliTest(main);
expect(
await runCli(
["verify-fix", "Synthetic finding", "--auth", "chatgpt", "--json"],
dependencies({
currentDirectory: directory,
onCodex: (_args, output) =>
runCodexSkillCommand(
["-e", source],
{ ...output!, auth: undefined },
{ command: process.execPath },
{},
),
}),
),
).toBe(2);
expect(JSON.parse(stdout.text())).toMatchObject({
status: "failed",
code: "VERIFICATION_FAILED",
message: expect.stringContaining(message),
});
expect(stderr.text()).toContain(message);
});

test.each([
["fixed", 0],
["still_vulnerable", 1],
Expand Down Expand Up @@ -658,7 +757,10 @@ describe("read-only finding verification", () => {
),
).toBe(2);

expect(stdout.text()).toBe("");
expect(JSON.parse(stdout.text())).toMatchObject({
status: "failed",
code: "VERIFICATION_FAILED",
});
expect(stderr.text()).toContain(
"an evidence-backed verification result for every finding",
);
Expand Down
Loading