From fac562dcb5bc5057d39f26099c5aa01666ede753 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 22:17:11 +0900 Subject: [PATCH 1/7] fix(test): stop the preload importing bun:test and crashing the runtime Bun re-loads the bunfig preload for every test file under --isolate (reload_entry_point_for_test_runner -> load_preloads), so a 219-file Windows shard evaluates tests/preload.ts 219 times in one process. #4796 added 'import { afterAll } from "bun:test"' to that graph to move temp-root cleanup onto the test lifecycle. On the 67th re-load Bun dereferenced a dead JSPromise in its own module loader and took SIGSEGV at 0x10, inside load_preloads and before the file's first test ran. Windows shard 5/6 failed that way on both attempts of dispatch runs 35087572377 (dc9d1fabc8) and 35093667426 (89bdf5fa4a). Linux shards took the same fault as exit 139 at five unrelated batch boundaries, where the batch runner hid it by re-running each file alone: one preload load never reaches the second that faults. Run 35054231781 at 8a1b01011d ran the same files in the same order, without the bun:test edge, and was green. Cleanup returns to the process exit listener, now on a 250ms removal budget because there is one listener per test file and the 15s default that a foreground teardown uses would let a contended exit outlive the job timeout. The containment and reclamation halves of #4796 are untouched, and reclamation does not depend on a handler firing. tests/ci-workflows/test-preload-module-graph.test.ts walks the preload's load-time graph, fails on any bun:test edge with the chain that introduced it, and is driven red by a fixture one hop away. --- scripts/test-layout/layout.json | 1 + scripts/test-temp.ts | 2 +- scripts/test.ts | 12 +- .../test-preload-module-graph.test.ts | 127 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + tests/preload.ts | 44 +++++- 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 tests/ci-workflows/test-preload-module-graph.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2106a9614be..f099fb5790f 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1350,6 +1350,7 @@ "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", + "test-preload-module-graph.test.ts": "ci-workflows", "test-runner.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", diff --git a/scripts/test-temp.ts b/scripts/test-temp.ts index 3915fafffad..8b80c1e64a6 100644 --- a/scripts/test-temp.ts +++ b/scripts/test-temp.ts @@ -51,7 +51,7 @@ export interface TestTempRecoveryResult { truncated: boolean; } -type RemoveTreeOptions = Readonly<{ +export type RemoveTreeOptions = Readonly<{ budgetMs?: number; delays?: readonly number[]; remove?: (path: string) => void; diff --git a/scripts/test.ts b/scripts/test.ts index 4406b080867..e484058091c 100644 --- a/scripts/test.ts +++ b/scripts/test.ts @@ -14,12 +14,18 @@ import { recoverStaleTestTempArtifactsOnce, removeTestTempTree, writeTestTempOwner, + type RemoveTreeOptions, } from "./test-temp"; export interface IsolatedTestEnvironment { root: string; env: Record; - cleanup(): void; + /** + * Remove this root. The caller chooses the removal budget because the two callers pay for it + * differently: a foreground teardown can afford the default long tail, while a process `exit` + * listener runs synchronously and there is one per test file (`tests/preload.ts`). + */ + cleanup(options?: RemoveTreeOptions): void; } export function createIsolatedTestEnvironment( @@ -82,8 +88,8 @@ export function createIsolatedTestEnvironment( TMP: containedTemp, TMPDIR: containedTemp, }, - cleanup() { - removeTestTempTree(root); + cleanup(options) { + removeTestTempTree(root, options); }, }; } diff --git a/tests/ci-workflows/test-preload-module-graph.test.ts b/tests/ci-workflows/test-preload-module-graph.test.ts new file mode 100644 index 00000000000..d030b55b0b7 --- /dev/null +++ b/tests/ci-workflows/test-preload-module-graph.test.ts @@ -0,0 +1,127 @@ +/** + * The bunfig preload may not reach `bun:test`. + * + * `bunfig.toml` names `tests/preload.ts`, and Bun re-loads it for every test file: `--isolate` + * gives each file its own realm and `reload_entry_point_for_test_runner` calls `load_preloads` + * again each time. A 219-file Windows shard therefore evaluates that module graph 219 times in + * one process, which makes the preload the single most re-entered graph in the repository and + * the wrong place to name a runtime built-in. + * + * #4796 named one anyway — `import { afterAll } from "bun:test"`, to register temp-root cleanup + * on the test lifecycle instead of on process exit. On the 67th re-load Bun dereferenced a dead + * JSPromise inside its own module loader and took SIGSEGV: + * + * panic(main thread): Segmentation fault at address 0x10 + * JSC::JSPromise::status -> JSC::JSModuleLoader::loadModule -> JSC::moduleLoadTopSettled + * -> bun_runtime::jsc_hooks::load_preloads + * + * Windows shard 5/6 of dispatch runs 35087572377 and 35093667426 failed that way on both + * attempts; Linux shards took the same fault as exit 139 at five unrelated batch boundaries, + * where the batch runner hid it by re-running each file alone — one preload load never reaches + * the second that faults, so singleton isolation always "passes". + * + * This guard is a static reach test, not a lint on one line: the edge that crashed the runtime + * would do so just as well one hop away, inside `scripts/test.ts` or any helper the preload + * pulls in. It shares `runtimeImportEdges`/`resolveSpec` with the other boundary guards for the + * reason recorded in `tests/helpers/import-graph.ts` — a second copy of the matcher cannot fail + * when the original drifts. + */ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSpec, runtimeImportEdges, slashed } from "../helpers/import-graph"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { repoPath } from "../helpers/repo-root"; + +const PRELOAD = repoPath("tests", "preload.ts"); + +/** A specifier the walk could not resolve to a file — every built-in arrives this way. */ +interface BareEdge { + readonly specifier: string; + /** Entry-first chain of slash-spelled paths ending at the file that names the specifier. */ + readonly chain: readonly string[]; +} + +function chainTo(previous: ReadonlyMap, leaf: string): string[] { + const chain: string[] = []; + let node: string | null = leaf; + while (node) { + chain.push(slashed(node)); + node = previous.get(node) ?? null; + } + return chain.reverse(); +} + +/** + * Every bare specifier reachable from `entry` over load-time edges. + * + * A dynamic `import()` is skipped for the same reason the Lab boundary skips it: a deferred + * edge is only entered if that branch runs, and deferring is the remedy rather than the defect. + */ +function bareSpecifiersReachableFrom(entry: string): BareEdge[] { + const previous = new Map([[entry, null]]); + const queue = [entry]; + const found: BareEdge[] = []; + while (queue.length > 0) { + const current = queue.shift()!; + let source: string; + try { + source = readFileSync(current, "utf8"); + } catch { + continue; + } + for (const edge of runtimeImportEdges(source)) { + if (edge.dynamic) continue; + const next = resolveSpec(edge.spec, current); + if (next === null) { + found.push({ specifier: edge.spec, chain: chainTo(previous, current) }); + continue; + } + if (previous.has(next)) continue; + previous.set(next, current); + queue.push(next); + } + } + return found; +} + +describe("the bunfig preload's module graph", () => { + test("never reaches bun:test", () => { + const offenders = bareSpecifiersReachableFrom(PRELOAD) + .filter(edge => edge.specifier === "bun:test") + .map(edge => edge.chain.join(" -> ")); + // Printed as the chain so a future violation names the hop that reintroduced it rather + // than only reporting that something, somewhere, did. + expect(offenders).toEqual([]); + }); + + test("the preload itself declares no bun:test import", () => { + const specifiers = runtimeImportEdges(readFileSync(PRELOAD, "utf8")).map(edge => edge.spec); + expect(specifiers).not.toContain("bun:test"); + }); + + test("the walk catches a bun:test edge introduced one hop away", () => { + // Attack the guard rather than trust it: without this, a walker that silently stopped at + // the first file would report the repository clean for every possible violation. + const directory = mkdtempSync(join(tmpdir(), "ocx-preload-graph-")); + try { + writeFileSync(join(directory, "entry.ts"), 'import { hook } from "./hop";\nexport { hook };\n'); + writeFileSync(join(directory, "hop.ts"), 'import { afterAll } from "bun:test";\nexport const hook = afterAll;\n'); + const offenders = bareSpecifiersReachableFrom(join(directory, "entry.ts")) + .filter(edge => edge.specifier === "bun:test"); + expect(offenders).toHaveLength(1); + expect(offenders[0]!.chain.map(path => path.split("/").at(-1))).toEqual(["entry.ts", "hop.ts"]); + } finally { + removeTreeWithRetry(directory); + } + }); + + test("the preload still reclaims its temp root through a process exit listener", () => { + // The crash is also curable by deleting the cleanup outright, which would hand #4762 its + // leak back. Pin the surviving mechanism so that repair cannot pass as this one. + const source = readFileSync(PRELOAD, "utf8"); + expect(source).toContain('process.on("exit", cleanupIsolatedRoot)'); + expect(source).toMatch(/isolated\.cleanup\(/); + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f5c49f9959c..d0df9e9555a 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1178,6 +1178,7 @@ "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", + "test-preload-module-graph.test.ts": "ci-workflows", "test-runner.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", diff --git a/tests/preload.ts b/tests/preload.ts index 50e65eff16d..df3e9d692ae 100644 --- a/tests/preload.ts +++ b/tests/preload.ts @@ -25,8 +25,33 @@ * `launchctl bootout gui//com.opencodex.proxy` reaches launchd — so neither cares * what HOME says. `assertLiveServiceManagerAllowed` in `src/service.ts` is the guard for * that, armed by the same flag set below. + * + * NOTHING IN THIS FILE'S IMPORT GRAPH MAY IMPORT `bun:test`. + * + * Bun re-loads this preload for EVERY test file: `--isolate` gives each file its own realm, + * and `VirtualMachine::reload_entry_point_for_test_runner` calls `load_preloads` again each + * time. Adding `import { afterAll } from "bun:test"` here (#4796, dc9d1fabc8) put a runtime + * built-in into that repeatedly re-entered graph, and on the 67th re-load Bun dereferenced a + * dead JSPromise inside its module loader and took SIGSEGV: + * + * panic(main thread): Segmentation fault at address 0x10 + * WTF::CompactPointerTuple::type + * JSC::JSPromise::status <- null cell + * JSC::JSPromise::performPromiseThenWithInternalMicrotask + * JSC::JSModuleLoader::loadModule + * JSC::moduleLoadTopSettled + * bun_runtime::jsc_hooks::load_preloads <- THIS FILE + * bun_jsc::virtual_machine::VirtualMachine::reload_entry_point_for_test_runner + * + * Windows shard 5/6 of dispatch runs 35087572377 and 35093667426 crashed there on both + * attempts, always at the file boundary and never inside a test; Linux shards took the same + * fault as exit 139 at five unrelated batch boundaries. Every file involved passes when it is + * the only file in its process — one preload load cannot reach the second re-load that faults. + * The immediately preceding dispatch run, 35054231781 at 8a1b01011d, ran the same files in the + * same order with no `bun:test` edge here and was green. + * + * `tests/ci-workflows/test-preload-module-graph.test.ts` is the guard that keeps the edge out. */ -import { afterAll } from "bun:test"; import { isTestHomeGuardArmed, protectedHomeForTests } from "../src/lib/test-home-guard"; import { createIsolatedTestEnvironment } from "../scripts/test"; import { @@ -116,17 +141,26 @@ if (process.platform === "win32" && lockPath && runLock.owner) { } // Clean up only the root this preload created. The `bun run test` wrapper owns its own. -// Bun test workers do not reliably run process `exit` handlers, so the test lifecycle hook -// is primary; the process hook remains a best-effort fallback for setup failures. +// +// A process `exit` listener, and deliberately not a `bun:test` lifecycle hook — reaching for +// `afterAll` is what crashed the runtime, for the reason recorded above the import list. +// +// Because the preload is re-loaded per test file, each file registers a listener for the root +// it created, and they all fire at process exit. A listener runs synchronously, so this pass +// takes a short removal budget rather than the 15s `removeTestTempTree` spends on a foreground +// teardown: a Windows shard holds ~219 of these, and a contended exit paying the full budget +// each would outlive the job timeout it is supposed to finish inside. Roots this pass cannot +// release are not lost — `recoverStaleTestTempArtifactsOnce` reclaims owned roots on a later +// run, and that half of #4796 does not depend on any handler firing. +const EXIT_CLEANUP_BUDGET_MS = 250; let cleanupComplete = false; const cleanupIsolatedRoot = () => { if (cleanupComplete) return; try { - isolated.cleanup(); + isolated.cleanup({ budgetMs: EXIT_CLEANUP_BUDGET_MS }); cleanupComplete = true; } catch { // The wrapper contains this root, and a later bare run reclaims it after the grace period. } }; -afterAll(cleanupIsolatedRoot); process.on("exit", cleanupIsolatedRoot); From 383fed2c377e38db1d74ce9c1b2cc723e21c47ea Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 22:34:51 +0900 Subject: [PATCH 2/7] Revert "fix(test): stop the preload importing bun:test and crashing the runtime" This reverts commit fac562dcb5bc5057d39f26099c5aa01666ede753. --- scripts/test-layout/layout.json | 1 - scripts/test-temp.ts | 2 +- scripts/test.ts | 12 +- .../test-preload-module-graph.test.ts | 127 ------------------ tests/fixtures/test-layout-expected.json | 1 - tests/preload.ts | 44 +----- 6 files changed, 9 insertions(+), 178 deletions(-) delete mode 100644 tests/ci-workflows/test-preload-module-graph.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index f099fb5790f..2106a9614be 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1350,7 +1350,6 @@ "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", - "test-preload-module-graph.test.ts": "ci-workflows", "test-runner.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", diff --git a/scripts/test-temp.ts b/scripts/test-temp.ts index 8b80c1e64a6..3915fafffad 100644 --- a/scripts/test-temp.ts +++ b/scripts/test-temp.ts @@ -51,7 +51,7 @@ export interface TestTempRecoveryResult { truncated: boolean; } -export type RemoveTreeOptions = Readonly<{ +type RemoveTreeOptions = Readonly<{ budgetMs?: number; delays?: readonly number[]; remove?: (path: string) => void; diff --git a/scripts/test.ts b/scripts/test.ts index e484058091c..4406b080867 100644 --- a/scripts/test.ts +++ b/scripts/test.ts @@ -14,18 +14,12 @@ import { recoverStaleTestTempArtifactsOnce, removeTestTempTree, writeTestTempOwner, - type RemoveTreeOptions, } from "./test-temp"; export interface IsolatedTestEnvironment { root: string; env: Record; - /** - * Remove this root. The caller chooses the removal budget because the two callers pay for it - * differently: a foreground teardown can afford the default long tail, while a process `exit` - * listener runs synchronously and there is one per test file (`tests/preload.ts`). - */ - cleanup(options?: RemoveTreeOptions): void; + cleanup(): void; } export function createIsolatedTestEnvironment( @@ -88,8 +82,8 @@ export function createIsolatedTestEnvironment( TMP: containedTemp, TMPDIR: containedTemp, }, - cleanup(options) { - removeTestTempTree(root, options); + cleanup() { + removeTestTempTree(root); }, }; } diff --git a/tests/ci-workflows/test-preload-module-graph.test.ts b/tests/ci-workflows/test-preload-module-graph.test.ts deleted file mode 100644 index d030b55b0b7..00000000000 --- a/tests/ci-workflows/test-preload-module-graph.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -/** - * The bunfig preload may not reach `bun:test`. - * - * `bunfig.toml` names `tests/preload.ts`, and Bun re-loads it for every test file: `--isolate` - * gives each file its own realm and `reload_entry_point_for_test_runner` calls `load_preloads` - * again each time. A 219-file Windows shard therefore evaluates that module graph 219 times in - * one process, which makes the preload the single most re-entered graph in the repository and - * the wrong place to name a runtime built-in. - * - * #4796 named one anyway — `import { afterAll } from "bun:test"`, to register temp-root cleanup - * on the test lifecycle instead of on process exit. On the 67th re-load Bun dereferenced a dead - * JSPromise inside its own module loader and took SIGSEGV: - * - * panic(main thread): Segmentation fault at address 0x10 - * JSC::JSPromise::status -> JSC::JSModuleLoader::loadModule -> JSC::moduleLoadTopSettled - * -> bun_runtime::jsc_hooks::load_preloads - * - * Windows shard 5/6 of dispatch runs 35087572377 and 35093667426 failed that way on both - * attempts; Linux shards took the same fault as exit 139 at five unrelated batch boundaries, - * where the batch runner hid it by re-running each file alone — one preload load never reaches - * the second that faults, so singleton isolation always "passes". - * - * This guard is a static reach test, not a lint on one line: the edge that crashed the runtime - * would do so just as well one hop away, inside `scripts/test.ts` or any helper the preload - * pulls in. It shares `runtimeImportEdges`/`resolveSpec` with the other boundary guards for the - * reason recorded in `tests/helpers/import-graph.ts` — a second copy of the matcher cannot fail - * when the original drifts. - */ -import { describe, expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { resolveSpec, runtimeImportEdges, slashed } from "../helpers/import-graph"; -import { removeTreeWithRetry } from "../helpers/remove-tree"; -import { repoPath } from "../helpers/repo-root"; - -const PRELOAD = repoPath("tests", "preload.ts"); - -/** A specifier the walk could not resolve to a file — every built-in arrives this way. */ -interface BareEdge { - readonly specifier: string; - /** Entry-first chain of slash-spelled paths ending at the file that names the specifier. */ - readonly chain: readonly string[]; -} - -function chainTo(previous: ReadonlyMap, leaf: string): string[] { - const chain: string[] = []; - let node: string | null = leaf; - while (node) { - chain.push(slashed(node)); - node = previous.get(node) ?? null; - } - return chain.reverse(); -} - -/** - * Every bare specifier reachable from `entry` over load-time edges. - * - * A dynamic `import()` is skipped for the same reason the Lab boundary skips it: a deferred - * edge is only entered if that branch runs, and deferring is the remedy rather than the defect. - */ -function bareSpecifiersReachableFrom(entry: string): BareEdge[] { - const previous = new Map([[entry, null]]); - const queue = [entry]; - const found: BareEdge[] = []; - while (queue.length > 0) { - const current = queue.shift()!; - let source: string; - try { - source = readFileSync(current, "utf8"); - } catch { - continue; - } - for (const edge of runtimeImportEdges(source)) { - if (edge.dynamic) continue; - const next = resolveSpec(edge.spec, current); - if (next === null) { - found.push({ specifier: edge.spec, chain: chainTo(previous, current) }); - continue; - } - if (previous.has(next)) continue; - previous.set(next, current); - queue.push(next); - } - } - return found; -} - -describe("the bunfig preload's module graph", () => { - test("never reaches bun:test", () => { - const offenders = bareSpecifiersReachableFrom(PRELOAD) - .filter(edge => edge.specifier === "bun:test") - .map(edge => edge.chain.join(" -> ")); - // Printed as the chain so a future violation names the hop that reintroduced it rather - // than only reporting that something, somewhere, did. - expect(offenders).toEqual([]); - }); - - test("the preload itself declares no bun:test import", () => { - const specifiers = runtimeImportEdges(readFileSync(PRELOAD, "utf8")).map(edge => edge.spec); - expect(specifiers).not.toContain("bun:test"); - }); - - test("the walk catches a bun:test edge introduced one hop away", () => { - // Attack the guard rather than trust it: without this, a walker that silently stopped at - // the first file would report the repository clean for every possible violation. - const directory = mkdtempSync(join(tmpdir(), "ocx-preload-graph-")); - try { - writeFileSync(join(directory, "entry.ts"), 'import { hook } from "./hop";\nexport { hook };\n'); - writeFileSync(join(directory, "hop.ts"), 'import { afterAll } from "bun:test";\nexport const hook = afterAll;\n'); - const offenders = bareSpecifiersReachableFrom(join(directory, "entry.ts")) - .filter(edge => edge.specifier === "bun:test"); - expect(offenders).toHaveLength(1); - expect(offenders[0]!.chain.map(path => path.split("/").at(-1))).toEqual(["entry.ts", "hop.ts"]); - } finally { - removeTreeWithRetry(directory); - } - }); - - test("the preload still reclaims its temp root through a process exit listener", () => { - // The crash is also curable by deleting the cleanup outright, which would hand #4762 its - // leak back. Pin the surviving mechanism so that repair cannot pass as this one. - const source = readFileSync(PRELOAD, "utf8"); - expect(source).toContain('process.on("exit", cleanupIsolatedRoot)'); - expect(source).toMatch(/isolated\.cleanup\(/); - }); -}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d0df9e9555a..f5c49f9959c 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1178,7 +1178,6 @@ "terminal-guard-server.test.ts": "server", "terminal-guard.test.ts": "server", "test-home-guard.test.ts": "ci-workflows", - "test-preload-module-graph.test.ts": "ci-workflows", "test-runner.test.ts": "ci-workflows", "thought-signature-credential-scope.test.ts": "responses", "token-estimate.test.ts": "lib", diff --git a/tests/preload.ts b/tests/preload.ts index df3e9d692ae..50e65eff16d 100644 --- a/tests/preload.ts +++ b/tests/preload.ts @@ -25,33 +25,8 @@ * `launchctl bootout gui//com.opencodex.proxy` reaches launchd — so neither cares * what HOME says. `assertLiveServiceManagerAllowed` in `src/service.ts` is the guard for * that, armed by the same flag set below. - * - * NOTHING IN THIS FILE'S IMPORT GRAPH MAY IMPORT `bun:test`. - * - * Bun re-loads this preload for EVERY test file: `--isolate` gives each file its own realm, - * and `VirtualMachine::reload_entry_point_for_test_runner` calls `load_preloads` again each - * time. Adding `import { afterAll } from "bun:test"` here (#4796, dc9d1fabc8) put a runtime - * built-in into that repeatedly re-entered graph, and on the 67th re-load Bun dereferenced a - * dead JSPromise inside its module loader and took SIGSEGV: - * - * panic(main thread): Segmentation fault at address 0x10 - * WTF::CompactPointerTuple::type - * JSC::JSPromise::status <- null cell - * JSC::JSPromise::performPromiseThenWithInternalMicrotask - * JSC::JSModuleLoader::loadModule - * JSC::moduleLoadTopSettled - * bun_runtime::jsc_hooks::load_preloads <- THIS FILE - * bun_jsc::virtual_machine::VirtualMachine::reload_entry_point_for_test_runner - * - * Windows shard 5/6 of dispatch runs 35087572377 and 35093667426 crashed there on both - * attempts, always at the file boundary and never inside a test; Linux shards took the same - * fault as exit 139 at five unrelated batch boundaries. Every file involved passes when it is - * the only file in its process — one preload load cannot reach the second re-load that faults. - * The immediately preceding dispatch run, 35054231781 at 8a1b01011d, ran the same files in the - * same order with no `bun:test` edge here and was green. - * - * `tests/ci-workflows/test-preload-module-graph.test.ts` is the guard that keeps the edge out. */ +import { afterAll } from "bun:test"; import { isTestHomeGuardArmed, protectedHomeForTests } from "../src/lib/test-home-guard"; import { createIsolatedTestEnvironment } from "../scripts/test"; import { @@ -141,26 +116,17 @@ if (process.platform === "win32" && lockPath && runLock.owner) { } // Clean up only the root this preload created. The `bun run test` wrapper owns its own. -// -// A process `exit` listener, and deliberately not a `bun:test` lifecycle hook — reaching for -// `afterAll` is what crashed the runtime, for the reason recorded above the import list. -// -// Because the preload is re-loaded per test file, each file registers a listener for the root -// it created, and they all fire at process exit. A listener runs synchronously, so this pass -// takes a short removal budget rather than the 15s `removeTestTempTree` spends on a foreground -// teardown: a Windows shard holds ~219 of these, and a contended exit paying the full budget -// each would outlive the job timeout it is supposed to finish inside. Roots this pass cannot -// release are not lost — `recoverStaleTestTempArtifactsOnce` reclaims owned roots on a later -// run, and that half of #4796 does not depend on any handler firing. -const EXIT_CLEANUP_BUDGET_MS = 250; +// Bun test workers do not reliably run process `exit` handlers, so the test lifecycle hook +// is primary; the process hook remains a best-effort fallback for setup failures. let cleanupComplete = false; const cleanupIsolatedRoot = () => { if (cleanupComplete) return; try { - isolated.cleanup({ budgetMs: EXIT_CLEANUP_BUDGET_MS }); + isolated.cleanup(); cleanupComplete = true; } catch { // The wrapper contains this root, and a later bare run reclaims it after the grace period. } }; +afterAll(cleanupIsolatedRoot); process.on("exit", cleanupIsolatedRoot); From 43734a0164a9c77f9d2689dca13922a8ab6d0259 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 22:35:50 +0900 Subject: [PATCH 3/7] fix(runtime): pin Bun back to 1.4.0 until the 1.4.2 test-runner crash is fixed Bun 1.4.2 segfaults while re-loading the bunfig preload. Under --isolate Bun re-enters tests/preload.ts once per test file, and the fault is in that path: load_preloads -> wait_for_promise -> drainMicrotasks -> moduleLoadTopSettled -> JSModuleLoader::loadModule -> JSPromise::performPromiseThenWithInternalMicrotask -> JSPromise::status, dereferencing a dead JSCell. Symbolicated from the bun.report link the Windows shard printed. It is a crash in the interpreter; no test content avoids it, and it needs at least two preload loads in one process. A/B, two Linux CI runs 40 minutes apart either side of #4064 (02bc10e8af), the commit that moved the pin to 1.4.2: 34274464811 6bd3274ab2 bun 1.4.0 0 panics / 4 shards 34278407438 02bc10e8af bun 1.4.2 12 panics / 4 shards Also clean on 1.4.0 at ff524ae635 (34273011879); crashing on 1.4.2 at 365e1ca51d, db8996df7e and every sampled tree from 2026-09-10 onward. Windows shard 5/6 failed on both attempts of runs 35087572377, 35093667426 and 35098735960, always at the 67th file. The file at that position changed from codex-prompt-layers.test.ts to codex-prompt-layers-read.test.ts between candidates, and moving codex-prompt-layers.test.ts into shard 1 left shard 1 green, so the crash is bound to the preload re-load count and not to any test file. Reverses only the version-bearing hunks of 02bc10e8af: package.json, bun.lock, Dockerfile, and the install-scripts source oracle that pins both package.json values. The unrelated workflow catch-up in that commit is left alone. 1.4.1 is untested. This pin is a claim about 1.4.0, which has direct CI evidence on seven trees, and about 1.4.2, which crashes on every tree sampled. It is not a claim that 1.4.1 is affected: no run in this repository ever used it. Whoever moves this line next should treat 1.4.1 as an open question worth checking, not as known-bad, and should prove any move with a green lane=all dispatch rather than an ordinary PR run, because the Linux batch sweep hides this class. --- Dockerfile | 2 +- bun.lock | 34 +++++++++++----------- package.json | 4 +-- tests/ci-workflows/install-scripts.test.ts | 23 +++++++++++++-- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8d3e72c6194..5987bfa991d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # Keep the runtime aligned with package.json and pin the multi-platform image index. -ARG BUN_IMAGE=oven/bun:1.4.2@sha256:9114c058aeae42162ee16dd5084b95fe9473970bb6bcb5b232ab1630f0546895 +ARG BUN_IMAGE=oven/bun:1.4.0@sha256:5ff609364c049b54eb0ff560ec96319729a972078ef2c755d758f0c6ef89c2d6 FROM ${BUN_IMAGE} AS build WORKDIR /home/bun/app diff --git a/bun.lock b/bun.lock index 6161766917a..c4619c28666 100644 --- a/bun.lock +++ b/bun.lock @@ -8,11 +8,11 @@ "@bufbuild/protobuf": "^2.14.0", "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/keyring": "1.3.0", - "bun": "1.4.2", + "bun": "1.4.0", "zod": "4.4.3", }, "devDependencies": { - "@types/bun": "1.4.2", + "@types/bun": "1.4.0", "typescript": "7.0.2", }, }, @@ -60,31 +60,31 @@ "@napi-rs/keyring-win32-x64-msvc": ["@napi-rs/keyring-win32-x64-msvc@1.3.0", "", { "os": "win32", "cpu": "x64" }, "sha512-4DnCWXwDc0HRKwyRlG5y0VhKZW2tNRQfKKfyj6IX/KWfDNyq9hn4n+GL1auyDcOO/v8PwnhmYo2+rOOqCkvvOg=="], - "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.4.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-MXdZkP1featqxZ+/VTXWG1BVjM4OGBehVY2Q88EeUj/7L0UMeCGItmyPYTN+wxvlGJ6F66JEtzsw+GvQWewnag=="], + "@oven/bun-darwin-aarch64": ["@oven/bun-darwin-aarch64@1.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-GCpf8QuFLsyioVawP5HrMxA1ZRBlu6Hq9RNnSc3UTUWAzIxBso9trjoZczw1HdgpqSssFkszfIV2zmOzFTjhkw=="], - "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.4.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-gZTxZuLjkUhAWjTETu3tw0WhsEdNkJ64daj60ybhPf835a2yollV3yTkK9JozvzKPx4TRFzLSl8C+U525pxVbw=="], + "@oven/bun-darwin-x64": ["@oven/bun-darwin-x64@1.4.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-cIrhwOr0SPEraewznhC+c/k6TG8bwFn5uZ4EJuXwjiKJLcAF36q7/bGjWkeXSe48JwMcPRUR054JXF7+cRwSSA=="], - "@oven/bun-freebsd-aarch64": ["@oven/bun-freebsd-aarch64@1.4.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-SMNItMw1Z8QeeQVKnw8jA7xQNkeXdP+OPgin4Wi/QTx/B8RHHLnuZfqmFy7NtVeT2NF0kKYppW4WWd2CCYZjhQ=="], + "@oven/bun-freebsd-aarch64": ["@oven/bun-freebsd-aarch64@1.4.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-09x7wnjMR6M5KGBDBhVl2CpfoCIQOkVDbPX2KfIhpXv4N6grbWE7dfLPw/Ydi9gaUMGhU7UKhoz444Nu6RCycA=="], - "@oven/bun-freebsd-x64": ["@oven/bun-freebsd-x64@1.4.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-THbPKXhO54N0DpFRKZNDZpQ7dpbX0bWASuARckAUS9wRtFIHsiY+uULXJvxJGo2YD1YewvXQ4G8Fj7XT5oBCiw=="], + "@oven/bun-freebsd-x64": ["@oven/bun-freebsd-x64@1.4.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-dRwzti/qJqV1HWplU27iUWUqp+f2DtFSf2yqQKSb+HH2dDOC//Uqd9u/A5h1DMsLszfP5OGP9UwQIKxVwFODaA=="], - "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-3BBP9ovJ2RGHFH6Ae1CAtxNtG1+YY6GD6rmYbsUosoAk9+OEl6zeDQ/k4fBkc6dYOJCtWnx8hUxzNzQATSmvYQ=="], + "@oven/bun-linux-aarch64": ["@oven/bun-linux-aarch64@1.4.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-Y5yAtCbHK6JjprXEtkdklDQFPADgs+CkfcliyY5g4JJ8baGHyQSrfpSkX3XVJ2C+aBLsdwNDdW+oczMsAwx6uA=="], - "@oven/bun-linux-aarch64-android": ["@oven/bun-linux-aarch64-android@1.4.2", "", { "os": "android", "cpu": "arm64" }, "sha512-3mZKO2rhsNgbAUtAHC1UKUlF2zTxFraDZT/Elv8wzyH0fJL9h+Iv3TgB9lO63w89PRn3eFe+NRA1bhVgikKNPQ=="], + "@oven/bun-linux-aarch64-android": ["@oven/bun-linux-aarch64-android@1.4.0", "", { "os": "android", "cpu": "arm64" }, "sha512-HpPIxJfDNPBPhiBNMyZoo/dOLijARfsx5j72vNuLtaTvl0Hh7HUculxjsOQ2WSyGoCgqXMEr1Qqjab1im9u1RA=="], - "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.4.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-+Sm6y+lSiSFBOtXmnekp5Q6n1tUKlyv71FCPWBc61Cgb14T5eBs8SN/nh4MUCOKzONkI3O+as3MGUgikS4aCBQ=="], + "@oven/bun-linux-aarch64-musl": ["@oven/bun-linux-aarch64-musl@1.4.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RUjAAkJ/CdNV++zVxyANWshPc73CECYsfhk0fWAkoJjtywxJ2BwXzI6nopBBDMfs0HS+fhRGn6zGwU8ccxLeJg=="], - "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-9/E/UXOTpSo3YsV5g+FhtTd/qTpiWoKuxS12cqtuYA1ssu9fRAoPQnipFgGyck3tWO63iUdxBiygq+kELFawng=="], + "@oven/bun-linux-x64": ["@oven/bun-linux-x64@1.4.0", "", { "os": "linux", "cpu": "x64" }, "sha512-Du44zebtPXJujvMLmtIxEQ6ykOhYt7L/Q+YIGVm+Yy+Pj/fpOnq60ggwIpKp/pGAFbYHNiTrA3JTjuZ9MTbZIg=="], - "@oven/bun-linux-x64-android": ["@oven/bun-linux-x64-android@1.4.2", "", { "os": "android", "cpu": "x64" }, "sha512-6HC5tzcC79113n2IHCTJMWv+HsQImv4ZFEK2XpYLxY6HbT8tM4cUM2Zv1bHZBQsS3jv/zYBamDJ1UX7If0d5tw=="], + "@oven/bun-linux-x64-android": ["@oven/bun-linux-x64-android@1.4.0", "", { "os": "android", "cpu": "x64" }, "sha512-u++KyLlfMn36yWz+AgJs+fZtS46UFDNpSSZhrcitkytONtNwq0X6Q9BDVEFXxYl/+Eec0xme1rb6MgW+U35WeA=="], - "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.4.2", "", { "os": "linux", "cpu": "x64" }, "sha512-vVTKUg1bnPhRP/Hp73jIVoFh2vPFNYEqYX0ERKfZBOQEEHitNAeukZzzuUDZS0SoDCIpuWUGSpd/CDMbjdR+Uw=="], + "@oven/bun-linux-x64-musl": ["@oven/bun-linux-x64-musl@1.4.0", "", { "os": "linux", "cpu": "x64" }, "sha512-C1Dv+ISL8YKEKM9jAHzNifOcRUoziy6UMxh+yVXjUCP6QnbRhENDHLaIWWkQZJyBLTn0I3xozflorAlHiGzGqA=="], - "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.4.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-8EJ1ST7339WJE3poPW5nBgVW/lWf9HBz4W27ZUNhburKmcBLOByPyE6DP9fHD8FQGm5c+ilUN2hX1mrW0jxq9Q=="], + "@oven/bun-windows-aarch64": ["@oven/bun-windows-aarch64@1.4.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-FBAYaQpJBP0asgqzL6NFUfjdQqsV+kvTpJ/eWxPKj+RcDgIfPSuE8kvQuPYu5pa8u8JTujYMjmuyvHxVuQsInA=="], - "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.4.2", "", { "os": "win32", "cpu": "x64" }, "sha512-+bN6OuVld/9diT/RLSXSW7JE6CvNE3gL9XsAEjULi1nUsXd6DNO6GuA9jNdNb3r8PdJFnYHr5aypNV1Oj3Rd9g=="], + "@oven/bun-windows-x64": ["@oven/bun-windows-x64@1.4.0", "", { "os": "win32", "cpu": "x64" }, "sha512-jRKv1NPLznMSZY5BEWciMF7zv0Tiyo2pQSxAJ3w+YWJ6y3VWNJQQQdLlV5Jx8lbOFDrJdrc9dD3GV17k3BP41A=="], - "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + "@types/bun": ["@types/bun@1.4.0", "", { "dependencies": { "bun-types": "1.4.0" } }, "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ=="], "@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], @@ -136,9 +136,9 @@ "body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="], - "bun": ["bun@1.4.2", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.4.2", "@oven/bun-darwin-x64": "1.4.2", "@oven/bun-freebsd-aarch64": "1.4.2", "@oven/bun-freebsd-x64": "1.4.2", "@oven/bun-linux-aarch64": "1.4.2", "@oven/bun-linux-aarch64-android": "1.4.2", "@oven/bun-linux-aarch64-musl": "1.4.2", "@oven/bun-linux-x64": "1.4.2", "@oven/bun-linux-x64-android": "1.4.2", "@oven/bun-linux-x64-musl": "1.4.2", "@oven/bun-windows-aarch64": "1.4.2", "@oven/bun-windows-x64": "1.4.2" }, "os": [ "!aix", "!sunos", "!openbsd", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-TrSXo6HJfIEaczpb3kjX82I2pL47vK1QUNmHRCUdz9IzaOwa9lzOXSWwu2l18YHE3sNfGRapVLd4nNm+22vVVA=="], + "bun": ["bun@1.4.0", "", { "optionalDependencies": { "@oven/bun-darwin-aarch64": "1.4.0", "@oven/bun-darwin-x64": "1.4.0", "@oven/bun-freebsd-aarch64": "1.4.0", "@oven/bun-freebsd-x64": "1.4.0", "@oven/bun-linux-aarch64": "1.4.0", "@oven/bun-linux-aarch64-android": "1.4.0", "@oven/bun-linux-aarch64-musl": "1.4.0", "@oven/bun-linux-x64": "1.4.0", "@oven/bun-linux-x64-android": "1.4.0", "@oven/bun-linux-x64-musl": "1.4.0", "@oven/bun-windows-aarch64": "1.4.0", "@oven/bun-windows-x64": "1.4.0" }, "os": [ "!aix", "!sunos", "!openbsd", ], "cpu": [ "x64", "arm64", ], "bin": { "bun": "bin/bun.exe", "bunx": "bin/bunx.exe" } }, "sha512-iRiFkc2W7UVpCyZXO9tod45TP9QCyN19fWqbpeN/jaM/K7uzeHYx/OSPsahMJazGKBgPsnxRt+4Jc43d8BcHZw=="], - "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], diff --git a/package.json b/package.json index a856df46861..22449e507ac 100644 --- a/package.json +++ b/package.json @@ -76,11 +76,11 @@ "@bufbuild/protobuf": "^2.14.0", "@modelcontextprotocol/sdk": "^1.30.0", "@napi-rs/keyring": "1.3.0", - "bun": "1.4.2", + "bun": "1.4.0", "zod": "4.4.3" }, "devDependencies": { - "@types/bun": "1.4.2", + "@types/bun": "1.4.0", "typescript": "7.0.2" }, "overrides": { diff --git a/tests/ci-workflows/install-scripts.test.ts b/tests/ci-workflows/install-scripts.test.ts index fc419501155..2d1436d4726 100644 --- a/tests/ci-workflows/install-scripts.test.ts +++ b/tests/ci-workflows/install-scripts.test.ts @@ -65,10 +65,29 @@ describe("install scripts", () => { expect(pkg.main).toBe("./bin/package-main.mjs"); expect(pkg.exports?.["."]?.bun).toBe("./src/index.ts"); expect(pkg.exports?.["."]?.default).toBe("./bin/package-main.mjs"); - expect(pkg.dependencies?.bun).toBe("1.4.2"); + // Bun stays at 1.4.0 until 1.4.2's test-runner crash is fixed upstream. 1.4.2 segfaults + // while RE-LOADING the bunfig preload that `--isolate` re-enters once per test file: + // `load_preloads -> JSModuleLoader::loadModule -> JSPromise::status` dereferences a dead + // promise and the process dies with "Segmentation fault at address 0x10". It is a crash in + // the interpreter, not a test result, and no test content avoids it. + // + // The A/B is two Linux CI runs 40 minutes apart, either side of #4064 (02bc10e8af), which + // is the commit that moved this line to 1.4.2: + // run 34274464811, 6bd3274ab2, Bun 1.4.0 -- 0 panics across all four shards + // run 34278407438, 02bc10e8af, Bun 1.4.2 -- 12 panics across all four shards + // Every sampled 1.4.0 tree is clean and every sampled 1.4.2 tree crashes 12-14 times per + // run. Linux hid it because `scripts/ci/run-bun-test-batches.sh` re-runs a crashed batch one + // file per process, and one preload load cannot reach the second that faults, so the sweep + // always "passes". Windows had no such sweep: shard 5/6 died on both attempts of runs + // 35087572377, 35093667426 and 35098735960, always at the 67th file, and the file sitting at + // that position changed between them. + // + // Moving this back to 1.4.2, or on to a later release, needs a green `lane=all` dispatch as + // the evidence -- an ordinary PR run cannot show it, because the Linux sweep masks it. + expect(pkg.dependencies?.bun).toBe("1.4.0"); expect(pkg.dependencies?.zod).toBe("4.4.3"); expect(pkg.devDependencies?.typescript).toBe("7.0.2"); - expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.2"); + expect(pkg.devDependencies?.["@types/bun"]).toBe("1.4.0"); expect(pkg.scripts?.dev).toBe("bun run src/cli/index.ts start"); expect(pkg.scripts?.["dev:proxy"]).toBe("bun run src/cli/index.ts start"); expect(pkg.scripts?.["dev:gui"]).toBe("cd gui && bun run dev"); From 5cff4711f85cf4976063f0e563d443f3f4cf7845 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 23:16:09 +0900 Subject: [PATCH 4/7] ci: fail on a Bun runtime crash instead of sweeping it, and share one classifier Linux CI had been segfaulting twelve to fourteen times per run since 2026-09-08 and reporting green every time. run-bun-test-batches.sh classified the crash, re-ran the batch one file per process, and returned success when that sweep passed. The sweep was never a retry of a flaky test: one file per process is a configuration in which this class of defect cannot occur, so it was guaranteed to pass and guaranteed to report nothing. That is why a Bun 1.4.2 regression reached a release candidate with only the Windows lane, which has no sweep, showing it. The sweep is kept, because proving every file passes alone is the useful half: it says the defect is in multi-file process state, not in a test. What changes is the verdict. A runtime crash now records its batch, finishes the shard so one run attributes every crash site rather than the first, and exits non-zero with an error annotation. A sweep that finds a genuinely failing file still reports that file immediately. A timeout is unchanged and may still recover, because a timeout is a load condition. The classifier itself was four copies: one in the batch runner and three inline in ci.yml. ci-workflows.test.ts pinned them in sync, which only detects the drift it was written to expect - #2152 had already broken one copy by anchoring on panic(thread 2852) when Bun also emits panic(main thread). They collapse into scripts/ci/bun-crash-signatures.sh, which every lane sources. The Windows and macOS legs gain the 132-139 status arm they never had, so a silent fatal signal with no banner is now caught there too. Exit 3 is covered through the signature arm rather than the status list. It is what Windows Bun returns alongside a SIGSEGV panic, but unlike 128+N it is an ordinary small exit code any process may return, and trusting it bare would reclassify a real failure as a crash and hide it - the same mistake in the opposite direction. The banner corroborates it. tests/ci-workflows/ci-bun-crash-classifier.test.ts owns both contracts. The macOS lane harness now copies the real classifier into its sandbox rather than a stand-in, and its signature cases exit 3 instead of 139 so the signature arm is what they actually exercise - with 139 the status arm matched first and those cases would have passed against an empty list. --- .github/workflows/ci.yml | 25 +-- scripts/ci/bun-crash-signatures.sh | 68 +++++++++ scripts/ci/run-bun-test-batches.sh | 63 ++++---- scripts/test-layout/layout.json | 1 + .../ci-bun-crash-classifier.test.ts | 142 ++++++++++++++++++ tests/ci-workflows/ci-workflows.test.ts | 35 +---- tests/ci-workflows/macos-serial-lanes.test.ts | 21 ++- tests/fixtures/test-layout-expected.json | 1 + 8 files changed, 286 insertions(+), 70 deletions(-) create mode 100755 scripts/ci/bun-crash-signatures.sh create mode 100644 tests/ci-workflows/ci-bun-crash-classifier.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 00abe5ec24d..ba7a41c7633 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -535,9 +535,10 @@ jobs: # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, # so the same crash that Linux absorbs failed the whole promotion here. # - # Keep the signature list in sync with `is_bun_runtime_crash` in - # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on - # the first attempt — only the crash signature is retried, exactly once. + # The classifier is `is_bun_runtime_crash` from scripts/ci/bun-crash-signatures.sh, which + # this leg, the Windows leg, the macOS control and the Linux batch runner all source. It used + # to be four inline copies kept in sync by a test; one definition cannot drift. An assertion + # failure still fails on the first attempt — only a crash is retried, exactly once. - name: Test env: MACOS_TEST_SHARD: ${{ matrix.shard }} @@ -546,6 +547,8 @@ jobs: # errexit so a Bun crash reaches PIPESTATUS and the bounded retry. set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh run_macos_suite() { local suite_log suite_status attempt @@ -559,7 +562,7 @@ jobs: rm -f "$suite_log" return 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." rm -f "$suite_log" return "$suite_status" @@ -685,15 +688,17 @@ jobs: # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, # so the same crash that Linux absorbs failed the whole promotion here. # - # Keep the signature list in sync with `is_bun_runtime_crash` in - # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on - # the first attempt — only the crash signature is retried, exactly once. + # The classifier is `is_bun_runtime_crash` from scripts/ci/bun-crash-signatures.sh, shared + # with every other lane. An assertion failure still fails on the first attempt — only a + # crash is retried, exactly once. - name: Test run: | # GitHub Actions starts bash `run:` blocks with `-e`. Disable # errexit so a Bun crash reaches PIPESTATUS and the bounded retry. set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)" for attempt in 1 2; do # --timeout: Bun's default 5s per-test ceiling is the recurring flake @@ -708,7 +713,7 @@ jobs: if [ "$suite_status" -eq 0 ]; then exit 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::macOS suite failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." exit "$suite_status" fi @@ -834,6 +839,8 @@ jobs: run: | set +e set -uo pipefail + # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. + source scripts/ci/bun-crash-signatures.sh suite_log="$(mktemp -t ocx-windows-suite.XXXXXX)" for attempt in 1 2; do bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/6 2>&1 | tee "$suite_log" @@ -841,7 +848,7 @@ jobs: if [ "$suite_status" -eq 0 ]; then exit 0 fi - if ! grep -Eqi 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' "$suite_log"; then + if ! is_bun_runtime_crash "$suite_status" "$suite_log"; then echo "::error::Windows shard ${{ matrix.shard }}/6 failed on attempt ${attempt} (exit ${suite_status}); assertion failures are not retried." exit "$suite_status" fi diff --git a/scripts/ci/bun-crash-signatures.sh b/scripts/ci/bun-crash-signatures.sh new file mode 100755 index 00000000000..476f6539c6f --- /dev/null +++ b/scripts/ci/bun-crash-signatures.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# The single definition of "this was a Bun runtime crash, not a test result". +# +# There were four copies of this list: one in the Linux batch runner and three inline in ci.yml +# (platform-windows, platform-macos, macos-control). ci-workflows.test.ts pinned them in sync +# rather than removing the duplication, because #2152 had already broken one copy by anchoring on +# `panic(thread 2852)` when Bun also emits `panic(main thread)` for the same class, and half the +# crashes stopped matching. Pinning four copies in sync only detects the drift it was written to +# expect; one definition cannot drift at all. +# +# Source it from the repository root: source scripts/ci/bun-crash-signatures.sh + +# shellcheck shell=bash + +# Sourced by nested shells in the same job, so a second source must be a no-op rather than a +# readonly-reassignment error. +if [[ -n "${OCX_BUN_CRASH_SIGNATURES_LOADED:-}" ]]; then + return 0 +fi +OCX_BUN_CRASH_SIGNATURES_LOADED=1 + +# Never anchor on the thread-numbered form. `Internal assertion failure` is the stable fingerprint +# recorded in devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md. +OCX_BUN_CRASH_SIGNATURE_PATTERN='oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' + +# True when the process output carries a Bun panic banner. +bun_log_has_crash_signature() { + grep -Eqi "$OCX_BUN_CRASH_SIGNATURE_PATTERN" "$1" +} + +# True for a status that can only be a fatal signal. +# +# 128+N for SIGILL/SIGABRT/SIGBUS/SIGKILL/SIGSEGV and neighbours. Windows exit 3 is deliberately +# NOT in this list: it is the status a Windows Bun returns alongside a SIGSEGV panic, but unlike +# 132-139 it is an ordinary small exit code any process may return for its own reasons. Trusting +# it bare would reclassify a real test failure as a crash and hide it, which is the exact mistake +# this file exists to stop. Exit 3 is still covered, through the signature arm below, which is +# corroborated by the panic banner Bun actually printed -- that is how the Windows shard 5/6 +# crashes of runs 35087572377, 35093667426 and 35098735960 are recognised. +bun_status_is_crash_code() { + case "$1" in + 132|133|134|135|136|137|139) return 0 ;; + esac + return 1 +} + +# The shared predicate: is_bun_runtime_crash +is_bun_runtime_crash() { + local status="$1" + local log_file="$2" + + if bun_status_is_crash_code "$status"; then + return 0 + fi + + # Bun 1.3.14 can surface a Linux epoll registration failure as exit 1, even though the failure + # comes from Bun's internal WriteStream setup rather than a test assertion. Treat only that + # narrow runtime signature as a crash. + if [[ "$status" == "1" ]] \ + && grep -Fq '# Unhandled error between tests' "$log_file" \ + && grep -Fq 'error: EEXIST: file already exists, epoll_ctl' "$log_file" \ + && grep -Fq 'at new WriteStream (internal:fs/streams:' "$log_file"; then + return 0 + fi + + bun_log_has_crash_signature "$log_file" +} + diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index ae34310258f..147c87c9804 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -11,6 +11,10 @@ readonly BATCH_KILL_GRACE_SECONDS="${BUN_TEST_BATCH_KILL_GRACE_SECONDS:-15}" # bundled stable runtime anyway, and report a qualification it never performed. readonly BUN_BIN="${OPENCODEX_BUN_PATH:-bun}" +# One definition of the crash classifier, shared with the Windows and macOS legs in ci.yml. +# shellcheck source=scripts/ci/bun-crash-signatures.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/bun-crash-signatures.sh" + usage() { echo "usage: $0 " >&2 exit 64 @@ -64,31 +68,6 @@ is_general_test_file() { esac } -is_bun_runtime_crash() { - local status="$1" - local log_file="$2" - - case "$status" in - 132|133|134|135|136|137|139) - return 0 - ;; - esac - - # Bun 1.3.14 can surface a Linux epoll registration failure as exit 1, - # even though the failure comes from Bun's internal WriteStream setup rather - # than a test assertion. Treat only that narrow runtime signature as a crash. - if (( status == 1 )) \ - && grep -Fq '# Unhandled error between tests' "$log_file" \ - && grep -Fq 'error: EEXIST: file already exists, epoll_ctl' "$log_file" \ - && grep -Fq 'at new WriteStream (internal:fs/streams:' "$log_file"; then - return 0 - fi - - grep -Eqi \ - 'oh no: Bun has crashed|Internal assertion failure|Segmentation fault at address|Illegal instruction|Bus error|Aborted \(core dumped\)' \ - "$log_file" -} - LAST_FAILURE_KIND="" run_test_once() { @@ -189,7 +168,14 @@ recover_batch_file_by_file() { return "$status" done - echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed under singleton isolation after the original ${batch_failure_kind}; continuing." + if [[ "$batch_failure_kind" == "runtime" ]]; then + # Not "recovered". One file per process is a configuration in which this class of defect + # cannot occur, so the sweep was always going to pass and always going to report nothing. + # What it does prove is that the files themselves are sound, which is the half worth keeping. + echo "::error::Shard ${SHARD_SPEC} batch ${batch_number} crashed the Bun runtime. Every file in it then passed alone, so the defect is in multi-file process state, not in any test." + else + echo "::warning::Shard ${SHARD_SPEC} batch ${batch_number} passed under singleton isolation after the original ${batch_failure_kind}; continuing." + fi return 0 } @@ -218,7 +204,12 @@ fi readonly TOTAL_BATCHES=$(( (${#SELECTED_FILES[@]} + BATCH_SIZE - 1) / BATCH_SIZE )) echo "Shard ${SHARD_SPEC}: ${#SELECTED_FILES[@]} files in ${TOTAL_BATCHES} primary Bun processes (batch size <= ${BATCH_SIZE}, timeout ${BATCH_TIMEOUT_SECONDS}s)." -echo "Runtime crashes and timeouts fall back to one-file-per-process isolation; assertion/test failures do not retry." +echo "Timeouts fall back to one-file-per-process isolation and may recover; assertion/test failures do not retry." +echo "A Bun runtime crash is swept one-file-per-process for attribution and then FAILS this shard: it is a defect in the interpreter, and a green report would be a lie." + +# Every batch that crashed the runtime, so one run attributes all of them instead of only the +# first. Linux was producing twelve to fourteen of these per run while reporting success. +CRASHED_BATCHES=() for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do start=$(( batch_index * BATCH_SIZE )) @@ -237,8 +228,22 @@ for ((batch_index = 0; batch_index < TOTAL_BATCHES; batch_index += 1)); do failure_kind="$LAST_FAILURE_KIND" if recover_batch_file_by_file "$batch_number" "$failure_kind" "${batch[@]}"; then - continue + recovery_status=0 else - exit $? + recovery_status=$? + fi + + if [[ "$failure_kind" == "runtime" ]]; then + CRASHED_BATCHES+=("$batch_number") + fi + + # A sweep that found a real failing file still reports that file, and immediately. + if (( recovery_status != 0 )); then + exit "$recovery_status" fi done + +if (( ${#CRASHED_BATCHES[@]} > 0 )); then + echo "::error::Shard ${SHARD_SPEC} crashed the Bun runtime in batch(es): ${CRASHED_BATCHES[*]}. Each batch was re-run one file per process and every file passed, so no test is at fault -- the interpreter is. Failing rather than reporting green." + exit 1 +fi diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2106a9614be..96e44b509cb 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -319,6 +319,7 @@ "chatgpt-oauth.test.ts": "oauth", "chatgpt-token-expiry.test.ts": "oauth", "chutes-provider.test.ts": "providers", + "ci-bun-crash-classifier.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", diff --git a/tests/ci-workflows/ci-bun-crash-classifier.test.ts b/tests/ci-workflows/ci-bun-crash-classifier.test.ts new file mode 100644 index 00000000000..cacdb6cf165 --- /dev/null +++ b/tests/ci-workflows/ci-bun-crash-classifier.test.ts @@ -0,0 +1,142 @@ +/** + * The Bun crash classifier is one definition, every lane sources it, and a crash fails the shard. + * + * Two separate defects are pinned here. + * + * The first is duplication. The signature list used to exist four times: in the Linux batch + * runner and inline in three ci.yml legs. #2152 broke one copy by anchoring on + * `panic(thread 2852)` when Bun also emits `panic(main thread)` for the same class, so half the + * crashes stopped matching in that lane alone. ci-workflows.test.ts responded by pinning the four + * copies in sync, which only ever detects the drift it was written to expect. One definition + * cannot drift, so the contract is now that the copies do not exist. + * + * The second is masking, and it is the one that cost a release. `run-bun-test-batches.sh` + * classified exit 139 as a runtime crash, re-ran the batch one file per process, and reported + * success when that sweep passed. The sweep is not a retry of a flaky test: one file per process + * is a configuration in which this class of defect cannot occur, so it was guaranteed to pass and + * guaranteed to report nothing. Linux CI segfaulted twelve to fourteen times per run from + * 2026-09-08 while reporting green, and the Windows lane -- which has no sweep -- was the only + * place the Bun 1.4.2 regression was visible at all. The sweep is kept for attribution; the shard + * now fails regardless of its result. + */ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { repoPath } from "../helpers/repo-root"; + +const CLASSIFIER_REL = "scripts/ci/bun-crash-signatures.sh"; +const SOURCE_LINE = `source ${CLASSIFIER_REL}`; + +/** Every signature that means "the interpreter died", not "a test failed". */ +const CRASH_SIGNATURES = [ + "oh no: Bun has crashed", + "Internal assertion failure", + "Segmentation fault at address", + "Illegal instruction", + "Bus error", +] as const; + +function read(...segments: string[]): string { + return readFileSync(repoPath(...segments), "utf8"); +} + +/** The literal text of one `run:` block, located by a line only that block contains. */ +function runBlockContaining(workflow: string, anchor: string): string { + const index = workflow.indexOf(anchor); + expect(`anchor present: ${anchor}`).toBe(`anchor present: ${anchor}`); + expect(index).toBeGreaterThan(-1); + const start = workflow.lastIndexOf("run: |", index); + expect(start).toBeGreaterThan(-1); + const end = workflow.indexOf("\n - name:", index); + return workflow.slice(start, end === -1 ? undefined : end); +} + +describe("the Bun crash classifier is shared", () => { + const classifier = read("scripts", "ci", "bun-crash-signatures.sh"); + const batchScript = read("scripts", "ci", "run-bun-test-batches.sh"); + const workflow = read(".github", "workflows", "ci.yml"); + + const lanes = { + windows: runBlockContaining(workflow, "bun test --isolate --timeout 60000 tests --shard=${{ matrix.shard }}/6"), + "macos-shard": runBlockContaining(workflow, "run_macos_suite tests"), + "macos-control": runBlockContaining(workflow, "bun test --isolate --timeout 60000 tests 2>&1"), + }; + + test("the signatures exist in the classifier", () => { + for (const signature of CRASH_SIGNATURES) { + expect(`classifier:${signature}:${classifier.includes(signature)}`).toBe(`classifier:${signature}:true`); + } + }); + + test("no lane and no script carries an inline copy of them", () => { + const others: Array = [ + ["batch-script", batchScript], + ...Object.entries(lanes), + ]; + for (const [name, text] of others) { + for (const signature of CRASH_SIGNATURES) { + expect(`${name}:inline:${signature}:${text.includes(signature)}`) + .toBe(`${name}:inline:${signature}:false`); + } + } + }); + + test("every lane sources the classifier and calls the shared predicate", () => { + for (const [name, text] of Object.entries(lanes)) { + expect(`${name}:sources:${text.includes(SOURCE_LINE)}`).toBe(`${name}:sources:true`); + expect(`${name}:calls:${text.includes("is_bun_runtime_crash \"$suite_status\" \"$suite_log\"")}`) + .toBe(`${name}:calls:true`); + } + expect(batchScript).toContain("bun-crash-signatures.sh"); + expect(batchScript).toContain('is_bun_runtime_crash "$status" "$log_file"'); + }); + + test("the thread-numbered panic form is the anchor nowhere", () => { + // `panic(thread 2852)` and `panic(main thread)` are the same class (#2152). + for (const [name, text] of [["classifier", classifier], ["batch-script", batchScript], ...Object.entries(lanes)] as Array) { + expect(`${name}:${text.includes("panic\\(thread")}`).toBe(`${name}:false`); + } + }); + + test("fatal signal codes classify on the status alone, and exit 3 never does", () => { + // 128+N is unambiguous. 3 is an ordinary small exit code any process may return, so it is + // recognised only when Bun also printed a panic banner -- which is how the Windows shard 5/6 + // crashes of runs 35087572377, 35093667426 and 35098735960 are caught. Trusting 3 bare would + // reclassify a real failure as a crash and hide it, which is the mistake this file prevents. + expect(classifier).toContain("132|133|134|135|136|137|139) return 0 ;;"); + expect(classifier).not.toMatch(/^\s*3\|/m); + expect(classifier).not.toContain("|3)"); + }); +}); + +describe("a Bun runtime crash fails the Linux shard", () => { + const batchScript = read("scripts", "ci", "run-bun-test-batches.sh"); + + test("crashed batches are collected and the shard exits non-zero", () => { + expect(batchScript).toContain("CRASHED_BATCHES=()"); + expect(batchScript).toContain('CRASHED_BATCHES+=("$batch_number")'); + expect(batchScript).toContain("if (( ${#CRASHED_BATCHES[@]} > 0 )); then"); + // The failure is an error annotation and a non-zero exit, not a warning and a green shard. + const tail = batchScript.slice(batchScript.indexOf("if (( ${#CRASHED_BATCHES[@]} > 0 )); then")); + expect(tail).toContain("::error::"); + expect(tail).toContain("exit 1"); + }); + + test("the singleton sweep reports a crash as an error rather than a recovery", () => { + expect(batchScript).toContain('if [[ "$batch_failure_kind" == "runtime" ]]; then'); + // The old wording promised recovery. A crash may not be announced that way again. + const sweepEnd = batchScript.slice(batchScript.indexOf("recover_batch_file_by_file")); + expect(sweepEnd).not.toContain("passed under singleton isolation after the original runtime"); + }); + + test("a real failing file found by the sweep still reports that file immediately", () => { + // Failing on the crash must not swallow an assertion the sweep genuinely attributed. + expect(batchScript).toContain("Singleton isolation identified ${file} as a failing test file."); + expect(batchScript).toContain("if (( recovery_status != 0 )); then"); + }); + + test("a timeout may still recover, because a timeout is a load condition", () => { + expect(batchScript).toContain('if [[ "$LAST_FAILURE_KIND" != "runtime" && "$LAST_FAILURE_KIND" != "timeout" ]]; then'); + expect(batchScript).toContain("passed under singleton isolation after the original ${batch_failure_kind}; continuing."); + }); +}); + diff --git a/tests/ci-workflows/ci-workflows.test.ts b/tests/ci-workflows/ci-workflows.test.ts index b098220521e..45763262634 100644 --- a/tests/ci-workflows/ci-workflows.test.ts +++ b/tests/ci-workflows/ci-workflows.test.ts @@ -271,8 +271,8 @@ describe("GitHub Actions hardening", () => { // must disable errexit before the crash-prone command or exit 133 aborts // the step before PIPESTATUS can be inspected and the retry can run. expect(hasExactShellCommand(macosTestRun, "set +e")).toBe(true); - expect(macosTestRun).toContain("Segmentation fault at address"); - expect(macosTestRun).toContain("oh no: Bun has crashed"); + // The crash signatures themselves moved to scripts/ci/bun-crash-signatures.sh; that one + // definition and every lane that sources it are pinned by ci-bun-crash-classifier.test.ts. expect(macosTestRun).toContain("assertion failures are not retried"); expect(macosTestRun).toContain("failing after one retry"); // `for attempt in 1 2` — one retry, never an unbounded loop. @@ -355,35 +355,10 @@ describe("GitHub Actions hardening", () => { expect(winSteps.some(step => step.if === "runner.environment == 'self-hosted'" && step.run?.includes("git clean -xffd"))).toBe(true); - // The three crash-signature lists must stay identical, and they must not key on - // `panic(thread`. - // - // Bun emits BOTH `panic(thread 2852)` and `panic(main thread)` for the same class of - // failure, so a grep anchored on the numbered form silently misses half of them and the - // shard fails on a crash it was supposed to retry. This repository already learned that - // once — `devlog/_fin/260731_pr_issue_triage_round/050_windows_ci_flake_rca.md` names - // `Internal assertion failure` as the stable fingerprint — and #2152 reintroduced it. - // Three copies of one list is the real hazard, so pin the sync rather than the text. - const crashSignatures = [ - "oh no: Bun has crashed", - "Internal assertion failure", - "Segmentation fault at address", - "Illegal instruction", - "Bus error", - ]; + // The crash-signature list lives in exactly one file now, and every lane sources it. + // ci-bun-crash-classifier.test.ts owns that contract, including the rule that no lane may + // reintroduce an inline copy and that a runtime crash fails the shard instead of being swept. const windowsTestRun = windowsTestSteps[0]?.run ?? ""; - const batchScript = await readText("scripts/ci/run-bun-test-batches.sh"); - for (const signature of crashSignatures) { - expect(`macos:${signature}:${macosTestRun.includes(signature)}`).toBe(`macos:${signature}:true`); - expect(`macos-control:${signature}:${macosControlTestRun.includes(signature)}`).toBe(`macos-control:${signature}:true`); - expect(`windows:${signature}:${windowsTestRun.includes(signature)}`).toBe(`windows:${signature}:true`); - expect(`script:${signature}:${batchScript.includes(signature)}`).toBe(`script:${signature}:true`); - } - // The thread-numbered form must not be the anchor anywhere. - expect(macosTestRun).not.toContain("panic\\(thread"); - expect(macosControlTestRun).not.toContain("panic\\(thread"); - expect(windowsTestRun).not.toContain("panic\\(thread"); - expect(batchScript).not.toContain("panic\\(thread"); // Windows carries the same bounded retry as macOS: one attempt, crash-only. expect(hasExactShellCommand(windowsTestRun, "set +e")).toBe(true); diff --git a/tests/ci-workflows/macos-serial-lanes.test.ts b/tests/ci-workflows/macos-serial-lanes.test.ts index c9677cddf4d..b290617f8b4 100644 --- a/tests/ci-workflows/macos-serial-lanes.test.ts +++ b/tests/ci-workflows/macos-serial-lanes.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { spawn, type ChildProcessByStdio } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { basename, dirname, join } from "node:path"; import type { Readable } from "node:stream"; @@ -19,6 +19,9 @@ const SERIAL_FILES = [ const GENERAL_FILES = ["general/ordinary.test.ts", "general/falcon-extra.test.ts"]; const ASSERTION_STATUS = 23; const CRASH_STATUS = 139; +// A status the classifier cannot recognise on its own, so a crash carrying it is only detected +// through the panic banner. 139 is a fatal signal and matches on the code alone. +const SIGNATURE_ONLY_CRASH_STATUS = 3; const CRASH_SIGNATURES = [ "oh no: Bun has crashed", "Internal assertion failure", @@ -98,6 +101,13 @@ process.exit(0); function createFixture(directory: string, options: FixtureOptions): void { mkdirSync(join(directory, "bin")); mkdirSync(join(directory, "tmp")); + // The lane sources its crash classifier from the working directory, so the sandbox gets the + // REAL file rather than a stand-in. That is deliberate: the harness executes the actual run + // block, so a copy here would let the block and the classifier drift apart unnoticed, which is + // the exact failure mode that collapsing four inline signature lists into one file removed. + mkdirSync(join(directory, "scripts", "ci"), { recursive: true }); + copyFileSync(repoPath("scripts", "ci", "bun-crash-signatures.sh"), + join(directory, "scripts", "ci", "bun-crash-signatures.sh")); for (const file of [...SERIAL_FILES, ...GENERAL_FILES]) { if (file === options.missing) continue; mkdirSync(dirname(join(directory, "tests", file)), { recursive: true }); @@ -304,7 +314,14 @@ describe.skipIf(process.platform === "win32")("macOS serial lane shell ownership for (const [caseIndex, signature] of CRASH_SIGNATURES.entries()) { test(`${target}: retries one runtime crash (case ${caseIndex + 1}), then finishes`, async () => { - const run = await runShard(1, { target, outcomes: ["crash"], crashSignature: signature }); + // Exit 3, not 139, so the SIGNATURE arm of the shared classifier is what is under test. + // With 139 the status arm matches first and this case would pass even if the signature + // list were empty -- which is how a lane can carry a broken list and look covered (#2152). + // Exit 3 is also the real Windows shape: Bun prints the panic banner and returns 3, + // and a bare 3 must NOT be treated as a crash, so the banner is doing the work here. + const run = await runShard(1, { + target, outcomes: ["crash"], crashSignature: signature, crashStatus: SIGNATURE_ONLY_CRASH_STATUS, + }); expect(run.status, run.output).toBe(0); const calls = testCalls(run); const attempts = calls.filter(call => targets(call, target)); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index f5c49f9959c..c1f826b816f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -151,6 +151,7 @@ "chatgpt-oauth.test.ts": "oauth", "chatgpt-token-expiry.test.ts": "oauth", "chutes-provider.test.ts": "providers", + "ci-bun-crash-classifier.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", From 0660ce45727e4b79ad30a7eaaf256fcbcf480b19 Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 23:21:31 +0900 Subject: [PATCH 5/7] fix(test): stop charging Windows cold-start and a credential race to two assertions Two unrelated Windows failures, both of which blocked a green lane=all dispatch just as hard as the Bun crash did. client-injection-guard: the first child-spawning scenario in the file paid a one-time cost and was billed for it. Run 35098735960 spent 30.446s in "deny" and hit the 30s deadline, then ran its siblings in 17.038s and 1.3-2.0s; run 35093667426 paid 3.814s first against 1.6-2.7s later. The deadline is a guard-behaviour budget, so first-touch transpilation, SQLite creation and Windows ACL work do not belong inside it. A beforeAll now walks the real path once and throws the result away. It swallows its own errors on purpose: the hook asserts nothing, and letting it throw would turn one broken scenario into seven failures pointing at a warm-up. server-search: the exact-search 429 case did not receive a 429 at all. It received a local 401, which the fake upstream cannot produce - it only ever returns 429 - so the request never reached the fixture. Credential resolution failed first. The case was starting a whole server between writing the fixture credential store and reading it, widening the window in which the process-wide OPENCODEX_HOME can move underneath it, and that startup is unrelated to the routing contract being asserted. It now calls handleSearch directly, which is what every other case in this file already does. Account selection, the public selector, cooldown state and the no-failover assertion are unchanged; the request-log lookup becomes a direct read of the log context the handler fills. Neither is a raised limit. The 30s guard deadline and the 45s composed watchdog both stay exactly where they were. The "fixture upstream connection reset" and the compact JSON parse error in the same job belong to server-auth.test.ts and a malformed-body case that both passed; they are log noise, not this failure. --- .../client-injection-guard.test.ts | 31 ++++++++++++++++--- tests/server/server-search.test.ts | 25 ++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/tests/codex-integration/client-injection-guard.test.ts b/tests/codex-integration/client-injection-guard.test.ts index 829db8a5a00..262ede77b0e 100644 --- a/tests/codex-integration/client-injection-guard.test.ts +++ b/tests/codex-integration/client-injection-guard.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, beforeAll, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync, realpathSync } from "node:fs"; import { tmpdir } from "node:os"; import { delimiter, dirname, join } from "node:path"; @@ -134,7 +134,7 @@ async function within(promise: Promise, milliseconds: number): Promise })]); } finally { if (timer !== undefined) clearTimeout(timer); } } -async function runScenario(mode: string): Promise> { +async function runScenario(mode: string, childDeadlineMs = 30_000): Promise> { const root = mkdtempSync(join(tmpdir(), "ocx-client-guard-")); roots.push(root); const codex = join(root, "codex"); const ocx = join(root, "ocx"); const desktop = join(root, "desktop"); for (const directory of [codex, ocx, desktop]) mkdirSync(directory, { recursive: true }); @@ -171,14 +171,14 @@ async function runScenario(mode: string): Promise> { children.push(child); const stdout = new Response(child.stdout).text(); const stderr = new Response(child.stderr).text(); - const code = await within(child.exited, 30_000); + const code = await within(child.exited, childDeadlineMs); const output = await within(Promise.all([stdout, stderr]), 5_000); const result = JSON.parse(output[0].trim().split("\n").at(-1) ?? "{}"); if (code !== 0) throw new Error("injection fixture failed: " + String(result.fatal ?? output[1])); return result; } -afterEach(async () => { +async function cleanupFixtures(): Promise { const errors: unknown[] = []; for (const child of children.splice(0)) { try { @@ -193,7 +193,28 @@ afterEach(async () => { try { removeTreeWithRetry(root); } catch (error) { errors.push(error); } } if (errors.length) throw new AggregateError(errors, "injection fixture cleanup failed"); -}, 30_000); +} + +beforeAll(async () => { + try { + // Windows run 35098735960 spent 30.446s in the first child, then 17.038s and + // 1.3-2.0s in its siblings. Run 35093667426 likewise paid 3.814s first versus + // 1.6-2.7s later. Exercise the real write/SQLite/ACL path once before a scenario's + // 30s execution budget so Bun and Windows first-touch work is not timed + // as guard behavior. The 45s warm-up ceiling is the file's existing test budget. + await runScenario("deny", 45_000); + } catch { + // Deliberately swallowed. This hook exists only to pay first-touch cost; it asserts + // nothing. Letting it throw would convert one broken scenario into seven failures whose + // messages all point at a warm-up rather than at the guard, and the real "deny" test below + // reproduces any genuine breakage with its own assertions. A warm-up that merely ran out of + // its own budget on a loaded runner must not fail a file it was added to stabilise. + } finally { + await cleanupFixtures(); + } +}, 55_000); + +afterEach(cleanupFixtures, 30_000); for (const mode of ["deny", "queued-native", "legacy", "external", "malformed", "async", "async-reject"]) { test("client commit guard preserves every routing artifact (" + mode + ")", async () => { diff --git a/tests/server/server-search.test.ts b/tests/server/server-search.test.ts index 721ca155593..667979427ca 100644 --- a/tests/server/server-search.test.ts +++ b/tests/server/server-search.test.ts @@ -315,16 +315,21 @@ test("an account-qualified search model uses that exact account and sends the ba test("an exact search 429 never switches to the active Pool account and reports only its public selector", async () => { const captured: CapturedRequest[] = []; const upstream = fakeSearchUpstream(captured, 429, { error: { message: "rate limited" } }); - saveConfig(exactSearchConfig()); + const config = exactSearchConfig(); + saveConfig(config); saveExactSearchCredentials(); - const server = startServer(0); try { - const requestExactSearch = () => fetch(new URL("/v1/alpha/search", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ id: "search-session", model: "side/gpt-test" }), - }); + // Run 35093667426 returned a local 401 before this fixture could return its 429. + // Full server startup is unrelated to this routing contract and widens the interval + // between writing and reading the credential store selected by process-wide + // OPENCODEX_HOME. Call the handler while that fixture home is current. + const logCtx = { model: "", provider: "" }; + const requestExactSearch = () => handleSearch( + alphaSearchRequest({ id: "search-session", model: "side/gpt-test" }), + config, + logCtx, + ); const first = await requestExactSearch(); expect(first.status).toBe(429); @@ -342,11 +347,9 @@ test("an exact search 429 never switches to the active Pool account and reports expect(captured).toHaveLength(1); expect(loadConfig().activeCodexAccountId).toBe("pool-b"); expect(getCodexUpstreamHealth("pool-b")).toBeNull(); - const entry = getRequestLogEntries().findLast(candidate => candidate.model === "side/gpt-test"); - expect(entry?.provider).toBe("openai-side"); - expect(JSON.stringify(entry)).not.toContain("pool-a"); + expect(logCtx.provider).toBe("openai-side"); + expect(JSON.stringify(logCtx)).not.toContain("pool-a"); } finally { - await server.stop(true); await upstream.stop(true); } }); From d0a79d6c964befd1d6b1aae90f82255b5298b13b Mon Sep 17 00:00:00 2001 From: JUN Date: Wed, 16 Sep 2026 23:23:12 +0900 Subject: [PATCH 6/7] fix(server): stop an explicitly OFF Codex startup from taking the catalog lock Windows shard 4/6 of run 35093667426 failed the WP13 A-reduced composed acceptance case after 47,774 ms with "timed out waiting for runtime-port record; child exit=null" and both child pipes still open. Healthy controls ran the same case in 13,962 ms and 13,718 ms, so the child spent 33.8 s somewhere between spawn and publishing runtime-port.json. That number has an owner. Startup cache invalidation took catalog lock K before it checked desired state, and K resolves the effective SID and LocalAppData through PowerShell, each on a 30,000 ms budget (src/codex/user-identity.ts, src/lib/windows-user-principal.ts). Results are cached per process only after they succeed, so a cold resolution that runs out its budget under six concurrent shards is paid in full. The case that failed is the one where Codex is explicitly OFF, which is precisely the case that had no reason to take K at all. An explicit OFF now short-circuits before the lock. The permit callback still re-reads intent under K, so a disable racing an enabled startup is unchanged, and enabled startup behaviour is untouched. Absence still means ON, so only an explicit false takes the new path. The composed fixture gains the assertion that an OFF startup creates no catalog-lock database, which is red against the previous code because the database was opened before the callback could look at intent. It also gains incremental stdout/stderr capture and PID/runtime-record diagnostics: the old failure printed "", which proved only that EOF had not arrived and not that the child had said nothing. If another pre-record path stalls, the next log distinguishes pre-PID, post-PID, malformed-record and post-runtime health failures instead of reporting one bare timeout. The 45 s watchdog is unchanged, and src/server/index.ts stays exactly at its 893-line ratchet. --- src/server/index.ts | 10 +- structure/runtime.md | 3 + .../codex-composed-acceptance.test.ts | 95 +++++++++++++++---- 3 files changed, 86 insertions(+), 22 deletions(-) diff --git a/src/server/index.ts b/src/server/index.ts index 4892f4aaf35..6cd079fd717 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -268,11 +268,11 @@ export function startServer(port?: number, deps: StartServerDeps = {}): Server; }; +type CapturedChildStream = { + completed: Promise; + snapshot: () => string; + closed: () => boolean; +}; + +/** Drain a child pipe while retaining the bytes already emitted before EOF. */ +function captureChildStream(stream: ReadableStream): CapturedChildStream { + let text = ""; + let closed = false; + const completed = (async () => { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + text += decoder.decode(chunk.value, { stream: true }); + } + text += decoder.decode(); + return text; + } finally { + closed = true; + reader.releaseLock(); + } + })(); + return { completed, snapshot: () => text, closed: () => closed }; +} + /** A byte manifest: paths plus bytes, not mtimes or parsed JSON. */ function manifest(root: string): Record { const entries: Record = {}; @@ -130,6 +160,8 @@ class Fixture { readonly managementToken = "composed-admin-token"; readonly lockPath: string; readonly lockAllowlist: string[]; + readonly catalogLockPath: string; + readonly catalogLockAllowlist: string[]; readonly serviceManagerEnv: Record; readonly serviceManagerPreloadPath: string | undefined; readonly powerShellCacheEnv: Record = {}; @@ -168,9 +200,18 @@ class Fixture { rmSync(this.root, { recursive: true, force: true, maxRetries: 3, retryDelay: 50 }); throw error; } - this.lockPath = resolveCodexCoordinatorDatabasePath(resolveEffectiveUserIdentity(), realpathSync.native(this.codex)); + const identity = resolveEffectiveUserIdentity(); + const canonicalCodexHome = realpathSync.native(this.codex); + this.lockPath = resolveCodexCoordinatorDatabasePath(identity, canonicalCodexHome); this.lockAllowlist = [this.lockPath, `${this.lockPath}-journal`, `${this.lockPath}-wal`, `${this.lockPath}-shm`]; - for (const path of this.lockAllowlist) { + this.catalogLockPath = resolveCodexCatalogSerializationDatabasePath(identity, canonicalCodexHome); + this.catalogLockAllowlist = [ + this.catalogLockPath, + `${this.catalogLockPath}-journal`, + `${this.catalogLockPath}-wal`, + `${this.catalogLockPath}-shm`, + ]; + for (const path of [...this.lockAllowlist, ...this.catalogLockAllowlist]) { if (existsSync(path)) throw new Error(`lock preflight found pre-existing case path: ${path}`); } writeFileSync(join(this.codex, "config.toml"), 'model = "gpt-5"\n'); @@ -260,23 +301,35 @@ class Fixture { async start(): Promise { const child = this.spawnCli(["start"]); + const pidPath = join(this.ocx, "ocx.pid"); const runtimePath = join(this.ocx, "runtime-port.json"); - // Capture the child's streams while we wait. Without this, a start that dies for a - // concrete reason — a throw, a port bind refusal, a missing artifact — surfaces only as - // "timed out waiting for runtime-port record", which is the symptom and never the cause. - // That is exactly how the Windows failures read for two CI rounds. - const stderr = new Response(child.stderr).text(); - const stdout = new Response(child.stdout).text(); + // Run 35093667426 waited the full 45 s Windows watchdog with the child alive, but + // Response(stream).text() reported only "still open": it cannot reveal bytes until EOF. + // Healthy controls in 35054231781 and 35098735960 finished this whole case in ~14 s, so + // preserve the budget and expose the child's actual progress plus its two startup records. + const stderr = captureChildStream(child.stderr); + const stdout = captureChildStream(child.stdout); const diagnose = async (label: string): Promise => { const exited = child.exitCode ?? (await Promise.race([ child.exited, new Promise(resolve => setTimeout(() => resolve(null), 500)), ])); - const [err, out] = await Promise.all([ - Promise.race([stderr, new Promise(resolve => setTimeout(() => resolve(""), 500))]), - Promise.race([stdout, new Promise(resolve => setTimeout(() => resolve(""), 500))]), - ]); - throw new Error(`${label}; child exit=${String(exited)}\n--- stderr ---\n${err.slice(-4000)}\n--- stdout ---\n${out.slice(-2000)}`); + let pidRecord = existsSync(pidPath) ? "present(unreadable)" : "missing"; + try { pidRecord = `present(${readFileSync(pidPath, "utf8").trim()})`; } catch { /* diagnostic only */ } + let runtimeRecord = existsSync(runtimePath) ? "present(unreadable)" : "missing"; + try { + const record = JSON.parse(readFileSync(runtimePath, "utf8")) as Partial; + runtimeRecord = `present(pid=${String(record.pid)}, port=${String(record.port)}, matches-child=${record.pid === child.pid})`; + } catch { /* diagnostic only; never print the record's attestation secret */ } + const streamText = (capture: CapturedChildStream, limit: number) => { + const value = capture.snapshot().slice(-limit); + return value || `<${capture.closed() ? "closed" : "open"}; no output captured>`; + }; + throw new Error( + `${label}; child exit=${String(exited)}; pid-record=${pidRecord}; runtime-record=${runtimeRecord}` + + `\n--- stderr (${stderr.closed() ? "closed" : "open"}) ---\n${streamText(stderr, 4000)}` + + `\n--- stdout (${stdout.closed() ? "closed" : "open"}) ---\n${streamText(stdout, 2000)}`, + ); }; const runtime = await waitFor(() => { if (!existsSync(runtimePath)) return null; @@ -297,9 +350,9 @@ class Fixture { } catch { return null; } - }, "child /healthz"); + }, "child /healthz").catch(() => diagnose("timed out waiting for child /healthz")); expect(health).toMatchObject({ pid: child.pid, port: runtime.port }); - return { process: child, runtime, stdout, stderr }; + return { process: child, runtime, stdout: stdout.completed, stderr: stderr.completed }; } async stop(server: StartedServer): Promise { @@ -368,9 +421,13 @@ class Fixture { } // Re-resolve before the limited four-name removal: never glob or inspect a // shared runtime namespace beyond the exact identities this case created. - const checked = resolveCodexCoordinatorDatabasePath(resolveEffectiveUserIdentity(), realpathSync.native(this.codex)); + const identity = resolveEffectiveUserIdentity(); + const canonicalCodexHome = realpathSync.native(this.codex); + const checked = resolveCodexCoordinatorDatabasePath(identity, canonicalCodexHome); if (checked !== this.lockPath) throw new Error("lock teardown identity changed"); - for (const path of this.lockAllowlist) { + const checkedCatalog = resolveCodexCatalogSerializationDatabasePath(identity, canonicalCodexHome); + if (checkedCatalog !== this.catalogLockPath) throw new Error("catalog lock teardown identity changed"); + for (const path of [...this.lockAllowlist, ...this.catalogLockAllowlist]) { if (existsSync(path)) unlinkSync(path); } removeTreeWithRetry(this.root); @@ -449,6 +506,10 @@ describe("WP13 composed toggle acceptance", () => { const before = manifest(fx.codex); const server = await fx.start(); try { + // OFF must short-circuit before K. On Windows, merely resolving K starts separate + // SID and LocalAppData PowerShell children with 30 s budgets each; run 35093667426 + // exceeded healthy controls by 33.8 s before the runtime-port watchdog fired at 45 s. + expect(existsSync(fx.catalogLockPath)).toBe(false); expect(manifest(fx.codex)).toEqual(before); for (const argv of [["ensure"], ["restore"]]) { const result = await fx.runCli(argv); From aa6205f50dd08c153129361c7cae82f9c1dea103 Mon Sep 17 00:00:00 2001 From: JUN Date: Thu, 17 Sep 2026 00:02:35 +0900 Subject: [PATCH 7/7] fix(test): wait for the icacls child before removing the management-auth home Windows shard 1/6 of run 35108652486 threw out of this file's afterEach: error: EPERM: operation not permitted, rm ...\Temp\opencodex-test-WXhR6c\tmp\ocx-management-auth-fDchUb at removeTestTempTree (scripts/test-temp.ts:198) at removeTreeWithRetry (tests/helpers/remove-tree.ts:27) at tests/server/server-management-auth.test.ts:206 It is a leaked handle, not a timing race, and the budget was never the answer. hardenConfigDir() spawns icacls.exe, which holds the directory open until it exits, and Windows file locking is mandatory. #4789 filed this exact failure at this exact line when the retry budget was 2.5s; #4796 raised it to a 15s exponential schedule, and this run exhausted all 15 seconds and still got EPERM. Six times the wait changed nothing, because the handle does not close on the remover's schedule. The process that started the child has to wait for it. src/config/paths.ts already documents this and already ships the wait. The afterEach now awaits flushConfigDirHardeningForTests() first. Twenty other test files do the same; this one starts servers and drives the ACL path hardest and did not. The all-directories variant is the required one. server.stop already flushes, but through flushConfigDirHardening(), which defaults to getConfigDir() read at stop time - and this hook restores OPENCODEX_HOME to the real home a few lines before the removal, so a directory-scoped flush would settle the wrong tree and leave this one held. Three things deliberately unchanged. The 15s budget stays: a wider retry would be treating the symptom, and this run proves it does not work anyway. The fake icacls runners this file installs are all synchronous, so awaiting the flush cannot hang on one. And removeTestTempTree keeps throwing rather than reporting and continuing - that throw is what surfaced the leak, and softening it would re-hide exactly the class of defect the batch-runner change stopped hiding this morning. 79 other test files start a server, remove a tree in a cleanup hook, and never flush. None has failed this way, because this is the file that drives icacls deliberately, so they are left alone rather than edited blind. The general guard belongs in the removal helper and is a separate change. --- tests/server/server-management-auth.test.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/server/server-management-auth.test.ts b/tests/server/server-management-auth.test.ts index 2e5f3db0616..0443c3329c3 100644 --- a/tests/server/server-management-auth.test.ts +++ b/tests/server/server-management-auth.test.ts @@ -7,6 +7,7 @@ import { mkdtempSync, readFileSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { getConfigPath, saveConfig } from "../../src/config"; +import { flushConfigDirHardeningForTests } from "../../src/config/paths"; import { clearContextSessionOwnersForTests } from "../../src/codex/context-owner"; import { resetContextRelayActivationForTests } from "../../src/codex/context-compat"; import { startServer } from "../../src/server"; @@ -189,7 +190,23 @@ beforeEach(() => { process.env.OPENCODEX_ADMIN_AUTH_TOKEN = "admin-secret"; }); -afterEach(() => { +afterEach(async () => { + // Settle every in-flight config-directory harden before anything here removes a directory. + // + // `hardenConfigDir()` spawns `icacls.exe`, which holds the directory open until it exits, and + // Windows file locking is mandatory: removing that tree while the child lives returns EPERM no + // matter how long the caller waits. Windows shard 1/6 of run 35108652486 proved the waiting is + // not the answer -- it exhausted the full 15s exponential budget and still threw + // `EPERM: operation not permitted, rm .../tmp/ocx-management-auth-fDchUb` out of this hook. + // #4789 filed the same failure at this same line when the budget was 2.5s, and raising it to + // 15s in #4796 bought six times the wait and changed nothing, because the handle was never + // going to close on its own schedule. The process that started the child has to wait for it. + // + // The all-directories variant is the required one. `server.stop` already flushes, but through + // `flushConfigDirHardening()`, which defaults to `getConfigDir()` read at stop time -- and this + // hook moves OPENCODEX_HOME back to the developer's real home a few lines below, so a + // directory-scoped flush here would settle the wrong tree and leave this one held. + await flushConfigDirHardeningForTests(); resetContextRelayActivationForTests(); if (previousCodexHome === undefined) delete process.env.CODEX_HOME; else process.env.CODEX_HOME = previousCodexHome;