From a9a073dc3c653434051f47739aa48cda23591f21 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 30 Oct 2026 14:24:41 +0900 Subject: [PATCH 1/2] fix(ci): distinguish install failures from owned deadlines --- scripts/ci/bun-install-frozen.sh | 192 +++++++++++--- .../ci-bun-install-frozen.test.ts | 249 +++++++++++++----- 2 files changed, 349 insertions(+), 92 deletions(-) diff --git a/scripts/ci/bun-install-frozen.sh b/scripts/ci/bun-install-frozen.sh index 0cc559d30da..d9e8d9ef3a3 100755 --- a/scripts/ci/bun-install-frozen.sh +++ b/scripts/ci/bun-install-frozen.sh @@ -1,15 +1,71 @@ #!/usr/bin/env bash -# Frozen-lockfile install with a per-attempt deadline and one total budget. -# -# A registry or runner stall can leave `bun install` printing only its banner until the job's -# timeout cancels the run, which reads as a failed check unrelated to the change under test. -# Each attempt therefore runs under a process deadline, and only an attempt that the deadline -# killed is retried. Any other failure, such as a lockfile that no longer matches, fails at once. -# All directories share one total budget, so retries end well inside the shortest job timeout -# that calls this (5 minutes) and leave time for setup, the GUI build and the tests. -# -# Usage: bash scripts/ci/bun-install-frozen.sh [dir ...] (default: the current directory) -set -euo pipefail +# Frozen installs: one witnessed deadline, a shared budget, and no PID-directed cleanup. +# GNU timeout/gtimeout bounds a fixed reader, never the install's own exit status. +# Bootstrap FIFO names disappear before install code runs; this is not a sandbox against +# arbitrary same-user process inspection. Cleanup covers descendants that remain in the group. +set -eu + +own_group() { + local stat state ppid group rest + if [[ -r /proc/$$/stat ]]; then + IFS= read -r stat < /proc/$$/stat || return 1 + read -r state ppid group rest <<< "${stat##*) }" + else + read -r group <<< "$(ps -o pgid= -p "$$")" || return 1 + fi + [[ "$group" == "$$" ]] +} + +keeper() { + local boot="$1" timer="$2" grace="$3" dir="$4" deadline token completion rc + shift 4 + set +m + own_group || exit 70 # No signal 0 until this process is its group leader. + trap ':' TERM + # O_RDWR bootstrap avoids a blocked open if the other endpoint dies during setup. + exec 6<>"$boot/outcome" 8>"$boot/outcome" + exec 6>&- + exec 6<>"$boot/lifeline" 9<"$boot/lifeline" + exec 6>&- + printf 'ready %s\n' "$$" >&8 + IFS=' ' read -r -t 5 token deadline <&9 || exit 70 + [[ "$token" == go && "$deadline" =~ ^[1-9][0-9]*$ ]] || exit 70 + + # Only the parent owns a lifeline writer. Its death/cancellation is EOF, never a PID kill. + ( exec 8>&-; trap '' TERM; cat <&9 >/dev/null; kill -TERM 0; sleep 0.2; kill -KILL 0 ) >/dev/null 2>&1 & + exec 9<&- # Only the EOF helper needs the read endpoint now. + if ! cd "$dir"; then printf 'fail 66\n' >&8; kill -KILL 0; fi + exec 7>&1 + report_install() { + local status=0 + exec 8>&- + # The command gets neither a control descriptor nor the result pipe's writer. + "$@" >&7 7>&- || status=$? + printf '%s\n' "$status" + } + rc=0 + # The fixed reader exits only 0/70; install 124/137 are pipe data, not timer statuses. + # Process substitution does not wait for a stalled reporter when the reader times out. + completion=$("$timer" --foreground "${deadline}s" bash -c ' + IFS= read -r status || exit 70 + [[ "$status" =~ ^(0|[1-9][0-9]{0,2})$ && $status -le 255 ]] || exit 70 + printf "%s" "$status" + ' < <(report_install "$@") 7>&- 8>&-) || rc=$? + if [[ $rc -eq 0 ]]; then + printf 'result %s\n' "$completion" >&8 + elif [[ $rc -eq 124 ]]; then + printf 'deadline 124\n' >&8 + kill -TERM 0 + sleep "$grace" + else + printf 'fail 70\n' >&8 + fi + # This live process pins the group until its last action, including ordinary success. + kill -KILL 0 + exit 70 +} + +if [[ "${1:-}" == --ocx-install-keeper ]]; then shift; keeper "$@"; exit 70; fi readonly ATTEMPT_SECONDS="${BUN_INSTALL_ATTEMPT_SECONDS:-45}" readonly TOTAL_SECONDS="${BUN_INSTALL_TOTAL_SECONDS:-150}" @@ -17,42 +73,112 @@ readonly MIN_ATTEMPT_SECONDS="${BUN_INSTALL_MIN_ATTEMPT_SECONDS:-5}" readonly KILL_GRACE_SECONDS="${BUN_INSTALL_KILL_GRACE_SECONDS:-5}" readonly MAX_ATTEMPTS="${BUN_INSTALL_MAX_ATTEMPTS:-3}" readonly BUN_INSTALL_BIN="${BUN_INSTALL_BIN:-bun}" +readonly SCRIPT_PATH="${BASH_SOURCE[0]}" +for value in "$ATTEMPT_SECONDS" "$TOTAL_SECONDS" "$MIN_ATTEMPT_SECONDS" "$KILL_GRACE_SECONDS" "$MAX_ATTEMPTS"; do + [[ "$value" =~ ^[1-9][0-9]*$ && ${#value} -le 6 ]] || { echo '::error::Invalid install deadline configuration.'; exit 70; } +done +TIMEOUT_BIN="" +for candidate in timeout gtimeout; do + if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --signal=TERM --kill-after=1s 1s true >/dev/null 2>&1; then + TIMEOUT_BIN=$(command -v "$candidate"); break + fi +done +[[ -n "$TIMEOUT_BIN" ]] || { echo '::error::GNU timeout is required to bound bun install.' >&2; exit 69; } -if ! command -v timeout >/dev/null 2>&1 || ! timeout --signal=TERM --kill-after=1s 1s true >/dev/null 2>&1; then - echo "::error::GNU timeout is required to bound bun install." >&2 - exit 69 -fi +ATTEMPT_VERDICT=fail +ATTEMPT_STATUS=70 +run_attempt() { + local dir="$1" deadline="$2" boot keeper_pid keeper_status=70 tag="" value="" extra="" + local cancel_status=0 interrupted=no drained=no remaining started=no monitor=no + shift 2 + boot=$(mktemp -d "${TMPDIR:-/tmp}/ocx-bun-install.XXXXXX") || return 70 + mkfifo "$boot/outcome" "$boot/lifeline" || return 70 + exec 8<>"$boot/outcome" 9<>"$boot/lifeline" + trap 'cancel_status=143; interrupted=yes; exec 9>&-' TERM + trap 'cancel_status=130; interrupted=yes; exec 9>&-' INT + trap 'cancel_status=129; interrupted=yes; exec 9>&-' HUP + [[ $- == *m* ]] && monitor=yes + set -m + bash "$SCRIPT_PATH" --ocx-install-keeper "$boot" "$TIMEOUT_BIN" "$KILL_GRACE_SECONDS" "$dir" "$@" 8>&- 9>&- & + keeper_pid=$! + [[ "$monitor" == yes ]] || set +m + ATTEMPT_VERDICT=fail; ATTEMPT_STATUS=70 + if IFS=' ' read -r -t 5 tag value extra <&8 && [[ "$tag" == ready && "$value" == "$keeper_pid" && -z "$extra" ]]; then + # After the handshake, unlink every IPC name before permitting any install code. + if rm "$boot/outcome" "$boot/lifeline" 8>&- 9>&- && rmdir "$boot" 8>&- 9>&-; then + remaining=$(( TOTAL_SECONDS - SECONDS - KILL_GRACE_SECONDS - 2 )) + (( remaining < deadline )) && deadline=$remaining + if (( deadline >= MIN_ATTEMPT_SECONDS && cancel_status == 0 )); then + if printf 'go %s\n' "$deadline" >&9; then + started=yes + tag=""; value=""; extra="" + if IFS=' ' read -r -t "$(( deadline + KILL_GRACE_SECONDS ))" tag value extra <&8 && [[ -z "$extra" ]]; then + case "$tag:$value" in + result:*) + if [[ "$value" =~ ^(0|[1-9][0-9]{0,2})$ && $value -le 255 ]]; then + ATTEMPT_STATUS=$value + [[ $value -eq 0 ]] && ATTEMPT_VERDICT=ok + fi ;; + deadline:124) ATTEMPT_VERDICT=deadline; ATTEMPT_STATUS=124 ;; + fail:66) ATTEMPT_STATUS=66 ;; + esac + fi + fi + else + ATTEMPT_VERDICT=budget; ATTEMPT_STATUS=124 + fi + fi + fi + # A valid outcome keeps the lifeline open through the keeper's own complete grace cycle. + if [[ "$ATTEMPT_VERDICT" == fail && $ATTEMPT_STATUS -eq 70 ]] || [[ "$started" == no ]]; then exec 9>&-; fi + while :; do + interrupted=no + keeper_status=0; wait "$keeper_pid" 2>/dev/null || keeper_status=$? + [[ "$interrupted" == yes ]] || break + done + # EOF is not a drain proof. Reuse can cause a conservative failure, never a foreign kill. + for value in 1 2 3 4 5 6 7 8 9 10; do + if ! kill -0 -- "-$keeper_pid" 2>/dev/null; then drained=yes; break; fi + sleep 0.1 8>&- 9>&- + done + exec 9>&- 8>&- + trap - TERM INT HUP + if [[ -d "$boot" ]]; then + for value in outcome lifeline; do [[ ! -e "$boot/$value" ]] || rm "$boot/$value"; done + rmdir "$boot" + fi + if (( cancel_status > 0 )); then + ATTEMPT_VERDICT=cancel; ATTEMPT_STATUS=$cancel_status + elif [[ "$drained" != yes ]]; then + ATTEMPT_VERDICT=drain_fail; ATTEMPT_STATUS=70 + elif [[ "$started" == yes && $keeper_status -ne 137 ]]; then + ATTEMPT_VERDICT=fail; ATTEMPT_STATUS=70 + fi +} install_dir() { - local dir="$1" attempt status remaining deadline - # Never empty: macOS ships bash 3.2, where expanding an empty array under `set -u` aborts - # with "unbound variable". + local dir="$1" attempt deadline remaining + # Keep this array nonempty for Bash 3.2 with nounset. local -a install_cmd=("$BUN_INSTALL_BIN" install --frozen-lockfile) for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do - # The kill grace counts against the budget too, so the worst case stays at TOTAL_SECONDS. - remaining=$(( TOTAL_SECONDS - SECONDS - KILL_GRACE_SECONDS )) + remaining=$(( TOTAL_SECONDS - SECONDS - KILL_GRACE_SECONDS - 2 )) if (( remaining < MIN_ATTEMPT_SECONDS )); then echo "::error::bun install --frozen-lockfile in ${dir} has no time left in its ${TOTAL_SECONDS}s budget." return 124 fi deadline=$(( remaining < ATTEMPT_SECONDS ? remaining : ATTEMPT_SECONDS )) - status=0 - (cd "$dir" && timeout --signal=TERM --kill-after="${KILL_GRACE_SECONDS}s" "${deadline}s" \ - "${install_cmd[@]}") || status=$? - if [[ $status -eq 0 ]]; then return 0; fi - # 124: the deadline fired and the install exited on TERM; 137: it needed the KILL. - if [[ $status -ne 124 && $status -ne 137 ]]; then - echo "::error::bun install --frozen-lockfile failed in ${dir} (exit ${status}); not retrying." - return "$status" - fi - echo "::warning::bun install --frozen-lockfile in ${dir} exceeded ${deadline}s (attempt ${attempt}/${MAX_ATTEMPTS})." - # A killed install may have extracted bun but not finished its trusted postinstall. - # Plain install can then report success while leaving bin/bun.exe a placeholder. + run_attempt "$dir" "$deadline" "${install_cmd[@]}" + case "$ATTEMPT_VERDICT" in + ok) return 0 ;; + cancel) return "$ATTEMPT_STATUS" ;; + budget) echo "::error::bun install --frozen-lockfile in ${dir} has no time left in its ${TOTAL_SECONDS}s budget."; return 124 ;; + deadline) echo "::warning::bun install --frozen-lockfile in ${dir} exceeded ${deadline}s (attempt ${attempt}/${MAX_ATTEMPTS})." ;; + *) echo "::error::bun install --frozen-lockfile failed in ${dir} (exit ${ATTEMPT_STATUS}, ${ATTEMPT_VERDICT}); not retrying."; return "$ATTEMPT_STATUS" ;; + esac install_cmd=("$BUN_INSTALL_BIN" install --frozen-lockfile --force) done echo "::error::bun install --frozen-lockfile in ${dir} exceeded its deadline ${MAX_ATTEMPTS} times." return 124 } - if [[ $# -eq 0 ]]; then set -- .; fi for dir in "$@"; do install_dir "$dir"; done diff --git a/tests/ci-workflows/ci-bun-install-frozen.test.ts b/tests/ci-workflows/ci-bun-install-frozen.test.ts index bec3dd7292f..76e64fe30a5 100644 --- a/tests/ci-workflows/ci-bun-install-frozen.test.ts +++ b/tests/ci-workflows/ci-bun-install-frozen.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn, spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { repoPath } from "../helpers/repo-root"; @@ -39,53 +39,212 @@ describe("CI frozen installs are bounded", () => { }); }); -// The script targets the Linux CI runners and needs GNU timeout. +// These process-group assertions need POSIX; a Windows skip is not Linux coverage. describe.skipIf(process.platform === "win32")("bun-install-frozen.sh retry policy", () => { const roots: string[] = []; - afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + const groups = (root: string) => existsSync(join(root, "groups")) + ? [...new Set(readFileSync(join(root, "groups"), "utf8").trim().split(/\s+/).filter((s) => /^[1-9]\d*$/.test(s)))] : []; + const drained = (root: string) => spawnSync("bash", ["-c", + 'for group; do if kill -0 -- "-$group" 2>/dev/null; then exit 1; fi; done', + "_", ...groups(root)], { timeout: 2_000 }).status === 0; + afterEach(() => { + for (const root of roots.splice(0)) { + // Never delete a fixture against a live attempt, or infer drain merely from pipe EOF. + if (!drained(root)) throw new Error("Attempt group remains; fixture retained: " + root); + rmSync(root, { recursive: true }); + } + }); - /** A fake bun whose Nth invocation behaves as listed: "hang", "ok" or an exit code. */ - function run(plan: string[], options: { dirs?: number; env?: Record } = {}) { - const root = mkdtempSync(join(tmpdir(), "ocx-bun-install-")); + function fixture(plan: string[], env: Record = {}) { + const root = mkdtempSync(join(tmpdir(), "ocx-bun-install-case-")); roots.push(root); - const count = join(root, "count"), fake = join(root, "fake-bun"); + mkdirSync(join(root, "tmp")); writeFileSync(join(root, "plan"), plan.join("\n") + "\n"); + const fake = join(root, "fake-bun"); writeFileSync(fake, [ "#!/usr/bin/env bash", - `n=$(( $(cat "${count}" 2>/dev/null || echo 0) + 1 )); echo "$n" > "${count}"`, - `step=$(sed -n "${"$"}{n}p" "${join(root, "plan")}")`, + 'if [[ -r /proc/$$/stat ]]; then', + ' IFS= read -r stat < /proc/$$/stat; rest="${stat##*) }"; read -r state parent group rest <<< "$rest"', + 'else read -r group <<< "$(ps -o pgid= -p "$$")"; fi', + 'printf "%s\\n" "$group" >> "$TEST_ROOT/groups"', + 'n=$(( $(cat "$TEST_ROOT/count" 2>/dev/null || echo 0) + 1 )); echo "$n" > "$TEST_ROOT/count"', + 'step=$(sed -n "$' + '{n}p" "$TEST_ROOT/plan")', 'echo "fake bun $* in $PWD"', - 'case "$step" in hang) exec sleep 30 ;; ok) exit 0 ;; *) exit "$step" ;; esac', + 'case "$step" in', + " hang) exec sleep 30 ;;", + " hang-ignore) trap '' TERM; exec sleep 30 ;;", + " term-marker) trap 'echo TERM > \"$TEST_ROOT/termed\"; exit 0' TERM; sleep 30 ;;", + " sigkill) kill -KILL $$ ;;", + " spawn-ignore) ( trap '' TERM; exec sleep 30 ) & exec sleep 30 ;;", + " spawn-leak) ( exec sleep 30 >/dev/null 2>&1 ) & exit 0 ;;", + ' forge-fd) for fd in 6 7 8 9; do if ( printf "result 0\\n" >&"$fd" ) 2>/dev/null; then echo "$fd" > "$TEST_ROOT/leaked"; exit 99; fi; done; exit 42 ;;', + ' metadata) mkdir "$TMPDIR/ocx-bun-install.forged"; mkdir "$TMPDIR/ocx-bun-install.forged/decision"; echo 0 > "$TMPDIR/ocx-bun-install.forged/result"; echo "deadline 0" > "$TMPDIR/ocx-bun-install.forged/witness"; exit 42 ;;', + ' kill-keeper) kill -KILL "$group"; exec sleep 30 ;;', + " ok) exit 0 ;;", + ' *) exit "$step" ;;', + "esac", ].join("\n")); chmodSync(fake, 0o755); - const dirs = Array.from({ length: options.dirs ?? 1 }, () => root); + return { + root, count: join(root, "count"), + env: { ...process.env, TEST_ROOT: root, TMPDIR: join(root, "tmp"), + BUN_INSTALL_BIN: fake, BUN_INSTALL_ATTEMPT_SECONDS: "1", + BUN_INSTALL_KILL_GRACE_SECONDS: "1", BUN_INSTALL_MIN_ATTEMPT_SECONDS: "1", ...env }, + }; + } + + function run(plan: string[], options: { dirs?: number; env?: Record } = {}) { + const f = fixture(plan, options.env); + const dirs = Array.from({ length: options.dirs ?? 1 }, (_, n) => { + const dir = join(f.root, "dir-" + n); mkdirSync(dir); return dir; + }); const started = Date.now(); - const result = spawnSync("bash", [script, ...dirs], { - encoding: "utf8", timeout: 30_000, - env: { ...process.env, BUN_INSTALL_BIN: fake, BUN_INSTALL_ATTEMPT_SECONDS: "1", BUN_INSTALL_KILL_GRACE_SECONDS: "1", - BUN_INSTALL_MIN_ATTEMPT_SECONDS: "1", ...options.env }, + const child = spawnSync("bash", [script, ...dirs], { + encoding: "utf8", timeout: 15_000, env: f.env, }); - return { status: result.status, out: result.stdout + result.stderr, elapsedMs: Date.now() - started, - calls: existsSync(count) ? Number(readFileSync(count, "utf8")) : 0 }; + expect(child.error, child.stdout + child.stderr).toBeUndefined(); + expect(drained(f.root), "owned group remains").toBe(true); + return { status: child.status, out: child.stdout + child.stderr, root: f.root, + elapsedMs: Date.now() - started, + calls: existsSync(f.count) ? Number(readFileSync(f.count, "utf8")) : 0 }; } - test("an attempt killed by its deadline is retried and the next success passes", () => { - const result = run(["hang", "ok"]); - expect(result.status, result.out).toBe(0); - expect(result.calls).toBe(2); - expect(result.out).toContain("::warning::bun install --frozen-lockfile"); - expect(result.out).toContain("fake bun install --frozen-lockfile --force"); - expect(result.out.split("fake bun ")[1]).toStartWith("install --frozen-lockfile in "); + test.each(["1", "124", "137", "sigkill"])("ordinary install failure %s never retries", (step) => { + const r = run([step, "ok"]); + expect(r.status, r.out).toBe(step === "sigkill" ? 137 : Number(step)); + expect(r.calls).toBe(1); + expect(r.out).toContain("not retrying"); + expect(r.out).not.toContain("::warning::"); }); - test("a genuine install failure after a timeout still fails without another retry", () => { - const result = run(["hang", "42", "ok"]); - expect(result.status, result.out).toBe(42); - expect(result.calls).toBe(2); - expect(result.out).toContain("::error::bun install --frozen-lockfile failed"); - expect(result.out).toContain("not retrying"); + test.each(["hang", "hang-ignore", "spawn-ignore"])("deadline retries %s after its full grace", (step) => { + const r = run([step, "ok"], { env: { BUN_INSTALL_KILL_GRACE_SECONDS: "2" } }); + expect(r.status, r.out).toBe(0); + expect(r.calls).toBe(2); + expect(r.elapsedMs).toBeGreaterThanOrEqual(2_800); + expect(r.elapsedMs).toBeLessThan(12_000); + expect(r.out).toContain("::warning::"); + expect(r.out).toContain("fake bun install --frozen-lockfile --force"); + expect(r.out.split("fake bun ")[1]).toStartWith("install --frozen-lockfile in "); + }, 15_000); + + test("TERM remains catchable and its full grace elapses before completion", () => { + const r = run(["term-marker"], { env: { + BUN_INSTALL_MAX_ATTEMPTS: "1", BUN_INSTALL_KILL_GRACE_SECONDS: "2", + } }); + expect(r.status, r.out).toBe(124); + expect(r.calls).toBe(1); + const marker = join(r.root, "termed"); + expect(existsSync(marker)).toBe(true); + expect(Date.now() - statSync(marker).mtimeMs).toBeGreaterThanOrEqual(1_800); + }, 10_000); + + test("a failure after a witnessed deadline stops further retries", () => { + const r = run(["hang", "42", "ok"]); + expect(r.status, r.out).toBe(42); + expect(r.calls).toBe(2); + expect(r.out).toContain("not retrying"); + }, 10_000); + + test("normal success drains descendants even when they closed output", () => { + const r = run(["spawn-leak"]); + expect(r.status, r.out).toBe(0); + expect(r.calls).toBe(1); + expect(r.elapsedMs).toBeLessThan(5_000); + }); + + test("the install inherits no result or lifeline descriptor", () => { + const r = run(["forge-fd", "ok"]); + expect(r.status, r.out).toBe(42); + expect(r.calls).toBe(1); + expect(existsSync(join(r.root, "leaked"))).toBe(false); }); + test("writable decision/result/witness files have no classification authority", () => { + const r = run(["metadata", "ok"]); + expect(r.status, r.out).toBe(42); + expect(r.calls).toBe(1); + expect(r.out).not.toContain("::warning::"); + expect(readFileSync(join(r.root, "tmp", "ocx-bun-install.forged", "result"), "utf8")).toBe("0\n"); + }); + + test("keeper death fails closed and leaves no attempt group", () => { + const r = run(["kill-keeper", "ok"]); + expect(r.status, r.out).toBe(70); + expect(r.calls).toBe(1); + expect(r.elapsedMs).toBeLessThan(8_000); + expect(r.out).not.toContain("::warning::"); + }, 10_000); + + test("three witnessed deadlines stop at max attempts", () => { + const r = run(["hang", "hang", "hang", "ok"]); + expect(r.status, r.out).toBe(124); + expect(r.calls).toBe(3); + expect(r.out).toContain("exceeded its deadline 3 times"); + }, 15_000); + + test("successful root and stalled GUI share the total budget, including cleanup", () => { + const r = run(["ok", "hang", "hang", "hang"], { dirs: 2, env: { + BUN_INSTALL_ATTEMPT_SECONDS: "3", BUN_INSTALL_TOTAL_SECONDS: "9", + } }); + expect(r.status, r.out).toBe(124); + expect(r.calls).toBeGreaterThanOrEqual(2); + expect(r.calls).toBeLessThan(4); + expect(r.elapsedMs).toBeLessThan(10_500); + expect(r.out).toContain("dir-1"); + expect(r.out).toContain("budget"); + }, 15_000); + + async function interrupt(signal: "SIGTERM" | "SIGINT" | "SIGHUP" | "SIGKILL", duringGrace = false) { + const f = fixture([duringGrace ? "term-marker" : "hang", "ok"], { + BUN_INSTALL_ATTEMPT_SECONDS: duringGrace ? "1" : "8", BUN_INSTALL_KILL_GRACE_SECONDS: "3", + }); + const child = spawn("bash", [script, f.root], { env: f.env }); + let out = ""; + child.stdout.on("data", (d) => { out += d; }); + child.stderr.on("data", (d) => { out += d; }); + let closed = false; + const completion = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, sig) => { + closed = true; + resolve(code ?? (sig === "SIGKILL" ? 137 : sig === "SIGTERM" ? 143 : -1)); + }); + }); + let timer: ReturnType | undefined; + try { + const readyBy = Date.now() + 5_000; + const ready = duringGrace ? join(f.root, "termed") : f.count; + while (!existsSync(ready) && !closed && Date.now() < readyBy) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(existsSync(ready), out).toBe(true); + expect(closed, out).toBe(false); + const stopped = Date.now(); + child.kill(signal); + const status = await Promise.race([completion, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("No parent exit/output EOF")), 5_000); + })]); + // SIGKILL cannot wait for its descendants, so observe the group separately. + while (!drained(f.root) && Date.now() - stopped < 4_000) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(status, out).toBe({ SIGTERM: 143, SIGINT: 130, SIGHUP: 129, SIGKILL: 137 }[signal]); + expect(drained(f.root), out).toBe(true); + expect(Number(readFileSync(f.count, "utf8"))).toBe(1); + expect(out).not.toContain("::warning::"); + } finally { + if (timer) clearTimeout(timer); + if (!closed && child.exitCode === null && child.signalCode === null) child.kill("SIGTERM"); + if (!closed) await completion; + } + } + + test.each(["SIGTERM", "SIGINT", "SIGHUP", "SIGKILL"] as const)("parent %s stops its attempt without a retry", + (signal) => interrupt(signal), 15_000); + test("parent cancellation outranks an already recorded deadline during its grace", + () => interrupt("SIGTERM", true), 15_000); + test("retry completes a trusted postinstall interrupted after package extraction", () => { const root = mkdtempSync(join(tmpdir(), "ocx-bun-postinstall-")); roots.push(root); @@ -125,32 +284,4 @@ describe.skipIf(process.platform === "win32")("bun-install-frozen.sh retry polic expect(readFileSync(join(root, "bun.lock"), "utf8")).toBe(lockBefore); }, 30_000); - test("an ordinary install failure such as a lockfile mismatch is not retried", () => { - const result = run(["1", "ok"]); - expect(result.status, result.out).toBe(1); - expect(result.calls).toBe(1); - expect(result.out).toContain("not retrying"); - }); - - test("three deadline kills fail with an error annotation", () => { - const result = run(["hang", "hang", "hang", "ok"]); - expect(result.status, result.out).toBe(124); - expect(result.calls).toBe(3); - expect(result.out).toContain("::error::bun install --frozen-lockfile in"); - }); - - test("every attempt runs the frozen-lockfile install", () => { - const result = run(["ok"]); - expect(result.status, result.out).toBe(0); - expect(result.out).toContain("fake bun install --frozen-lockfile"); - }); - - test("root and GUI stalls share one total budget instead of three attempts each", () => { - // Per-directory retries alone would allow 6 attempts of 2 s plus grace; the 4 s total ends after 2. - const result = run(Array(8).fill("hang"), { dirs: 2, env: { BUN_INSTALL_ATTEMPT_SECONDS: "2", BUN_INSTALL_TOTAL_SECONDS: "4" } }); - expect(result.status, result.out).toBe(124); - expect(result.calls).toBe(2); - expect(result.elapsedMs).toBeLessThan(7_000); - expect(result.out).toContain("budget"); - }); }); From 78fca1448351ed698e9ba4a922e393d93b9806e6 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Fri, 30 Oct 2026 15:24:49 +0900 Subject: [PATCH 2/2] fix(ci): probe required timeout foreground support --- scripts/ci/bun-install-frozen.sh | 2 +- .../ci-bun-install-frozen.test.ts | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/scripts/ci/bun-install-frozen.sh b/scripts/ci/bun-install-frozen.sh index d9e8d9ef3a3..e6d7bc3aec0 100755 --- a/scripts/ci/bun-install-frozen.sh +++ b/scripts/ci/bun-install-frozen.sh @@ -79,7 +79,7 @@ for value in "$ATTEMPT_SECONDS" "$TOTAL_SECONDS" "$MIN_ATTEMPT_SECONDS" "$KILL_G done TIMEOUT_BIN="" for candidate in timeout gtimeout; do - if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --signal=TERM --kill-after=1s 1s true >/dev/null 2>&1; then + if command -v "$candidate" >/dev/null 2>&1 && "$candidate" --foreground 1s true >/dev/null 2>&1; then TIMEOUT_BIN=$(command -v "$candidate"); break fi done diff --git a/tests/ci-workflows/ci-bun-install-frozen.test.ts b/tests/ci-workflows/ci-bun-install-frozen.test.ts index 76e64fe30a5..cb969eaf5ef 100644 --- a/tests/ci-workflows/ci-bun-install-frozen.test.ts +++ b/tests/ci-workflows/ci-bun-install-frozen.test.ts @@ -109,6 +109,51 @@ describe.skipIf(process.platform === "win32")("bun-install-frozen.sh retry polic calls: existsSync(f.count) ? Number(readFileSync(f.count, "utf8")) : 0 }; } + function timeoutCandidates(f: ReturnType, compatibleFallback: boolean) { + const bin = join(f.root, "timeout-bin"); + mkdirSync(bin); + for (const name of ["timeout", "gtimeout"]) { + writeFileSync(join(bin, name), [ + "#!/usr/bin/env bash", + 'if [[ "$1" == --signal=TERM ]]; then exit 0; fi', + 'if [[ "$1" == --foreground ]]; then', + ...(name === "gtimeout" && compatibleFallback + ? [' shift 2', ' exec "$@"'] + : [' exit 64']), + "fi", + "exit 64", + ].join("\n")); + chmodSync(join(bin, name), 0o755); + } + return { ...f.env, PATH: `${bin}:${f.env.PATH ?? ""}` }; + } + + test("rejects candidates without foreground support before starting an install", () => { + const f = fixture(["ok"]); + const dir = join(f.root, "dir"); mkdirSync(dir); + const child = spawnSync("bash", [script, dir], { + encoding: "utf8", timeout: 10_000, env: timeoutCandidates(f, false), + }); + expect(child.error, child.stdout + child.stderr).toBeUndefined(); + expect(child.status, child.stdout + child.stderr).toBe(69); + expect(child.stdout + child.stderr).toContain("GNU timeout is required"); + expect(existsSync(f.count)).toBe(false); + expect(drained(f.root)).toBe(true); + }); + + test("falls back to gtimeout when timeout lacks foreground support", () => { + const f = fixture(["ok"]); + const dir = join(f.root, "dir"); mkdirSync(dir); + const child = spawnSync("bash", [script, dir], { + encoding: "utf8", timeout: 10_000, env: timeoutCandidates(f, true), + }); + expect(child.error, child.stdout + child.stderr).toBeUndefined(); + expect(child.status, child.stdout + child.stderr).toBe(0); + expect(existsSync(f.count)).toBe(true); + expect(readFileSync(f.count, "utf8")).toBe("1\n"); + expect(drained(f.root)).toBe(true); + }); + test.each(["1", "124", "137", "sigkill"])("ordinary install failure %s never retries", (step) => { const r = run([step, "ok"]); expect(r.status, r.out).toBe(step === "sigkill" ? 137 : Number(step));