From f073b7fb3f77351fcf33674e34b6ddd32e78de2e Mon Sep 17 00:00:00 2001 From: Willem Date: Mon, 19 Oct 2026 09:28:02 +0200 Subject: [PATCH] Add agent churn bench for main event-loop lag (#1661) Runs N fake stream-json claude agents through the real runner, Store and IPC against a /tmp copy of the store, and reports main event-loop delay, main CPU and renderer-timed threads.list round-trip per level. Co-Authored-By: Claude Opus 5.5 --- scripts/perf/agent-churn-bench.js | 206 ++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 scripts/perf/agent-churn-bench.js diff --git a/scripts/perf/agent-churn-bench.js b/scripts/perf/agent-churn-bench.js new file mode 100644 index 000000000..7ea131143 --- /dev/null +++ b/scripts/perf/agent-churn-bench.js @@ -0,0 +1,206 @@ +"use strict"; +// Main-process responsiveness under agent churn (#1661, step 1 of #418). +// +// ./node_modules/.bin/electron scripts/perf/agent-churn-bench.js /tmp/fstore [counts] [seconds] +// +// BENCH_BIG_THREADS=1 streams into the copy's largest transcripts (the worst +// case). BENCH_REAL_RENDERER=1 loads dist/index.html (npx vite build first) +// instead of a blank page with the preload. +// +// /tmp/fstore is a COPY of the data dir, as for store-stream-bench.js: +// mkdir /tmp/fstore +// cp -c ~/Library/Application\ Support/Solenta/coder-store.json /tmp/fstore/ +// cp -Rc ~/Library/Application\ Support/Solenta/messages /tmp/fstore/ +// The bench clones it again and works on the clone. counts defaults to +// 0,1,5,10,20 concurrent agents; seconds (default 20) is the sample per level. +// +// Same wiring as electron/smoke.js (Store + runner + registerIpc + a sandboxed +// window on preload.js), with no main.js boot: no orphan sweep, no auto-resume, +// no retention timers. Each agent is a fake claude CLI streaming stream-json +// partials (~40 text deltas/s, a tool call every ~2 s) through the real +// runner-provider-claude.js path, so main does the real JSON parse, Store +// upserts, 250 ms partial saves and thread:updated pushes. +// +// Reported per level: main event-loop delay (monitorEventLoopDelay), main CPU, +// and threads.list round-trip measured in the renderer every 100 ms. + +const { app, BrowserWindow, ipcMain, dialog } = require("electron"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { execFileSync } = require("node:child_process"); +const { monitorEventLoopDelay } = require("node:perf_hooks"); +const { pathToFileURL } = require("node:url"); + +const src = path.resolve(process.argv[2] || "/tmp/fstore"); +const counts = (process.argv[3] || "0,1,5,10,20").split(",").map(Number); +const seconds = Number(process.argv[4] || 20); +const tmpRoots = [os.tmpdir(), "/tmp", "/private/tmp"].map((p) => fs.realpathSync(p)); +if (!tmpRoots.some((root) => fs.realpathSync(src).startsWith(root + path.sep))) { + console.error(`refusing ${src}: the bench only runs on a copy under /tmp`); + process.exit(2); +} +if (!fs.existsSync(path.join(src, "coder-store.json"))) { + console.error(`no coder-store.json in ${src}`); + process.exit(2); +} + +const work = fs.mkdtempSync(path.join(os.tmpdir(), "churn-bench-")); +try { + execFileSync("cp", ["-Rc", `${src}/.`, work]); +} catch { + fs.cpSync(src, work, { recursive: true }); +} +app.setPath("userData", work); +process.env.SOLENTA_SKIP_USERDATA_MIGRATION = "1"; + +// Fake claude: streams until killed by runs.stop. +const fakeClaude = path.join(work, "bench-fake-claude"); +fs.writeFileSync(fakeClaude, `#!/usr/bin/env node +"use strict"; +const delay = (ms) => new Promise((r) => setTimeout(r, ms)); +const emit = (o) => process.stdout.write(JSON.stringify(o) + "\\n"); +const ev = (event) => emit({ type: "stream_event", event }); +(async () => { + emit({ type: "system", subtype: "init", session_id: "bench-" + process.pid, model: "bench" }); + for (let turn = 0; ; turn++) { + ev({ type: "message_start", message: { id: "m" + turn } }); + ev({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }); + let text = ""; + for (let i = 0; i < 72; i++) { + const d = "lorem ipsum dolor sit amet " + i + " "; + text += d; + ev({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: d } }); + await delay(25); + } + ev({ type: "content_block_stop", index: 0 }); + const id = "toolu_" + process.pid + "_" + turn; + const input = { command: "ls -la " + turn }; + ev({ type: "content_block_start", index: 1, content_block: { type: "tool_use", id, name: "Bash", input: {} } }); + ev({ type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: JSON.stringify(input) } }); + ev({ type: "content_block_stop", index: 1 }); + emit({ type: "assistant", message: { content: [{ type: "text", text }, { type: "tool_use", id, name: "Bash", input }] } }); + ev({ type: "message_stop" }); + await delay(200); + emit({ type: "user", message: { content: [{ type: "tool_result", tool_use_id: id, content: "total 0\\n".repeat(20), is_error: false }] } }); + } +})(); +`, { mode: 0o755 }); +process.env.CODER_CLAUDE_BIN = fakeClaude; + +const pct = (sorted, p) => sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))] : NaN; +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +app.whenReady().then(async () => { + const { Store } = require("../../electron/store.js"); + const { createRunner } = require("../../electron/runner.js"); + const { registerIpc } = require("../../electron/ipc.js"); + const core = await import(pathToFileURL(path.join(__dirname, "../../core/dist/index.js")).href); + + const store = new Store(path.join(work, "coder-store.json")); + const broadcast = (channel, payload) => { + for (const w of BrowserWindow.getAllWindows()) { + if (!w.isDestroyed()) w.webContents.send(channel, payload); + } + }; + const runner = createRunner({ store, core, pushFn: broadcast, tickMs: 700 }); + registerIpc({ ipcMain, dialog, store, runner, broadcast, worktreeBase: path.join(work, "worktrees"), userDataPath: work }); + + const win = new BrowserWindow({ + show: false, + webPreferences: { preload: path.join(__dirname, "../../electron/preload.js"), contextIsolation: true, sandbox: true }, + }); + const dist = path.join(__dirname, "../../dist/index.html"); + const blank = path.join(work, "blank.html"); + fs.writeFileSync(blank, "bench"); + const page = process.env.BENCH_REAL_RENDERER ? dist : blank; + await win.loadFile(page); + const js = (expr) => win.webContents.executeJavaScript(expr, true); + + // A scratch git repo, never one of the store copy's real project paths. + const repo = fs.mkdtempSync(path.join(os.tmpdir(), "churn-bench-repo-")); + execFileSync("git", ["init", "-q", repo]); + execFileSync("git", ["-C", repo, "-c", "user.email=b@b", "-c", "user.name=b", "commit", "-q", "--allow-empty", "-m", "init"]); + const project = await js(`window.coder.projects.add(${JSON.stringify(repo)})`); + + console.log(`store: ${(fs.statSync(path.join(work, "coder-store.json")).size / 1048576).toFixed(1)} MiB envelope, ${store.getThreads().length} threads; renderer: ${path.basename(page)}`); + console.log("agents | loop p50 | loop p99 | loop max | main CPU | list p50 | list p99 | list max | pushes/s | msgs"); + + const histogram = monitorEventLoopDelay({ resolution: 1 }); + let pushes = 0; + const realSend = win.webContents.send.bind(win.webContents); + win.webContents.send = (ch, ...a) => { if (ch === "thread:updated") pushes++; return realSend(ch, ...a); }; + + // BENCH_BIG_THREADS=1: stream into the copy's largest transcripts instead of + // fresh threads, so each partial save rewrites a multi-MB shard. They are + // moved onto the scratch project first so no run touches a real repo. + const known = new Set(store.getThreads().map((t) => t.id)); + const bigThreads = process.env.BENCH_BIG_THREADS + ? fs.readdirSync(path.join(work, "messages")) + .map((f) => ({ id: f.replace(/\.json$/, ""), size: fs.statSync(path.join(work, "messages", f)).size })) + .filter((s) => known.has(s.id)) + .sort((a, b) => b.size - a.size) + : []; + for (const { id } of bigThreads.slice(0, Math.max(...counts))) { + store.updateThread(id, { + projectId: project.id, provider: "claude", model: null, sessionId: null, worktreePath: null, + permissionMode: "default", archived: false, settledOverride: null, status: "idle", queued: null, + }); + } + if (bigThreads.length) { + const top = bigThreads.slice(0, Math.max(...counts)).map((s) => s.size / 1048576); + console.log(`big threads: shards ${top[top.length - 1].toFixed(1)}–${top[0].toFixed(1)} MiB`); + } + + for (const n of counts) { + const levelStart = Date.now(); + const threadIds = []; + for (let i = 0; i < n; i++) { + const id = bigThreads.length + ? bigThreads[i].id + : (await js(`window.coder.threads.create(${JSON.stringify({ projectId: project.id, title: `bench ${n}/${i}` })})`)).id; + threadIds.push(id); + await js(`window.coder.runs.start(${JSON.stringify({ threadId: id, prompt: "bench" })})`); + } + await sleep(3000); // warm-up: spawn, init, first saves + + await js(`window.__rtt = []; window.__stop = false; (async () => { + while (!window.__stop) { + const t = performance.now(); + await window.coder.threads.list(); + window.__rtt.push(performance.now() - t); + await new Promise((r) => setTimeout(r, 100)); + } + })(); 0`); + histogram.reset(); + histogram.enable(); + pushes = 0; + const cpu0 = process.cpuUsage(); + const t0 = performance.now(); + await sleep(seconds * 1000); + const wallMs = performance.now() - t0; + const cpu = process.cpuUsage(cpu0); + histogram.disable(); + const rtt = (await js(`window.__stop = true; window.__rtt`)).sort((a, b) => a - b); + + for (const id of threadIds) await js(`window.coder.runs.stop(${JSON.stringify({ threadId: id })})`); + const msgs = threadIds.reduce((sum, id) => sum + store.getMessages(id).filter((m) => m.createdAt >= levelStart).length, 0); + const ms = (ns) => (ns / 1e6).toFixed(1).padStart(8); + const r = (v) => v.toFixed(1).padStart(8); + console.log( + `${String(n).padStart(6)} |${ms(histogram.percentile(50))} |${ms(histogram.percentile(99))} |${ms(histogram.max)} |` + + `${(((cpu.user + cpu.system) / 1000 / wallMs) * 100).toFixed(0).padStart(7)}% |` + + `${r(pct(rtt, 0.5))} |${r(pct(rtt, 0.99))} |${r(rtt[rtt.length - 1] || NaN)} |${(pushes / (wallMs / 1000)).toFixed(0).padStart(9)} |${String(msgs).padStart(6)}`, + ); + + await store.flushPending(); + await sleep(2000); + } + + fs.rmSync(work, { recursive: true, force: true, maxRetries: 8, retryDelay: 10 }); + fs.rmSync(repo, { recursive: true, force: true }); + app.exit(0); +}).catch((err) => { + console.error(err); + app.exit(1); +});