From 044f819fd78c6149eec2f7652382671bcfad84fa Mon Sep 17 00:00:00 2001 From: Tim Biggs Date: Sun, 1 Nov 2026 23:13:44 +0000 Subject: [PATCH] fix(server): Claude threads get a reminder to mark background work they wait on An agent that started a background waiter could end its turn without calling t3_background_wait, so the sidebar showed the thread as available while it was still waiting. A PostToolUse hook now repeats the instruction when a background Bash or Monitor call starts in the top-level thread. The mcp-tool-use eval applies T3's PostToolUse hooks and gains a case for a waiter restarted while handling a delegate's wake. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/ClaudeAdapterV2.test.ts | 42 +++++++++++++ .../Adapters/ClaudeAdapterV2.ts | 35 ++++++++++- .../mcp-tool-use/cases/background-work.ndjson | 1 + evals/mcp-tool-use/run.test.ts | 11 ++++ evals/mcp-tool-use/run.ts | 63 ++++++++++++++++++- 5 files changed, 148 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 0ce627d4e7747..f4f9dcaf61c34 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -692,6 +692,48 @@ describe("ClaudeAdapterV2 device instructions", () => { assert.notInclude(append({ mcp: false, deviceToolsAvailable: true }), "## T3 Code devices"); }); + it("reminds the thread to mark background work only when a background task starts", async () => { + const reminded = async (input: { + readonly mcp?: boolean; + readonly toolName: string; + readonly toolInput: unknown; + readonly agentId?: string; + }) => { + const matchers = ( + options({ mcp: input.mcp ?? true, deviceToolsAvailable: false }).hooks?.PostToolUse ?? [] + ).filter((matcher) => new RegExp(`^(?:${matcher.matcher})$`).test(input.toolName)); + const outputs = await Promise.all( + matchers + .flatMap((matcher) => matcher.hooks) + .map((hook) => + hook( + { + hook_event_name: "PostToolUse", + tool_name: input.toolName, + tool_input: input.toolInput, + tool_response: "", + tool_use_id: "toolu-1", + session_id: "native-thread-claude-devices", + transcript_path: "/transcript.jsonl", + cwd: "/workspace", + ...(input.agentId === undefined ? {} : { agent_id: input.agentId }), + }, + "toolu-1", + { signal: new AbortController().signal }, + ), + ), + ); + return outputs.some((output) => "hookSpecificOutput" in output); + }; + const background = { command: "npm test", run_in_background: true }; + assert.isTrue(await reminded({ toolName: "Bash", toolInput: background })); + assert.isTrue(await reminded({ toolName: "Monitor", toolInput: { command: "tail -f log" } })); + assert.isFalse(await reminded({ toolName: "Bash", toolInput: { command: "npm test" } })); + assert.isFalse(await reminded({ toolName: "Read", toolInput: { run_in_background: true } })); + assert.isFalse(await reminded({ toolName: "Bash", toolInput: background, agentId: "agent-1" })); + assert.isFalse(await reminded({ mcp: false, toolName: "Bash", toolInput: background })); + }); + it("gives native subagents the device block only in a session with device access", async () => { // Subagents start from their task prompt, not the parent's system prompt. const subagentContext = (input: { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3789bc477b8b1..e701d24506560 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -128,6 +128,7 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Predicate from "effect/Predicate"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -1462,6 +1463,27 @@ const subagentDeviceInstructions: HookCallback = () => }, }); +const CLAUDE_BACKGROUND_WAIT_REMINDER = + "If you will report on this task's result, call `t3_background_wait` with its task ID before you end your turn. If you are leaving it running, do not mark it."; + +// The system prompt's line fades in long threads, so repeat it where the decision happens. +// A Monitor always runs in the background. Subagents' work is not the thread's to mark. +const backgroundWaitReminder: HookCallback = (input) => + Promise.resolve( + input.hook_event_name === "PostToolUse" && + input.agent_id === undefined && + (input.tool_name === "Monitor" || + (Predicate.isObject(input.tool_input) && + Reflect.get(input.tool_input, "run_in_background") === true)) + ? { + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: CLAUDE_BACKGROUND_WAIT_REMINDER, + }, + } + : {}, + ); + export function makeClaudeQueryOptions(input: { readonly modelSelection: ModelSelection; readonly nativeThreadId: string; @@ -1604,9 +1626,16 @@ export function makeClaudeQueryOptions(input: { ...launchAppendSystemPrompt, ].join("\n\n"), }, - ...(deviceInstructions - ? { hooks: { SubagentStart: [{ hooks: [subagentDeviceInstructions] }] } } - : {}), + ...(input.mcpServers === undefined + ? {} + : { + hooks: { + PostToolUse: [{ matcher: "Bash|Monitor", hooks: [backgroundWaitReminder] }], + ...(deviceInstructions + ? { SubagentStart: [{ hooks: [subagentDeviceInstructions] }] } + : {}), + }, + }), ...(Object.keys(extraArgs).length === 0 ? {} : { extraArgs }), }; const additionalDirectories = [ diff --git a/evals/mcp-tool-use/cases/background-work.ndjson b/evals/mcp-tool-use/cases/background-work.ndjson index b359451112738..77b5e22c978b0 100644 --- a/evals/mcp-tool-use/cases/background-work.ndjson +++ b/evals/mcp-tool-use/cases/background-work.ndjson @@ -1,3 +1,4 @@ {"id": "dev-server-not-waited-on", "area": "background-work", "kind": "guard", "pattern": "does not hold the thread for a dev server it leaves running", "experience": "A dev server marked as waited on would keep the thread in Working, with no completion notice, until the server exits.", "rule": "Do not mark work you leave running, or PRs.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"scripts\": { \"dev\": \"vite --port 5173\", \"build\": \"vite build\" },\n \"devDependencies\": { \"vite\": \"^7.1.0\" }\n}\n"}}, "shellMocks": [{"match": "npm run dev|pnpm (run )?dev|vite", "output": "Command running in background with ID: b7k2m9x4q. Output is being written to: /tmp/claude/tasks/b7k2m9x4q.output"}]}, "prompt": "Start the dev server in the background so I can click around the site, then hand back to me.", "expect": {"notCalls": ["t3_background_wait"]}, "maxSteps": 8, "source": {"pr": "tebiggs/t3code#14"}} {"id": "background-deploy-waited-on", "area": "background-work", "kind": "guard", "pattern": "holds the thread for a background deploy it will report on", "experience": "A deploy ran in the background after the agent ended its turn. Automatic settlement after the merged PR detached the session and killed the deploy and its waiter, and its failed smoke check surfaced hours later. Marking the job keeps the thread working until its result arrives.", "rule": "To wait for a background command or Monitor you started (tests, a build), call t3_background_wait with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"scripts\": { \"dev\": \"vite --port 5173\", \"build\": \"vite build\" },\n \"devDependencies\": { \"vite\": \"^7.1.0\" }\n}\n", "scripts/deploy.sh": "#!/bin/bash\nset -e\nnpm run build\n./scripts/publish.sh\n"}}, "shellMocks": [{"match": "deploy", "output": "Command running in background with ID: bq4w8e1rt. Output is being written to: /tmp/claude/tasks/bq4w8e1rt.output"}]}, "prompt": "The PR is merged. Run ./scripts/deploy.sh in the background; it takes about ten minutes. Tell me how it went when it finishes.", "expect": {"calls": [{"tool": "Bash", "args": {"run_in_background": true}}, {"tool": "t3_background_wait"}], "callsMustNot": [{"tool": "t3_background_wait", "args": {"waiting": false}}]}, "maxSteps": 8, "source": {"pr": "tebiggs/t3code#14"}} {"id": "detached-deploy-waiter-waited-on", "area": "background-work", "kind": "guard", "pattern": "holds the thread for a detached deploy's waiter", "experience": "Long deploys run detached with setsid so a closed session cannot kill them, and a background `tail --pid` waiter tells the agent when they exit. An unmarked waiter let automatic settlement after a merged PR kill it, and the deploy's failed smoke check surfaced only at the next user message.", "rule": "To wait for a background command or Monitor you started (tests, a build), call t3_background_wait with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"scripts\": { \"dev\": \"vite --port 5173\", \"build\": \"vite build\" },\n \"devDependencies\": { \"vite\": \"^7.1.0\" }\n}\n", "scripts/deploy.sh": "#!/bin/bash\nset -e\nnpm run build\n./scripts/publish.sh\n./scripts/smoke.sh\n"}}, "shellMocks": [{"match": "setsid", "output": "48213"}, {"match": "tail --pid", "output": "Command running in background with ID: bw5t2n8kd. Output is being written to: /tmp/claude/tasks/bw5t2n8kd.output"}]}, "prompt": "The PR is merged, so deploy it. The deploy can outlast your shell's time limit, so start it detached: `setsid nohup ./scripts/deploy.sh > /tmp/deploy.log 2>&1 < /dev/null & echo $!`. Then wait for that pid with `tail --pid= -f /dev/null` in the background, so you hear when it exits, and tell me how the deploy went.", "expect": {"calls": [{"tool": "Bash", "args": {"run_in_background": true}}, {"tool": "t3_background_wait"}], "callsMustNot": [{"tool": "t3_background_wait", "args": {"waiting": false}}]}, "maxSteps": 8, "source": {"pr": "tebiggs/t3code#14"}} +{"id": "waiter-beside-delegate-waited-on", "area": "background-work", "kind": "bad", "pattern": "holds the thread for a gates waiter it restarts while handling a delegate's wake", "experience": "An orchestrator restarted detached test gates and a background `tail --pid` waiter while handling a delegated review's result, and started the next review round beside them. It never marked the waiter, so once the review finished the sidebar showed the thread as available while it was still waiting to merge.", "rule": "To wait for a background command or Monitor you started (tests, a build), call t3_background_wait with its task ID before you end your turn.", "privacy": "commit-safe", "setup": {"role": "top", "harness": "claude", "world": {"files": {"package.json": "{\n \"name\": \"app\",\n \"private\": true,\n \"scripts\": { \"test\": \"vitest run\", \"typecheck\": \"tsc --noEmit\" }\n}\n", "scripts/gates.sh": "#!/bin/bash\nset -e\nnpm run typecheck\nnpm test\n", "spec.md": "# Payday\n\nA busy marketeer may be marked only if every salaried card that is not busy is marked and the seat cannot pay.\n"}, "branch": "fix/payday-firing", "status": "", "commits": [{"sha": "24fa703", "subject": "docs(spec): payday firing marks"}], "pullRequest": {"number": 42, "state": "OPEN", "headRefName": "fix/payday-firing", "headRefOid": "24fa703", "note": "This worktree has the PR branch checked out at its head. No one else has pushed to it."}}, "shellMocks": [{"match": "setsid", "output": "48297"}, {"match": "tail --pid", "output": "Command running in background with ID: bw5t2n8kd. Output is being written to: /tmp/claude/tasks/bw5t2n8kd.output"}], "context": "Long jobs in this project: a job that can outlast the shell's time limit runs detached with setsid. Also start `tail --pid= -f /dev/null` with run_in_background. That keeps the session alive and notifies you when the job exits.", "priorTurns": [{"role": "user", "text": "Keep the current design. Once PR #42 is merged, hand off to a new thread for the implementation."}, {"role": "assistant", "text": "PR #42 has the updated spec. The Codex review and the gates are both running on head 24fa7033. Once they pass, I'll merge it and hand the implementation to a new thread.", "toolCalls": [{"tool": "Bash", "input": {"command": "setsid nohup ./scripts/gates.sh > /tmp/gates.log 2>&1 < /dev/null & echo $!", "description": "Start the gates detached"}, "result": "48190"}, {"tool": "delegate_task", "input": {"title": "Review PR 42 round 1", "role": "review", "clientRequestId": "pr42-review-round1", "target": {"providerInstanceId": "codex", "model": "gpt-6.1-sol"}, "task": "Review PR #42 at head 24fa7033. You wrote none of this diff and must not edit the branch."}, "result": {"taskId": "node:delegated-task:round1", "childThreadId": "thread:delegated-task:round1", "status": "running", "workState": "working", "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}, {"tool": "Bash", "input": {"command": "tail --pid=48190 -f /dev/null; grep -E '^\\[gate\\]' /tmp/gates.log", "run_in_background": true, "description": "Wait for the gates"}, "result": "Command running in background with ID: b0z7fob8f. Output is being written to: /tmp/claude/tasks/b0z7fob8f.output"}]}]}, "prompt": "Delegated task \"Review PR 42 round 1\" finished.\n\nVerdict: changes needed. F1, blocking: spec.md lets a free busy marketing trainee be marked to fire, which cannot help pay the bill. Require the busy card itself to draw a salary this turn.\n\nApply the fix, stop the gates running on the old head (pid 48190), restart them and their waiter on the new head, and start round 2 of the review.", "expect": {"calls": [{"tool": "delegate_task"}, {"tool": "Bash", "args": {"run_in_background": true}}, {"tool": "t3_background_wait"}], "callsMustNot": [{"tool": "t3_background_wait", "args": {"waiting": false}}]}, "maxSteps": 14, "source": {"pr": "tebiggs/t3code#24"}} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 69de4f0c6c4b2..413ab8f88a483 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -24,6 +24,7 @@ import { shellRequest, summarize, t3SystemAppend, + t3PostToolUseReminders, t3SubagentContext, withDevices, } from "./run.ts"; @@ -780,6 +781,16 @@ it("gives native children hook context while excluding the parent's T3 append an assert.include(shown, testCase.prompt); }); +it("adds T3's background reminder under a template's renamed Bash, for the top thread only", async () => { + const uses = (input: unknown) => [{ id: "toolu_1", name: "mcp__select_sight__term_Bash", input }]; + const reminders = (input: unknown, nativeSubagent = false) => + t3PostToolUseReminders("claude-sonnet-5-5", uses(input), { nativeSubagent }); + const background = { command: "npm test", run_in_background: true }; + assert.lengthOf(await reminders(background), 1); + assert.isEmpty(await reminders({ command: "npm test" })); + assert.isEmpty(await reminders(background, true)); +}); + // Round-5 adversarial review counterexamples. it.each([ ["Use device_open. Use ssh build-mac for screenshots.", "mixed"], diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 1bb42346d650a..4023cdbab9375 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -465,6 +465,54 @@ export const t3SubagentContext = async (model: Model, device = true) => { .join("\n\n"); }; +/** + * The reminders T3's PostToolUse hooks add after these tool calls, as Claude Code words them. + * Empty when T3 registers no such hook, as on text from before the hook existed. + */ +export const t3PostToolUseReminders = async ( + model: Model, + uses: ReadonlyArray<{ readonly id: string; readonly name: string; readonly input: unknown }>, + options: { readonly nativeSubagent: boolean }, +) => { + const matchers = claudeQueryOptions(model).hooks?.PostToolUse ?? []; + const outputs = await Promise.all( + uses.flatMap((use) => { + // Templates rename native tools. Hooks see Claude Code's own names. + const toolName = normalizeToolName(use.name); + return matchers + .filter((m) => new RegExp(`^(?:${m.matcher ?? ".*"})$`).test(toolName)) + .flatMap((m) => m.hooks) + .map(async (hook) => { + const output = await hook( + { + hook_event_name: "PostToolUse", + session_id: "mcp-tool-use-eval", + transcript_path: "", + cwd: "", + tool_name: toolName, + tool_input: use.input, + tool_response: "", + tool_use_id: use.id, + ...(options.nativeSubagent ? { agent_id: "mcp-tool-use-eval" } : {}), + }, + use.id, + { signal: AbortSignal.timeout(10_000) }, + ); + return "hookSpecificOutput" in output && + output.hookSpecificOutput?.hookEventName === "PostToolUse" && + output.hookSpecificOutput.additionalContext + ? [ + reminder( + `PostToolUse:${toolName} hook additional context: ${output.hookSpecificOutput.additionalContext}`, + ), + ] + : []; + }); + }), + ); + return outputs.flat(); +}; + /** * Size of the context T3 adds: the system text every Claude thread gets, and the tool * descriptions and schemas (loaded on demand here, sent up front by harnesses without @@ -1816,7 +1864,20 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( ); // Claude Code adds this text after a ToolSearch that loaded tools. const loaded = results.some((result) => Array.isArray(result.content)); - messages.push({ role: "user", content: [...results, ...(loaded ? [text(TOOL_LOADED)] : [])] }); + // PostToolUse fires only for calls that ran and did not fail. + const hookReminders = input + ? [] + : yield* Effect.promise(() => + t3PostToolUseReminders( + config.model, + uses.filter((_, index) => results[index]?.is_error !== true), + { nativeSubagent: testCase.setup.role === "native-subagent" }, + ), + ); + messages.push({ + role: "user", + content: [...results, ...(loaded ? [text(TOOL_LOADED)] : []), ...hookReminders], + }); if (failedEarly(testCase.expect, { calls, text: said.join("\n\n"), stopReason })) { stopReason = "failed_early"; break;