Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/server/src/mcp/toolkits/project/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ const ThreadLaunchTool = Tool.make("t3_thread_launch", {
workspaceStrategy: Schema.optional(
OrchestrationV2ThreadLaunchWorkspaceStrategy.annotate({
description:
"'worktree': new checkout from `baseRef`. 'existing_worktree': bind `worktreePath`, a project worktree. 'root': project checkout, the default, not your worktree. Stack: parent branch as `baseRef`, `startFromOrigin: false`. Uncommitted edits are not copied.",
"'worktree': new checkout from `baseRef`. Honor a requested base. Otherwise, independent work uses the default branch as `remote/branch` (e.g. `origin/main`) with `startFromOrigin: true`; with no remote, use the local default branch. Required unpublished commits and dependent or stacked work use their local base with `startFromOrigin: false`. 'existing_worktree': bind `worktreePath`, a project worktree. 'root': project checkout, the default, not your worktree. Uncommitted edits are not copied.",
}),
),
message: Schema.optional(
Expand Down
6 changes: 3 additions & 3 deletions apps/server/src/provider/agentContextBudget.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 5133,
"codexAdditionalContextBytes": 6863,
"mcpToolListJsonBytes": 47809
"claudeSystemAppendBytes": 5366,
"codexAdditionalContextBytes": 7096,
"mcpToolListJsonBytes": 48051
}
25 changes: 21 additions & 4 deletions apps/server/src/textGeneration/CodexTextGeneration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ interface FakeCodexInput {
stdinMustContain?: string;
stdinMustNotContain?: string;
mcpServers?: ReadonlyArray<{ name: string; enabled: boolean }> | number;
requiredCwd?: string;
requireInstructions?: string;
}

function makeFakeCodexBinary(dir: string, input: FakeCodexInput) {
Expand All @@ -67,6 +69,8 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) {
stdinMustContain: input.stdinMustContain ?? null,
stdinMustNotContain: input.stdinMustNotContain ?? null,
mcpServers: input.mcpServers ?? [],
requiredCwd: input.requiredCwd ?? null,
requireInstructions: input.requireInstructions ?? null,
stderr: input.stderr ?? null,
output: input.output,
exitCode: input.exitCode ?? 0,
Expand Down Expand Up @@ -100,13 +104,15 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) {
"let seenImage = false;",
'let seenServiceTier = "";',
'let seenReasoningEffort = "";',
'let instructions = "";',
"for (let index = 0; index < args.length; index += 1) {",
' if (args[index] === "--image") {',
" index += 1;",
" if (args[index]) seenImage = true;",
' } else if (args[index] === "--config") {',
" index += 1;",
' const value = args[index] ?? "";',
' if (value.startsWith("developer_instructions=")) instructions = JSON.parse(value.slice("developer_instructions=".length));',
' if (value.startsWith("service_tier=")) seenServiceTier = value;',
' if (value.startsWith("model_reasoning_effort=")) seenReasoningEffort = value;',
' } else if (args[index] === "--output-last-message") {',
Expand All @@ -121,6 +127,8 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) {
' process.stderr.write(message + "\\n");',
" process.exit(code);",
"}",
'if (check.requiredCwd !== null && process.cwd() !== check.requiredCwd) fail("cannot inspect checkout", 10);',
'if (check.requireInstructions !== null && !instructions.includes(check.requireInstructions)) fail("missing writer task", 11);',
"if (check.requireArg !== null && !originalArgs.includes(` ${check.requireArg} `)) {",
' fail("missing arg: " + check.requireArg, 8);',
"}",
Expand Down Expand Up @@ -341,10 +349,14 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => {
),
);

it.effect("generates titles with the user's MCP servers and plugins off", () =>
it.effect("preserves custom PR instructions while allowing checkout inspection", () =>
withFakeCodexEnv(
{
output: JSON.stringify({ title: "Remove default shapes" }),
output: JSON.stringify({ title: "Remove default shapes", body: "Removes default shapes." }),
launchArgs: "--config 'developer_instructions=\"Use TEAM: as the PR title prefix\"'",
requiredCwd: process.cwd(),
requireInstructions: "Use TEAM: as the PR title prefix",
forbidArg: "--config features.shell_tool=false",
mcpServers: [
{ name: "augmenter", enabled: true },
{ name: "my.server", enabled: true },
Expand All @@ -355,9 +367,14 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => {
},
(textGeneration) =>
Effect.gen(function* () {
const result = yield* textGeneration.generateThreadTitle({
const result = yield* textGeneration.generatePrContent({
cwd: process.cwd(),
message: "Augmenter task 1. Use the Augmenter MCP: link_thread when you start.",
baseBranch: "main",
headBranch: "remove-default-shapes",
commitSummary: "Remove default shapes",
diffSummary: "shapes.ts | 2 --",
diffPatch: "-addDefaultShapes();",
threadContext: "user: in a few minutes PR will be up then $babysit-pr",
modelSelection: DEFAULT_TEST_MODEL_SELECTION,
});
expect(result.title).toBe("Remove default shapes");
Expand Down
1 change: 1 addition & 0 deletions evals/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Write or run an eval when a change affects what a model sees or does: agent inst
| ------------------------------- | ---------------------------------------------------------------------------------------------------- |
| [mcp-tool-use](mcp-tool-use/) | Whether agents use T3's MCP tools correctly, including device routing and recovery, in fake threads. |
| [thread-titles](thread-titles/) | Whether generated thread titles name the right subject. |
| [pr-content](pr-content/) | Whether PR generation describes the diff when project instructions try to redirect it. |
| [voice-cleanup](voice-cleanup/) | Whether optional dictation cleanup edits only the spoken text and preserves its meaning. |
| [test-signal](test-signal/) | Whether a cheap model running the test-signal check flags low-signal tests. |

Expand Down
4 changes: 4 additions & 0 deletions evals/mcp-tool-use/cases/threads-vs-tasks.ndjson
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,7 @@
{"id": "tt-control-agent-config", "area": "orchestration", "kind": "control", "pattern": "Domain agent configuration lookup", "experience": "A question about the app's agent label setting is ordinary code work; it starts no agent.", "rule": "Ordinary coding tasks need no T3 tools.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n"}}, "mocks": {"t3_thread_launch": {"threadId": "11111111-1111-4111-8111-111111111111", "projectId": "33333333-3333-4333-8333-333333333333", "modelSelection": {"instanceId": "codex", "model": "gpt-6.1-sol", "options": []}, "runId": "run-acme-thread", "status": "running"}, "t3_thread_read": {"thread": {"threadId": "11111111-1111-4111-8111-111111111111", "title": "Acme cache worker", "status": "running"}, "items": [{"position": 1, "type": "assistant_message", "text": "Checking the cache export."}], "nextPosition": 1, "hasMore": false}, "t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Read src/agent-config.ts and tell me the current default agent label in this application.", "expect": {"notCalls": ["t3_thread_launch", "create_threads", "delegate_task", "Agent", "Workflow", "orchestrator_capabilities"]}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard"}, "notes": "Synthetic Acme scenario; no field transcript copied."}
{"id": "tt-background-research-no-agent-word", "area": "orchestration", "kind": "guard", "pattern": "bare 'in the background' ask: delegate_task", "experience": "Asked to look into something in the background without the word 'agent', the agent must start a delegate_task, not a background native Agent or a new thread.", "rule": "Background agent work goes to delegate_task; Claude's Agent runs with run_in_background:false. / t3_thread_launch and create_threads make new top-level threads, only when the user asks for a new thread.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "main", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/cache/export.ts": "export function exportCache(entries: Map<string, string>) {\n return [...entries].filter(([, value]) => value !== '').map(([key, value]) => ({ key, value }));\n}\n"}}}, "prompt": "In the background, find out why src/cache/export.ts drops entries that docs/cache.md says must be kept, and report back when you know. I'll keep going on the release notes here.", "expect": {"calls": [{"tool": "delegate_task", "args": {"task": "~cache"}}], "notCalls": ["t3_thread_launch", "create_threads", "Workflow", "t3_thread_wait"], "callsMustNot": [{"tool": "Agent", "args": {"run_in_background": true}}, {"tool": "delegate_task", "args": {"mode": "wait"}}], "maxCalls": {"task_status": 0}}, "source": {"request": "juliusmarminge (maintainer) request, 2026-10-29; regression guard", "review": "council review round 1: bare background ask without 'agent'"}, "notes": "Synthetic Acme scenario. Pairs with tt-background-agent; checks the rule does not depend on the word 'agent'."}
{"id":"tt-control-background-test-run","area":"orchestration","kind":"control","pattern":"background shell command is not a delegate","experience":"Asked to run the test suite in the background while refactoring, the agent should use an ordinary background shell command, not a delegate, subagent or thread.","rule":"Ordinary coding tasks need no T3 tools.","privacy":"commit-safe","setup":{"role":"top","context":"Repository: /home/user/app (acme/app). This is the local machine.","world":{"repo":"acme/app","branch":"main","status":"","files":{"package.json":"{\"scripts\":{\"test\":\"vitest run\"}}\n","src/export/csv.ts":"export function toCsv(rows: Row[]) {\n return rows.map((r) => [r.id, r.total].join(',')).join('\\n');\n}\n"}}},"prompt":"Kick off the full test suite in the background, and meanwhile rename toCsv to rowsToCsv in src/export/csv.ts.","expect":{"notCalls":["delegate_task","t3_thread_launch","create_threads","Workflow","orchestrator_capabilities"],"callsMustNot":[{"tool":"Agent","args":{"run_in_background":true}}],"calls":[{"tool":"Bash","args":{"command":"~test"}}]},"source":{"request":"juliusmarminge (maintainer) request, 2026-10-29; regression guard","review":"council review round 1: over-correction control for 'background'"},"notes":"Synthetic Acme scenario. 'Background' here means a shell command, not an agent."}
{"id": "tt-independent-worktree-default-base", "area": "orchestration", "kind": "bad", "pattern": "Another thread and PR must use thread workspace binding", "experience": "An agent launching unrelated work explicitly chose its current feature branch, accidentally creating a stacked PR.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. Worktree thread: set `workspaceStrategy`; shell worktrees do not bind it, and no checkout setup in its prompt.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine. Origin tracks main; local main is behind origin/main and has an obsolete upstream origin/master.", "world": {"repo": "acme/app", "branch": "feature/error-display", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n", "AGENTS.md": "The repository default branch is main.\n"}}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Keep fixing the error display here. Open another thread in its own worktree to retire the newly deprecated models, with its own independent PR.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"workspaceStrategy.type": "worktree", "workspaceStrategy.baseRef": "origin/main", "message": "~retir", "workspaceStrategy.startFromOrigin": true}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "shellMustNot": ["git\\s+worktree\\s+add"], "callsMustNot": [{"tool": "t3_thread_launch", "args": {"message": "~git worktree add"}}], "newThreads": ["~retir"]}, "source": {"request": "Reported unintended stack during independent model-retirement work, November 2026"}, "notes": "Synthetic Acme scenario. Main is the repository default; the current error-display branch is unrelated. Origin/main is newer than local main, whose upstream is origin/master."}
{"id": "tt-dependent-worktree-parent-base", "area": "orchestration", "kind": "guard", "pattern": "Another thread and PR must use thread workspace binding", "experience": "Explicitly dependent work must keep the requested parent branch as its base.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. Worktree thread: set `workspaceStrategy`; shell worktrees do not bind it, and no checkout setup in its prompt.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine.", "world": {"repo": "acme/app", "branch": "feature/error-display", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n", "AGENTS.md": "The repository default branch is main.\n"}}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Open another thread in a worktree to extend this error-display fix with recovery controls. Stack its PR on this branch because it depends on these changes.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"workspaceStrategy.type": "worktree", "workspaceStrategy.baseRef": "feature/error-display"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "shellMustNot": ["git\\s+worktree\\s+add"], "callsMustNot": [{"tool": "t3_thread_launch", "args": {"message": "~git worktree add"}}, {"tool": "t3_thread_launch", "args": {"workspaceStrategy.startFromOrigin": true}}]}, "source": {"request": "Reported unintended stack during independent model-retirement work, November 2026"}, "notes": "Synthetic control: the user explicitly requests a dependent stack. Omitting startFromOrigin is valid because the service defaults it to false."}
{"id": "tt-independent-requested-base", "area": "orchestration", "kind": "guard", "pattern": "Another thread and PR must use thread workspace binding", "experience": "Independent-work defaults must preserve an explicitly requested base or a local-only repository.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. Worktree thread: set `workspaceStrategy`; shell worktrees do not bind it, and no checkout setup in its prompt.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine. Local release/1.0 exists; origin/main is the repository default.", "world": {"repo": "acme/app", "branch": "feature/error-display", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n", "AGENTS.md": "The repository default branch is main.\n"}}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Open another thread in its own worktree for an independent fix to model retirement. Base it on release/1.0, not main.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"workspaceStrategy.type": "worktree", "workspaceStrategy.baseRef": "release/1.0", "message": "~retir"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "shellMustNot": ["git\\s+worktree\\s+add"], "callsMustNot": [{"tool": "t3_thread_launch", "args": {"message": "~git worktree add"}}], "newThreads": ["~retir"]}, "source": {"request": "Reported unintended stack during independent model-retirement work, November 2026"}, "notes": "Synthetic guard for existing worktree behavior."}
{"id": "tt-independent-local-default-no-remote", "area": "orchestration", "kind": "guard", "pattern": "Another thread and PR must use thread workspace binding", "experience": "Independent-work defaults must preserve an explicitly requested base or a local-only repository.", "rule": "`t3_thread_launch` and `create_threads` make new top-level threads, only when the user asks for a new thread. \"Subagent\", \"delegate\", and \"parallel\" never mean one. Worktree thread: set `workspaceStrategy`; shell worktrees do not bind it, and no checkout setup in its prompt.", "privacy": "commit-safe", "setup": {"role": "top", "context": "Repository: /home/user/app (acme/app). This is the local machine. Main is the local default branch. No Git remotes are configured.", "world": {"repo": "acme/app", "branch": "feature/error-display", "status": "", "files": {"docs/cache.md": "Cache exports must preserve keys.\n", "src/thread-pool.ts": "export const workerCount = 4;\n", "src/task-queue.ts": "export const queueLimit = 10;\n", "src/agent-config.ts": "export const agentLabel = \"Assistant\";\n", "AGENTS.md": "The repository default branch is main.\n"}, "remotes": []}, "mocks": {"t3_thread_wait": {"threadId": "11111111-1111-4111-8111-111111111111", "runId": "run-acme-thread", "status": "running", "timedOut": true, "next": "Completion wakes this thread, so end your turn. Do not wait on it or poll it."}}}, "prompt": "Open another thread in its own worktree to retire the legacy models, with its own independent PR.", "expect": {"calls": [{"tool": "t3_thread_launch", "args": {"workspaceStrategy.type": "worktree", "workspaceStrategy.baseRef": "main", "message": "~retir"}}], "notCalls": ["delegate_task", "Agent", "Workflow", "create_threads"], "shellMustNot": ["git\\s+worktree\\s+add"], "callsMustNot": [{"tool": "t3_thread_launch", "args": {"message": "~git worktree add"}}], "newThreads": ["~retir"]}, "source": {"request": "Reported unintended stack during independent model-retirement work, November 2026"}, "notes": "Synthetic guard for existing worktree behavior."}
1 change: 1 addition & 0 deletions evals/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"mcp-tool-use:trim-capture": "node mcp-tool-use/trim-capture.ts",
"mcp-tool-use:regrade": "node mcp-tool-use/regrade.ts",
"thread-titles": "node --env-file-if-exists=../.env thread-titles/run.ts",
"pr-content": "node --env-file-if-exists=../.env pr-content/run.ts",
"voice-cleanup": "node --env-file-if-exists=../.env voice-cleanup/run.ts",
"test-signal": "node --env-file-if-exists=../.env test-signal/run.ts",
"typecheck": "effect-tsgo tsrs --noEmit",
Expand Down
Loading
Loading