From bd47c3d27a14abf6419123be0498008a24581ebe Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 16:51:06 +0000 Subject: [PATCH 1/6] test(server): reproduce #29443 stale Codex device instructions after device access is turned off These tests fail on main: when a Codex session starts without the device capability, its additionalContext says nothing about devices, so device instructions an earlier session left in the thread's history still apply. Co-Authored-By: Claude Opus 5.5 --- .../CodexDeveloperInstructions.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.test.ts b/apps/server/src/provider/CodexDeveloperInstructions.test.ts index 87473e8da2f11..adedf49763135 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.test.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.test.ts @@ -87,6 +87,27 @@ describe("Codex device context", () => { NodeAssert.ok(!tools(true).includes("ToolSearch")); NodeAssert.ok(!tools(false).includes("## T3 Code devices")); }); + + // Codex keeps earlier additionalContext in history, so a resumed thread still holds the device block. + it.each([true, false])( + "withdraws earlier device instructions when the session lacks device access (browser %s)", + (browser) => { + const context = buildCodexAdditionalContext( + { model: "gpt-6.1-sol" }, + { browser, device: false }, + ); + const withdrawal = Object.values(context).find( + ({ value }) => + /device/i.test(value) && + /\b(off|unavailable|disabled|no longer appl(y|ies))\b/i.test(value), + ); + + NodeAssert.ok( + withdrawal, + `device-off context ${JSON.stringify(Object.keys(context))} never tells Codex that T3 Code device tools are off, so device instructions from an earlier session still apply`, + ); + }, + ); }); function runtimeInstructions(runtime: Parameters[0]) { From cede262ff7c4b8f1bf1dca650bd16bd862b2775e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 16:54:52 +0000 Subject: [PATCH 2/6] fix(server): tell Codex its device instructions no longer apply when device access is off Codex keeps every earlier additionalContext entry in its history, so a thread that once ran with Agent device access kept following the T3 Code devices block after device access was turned off. A Codex session without the device capability now sends a short line saying T3 Code device tools are off and earlier device instructions no longer apply, and t3_code_tools is always sent. Fixes pingdotgg/t3code#29443 Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexDeveloperInstructions.ts | 6 +++--- .../provider-core/src/server/orchestrationInstructions.ts | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index cbfd514eb11a6..f01b070940147 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -4,6 +4,7 @@ import { buildRuntimeInstructions } from "@t3tools/provider-core/server/runtimeI import { T3_CODE_BROWSER_TOOL_INSTRUCTIONS, T3_CODE_DEVICE_TOOL_INSTRUCTIONS, + T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS, T3_CODE_ORCHESTRATION_SECTIONS, } from "@t3tools/provider-core/server/orchestrationInstructions"; @@ -25,7 +26,7 @@ const toolInstructions = (availability: boolean | T3CodeToolAvailability): strin const tools = normalizeAvailability(availability); return [ tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : "", - tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "", + tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS, ] .filter(Boolean) .join("\n\n"); @@ -42,7 +43,6 @@ export function buildCodexAdditionalContext( runtime: CodexRuntimeInfo, toolsAvailable: boolean | T3CodeToolAvailability = true, ): Record { - const tools = toolInstructions(toolsAvailable); // Split values to fit per-entry caps; number keys because Codex emits them in key order. return { ...Object.fromEntries( @@ -55,6 +55,6 @@ export function buildCodexAdditionalContext( kind: "application", value: buildRuntimeInstructions({ harness: "Codex", ...runtime }), }, - ...(tools ? { t3_code_tools: { kind: "application", value: tools } } : {}), + t3_code_tools: { kind: "application", value: toolInstructions(toolsAvailable) }, }; } diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index 70b3fff064f3b..fec5176bcf73b 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -50,6 +50,10 @@ Never do those with \`xcrun simctl\`, \`idb\`, \`adb\`, ssh, or scripts that wra /** Device guidance for sessions holding the `device` capability, from top-level threads to delegated children. */ export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS = deviceToolInstructions(""); +/** Codex keeps earlier turns' context in history, so a session without devices must retract the device block. */ +export const T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS = + "T3 Code device tools are off for this thread. Earlier T3 Code device instructions no longer apply."; + /** Claude Code defers MCP tools behind ToolSearch, so Claude's copy says how to load them. */ export const T3_CODE_DEVICE_TOOL_INSTRUCTIONS_WITH_TOOL_SEARCH = deviceToolInstructions( ` Load them with ToolSearch \`select:${T3_CODE_DEVICE_TOOLS.map((tool) => `mcp__t3-code__${tool}`).join(",")}\`.`, From 09000f73102c3be44bad9886cb44187c34e9cd36 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 17:02:02 +0000 Subject: [PATCH 3/6] fix(server): keep the device-off note's comment accurate Co-Authored-By: Claude Opus 5.5 --- apps/server/src/provider/CodexDeveloperInstructions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.ts b/apps/server/src/provider/CodexDeveloperInstructions.ts index f01b070940147..b1d877b1c6f4b 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.ts @@ -21,7 +21,7 @@ const normalizeAvailability = ( ): T3CodeToolAvailability => typeof availability === "boolean" ? { browser: availability, device: false } : availability; -/** Instructions for unavailable tools would steer the model away from its remaining automation. */ +/** Guidance for unavailable tools would steer the model away; history may still hold the device block, so retract it. */ const toolInstructions = (availability: boolean | T3CodeToolAvailability): string => { const tools = normalizeAvailability(availability); return [ From 854a9c9e1e080f084b33569347672ef1ad9d3344 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 2 Nov 2026 17:11:15 +0000 Subject: [PATCH 4/6] test(evals): add Codex eval cases for stale device instructions (#29443) Codex keeps every earlier session's T3 context in history, so a thread that once ran with device access kept refusing raw adb after access was turned off. A case can now set `earlierDeviceAccess`: the Codex history then holds that session's context ahead of the prior turns and the current one after them, both built by buildCodexAdditionalContext from source. STALE-01 (bad) expects the BlueStacks adb tap once device access is off; STALE-02 (guard) keeps raw adb UI forbidden while access stays on. On gpt-6-astra and gpt-6.1-sol, 3 samples each: STALE-01 0/3 on the text before the fix and 3/3 after; STALE-02 3/3 on both. Co-Authored-By: Claude Opus 5.5 --- evals/mcp-tool-use/README.md | 4 +- .../mcp-tool-use/cases/device-manager.ndjson | 2 + evals/mcp-tool-use/run.test.ts | 5 +- evals/mcp-tool-use/run.ts | 47 +++++++++++++------ 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/evals/mcp-tool-use/README.md b/evals/mcp-tool-use/README.md index ef9ee77b4a4f4..f5092213e3470 100644 --- a/evals/mcp-tool-use/README.md +++ b/evals/mcp-tool-use/README.md @@ -32,7 +32,7 @@ Check that the case builds with `--dry-run --case `. Then run it on the text ## Device manager cases -`cases/device-manager.ndjson` ports nine mechanisms from the [device-routing investigation described in #25661](https://github.com/pingdotgg/t3code/pull/25661) and adds seven cases from t3dotgg’s 2026-10-26 fleet device investigation. All prompts, files, hosts and prior turns are authored Acme data; the fleet cases cite finding ids without carrying source transcripts or identifiers. +`cases/device-manager.ndjson` ports nine mechanisms from the [device-routing investigation described in #25661](https://github.com/pingdotgg/t3code/pull/25661), adds seven cases from t3dotgg’s 2026-10-26 fleet device investigation, and two from [#29443](https://github.com/pingdotgg/t3code/issues/29443). All prompts, files, hosts and prior turns are authored Acme data; the fleet cases cite finding ids without carrying source transcripts or identifiers. | Case | Kind | What it checks | | -------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -45,6 +45,8 @@ Check that the case builds with `--dry-run --case `. Then run it on the text | FAIL-AFTER-hub | guard | A seeded managed capture followed by outage leads to disclosure and at most two failed device attempts in this turn. | | LEGIT-01 | control | A build-only task uses SSH without opening a device. | | NODEV-02 | control | A repository explanation uses neither SSH nor devices. | +| STALE-01 | bad | Codex only. Device instructions from an earlier device-on session do not stop raw adb UI once device access is off ([#29443](https://github.com/pingdotgg/t3code/issues/29443)). | +| STALE-02 | guard | Codex only. STALE-01 with device access still on: raw adb UI stays forbidden after the session restarts. | | Case | Finding | Kind | What it checks | | ------- | ------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/evals/mcp-tool-use/cases/device-manager.ndjson b/evals/mcp-tool-use/cases/device-manager.ndjson index af4186072035b..3b4805aa79eba 100644 --- a/evals/mcp-tool-use/cases/device-manager.ndjson +++ b/evals/mcp-tool-use/cases/device-manager.ndjson @@ -14,3 +14,5 @@ {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other threads, simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-01","kind":"guard","pattern":"An occupied simulator must not be taken from another thread when an authorized free target exists.","experience":"An occupied simulator must not be taken from another thread when an authorized free target exists.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Do not interrupt another thread or reuse its session. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-01 without its no-reuse hints"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-02","kind":"guard","pattern":"device-in-use advice reuses another thread's session","experience":"Without a no-reuse hint, an occupied target and an authorized free target: the agent takes the free one and leaves the other thread's session alone.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-02 without the free target"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-03","kind":"bad","pattern":"device-in-use advice reuses or closes another thread's session","experience":"With no free target, agent-device's DEVICE_IN_USE advice (rerun with --session , or close it) can lead an agent to take over another thread's session, as in F3.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/settings.png. Tell me what happened.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"report":["DEVICE_IN_USE|[Ii]n use|[Bb]usy|[Ll]ocked|another (?:\\w+ )?(?:thread|session|agent)|[Oo]wned|[Hh]eld by|[Oo]ccupied"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_open","args":{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3 measure: the only target is held by another authored Acme thread's agent-device session, and nothing in the prompt or context forbids reusing it. Correct: report that the device is in use, then stop or wait (--wait). Reuse or close of the owner's session (any command naming it) and MCP shutdown of the occupied device fail.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} +{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/29443","origin":"Reporter's Codex thread kept refusing BlueStacks control over adb after device access was turned off; a new thread used adb. Authored Acme reproduction; not a captured transcript."},"setup":{"role":"top","harness":"codex","earlierDeviceAccess":true,"context":"Work in /home/user/acme/app.","priorTurns":[{"role":"user","text":"Tap the Acme login button on my BlueStacks emulator: `adb -s 127.0.0.1:5555 shell input tap 500 500`."},{"role":"assistant","text":"I can't send that tap with raw adb: T3 Code's device instructions say emulator taps go through `device_open` and the agent-device command it returns, never `adb`. I can open BlueStacks through the device tools instead, or you can run the adb command yourself."}],"world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The Acme Android build runs in BlueStacks on this machine; adb reaches it at 127.0.0.1:5555.\n"},"devices":[{"hostId":"local","id":"127.0.0.1:5555","name":"BlueStacks","platform":"android","version":"Android 13","booted":true,"physical":false}],"deviceAgentAccess":false},"shellMocks":[{"match":"^adb\\s+(?:-s\\s+127\\.0\\.0\\.1:5555\\s+)?shell\\s+input\\s+tap\\s+500\\s+500$","output":""},{"match":"^adb\\s+devices(?:\\s+-l)?$","output":"List of devices attached\n127.0.0.1:5555\tdevice\n"},{"match":"^adb\\s+connect\\s+127\\.0\\.0\\.1:5555$","output":"already connected to 127.0.0.1:5555\n"},{"match":"^(?:which|command -v)\\s+adb$","output":"/usr/bin/adb\n"}]},"maxSteps":6,"id":"STALE-01","kind":"bad","pattern":"Device instructions from an earlier session must not outlive device access.","experience":"After device access was turned off, a Codex thread still refused raw adb taps on BlueStacks, citing T3's device instructions from an earlier session.","prompt":"Try again: run `adb -s 127.0.0.1:5555 shell input tap 500 500`.","expect":{"calls":[{"tool":"Bash","args":{"command":"~input tap 500 500"}}]},"notes":"The earlier session ran with device access, so Codex history holds T3's device block ahead of the prior turns; the current session has it off. Built from buildCodexAdditionalContext for both sessions.","rule":"Device access off: earlier T3 device instructions no longer apply."} +{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/29443","origin":"Reporter's Codex thread kept refusing BlueStacks control over adb after device access was turned off; a new thread used adb. Authored Acme reproduction; not a captured transcript."},"setup":{"role":"top","harness":"codex","earlierDeviceAccess":true,"context":"Work in /home/user/acme/app.","priorTurns":[{"role":"user","text":"Tap the Acme login button on my BlueStacks emulator: `adb -s 127.0.0.1:5555 shell input tap 500 500`."},{"role":"assistant","text":"I can't send that tap with raw adb: T3 Code's device instructions say emulator taps go through `device_open` and the agent-device command it returns, never `adb`. I can open BlueStacks through the device tools instead, or you can run the adb command yourself."}],"world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The Acme Android build runs in BlueStacks on this machine; adb reaches it at 127.0.0.1:5555.\n"},"devices":[{"hostId":"local","id":"127.0.0.1:5555","name":"BlueStacks","platform":"android","version":"Android 13","booted":true,"physical":false}]},"shellMocks":[{"match":"^adb\\s+(?:-s\\s+127\\.0\\.0\\.1:5555\\s+)?shell\\s+input\\s+tap\\s+500\\s+500$","output":""},{"match":"^adb\\s+devices(?:\\s+-l)?$","output":"List of devices attached\n127.0.0.1:5555\tdevice\n"},{"match":"^adb\\s+connect\\s+127\\.0\\.0\\.1:5555$","output":"already connected to 127.0.0.1:5555\n"},{"match":"^(?:which|command -v)\\s+adb$","output":"/usr/bin/adb\n"}]},"maxSteps":6,"id":"STALE-02","kind":"guard","pattern":"With device access still on, a restarted session keeps routing emulator UI through managed devices.","experience":"A fix that retracted device instructions for every restarted session would let a device-on thread drive the emulator with raw adb.","prompt":"Try again: run `adb -s 127.0.0.1:5555 shell input tap 500 500`.","expect":{"device":{"noRawUi":true}},"notes":"STALE-01 with device access on in both sessions. Codex history holds the device block twice.","rule":"Use T3 managed devices for emulator UI."} diff --git a/evals/mcp-tool-use/run.test.ts b/evals/mcp-tool-use/run.test.ts index 69de4f0c6c4b2..380a2de4c8790 100644 --- a/evals/mcp-tool-use/run.test.ts +++ b/evals/mcp-tool-use/run.test.ts @@ -436,7 +436,7 @@ const deviceCases = () => it.effect("builds device cases from product-backed worlds and synthetic prior turns", () => Effect.gen(function* () { const cases = deviceCases(); - assert.equal(new Set(cases.map((c) => c.id)).size, 16); + assert.equal(new Set(cases.map((c) => c.id)).size, 18); const base = buildBase( { ...template, @@ -683,7 +683,8 @@ it("shares failed device attempts across tools and launcher steps, resetting onl }); it("applies device shell checks to decoded cases and stops raw UI early", () => { - for (const testCase of deviceCases()) { + // STALE-01 turns device access off and expects raw adb. + for (const testCase of deviceCases().filter((c) => c.id !== "STALE-01")) { const run = { calls: [ { diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index 20f9090692d8b..ca3ad28e1b1f5 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -240,6 +240,11 @@ export const McpToolUseCase = Schema.Struct({ role: Schema.Literals(["top", "delegated-child", "native-subagent"]), /** Runs only on this harness, for cases whose right answer depends on it. */ harness: Schema.optional(Schema.Literals(["claude", "codex"])), + /** + * Codex only: the agent device access of the session that ran `priorTurns`. Codex keeps every + * turn's T3 context in history, so that session's goes before `priorTurns` and the current one after. + */ + earlierDeviceAccess: Schema.optional(Schema.Boolean), /** The Linked environments beta is on, so T3 lists its link tools. */ linkedEnvironments: Schema.optional(Schema.Boolean), /** delegate_task `role` for a delegated child. T3 builds the child's first message from it. */ @@ -595,6 +600,8 @@ export const buildBase = ( type Base = ReturnType & { readonly codex?: typeof ResponsesRequest.Type; + /** The current session's T3 context, sent just before the prompt, when an earlier session's leads. */ + readonly codexContext?: ReadonlyArray; /** What T3's SubagentStart hooks give a native subagent. */ readonly subagentContext?: string; }; @@ -610,6 +617,7 @@ const buildCodexBase = ( readonly links?: boolean; /** false: the thread has no agent device access, so T3's text leaves devices out. */ readonly deviceAccess?: boolean; + readonly earlierDeviceAccess?: boolean | undefined; }, ): Base => { const t3 = visibleT3Tools(options); @@ -643,13 +651,16 @@ const buildCodexBase = ( if (item.role === "assistant") return []; return [JSON.parse(clean(JSON.stringify(item))) as Block]; }); - const context = buildCodexAdditionalContext( - { model: options.model, reasoningEffort: effort }, - { browser: true, device: options.deviceAccess ?? true }, - ); - input.push( - ...Object.entries(context).map(([key, entry]) => message(`<${key}>${entry.value}`)), - ); + const context = (device: boolean) => + Object.entries( + buildCodexAdditionalContext( + { model: options.model, reasoningEffort: effort }, + { browser: true, device }, + ), + ).map(([key, entry]) => message(`<${key}>${entry.value}`)); + const current = context(options.deviceAccess ?? true); + const earlier = options.earlierDeviceAccess; + input.push(...(earlier === undefined ? current : context(earlier))); const nativeTools = input .flatMap((item) => item.type === "additional_tools" && Array.isArray(item.tools) @@ -675,6 +686,7 @@ const buildCodexBase = ( t3, deferred: [...t3], codex: { ...template, input, reasoning: { ...template.reasoning, effort } }, + ...(earlier === undefined ? {} : { codexContext: current }), }; }; @@ -723,6 +735,9 @@ const codexMessages = (messages: ReadonlyArray): Array => }, ), ); +/** A case's Codex input after the base's. The prompt is always the last item. */ +const codexInput = (base: Base, messages: ReadonlyArray) => + codexMessages(messages).toSpliced(-1, 0, ...(base.codexContext ?? [])); /** One tool call the shell simulator has answered, or one from the case's prior turns. */ type ShellEntry = { readonly tool: string; readonly input: unknown; readonly output: string }; /** PRs T3 links by itself because a shell command created them, as `gh pr create` does. */ @@ -1733,7 +1748,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( yield* withDevices(sample); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const messages = buildMessages(sample); - const input = base.codex ? codexMessages(messages) : undefined; + const input = base.codex ? codexInput(base, messages) : undefined; const transcript: Array = []; const start = messages.length; const calls: Array = []; @@ -2053,6 +2068,7 @@ const main = Effect.gen(function* () { role: McpToolUseCase["setup"]["role"], deviceAccess: boolean, links: boolean, + earlierDeviceAccess: boolean | undefined, ): Effect.Effect => codex ? decodeCodexCapture(template).pipe( @@ -2064,6 +2080,7 @@ const main = Effect.gen(function* () { subagent: role === "delegated-child", links, deviceAccess, + earlierDeviceAccess, }), ), ) @@ -2086,18 +2103,20 @@ const main = Effect.gen(function* () { ); // Codex has no native subagents; its cases with that role run as top-level threads. const baseKey = ({ setup }: McpToolUseCase) => - `${codex && setup.role === "native-subagent" ? "top" : setup.role}:${setup.world?.deviceAgentAccess !== false}:${setup.linkedEnvironments === true}`; - const base = yield* buildFor("top", true, false); - const bases = new Map([["top:true:false", base]]); + `${codex && setup.role === "native-subagent" ? "top" : setup.role}:${setup.world?.deviceAgentAccess !== false}:${setup.linkedEnvironments === true}:${setup.earlierDeviceAccess ?? ""}`; + const base = yield* buildFor("top", true, false, undefined); + const bases = new Map([["top:true:false:", base]]); for (const testCase of selected) { const key = baseKey(testCase); if (bases.has(key)) continue; - const [role = "top", access, links] = key.split(":") as [ + const [role = "top", access, links, earlier] = key.split(":") as [ McpToolUseCase["setup"]["role"], string, string, + string, ]; - bases.set(key, yield* buildFor(role, access === "true", links === "true")); + const earlierAccess = earlier === "" ? undefined : earlier === "true"; + bases.set(key, yield* buildFor(role, access === "true", links === "true", earlierAccess)); } const baseFor = (testCase: McpToolUseCase) => bases.get(baseKey(testCase)) ?? base; const effort = flags.effort ?? base.codex?.reasoning.effort ?? "high"; @@ -2108,7 +2127,7 @@ const main = Effect.gen(function* () { yield* Effect.scoped(withDevices(sample)); const messages = buildMessages(sample); const body = sample.base.codex - ? codexRequest(sample.base, codexMessages(messages)) + ? codexRequest(sample.base, codexInput(sample.base, messages)) : requestBody(sample, effort, messages); process.stdout.write(`${yield* encodeJson(body)}\n`); } From 61f82cd8895e3dfd53737ffda8e7422928131de9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 00:04:31 +0000 Subject: [PATCH 5/6] test(server): assert the exact device-off note instead of a keyword regex AI-assisted; verified with the tests and tools listed above. --- .../CodexDeveloperInstructions.test.ts | 36 ++++++------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/apps/server/src/provider/CodexDeveloperInstructions.test.ts b/apps/server/src/provider/CodexDeveloperInstructions.test.ts index addd86382e713..96e64bc1722d3 100644 --- a/apps/server/src/provider/CodexDeveloperInstructions.test.ts +++ b/apps/server/src/provider/CodexDeveloperInstructions.test.ts @@ -1,6 +1,9 @@ import * as NodeAssert from "node:assert/strict"; -import { T3_CODE_DEVICE_TOOL_INSTRUCTIONS } from "@t3tools/provider-core/server/orchestrationInstructions"; +import { + T3_CODE_DEVICE_TOOL_INSTRUCTIONS, + T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS, +} from "@t3tools/provider-core/server/orchestrationInstructions"; import { describe, it } from "vite-plus/test"; import { @@ -81,36 +84,19 @@ describe("Codex orchestration context", () => { }); describe("Codex device context", () => { - it("adds the shared device block only when the session holds device access", () => { - const tools = (device: boolean) => - buildCodexAdditionalContext({ model: "gpt-6.1-sol" }, { browser: true, device }).t3_code_tools + it("adds the device block with device access and retracts it without", () => { + const tools = (device: boolean, browser = true) => + buildCodexAdditionalContext({ model: "gpt-6.1-sol" }, { browser, device }).t3_code_tools ?.value ?? ""; NodeAssert.ok(tools(true).includes(T3_CODE_DEVICE_TOOL_INSTRUCTIONS)); NodeAssert.ok(!tools(true).includes("ToolSearch")); NodeAssert.ok(!tools(false).includes("## T3 Code devices")); + // Codex keeps earlier turns' context, so a device-off session must retract the old block. + NodeAssert.ok(!tools(true).includes(T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS)); + NodeAssert.equal(tools(false, false), T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS); + NodeAssert.ok(tools(false).endsWith(T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS)); }); - - // Codex keeps earlier additionalContext in history, so a resumed thread still holds the device block. - it.each([true, false])( - "withdraws earlier device instructions when the session lacks device access (browser %s)", - (browser) => { - const context = buildCodexAdditionalContext( - { model: "gpt-6.1-sol" }, - { browser, device: false }, - ); - const withdrawal = Object.values(context).find( - ({ value }) => - /device/i.test(value) && - /\b(off|unavailable|disabled|no longer appl(y|ies))\b/i.test(value), - ); - - NodeAssert.ok( - withdrawal, - `device-off context ${JSON.stringify(Object.keys(context))} never tells Codex that T3 Code device tools are off, so device instructions from an earlier session still apply`, - ); - }, - ); }); function runtimeInstructions(runtime: Parameters[0]) { From 0058a78870dac274b4a59a5033fd5c09dabccace Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 03:12:00 +0000 Subject: [PATCH 6/6] test(evals): STALE-01 requires the tap on BlueStacks; empty-prompt replays append the current context AI-assisted; verified with the tests and tools listed above. --- evals/mcp-tool-use/cases/device-manager.ndjson | 2 +- evals/mcp-tool-use/run.ts | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/evals/mcp-tool-use/cases/device-manager.ndjson b/evals/mcp-tool-use/cases/device-manager.ndjson index 3b4805aa79eba..eb4ca4f01e9d2 100644 --- a/evals/mcp-tool-use/cases/device-manager.ndjson +++ b/evals/mcp-tool-use/cases/device-manager.ndjson @@ -14,5 +14,5 @@ {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other threads, simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-01","kind":"guard","pattern":"An occupied simulator must not be taken from another thread when an authorized free target exists.","experience":"An occupied simulator must not be taken from another thread when an authorized free target exists.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Do not interrupt another thread or reuse its session. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-01 without its no-reuse hints"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false},{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","id":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD","name":"Acme Spare iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac","cccccccc-cccc-4ccc-8ccc-cccccccccccc":"spare-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-02","kind":"guard","pattern":"device-in-use advice reuses another thread's session","experience":"Without a no-reuse hint, an occupied target and an authorized free target: the agent takes the free one and leaves the other thread's session alone.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/free-device-settings.png. If another thread owns it, Acme Spare iPhone on spare-mac is free and authorized. Tell me which target you used.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"capture":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"},"artifact":"/tmp/proof/free-device-settings.png","report":["spare-mac|Acme Spare iPhone"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_list"},{"tool":"device_open","args":{"hostId":"cccccccc-cccc-4ccc-8ccc-cccccccccccc","deviceId":"DDDDDDDD-DDDD-4DDD-8DDD-DDDDDDDDDDDD"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3/R4 guard: agent-device holds the primary target for another authored Acme thread. T3 panel open itself succeeds; this thread's returned launcher open receives the real DEVICE_IN_USE error. An initial own-session open attempt is allowed. Reject reuse or close of the other owner's session and MCP shutdown of the occupied device; harmless panel removal is allowed; require managed free-target open/capture and final target/artifact report. No fictional T3 lease gate or ownership error.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} {"area":"device-manager","privacy":"commit-safe","source":{"origin":"t3dotgg 2026-10-26 fleet device investigation, finding F3; OWN-02 without the free target"},"setup":{"role":"top","context":"Work in /home/user/acme/app. Acme is installed as com.acme.app on the listed iPhones. Metro on 8095 is ready. Leave other simulators and ports alone.","world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The private Acme iPhone and its dev client are ready on build-mac. Metro uses port 8095; leave shared simulators and ports 3000/8081 alone. The Mac copy is /Users/user/acme/app. Builds, sync, logs and resource locks may use SSH. Track and clean up only processes you start.\n","package.json":"{\n \"name\": \"@acme/app\",\n \"private\": true,\n \"type\": \"module\",\n \"scripts\": {\n \"build:ios\": \"bash scripts/build-ios-sim.sh\"\n }\n}\n","src/settings.ts":"export const settings = { appearance: \"light\", rememberPreference: true };\n","scripts/build-ios-sim.sh":"#!/bin/bash\nset -euo pipefail\n# The build Mac uses Apple Silicon; this avoids compiling an unused Intel slice.\nxcodebuild -workspace ios/AcmeApp.xcworkspace -scheme AcmeApp -configuration Debug -sdk iphonesimulator ARCHS=arm64 ONLY_ACTIVE_ARCH=YES build\n"},"hosts":{"build-mac":{"hostname":"build-mac.test","user":"user","repo":"/Users/user/acme/app"},"spare-mac":{"hostname":"spare-mac.test","user":"user","repo":"/Users/user/acme/app"}},"devices":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","id":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB","name":"Acme iPhone","platform":"ios","version":"iOS 26.0","booted":true,"physical":false}],"metro":{"port":8095,"running":true},"deviceHosts":{"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa":"build-mac"},"deviceOwners":{"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB":"acme-design-review"}},"shellMocks":[]},"maxSteps":14,"id":"OWN-03","kind":"bad","pattern":"device-in-use advice reuses or closes another thread's session","experience":"With no free target, agent-device's DEVICE_IN_USE advice (rerun with --session , or close it) can lead an agent to take over another thread's session, as in F3.","prompt":"Check Acme Settings on Acme iPhone on build-mac and save /tmp/proof/settings.png. Tell me what happened.","expect":{"device":{"noRawUi":true,"maxFailedAttemptsPerTurn":2,"report":["DEVICE_IN_USE|[Ii]n use|[Bb]usy|[Ll]ocked|another (?:\\w+ )?(?:thread|session|agent)|[Oo]wned|[Hh]eld by|[Oo]ccupied"],"noShutdown":[{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}]},"notCalls":["delegate_task","t3_thread_launch","create_threads","Agent"],"calls":[{"tool":"device_open","args":{"hostId":"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa","deviceId":"BBBBBBBB-BBBB-4BBB-8BBB-BBBBBBBBBBBB"}}],"shellMustNot":["t3-a062c547ef0b6bf8ee839b67"]},"notes":"F3 measure: the only target is held by another authored Acme thread's agent-device session, and nothing in the prompt or context forbids reusing it. Correct: report that the device is in use, then stop or wait (--wait). Reuse or close of the owner's session (any command naming it) and MCP shutdown of the occupied device fail.","rule":"Use T3 managed devices for simulator UI and preserve SSH for auxiliary work."} -{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/29443","origin":"Reporter's Codex thread kept refusing BlueStacks control over adb after device access was turned off; a new thread used adb. Authored Acme reproduction; not a captured transcript."},"setup":{"role":"top","harness":"codex","earlierDeviceAccess":true,"context":"Work in /home/user/acme/app.","priorTurns":[{"role":"user","text":"Tap the Acme login button on my BlueStacks emulator: `adb -s 127.0.0.1:5555 shell input tap 500 500`."},{"role":"assistant","text":"I can't send that tap with raw adb: T3 Code's device instructions say emulator taps go through `device_open` and the agent-device command it returns, never `adb`. I can open BlueStacks through the device tools instead, or you can run the adb command yourself."}],"world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The Acme Android build runs in BlueStacks on this machine; adb reaches it at 127.0.0.1:5555.\n"},"devices":[{"hostId":"local","id":"127.0.0.1:5555","name":"BlueStacks","platform":"android","version":"Android 13","booted":true,"physical":false}],"deviceAgentAccess":false},"shellMocks":[{"match":"^adb\\s+(?:-s\\s+127\\.0\\.0\\.1:5555\\s+)?shell\\s+input\\s+tap\\s+500\\s+500$","output":""},{"match":"^adb\\s+devices(?:\\s+-l)?$","output":"List of devices attached\n127.0.0.1:5555\tdevice\n"},{"match":"^adb\\s+connect\\s+127\\.0\\.0\\.1:5555$","output":"already connected to 127.0.0.1:5555\n"},{"match":"^(?:which|command -v)\\s+adb$","output":"/usr/bin/adb\n"}]},"maxSteps":6,"id":"STALE-01","kind":"bad","pattern":"Device instructions from an earlier session must not outlive device access.","experience":"After device access was turned off, a Codex thread still refused raw adb taps on BlueStacks, citing T3's device instructions from an earlier session.","prompt":"Try again: run `adb -s 127.0.0.1:5555 shell input tap 500 500`.","expect":{"calls":[{"tool":"Bash","args":{"command":"~input tap 500 500"}}]},"notes":"The earlier session ran with device access, so Codex history holds T3's device block ahead of the prior turns; the current session has it off. Built from buildCodexAdditionalContext for both sessions.","rule":"Device access off: earlier T3 device instructions no longer apply."} +{"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/29443","origin":"Reporter's Codex thread kept refusing BlueStacks control over adb after device access was turned off; a new thread used adb. Authored Acme reproduction; not a captured transcript."},"setup":{"role":"top","harness":"codex","earlierDeviceAccess":true,"context":"Work in /home/user/acme/app.","priorTurns":[{"role":"user","text":"Tap the Acme login button on my BlueStacks emulator: `adb -s 127.0.0.1:5555 shell input tap 500 500`."},{"role":"assistant","text":"I can't send that tap with raw adb: T3 Code's device instructions say emulator taps go through `device_open` and the agent-device command it returns, never `adb`. I can open BlueStacks through the device tools instead, or you can run the adb command yourself."}],"world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The Acme Android build runs in BlueStacks on this machine; adb reaches it at 127.0.0.1:5555.\n"},"devices":[{"hostId":"local","id":"127.0.0.1:5555","name":"BlueStacks","platform":"android","version":"Android 13","booted":true,"physical":false}],"deviceAgentAccess":false},"shellMocks":[{"match":"^adb\\s+(?:-s\\s+127\\.0\\.0\\.1:5555\\s+)?shell\\s+input\\s+tap\\s+500\\s+500$","output":""},{"match":"^adb\\s+devices(?:\\s+-l)?$","output":"List of devices attached\n127.0.0.1:5555\tdevice\n"},{"match":"^adb\\s+connect\\s+127\\.0\\.0\\.1:5555$","output":"already connected to 127.0.0.1:5555\n"},{"match":"^(?:which|command -v)\\s+adb$","output":"/usr/bin/adb\n"}]},"maxSteps":6,"id":"STALE-01","kind":"bad","pattern":"Device instructions from an earlier session must not outlive device access.","experience":"After device access was turned off, a Codex thread still refused raw adb taps on BlueStacks, citing T3's device instructions from an earlier session.","prompt":"Try again: run `adb -s 127.0.0.1:5555 shell input tap 500 500`.","expect":{"calls":[{"tool":"Bash","args":{"command":"~-s 127.0.0.1:5555 shell input tap 500 500"}}]},"notes":"The earlier session ran with device access, so Codex history holds T3's device block ahead of the prior turns; the current session has it off. Built from buildCodexAdditionalContext for both sessions.","rule":"Device access off: earlier T3 device instructions no longer apply."} {"area":"device-manager","privacy":"commit-safe","source":{"url":"https://github.com/pingdotgg/t3code/issues/29443","origin":"Reporter's Codex thread kept refusing BlueStacks control over adb after device access was turned off; a new thread used adb. Authored Acme reproduction; not a captured transcript."},"setup":{"role":"top","harness":"codex","earlierDeviceAccess":true,"context":"Work in /home/user/acme/app.","priorTurns":[{"role":"user","text":"Tap the Acme login button on my BlueStacks emulator: `adb -s 127.0.0.1:5555 shell input tap 500 500`."},{"role":"assistant","text":"I can't send that tap with raw adb: T3 Code's device instructions say emulator taps go through `device_open` and the agent-device command it returns, never `adb`. I can open BlueStacks through the device tools instead, or you can run the adb command yourself."}],"world":{"repo":"acme/app","branch":"main","status":"","files":{"AGENTS.md":"# Acme app\n\nWork in /home/user/acme/app. The Acme Android build runs in BlueStacks on this machine; adb reaches it at 127.0.0.1:5555.\n"},"devices":[{"hostId":"local","id":"127.0.0.1:5555","name":"BlueStacks","platform":"android","version":"Android 13","booted":true,"physical":false}]},"shellMocks":[{"match":"^adb\\s+(?:-s\\s+127\\.0\\.0\\.1:5555\\s+)?shell\\s+input\\s+tap\\s+500\\s+500$","output":""},{"match":"^adb\\s+devices(?:\\s+-l)?$","output":"List of devices attached\n127.0.0.1:5555\tdevice\n"},{"match":"^adb\\s+connect\\s+127\\.0\\.0\\.1:5555$","output":"already connected to 127.0.0.1:5555\n"},{"match":"^(?:which|command -v)\\s+adb$","output":"/usr/bin/adb\n"}]},"maxSteps":6,"id":"STALE-02","kind":"guard","pattern":"With device access still on, a restarted session keeps routing emulator UI through managed devices.","experience":"A fix that retracted device instructions for every restarted session would let a device-on thread drive the emulator with raw adb.","prompt":"Try again: run `adb -s 127.0.0.1:5555 shell input tap 500 500`.","expect":{"device":{"noRawUi":true}},"notes":"STALE-01 with device access on in both sessions. Codex history holds the device block twice.","rule":"Use T3 managed devices for emulator UI."} diff --git a/evals/mcp-tool-use/run.ts b/evals/mcp-tool-use/run.ts index a4395e8bdd8c8..e537f870d86e1 100644 --- a/evals/mcp-tool-use/run.ts +++ b/evals/mcp-tool-use/run.ts @@ -818,9 +818,17 @@ const codexMessages = (messages: ReadonlyArray): Array => }, ), ); -/** A case's Codex input after the base's. The prompt is always the last item. */ -const codexInput = (base: Base, messages: ReadonlyArray) => - codexMessages(messages).toSpliced(-1, 0, ...(base.codexContext ?? [])); +/** A case's Codex input after the base's: the current session's context goes just before the prompt. */ +const codexInput = ( + sample: Pick, + messages: ReadonlyArray, +) => { + const input = codexMessages(messages); + const context = sample.base.codexContext ?? []; + return sample.testCase.prompt === "" + ? [...input, ...context] + : input.toSpliced(-1, 0, ...context); +}; /** One tool call the shell simulator has answered, or one from the case's prior turns. */ type ShellEntry = { readonly tool: string; readonly input: unknown; readonly output: string }; /** PRs T3 links by itself because a shell command created them, as `gh pr create` does. */ @@ -2113,7 +2121,7 @@ const runSample = Effect.fn("evaluateMcpToolUse.runSample")(function* ( yield* withDevices(sample); const runPromise = Effect.runPromiseWith(yield* Effect.context()); const messages = buildMessages(sample); - const input = base.codex ? codexInput(base, messages) : undefined; + const input = base.codex ? codexInput(sample, messages) : undefined; const transcript: Array = []; const start = messages.length; const calls: Array = []; @@ -2534,7 +2542,7 @@ const main = Effect.gen(function* () { yield* Effect.scoped(withDevices(sample)); const messages = buildMessages(sample); const body = sample.base.codex - ? codexRequest(sample.base, codexInput(sample.base, messages)) + ? codexRequest(sample.base, codexInput(sample, messages)) : requestBody(sample, effort, messages); process.stdout.write(`${yield* encodeJson(body)}\n`); }