Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions apps/server/src/provider/CodexDeveloperInstructions.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import * as NodeAssert from "node:assert/strict";

import { T3_CODE_DEVICE_TOOL_INSTRUCTIONS } from "@t3tools/provider-core/server/orchestrationInstructions";
import {
T3_CODE_DEVICE_TOOL_INSTRUCTIONS,
T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS,
} from "@t3tools/provider-core/server/orchestrationInstructions";
import { describe, it } from "vite-plus/test";

import {
Expand Down Expand Up @@ -81,14 +84,18 @@ describe("Codex orchestration context", () => {
});

describe("Codex device context", () => {
it("adds the shared device block only when the session holds device access", () => {
const tools = (device: boolean) =>
buildCodexAdditionalContext({ model: "gpt-6.1-sol" }, { browser: true, device }).t3_code_tools
it("adds the device block with device access and retracts it without", () => {
const tools = (device: boolean, browser = true) =>
buildCodexAdditionalContext({ model: "gpt-6.1-sol" }, { browser, device }).t3_code_tools
?.value ?? "";

NodeAssert.ok(tools(true).includes(T3_CODE_DEVICE_TOOL_INSTRUCTIONS));
NodeAssert.ok(!tools(true).includes("ToolSearch"));
NodeAssert.ok(!tools(false).includes("## T3 Code devices"));
// Codex keeps earlier turns' context, so a device-off session must retract the old block.
NodeAssert.ok(!tools(true).includes(T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS));
NodeAssert.equal(tools(false, false), T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS);
NodeAssert.ok(tools(false).endsWith(T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS));
});
});

Expand Down
8 changes: 4 additions & 4 deletions apps/server/src/provider/CodexDeveloperInstructions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { buildRuntimeInstructions } from "@t3tools/provider-core/server/runtimeI
import {
T3_CODE_BROWSER_TOOL_INSTRUCTIONS,
T3_CODE_DEVICE_TOOL_INSTRUCTIONS,
T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS,
T3_CODE_ORCHESTRATION_SECTIONS,
} from "@t3tools/provider-core/server/orchestrationInstructions";

Expand All @@ -20,12 +21,12 @@ const normalizeAvailability = (
): T3CodeToolAvailability =>
typeof availability === "boolean" ? { browser: availability, device: false } : availability;

/** Instructions for unavailable tools would steer the model away from its remaining automation. */
/** Guidance for unavailable tools would steer the model away; history may still hold the device block, so retract it. */
const toolInstructions = (availability: boolean | T3CodeToolAvailability): string => {
const tools = normalizeAvailability(availability);
return [
tools.browser ? T3_CODE_BROWSER_TOOL_INSTRUCTIONS : "",
tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : "",
tools.device ? T3_CODE_DEVICE_TOOL_INSTRUCTIONS : T3_CODE_DEVICE_TOOLS_OFF_INSTRUCTIONS,
]
.filter(Boolean)
.join("\n\n");
Expand All @@ -42,7 +43,6 @@ export function buildCodexAdditionalContext(
runtime: CodexRuntimeInfo,
toolsAvailable: boolean | T3CodeToolAvailability = true,
): Record<string, V2TurnStartParams__AdditionalContextEntry> {
const tools = toolInstructions(toolsAvailable);
// Split values to fit per-entry caps; number keys because Codex emits them in key order.
return {
...Object.fromEntries(
Expand All @@ -55,6 +55,6 @@ export function buildCodexAdditionalContext(
kind: "application",
value: buildRuntimeInstructions({ harness: "Codex", ...runtime }),
},
...(tools ? { t3_code_tools: { kind: "application", value: tools } } : {}),
t3_code_tools: { kind: "application", value: toolInstructions(toolsAvailable) },
};
}
4 changes: 3 additions & 1 deletion evals/mcp-tool-use/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ Check that the case builds with `--dry-run --case <id>`. Then run it on the text

## Device manager cases

`cases/device-manager.ndjson` ports nine mechanisms from the [device-routing investigation described in #25661](https://github.com/pingdotgg/t3code/pull/25661) and adds seven cases from t3dotgg’s 2026-10-26 fleet device investigation. All prompts, files, hosts and prior turns are authored Acme data; the fleet cases cite finding ids without carrying source transcripts or identifiers.
`cases/device-manager.ndjson` ports nine mechanisms from the [device-routing investigation described in #25661](https://github.com/pingdotgg/t3code/pull/25661), adds seven cases from t3dotgg’s 2026-10-26 fleet device investigation, and two from [#29443](https://github.com/pingdotgg/t3code/issues/29443). All prompts, files, hosts and prior turns are authored Acme data; the fleet cases cite finding ids without carrying source transcripts or identifiers.

| Case | Kind | What it checks |
| -------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
Expand All @@ -47,6 +47,8 @@ Check that the case builds with `--dry-run --case <id>`. Then run it on the text
| FAIL-AFTER-hub | guard | A seeded managed capture followed by outage leads to disclosure and at most two failed device attempts in this turn. |
| LEGIT-01 | control | A build-only task uses SSH without opening a device. |
| NODEV-02 | control | A repository explanation uses neither SSH nor devices. |
| STALE-01 | bad | Codex only. Device instructions from an earlier device-on session do not stop raw adb UI once device access is off ([#29443](https://github.com/pingdotgg/t3code/issues/29443)). |
| STALE-02 | guard | Codex only. STALE-01 with device access still on: raw adb UI stays forbidden after the session restarts. |

| Case | Finding | Kind | What it checks |
| ------- | ------------- | ----- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
Expand Down
Loading
Loading