test(evals): measure how agents get secrets only the user has

evals/mcp-tool-use/agent-device-help.txt

@@ -1,5 +1,5 @@
 agent-device 0.21.31 prints this help; apps/server/src/device/deviceToolVersions.json pins that version. Regenerate from
-the repo root with: node scripts/node_modules/agent-device/bin/agent-device.mjs help [commands|record|screenshot|open|settings]
+the repo root with: node scripts/node_modules/agent-device/bin/agent-device.mjs help [commands|record|screenshot|open|settings|fill]
 
 $ agent-device help
 agent-device <command> [args] [--json]
@@ -260,3 +260,21 @@ Settings:
 
 Command flags:
   --app <id-or-name>  Target an app without opening it: doctor verifies an installed app by id or name; settings applies an app-scoped change (permission, iOS location) to a bundle id or package
+
+$ agent-device help fill
+Usage:
+  agent-device fill <x> <y> <text> | fill <@ref|selector> <text> | fill <x y|@ref|selector> --text-stdin
+
+Replace text in a UI input selected by snapshot ref, selector, or coordinates. Pass an empty text to clear the field. Inspect unconfirmed evidence and assert the expected value or resulting screen before continuing. Prefer refs or selectors after snapshot; use recordAs to keep sensitive text out of a recorded replay while sending it to the live app. Every positional after an @ref is the replacement text, so fill @e57 good morning enters "good morning"; quote the text when the shell must preserve exact whitespace. Clear a field with an empty text argument: fill @e57 "" (the argument must be present — fill @e57 alone is a missing argument, not a clear). When visible label text also matches a non-input element, constrain the target with editable=true, for example fill 'label="Email" editable=true' "qa@example.com". To keep a secret out of argv, omit the text and pipe it in: printf %s "$PASSWORD" | fill @e57 --text-stdin (max 64 KiB, one trailing newline removed).
+
+Command flags:
+  --delay-ms <ms>      Delay between typed characters
+  --record-as <VAR>    Fill: send the live text but publish ${VAR} in an armed .ad recording
+  --text-stdin         Fill: read the text from stdin (max 64 KiB, one trailing newline removed) instead of argv
+  --verify             Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot
+  --settle             After the action, wait for the UI to go quiet and return the settled diff vs the pre-action tree in the same response (best-effort; never fails the action)
+  --settle-quiet <ms>  Settle: quiet window the UI must hold to count as settled (default 500ms; 300ms when the settled tree comes from the iOS simulator snapshot bridge). On iOS, a broad screen replacement such as a modal dismissal is confirmed for at least 1500ms whatever window is asked for, within the --timeout settle deadline; a replacement from a screen the pre-action capture read as presenting nothing, such as a tab switch, gets no 1500ms confirmation and keeps the asked window
+  --timeout <ms>       Boot/Open/Prepare: startup budget covering the device boot (and runner preparation for prepare). Replay/Snapshot/Test: maximum wall-clock time for the command or attempt. With --settle: the settle-wait deadline (default 10s)
+  --depth, -d <depth>  Snapshot: limit snapshot depth
+  --scope, -s <scope>  Snapshot: scope snapshot to label/identifier
+  --raw                Snapshot: raw node output

evals/mcp-tool-use/devices.test.ts

@@ -224,6 +224,9 @@ it("prints agent-device's own help for a lone help command on a returned launche
   assert.equal(asked(`${launcher} help open ${target}`), help("open"));
   assert.equal(asked(`${launcher} --help 2>&1`), help(""));
   assert.equal(asked(`${launcher} help settings --session=t3-1`), help("settings"));
+  // A password reaches fill from stdin, so request_secret's file never enters argv.
+  assert.equal(asked(`${launcher} fill --help ${target}`), help("fill"));
+  assert.include(help("fill"), "fill <x y|@ref|selector> --text-stdin");
   // Anything else stays with the simulator: other topics, pipes, aliases, other commands.
   assert.isUndefined(asked(`${launcher} metro --help ${target}`));
   assert.isUndefined(asked(`${launcher} help record 2>&1 | head -40`));

evals/mcp-tool-use/regrade.ts

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// @effect-diagnostics nodeBuiltinImport:off globalConsole:off
+/**
+ * Grades saved samples again with this checkout's cases and graders: `pnpm mcp-tool-use:regrade
+ * <results.jsonl>... [--cases <file>]` prints pass counts per case, old → new. Rows with an error
+ * are skipped. A sample that stopped early and passes now is "cut short": its turn never finished.
+ */
+import * as NodeFS from "node:fs";
+import * as NodeURL from "node:url";
+import * as NodeUtil from "node:util";
+import * as Schema from "effect/Schema";
+
+import { gradeSample, McpToolUseCase, pastedText } from "./run.ts";
+
+const { values, positionals } = NodeUtil.parseArgs({
+  allowPositionals: true,
+  options: { cases: { type: "string", multiple: true } },
+});
+const caseFiles =
+  values.cases ??
+  NodeFS.readdirSync(new URL("./cases", import.meta.url))
+    .filter((name) => name.endsWith(".ndjson"))
+    .map((name) => NodeURL.fileURLToPath(new URL(`./cases/${name}`, import.meta.url)));
+const decodeCase = Schema.decodeUnknownSync(Schema.fromJsonString(McpToolUseCase));
+const lines = (file: string) =>
+  NodeFS.readFileSync(file, "utf8")
+    .split("\n")
+    .filter((line) => line.trim() !== "");
+const cases = new Map(
+  caseFiles.flatMap(lines).map((line) => [decodeCase(line).id, decodeCase(line)]),
+);
+
+for (const file of positionals) {
+  const counts = new Map<string, { n: number; was: number; now: number; cut: number }>();
+  for (const line of lines(file)) {
+    const row = JSON.parse(line);
+    const testCase = cases.get(row.id);
+    if (testCase === undefined || row.error !== undefined) continue;
+    const failed = gradeSample(testCase.expect, row, pastedText(testCase)).filter(
+      (check) => !check.pass,
+    );
+    const cut = failed.length === 0 && row.stopReason === "failed_early";
+    const pass = failed.length === 0 && !cut;
+    const count = counts.get(row.id) ?? { n: 0, was: 0, now: 0, cut: 0 };
+    count.n++;
+    if (row.pass) count.was++;
+    if (pass) count.now++;
+    if (cut) count.cut++;
+    counts.set(row.id, count);
+    if (pass !== row.pass || cut)
+      console.log(
+        `${row.id}#${count.n} ${row.pass ? "pass" : "FAIL"} → ${cut ? "cut short" : pass ? "pass" : "FAIL"}${failed.length > 0 ? `: ${failed.map((check) => check.check.slice(0, 100)).join(" | ")}` : ""}`,
+      );
+  }
+  console.log(`\n${file}\n| case | was | now |\n| --- | --- | --- |`);
+  for (const [id, { n, was, now, cut }] of counts)
+    console.log(
+      `| ${id} | ${was}/${n} | ${now}/${n}${cut > 0 ? ` (${cut} cut short)` : ""}${was === now ? "" : " *"} |`,
+    );
+}