From 29b68f5c70d1496559af28c76145c61e9a5700ad Mon Sep 17 00:00:00 2001 From: bft-codebot Date: Wed, 29 Apr 2026 00:25:04 +0000 Subject: [PATCH] sync(bfmono): feat(gambit): add browser introspection live commands (+19 more) (bfmono@d6a165d66) This PR is an automated gambitmono sync of bfmono Gambit packages. - Source: `packages/gambit/` - Core: `packages/gambit/packages/gambit-core/` - bfmono rev: d6a165d66 Changes: - 668e393de feat(gambit): add browser introspection live commands - 4d8a6ad7b feat(workloop): bridge runtime Codex refresh to host services - 81ac0db58 feat(gambit): add live browser pointer refs - 62e132a24 chore(browser): move runtime out of Gambit - e6c80f928 test(gambit): remove legacy chat suites - 49c9b67e3 chore(gambit): remove dead chat provider adapters - 8b4af5051 fix(gambit): preserve responses continuation context - d9127ae36 feat(gambit): expose structured responses runtime - 522e10d5e feat(gambit): make provider execution responses-only - 13bfdfe34 fix(gambit): pass codex runtime tools - 477a6e8fa fix(gambit): preserve structured text spacing - c31a35389 fix(gambit): read workloop codex auth bundle env - ca19ea448 chore(workloop): rename desktop app from bfdesktop - 0715225a7 fix(gambit): parse mock Codex string ids portably - e4fad187d fix(browser-runtime): use repo-relative automation imports - 549d9dafa fix(browser-runtime): use explicit Deno exports - 46d961d01 fix(browser-runtime): detect stale live sessions - bd2b9d937 fix(bfdesktop): route Chief browser MCP tool calls - 63b35d1cc feat(bfdesktop): add Chief runtime browser tools - 0e38141eb fix(gambit): preserve PATH for codex shell commands Do not edit this repo directly; make changes in bfmono and re-run the sync. --- ...6-04-28-browser-introspection-tools-AAR.md | 109 ++++++++++ ...026-04-28-browser-pointer-targeting-AAR.md | 94 ++++++++ src/codex_preflight.ts | 112 +++++++--- src/runtime_host_service.ts | 203 ++++++++++++++++++ 4 files changed, 493 insertions(+), 25 deletions(-) create mode 100644 reviews/2026-04-28-browser-introspection-tools-AAR.md create mode 100644 reviews/2026-04-28-browser-pointer-targeting-AAR.md create mode 100644 src/runtime_host_service.ts diff --git a/reviews/2026-04-28-browser-introspection-tools-AAR.md b/reviews/2026-04-28-browser-introspection-tools-AAR.md new file mode 100644 index 00000000..e71108d9 --- /dev/null +++ b/reviews/2026-04-28-browser-introspection-tools-AAR.md @@ -0,0 +1,109 @@ +# Browser Runtime Introspection Tools AAR + +## Context + +Workloop needed more browser read tools after the pointer-targeting pass: +waiting, page state, text/region queries, point inspection, viewport controls, +highlighting, link extraction, and recent console/network diagnostics. The +Workloop tool boundary should remain narrow and must not expose arbitrary +browser eval. + +Because Workloop routes browser commands through the shared Gambit browser +runtime, the live-session command contract needed to support those capabilities. + +## Intent + +- Purpose: add structured browser live-session introspection commands that + Workloop can expose as narrow coworker tools. +- End State: live sessions support read-only page state, text, region, element, + viewport, point, link, console, and network-failure inspection, plus bounded + wait and highlight commands. +- Constraints / Tradeoffs: keep arbitrary eval out of the Workloop-facing + surface; keep the runtime command payloads explicit and typed; preserve + existing query/ref/mouse commands. +- Phase (if applicable): Workloop browser-use Phase 2 support with Phase 6 + screenshot/debug evidence support. + +## What Happened + +Extended `BrowserLiveSessionCommand` with wait, page-state, text query, region +query, element description, stable-layout wait, viewport measurement/resize, +point inspection, highlight, link extraction, console read, and network failure +read commands. + +The live daemon now keeps bounded in-memory rings for console messages and +failed network requests. It can derive visible text blocks, element boxes, +visible links with `href`, and the element stack at a coordinate. Highlighting +adds a visible overlay that can be captured by the existing screenshot command. + +The new DOM-inspection helpers live in `liveSessionInspection.ts`, keeping +`liveSessionDaemon.ts` below the repository file-length hard limit. + +## Delta Analysis + +The earlier runtime supported enough structure to click a known target, but it +did not provide enough state for robust multi-step page diagnosis. Adding these +commands to the runtime keeps Workloop's tool wrapper simple and lets future +CLI/operator surfaces reuse the same underlying behavior if needed. + +The implementation intentionally does not remove the runtime's existing low +level eval command because it predates Workloop and is useful for developer +verification. Workloop still does not expose it as an assistant tool. + +## Initiative Assessment + +Disciplined initiative: the command contract was extended in one place and the +Workloop wrapper simply forwards typed commands. + +Disciplined initiative: link extraction now includes `href`, which avoids +forcing callers to infer navigation targets from text and coordinates. + +Disciplined initiative: verification exercised a real browser session and +captured console/network failure events rather than relying only on typechecks. + +## Weaknesses In Intent + +The runtime still has duplicated accessible-name logic between query and element +description. That is acceptable for this pass, but a follow-up should extract a +shared helper if more DOM description behavior is added. + +## What We Will Sustain + +- Keep Workloop-facing commands typed and narrow. +- Preserve mouse/ref based actuation as the default action model. +- Keep diagnostic event buffers bounded so sessions do not grow unbounded in + long tasks. + +## What We Will Improve + +- Add stale-ref generation semantics if real traces show refs reused after + navigation. +- Consider exposing these commands in the developer CLI only if operator + workflows need them directly. + +## Ownership And Follow-Up + +- Owner: Gambit browser runtime maintainers. +- Action: monitor Workloop browser-use traces for missed controls, stale refs, + and whether console/network diagnostics explain page failures. +- Target date: next browser-runtime polish pass. + +## Verification Evidence + +- `deno fmt apps/workloop/sidecar/chief_runtime_browser_tools.ts apps/workloop/sidecar/chief_runtime_browser_tools_test.ts apps/workloop/sidecar/chief_runtime_workloop_tools_test.ts packages/browser-runtime/src/liveControl.ts packages/browser-runtime/src/liveSessionDaemon.ts packages/browser-runtime/src/liveSessionInspection.ts` + passed as part of the final formatting run with `Checked 10 files`. +- `deno check --config packages/browser-runtime/deno.json packages/browser-runtime/src/liveSessionDaemon.ts packages/browser-runtime/src/liveSessionInspection.ts` + passed. +- `deno test -A --config packages/browser-runtime/deno.json packages/browser-runtime/src/liveControl.test.ts packages/browser-runtime/src/liveSessionDaemon.test.ts` + passed: 8 tests. +- Live smoke: a headless session named `workloop-js-tools-smoke` validated all + new runtime commands on a local `data:` page. The run extracted a visible link + with `href` `https://example.com/docs`, captured console event + `smoke-console-error`, captured failed request + `http://127.0.0.1:9/missing-smoke.png`, and wrote screenshot evidence to +- Live smoke after the file split repeated the same command sequence and wrote + screenshot evidence to + `/Users/randallb/code/bolt-foundry/codebot-workspaces/shared/bft-e2e/browser-live-workloop-js-tools-smoke/__latest__/screenshots/2026-04-28T22-23-36-724Z_browser-js-tools-smoke-refactor.png`. +- `direnv exec . bft precommit` passed after splitting Gambit and Workloop + commits: codegen produced no tracked changes, format/lint/typecheck passed, + and the full test run reported `1484 passed`, `0 failed`, and `3 ignored`. diff --git a/reviews/2026-04-28-browser-pointer-targeting-AAR.md b/reviews/2026-04-28-browser-pointer-targeting-AAR.md new file mode 100644 index 00000000..5253a268 --- /dev/null +++ b/reviews/2026-04-28-browser-pointer-targeting-AAR.md @@ -0,0 +1,94 @@ +# Browser Runtime Pointer Targeting AAR + +## Context + +The shared browser runtime had live-session click support for selectors and raw +coordinates. Workloop needed a safer higher-level tool surface where agents +could inspect visible controls, choose a target, move the browser mouse there, +and click without exposing arbitrary page evaluation. + +Because Workloop browser tools depend on the Gambit browser runtime, the runtime +needed to provide the underlying query/ref/mouse contract before the Workloop +tool wrapper could expose it. + +## Intent + +- Purpose: support reliable pointer-based browser targeting from the shared + browser runtime. +- End State: live sessions can query visible interactive elements, return + short-lived refs, move the mouse to refs/selectors/coordinates, click via the + mouse, and include the tracked cursor in screenshots. +- Constraints / Tradeoffs: keep arbitrary eval out of the Workloop agent tool + surface; use DOM inspection only inside the runtime to derive visible + coordinates; preserve existing selector and coordinate callers. +- Phase (if applicable): Workloop browser-use Phase 2 support with Phase 6 + screenshot evidence support. + +## What Happened + +Added a `query` live-session command and ref fields for `mouse-move` and +`click`. Query uses constrained runtime-internal DOM inspection to find visible +interactive elements and returns refs such as `e1` for follow-up commands. + +Changed live-session click handling to resolve refs and selectors to center +points, move the Playwright mouse there, and click with `page.mouse.click`. +Screenshots now temporarily render the tracked cursor position before capture. + +Updated the browser CLI so operator workflows can exercise `live query`, +`live mouse move --ref`, and `live click --ref`. + +## Delta Analysis + +The previous selector/coordinate API was too low-level for agents that need to +decide among visually similar controls. Returning short-lived refs gives the +agent enough structure to choose a target while keeping actuation mouse-based. + +The query command intentionally implements a pragmatic accessible-name subset +rather than a full accessibility tree. That keeps the change small and suitable +for the current failure mode. + +## Initiative Assessment + +Disciplined initiative: the runtime preserved existing selector and coordinate +contracts while adding refs as an optional, safer target handoff. + +Disciplined initiative: screenshot cursor rendering was kept as a temporary +artifact-time overlay rather than a durable page mutation. + +## Weaknesses In Intent + +No material weaknesses identified for the runtime slice. Future intent should +say whether a full accessibility-tree source is required. + +## What We Will Sustain + +- Keep browser actuation mouse-based. +- Keep query/read support narrow and structured. +- Keep runtime CLI support aligned with programmatic live-session commands. + +## What We Will Improve + +- Add stronger stale-ref semantics if query refs are reused after navigation in + real traces. +- Replace the accessible-name subset with a browser accessibility-tree source if + query misses become common. + +## Ownership And Follow-Up + +- Owner: Gambit browser runtime maintainers. +- Action: monitor Workloop browser traces for query quality and stale-ref + frequency. +- Target date: next browser-runtime polish pass. + +## Verification Evidence + +- `deno fmt` on the touched browser runtime and mirrored browser files passed. +- `deno check --config packages/browser-runtime/deno.json packages/browser-runtime/src/liveSessionDaemon.ts packages/browser-runtime/src/browserCli.ts` + passed. +- `deno test -A --config packages/browser-runtime/deno.json packages/browser-runtime/src/liveControl.test.ts packages/browser-runtime/src/liveSessionDaemon.test.ts` + passed: 8 tests. +- Live smoke: a headless session queried a checkbox by role/name, returned ref + `e1`, moved to it, captured a screenshot with the visible cursor overlay, + clicked it with the mouse path, and verified the checkbox became checked. +- `direnv exec . bft precommit` passed the full repo gate: codegen no tracked + changes, format, lint, typecheck, and 1479 tests passed with 3 ignored. diff --git a/src/codex_preflight.ts b/src/codex_preflight.ts index 14f1fe93..7178d3bf 100644 --- a/src/codex_preflight.ts +++ b/src/codex_preflight.ts @@ -4,6 +4,12 @@ import { summarizeCodexAuthBundle, } from "./codex_auth.ts"; import { logCodexAppServerDebug } from "./codex_app_server_debug.ts"; +import { + callRuntimeHostService, + CODEX_REFRESH_HOST_SERVICE_METHOD, + type CodexRefreshHostServiceResult, + RUNTIME_HOST_SERVICE_SOCKET_ENV, +} from "./runtime_host_service.ts"; const CODEX_BIN_ENV = "GAMBIT_CODEX_BIN"; export const MINIMUM_SUPPORTED_CODEX_CLI_VERSION = "0.121.0"; @@ -82,15 +88,37 @@ async function appServerPreflightRequestResult(input: { }; }> { if (input.method === "account/chatgptAuthTokens/refresh") { - const refreshed = await refreshCodexChatgptAuthTokens({ - bundle: input.bundle, - previousAccountId: typeof input.params.previousAccountId === "string" - ? input.params.previousAccountId - : null, - reason: typeof input.params.reason === "string" && input.params.reason + const previousAccountId = typeof input.params.previousAccountId === "string" + ? input.params.previousAccountId + : null; + const reason = + typeof input.params.reason === "string" && input.params.reason ? input.params.reason - : "account/chatgptAuthTokens/refresh", - }); + : "account/chatgptAuthTokens/refresh"; + const hostServiceSocket = Deno.env.get(RUNTIME_HOST_SERVICE_SOCKET_ENV) + ?.trim(); + const hostRefreshed = hostServiceSocket + ? await callRuntimeHostService({ + method: CODEX_REFRESH_HOST_SERVICE_METHOD, + params: { + previousAccountId, + reason, + }, + }) + : null; + const refreshed = hostRefreshed + ? { + ...input.bundle, + accessToken: hostRefreshed.accessToken, + chatgptAccountId: hostRefreshed.chatgptAccountId, + chatgptPlanType: hostRefreshed.chatgptPlanType, + lastRefresh: new Date().toISOString(), + } + : await refreshCodexChatgptAuthTokens({ + bundle: input.bundle, + previousAccountId, + reason, + }); return { bundle: refreshed, result: { @@ -111,6 +139,17 @@ async function appServerPreflightRequestResult(input: { }; } +async function refreshCodexPreflightViaHost(input: { + previousAccountId?: string | null; + reason: string; +}): Promise { + if (!Deno.env.get(RUNTIME_HOST_SERVICE_SOCKET_ENV)?.trim()) return null; + return await callRuntimeHostService({ + method: CODEX_REFRESH_HOST_SERVICE_METHOD, + params: input, + }); +} + async function readLegacyCodexLoginStatus(): Promise { const codexBin = Deno.env.get(CODEX_BIN_ENV)?.trim() || "codex"; const codexVersion = await readCodexCliVersion(); @@ -337,26 +376,49 @@ export async function readCodexLoginStatus(): Promise { capabilities: { experimentalApi: true }, }); await writeMessage({ method: "initialized", params: {} }); - await request("account/login/start", { - accessToken: bundle.accessToken, - chatgptAccountId: bundle.chatgptAccountId, - chatgptPlanType: bundle.chatgptPlanType, - type: "chatgptAuthTokens", - }); - const result = await request("account/read", { - type: "chatgptAuthTokens", - }) as Record< - string, - unknown - >; + let loginBundle = bundle; + const loginAndRead = async () => { + await request("account/login/start", { + accessToken: loginBundle.accessToken, + chatgptAccountId: loginBundle.chatgptAccountId, + chatgptPlanType: loginBundle.chatgptPlanType, + type: "chatgptAuthTokens", + }); + return await request("account/read", { + type: "chatgptAuthTokens", + }) as Record; + }; + let result = await loginAndRead(); const account = asRecord(result.account); - const requiresOpenaiAuth = result.requiresOpenaiAuth === true; - const confirmedAccountId = typeof account.id === "string" + let requiresOpenaiAuth = result.requiresOpenaiAuth === true; + let confirmedAccountId = typeof account.id === "string" ? account.id.trim() : ""; - const planType = typeof account.planType === "string" - ? account.planType - : bundle.chatgptPlanType; + if (requiresOpenaiAuth || !confirmedAccountId) { + const refreshed = await refreshCodexPreflightViaHost({ + previousAccountId: confirmedAccountId || bundle.chatgptAccountId, + reason: "codex-preflight-account-read-stale", + }); + if (refreshed) { + loginBundle = { + ...loginBundle, + accessToken: refreshed.accessToken, + chatgptAccountId: refreshed.chatgptAccountId, + chatgptPlanType: refreshed.chatgptPlanType, + lastRefresh: new Date().toISOString(), + }; + result = await loginAndRead(); + const retryAccount = asRecord(result.account); + requiresOpenaiAuth = result.requiresOpenaiAuth === true; + confirmedAccountId = typeof retryAccount.id === "string" + ? retryAccount.id.trim() + : ""; + } + } + const finalAccount = asRecord(result.account); + const planType = typeof finalAccount.planType === "string" + ? finalAccount.planType + : loginBundle.chatgptPlanType; const hasConfirmedAccountId = confirmedAccountId.length > 0; return { codexLoggedIn: hasConfirmedAccountId, diff --git a/src/runtime_host_service.ts b/src/runtime_host_service.ts new file mode 100644 index 00000000..1f33096c --- /dev/null +++ b/src/runtime_host_service.ts @@ -0,0 +1,203 @@ +export const RUNTIME_HOST_SERVICE_SOCKET_ENV = + "WORKLOOP_RUNTIME_HOST_SERVICE_SOCKET"; +export const RUNTIME_HOST_SERVICE_TOKEN_ENV = + "WORKLOOP_RUNTIME_HOST_SERVICE_TOKEN"; + +export const CODEX_REFRESH_HOST_SERVICE_METHOD = + "providerAuth.codex.refreshChatgptTokens"; + +export type RuntimeHostServiceFailureReason = + | "host_auth_missing" + | "login_required" + | "refresh_failed" + | "invalid_request" + | "unknown_method"; + +export type CodexRefreshHostServiceParams = { + previousAccountId?: string | null; + reason: string; +}; + +export type CodexRefreshHostServiceResult = { + accessToken: string; + chatgptAccountId: string; + chatgptPlanType: string | null; + type: "chatgptAuthTokens"; +}; + +export type RuntimeHostServiceRequest = { + id: string; + method: string; + params: Record; + token: string; + type: "request"; +}; + +export type RuntimeHostServiceError = { + code: RuntimeHostServiceFailureReason; + message: string; +}; + +export type RuntimeHostServiceResponse = { + error: RuntimeHostServiceError | null; + id: string; + result: unknown | null; +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function normalizeOptionalString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 + ? value.trim() + : null; +} + +function normalizeRequiredString( + value: unknown, + label: string, +): string { + const normalized = normalizeOptionalString(value); + if (!normalized) { + throw new Error(`host service request is missing ${label}.`); + } + return normalized; +} + +export function validateCodexRefreshHostServiceParams( + value: unknown, +): CodexRefreshHostServiceParams { + if (!isRecord(value)) { + throw new Error("host service params must be a JSON object."); + } + const previousAccountId = value.previousAccountId == null + ? null + : normalizeOptionalString(value.previousAccountId); + if (value.previousAccountId != null && previousAccountId == null) { + throw new Error( + "providerAuth.codex.refreshChatgptTokens previousAccountId must be a non-empty string when provided.", + ); + } + return { + previousAccountId, + reason: normalizeRequiredString(value.reason, "reason"), + }; +} + +export function validateCodexRefreshHostServiceResult( + value: unknown, +): CodexRefreshHostServiceResult { + if (!isRecord(value)) { + throw new Error("host service result must be a JSON object."); + } + const type = normalizeRequiredString(value.type, "type"); + if (type !== "chatgptAuthTokens") { + throw new Error(`unexpected Codex host service result type: ${type}`); + } + return { + accessToken: normalizeRequiredString(value.accessToken, "accessToken"), + chatgptAccountId: normalizeRequiredString( + value.chatgptAccountId, + "chatgptAccountId", + ), + chatgptPlanType: value.chatgptPlanType == null + ? null + : normalizeRequiredString(value.chatgptPlanType, "chatgptPlanType"), + type, + }; +} + +export function validateRuntimeHostServiceMethodAndParams(input: { + method: string; + params: unknown; +}): { + method: typeof CODEX_REFRESH_HOST_SERVICE_METHOD; + params: CodexRefreshHostServiceParams; +} { + if (input.method !== CODEX_REFRESH_HOST_SERVICE_METHOD) { + throw new Error(`unsupported runtime host service method: ${input.method}`); + } + return { + method: CODEX_REFRESH_HOST_SERVICE_METHOD, + params: validateCodexRefreshHostServiceParams(input.params), + }; +} + +async function readFirstLine( + readable: ReadableStream, +): Promise { + const reader = readable.getReader(); + const decoder = new TextDecoder(); + let text = ""; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + text += decoder.decode(value, { stream: true }); + const newlineIndex = text.indexOf("\n"); + if (newlineIndex >= 0) return text.slice(0, newlineIndex); + } + text += decoder.decode(); + return text.length > 0 ? text : null; + } finally { + reader.releaseLock(); + } +} + +async function writeJsonLine( + writable: WritableStream, + payload: unknown, +): Promise { + const writer = writable.getWriter(); + try { + await writer.write( + new TextEncoder().encode(`${JSON.stringify(payload)}\n`), + ); + } finally { + writer.releaseLock(); + } +} + +export async function callRuntimeHostService(input: { + method: typeof CODEX_REFRESH_HOST_SERVICE_METHOD; + params: CodexRefreshHostServiceParams; + socketPath?: string | null; + token?: string | null; +}): Promise { + const socketPath = input.socketPath?.trim() || + Deno.env.get(RUNTIME_HOST_SERVICE_SOCKET_ENV)?.trim(); + const token = input.token?.trim() || + Deno.env.get(RUNTIME_HOST_SERVICE_TOKEN_ENV)?.trim(); + if (!socketPath || !token) { + throw new Error("Workloop host service bridge is not configured."); + } + const request: RuntimeHostServiceRequest = { + id: crypto.randomUUID(), + method: input.method, + params: input.params, + token, + type: "request", + }; + const conn = await Deno.connect({ transport: "unix", path: socketPath }); + try { + await writeJsonLine(conn.writable, request); + const line = await readFirstLine(conn.readable); + if (!line) { + throw new Error( + "Workloop host service bridge closed without a response.", + ); + } + const response = JSON.parse(line) as RuntimeHostServiceResponse; + if (response.error) { + throw new Error(`${response.error.code}: ${response.error.message}`); + } + return validateCodexRefreshHostServiceResult(response.result); + } finally { + try { + conn.close(); + } catch { + // ignore close races + } + } +}