Fix #1645: stop the overview cache from burning the GitHub GraphQL quota

apps/web/src/components/WorkView.tsx

@@ -112,6 +112,13 @@ export function WorkView({ state, onRefresh, onSelectTab }: WorkViewProps) {
           onSelectTab={id => onSelectTab?.(id)}
         />
 
+        {overview?.forgeStatus === 'rate-limited' && (
+          <p className="work-unavailable">
+            Forge rate limited — PRs, backlog and recently closed may be stale until{' '}
+            {overview.forgeResetAt ? new Date(overview.forgeResetAt).toLocaleTimeString() : 'the budget resets'}
+          </p>
+        )}
+
         {/* Needs Attention */}
         <section className="work-section">
           <h3 className="work-section-title">Needs Attention</h3>

codev/projects/bugfix-1645-tower-overview-cache-burns-the/status.yaml

@@ -0,0 +1,21 @@
+id: bugfix-1645
+title: tower-overview-cache-burns-the
+protocol: bugfix
+phase: pr
+plan_phases: []
+current_plan_phase: null
+gates:
+  pr:
+    status: pending
+    requested_at: '2026-09-08T08:39:17.850Z'
+iteration: 1
+build_complete: false
+history: []
+started_at: '2026-09-08T08:02:34.019Z'
+updated_at: '2026-09-08T08:39:30.355Z'
+pr_ready_for_human: true
+pr_history:
+  - phase: pr
+    pr_number: 1652
+    branch: builder/bugfix-1645
+    created_at: '2026-09-08T08:39:30.354Z'

codev/state/bugfix-1645_thread.md

@@ -0,0 +1,108 @@
+# bugfix-1645 — builder thread (rebuild lane)
+
+Issue #1645: Tower's OverviewCache burns the whole GitHub GraphQL quota.
+Spec = architect's rebuild prescription on the issue (comment 5581422713) + the
+46-item pitfall list (comment 5581455959). PR #1646 is superseded; not building on it.
+
+## 2026-09-08 — investigate
+
+### Reproduced (read-only against the production Tower, pid 21829)
+- Sampled `gh` descendants of the Tower pid at ~1 Hz: 30 distinct spawns in ~14 s
+  (≈ 2/s ≈ 7,000/h). Every spawn is one of the four overview concepts
+  (`gh issue list --limit 200`, `gh issue list --state closed --search closed:>…`,
+  `gh pr list --state merged --search merged:>…`, `gh pr list --json …`).
+- Real GraphQL budget, from `gh api graphql -i` headers: `X-Ratelimit-Used: 1420,
+  Remaining: 3580` five minutes into the window (reset 1788858149). `gh api rate_limit
+  --jq .resources.graphql` reported `used: 0, remaining: 5000` at the same instant —
+  confirms the "misleading REST probe" note; a healthy reading from it is worthless.
+- The dashboard polls `/api/overview` every 2.5 s (`apps/web/src/hooks/useOverview.ts`),
+  the VS Code sidebar every 60 s; the 30 s TTL governs spend, the poll governs the
+  failure-amplifier.
+
+### Root cause (packages/codev)
+1. `src/agent-farm/servers/overview.ts` `OverviewCache.fetch*Cached`: `if (data !== null) cache.set(...)`
+   — failures are never cached, so while gh fails every poll re-spawns all four commands.
+2. `src/lib/forge.ts` `executeForgeCommand`: every failure collapses to `null`; stderr/exit
+   code never reach the caller, so nothing can tell "rate limited" from "gh missing".
+3. No single-flight: concurrent polls (2.5 s dashboard + VS Code + cloud) each start their
+   own batch while a previous one is in flight.
+4. `invalidate()` (called by porch after every mutating command, by VS Code, by cleanup)
+   clears every cache for every workspace, so churn bypasses the TTL entirely.
+5. TTL 30 s × 4 concepts × 13 workspaces ≈ 6,240 calls/h even with everything healthy.
+
+### Measured GraphQL cost (for the record; collapse is #1647, out of scope)
+A single combined GraphQL query (open PRs + open issues + closed-24h search + merged-24h
+search, first:100 each) costs **4 points** per `rateLimit.cost`. `{owner}`/`{repo}`
+placeholders work in `gh api graphql -F` and inside REST `search/issues?q=`.
+
+### Scope decision
+Frozen to the architect's six items (negative cache+backoff, per-backend suspension,
+single-flight w/ generation tag, TTLs 180/600/3600 + list-only debounced invalidate,
+forgeStatus/forgeResetAt payload, `# forge-executable:` on the 8 builtin-resolving
+scripts). Doctor check, #1647, #1648, #1650 are out. Estimated production diff ≈ 300 lines.
+
+## 2026-09-08 — fix (commit 92b5881f8, pushed)
+
+Shape, in dependency order (all inside the architect's six-item scope):
+- `lib/forge.ts`: `executeForgeCommandDetailed` keeps `{stderr, exitCode}` and never rejects
+  (config resolution moved inside the try — a malformed `.codev/config.json` used to be
+  reachable as an unhandled rejection, which tower-server turns into `process.exit(1)`).
+  `resolveForgeBackend` keys a concept by its resolved executable basename (`gh`), falling
+  back to the provider for generic transports (`curl`) and custom script paths.
+- `lib/forge-rate-limit.ts` (new): `ForgeRateLimiter` — per-budget suspension, 15 min
+  fallback, one advisory `gh api rate_limit` probe per fresh suspension (trusted only when
+  it reports `remaining === 0`), strict-`>` success clearing. Budget key: `gh` for the four
+  lists, `gh:rest` for `user-identity`/`auth-status`.
+- `servers/overview.ts`: one `cachedFetch` path — TTL 180/600/3600 s, negative entries with
+  a per-`<workspace>:<budget>` window (60 s doubling to 15 min, once per elapsed window),
+  suspension gate before any spawn, single-flight per `<workspace>:<concept>` with an
+  identity-checked cleanup, `invalidate()` = epoch stamp honoured by positive open-list
+  entries only, debounced 60 s per entry (hence per workspace). Payload gains
+  `forgeStatus` + `forgeResetAt`; error text names the real backend.
+- Scripts: `github/pr-list` no longer pipes gh into jq (POSIX sh has no pipefail);
+  `# forge-executable:` on gitlab/issue-search + the 7 linear scripts.
+- Dashboard: WorkView shows a rate-limited banner with the reset time.
+
+Surprise caught by my own Linear pitfall test: a *success* from the same batch that
+triggered the suspension (recently-merged succeeded while pr-list was rate-limited)
+cleared it, because `startedAt <= dispatchedAt` was true in the same millisecond. Pitfall
+B10 in the list is exactly this; comparison is now strict `>`.
+
+Production diff: 448+/125− over 17 files (overview.ts 163+/91−, forge-rate-limit.ts 105,
+forge.ts 76+/11−, github.ts 69+/18−). Tests: 17 new (10 cache-level pitfall tests, 6
+limiter unit tests, 1 real-`gh`-on-PATH harness) + 5 forge tests, 7 existing tests
+re-pinned to the new TTL/debounce semantics with an injected clock.
+
+Full suite first run: 67 failures in 12 files, all "embedded skeleton not found" — the
+worktree had no `dist/` (skeleton bundle). Rebuilding the package and re-running.
+
+### Verification (fix phase close)
+- Full suite after `pnpm --filter @cluesmith/codev build`: 288 files / 5767 tests passed,
+  48 skipped, 0 failed. (The earlier 67 failures were the unbuilt worktree: embedded
+  skeleton missing from `dist/`.) `tsc --noEmit` clean for packages/codev and apps/web.
+- Revert-based vacuity check (HEAD committed + pushed first; each guard patched in place,
+  test run, file restored from HEAD, tree confirmed clean): 16 guards → 16 RED. Table in the
+  PR body. One design item has no reachable red state: the identity check on single-flight
+  cleanup (`inflight.get(key) === flight`) is defensive only — nothing in this design can
+  register a newer flight while an older one is still in the map, because invalidate()
+  never touches `inflight`. Kept as a two-line guard; stated, not faked.
+- Dashboard banner rendered in Chromium via Playwright against the vite dev server on
+  :5199 with `/api/**` intercepted (overview fixture `forgeStatus: 'rate-limited'`), so no
+  request reached the live Tower. Banner text and the per-section messages visible.
+- `gh api rate_limit` misreport confirmed on this account during the investigation:
+  REST said `used: 0` while `gh api graphql -i` headers said `Used: 1420`. The probe is
+  therefore advisory (trusted only on `remaining === 0`), exactly as prescribed.
+
+## 2026-09-08 — pr phase
+
+PR #1652 opened via REST (`gh api repos/…/pulls`): the `pr-create` concept failed on the
+exhausted GraphQL bucket — the bug itself. Same for CMAP: `consult --type pr` resolves the
+PR through `pr-search` (`gh pr list --search`, GraphQL) and `pr-view`/`pr-diff`, so all
+three lanes first returned "No PR found". Re-ran them with a REST-only `gh` shim on PATH
+(scratchpad only, nothing committed) answering `pr list/view/diff` from `gh api`.
+
+CMAP round 1: gemini APPROVE, codex APPROVE, claude APPROVE. Tree clean after every lane.
+Claude's non-blocking notes: banner copy over-claimed staleness (fixed: "may be stale");
+freshness trade-off (lists 180 s, searches 600 s never invalidated) wants an architect ack;
+`isRateLimitError` doesn't match the abuse-detection 403 (falls to backoff — acceptable);
+pre-existing missing `Array.isArray` before `prs.map` (same on main; follow-up material).

packages/codev/scripts/forge/github/pr-list.sh

@@ -7,5 +7,9 @@
 # as objects (users carry `login`; teams carry `slug`/`name` and no `login`), so
 # `.login // empty` keeps user reviewers and drops team reviewers — matching the
 # `reviewRequests: string[]` contract in PrListItem (forge-contracts.ts).
-exec gh pr list --json number,title,url,reviewDecision,body,createdAt,author,reviewRequests,isDraft \
-  | jq '[.[] | .reviewRequests = [.reviewRequests[].login // empty]]'
+#
+# Not `gh … | jq`: POSIX sh has no pipefail, so the caller would see jq's exit
+# status and a rate-limited gh would look like a successful empty list (#1645).
+# forge-executable: gh
+out="$(gh pr list --json number,title,url,reviewDecision,body,createdAt,author,reviewRequests,isDraft)" || exit 1
+printf '%s' "$out" | jq '[.[] | .reviewRequests = [.reviewRequests[].login // empty]]'

packages/codev/scripts/forge/gitlab/issue-search.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: issue-search (GitLab via glab CLI)
+# forge-executable: glab
 #
 # ⚠️ UNVERIFIED — mirrors gitlab/issue-list.sh (raw `glab ... --output json`,
 #    no field normalization) plus a state flag and a `body` field surfaced from

packages/codev/scripts/forge/linear/auth-status.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: auth-status (Linear via GraphQL API)
+# forge-executable: curl
 # Output: exit code (0 = authenticated)
 set -e
 

packages/codev/scripts/forge/linear/issue-comment.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: issue-comment (Linear via GraphQL API)
+# forge-executable: curl
 # Input: CODEV_ISSUE_ID, CODEV_COMMENT_BODY
 # Output: exit code only
 set -e

packages/codev/scripts/forge/linear/issue-list.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: issue-list (Linear via GraphQL API)
+# forge-executable: curl
 # Input: CODEV_LINEAR_TEAM (team key, e.g. "ENG")
 # Output: JSON [{number, title, url, labels, createdAt, author, assignees}]
 set -e

packages/codev/scripts/forge/linear/issue-search.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: issue-search (Linear via GraphQL API)
+# forge-executable: curl
 #
 # ⚠️ UNVERIFIED — mirrors linear/issue-list.sh with `description` added to the
 #    query (mapped to `body`) and the state filter parameterized. No Linear

packages/codev/scripts/forge/linear/issue-view.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: issue-view (Linear via GraphQL API)
+# forge-executable: curl
 # Input: CODEV_ISSUE_ID (e.g. "ENG-123")
 # Output: JSON {title, body, state, url, author, createdAt, assignees, labels, milestone, comments[]}
 #

packages/codev/scripts/forge/linear/recently-closed.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: recently-closed (Linear via GraphQL API)
+# forge-executable: curl
 # Input: CODEV_LINEAR_TEAM, CODEV_SINCE_DATE (optional, ISO date)
 # Output: JSON [{number, title, url, labels, createdAt, closedAt}]
 set -e

packages/codev/scripts/forge/linear/user-identity.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 # Forge concept: user-identity (Linear via GraphQL API)
+# forge-executable: curl
 # Output: plain text display name
 set -e
 

packages/codev/src/__tests__/forge-rate-limit.test.ts

@@ -0,0 +1,62 @@
+/**
+ * #1645: ForgeRateLimiter — what may set and clear a per-budget suspension.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { ForgeRateLimiter, budgetKeyFor, isRateLimitError, SUSPENSION_FALLBACK_MS } from '../lib/forge-rate-limit.js';
+
+const GRAPHQL = 'gh: API rate limit already exceeded for user ID 1';
+
+describe('ForgeRateLimiter (#1645)', () => {
+  it('recognises GitHub rate-limit errors and nothing else', () => {
+    expect(isRateLimitError(GRAPHQL)).toBe(true);
+    expect(isRateLimitError('{"errors":[{"type":"RATE_LIMITED"}]}')).toBe(true);
+    expect(isRateLimitError('You have exceeded a secondary rate limit')).toBe(true);
+    expect(isRateLimitError('no git remotes found')).toBe(false);
+    expect(isRateLimitError('gh: command not found')).toBe(false);
+  });
+
+  it('keys gh REST concepts on their own budget; other backends are one budget', () => {
+    expect(budgetKeyFor('gh', 'user-identity')).toBe('gh:rest');
+    expect(budgetKeyFor('GH', 'pr-list')).toBe('gh');
+    expect(budgetKeyFor('linear', 'user-identity')).toBe('linear');
+  });
+
+  it('suspends only on a rate-limit error, for the fallback window, without a probe', () => {
+    const limiter = new ForgeRateLimiter();
+    expect(limiter.noteFailure('gh', 'gh', '/ws', 'no git remotes found', 1000)).toBe(false);
+    expect(limiter.isSuspended('gh', 1000)).toBe(false);
+    expect(limiter.noteFailure('gh', 'gh', '/ws', GRAPHQL, 1000)).toBe(true);
+    expect(limiter.isSuspended('gh', 1000)).toBe(true);
+    expect(limiter.resetAt('gh', 1000)).toBe(1000 + SUSPENSION_FALLBACK_MS);
+    expect(limiter.isSuspended('gh', 1000 + SUSPENSION_FALLBACK_MS)).toBe(false);
+  });
+
+  it('a success from the same tick as the suspension does not clear it; a later one does', () => {
+    const limiter = new ForgeRateLimiter();
+    limiter.noteFailure('gh', 'gh', '/ws', GRAPHQL, 5000);
+    limiter.noteSuccess('gh', 5000);
+    expect(limiter.isSuspended('gh', 5001)).toBe(true);
+    limiter.noteSuccess('gh', 4000);
+    expect(limiter.isSuspended('gh', 5001)).toBe(true);
+    limiter.noteSuccess('gh', 5001);
+    expect(limiter.isSuspended('gh', 5002)).toBe(false);
+  });
+
+  it('a success on another budget never clears this one', () => {
+    const limiter = new ForgeRateLimiter();
+    limiter.noteFailure('gh', 'gh', '/ws', GRAPHQL, 1000);
+    limiter.noteSuccess('gh:rest', 9000);
+    expect(limiter.isSuspended('gh', 9001)).toBe(true);
+  });
+
+  it('runs the probe once per fresh suspension and only for gh', async () => {
+    const calls: string[] = [];
+    const limiter = new ForgeRateLimiter(async (backend) => { calls.push(backend); return null; });
+    limiter.noteFailure('gh', 'gh', '/ws', GRAPHQL, 1000);
+    limiter.noteFailure('gh', 'gh', '/ws', GRAPHQL, 1001);
+    limiter.noteFailure('glab', 'glab', '/ws', GRAPHQL, 1002);
+    await new Promise(r => setTimeout(r, 0));
+    expect(calls).toEqual(['gh']);
+  });
+});

packages/codev/src/__tests__/forge.test.ts

@@ -20,6 +20,8 @@ import {
   validateForgeConfig,
   loadForgeConfig,
   resolveAllConcepts,
+  executeForgeCommandDetailed,
+  resolveForgeBackend,
 } from '../lib/forge.js';
 
 // =============================================================================
@@ -693,3 +695,73 @@ describe('resolveAllConcepts', () => {
     expect(issueView?.executable).toBe('./bin/my-forge');
   });
 });
+
+// =============================================================================
+// #1645: failures keep their stderr/exit code; budgets key by resolved backend
+// =============================================================================
+
+describe('executeForgeCommandDetailed (#1645)', () => {
+  const SCRIPT = join(MOCK_SCRIPTS_DIR, 'rate-limited.sh');
+  beforeAll(() => {
+    writeFileSync(SCRIPT, '#!/bin/sh\necho "gh: API rate limit already exceeded for user ID 1" >&2\nexit 1\n');
+    chmodSync(SCRIPT, 0o755);
+  });
+
+  it('surfaces stderr and the exit code of a failing concept', async () => {
+    const result = await executeForgeCommandDetailed('issue-view', {}, { forgeConfig: { 'issue-view': SCRIPT } });
+    expect(result.data).toBeNull();
+    expect(result.error?.exitCode).toBe(1);
+    expect(result.error?.stderr).toContain('rate limit already exceeded');
+  });
+
+  it('reports a disabled concept as an error without executing', async () => {
+    const result = await executeForgeCommandDetailed('issue-view', {}, { forgeConfig: { 'issue-view': null } });
+    expect(result).toEqual({ data: null, error: { message: expect.stringContaining('disabled'), stderr: '', exitCode: null } });
+  });
+
+  it('reports a malformed .codev/config.json as an error instead of rejecting', async () => {
+    const broken = join(TEST_DIR, 'broken-config');
+    mkdirSync(join(broken, '.codev'), { recursive: true });
+    writeFileSync(join(broken, '.codev', 'config.json'), '{ not json');
+    await expect(executeForgeCommandDetailed('pr-list', {}, { workspaceRoot: broken })).resolves.toMatchObject({ data: null });
+  });
+});
+
+describe('resolveForgeBackend (#1645)', () => {
+  it('keys the github default scripts by their CLI, gh', () => {
+    expect(resolveForgeBackend('pr-list', { forgeConfig: null })).toBe('gh');
+    expect(resolveForgeBackend('user-identity', { forgeConfig: null })).toBe('gh');
+  });
+
+  it("keys a Linear workspace's pr-list (falls through to github) by gh, and its issue-list by provider", () => {
+    expect(resolveForgeBackend('pr-list', { forgeConfig: { provider: 'linear' } })).toBe('gh');
+    expect(resolveForgeBackend('issue-list', { forgeConfig: { provider: 'linear' } })).toBe('linear');
+  });
+
+  it('keys an absolute CLI path like the bare command, lowercased', () => {
+    expect(resolveForgeBackend('pr-list', { forgeConfig: { 'pr-list': '/usr/local/bin/GH pr list --json number' } })).toBe('gh');
+  });
+
+  it('keys generic transports and custom script paths by provider', () => {
+    expect(resolveForgeBackend('pr-list', { forgeConfig: { provider: 'gitlab', 'pr-list': 'curl -s https://forge.example/prs' } })).toBe('gitlab');
+    expect(resolveForgeBackend('pr-list', { forgeConfig: { 'pr-list': '/home/me/my-forge-script' } })).toBe('github');
+  });
+
+  it('returns null for a disabled concept', () => {
+    expect(resolveForgeBackend('pr-list', { forgeConfig: { 'pr-list': null } })).toBeNull();
+  });
+});
+
+describe('shipped provider scripts declare their executable (#1645)', () => {
+  const BUILTINS = new Set(['set', 'if', 'case', 'echo', 'printf', 'test', '[', 'exit', 'export', 'local', '.', 'source']);
+
+  it('no built-in provider resolves a concept to a shell builtin', () => {
+    for (const provider of getKnownProviders()) {
+      for (const r of resolveAllConcepts({ provider })) {
+        if (r.command === null) continue;
+        expect(r.executable, `${provider}/${r.concept} resolved to '${r.executable}'`).not.toBeNull();
+        expect(BUILTINS.has(r.executable as string), `${provider}/${r.concept} resolved to builtin '${r.executable}'`).toBe(false);
+      }
+    }
+  });
+});