From 7d5880c5c666da50dc51559cbf5b1b18f45cf19b Mon Sep 17 00:00:00 2001 From: fixfon <12512757+fixfon@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:15:18 +0300 Subject: [PATCH] fix(usage): count Claude advisor calls under the advisor model Claude Code lists advisor calls only in usage.iterations. The top-level usage covers the executor alone, so advisor tokens never reached the usage totals. Each advisor_message iteration now becomes its own record under the advisor model, keyed apart from the executor record so the repeated content-block lines of one message still count once. Cached Claude transcripts are re-parsed once (scan cache v10) so existing history picks up advisor calls. Fixes #24129 --- .../src/provider/Drivers/claudeUsage.test.ts | 138 +++++++++++++++++- .../src/provider/Drivers/claudeUsage.ts | 60 ++++++-- apps/server/src/usage/UsageService.test.ts | 2 +- apps/server/src/usage/usageScanCache.test.ts | 18 ++- apps/server/src/usage/usageScanCache.ts | 13 +- 5 files changed, 205 insertions(+), 26 deletions(-) diff --git a/apps/server/src/provider/Drivers/claudeUsage.test.ts b/apps/server/src/provider/Drivers/claudeUsage.test.ts index 1db2197a53e5..f683808a4784 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.test.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.test.ts @@ -32,9 +32,9 @@ function claudeLine(overrides: { describe("parseClaudeLine", () => { it("extracts token totals and a dedupe key", () => { - const record = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); + const [record] = parseClaudeLine(claudeLine({ messageId: "msg_1", contentType: "text" })); - expect(record).not.toBeNull(); + expect(record).toBeDefined(); expect(record?.provider).toBe("claude"); expect(record?.model).toBe("claude-fable-5"); expect(record?.totals).toEqual({ @@ -51,8 +51,8 @@ describe("parseClaudeLine", () => { it("gives every content block of one message the same dedupe key", () => { // T3 Code writes one record per content block, each repeating the parent // message's full usage. Summing them would overcount ~2.4x on real data. - const text = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); - const toolUse = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); + const [text] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "text" })); + const [toolUse] = parseClaudeLine(claudeLine({ messageId: "msg_2", contentType: "tool_use" })); expect(text?.dedupeKey).toBe(toolUse?.dedupeKey); expect(text?.totals).toEqual(toolUse?.totals); @@ -87,3 +87,133 @@ describe("parseClaudeLine", () => { expect(claudeUsageFormat.parseProjected(synthetic, undefined)).toEqual([]); }); }); + +describe("Claude advisor iterations", () => { + const advisorTurn = { + type: "assistant", + timestamp: "2026-10-22T01:31:11.984Z", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + requestId: "req_1", + message: { + id: "msg_1", + model: "claude-opus-5-5", + usage: { + input_tokens: 4, + cache_read_input_tokens: 118320, + cache_creation_input_tokens: 2676, + output_tokens: 4506, + speed: "standard", + iterations: [ + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 58352, + cache_creation_input_tokens: 1616, + output_tokens: 141, + }, + { + type: "advisor_message", + model: "claude-fable-5-1", + input_tokens: 54204, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + output_tokens: 4229, + }, + { + type: "message", + input_tokens: 2, + cache_read_input_tokens: 59968, + cache_creation_input_tokens: 1060, + output_tokens: 4365, + }, + ], + }, + }, + }; + const withIterations = (iterations: readonly unknown[]) => ({ + ...advisorTurn, + message: { ...advisorTurn.message, usage: { ...advisorTurn.message.usage, iterations } }, + }); + const parse = (turn: unknown) => claudeUsageFormat.parseLine(JSON.stringify(turn), undefined); + + it("reports the advisor model's tokens as their own record beside the unchanged executor record", () => { + const executor = { + provider: "claude", + timestampMs: Date.parse("2026-10-22T01:31:11.984Z"), + model: "claude-opus-5-5", + sessionId: "5a128faa-8253-489e-b935-6c08e8e670c0", + totals: { + uncachedInputTokens: 4, + cachedInputTokens: 118320, + cacheCreationTokens: 2676, + outputTokens: 4506, + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: "standard", + dedupeKey: "msg_1:req_1", + }; + const expected = [ + executor, + { + ...executor, + model: "claude-fable-5-1", + totals: { + uncachedInputTokens: 54204, + cachedInputTokens: 0, + cacheCreationTokens: 0, + outputTokens: 4229, + reasoningTokens: 0, + }, + dedupeKey: expect.any(String), + }, + ]; + + expect(parse(advisorTurn)).toEqual(expected); + expect(claudeUsageFormat.parseProjected(advisorTurn, undefined)).toEqual(expected); + }); + + it("keys the advisor record apart from the executor and the same across a message's content blocks", () => { + const block = (type: string) => + parse({ ...advisorTurn, message: { ...advisorTurn.message, content: [{ type }] } }); + const [executorKey, advisorKey] = block("text").map((record) => record.dedupeKey); + + expect(advisorKey).toEqual(expect.any(String)); + expect(advisorKey).not.toBe(executorKey); + expect(block("tool_use").map((record) => record.dedupeKey)).toEqual([executorKey, advisorKey]); + + const unkeyed = parse({ + ...advisorTurn, + requestId: undefined, + message: { ...advisorTurn.message, id: undefined }, + }); + expect(unkeyed.map((record) => record.dedupeKey)).toEqual([null, null]); + }); + + it("emits only the executor record without an advisor iteration", () => { + const executorIterations = advisorTurn.message.usage.iterations.filter( + (iteration) => iteration.type === "message", + ); + + expect(parse(withIterations(executorIterations))).toHaveLength(1); + expect(claudeUsageFormat.parseProjected(withIterations([]), undefined)).toHaveLength(1); + expect( + claudeUsageFormat.parseLine( + claudeLine({ messageId: "msg_3", contentType: "text" }), + undefined, + ), + ).toHaveLength(1); + }); + + it("ignores advisor iterations without a usable model and malformed entries", () => { + const unusable = withIterations([ + null, + "advisor_message", + { type: "advisor_message", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + { type: "advisor_message", model: "", input_tokens: 5 }, + ]); + + expect(parse(unusable).map((record) => record.model)).toEqual(["claude-opus-5-5"]); + }); +}); diff --git a/apps/server/src/provider/Drivers/claudeUsage.ts b/apps/server/src/provider/Drivers/claudeUsage.ts index c5ff71a5c839..aca7696d31df 100644 --- a/apps/server/src/provider/Drivers/claudeUsage.ts +++ b/apps/server/src/provider/Drivers/claudeUsage.ts @@ -20,35 +20,35 @@ export const CLAUDE_SYNTHETIC_MODEL = ""; * the same complete `usage` object for the parent message. Summing them overcounts by roughly * 2.4x on a real workload, so the caller must drop repeats by `dedupeKey` and keep the first. */ -export function parseClaudeLine(line: string): UsageRecord | null { +export function parseClaudeLine(line: string): readonly UsageRecord[] { let parsed: unknown; try { parsed = JSON.parse(line); } catch { - return null; + return []; } return parseClaudeRecord(parsed); } -function parseClaudeRecord(parsed: unknown): UsageRecord | null { - if (typeof parsed !== "object" || parsed === null) return null; +function parseClaudeRecord(parsed: unknown): readonly UsageRecord[] { + if (typeof parsed !== "object" || parsed === null) return []; const record = parsed as Record; - if (record["type"] !== "assistant") return null; + if (record["type"] !== "assistant") return []; const message = record["message"]; - if (typeof message !== "object" || message === null) return null; + if (typeof message !== "object" || message === null) return []; const messageRecord = message as Record; const usage = messageRecord["usage"]; - if (typeof usage !== "object" || usage === null) return null; + if (typeof usage !== "object" || usage === null) return []; const usageRecord = usage as Record; const timestampMs = parseTimestampMs(record["timestamp"]); - if (timestampMs === null) return null; + if (timestampMs === null) return []; const model = typeof messageRecord["model"] === "string" ? messageRecord["model"] : ""; - if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return null; + if (model.length === 0 || model === CLAUDE_SYNTHETIC_MODEL) return []; const messageId = typeof messageRecord["id"] === "string" ? messageRecord["id"] : null; const requestId = typeof record["requestId"] === "string" ? record["requestId"] : null; @@ -57,7 +57,7 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { const cost = record["costUSD"]; - return { + const executor: UsageRecord = { provider: "claude", timestampMs, model, @@ -74,10 +74,40 @@ function parseClaudeRecord(parsed: unknown): UsageRecord | null { speed: usageRecord["speed"] === "fast" ? "fast" : "standard", dedupeKey, }; -} -const orEmpty = (record: UsageRecord | null): readonly UsageRecord[] => - record === null ? [] : [record]; + // Top-level `usage` is the executor's alone. Advisor calls run on another model at its own rates + // and appear only in `iterations`, so each one is its own record under that model. + const records = [executor]; + const iterations: readonly unknown[] = Array.isArray(usageRecord["iterations"]) + ? usageRecord["iterations"] + : []; + for (const [index, iteration] of iterations.entries()) { + if (typeof iteration !== "object" || iteration === null) continue; + const advisor = iteration as Record; + if (advisor["type"] !== "advisor_message") continue; + + const advisorModel = typeof advisor["model"] === "string" ? advisor["model"] : ""; + if (advisorModel.length === 0 || advisorModel === CLAUDE_SYNTHETIC_MODEL) continue; + + records.push({ + provider: "claude", + timestampMs, + model: advisorModel, + sessionId: executor.sessionId, + totals: { + uncachedInputTokens: tokenCount(advisor["input_tokens"]), + cachedInputTokens: tokenCount(advisor["cache_read_input_tokens"]), + cacheCreationTokens: tokenCount(advisor["cache_creation_input_tokens"]), + outputTokens: tokenCount(advisor["output_tokens"]), + reasoningTokens: 0, + }, + reportedCostUsd: null, + speed: executor.speed, + dedupeKey: dedupeKey === null ? null : `${dedupeKey}:advisor:${index}`, + }); + } + return records; +} export const claudeUsageFormat: TranscriptUsageFormat = { selectFields: { @@ -89,8 +119,8 @@ export const claudeUsageFormat: TranscriptUsageFormat = { message: { id: true, model: true, usage: true }, }, usageMarkers: ['"usage"'], - parseLine: (line) => orEmpty(parseClaudeLine(line)), - parseProjected: (projected) => orEmpty(parseClaudeRecord(projected)), + parseLine: (line) => parseClaudeLine(line), + parseProjected: (projected) => parseClaudeRecord(projected), }; export const claudeUsageReader: ProviderUsageReader = { diff --git a/apps/server/src/usage/UsageService.test.ts b/apps/server/src/usage/UsageService.test.ts index 0fd7e4f6b803..acfd82a6f29e 100644 --- a/apps/server/src/usage/UsageService.test.ts +++ b/apps/server/src/usage/UsageService.test.ts @@ -1980,7 +1980,7 @@ describe("UsageService", () => { 60 + 20 * (version === 4 ? 1 : 2), ); yield* upgraded.awaitPersisted; - assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v9.json"); + assert.strictEqual(SCAN_CACHE_FILE_NAME, "usage-scan-cache-v10.json"); assert.strictEqual( yield* Effect.promise(() => NodeFSP.readFile(legacyPath, "utf8")), legacy, diff --git a/apps/server/src/usage/usageScanCache.test.ts b/apps/server/src/usage/usageScanCache.test.ts index 22eec143d8c7..d7b191fe0af9 100644 --- a/apps/server/src/usage/usageScanCache.test.ts +++ b/apps/server/src/usage/usageScanCache.test.ts @@ -219,7 +219,11 @@ describe("scan cache round trip", () => { }; const restored = decodeScanCache(JSON.parse(JSON.stringify(legacy)), TEST_FORMAT_MAP); - expect(restored.get("/gone.jsonl")).toEqual(original.get("/gone.jsonl")); + expect(restored.get("/gone.jsonl")).toEqual({ + ...original.get("/gone.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); expect(restored.get("/gone-codex.jsonl")).toEqual({ ...original.get("/gone-codex.jsonl"), size: -1, @@ -246,6 +250,18 @@ describe("scan cache round trip", () => { }, ); + it("re-parses a Claude transcript cached before advisor usage was counted", () => { + const original = cacheWith([["/claude.jsonl", 100, [record()]]]); + const v9 = { ...encodeScanCache(original), version: 9 }; + + const restored = decodeScanCache(JSON.parse(JSON.stringify(v9)), TEST_FORMAT_MAP); + expect(restored.get("/claude.jsonl")).toEqual({ + ...original.get("/claude.jsonl"), + size: -1, + position: { resumeOffset: 0, guardLength: 0, guardHash: 0, state: null }, + }); + }); + it("rewrites only changed entries and still restores the whole cache", () => { const write = makeScanCacheWriter(); const cache = cacheWith([ diff --git a/apps/server/src/usage/usageScanCache.ts b/apps/server/src/usage/usageScanCache.ts index 3e9f3a92596a..3a5d910ed3a0 100644 --- a/apps/server/src/usage/usageScanCache.ts +++ b/apps/server/src/usage/usageScanCache.ts @@ -9,7 +9,7 @@ import * as Schema from "effect/Schema"; import { CLAUDE_SYNTHETIC_MODEL } from "../provider/Drivers/claudeUsage.ts"; import { GUARD_LENGTH, type TranscriptParsePosition } from "./usageTranscriptReader.ts"; -const USAGE_SCAN_CACHE_VERSION = 9 as const; +const USAGE_SCAN_CACHE_VERSION = 10 as const; const SPEED_COMPATIBLE_SINCE_VERSION = 4; /** @@ -17,8 +17,9 @@ const SPEED_COMPATIBLE_SINCE_VERSION = 4; * replace it, dropping saved usage for deleted transcripts. Separate files keep * both. */ -export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v9.json"; +export const SCAN_CACHE_FILE_NAME = "usage-scan-cache-v10.json"; export const LEGACY_SCAN_CACHE_FILE_NAMES = [ + "usage-scan-cache-v9.json", "usage-scan-cache-v8.json", "usage-scan-cache-v7.json", "usage-scan-cache-v6.json", @@ -313,10 +314,12 @@ export function decodeScanCache( } // Keep history for deleted rollouts, but re-parse incompatible reducer state. // v9 repairs skipped initial Codex requests; clean v8 entries can still resume. + // v10 adds Claude advisor calls, which no older Claude entry holds. const legacy = - format.state !== undefined && - (version < 6 || - (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0)))); + (entry.p === "claude" && version < 10) || + (format.state !== undefined && + (version < 6 || + (entry.p === "codex" && (version < 8 || (version < 9 && malformedRecords > 0))))); // A corrupt state disqualifies the entry: resuming with it would attach // appended usage to the wrong model or replay fork-copied history. if (!legacy && !isValidState.get(entry.p)?.(entry.cs)) continue;