From c96abb78a7000761e2aa9b51b59c377873aafcae Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Thu, 6 Aug 2026 09:16:13 +0900 Subject: [PATCH] fix(responses): give the bounded JSON read a first-byte deadline distinct from inter-chunk inactivity (#1065) Thinking-capable models can send early or chunked headers and then spend more than 30 seconds generating before emitting the first JSON body byte. Keep the 30-second inactivity deadline once body flow starts, but give the bounded JSON call site its existing 180-second total budget for the first byte. The total deadline wins the no-byte race, so a dead upstream remains bounded. --- src/lib/bounded-body.ts | 4 +- src/server/responses/core.ts | 16 ++++--- tests/bounded-body.test.ts | 83 ++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/src/lib/bounded-body.ts b/src/lib/bounded-body.ts index 2cb7ea39b00..87378cec326 100644 --- a/src/lib/bounded-body.ts +++ b/src/lib/bounded-body.ts @@ -23,6 +23,8 @@ export interface BoundedBodyOptions { totalTimeoutMs?: number; /** Deadline between non-empty raw chunks. Exposed for focused tests. */ inactivityTimeoutMs?: number; + /** Deadline for the first non-empty raw chunk. Defaults to inactivityTimeoutMs. */ + firstByteTimeoutMs?: number; } export interface BoundedBodyResult { @@ -129,7 +131,7 @@ export async function readBoundedResponseBody( let cancelReason: unknown; const total = timeoutPromise(options.totalTimeoutMs ?? BOUNDED_BODY_TIMEOUT_MS, TOTAL_TIMEOUT); let inactivity = timeoutPromise( - options.inactivityTimeoutMs ?? BOUNDED_BODY_TIMEOUT_MS, + options.firstByteTimeoutMs ?? options.inactivityTimeoutMs ?? BOUNDED_BODY_TIMEOUT_MS, INACTIVITY_TIMEOUT, ); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 29b087424bc..56d8505404c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -747,10 +747,18 @@ const UNREADABLE_ENCRYPTED_AGENT_TASK_MESSAGE = // branch of the passthrough return path). 32 MiB matches the continuation snapshot read // bound and is far above any legitimate non-streaming completion, including base64 image // payloads. The stall deadlines only govern the body transfer β€” generation time before -// the response headers is untouched. +// the response headers is untouched. Generation after early/chunked headers but before +// the first body byte previously used the 30-second inactivity deadline; this call site +// gives it the full body deadline instead. const MAX_UPSTREAM_JSON_BODY_BYTES = 32 * 1024 * 1024; const UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS = 180_000; const UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS = 30_000; +export const UPSTREAM_JSON_BODY_READ_OPTIONS = { + maxBytes: MAX_UPSTREAM_JSON_BODY_BYTES, + totalTimeoutMs: UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS, + inactivityTimeoutMs: UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS, + firstByteTimeoutMs: UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS, +}; function unreadableEncryptedAgentTaskResponse(): Response { return new Response( @@ -2226,11 +2234,7 @@ async function handleResponsesInner( // without limit. This path is no longer rare β€” WebSocket turns for models whose // streaming terminal event is unreliable are deliberately answered with bounded JSON. // Oversize and stall deadlines both fail closed; a partial body is never parsed. - const bounded = await readBoundedResponseBody(upstreamResponse, { - maxBytes: MAX_UPSTREAM_JSON_BODY_BYTES, - totalTimeoutMs: UPSTREAM_JSON_BODY_TOTAL_TIMEOUT_MS, - inactivityTimeoutMs: UPSTREAM_JSON_BODY_INACTIVITY_TIMEOUT_MS, - }); + const bounded = await readBoundedResponseBody(upstreamResponse, UPSTREAM_JSON_BODY_READ_OPTIONS); if (bounded.oversized) { return formatErrorResponse(502, "upstream_error", "upstream JSON response exceeded the safe body limit"); } diff --git a/tests/bounded-body.test.ts b/tests/bounded-body.test.ts index 41584c643c9..2bf9efff8ef 100644 --- a/tests/bounded-body.test.ts +++ b/tests/bounded-body.test.ts @@ -4,6 +4,7 @@ import { boundedBodyBufferGrowthsForTests, readBoundedResponseBody, } from "../src/lib/bounded-body"; +import { UPSTREAM_JSON_BODY_READ_OPTIONS } from "../src/server/responses/core"; const encoder = new TextEncoder(); @@ -18,6 +19,11 @@ function responseFromChunks(...chunks: Uint8Array[]): Response { } describe("readBoundedResponseBody", () => { + test("the bounded JSON caller allows a full total deadline for its first byte", () => { + expect(UPSTREAM_JSON_BODY_READ_OPTIONS.firstByteTimeoutMs) + .toBe(UPSTREAM_JSON_BODY_READ_OPTIONS.totalTimeoutMs); + }); + test("reads multiple chunks and flushes split UTF-8", async () => { const bytes = encoder.encode("alpha ν•œκΈ€ 🌍"); const response = responseFromChunks(bytes.subarray(0, 8), bytes.subarray(8, 11), bytes.subarray(11)); @@ -52,6 +58,83 @@ describe("readBoundedResponseBody", () => { expect(cancelled).toBe(true); }); + test("allows the first byte to arrive after the inter-chunk inactivity deadline", async () => { + const response = new Response(new ReadableStream({ + start(controller) { + setTimeout(() => { + controller.enqueue(encoder.encode("first byte")); + controller.close(); + }, 30); + }, + })); + + const result = await readBoundedResponseBody(response, { + totalTimeoutMs: 100, + inactivityTimeoutMs: 15, + firstByteTimeoutMs: 60, + }); + + expect(result.text).toBe("first byte"); + expect(result.truncated).toBe(false); + expect(result.inactivityTimedOut).toBe(false); + }); + + test("times out when the first byte misses its dedicated deadline", async () => { + const response = new Response(new ReadableStream({})); + + const result = await readBoundedResponseBody(response, { + totalTimeoutMs: 100, + inactivityTimeoutMs: 60, + firstByteTimeoutMs: 15, + }); + + expect(result.inactivityTimedOut).toBe(true); + expect(result.totalTimedOut).toBe(false); + }); + + test("keeps the inter-chunk inactivity deadline after the first byte", async () => { + let timer: ReturnType | undefined; + const response = new Response(new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("first")); + timer = setTimeout(() => controller.enqueue(encoder.encode("late")), 30); + }, + cancel() { + if (timer) clearTimeout(timer); + }, + })); + + const result = await readBoundedResponseBody(response, { + totalTimeoutMs: 100, + inactivityTimeoutMs: 15, + firstByteTimeoutMs: 60, + }); + + expect(result.text).toBe("first"); + expect(result.truncated).toBe(true); + expect(result.inactivityTimedOut).toBe(true); + }); + + test("uses the inactivity deadline for the first byte by default", async () => { + let timer: ReturnType | undefined; + const response = new Response(new ReadableStream({ + start(controller) { + timer = setTimeout(() => controller.enqueue(encoder.encode("late")), 30); + }, + cancel() { + if (timer) clearTimeout(timer); + }, + })); + + const result = await readBoundedResponseBody(response, { + totalTimeoutMs: 100, + inactivityTimeoutMs: 15, + }); + + expect(result.inactivityTimedOut).toBe(true); + expect(result.totalTimedOut).toBe(false); + }); + test("a partial body followed by silence times out and flushes UTF-8", async () => { const response = new Response(new ReadableStream({ start(controller) {