From 50dab161609df9e762384b4a8e7042e1322e6cdb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 22:05:10 +0000 Subject: [PATCH 1/5] test(client-runtime): reproduce #33089 stalled setup survives tab return A connection attempt that stalls while a web tab is hidden keeps running after an `application-active` wakeup, so the client waits out the 15 s establishment timeout instead of starting a fresh attempt. This test reproduces pingdotgg/t3code#33089. AI-assisted; verified with the tests and tools listed above. --- .../src/connection/supervisor.test.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index f284c48e82772..c3d6483968360 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -683,6 +683,41 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("restarts connection setup that stalled while a web tab was hidden", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* harness.closeLatestSession(); + yield* awaitState( + supervisor.state, + (state) => state.phase === "backoff" && state.attempt === 1, + ); + yield* TestClock.adjust("3 seconds"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connecting" && state.attempt === 2, + ); + + yield* harness.wake("application-active"); + yield* TestClock.adjust("1 second"); + + expect( + yield* Ref.get(harness.prepareCount), + "a visible tab should start a fresh attempt instead of waiting out the stalled one", + ).toBe(3); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1, + ); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("probes the active session without reconnecting on application activation", () => Effect.gen(function* () { const probeCount = yield* Ref.make(0); From 40f203054ca135ae11695e8bb9b997360580747a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 22:08:42 +0000 Subject: [PATCH 2/5] fix(client-runtime): returning to a web tab restarts a stalled reconnect A reconnect that started while an Android web tab was hidden can stall on a request that never settles. Returning to the tab emitted application-active, which the in-flight attempt ignored, so recovery waited out the 15 s setup timeout and the backoff after it. application-active now interrupts an in-flight attempt and resets the retry ladder when that attempt is a reconnect (attempt > 1 or an earlier failure), the same condition the UI uses to show "Reconnecting". A first attempt still ignores it, and mobile's application-active-probe is unchanged. AI-assisted; verified with the tests and tools listed above. --- docs/internals/connection-runtime.md | 3 ++- packages/client-runtime/src/connection/supervisor.ts | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/internals/connection-runtime.md b/docs/internals/connection-runtime.md index 698ddac7a73f9..464541ffb5b79 100644 --- a/docs/internals/connection-runtime.md +++ b/docs/internals/connection-runtime.md @@ -32,7 +32,8 @@ suspension's probe that times out also replaces the socket regardless of the network report. The web and desktop foreground probe skips the fresh request, because a busy host can answer it before a queued probe. Foregrounding also wakes a pending retry immediately. A long suspension restarts an in-flight -attempt; a short one leaves it alone. +attempt, and so does a web or desktop tab returning to view during a reconnect; +a short mobile suspension and a first attempt leave it alone. The [registry](../../packages/client-runtime/src/connection/registry.ts) scopes connections by environment. An involuntary disconnect retains the registration diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 8f7536605810b..3c30c4f774aba 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -579,7 +579,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ); }); - const waitForEstablishmentInterrupt = Effect.fnUntraced(function* () { + const waitForEstablishmentInterrupt = Effect.fnUntraced(function* (reconnecting: boolean) { // A primary attempt keeps running through an offline report, so the // network returning restarts it in case it stalled meanwhile. let deviceOffline = (yield* connectivity.status) === "offline"; @@ -608,6 +608,11 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( if (next.reason === "application-active-long") { return true; } + // A reconnect that started while a web tab was hidden can stall on + // a request that never settles, so returning to the tab restarts it. + if (next.reason === "application-active" && reconnecting) { + return true; + } if (next.reason === "credentials-changed" && usesRelay) { yield* logManagedRelayAccountChange; return false; @@ -899,7 +904,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( exit, })), ), - waitForEstablishmentInterrupt().pipe( + // The same condition labels the attempt "reconnecting" for the user. + waitForEstablishmentInterrupt(attempt > 1 || lastFailure !== null).pipe( Effect.map((resetRetry): EstablishmentEvent => ({ _tag: "Interrupted", resetRetry, From 0182a07e273656570f3e6d184ce1c936ca808a15 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 22:26:25 +0000 Subject: [PATCH 3/5] fix(client-runtime): a restarted T3 Connect attempt starts a fresh credential renewal Credential renewal is shared between callers, so the attempt the supervisor starts after a tab returns joined the renewal its stalled predecessor was waiting on and still waited out the 30 second renewal timeout. An interrupted connection attempt now forgets the pending renewal it was waiting on without interrupting it: HTTP requests that joined it keep waiting, and the next attempt issues its own bootstrap and token exchange. AI-assisted; verified with the tests and tools listed above. --- docs/internals/connection-runtime.md | 4 +- .../src/authorization/layer.test.ts | 46 +++++++++++++++++++ .../src/authorization/service.ts | 46 ++++++++++++------- .../src/connection/supervisor.ts | 3 +- 4 files changed, 79 insertions(+), 20 deletions(-) diff --git a/docs/internals/connection-runtime.md b/docs/internals/connection-runtime.md index 464541ffb5b79..dd5128bd32905 100644 --- a/docs/internals/connection-runtime.md +++ b/docs/internals/connection-runtime.md @@ -33,7 +33,9 @@ network report. The web and desktop foreground probe skips the fresh request, because a busy host can answer it before a queued probe. Foregrounding also wakes a pending retry immediately. A long suspension restarts an in-flight attempt, and so does a web or desktop tab returning to view during a reconnect; -a short mobile suspension and a first attempt leave it alone. +a short mobile suspension and a first attempt leave it alone. A restarted attempt +starts its own T3 Connect credential renewal instead of joining the one the +replaced attempt was waiting on, which may be stalled too. The [registry](../../packages/client-runtime/src/connection/registry.ts) scopes connections by environment. An involuntary disconnect retains the registration diff --git a/packages/client-runtime/src/authorization/layer.test.ts b/packages/client-runtime/src/authorization/layer.test.ts index 9ed7b9a63e9d3..bf1dd0f7c1628 100644 --- a/packages/client-runtime/src/authorization/layer.test.ts +++ b/packages/client-runtime/src/authorization/layer.test.ts @@ -860,6 +860,52 @@ describe("RemoteEnvironmentAuthorization", () => { }), ); + it.effect("starts a fresh renewal for a connection attempt that replaces a stalled one", () => + Effect.gen(function* () { + const stalled = yield* Deferred.make(); + const release = yield* Deferred.make(); + const bootstraps = yield* Ref.make(0); + const harness = yield* makeHarness({ + beforeBootstrap: Ref.getAndUpdate(bootstraps, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 0 + ? Deferred.succeed(stalled, undefined).pipe(Effect.andThen(Deferred.await(release))) + : Effect.void, + ), + ), + responses: [ + Response.json(DESCRIPTOR), + accessToken("fresh-token"), + websocketTicket("fresh-ticket"), + Response.json(DESCRIPTOR), + accessToken("late-token"), + ], + }); + yield* Effect.gen(function* () { + const remote = yield* RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization; + const stalledAttempt = yield* remote + .authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID }) + .pipe(Effect.forkChild); + yield* Deferred.await(stalled); + const request = yield* remote + .authorizeDpopHttp({ expectedEnvironmentId: ENVIRONMENT_ID }) + .pipe(Effect.forkChild); + yield* Queue.takeN(harness.tokenReads, 2); + yield* Fiber.interrupt(stalledAttempt); + + const authorized = yield* remote.authorizeDpop({ expectedEnvironmentId: ENVIRONMENT_ID }); + expect(authorized.socketUrl).toContain("wsTicket=fresh-ticket"); + expect(yield* Ref.get(harness.bootstrapCalls)).toBe(2); + + yield* Deferred.succeed(release, undefined); + const late = yield* Fiber.join(request); + expect(late.httpAuthorization.accessToken, "a request that joined it keeps waiting").toBe( + "late-token", + ); + }).pipe(Effect.provide(harness.layer)); + }), + ); + it.effect("does not block another environment behind a pending renewal", () => Effect.gen(function* () { const started = yield* Deferred.make(); diff --git a/packages/client-runtime/src/authorization/service.ts b/packages/client-runtime/src/authorization/service.ts index 9b4d9c2f1f892..45116eba984cb 100644 --- a/packages/client-runtime/src/authorization/service.ts +++ b/packages/client-runtime/src/authorization/service.ts @@ -341,8 +341,15 @@ export const make = Effect.gen(function* () { }, ); + const forgetPendingToken = (environmentId: EnvironmentId, id: object) => { + if (pendingTokens.get(environmentId)?.id === id) { + pendingTokens.delete(environmentId); + } + }; + const getDpopToken = Effect.fn("clientRuntime.connection.remote.getDpopToken")(function* ( input: Parameters[0], + connectionAttempt = false, ) { const session = yield* cloudSession.identity; if (Option.isNone(session)) { @@ -380,14 +387,14 @@ export const make = Effect.gen(function* () { accessToken: cached.value.accessToken, identity, }); - return { token: Effect.succeed(cached.value), fromCache: true }; + return { token: Effect.succeed(cached.value), pendingId: undefined, fromCache: true }; } if (input.rejectedAccessToken !== undefined) { yield* removeRejectedToken(input.expectedEnvironmentId, input.rejectedAccessToken); } const pending = pendingTokens.get(input.expectedEnvironmentId); if (pending?.identity === identity && pending.thumbprint === thumbprint) { - return { token: pending.token, fromCache: false }; + return { token: pending.token, pendingId: pending.id, fromCache: false }; } const id = {}; @@ -412,9 +419,7 @@ export const make = Effect.gen(function* () { accessToken: token.accessToken, identity, }); - if (pendingTokens.get(input.expectedEnvironmentId)?.id === id) { - pendingTokens.delete(input.expectedEnvironmentId); - } + forgetPendingToken(input.expectedEnvironmentId, id); return token; }), ), @@ -431,11 +436,7 @@ export const make = Effect.gen(function* () { }), Effect.ensuring( tokenLock.withPermits(1)( - Effect.sync(() => { - if (pendingTokens.get(input.expectedEnvironmentId)?.id === id) { - pendingTokens.delete(input.expectedEnvironmentId); - } - }), + Effect.sync(() => forgetPendingToken(input.expectedEnvironmentId, id)), ), ), // One caller's timeout must not cancel renewal for other HTTP requests. @@ -443,13 +444,24 @@ export const make = Effect.gen(function* () { ); const token = Fiber.join(fiber); pendingTokens.set(input.expectedEnvironmentId, { id, identity, thumbprint, token }); - return { token, fromCache: false }; + return { token, pendingId: id, fromCache: false }; }), ); yield* Effect.annotateCurrentSpan({ "connection.remote_token_cache": selected.fromCache ? "hit" : "miss", }); - const token = yield* selected.token; + const { pendingId } = selected; + const token = yield* connectionAttempt && pendingId !== undefined + ? // Attempts are interrupted to replace a stalled one, so the replacement + // starts its own renewal while other waiters keep this one. + selected.token.pipe( + Effect.onInterrupt(() => + tokenLock.withPermits(1)( + Effect.sync(() => forgetPendingToken(input.expectedEnvironmentId, pendingId)), + ), + ), + ) + : selected.token; yield* assertSession(identity); return { token, fromCache: selected.fromCache, identity }; }); @@ -485,7 +497,7 @@ export const make = Effect.gen(function* () { socketUrl, ...(descriptor === undefined ? {} : { descriptor }), }); - let selected = yield* getDpopToken(input); + let selected = yield* getDpopToken(input, true); if (selected.fromCache) { const cachedSocket = yield* createDpopSocketUrl( selected.token, @@ -505,10 +517,10 @@ export const make = Effect.gen(function* () { return yield* mapDpopSocketError(cachedSocket.failure); } yield* assertSession(selected.identity); - selected = yield* getDpopToken({ - ...input, - rejectedAccessToken: selected.token.accessToken, - }); + selected = yield* getDpopToken( + { ...input, rejectedAccessToken: selected.token.accessToken }, + true, + ); } const socket = yield* createDpopSocketUrl(selected.token, undefined, endpoint).pipe( Effect.result, diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 3c30c4f774aba..a3d35aea9ca4b 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -608,8 +608,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( if (next.reason === "application-active-long") { return true; } - // A reconnect that started while a web tab was hidden can stall on - // a request that never settles, so returning to the tab restarts it. + // A reconnect may have stalled while the tab was hidden, so returning restarts it. if (next.reason === "application-active" && reconnecting) { return true; } From aebb2e6ec6870906fa0fee0bd5553ab9ae4de15d Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 22:35:53 +0000 Subject: [PATCH 4/5] fix(client-runtime): a second tab return also restarts the replacement attempt The attempt that replaces a stalled reconnect starts at attempt 1 with no recorded failure, so a later tab return left it alone. It now counts as a reconnect too. AI-assisted; verified with the tests and tools listed above. --- .../client-runtime/src/connection/supervisor.test.ts | 11 ++++++++++- packages/client-runtime/src/connection/supervisor.ts | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index c3d6483968360..90f3019e12045 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -686,7 +686,8 @@ describe("EnvironmentSupervisor", () => { it.effect("restarts connection setup that stalled while a web tab was hidden", () => Effect.gen(function* () { const harness = yield* makeHarness({ - prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)), + prepare: (attempt) => + attempt === 2 || attempt === 3 ? Effect.never : Effect.succeed(PREPARED_CONNECTION), }); const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { initiallyDesired: true, @@ -711,6 +712,14 @@ describe("EnvironmentSupervisor", () => { yield* Ref.get(harness.prepareCount), "a visible tab should start a fresh attempt instead of waiting out the stalled one", ).toBe(3); + + yield* harness.wake("application-active"); + yield* TestClock.adjust("1 second"); + + expect( + yield* Ref.get(harness.prepareCount), + "returning again should also restart the replacement attempt", + ).toBe(4); yield* awaitState( supervisor.state, (state) => state.phase === "connected" && state.generation === 2 && state.attempt === 1, diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index a3d35aea9ca4b..d636c86a6cf69 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -903,8 +903,8 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( exit, })), ), - // The same condition labels the attempt "reconnecting" for the user. - waitForEstablishmentInterrupt(attempt > 1 || lastFailure !== null).pipe( + // Any attempt that follows a lost session, a failure, or a restart is a reconnect. + waitForEstablishmentInterrupt(attempt > 1 || lastFailure !== null || ignoreOffline).pipe( Effect.map((resetRetry): EstablishmentEvent => ({ _tag: "Interrupted", resetRetry, From d1e1bd663555e3bf05bc902d472a39720ca59fe2 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 8 Nov 2026 22:46:07 +0000 Subject: [PATCH 5/5] fix(client-runtime): any reconnect after a session came up restarts on tab return A lost session whose retry ladder was reset (a tab return during backoff, a failed health check) reconnects as attempt 1 with no recorded failure, so a later tab return left it stalled. Count every attempt after the first connection as a reconnect. AI-assisted; verified with the tests and tools listed above. --- docs/internals/connection-runtime.md | 6 ++-- .../src/connection/supervisor.test.ts | 35 +++++++++++++++++++ .../src/connection/supervisor.ts | 6 ++-- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/internals/connection-runtime.md b/docs/internals/connection-runtime.md index dd5128bd32905..8abe1fd4aa014 100644 --- a/docs/internals/connection-runtime.md +++ b/docs/internals/connection-runtime.md @@ -33,9 +33,9 @@ network report. The web and desktop foreground probe skips the fresh request, because a busy host can answer it before a queued probe. Foregrounding also wakes a pending retry immediately. A long suspension restarts an in-flight attempt, and so does a web or desktop tab returning to view during a reconnect; -a short mobile suspension and a first attempt leave it alone. A restarted attempt -starts its own T3 Connect credential renewal instead of joining the one the -replaced attempt was waiting on, which may be stalled too. +a short mobile suspension and a first attempt leave it alone. An interrupted +attempt stops sharing its pending T3 Connect credential renewal, so the next +attempt starts its own instead of joining one that may be stalled too. The [registry](../../packages/client-runtime/src/connection/registry.ts) scopes connections by environment. An involuntary disconnect retains the registration diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index 90f3019e12045..9b54137f3328a 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -727,6 +727,41 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("restarts a reconnect that a tab return started from backoff", () => + Effect.gen(function* () { + const harness = yield* makeHarness({ + prepare: (attempt) => (attempt === 2 ? Effect.never : Effect.succeed(PREPARED_CONNECTION)), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* harness.closeLatestSession(); + yield* awaitState( + supervisor.state, + (state) => state.phase === "backoff" && state.attempt === 1, + ); + yield* harness.wake("application-active"); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connecting" && state.attempt === 1, + ); + + yield* harness.wake("application-active"); + yield* TestClock.adjust("1 second"); + + expect( + yield* Ref.get(harness.prepareCount), + "a reconnect whose retry count was reset is still restarted by a tab return", + ).toBe(3); + yield* awaitState( + supervisor.state, + (state) => state.phase === "connected" && state.generation === 2, + ); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("probes the active session without reconnecting on application activation", () => Effect.gen(function* () { const probeCount = yield* Ref.make(0); diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index d636c86a6cf69..8ea1ef75e3cee 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -903,8 +903,10 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( exit, })), ), - // Any attempt that follows a lost session, a failure, or a restart is a reconnect. - waitForEstablishmentInterrupt(attempt > 1 || lastFailure !== null || ignoreOffline).pipe( + // Only the first attempt of the first connection is not a reconnect. + waitForEstablishmentInterrupt( + generation > 1 || attempt > 1 || lastFailure !== null || ignoreOffline, + ).pipe( Effect.map((resetRetry): EstablishmentEvent => ({ _tag: "Interrupted", resetRetry,