From b6114d76e59a6c261c4e18bec675166f5569cbcd Mon Sep 17 00:00:00 2001 From: Michiel de Jong Date: Fri, 25 Sep 2026 12:02:49 +0200 Subject: [PATCH] Keep a hidden tab's socket open, and hold presence sent during subscribe A hidden tab's timers fire about once a minute, so the 5 s liveness check first saw a quiet socket after 60 s of silence, past the 45 s deadline, and closed it without ever sending the KEEPALIVE probe. The client now closes only when a probe it sent has gone unanswered for 25 s, measured from when it was sent. After a reconnect the client subscribes to drive presence and announces itself straight away. The server's subscribe runs an async read check and keeps handling messages meanwhile, so the announcement arrived before the connection counted as a subscriber: it was dropped with "presence update from non-subscriber". The server now holds the latest update from a connection whose subscribe is still being checked, delivers it once the check passes, and drops it if the check fails. Closes #1800 Co-Authored-By: Claude Opus 5.5 --- TESTING_COVERAGE.md | 12 +++ browser/lib/src/liveness.test.ts | 101 ++++++++++++++++++++--- browser/lib/src/liveness.ts | 31 +++++--- browser/lib/src/websockets.ts | 24 +++--- docs/src/websockets.md | 8 +- server/src/commit_monitor.rs | 128 ++++++++++++++++++++++++++---- server/tests/it/drive_presence.rs | 78 ++++++++++++++++-- 7 files changed, 325 insertions(+), 57 deletions(-) diff --git a/TESTING_COVERAGE.md b/TESTING_COVERAGE.md index 713bb8d235..8a7a138b44 100644 --- a/TESTING_COVERAGE.md +++ b/TESTING_COVERAGE.md @@ -2284,3 +2284,15 @@ deployed app's base64 JSON format with an `atomic:agent:` subject through the same `Agent.fromSecret` parser used by the local welcome form. It verifies the identity and public key. Browser sign-in and data recovery are separate flows. + +## Hidden-tab liveness and presence after reconnect (#1800) + +`browser/lib/src/liveness.test.ts` replays the WS liveness timer at a hidden +tab's once-a-minute cadence and checks that a socket whose probes are answered +is never closed, while a dead one still closes one tick after its probe times +out. It does not drive a real browser's throttling. +`server/tests/it/drive_presence.rs` +(`presence_sent_right_after_subscribe_is_delivered`) sends a presence update +right behind `PRESENCE_SUBSCRIBE`, as every reconnect does, and checks it +reaches the other subscriber without a retry, and that an update held for a +refused subscribe is dropped. diff --git a/browser/lib/src/liveness.test.ts b/browser/lib/src/liveness.test.ts index f6c55b72ab..fe413f3aeb 100644 --- a/browser/lib/src/liveness.test.ts +++ b/browser/lib/src/liveness.test.ts @@ -1,8 +1,9 @@ import { describe, it } from 'vitest'; import { livenessAction, - LIVENESS_DEADLINE_MS, + LIVENESS_CHECK_MS, LIVENESS_IDLE_MS, + LIVENESS_PROBE_TIMEOUT_MS, } from './liveness.js'; /** @@ -12,25 +13,103 @@ import { */ describe('livenessAction', () => { it('does nothing while frames keep arriving', ({ expect }) => { - expect(livenessAction(0, false)).toBe('none'); - expect(livenessAction(LIVENESS_IDLE_MS - 1, false)).toBe('none'); + expect(livenessAction(0, undefined)).toBe('none'); + expect(livenessAction(LIVENESS_IDLE_MS - 1, undefined)).toBe('none'); }); it('probes once past the idle threshold', ({ expect }) => { - expect(livenessAction(LIVENESS_IDLE_MS, false)).toBe('probe'); + expect(livenessAction(LIVENESS_IDLE_MS, undefined)).toBe('probe'); }); it('does not probe twice while one is outstanding', ({ expect }) => { - expect(livenessAction(LIVENESS_IDLE_MS, true)).toBe('none'); - expect(livenessAction(LIVENESS_DEADLINE_MS - 1, true)).toBe('none'); + expect(livenessAction(LIVENESS_IDLE_MS, 0)).toBe('none'); + expect( + livenessAction( + LIVENESS_IDLE_MS + LIVENESS_PROBE_TIMEOUT_MS - 1, + LIVENESS_PROBE_TIMEOUT_MS - 1, + ), + ).toBe('none'); }); - it('closes once the deadline passes, probe or not', ({ expect }) => { - expect(livenessAction(LIVENESS_DEADLINE_MS, true)).toBe('close'); - expect(livenessAction(LIVENESS_DEADLINE_MS, false)).toBe('close'); + it('closes once a probe has gone unanswered past its timeout', ({ + expect, + }) => { + expect( + livenessAction( + LIVENESS_IDLE_MS + LIVENESS_PROBE_TIMEOUT_MS, + LIVENESS_PROBE_TIMEOUT_MS, + ), + ).toBe('close'); }); - it('keeps the deadline after the idle threshold', ({ expect }) => { - expect(LIVENESS_DEADLINE_MS).toBeGreaterThan(LIVENESS_IDLE_MS); + it('never closes on silence alone, however long', ({ expect }) => { + // A hidden tab's timer can first run minutes after the last frame. + // Without a probe out, the server has not been asked yet. + expect(livenessAction(10 * 60_000, undefined)).toBe('probe'); + }); +}); + +/** + * #1800: in a hidden tab Chrome batches timers to once a minute, so the 5 s + * liveness check ran every 60 s. Its first look at a quiet socket saw 60 s + * of silence, past the old 45 s deadline, and closed a healthy connection + * without ever sending the probe ("no frame from the server for 45000ms + * (probe unanswered)"). The app then showed "Working offline". + * + * Replays the WSClient timer loop against a server that answers every probe + * within a round trip, at the throttled cadence and at the normal one. + */ +describe('liveness under timer throttling', () => { + function simulate(tickMs: number, durationMs: number, rttMs: number) { + let now = 0; + let lastFrameAt = 0; + let probeSentAt: number | undefined; + let echoAt: number | undefined; + const actions: string[] = []; + + for (now = tickMs; now <= durationMs; now += tickMs) { + // Socket events are not throttled: an echo lands when it arrives. + if (echoAt !== undefined && echoAt <= now) { + lastFrameAt = echoAt; + probeSentAt = undefined; + echoAt = undefined; + } + + const action = livenessAction( + now - lastFrameAt, + probeSentAt === undefined ? undefined : now - probeSentAt, + ); + actions.push(action); + + if (action === 'probe') { + probeSentAt = now; + echoAt = now + rttMs; + } else if (action === 'close') { + break; + } + } + + return actions; + } + + it('keeps a healthy socket open when timers fire once a minute', ({ + expect, + }) => { + const actions = simulate(60_000, 30 * 60_000, 50); + expect(actions).not.toContain('close'); + expect(actions).toContain('probe'); + }); + + it('keeps a healthy socket open at the normal cadence', ({ expect }) => { + const actions = simulate(LIVENESS_CHECK_MS, 30 * 60_000, 50); + expect(actions).not.toContain('close'); + }); + + it('still closes a dead socket, one tick after the probe times out', ({ + expect, + }) => { + // The echo never comes. + const actions = simulate(60_000, 30 * 60_000, Infinity); + expect(actions).toEqual(['probe', 'close']); }); }); diff --git a/browser/lib/src/liveness.ts b/browser/lib/src/liveness.ts index 5468c5b877..91cd848c31 100644 --- a/browser/lib/src/liveness.ts +++ b/browser/lib/src/liveness.ts @@ -4,14 +4,21 @@ * alone whether to probe (`KEEPALIVE`) or give the socket up. The decision * is a pure function of two numbers, kept apart from the socket so it can * be tested without one. + * + * Silence alone never closes the socket; only an unanswered probe does. + * A hidden tab's timers are throttled (Chrome batches them to once a minute), + * so the check can first run long after the socket went quiet. Closing on + * silence then dropped a healthy connection without ever asking the server, + * every minute the tab stayed hidden (#1800). */ /** After this long without any inbound frame, send a `KEEPALIVE` probe. */ export const LIVENESS_IDLE_MS = 20_000; -/** After this long without any inbound frame, the socket is presumed dead - * and closed so the reconnect loop takes over. Comfortably more than the - * idle threshold plus a round trip, so one slow echo is not a disconnect. */ -export const LIVENESS_DEADLINE_MS = 45_000; +/** How long a probe may go unanswered before the socket is presumed dead + * and closed so the reconnect loop takes over. Measured from when the probe + * was sent, not from the last frame, so a late timer cannot skip the probe. + * Comfortably more than a round trip, so one slow echo is not a disconnect. */ +export const LIVENESS_PROBE_TIMEOUT_MS = 25_000; /** How often the liveness timer looks. */ export const LIVENESS_CHECK_MS = 5_000; @@ -19,16 +26,20 @@ export type LivenessAction = 'none' | 'probe' | 'close'; /** * What the liveness timer should do given how long the socket has been - * silent and whether a probe is already outstanding. Pure, so it is testable - * without a socket: `probe` once past the idle threshold, `close` once past - * the deadline, otherwise nothing. + * silent and how long the outstanding probe has gone unanswered (`undefined` + * when none is outstanding). Pure, so it is testable without a socket: + * `probe` once past the idle threshold, `close` once a probe has waited past + * its timeout, otherwise nothing. */ export function livenessAction( idleMs: number, - probeSent: boolean, + probeAgeMs: number | undefined, ): LivenessAction { - if (idleMs >= LIVENESS_DEADLINE_MS) return 'close'; - if (idleMs >= LIVENESS_IDLE_MS && !probeSent) return 'probe'; + if (probeAgeMs !== undefined) { + return probeAgeMs >= LIVENESS_PROBE_TIMEOUT_MS ? 'close' : 'none'; + } + + if (idleMs >= LIVENESS_IDLE_MS) return 'probe'; return 'none'; } diff --git a/browser/lib/src/websockets.ts b/browser/lib/src/websockets.ts index 35392085ef..d6fc4e3262 100644 --- a/browser/lib/src/websockets.ts +++ b/browser/lib/src/websockets.ts @@ -72,7 +72,7 @@ import { hexToBytes } from './value.js'; import { livenessAction, LIVENESS_CHECK_MS, - LIVENESS_DEADLINE_MS, + LIVENESS_PROBE_TIMEOUT_MS, } from './liveness.js'; import { perfMark, perfSpan } from './perf-trace.js'; @@ -248,13 +248,14 @@ export class WSClient { private _rbsrItemsQueue: Array<(items: Item[]) => void> = []; /** What the server said it speaks, from its `AUTH_OK` payload. */ private _serverCaps: string[] = []; - /** Liveness: when the last inbound frame arrived, whether a `KEEPALIVE` - * probe is outstanding, and the timer that checks both. A browser cannot + /** Liveness: when the last inbound frame arrived, when the outstanding + * `KEEPALIVE` probe was sent (`undefined` when none is), and the timer + * that checks both. A browser cannot * observe the server's protocol-level pings, so without this a socket * the network silently dropped stays "connected" until the next write * fails — every subscription push in between is lost. */ private _lastFrameAt = 0; - private _probeSent = false; + private _probeSentAt: number | undefined; private _livenessTimer: ReturnType | undefined; /** When true, all WS frames are logged to the console in human-readable form. */ @@ -1105,7 +1106,7 @@ export class WSClient { private handleMessage(ev: MessageEvent) { // Any inbound frame proves the socket is alive. this._lastFrameAt = Date.now(); - this._probeSent = false; + this._probeSentAt = undefined; if (ev.data instanceof ArrayBuffer) { this.handleBinary(new Uint8Array(ev.data)); @@ -1675,28 +1676,29 @@ export class WSClient { * Start the liveness timer for the current socket. Only probes a server * that advertised `keepalive` (older servers do not echo the frame, and a * probe that is never answered would make an idle socket look dead every - * `LIVENESS_DEADLINE_MS`). Anonymous sessions never authenticate, so they + * `LIVENESS_PROBE_TIMEOUT_MS`). Anonymous sessions never authenticate, so they * learn no capabilities and keep the pre-2026-09 reactive behaviour. */ private startLiveness() { this.stopLiveness(); this._lastFrameAt = Date.now(); - this._probeSent = false; + this._probeSentAt = undefined; this._livenessTimer = setInterval(() => { if (this.readyState !== WebSocket.OPEN) return; if (!this._serverCaps.includes('keepalive')) return; + const now = Date.now(); const action = livenessAction( - Date.now() - this._lastFrameAt, - this._probeSent, + now - this._lastFrameAt, + this._probeSentAt === undefined ? undefined : now - this._probeSentAt, ); if (action === 'probe') { - this._probeSent = true; + this._probeSentAt = now; this.sendBinary(encodeKeepalive()); } else if (action === 'close') { console.warn( - `[WS] no frame from the server for ${LIVENESS_DEADLINE_MS}ms (probe unanswered); closing so the reconnect loop takes over`, + `[WS] keepalive probe unanswered for ${LIVENESS_PROBE_TIMEOUT_MS}ms; closing so the reconnect loop takes over`, ); this.stopLiveness(); // Closing fires the `close` handler, which reports disconnection, diff --git a/docs/src/websockets.md b/docs/src/websockets.md index 55c9baa103..49e44803db 100644 --- a/docs/src/websockets.md +++ b/docs/src/websockets.md @@ -818,9 +818,11 @@ saturated renderer can stall pong delivery for seconds. A browser cannot observe protocol-level pings, so it has its own probe. When the server advertised `keepalive`, the client checks every 5 s: after 20 s -with no inbound frame it sends one `KEEPALIVE (0x41)`; if nothing has arrived -after 45 s it closes the socket and lets the reconnect loop take over. Any -inbound frame resets both. The server **echoes** `KEEPALIVE` verbatim, which +with no inbound frame it sends one `KEEPALIVE (0x41)`; if that probe is still +unanswered 25 s after it was sent, it closes the socket and lets the reconnect +loop take over. Any inbound frame resets both. Silence alone never closes the +socket: a hidden tab's timers can fire a minute apart, and the first check +after such a gap must ask the server rather than give up on it. The server **echoes** `KEEPALIVE` verbatim, which is the whole point of it. Against a server that did not advertise `keepalive` the client does not probe, since an unanswered probe would make every idle socket look dead. diff --git a/server/src/commit_monitor.rs b/server/src/commit_monitor.rs index 6ebc823b7f..b0805d9c72 100644 --- a/server/src/commit_monitor.rs +++ b/server/src/commit_monitor.rs @@ -90,6 +90,16 @@ pub struct CommitMonitor { #[allow(clippy::mutable_key_type)] presence: HashMap, Option>>, + /// Presence subscribes whose read check is still running, with the + /// latest update each such connection sent meanwhile. The check awaits a + /// store read, and the actor keeps handling messages while it does, so a + /// client's first update (sent right behind its `PRESENCE_SUBSCRIBE`, as + /// every reconnect does) routinely arrives before the connection is in + /// `presence`. It is held here and delivered once the check passes, or + /// dropped if it fails (#1800). + #[allow(clippy::mutable_key_type)] + pending_presence: + HashMap, Option>>, store: Db, vector_search_state: VectorSearchState, /// Set by every commit handler that may have queued a vector-index @@ -377,6 +387,12 @@ impl Handler for CommitMonitor { } self.presence .retain(|_, subscribers| !subscribers.is_empty()); + + for pending in self.pending_presence.values_mut() { + pending.remove(&msg.addr); + } + self.pending_presence + .retain(|_, pending| !pending.is_empty()); } } @@ -1026,6 +1042,17 @@ impl Handler for CommitMonitor { #[allow(clippy::mutable_key_type)] fn handle(&mut self, msg: SubscribePresence, _ctx: &mut Context) -> Self::Result { let store = self.store.clone(); + + // Note the subscribe before the read check yields, so an update that + // overtakes the check is held rather than refused. + self.pending_presence + .entry(msg.drive.clone()) + .or_default() + .entry(msg.addr.clone()) + .or_insert(None); + let pending_drive = msg.drive.clone(); + let pending_addr = msg.addr.clone(); + Box::pin( async move { if !msg.drive.is_local() { @@ -1074,8 +1101,12 @@ impl Handler for CommitMonitor { Some((msg.drive, msg.addr)) } .into_actor(self) - .map(|res, actor, _ctx| { - if let Some((drive, addr)) = res { + .map(move |res, actor, _ctx| { + // `None` when the connection closed or unsubscribed while the + // check ran: it must not be added back. + let held = actor.take_pending_presence(&pending_drive, &pending_addr); + + if let (Some((drive, addr)), Some(held)) = (res, held) { let subscribers = actor.presence.entry(drive.clone()).or_default(); // Bring the newcomer up to date: replay every other @@ -1095,8 +1126,12 @@ impl Handler for CommitMonitor { }); } - subscribers.entry(addr).or_insert(None); + subscribers.entry(addr.clone()).or_insert(None); tracing::debug!("Presence subscribed to {}", drive); + + if let Some(update) = held { + actor.accept_presence(&drive, &addr, update); + } } }), ) @@ -1107,6 +1142,8 @@ impl Handler for CommitMonitor { type Result = (); fn handle(&mut self, msg: UnsubscribePresence, _ctx: &mut Context) { + self.take_pending_presence(&msg.drive, &msg.addr); + if let Some(subscribers) = self.presence.get_mut(&msg.drive) { subscribers.remove(&msg.addr); @@ -1121,38 +1158,94 @@ impl Handler for CommitMonitor { type Result = (); fn handle(&mut self, msg: PresenceUpdate, _ctx: &mut Context) { - let Some(subscribers) = self.presence.get_mut(&msg.subject) else { - return; - }; - - let Some(sender) = msg.addr.as_ref() else { + let Some(sender) = msg.addr.clone() else { tracing::warn!("no addr in presence update for {}", msg.subject); return; }; + let update = CachedPresence { + agent: msg.agent, + update: msg.update, + }; // Only subscribers may broadcast — subscribing is where the drive // read-access check happens, so this is the auth gate. + let subscribed = self + .presence + .get(&msg.subject) + .is_some_and(|subscribers| subscribers.contains_key(&sender)); + if subscribed { + self.accept_presence(&msg.subject, &sender, update); + return; + } + + // Subscribed, but the read check has not finished: hold the latest + // update until it does. + if let Some(held) = self + .pending_presence + .get_mut(&msg.subject) + .and_then(|pending| pending.get_mut(&sender)) + { + *held = Some(update); + return; + } + + tracing::warn!("presence update from non-subscriber for {}", msg.subject); + } +} + +impl CommitMonitor { + /// Remove a connection's in-flight presence subscribe. The outer `Option` + /// is whether one was in flight; the inner one the update it sent + /// meanwhile, if any. + #[allow(clippy::mutable_key_type)] + fn take_pending_presence( + &mut self, + drive: &atomic_lib::Subject, + addr: &Addr, + ) -> Option> { + let pending = self.pending_presence.get_mut(drive)?; + let held = pending.remove(addr); + if pending.is_empty() { + self.pending_presence.remove(drive); + } + held + } + + /// Cache a subscriber's presence and fan it out to the drive's other + /// subscribers and to peers. The caller has checked `sender` subscribes. + fn accept_presence( + &mut self, + drive: &atomic_lib::Subject, + sender: &Addr, + update: CachedPresence, + ) { + let Some(subscribers) = self.presence.get_mut(drive) else { + return; + }; let Some(cached) = subscribers.get_mut(sender) else { - tracing::warn!("presence update from non-subscriber for {}", msg.subject); return; }; - *cached = Some(CachedPresence { - agent: msg.agent.clone(), - update: msg.update.clone(), - }); + *cached = Some(update.clone()); - // Relay to peers. Only local presence reaches here (the handler above - // requires a sender address), so there is no echo to guard against. + // Relay to peers. Only local presence reaches here (it always has a + // sender address), so there is no echo to guard against. if let Ok(agent) = self.store.get_default_agent() { atomic_lib::sync::peer::broadcast_ephemeral( atomic_lib::sync::protocol::ephemeral_kind::PRESENCE, - msg.subject.as_str(), + drive.as_str(), &agent.subject.to_string(), - &msg.update, + &update.update, None, ); } + let msg = PresenceUpdate { + subject: drive.clone(), + agent: update.agent, + update: update.update, + addr: Some(sender.clone()), + }; + for subscriber in subscribers.keys() { if subscriber == sender { continue; @@ -1201,6 +1294,7 @@ pub fn create_commit_monitor( drive_subscriptions: HashMap::new(), loro_subscriptions: HashMap::new(), presence: HashMap::new(), + pending_presence: HashMap::new(), store, vector_search_state, pending_commit: Arc::new(AtomicBool::new(false)), diff --git a/server/tests/it/drive_presence.rs b/server/tests/it/drive_presence.rs index 23152fccbe..c8ed5b784f 100644 --- a/server/tests/it/drive_presence.rs +++ b/server/tests/it/drive_presence.rs @@ -77,11 +77,9 @@ async fn presence_relays_caches_and_gates() -> AtomicResult<()> { // ----- Relay: Alice broadcasts, Bob receives, Alice gets no echo ----- // Bytes are opaque to the server; production sends // `EphemeralStore.encodeAll()`, a distinctive blob suffices here. - // The broadcaster admits an update only from a connection whose - // subscribe has finished its read check, and that check is a store - // read the actor runs concurrently; on a loaded host the first update - // can land before it. Production re-announces on a heartbeat, so do - // the same here rather than sleeping longer. + // An update that overtakes the subscribe's read check is held until + // the check passes (pinned by `presence_sent_right_after_subscribe_is_ + // delivered`); the retry only guards against a slow host here. let alice_state: Vec = b"alice-presence-state".to_vec(); let mut received = None; for _ in 0..10 { @@ -154,3 +152,73 @@ async fn presence_relays_caches_and_gates() -> AtomicResult<()> { Ok(()) } + +/// #1800: every (re)connect sends `PRESENCE_SUBSCRIBE` and then announces +/// itself straight away. The subscribe's read check awaits a store read and +/// the broadcaster keeps handling messages meanwhile, so that first update +/// used to arrive before the connection counted as a subscriber: the server +/// logged "presence update from non-subscriber" and dropped it, and peers did +/// not see the reconnected tab until its next heartbeat. The update must be +/// held until the check passes, and delivered once, with no retry. +#[tokio::test] +async fn presence_sent_right_after_subscribe_is_delivered() -> AtomicResult<()> { + let port = start_server("drive_presence_immediate"); + wait_for_server(port).await; + let server_url = format!("http://localhost:{}", port); + let ws_url = format!("ws://localhost:{}/ws", port); + + let client = Client::new(&server_url).await?; + let agent_a = client.new_agent("Alice").await?; + let drive = client.new_public_drive(&agent_a, "Presence Drive").await?; + + let client_b = Client::new(&server_url).await?; + let agent_b = client_b.new_agent("Bob").await?; + + // Bob is already watching the drive, settled. + let ws_b = WsClient::connect(&ws_url).await?; + ws_b.authenticate(&agent_b).await?; + let mut rx_b = ws_b.subscribe(); + ws_b.subscribe_presence(&drive).await?; + tokio::time::sleep(Duration::from_millis(300)).await; + + // Alice (re)connects and announces herself without waiting, as the + // browser client does after every reconnect. + for round in 0..5u8 { + let ws_a = WsClient::connect(&ws_url).await?; + ws_a.authenticate(&agent_a).await?; + let state = format!("alice-presence-{round}").into_bytes(); + ws_a.subscribe_presence(&drive).await?; + ws_a.send_presence_update(&drive, &state).await?; + + let received = recv_presence(&mut rx_b, &drive, 5).await; + assert_eq!( + received.as_deref(), + Some(state.as_slice()), + "round {round}: an update sent right behind PRESENCE_SUBSCRIBE must reach Bob" + ); + } + + // A connection that is refused must not have its held update relayed. + let private_drive = client.new_drive(&agent_a, "Private Drive").await?; + let ws_owner = WsClient::connect(&ws_url).await?; + ws_owner.authenticate(&agent_a).await?; + let mut rx_owner = ws_owner.subscribe(); + ws_owner.subscribe_presence(&private_drive).await?; + tokio::time::sleep(Duration::from_millis(300)).await; + + let ws_intruder = WsClient::connect(&ws_url).await?; + ws_intruder.authenticate(&agent_b).await?; + ws_intruder.subscribe_presence(&private_drive).await?; + ws_intruder + .send_presence_update(&private_drive, b"intruder") + .await?; + + assert!( + recv_presence(&mut rx_owner, &private_drive, 1) + .await + .is_none(), + "An update held for a subscribe that fails its read check must be dropped" + ); + + Ok(()) +}