diff --git a/packages/perps-controller/CHANGELOG.md b/packages/perps-controller/CHANGELOG.md index 8c0357ee573..8ae5c907fc9 100644 --- a/packages/perps-controller/CHANGELOG.md +++ b/packages/perps-controller/CHANGELOG.md @@ -11,6 +11,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bump `uuid` from `^14.0.0` to `^14.0.1` ([#11179](https://github.com/MetaMask/core/pull/11179)) +### Fixed + +- Prevent HyperLiquid `userFills` from going silent after a WebSocket reconnect when an in-flight subscribe races with `restoreSubscriptions` (same pending + generation pattern TWAP already uses) ([#11160](https://github.com/MetaMask/core/pull/11160)) + - Without this, live fills can keep attaching to a dead pre-reconnect listener (MetaMask/metamask-extension#47287). + ## [22.1.0] ### Added diff --git a/packages/perps-controller/src/services/HyperLiquidSubscriptionService.ts b/packages/perps-controller/src/services/HyperLiquidSubscriptionService.ts index c0d98dda8df..a45f687eff0 100644 --- a/packages/perps-controller/src/services/HyperLiquidSubscriptionService.ts +++ b/packages/perps-controller/src/services/HyperLiquidSubscriptionService.ts @@ -237,6 +237,27 @@ export class HyperLiquidSubscriptionService { // Order fill subscriptions keyed by accountId (normalized: undefined -> 'default') readonly #orderFillSubscriptions = new Map(); + // In-flight userFills subscribe calls, so concurrent subscribers / reconnect + // share one socket open instead of racing two opens onto the same map key. + readonly #pendingOrderFillSubscriptions = new Map>(); + + // Bumped per account by teardown and reconnect so an in-flight open for + // that account discards its subscription instead of rehydrating a cleared + // map (same race TWAP already guards; without it live fills can stick to a + // dead pre-reconnect listener — MetaMask/metamask-extension#47287). + readonly #orderFillSubscriptionGenerations = new Map(); + + #orderFillGeneration(normalizedAccountId: string): number { + return this.#orderFillSubscriptionGenerations.get(normalizedAccountId) ?? 0; + } + + #bumpOrderFillGeneration(normalizedAccountId: string): void { + this.#orderFillSubscriptionGenerations.set( + normalizedAccountId, + this.#orderFillGeneration(normalizedAccountId) + 1, + ); + } + // TWAP subscriptions keyed by accountId (normalized: undefined -> 'default') readonly #twapOrderSubscriptions = new Map(); @@ -2912,6 +2933,10 @@ export class HyperLiquidSubscriptionService { }); this.#orderFillSubscriptions.delete(normalizedAccountId); } + // Invalidate any open still in flight for this account: without this + // its continuation installs a live socket nobody is listening to. + this.#bumpOrderFillGeneration(normalizedAccountId); + this.#pendingOrderFillSubscriptions.delete(normalizedAccountId); } }; } @@ -2934,17 +2959,79 @@ export class HyperLiquidSubscriptionService { return; } - const subscriptionClient = this.#clientService.getSubscriptionClient(); + // Another caller is already opening this account's socket. Await it so + // subscribe + reconnect do not race two userFills opens onto one key. + // The owner of the pending open logs its failure, so swallow it here and + // fall through to open our own. Re-read after each wait: several waiters + // resume together and only the first should start the replacement open. + let pending = this.#pendingOrderFillSubscriptions.get(normalizedAccountId); + while (pending) { + await pending.catch(() => undefined); + if (this.#orderFillSubscriptions.has(normalizedAccountId)) { + return; + } + pending = this.#pendingOrderFillSubscriptions.get(normalizedAccountId); + } + + // A discarded open resolves without registering; if the last subscriber + // left meanwhile, opening now would cache frames nobody is listening to. + const subscribers = this.#orderFillSubscribers.get(normalizedAccountId); + if (!subscribers || subscribers.size === 0) { + return; + } + + // Captured once here, spanning the whole open including the client + // handshake. Capturing inside the open would let a bump that lands during + // the handshake be adopted as the baseline and silently forgotten. + const startGeneration = this.#orderFillGeneration(normalizedAccountId); + const subscriptionPromise = this.#createOrderFillSubscription( + normalizedAccountId, + accountId, + startGeneration, + ); + this.#pendingOrderFillSubscriptions.set( + normalizedAccountId, + subscriptionPromise, + ); + try { + await subscriptionPromise; + } finally { + // Delete by identity: a reconnect may have cleared the map and installed + // a newer open behind this one, which must not be evicted here. + if ( + this.#pendingOrderFillSubscriptions.get(normalizedAccountId) === + subscriptionPromise + ) { + this.#pendingOrderFillSubscriptions.delete(normalizedAccountId); + } + } + } + + /** + * Open one userFills subscription and register it for teardown. + * + * @param normalizedAccountId - Map key for this account ('default' when unset). + * @param accountId - Optional CAIP account ID to resolve the address from. + * @param startGeneration - This account's generation when the open began. + * @returns A promise that resolves once the subscription is registered. + */ + async #createOrderFillSubscription( + normalizedAccountId: string, + accountId: CaipAccountId | undefined, + startGeneration: number, + ): Promise { + // Bootstrap the client inline rather than recursing: a recursive call + // would re-read the generation after the handshake and adopt a bump that + // landed during it as its own baseline. + let subscriptionClient = this.#clientService.getSubscriptionClient(); if (!subscriptionClient) { await this.#clientService.ensureSubscriptionClient( this.#walletService.createWalletAdapter(), ); - const client = this.#clientService.getSubscriptionClient(); - if (!client) { + subscriptionClient = this.#clientService.getSubscriptionClient(); + if (!subscriptionClient) { throw new Error('SubscriptionClient not available'); } - await this.#ensureOrderFillISubscription(accountId); - return; } const userAddress = @@ -2954,6 +3041,13 @@ export class HyperLiquidSubscriptionService { const subscription = await subscriptionClient.userFills( { user: userAddress }, (data: UserFillsWsEvent) => { + // A frame queued before an unsubscribe or reconnect must not mutate + // the replacement's cache or reach its subscribers. + if ( + startGeneration !== this.#orderFillGeneration(normalizedAccountId) + ) { + return; + } // Build a Map for O(1) lookup instead of O(n) find per fill const orderMap = new Map(); if (this.#cachedOrders) { @@ -3019,6 +3113,19 @@ export class HyperLiquidSubscriptionService { }, ); + // Teardown, clearAll, or a reconnect may have run while this open was in + // flight. Rehydrating the map here would leave a socket with no + // subscribers, or clobber the live replacement — so drop ours instead. + const subscribers = this.#orderFillSubscribers.get(normalizedAccountId); + if ( + startGeneration !== this.#orderFillGeneration(normalizedAccountId) || + this.#orderFillSubscriptions.has(normalizedAccountId) || + !subscribers || + subscribers.size === 0 + ) { + subscription.unsubscribe().catch(() => undefined); + return; + } this.#orderFillSubscriptions.set(normalizedAccountId, subscription); } @@ -4956,8 +5063,19 @@ export class HyperLiquidSubscriptionService { // Re-establish order fill subscriptions if there are fill subscribers if (this.#orderFillSubscribers.size > 0) { - // Clear existing subscription references (they're dead after reconnection) + // Existing subscription references are dead after reconnection. Bump + // each account's generation first so a pre-reconnect open still in + // flight discards itself instead of occupying the map ahead of its + // replacement (MetaMask/metamask-extension#47287). + for (const normalizedAccountId of new Set([ + ...this.#orderFillSubscribers.keys(), + ...this.#orderFillSubscriptions.keys(), + ...this.#pendingOrderFillSubscriptions.keys(), + ])) { + this.#bumpOrderFillGeneration(normalizedAccountId); + } this.#orderFillSubscriptions.clear(); + this.#pendingOrderFillSubscriptions.clear(); // Re-establish subscriptions for all accountIds with subscribers // Note: normalizedAccountId is 'default' for undefined, need to convert back @@ -5192,6 +5310,15 @@ export class HyperLiquidSubscriptionService { ])) { this.#bumpTwapOrderGeneration(normalizedAccountId); } + // Same for order fills: bump before subscriber keys are cleared so an + // in-flight userFills open cannot rehydrate after clearAll. + for (const normalizedAccountId of new Set([ + ...this.#orderFillSubscribers.keys(), + ...this.#orderFillSubscriptions.keys(), + ...this.#pendingOrderFillSubscriptions.keys(), + ])) { + this.#bumpOrderFillGeneration(normalizedAccountId); + } // Clear all local subscriber collections this.#priceSubscribers.clear(); @@ -5204,13 +5331,14 @@ export class HyperLiquidSubscriptionService { this.#marketDataSubscribers.clear(); this.#orderBookSubscribers.clear(); - // Clear order fill subscriptions + // Clear order fill subscriptions; generations were bumped above. this.#orderFillSubscriptions.forEach((subscription) => { subscription.unsubscribe().catch(() => { // Ignore errors during cleanup }); }); this.#orderFillSubscriptions.clear(); + this.#pendingOrderFillSubscriptions.clear(); // Clear TWAP subscriptions; generations were bumped above, before the // subscriber keys naming those accounts were cleared. diff --git a/packages/perps-controller/tests/src/services/HyperLiquidSubscriptionService.streams.test.ts b/packages/perps-controller/tests/src/services/HyperLiquidSubscriptionService.streams.test.ts index 0a57397077c..feb695ea622 100644 --- a/packages/perps-controller/tests/src/services/HyperLiquidSubscriptionService.streams.test.ts +++ b/packages/perps-controller/tests/src/services/HyperLiquidSubscriptionService.streams.test.ts @@ -1456,6 +1456,244 @@ describe('HyperLiquidSubscriptionService', () => { }); }); }); + + it('re-establishes the venue subscription after a reconnect', async () => { + const callback = vi.fn(); + const unsubscribe = service.subscribeToOrderFills({ + callback, + }); + await vi.runAllTimersAsync(); + expect(mockSubscriptionClient.userFills).toHaveBeenCalledTimes(1); + + await service.restoreSubscriptions(); + await vi.runAllTimersAsync(); + + expect(mockSubscriptionClient.userFills).toHaveBeenCalledTimes(2); + unsubscribe(); + }); + + it('discards a pre-reconnect userFills open that resolves after restore', async () => { + // Race: subscribe starts an open that is still awaiting when reconnect + // clears the map and opens a replacement. The stale open must unsubscribe + // itself instead of overwriting the live post-reconnect subscription + // (MetaMask/metamask-extension#47287). + let resolveFirstOpen: + | ((subscription: { unsubscribe: ReturnType }) => void) + | undefined; + const firstSubscription = { + unsubscribe: vi.fn().mockResolvedValue(undefined), + }; + const secondSubscription = { + unsubscribe: vi.fn().mockResolvedValue(undefined), + }; + let openCount = 0; + mockSubscriptionClient.userFills.mockImplementation(() => { + openCount += 1; + if (openCount === 1) { + return new Promise((resolve) => { + resolveFirstOpen = resolve; + }); + } + return Promise.resolve(secondSubscription); + }); + + const callback = vi.fn(); + const unsubscribe = service.subscribeToOrderFills({ callback }); + // Let the first open start but not resolve. + await vi.waitFor(() => { + expect(openCount).toBe(1); + }); + + const restorePromise = service.restoreSubscriptions(); + await vi.waitFor(() => { + expect(openCount).toBe(2); + }); + + resolveFirstOpen?.(firstSubscription); + await restorePromise; + await vi.runAllTimersAsync(); + + expect(firstSubscription.unsubscribe).toHaveBeenCalled(); + expect(secondSubscription.unsubscribe).not.toHaveBeenCalled(); + expect(mockSubscriptionClient.userFills).toHaveBeenCalledTimes(2); + unsubscribe(); + }); + + describe('userFills open lifecycle', () => { + type ControlledOpen = { + emit: (event: unknown) => void; + resolve: () => void; + reject: (error: Error) => void; + subscription: { unsubscribe: ReturnType }; + }; + + const liveFill = (oid: number): unknown => ({ + user: '0x0000000000000000000000000000000000000000', + fills: [ + { + oid, + coin: 'BTC', + side: 'B', + sz: '0.1', + px: '50000', + fee: '5', + time: 1_700_000_000_000 + oid, + closedPnl: '0', + dir: 'Open Long', + feeToken: 'USDC', + startPosition: '0', + }, + ], + isSnapshot: false, + }); + + // Each userFills call returns a promise the test settles by hand, and + // exposes the listener the service registered so frames can be pushed. + const controlUserFillsOpens = (): ControlledOpen[] => { + const opens: ControlledOpen[] = []; + mockSubscriptionClient.userFills.mockImplementation( + (_params: unknown, callback: (event: unknown) => void) => { + const subscription = { + unsubscribe: vi.fn().mockResolvedValue(undefined), + }; + return new Promise((resolve, reject) => { + opens.push({ + emit: callback, + resolve: () => resolve(subscription), + reject, + subscription, + }); + }); + }, + ); + return opens; + }; + + const waitForOpens = async ( + opens: ControlledOpen[], + count: number, + ): Promise => { + await vi.waitFor(() => { + expect(opens).toHaveLength(count); + }); + }; + + it('delivers fills on the post-reconnect subscription and drops pre-reconnect frames', async () => { + const opens = controlUserFillsOpens(); + const callback = vi.fn(); + const unsubscribe = service.subscribeToOrderFills({ callback }); + await waitForOpens(opens, 1); + + const restorePromise = service.restoreSubscriptions(); + await waitForOpens(opens, 2); + opens[1].resolve(); + opens[0].resolve(); + await restorePromise; + await vi.runAllTimersAsync(); + + opens[0].emit(liveFill(1)); + expect(callback).not.toHaveBeenCalled(); + + opens[1].emit(liveFill(2)); + expect(callback).toHaveBeenCalledTimes(1); + expect(callback).toHaveBeenCalledWith( + [expect.objectContaining({ orderId: '2' })], + false, + ); + expect(opens[0].subscription.unsubscribe).toHaveBeenCalled(); + expect(opens[1].subscription.unsubscribe).not.toHaveBeenCalled(); + unsubscribe(); + }); + + it('opens its own userFills when the pending open it awaited rejects', async () => { + const opens = controlUserFillsOpens(); + const firstCallback = vi.fn(); + const secondCallback = vi.fn(); + const unsubscribeFirst = service.subscribeToOrderFills({ + callback: firstCallback, + }); + await waitForOpens(opens, 1); + const unsubscribeSecond = service.subscribeToOrderFills({ + callback: secondCallback, + }); + + opens[0].reject(new Error('socket closed')); + await waitForOpens(opens, 2); + opens[1].resolve(); + await vi.runAllTimersAsync(); + + opens[1].emit(liveFill(3)); + expect(firstCallback).toHaveBeenCalledTimes(1); + expect(secondCallback).toHaveBeenCalledTimes(1); + expect(mockSubscriptionClient.userFills).toHaveBeenCalledTimes(2); + unsubscribeFirst(); + unsubscribeSecond(); + }); + + it('does not open when the last subscriber left while a waiter was pending', async () => { + const opens = controlUserFillsOpens(); + const unsubscribeFirst = service.subscribeToOrderFills({ + callback: vi.fn(), + }); + await waitForOpens(opens, 1); + const unsubscribeSecond = service.subscribeToOrderFills({ + callback: vi.fn(), + }); + + unsubscribeFirst(); + unsubscribeSecond(); + opens[0].resolve(); + await vi.runAllTimersAsync(); + + expect(mockSubscriptionClient.userFills).toHaveBeenCalledTimes(1); + expect(opens[0].subscription.unsubscribe).toHaveBeenCalled(); + }); + + it('discards an in-flight open when the last subscriber leaves', async () => { + const opens = controlUserFillsOpens(); + const staleCallback = vi.fn(); + const liveCallback = vi.fn(); + const unsubscribeStale = service.subscribeToOrderFills({ + callback: staleCallback, + }); + await waitForOpens(opens, 1); + unsubscribeStale(); + + const unsubscribeLive = service.subscribeToOrderFills({ + callback: liveCallback, + }); + await waitForOpens(opens, 2); + + opens[0].emit(liveFill(4)); + expect(staleCallback).not.toHaveBeenCalled(); + expect(liveCallback).not.toHaveBeenCalled(); + + opens[1].resolve(); + opens[0].resolve(); + await vi.runAllTimersAsync(); + + expect(opens[0].subscription.unsubscribe).toHaveBeenCalled(); + expect(opens[1].subscription.unsubscribe).not.toHaveBeenCalled(); + opens[1].emit(liveFill(5)); + expect(liveCallback).toHaveBeenCalledTimes(1); + unsubscribeLive(); + }); + + it('discards an in-flight open when clearAll runs', async () => { + const opens = controlUserFillsOpens(); + const callback = vi.fn(); + service.subscribeToOrderFills({ callback }); + await waitForOpens(opens, 1); + + service.clearAll(); + opens[0].emit(liveFill(6)); + opens[0].resolve(); + await vi.runAllTimersAsync(); + + expect(callback).not.toHaveBeenCalled(); + expect(opens[0].subscription.unsubscribe).toHaveBeenCalled(); + }); + }); }); describe('Shared WebData3 Subscription', () => {