From b7c89152809e420a565bf58a27d5427bb76620ef Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 00:24:34 -0500 Subject: [PATCH 01/11] fix(server): keep slow device hosts from blocking other hosts DeviceService held its one lifecycle lock while a host started, so an SSH host that was slow to connect or still installing its hub (its bootstrap allows about 22 minutes) blocked readiness, listing and state publication for every other host, the local one included. Start hosts outside the lock; each host already serializes its own starts and stops. The lock now covers only the check that the start is still wanted and the ready publication. A start that finishes after device support or agent access was turned off stops that host again. Co-Authored-By: Claude Opus 5.5 --- .../server/src/device/DeviceMultiHost.test.ts | 71 +++++++++++++++++++ apps/server/src/device/DeviceService.ts | 64 ++++++++++++----- 2 files changed, 118 insertions(+), 17 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 7bbd8418de774..6904e3a979a85 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -147,3 +147,74 @@ it.effect("keeps hosts independent when serials collide and another host fails", ), ), ); + +it.effect("starts other hosts while one host is still starting", () => + Effect.gen(function* () { + const slowStarted = yield* Deferred.make(); + const finishSlow = yield* Deferred.make(); + const stops: string[] = []; + const host = (id: string, start: Effect.Effect): DeviceHost.DeviceHost["Service"] => { + const run = () => Effect.succeed({ stdout: "", stderr: "", code: 0 }); + const ready: DeviceHost.DeviceHostAgentReady = { + nodePath: process.execPath, + hub: { origin: `http://${id}` }, + devices: makeDeviceHubOperations({ + hostId: id, + origin: `http://${id}`, + http: HttpClient.make(() => Effect.die("unused")), + run, + androidAvailable: Effect.succeed(false), + }), + agentDevice: null, + run, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + return { + id, + label: id, + summary: Effect.succeed({ + id, + label: id, + kind: "ssh", + hubInstalled: true, + agentDeviceInstalled: true, + platforms: [], + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: () => start.pipe(Effect.as(ready)), + ensureAgentReady: () => start.pipe(Effect.as(ready)), + current: Effect.succeed(null), + stopAgent: Effect.void, + stop: Effect.sync(() => { + stops.push(id); + }), + }; + }; + const hosts = new Map([ + [ + "slow", + host( + "slow", + Deferred.succeed(slowStarted, undefined).pipe(Effect.andThen(Deferred.await(finishSlow))), + ), + ], + ["fast", host("fast", Effect.void)], + ]); + const service = yield* DeviceService.makeWithHosts(hosts).pipe( + Effect.provide(NodeCrypto.layer), + ); + const slow = yield* service.readiness("slow").pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(slowStarted); + expect((yield* service.readiness("fast")).hostId).toBe("fast"); + expect((yield* service.state).hostStatuses.fast?.status).toBe("ready"); + // Turning device support off while a host starts stops that host once it is up. + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finishSlow, undefined); + const error = yield* Fiber.join(slow); + expect(error._tag === "DeviceHostUnavailableError" && error.reason).toBe( + "Device support was turned off while the host was starting.", + ); + expect(stops).toEqual(["slow", "fast", "slow"]); + expect((yield* service.state).hostStatuses).toEqual({}); + }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 1ad95d17559bb..cfff0185a3353 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -262,6 +262,30 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* })), ); + // A host can take minutes to start (an SSH host may be installing its hub), + // so starting runs outside the lifecycle lock; each host serializes its own + // starts and stops. Under the lock, a start that is no longer wanted stops. + const confirmStarted = Effect.fn("DeviceService.confirmStarted")(function* ( + host: DeviceHost.DeviceHost["Service"], + agent: boolean, + ) { + const current = yield* readDeviceSettings; + if (!current.enabled || (agent && !current.agentAccessEnabled)) { + yield* current.enabled ? host.stopAgent : host.stop; + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + hostLabel: host.label, + reason: `${current.enabled ? "Agent device access" : "Device support"} was turned off while the host was starting.`, + }); + } + if (hosts.get(host.id) !== host) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + hostLabel: host.label, + reason: "Host configuration changed. Retry the operation.", + }); + }); + const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( function* (hostId) { const host = yield* resolveHost(hostId); @@ -302,19 +326,17 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* }), ), ); - if (hosts.get(host.id) !== host) - return yield* new DeviceHostUnavailableError({ - hostId: host.id, - hostLabel: host.label, - reason: "Host configuration changed. Retry the operation.", - }); - const { state } = yield* SynchronizedRef.get(stateRef); - if (state.hostStatuses[host.id]?.status !== "ready") { - yield* setHostStatus(host.id, { status: "ready" }); - } - return { hostId: host.id, ...ready }; + return yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* confirmStarted(host, false); + const { state } = yield* SynchronizedRef.get(stateRef); + if (state.hostStatuses[host.id]?.status !== "ready") { + yield* setHostStatus(host.id, { status: "ready" }); + } + return { hostId: host.id, ...ready }; + }), + ); }, - lifecycleLock.withPermit, ); const readinessIfSupported: DeviceService["Service"]["readinessIfSupported"] = Effect.fn( @@ -369,11 +391,19 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* }), ), ); - const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); - yield* publish((state) => ({ ...state, hosts: hostSummaries })); - yield* setHostStatus(host.id, { status: "ready" }); - return { hostId: host.id, ...ready }; - }, lifecycleLock.withPermit); + return yield* lifecycleLock.withPermit( + Effect.gen(function* () { + yield* confirmStarted(host, true); + const hostSummaries = yield* Effect.forEach( + hosts.values(), + (candidate) => candidate.summary, + ); + yield* publish((state) => ({ ...state, hosts: hostSummaries })); + yield* setHostStatus(host.id, { status: "ready" }); + return { hostId: host.id, ...ready }; + }), + ); + }); const currentReadiness: DeviceService["Service"]["currentReadiness"] = (hostId) => resolveHost(hostId).pipe( From 771ef5c20dff3a7cdf7c0fc697f279c14dee892c Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 02:01:45 -0500 Subject: [PATCH 02/11] fix(server): stop a device start that ends after support was turned off With starts outside the lifecycle lock, turning device support off can finish before a start takes its host's lock. If that start then failed or was interrupted, for example when the agent tools install failed after the hub came up, nothing stopped the hub, because the check ran only on success. However a start ends, it now rereads the settings under the lock, stops the host (or its agent) when it is no longer wanted, and drops the statuses it published after configure cleared them. A successful start still fails when it is unwanted or its host was replaced. Co-Authored-By: Claude Opus 5.5 --- .../server/src/device/DeviceMultiHost.test.ts | 90 ++++++++++++++++++- apps/server/src/device/DeviceService.ts | 40 ++++++++- 2 files changed, 125 insertions(+), 5 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 6904e3a979a85..121118ad6d8ec 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "@effect/vitest"; -import { ThreadId } from "@t3tools/contracts"; +import { LOCAL_DEVICE_HOST_ID, ThreadId, type DeviceHostSummary } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -218,3 +218,91 @@ it.effect("starts other hosts while one host is still starting", () => expect((yield* service.state).hostStatuses).toEqual({}); }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), ); + +it.effect("stops a start that fails after device support was turned off before it began", () => + Effect.gen(function* () { + const summaryEntered = yield* Deferred.make(); + const finishSummary = yield* Deferred.make(); + let pauseSummary = false; + let running = false; + const stops: string[] = []; + const summary: DeviceHostSummary = { + id: LOCAL_DEVICE_HOST_ID, + label: "Local", + kind: "local", + hubInstalled: true, + agentDeviceInstalled: false, + platforms: [{ platform: "android", available: true }], + }; + const run = () => Effect.succeed({ stdout: "", stderr: "", code: 0 }); + const ready: DeviceHost.DeviceHostReady = { + nodePath: process.execPath, + hub: { origin: "http://local" }, + devices: makeDeviceHubOperations({ + hostId: LOCAL_DEVICE_HOST_ID, + origin: "http://local", + http: HttpClient.make(() => Effect.die("unused")), + run, + androidAvailable: Effect.succeed(true), + }), + run, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + const host: DeviceHost.DeviceHost["Service"] = { + id: LOCAL_DEVICE_HOST_ID, + label: "Local", + summary: Effect.suspend(() => + pauseSummary + ? Deferred.succeed(summaryEntered, undefined).pipe( + Effect.andThen(Deferred.await(finishSummary)), + Effect.as(summary), + ) + : Effect.succeed(summary), + ), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: () => Effect.succeed(ready), + // Like LocalDeviceHost: the hub is up before the agent tools install fails. + ensureAgentReady: () => + Effect.sync(() => { + running = true; + }).pipe( + Effect.andThen( + Effect.fail( + new DeviceHost.DeviceHostError({ + hostId: LOCAL_DEVICE_HOST_ID, + step: "installing agent tools", + cause: new Error("offline"), + }), + ), + ), + ), + current: Effect.sync(() => (running ? ready : null)), + stopAgent: Effect.void, + stop: Effect.sync(() => { + stops.push("stop"); + running = false; + }), + }; + const service = yield* DeviceService.makeWithHosts(new Map([[host.id, host]])).pipe( + Effect.provide(NodeCrypto.layer), + ); + pauseSummary = true; + const starting = yield* service.agentReadinessIfSupported().pipe(Effect.flip, Effect.forkChild); + yield* Deferred.await(summaryEntered); + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finishSummary, undefined); + const error = yield* Fiber.join(starting); + expect(error._tag === "DeviceHostUnavailableError" && error.reason).toBe( + "Device support failed during installing agent tools.", + ); + expect(stops).toEqual(["stop", "stop"]); + expect(yield* service.currentReadiness()).toBeNull(); + const state = yield* service.state; + expect(state.hostStatus).toBe("disabled"); + expect(state.hostStatuses).toEqual({}); + }).pipe( + Effect.provide( + ServerSettings.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), + ), + ), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index cfff0185a3353..17eba85bbd7ba 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -264,20 +264,50 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* // A host can take minutes to start (an SSH host may be installing its hub), // so starting runs outside the lifecycle lock; each host serializes its own - // starts and stops. Under the lock, a start that is no longer wanted stops. + // starts and stops. Turning device support or agent access off can land + // before a start takes its host's lock, so however the start ends, an + // unwanted one stops again under the lock and drops the statuses it + // published after configure cleared them. + const stopIfRevoked = + (host: DeviceHost.DeviceHost["Service"], agent: boolean) => + (start: Effect.Effect) => + start.pipe( + Effect.ensuring( + lifecycleLock + .withPermit( + Effect.gen(function* () { + const current = yield* readDeviceSettings; + if (current.enabled && (!agent || current.agentAccessEnabled)) return; + yield* current.enabled ? host.stopAgent : host.stop; + yield* publish((state) => ({ + ...state, + ...(host.id === LOCAL_DEVICE_HOST_ID + ? { + hostStatus: current.enabled ? "idle" : "disabled", + hostStatusDetail: undefined, + } + : {}), + hostStatuses: Object.fromEntries( + Object.entries(state.hostStatuses).filter(([id]) => id !== host.id), + ), + })); + }), + ) + .pipe(Effect.ignore), + ), + ); + const confirmStarted = Effect.fn("DeviceService.confirmStarted")(function* ( host: DeviceHost.DeviceHost["Service"], agent: boolean, ) { const current = yield* readDeviceSettings; - if (!current.enabled || (agent && !current.agentAccessEnabled)) { - yield* current.enabled ? host.stopAgent : host.stop; + if (!current.enabled || (agent && !current.agentAccessEnabled)) return yield* new DeviceHostUnavailableError({ hostId: host.id, hostLabel: host.label, reason: `${current.enabled ? "Agent device access" : "Device support"} was turned off while the host was starting.`, }); - } if (hosts.get(host.id) !== host) return yield* new DeviceHostUnavailableError({ hostId: host.id, @@ -325,6 +355,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* cause: error, }), ), + stopIfRevoked(host, false), ); return yield* lifecycleLock.withPermit( Effect.gen(function* () { @@ -390,6 +421,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* cause: error, }), ), + stopIfRevoked(host, true), ); return yield* lifecycleLock.withPermit( Effect.gen(function* () { From de7be1e6db8480959fc98b583f72088abeb34789 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Thu, 22 Oct 2026 03:19:16 -0500 Subject: [PATCH 03/11] fix(server): keep device support off when a start fails after it is turned off The device list and host retry published a start's failure without checking settings, so a start that ended after device support was turned off marked the local host "failed", and the setup switch read that as enabled. They now publish under the lifecycle lock and only while device support is on. An agent start that ends after agent access was turned off returns no readiness, so a retry starts the device hub alone, as before the lock change. Co-Authored-By: Claude Opus 5.5 --- .../server/src/device/DeviceMultiHost.test.ts | 96 +++++++++++++++++++ apps/server/src/device/DeviceService.ts | 51 +++++----- 2 files changed, 123 insertions(+), 24 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 121118ad6d8ec..3381693058002 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -306,3 +306,99 @@ it.effect("stops a start that fails after device support was turned off before i ), ), ); + +// A local host whose `pausing` start waits until the test lets it finish. +const pausedLocalHost = (pausing: "ensureReady" | "ensureAgentReady") => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const finish = yield* Deferred.make(); + const calls: string[] = []; + const run = () => Effect.succeed({ stdout: "", stderr: "", code: 0 }); + const ready: DeviceHost.DeviceHostAgentReady = { + nodePath: process.execPath, + hub: { origin: "http://local" }, + devices: makeDeviceHubOperations({ + hostId: LOCAL_DEVICE_HOST_ID, + origin: "http://local", + http: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb(request, Response.json({ simulators: [], emulators: [] })), + ), + ), + run, + androidAvailable: Effect.succeed(false), + }), + agentDevice: null, + run, + helpers: { serveSimAxSettings: null, serveSimCli: null }, + }; + const start = (step: typeof pausing) => + step === pausing + ? Deferred.succeed(started, undefined).pipe( + Effect.andThen(Deferred.await(finish)), + Effect.as(ready), + ) + : Effect.succeed(ready); + const host: DeviceHost.DeviceHost["Service"] = { + id: LOCAL_DEVICE_HOST_ID, + label: "Local", + summary: Effect.succeed({ + id: LOCAL_DEVICE_HOST_ID, + label: "Local", + kind: "local", + hubInstalled: true, + agentDeviceInstalled: true, + platforms: [{ platform: "android", available: true }], + }), + platformAvailability: (platform) => Effect.succeed({ platform, available: true }), + ensureReady: () => start("ensureReady"), + ensureAgentReady: () => start("ensureAgentReady"), + current: Effect.succeed(null), + stopAgent: Effect.sync(() => { + calls.push("stopAgent"); + }), + stop: Effect.sync(() => { + calls.push("stop"); + }), + }; + return { host, started, finish, calls }; + }); + +it.effect("stays disabled when device support is turned off while listing starts a host", () => + Effect.gen(function* () { + const { host, started, finish, calls } = yield* pausedLocalHost("ensureReady"); + const service = yield* DeviceService.makeWithHosts(new Map([[host.id, host]])).pipe( + Effect.provide(NodeCrypto.layer), + ); + const listing = yield* service.list.pipe(Effect.forkChild); + yield* Deferred.await(started); + yield* service.configure({ enabled: false }); + yield* Deferred.succeed(finish, undefined); + yield* Fiber.join(listing); + expect(calls).toEqual(["stop", "stop"]); + const state = yield* service.state; + expect(state.hostStatus).toBe("disabled"); + expect(state.hostStatuses).toEqual({}); + }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), +); + +it.effect("retries without agent tools when agent access is turned off while they start", () => + Effect.gen(function* () { + const { host, started, finish, calls } = yield* pausedLocalHost("ensureAgentReady"); + const service = yield* DeviceService.makeWithHosts(new Map([[host.id, host]])).pipe( + Effect.provide(NodeCrypto.layer), + ); + const retrying = yield* service.retryHost(LOCAL_DEVICE_HOST_ID).pipe(Effect.forkChild); + yield* Deferred.await(started); + yield* service.configure({ agentAccessEnabled: false }); + yield* Deferred.succeed(finish, undefined); + const state = yield* Fiber.join(retrying); + expect(calls).toEqual(["stopAgent", "stopAgent"]); + expect(state.agentAccessEnabled).toBe(false); + expect(state.hostStatus).toBe("ready"); + }).pipe( + Effect.provide( + ServerSettings.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), + ), + ), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 17eba85bbd7ba..907544d5124f6 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -297,23 +297,20 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ), ); + // Whether a start that just ended is still wanted; a replaced host fails it. const confirmStarted = Effect.fn("DeviceService.confirmStarted")(function* ( host: DeviceHost.DeviceHost["Service"], agent: boolean, ) { const current = yield* readDeviceSettings; - if (!current.enabled || (agent && !current.agentAccessEnabled)) - return yield* new DeviceHostUnavailableError({ - hostId: host.id, - hostLabel: host.label, - reason: `${current.enabled ? "Agent device access" : "Device support"} was turned off while the host was starting.`, - }); + if (!current.enabled || (agent && !current.agentAccessEnabled)) return false; if (hosts.get(host.id) !== host) return yield* new DeviceHostUnavailableError({ hostId: host.id, hostLabel: host.label, reason: "Host configuration changed. Retry the operation.", }); + return true; }); const readiness: DeviceService["Service"]["readiness"] = Effect.fn("DeviceService.readiness")( @@ -359,7 +356,12 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ); return yield* lifecycleLock.withPermit( Effect.gen(function* () { - yield* confirmStarted(host, false); + if (!(yield* confirmStarted(host, false))) + return yield* new DeviceHostUnavailableError({ + hostId: host.id, + hostLabel: host.label, + reason: "Device support was turned off while the host was starting.", + }); const { state } = yield* SynchronizedRef.get(stateRef); if (state.hostStatuses[host.id]?.status !== "ready") { yield* setHostStatus(host.id, { status: "ready" }); @@ -425,7 +427,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ); return yield* lifecycleLock.withPermit( Effect.gen(function* () { - yield* confirmStarted(host, true); + if (!(yield* confirmStarted(host, true))) return null; const hostSummaries = yield* Effect.forEach( hosts.values(), (candidate) => candidate.summary, @@ -499,6 +501,21 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ); }); + // Checked under the lock configure publishes under, so a failure that ends + // after device support was turned off cannot mark the setting back on. + const publishStartFailure = (hostId: DeviceHostId, error: DeviceError) => + lifecycleLock + .withPermit( + Effect.gen(function* () { + if (!(yield* readDeviceSettings).enabled) return; + yield* setHostStatus(hostId, { + status: "failed", + detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, + }); + }), + ) + .pipe(Effect.ignore); + const list: DeviceService["Service"]["list"] = Effect.gen(function* () { if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; yield* Effect.forEach( @@ -507,14 +524,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* Effect.gen(function* () { const ready = yield* readinessIfSupported(host.id); if (ready) yield* refresh(ready); - }).pipe( - Effect.catch((error) => - setHostStatus(host.id, { - status: "failed", - detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, - }), - ), - ), + }).pipe(Effect.catch((error) => publishStartFailure(host.id, error))), { concurrency: 4 }, ); return (yield* SynchronizedRef.get(stateRef)).state; @@ -560,14 +570,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* const ready = (yield* agentReadinessIfSupported(hostId)) ?? (yield* readinessIfSupported(hostId)); if (ready) yield* refresh(ready); - }).pipe( - Effect.catch((error) => - setHostStatus(hostId, { - status: "failed", - detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, - }), - ), - ); + }).pipe(Effect.catch((error) => publishStartFailure(hostId, error))); return (yield* SynchronizedRef.get(stateRef)).state; }, ); From 8f85b9086ff9715c8fa1fe419868e69783718130 Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Fri, 23 Oct 2026 22:55:16 -0500 Subject: [PATCH 04/11] refactor(server): name the device start failure span Co-Authored-By: Claude Opus 5.5 --- apps/server/src/device/DeviceService.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 907544d5124f6..779f18ec9d43c 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -503,18 +503,18 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* // Checked under the lock configure publishes under, so a failure that ends // after device support was turned off cannot mark the setting back on. - const publishStartFailure = (hostId: DeviceHostId, error: DeviceError) => - lifecycleLock - .withPermit( - Effect.gen(function* () { - if (!(yield* readDeviceSettings).enabled) return; - yield* setHostStatus(hostId, { - status: "failed", - detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, - }); - }), - ) - .pipe(Effect.ignore); + const publishStartFailure = Effect.fn("DeviceService.publishStartFailure")( + function* (hostId: DeviceHostId, error: DeviceError) { + if (!(yield* readDeviceSettings).enabled) return; + yield* setHostStatus(hostId, { + status: "failed", + detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, + }); + }, + lifecycleLock.withPermit, + // Effect.fn also passes the call's arguments, which Effect.ignore would read as options. + (effect) => Effect.ignore(effect), + ); const list: DeviceService["Service"]["list"] = Effect.gen(function* () { if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; From d2177db2687e00b96c4ef003aef7e2b918e4cb6f Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Mon, 26 Oct 2026 15:57:34 +0000 Subject: [PATCH 05/11] fix(server): preserve device host ownership during concurrent startup --- .../server/src/device/DeviceMultiHost.test.ts | 156 +++++++++++------- apps/server/src/device/DeviceService.ts | 121 ++++++++------ apps/server/src/device/SshDeviceHost.test.ts | 13 +- apps/server/src/device/SshDeviceHost.ts | 20 ++- 4 files changed, 196 insertions(+), 114 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 3381693058002..843328d8be742 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -150,71 +150,24 @@ it.effect("keeps hosts independent when serials collide and another host fails", it.effect("starts other hosts while one host is still starting", () => Effect.gen(function* () { - const slowStarted = yield* Deferred.make(); - const finishSlow = yield* Deferred.make(); - const stops: string[] = []; - const host = (id: string, start: Effect.Effect): DeviceHost.DeviceHost["Service"] => { - const run = () => Effect.succeed({ stdout: "", stderr: "", code: 0 }); - const ready: DeviceHost.DeviceHostAgentReady = { - nodePath: process.execPath, - hub: { origin: `http://${id}` }, - devices: makeDeviceHubOperations({ - hostId: id, - origin: `http://${id}`, - http: HttpClient.make(() => Effect.die("unused")), - run, - androidAvailable: Effect.succeed(false), - }), - agentDevice: null, - run, - helpers: { serveSimAxSettings: null, serveSimCli: null }, - }; - return { - id, - label: id, - summary: Effect.succeed({ - id, - label: id, - kind: "ssh", - hubInstalled: true, - agentDeviceInstalled: true, - platforms: [], - }), - platformAvailability: (platform) => Effect.succeed({ platform, available: true }), - ensureReady: () => start.pipe(Effect.as(ready)), - ensureAgentReady: () => start.pipe(Effect.as(ready)), - current: Effect.succeed(null), - stopAgent: Effect.void, - stop: Effect.sync(() => { - stops.push(id); - }), - }; - }; - const hosts = new Map([ - [ - "slow", - host( - "slow", - Deferred.succeed(slowStarted, undefined).pipe(Effect.andThen(Deferred.await(finishSlow))), - ), - ], - ["fast", host("fast", Effect.void)], - ]); + const slowHost = yield* pausedHost("ensureReady", "slow"); + const fastHost = yield* pausedHost("ensureAgentReady", "fast"); + const hosts = new Map([slowHost.host, fastHost.host].map((host) => [host.id, host])); const service = yield* DeviceService.makeWithHosts(hosts).pipe( Effect.provide(NodeCrypto.layer), ); const slow = yield* service.readiness("slow").pipe(Effect.flip, Effect.forkChild); - yield* Deferred.await(slowStarted); + yield* Deferred.await(slowHost.started); expect((yield* service.readiness("fast")).hostId).toBe("fast"); expect((yield* service.state).hostStatuses.fast?.status).toBe("ready"); - // Turning device support off while a host starts stops that host once it is up. yield* service.configure({ enabled: false }); - yield* Deferred.succeed(finishSlow, undefined); + yield* Deferred.succeed(slowHost.finish, undefined); const error = yield* Fiber.join(slow); expect(error._tag === "DeviceHostUnavailableError" && error.reason).toBe( "Device support was turned off while the host was starting.", ); - expect(stops).toEqual(["slow", "fast", "slow"]); + expect(slowHost.calls).toEqual(["stop", "stop"]); + expect(fastHost.calls).toEqual(["stop"]); expect((yield* service.state).hostStatuses).toEqual({}); }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), ); @@ -307,8 +260,10 @@ it.effect("stops a start that fails after device support was turned off before i ), ); -// A local host whose `pausing` start waits until the test lets it finish. -const pausedLocalHost = (pausing: "ensureReady" | "ensureAgentReady") => +const pausedHost = ( + pausing: "ensureReady" | "ensureAgentReady", + id: string = LOCAL_DEVICE_HOST_ID, +) => Effect.gen(function* () { const started = yield* Deferred.make(); const finish = yield* Deferred.make(); @@ -316,10 +271,10 @@ const pausedLocalHost = (pausing: "ensureReady" | "ensureAgentReady") => const run = () => Effect.succeed({ stdout: "", stderr: "", code: 0 }); const ready: DeviceHost.DeviceHostAgentReady = { nodePath: process.execPath, - hub: { origin: "http://local" }, + hub: { origin: `http://${id}` }, devices: makeDeviceHubOperations({ - hostId: LOCAL_DEVICE_HOST_ID, - origin: "http://local", + hostId: id, + origin: `http://${id}`, http: HttpClient.make((request) => Effect.succeed( HttpClientResponse.fromWeb(request, Response.json({ simulators: [], emulators: [] })), @@ -340,12 +295,12 @@ const pausedLocalHost = (pausing: "ensureReady" | "ensureAgentReady") => ) : Effect.succeed(ready); const host: DeviceHost.DeviceHost["Service"] = { - id: LOCAL_DEVICE_HOST_ID, + id, label: "Local", summary: Effect.succeed({ - id: LOCAL_DEVICE_HOST_ID, + id, label: "Local", - kind: "local", + kind: id === LOCAL_DEVICE_HOST_ID ? "local" : "ssh", hubInstalled: true, agentDeviceInstalled: true, platforms: [{ platform: "android", available: true }], @@ -366,7 +321,7 @@ const pausedLocalHost = (pausing: "ensureReady" | "ensureAgentReady") => it.effect("stays disabled when device support is turned off while listing starts a host", () => Effect.gen(function* () { - const { host, started, finish, calls } = yield* pausedLocalHost("ensureReady"); + const { host, started, finish, calls } = yield* pausedHost("ensureReady"); const service = yield* DeviceService.makeWithHosts(new Map([[host.id, host]])).pipe( Effect.provide(NodeCrypto.layer), ); @@ -384,7 +339,7 @@ it.effect("stays disabled when device support is turned off while listing starts it.effect("retries without agent tools when agent access is turned off while they start", () => Effect.gen(function* () { - const { host, started, finish, calls } = yield* pausedLocalHost("ensureAgentReady"); + const { host, started, finish, calls } = yield* pausedHost("ensureAgentReady"); const service = yield* DeviceService.makeWithHosts(new Map([[host.id, host]])).pipe( Effect.provide(NodeCrypto.layer), ); @@ -402,3 +357,76 @@ it.effect("retries without agent tools when agent access is turned off while the ), ), ); + +it.effect("keeps the replacement ready when an old host finishes starting", () => + Effect.gen(function* () { + for (const fail of [false, true]) { + const { host, started, finish, calls } = yield* pausedHost("ensureReady"); + const pendingHost: DeviceHost.DeviceHost["Service"] = { + ...host, + ensureReady: (onPhase) => + host.ensureReady(onPhase).pipe( + Effect.tap(() => onPhase("starting")), + Effect.filterOrFail( + () => !fail, + () => + new DeviceHost.DeviceHostError({ + hostId: host.id, + step: "start", + cause: "offline", + }), + ), + ), + }; + const hosts = new Map([[host.id, pendingHost]]); + const service = yield* DeviceService.makeWithHosts(hosts).pipe( + Effect.provide(NodeCrypto.layer), + ); + const listing = yield* service.list.pipe(Effect.forkChild); + yield* Deferred.await(started); + hosts.set(host.id, { ...host }); + yield* service.setHostStatus(host.id, { status: "ready" }); + yield* Deferred.succeed(finish, undefined); + const state = yield* Fiber.join(listing); + expect(calls).toEqual(["stop"]); + expect(state.hostStatuses[host.id]?.status).toBe("ready"); + } + }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), +); + +it.effect("retries the hub when revoked agent startup fails", () => + Effect.gen(function* () { + const { host, started, finish } = yield* pausedHost("ensureAgentReady"); + const failingHost = { + ...host, + ensureAgentReady: () => + host + .ensureAgentReady(() => Effect.void) + .pipe( + Effect.andThen( + Effect.fail( + new DeviceHost.DeviceHostError({ + hostId: host.id, + step: "agent", + cause: "offline", + }), + ), + ), + ), + }; + const service = yield* DeviceService.makeWithHosts(new Map([[host.id, failingHost]])).pipe( + Effect.provide(NodeCrypto.layer), + ); + const retrying = yield* service.retryHost(host.id).pipe(Effect.forkChild); + yield* Deferred.await(started); + yield* service.configure({ agentAccessEnabled: false }); + yield* Deferred.succeed(finish, undefined); + const state = yield* Fiber.join(retrying); + expect(state.agentAccessEnabled).toBe(false); + expect(state.hostStatuses[host.id]?.status).toBe("ready"); + }).pipe( + Effect.provide( + ServerSettings.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), + ), + ), +); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index ce1e3d75acead..651cbc300eda1 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -241,33 +241,33 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* const setHostStatus = ( hostId: DeviceHostId, status: DeviceServiceState["hostStatuses"][string], + expectedHost?: DeviceHost.DeviceHost["Service"], ) => Effect.suspend(() => !hosts.has(hostId) ? SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)) - : publish((state) => ({ - ...state, - ...(hostId === LOCAL_DEVICE_HOST_ID - ? { hostStatus: status.status, hostStatusDetail: status.detail } - : {}), - hostStatuses: { - ...state.hostStatuses, - [hostId]: { - ...(state.hostStatuses[hostId]?.androidDiscoveryIncomplete - ? { androidDiscoveryIncomplete: true } - : {}), - ...status, - }, - }, - })), + : publish((state) => + expectedHost && hosts.get(hostId) !== expectedHost + ? state + : { + ...state, + ...(hostId === LOCAL_DEVICE_HOST_ID + ? { hostStatus: status.status, hostStatusDetail: status.detail } + : {}), + hostStatuses: { + ...state.hostStatuses, + [hostId]: { + ...(state.hostStatuses[hostId]?.androidDiscoveryIncomplete + ? { androidDiscoveryIncomplete: true } + : {}), + ...status, + }, + }, + }, + ), ); - // A host can take minutes to start (an SSH host may be installing its hub), - // so starting runs outside the lifecycle lock; each host serializes its own - // starts and stops. Turning device support or agent access off can land - // before a start takes its host's lock, so however the start ends, an - // unwanted one stops again under the lock and drops the statuses it - // published after configure cleared them. + // Host startup can take minutes; only revocation cleanup holds the shared lock. const stopIfRevoked = (host: DeviceHost.DeviceHost["Service"], agent: boolean) => (start: Effect.Effect) => @@ -277,6 +277,10 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* .withPermit( Effect.gen(function* () { const current = yield* readDeviceSettings; + if (hosts.get(host.id) !== host) { + yield* host.stop; + return; + } if (current.enabled && (!agent || current.agentAccessEnabled)) return; yield* current.enabled ? host.stopAgent : host.stop; yield* publish((state) => ({ @@ -297,7 +301,6 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ), ); - // Whether a start that just ended is still wanted; a replaced host fails it. const confirmStarted = Effect.fn("DeviceService.confirmStarted")(function* ( host: DeviceHost.DeviceHost["Service"], agent: boolean, @@ -326,17 +329,21 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* } const ready = yield* host .ensureReady((status, detail) => - setHostStatus(host.id, { status, detail }).pipe(Effect.asVoid), + setHostStatus(host.id, { status, detail }, host).pipe(Effect.asVoid), ) .pipe( Effect.tapError((error) => - setHostStatus(host.id, { - status: "failed", - detail: - error._tag === "DeviceHostError" && error.step === "probe" - ? sshProbeFailureMessage(error) - : error.message, - }), + setHostStatus( + host.id, + { + status: "failed", + detail: + error._tag === "DeviceHostError" && error.step === "probe" + ? sshProbeFailureMessage(error) + : error.message, + }, + host, + ), ), Effect.mapError( (error) => @@ -393,17 +400,21 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* return null; const ready = yield* host .ensureAgentReady((phase, detail) => - setHostStatus(host.id, { status: phase, detail }).pipe(Effect.asVoid), + setHostStatus(host.id, { status: phase, detail }, host).pipe(Effect.asVoid), ) .pipe( Effect.tapError((error) => - setHostStatus(host.id, { - status: "failed", - detail: - error._tag === "DeviceHostError" && error.step === "probe" - ? sshProbeFailureMessage(error) - : error.message, - }), + setHostStatus( + host.id, + { + status: "failed", + detail: + error._tag === "DeviceHostError" && error.step === "probe" + ? sshProbeFailureMessage(error) + : error.message, + }, + host, + ), ), Effect.mapError( (error) => @@ -504,9 +515,9 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* // Checked under the lock configure publishes under, so a failure that ends // after device support was turned off cannot mark the setting back on. const publishStartFailure = Effect.fn("DeviceService.publishStartFailure")( - function* (hostId: DeviceHostId, error: DeviceError) { - if (!(yield* readDeviceSettings).enabled) return; - yield* setHostStatus(hostId, { + function* (host: DeviceHost.DeviceHost["Service"], error: DeviceError) { + if (hosts.get(host.id) !== host || !(yield* readDeviceSettings).enabled) return; + yield* setHostStatus(host.id, { status: "failed", detail: error._tag === "DeviceHostUnavailableError" ? error.reason : error.message, }); @@ -524,7 +535,7 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* Effect.gen(function* () { const ready = yield* readinessIfSupported(host.id); if (ready) yield* refresh(ready); - }).pipe(Effect.catch((error) => publishStartFailure(host.id, error))), + }).pipe(Effect.catch((error) => publishStartFailure(host, error))), { concurrency: 4 }, ); return (yield* SynchronizedRef.get(stateRef)).state; @@ -564,13 +575,23 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* const retryHost: DeviceService["Service"]["retryHost"] = Effect.fn("DeviceService.retryHost")( function* (hostId) { - yield* resolveHost(hostId); + const host = yield* resolveHost(hostId); if (!(yield* readDeviceSettings).enabled) return (yield* SynchronizedRef.get(stateRef)).state; yield* Effect.gen(function* () { const ready = - (yield* agentReadinessIfSupported(hostId)) ?? (yield* readinessIfSupported(hostId)); + (yield* agentReadinessIfSupported(hostId).pipe( + Effect.catch((error) => + readDeviceSettings.pipe( + Effect.flatMap((current) => + current.enabled && current.agentAccessEnabled + ? Effect.fail(error) + : Effect.succeed(null), + ), + ), + ), + )) ?? (yield* readinessIfSupported(hostId)); if (ready) yield* refresh(ready); - }).pipe(Effect.catch((error) => publishStartFailure(hostId, error))); + }).pipe(Effect.catch((error) => publishStartFailure(host, error))); return (yield* SynchronizedRef.get(stateRef)).state; }, ); @@ -1133,9 +1154,13 @@ export const make = Effect.gen(function* () { ), ), (status, detail) => - service - .setHostStatus(host.id, { status, ...(detail ? { detail } : {}) }) - .pipe(Effect.asVoid), + Effect.suspend(() => + configured.get(host.id)?.scope === hostScope + ? service + .setHostStatus(host.id, { status, ...(detail ? { detail } : {}) }) + .pipe(Effect.asVoid) + : Effect.void, + ), ).pipe(Effect.provideService(Scope.Scope, hostScope), Effect.provide(hostContext)); hosts.set(host.id, instance); configured.set(host.id, { config: host, scope: hostScope }); diff --git a/apps/server/src/device/SshDeviceHost.test.ts b/apps/server/src/device/SshDeviceHost.test.ts index cde68d1b50c99..92ebda27d09c7 100644 --- a/apps/server/src/device/SshDeviceHost.test.ts +++ b/apps/server/src/device/SshDeviceHost.test.ts @@ -1,6 +1,8 @@ import { expect, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Net from "@t3tools/shared/Net"; +import * as Scope from "effect/Scope"; +import * as Exit from "effect/Exit"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -100,6 +102,8 @@ it.effect("preserves installed status after probes and cleans failed agent activ }); }), ); + const hostScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(hostScope, Exit.void)); const host = yield* SshDeviceHost.make( { kind: "ssh", id: "test", label: "Test", target: "test.example" }, () => @@ -113,6 +117,7 @@ it.effect("preserves installed status after probes and cleans failed agent activ ) : Effect.void, ).pipe( + Effect.provideService(Scope.Scope, hostScope), Effect.provide(Layer.mergeAll(ServerConfig.layerTest(home, home), Net.layer)), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService( @@ -179,7 +184,13 @@ it.effect("preserves installed status after probes and cleans failed agent activ for (const args of commandArgs) { for (const option of multiplexingOff) expect(args).not.toContain(option); } - yield* host.stop; + const retired = yield* host + .ensureAgentReady(() => Scope.close(hostScope, Exit.void)) + .pipe(Effect.result); + expect(retired._tag).toBe("Failure"); expect(forwards).toBe(0); + const modesAfterClose = [...modes]; + expect((yield* host.ensureReady(() => Effect.void).pipe(Effect.result))._tag).toBe("Failure"); + expect(modes).toEqual(modesAfterClose); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/apps/server/src/device/SshDeviceHost.ts b/apps/server/src/device/SshDeviceHost.ts index d7324d515b006..6d8208eaa0bcc 100644 --- a/apps/server/src/device/SshDeviceHost.ts +++ b/apps/server/src/device/SshDeviceHost.ts @@ -152,6 +152,18 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( ); const lock = yield* Semaphore.make(1); let stopped = false; + let retired = false; + const requireActive = Effect.suspend(() => + retired + ? Effect.fail( + new DeviceHost.DeviceHostError({ + hostId: config.id, + step: "starting retired host", + cause: new Error("Host configuration changed. Retry the operation."), + }), + ) + : Effect.void, + ); let activated = false; let wantsAgent = false; let ready: @@ -417,6 +429,7 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( const ensureReady: DeviceHost.DeviceHost["Service"]["ensureReady"] = (onPhase) => lock.withPermit( Effect.gen(function* () { + yield* requireActive; stopped = false; if (ready) return ready; summary = yield* provide(probe(config, owner)); @@ -447,6 +460,7 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( const changeAgent = (enabled: boolean) => lock.withPermit( Effect.gen(function* () { + yield* requireActive; wantsAgent = enabled; if (enabled && ready?.agentDevice) return { ...ready, agentDevice: ready.agentDevice }; if (!enabled && !ready?.agentDevice) return null; @@ -468,7 +482,11 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( ); }), ); - yield* Effect.addFinalizer(() => stop); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + retired = true; + }).pipe(Effect.andThen(stop)), + ); return { id: config.id, label: config.label, From 3cb8fc0676f3902559c2ae7981dcaf6a3ee28b72 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 01:48:45 +0000 Subject: [PATCH 06/11] test(server): allow host status updates during device close --- apps/server/src/device/DeviceService.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/device/DeviceService.test.ts b/apps/server/src/device/DeviceService.test.ts index 8c7b400a2c5de..732683b63eca5 100644 --- a/apps/server/src/device/DeviceService.test.ts +++ b/apps/server/src/device/DeviceService.test.ts @@ -1723,7 +1723,7 @@ describe("agent session release on close", () => { sessions.includes(session) ? "old" : sessions.includes(reopened) ? "reopened" : "none", ) .filter((label, index, labels) => label !== labels[index - 1]); - expect(published).toEqual(["none", "reopened"]); + expect(published[0] === "old" ? published.slice(1) : published).toEqual(["none", "reopened"]); }).pipe(Effect.scoped), ); }); From 5a1a98fc794d7bbf2c7a3a2eb651491bfb2b45dd Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 03:47:56 +0000 Subject: [PATCH 07/11] fix(server): invalidate stopped device host readiness --- apps/server/src/device/DeviceHost.ts | 11 +++- .../server/src/device/DeviceMultiHost.test.ts | 18 +++++- apps/server/src/device/DeviceService.ts | 62 ++++++++++--------- apps/server/src/device/EasDeviceHost.test.ts | 1 + apps/server/src/device/EasDeviceHost.ts | 4 ++ .../src/device/SimlockDeviceHost.test.ts | 8 +++ apps/server/src/device/SimlockDeviceHost.ts | 4 ++ apps/server/src/device/SshDeviceHost.test.ts | 7 ++- apps/server/src/device/SshDeviceHost.ts | 7 ++- 9 files changed, 85 insertions(+), 37 deletions(-) diff --git a/apps/server/src/device/DeviceHost.ts b/apps/server/src/device/DeviceHost.ts index 7a0962e06f18b..4889b6af70592 100644 --- a/apps/server/src/device/DeviceHost.ts +++ b/apps/server/src/device/DeviceHost.ts @@ -191,12 +191,19 @@ export class DeviceHost extends Context.Service< ) => Effect.Effect; readonly ensureReady: ( onPhase: (phase: "installing" | "starting", detail?: string) => Effect.Effect, - ) => Effect.Effect; + ) => Effect.Effect< + DeviceHostReady, + DeviceHostError | DeviceHostUnavailableError | NodeRuntimeUnavailableError + >; readonly ensureAgentReady: ( onPhase: (phase: "installing" | "starting", detail?: string) => Effect.Effect, ) => Effect.Effect< DeviceHostAgentReady, - DeviceHostError | DeviceHostTimeoutError | AgentDeviceStartError | NodeRuntimeUnavailableError + | DeviceHostError + | DeviceHostUnavailableError + | DeviceHostTimeoutError + | AgentDeviceStartError + | NodeRuntimeUnavailableError >; readonly current: Effect.Effect; readonly stopAgent: Effect.Effect; diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index c1a1c12b6785c..5e6ab401ba7fa 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -1,5 +1,10 @@ import { expect, it } from "@effect/vitest"; -import { LOCAL_DEVICE_HOST_ID, ThreadId, type DeviceHostSummary } from "@t3tools/contracts"; +import { + DeviceHostUnavailableError, + LOCAL_DEVICE_HOST_ID, + ThreadId, + type DeviceHostSummary, +} from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as Fiber from "effect/Fiber"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; @@ -145,6 +150,17 @@ it.effect("keeps hosts independent when serials collide and another host fails", expect((yield* service.state).sessions).toEqual([]); yield* service.agentReadinessIfSupported("a"); expect((yield* service.state).hostStatuses.a?.status).toBe("ready"); + const unavailable = new DeviceHostUnavailableError({ + hostId: "a", + reason: "Host configuration changed. Retry the operation.", + }); + hosts.set("a", { + ...host("a"), + ensureReady: () => Effect.fail(unavailable), + ensureAgentReady: () => Effect.fail(unavailable), + }); + expect(yield* service.readiness("a").pipe(Effect.flip)).toBe(unavailable); + expect(yield* service.agentReadinessIfSupported("a").pipe(Effect.flip)).toBe(unavailable); yield* service.configure({ enabled: false }); expect((yield* service.state).hostStatuses).toEqual({}); }).pipe( diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 7ded3ed46d547..7ed3fcac1a550 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -490,19 +490,20 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* host, ), ), - Effect.mapError( - (error) => - new DeviceHostUnavailableError({ - hostId: host.id, - hostLabel: host.label, - reason: - error._tag === "NodeRuntimeUnavailableError" - ? nodeRuntimeUnavailableMessage("Local device support") - : error.step === "probe" - ? sshProbeFailureMessage(error) - : `Device support failed during ${error.step}.`, - cause: error, - }), + Effect.mapError((error) => + error._tag === "DeviceHostUnavailableError" + ? error + : new DeviceHostUnavailableError({ + hostId: host.id, + hostLabel: host.label, + reason: + error._tag === "NodeRuntimeUnavailableError" + ? nodeRuntimeUnavailableMessage("Local device support") + : error.step === "probe" + ? sshProbeFailureMessage(error) + : `Device support failed during ${error.step}.`, + cause: error, + }), ), stopIfRevoked(host, false), ); @@ -561,23 +562,24 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* host, ), ), - Effect.mapError( - (error) => - new DeviceHostUnavailableError({ - hostId: host.id, - hostLabel: host.label, - reason: - error._tag === "NodeRuntimeUnavailableError" - ? nodeRuntimeUnavailableMessage("Local device support") - : error._tag === "DeviceHostTimeoutError" - ? `Agent tools did not start within ${error.timeoutMs} ms.` - : error._tag === "AgentDeviceStartError" - ? error.message - : error.step === "probe" - ? sshProbeFailureMessage(error) - : `Device support failed during ${error.step}.`, - cause: error, - }), + Effect.mapError((error) => + error._tag === "DeviceHostUnavailableError" + ? error + : new DeviceHostUnavailableError({ + hostId: host.id, + hostLabel: host.label, + reason: + error._tag === "NodeRuntimeUnavailableError" + ? nodeRuntimeUnavailableMessage("Local device support") + : error._tag === "DeviceHostTimeoutError" + ? `Agent tools did not start within ${error.timeoutMs} ms.` + : error._tag === "AgentDeviceStartError" + ? error.message + : error.step === "probe" + ? sshProbeFailureMessage(error) + : `Device support failed during ${error.step}.`, + cause: error, + }), ), stopIfRevoked(host, true), ); diff --git a/apps/server/src/device/EasDeviceHost.test.ts b/apps/server/src/device/EasDeviceHost.test.ts index b5825d3f973e7..a32636d2e1e5f 100644 --- a/apps/server/src/device/EasDeviceHost.test.ts +++ b/apps/server/src/device/EasDeviceHost.test.ts @@ -293,6 +293,7 @@ describe("EAS Simulator host", () => { ).toBeNull(); yield* Scope.close(scope, Exit.void); + expect(yield* host.current).toBeNull(); expect(expo.named("T3StopEasSimulatorSession").map((call) => call.variables.id)).toEqual([ "session-1", ]); diff --git a/apps/server/src/device/EasDeviceHost.ts b/apps/server/src/device/EasDeviceHost.ts index e82638dbd76fa..38f674a80c32d 100644 --- a/apps/server/src/device/EasDeviceHost.ts +++ b/apps/server/src/device/EasDeviceHost.ts @@ -819,10 +819,13 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice "EasDeviceHost.ensureReady", )(function* () { if (ready) return ready; + const startedIn = epoch; yield* adoptRecorded; // Without a credential the host is still ready: opening a device then // says how to sign in, and listing reaps once one is there. yield* FiberSet.run(background)(api.pipe(Effect.flatMap(reapLeftovers), Effect.ignore)); + if (epoch !== startedIn) + return yield* unavailable(`${config.label} was stopped while readiness started.`); ready = { nodePath, hub: { origin: "https://api.expo.dev" }, @@ -850,6 +853,7 @@ export const make = Effect.fn("EasDeviceHost.make")(function* (config: EasDevice // and stays in the ledger for the next server if this one closes first. const stopAll = Effect.gen(function* () { epoch++; + ready = null; // Interrupted starts stop the sessions they created before this returns. yield* FiberSet.clear(starts); yield* FiberSet.clear(background); diff --git a/apps/server/src/device/SimlockDeviceHost.test.ts b/apps/server/src/device/SimlockDeviceHost.test.ts index 1b670a06cd078..70cd87691d73e 100644 --- a/apps/server/src/device/SimlockDeviceHost.test.ts +++ b/apps/server/src/device/SimlockDeviceHost.test.ts @@ -241,7 +241,15 @@ describe("Simlock device host", () => { yield* ready.devices.shutdown({ id: leaseId, platform: "android" }); expect(pool.ran("close")).toHaveLength(1); expect((yield* ready.devices.list).devices.some((device) => device.booted)).toBe(false); + yield* host.stop; + expect(yield* host.current).toBeNull(); + const stoppedStart = yield* host.ensureReady(() => host.stop).pipe(Effect.result); + expect(stoppedStart._tag).toBe("Failure"); + expect(yield* host.current).toBeNull(); + yield* host.ensureReady(() => Effect.void); + expect(yield* host.current).not.toBeNull(); yield* Scope.close(scope, Exit.void); + expect(yield* host.current).toBeNull(); expect(pool.ran("close")).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); diff --git a/apps/server/src/device/SimlockDeviceHost.ts b/apps/server/src/device/SimlockDeviceHost.ts index 75476f0333ce3..262339474ef43 100644 --- a/apps/server/src/device/SimlockDeviceHost.ts +++ b/apps/server/src/device/SimlockDeviceHost.ts @@ -597,10 +597,13 @@ export const make = Effect.fn("SimlockDeviceHost.make")(function* ( "SimlockDeviceHost.ensureReady", )(function* (onPhase) { if (ready) return ready; + const startedIn = stops; // A missing token or unreachable gateway does not stop readiness: listing // and opening devices say what is wrong instead. yield* onPhase("installing"); cli = yield* agentDeviceCli; + if (stops !== startedIn) + return yield* unavailable("Device host was stopped while it was starting."); const { nodePath } = cli; ready = { nodePath, @@ -621,6 +624,7 @@ export const make = Effect.fn("SimlockDeviceHost.make")(function* ( const stopAll = Effect.gen(function* () { stops++; + ready = null; yield* FiberMap.clear(keepAlive); yield* Effect.forEach([...leases.values()], release, { discard: true, concurrency: 4 }); }).pipe(Effect.ignore); diff --git a/apps/server/src/device/SshDeviceHost.test.ts b/apps/server/src/device/SshDeviceHost.test.ts index 2f59f156c9ceb..e3cc615585bb9 100644 --- a/apps/server/src/device/SshDeviceHost.test.ts +++ b/apps/server/src/device/SshDeviceHost.test.ts @@ -201,7 +201,12 @@ it.effect("preserves installed status after probes and cleans failed agent activ expect(retired._tag).toBe("Failure"); expect(forwards).toBe(0); const modesAfterClose = [...modes]; - expect((yield* host.ensureReady(() => Effect.void).pipe(Effect.result))._tag).toBe("Failure"); + const unavailable = yield* host.ensureReady(() => Effect.void).pipe(Effect.flip); + expect(unavailable).toMatchObject({ + _tag: "DeviceHostUnavailableError", + reason: "Host configuration changed. Retry the operation.", + }); + expect(unavailable.cause).toBeUndefined(); expect(modes).toEqual(modesAfterClose); }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), ); diff --git a/apps/server/src/device/SshDeviceHost.ts b/apps/server/src/device/SshDeviceHost.ts index 63257d9cce209..ae868a263c3e5 100644 --- a/apps/server/src/device/SshDeviceHost.ts +++ b/apps/server/src/device/SshDeviceHost.ts @@ -1,5 +1,6 @@ import { type DeviceHostSummary, + DeviceHostUnavailableError, DevicePlatformAvailability, DeviceToolVersions, deviceToolInstallMessage, @@ -160,10 +161,10 @@ export const make = Effect.fn("SshDeviceHost.make")(function* ( const requireActive = Effect.suspend(() => retired ? Effect.fail( - new DeviceHost.DeviceHostError({ + new DeviceHostUnavailableError({ hostId: config.id, - step: "starting retired host", - cause: new Error("Host configuration changed. Retry the operation."), + hostLabel: config.label, + reason: "Host configuration changed. Retry the operation.", }), ) : Effect.void, From ec6a4e71eccccdb4521e4be95dbddab0b9332163 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Tue, 3 Nov 2026 03:51:51 +0000 Subject: [PATCH 08/11] fix(server): keep disabled device status during late startup --- .../server/src/device/DeviceMultiHost.test.ts | 11 +++- apps/server/src/device/DeviceService.ts | 61 +++++++++++-------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 5e6ab401ba7fa..f7eaa0b87a062 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -240,10 +240,19 @@ it.effect("stops a start that fails after device support was turned off before i platformAvailability: (platform) => Effect.succeed({ platform, available: true }), ensureReady: () => Effect.succeed(ready), // Like LocalDeviceHost: the hub is up before the agent tools install fails. - ensureAgentReady: () => + ensureAgentReady: (onPhase) => Effect.sync(() => { running = true; }).pipe( + Effect.andThen(onPhase("installing")), + Effect.tap(() => + service.state.pipe( + Effect.map((state) => { + expect(state.hostStatus).toBe("disabled"); + expect(state.hostStatuses).toEqual({}); + }), + ), + ), Effect.andThen( Effect.fail( new DeviceHost.DeviceHostError({ diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 7ed3fcac1a550..811b6d6705bfe 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -359,14 +359,25 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ); // Uninterruptible so a committed retirement always prunes, even if its caller is cancelled. - const publish = (update: (state: DeviceServiceState) => DeviceServiceState) => - SynchronizedRef.modifyEffect(stateRef, ({ state }) => { - const next = { ...update(state), revision: state.revision + 1 }; - const retired = state.sessions.some((session) => !next.sessions.includes(session)); - return PubSub.publish(statePubSub, next).pipe( - Effect.as([{ next, retired }, { state: next }] as const), - ); - }).pipe( + const publish = ( + update: (state: DeviceServiceState) => DeviceServiceState, + onlyWhileEnabled = false, + ) => + SynchronizedRef.modifyEffect(stateRef, ({ state }) => + Effect.gen(function* () { + const enabled = + !onlyWhileEnabled || + (yield* readDeviceSettings.pipe( + Effect.map((current) => current.enabled), + Effect.orElseSucceed(() => false), + )); + const next = { ...(enabled ? update(state) : state), revision: state.revision + 1 }; + const retired = state.sessions.some((session) => !next.sessions.includes(session)); + return yield* PubSub.publish(statePubSub, next).pipe( + Effect.as([{ next, retired }, { state: next }] as const), + ); + }), + ).pipe( Effect.tap(({ retired }) => (retired ? pruneLaunchers : Effect.void)), Effect.map(({ next }) => next), Effect.uninterruptible, @@ -391,24 +402,26 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* Effect.suspend(() => !hosts.has(hostId) ? SynchronizedRef.get(stateRef).pipe(Effect.map(({ state }) => state)) - : publish((state) => - expectedHost && hosts.get(hostId) !== expectedHost - ? state - : { - ...state, - ...(hostId === LOCAL_DEVICE_HOST_ID - ? { hostStatus: status.status, hostStatusDetail: status.detail } - : {}), - hostStatuses: { - ...state.hostStatuses, - [hostId]: { - ...(state.hostStatuses[hostId]?.androidDiscoveryIncomplete - ? { androidDiscoveryIncomplete: true } - : {}), - ...status, + : publish( + (state) => + expectedHost && hosts.get(hostId) !== expectedHost + ? state + : { + ...state, + ...(hostId === LOCAL_DEVICE_HOST_ID + ? { hostStatus: status.status, hostStatusDetail: status.detail } + : {}), + hostStatuses: { + ...state.hostStatuses, + [hostId]: { + ...(state.hostStatuses[hostId]?.androidDiscoveryIncomplete + ? { androidDiscoveryIncomplete: true } + : {}), + ...status, + }, }, }, - }, + true, ), ); From e35f4379fb025f6209ca33ae94346cac46a25df2 Mon Sep 17 00:00:00 2001 From: maria-rcks Date: Fri, 6 Nov 2026 11:01:11 +0000 Subject: [PATCH 09/11] test(server): use platformAvailable in device host fakes --- apps/server/src/device/DeviceMultiHost.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index dff687077a9df..06e40ca4c9623 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -221,7 +221,7 @@ it.effect("stops a start that fails after device support was turned off before i origin: "http://local", http: HttpClient.make(() => Effect.die("unused")), run, - androidAvailable: Effect.succeed(true), + platformAvailable: () => Effect.succeed(true), }), run, helpers: { serveSimAxSettings: null, serveSimCli: null }, @@ -315,7 +315,7 @@ const pausedHost = ( ), ), run, - androidAvailable: Effect.succeed(false), + platformAvailable: () => Effect.succeed(false), }), agentDevice: null, run, From 545b6d4bf320c6ad609d7908c9f0d7fe8252cc9a Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sun, 8 Nov 2026 10:04:05 +0000 Subject: [PATCH 10/11] fix(server): collect host summaries before agent readiness takes the lock Agent readiness read every host's summary while holding the shared lifecycle lock, so a slow summary on one host held up lifecycle work on the others. Collect the summaries first, as refresh already does. Co-Authored-By: Claude Opus 5.5 --- .../server/src/device/DeviceMultiHost.test.ts | 32 +++++++++++++++++++ apps/server/src/device/DeviceService.ts | 5 +-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 06e40ca4c9623..041e1497f5671 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -197,6 +197,38 @@ it.effect("starts other hosts while one host is still starting", () => }).pipe(Effect.provide(ServerSettings.layerTest({ enableDeviceSupport: true }))), ); +it.effect("reads host summaries outside the lifecycle lock when agent tools start", () => + Effect.gen(function* () { + const summaryEntered = yield* Deferred.make(); + const finishSummary = yield* Deferred.make(); + let pauseSummary = false; + const { host: agentHost } = yield* pausedHost("ensureReady", "agent"); + const { host: otherHost } = yield* pausedHost("ensureAgentReady", "other"); + const summary = Effect.suspend(() => + pauseSummary + ? Deferred.succeed(summaryEntered, undefined).pipe( + Effect.andThen(Deferred.await(finishSummary)), + Effect.andThen(otherHost.summary), + ) + : otherHost.summary, + ); + const hosts = new Map([agentHost, { ...otherHost, summary }].map((host) => [host.id, host])); + const service = yield* DeviceService.makeWithHosts(hosts).pipe( + Effect.provide(NodeCrypto.layer), + ); + pauseSummary = true; + const agent = yield* service.agentReadinessIfSupported("agent").pipe(Effect.forkChild); + yield* Deferred.await(summaryEntered); + expect((yield* service.readiness("other")).hostId).toBe("other"); + yield* Deferred.succeed(finishSummary, undefined); + expect((yield* Fiber.join(agent))?.hostId).toBe("agent"); + }).pipe( + Effect.provide( + ServerSettings.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), + ), + ), +); + it.effect("stops a start that fails after device support was turned off before it began", () => Effect.gen(function* () { const summaryEntered = yield* Deferred.make(); diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 2a1a7fbbdb9ca..42231fc8ff6f0 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -614,13 +614,10 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ), stopIfRevoked(host, true), ); + const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); return yield* lifecycleLock.withPermit( Effect.gen(function* () { if (!(yield* confirmStarted(host, true))) return null; - const hostSummaries = yield* Effect.forEach( - hosts.values(), - (candidate) => candidate.summary, - ); yield* publish((state) => ({ ...state, hosts: hostSummaries })); yield* setHostStatus(host.id, { status: "ready" }); return { hostId: host.id, ...ready }; From 67e3a530453075cd926d40faff08880c069d500b Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Wed, 11 Nov 2026 21:24:50 +0000 Subject: [PATCH 11/11] fix(server): agent readiness no longer brings back a host removed while summaries ran Agent readiness reads host summaries before it takes the lifecycle lock. If a host was removed or replaced in that window, publishing that snapshot put the old host back in the list. Publish it only when the hosts are unchanged. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/device/DeviceMultiHost.test.ts | 4 ++++ apps/server/src/device/DeviceService.ts | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/server/src/device/DeviceMultiHost.test.ts b/apps/server/src/device/DeviceMultiHost.test.ts index 041e1497f5671..7491155c203a7 100644 --- a/apps/server/src/device/DeviceMultiHost.test.ts +++ b/apps/server/src/device/DeviceMultiHost.test.ts @@ -220,8 +220,12 @@ it.effect("reads host summaries outside the lifecycle lock when agent tools star const agent = yield* service.agentReadinessIfSupported("agent").pipe(Effect.forkChild); yield* Deferred.await(summaryEntered); expect((yield* service.readiness("other")).hostId).toBe("other"); + yield* service.withLifecycleLock( + Effect.sync(() => hosts.delete("other")).pipe(Effect.andThen(service.refreshHosts)), + ); yield* Deferred.succeed(finishSummary, undefined); expect((yield* Fiber.join(agent))?.hostId).toBe("agent"); + expect((yield* service.state).hosts.map((summary) => summary.id)).toEqual(["agent"]); }).pipe( Effect.provide( ServerSettings.layerTest({ enableDeviceSupport: true, enableAgentDeviceAccess: true }), diff --git a/apps/server/src/device/DeviceService.ts b/apps/server/src/device/DeviceService.ts index 543ba3f1f846d..1aab127dd892d 100644 --- a/apps/server/src/device/DeviceService.ts +++ b/apps/server/src/device/DeviceService.ts @@ -623,11 +623,17 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function* ), stopIfRevoked(host, true), ); - const hostSummaries = yield* Effect.forEach(hosts.values(), (candidate) => candidate.summary); + const summarized = [...hosts.values()]; + const hostSummaries = yield* Effect.forEach(summarized, (candidate) => candidate.summary); return yield* lifecycleLock.withPermit( Effect.gen(function* () { if (!(yield* confirmStarted(host, true))) return null; - yield* publish((state) => ({ ...state, hosts: hostSummaries })); + // A reconcile while the summaries ran has already published the newer host list. + if ( + summarized.length === hosts.size && + summarized.every((candidate) => hosts.get(candidate.id) === candidate) + ) + yield* publish((state) => ({ ...state, hosts: hostSummaries })); yield* setHostStatus(host.id, { status: "ready" }); return { hostId: host.id, ...ready }; }),