fix(ssh): wait for the desktop app's server across an update restart

apps/server/src/cli/sshHelper.ts

@@ -125,7 +125,23 @@ const runtimePort = Command.make("runtime-port", {
   ),
 );
 
+/** Exits 0 when the file was modified less than `max-age-ms` ago. */
+const fileFresh = Command.make("file-fresh", {
+  file: Argument.String("file"),
+  maxAgeMs: Argument.Int("max-age-ms"),
+}).pipe(
+  Command.withHandler(({ file, maxAgeMs }) =>
+    Effect.sync(() => {
+      try {
+        process.exitCode = Date.now() - NodeFS.statSync(file).mtimeMs < maxAgeMs ? 0 : 1;
+      } catch {
+        process.exitCode = 1;
+      }
+    }),
+  ),
+);
+
 export const sshHelperCommand = Command.make("__ssh-helper").pipe(
   Command.unlisted,
-  Command.withSubcommands([pickPort, waitReady, runtimePort]),
+  Command.withSubcommands([pickPort, waitReady, runtimePort, fileFresh]),
 );

apps/server/src/cloud/updateHandoff.ts

@@ -1,6 +1,8 @@
-import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts";
+import {
+  DESKTOP_UPDATE_RESTART_MARKER_FILE,
+  DESKTOP_UPDATE_RESTART_MARKER_TTL_MS,
+} from "@t3tools/contracts";
 import * as Clock from "effect/Clock";
-import * as Duration from "effect/Duration";
 import * as Effect from "effect/Effect";
 import * as FileSystem from "effect/FileSystem";
 import * as Option from "effect/Option";
@@ -13,8 +15,6 @@ import {
   serviceStateHasPendingUpdate,
 } from "./serviceProtocol.ts";
 
-const DESKTOP_UPDATE_RESTART_MARKER_TTL = Duration.minutes(1);
-
 export const pendingServiceUpdateExists = Effect.gen(function* () {
   const config = yield* ServerConfig.ServerConfig;
   const fs = yield* FileSystem.FileSystem;
@@ -59,7 +59,6 @@ export const desktopUpdateRestartPending = Effect.gen(function* () {
   const now = yield* Clock.currentTimeMillis;
   return Option.match(marker.value.mtime, {
     onNone: () => false,
-    onSome: (writtenAt) =>
-      now - writtenAt.getTime() < Duration.toMillis(DESKTOP_UPDATE_RESTART_MARKER_TTL),
+    onSome: (writtenAt) => now - writtenAt.getTime() < DESKTOP_UPDATE_RESTART_MARKER_TTL_MS,
   });
 });

packages/contracts/src/desktopBootstrap.ts

@@ -43,3 +43,6 @@ export type DesktopBackendBootstrap = typeof DesktopBackendBootstrap.Type;
     away, so a backend that sees a fresh marker at shutdown keeps its managed
     tunnel. */
 export const DESKTOP_UPDATE_RESTART_MARKER_FILE = "desktop-update-restart";
+/** How long the marker counts as fresh: past this, the update restart failed
+    or happened long ago. */
+export const DESKTOP_UPDATE_RESTART_MARKER_TTL_MS = 60_000;

packages/ssh/src/tunnel.launchScript.test.ts

@@ -0,0 +1,131 @@
+// @effect-diagnostics nodeBuiltinImport:off - Runs the rendered launch script in a real shell.
+// @effect-diagnostics globalDate:off - Backdates the marker's mtime on the real file system.
+import * as NodeChildProcess from "node:child_process";
+import * as NodeFS from "node:fs";
+import * as NodeNet from "node:net";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+
+import { DESKTOP_UPDATE_RESTART_MARKER_FILE } from "@t3tools/contracts";
+import { afterEach, describe, expect, it } from "vite-plus/test";
+
+import { buildRemoteLaunchScript } from "./tunnel.ts";
+
+const STATE_KEY = "test";
+
+// Stands in for `t3 serve` and the desktop backend: answers HTTP on --port and,
+// as the desktop backend, records itself in the default runtime file.
+const FAKE_SERVER = `
+const fs = require("node:fs");
+const http = require("node:http");
+const args = process.argv.slice(2);
+const port = Number(args[args.indexOf("--port") + 1]);
+const runtimeFile = process.env.T3_FAKE_RUNTIME_FILE;
+setTimeout(() => {
+  http.createServer((_, res) => res.end("ok")).listen(port, "127.0.0.1", () => {
+    if (runtimeFile) {
+      fs.writeFileSync(runtimeFile, JSON.stringify({ pid: process.pid, port, origin: "http://127.0.0.1:" + port }));
+    }
+  });
+}, Number(process.env.T3_FAKE_DELAY_MS ?? 0));
+`;
+
+const directories: Array<string> = [];
+const pids: Array<number> = [];
+afterEach(() => {
+  for (const pid of pids.splice(0)) {
+    try {
+      process.kill(pid);
+    } catch {}
+  }
+  for (const directory of directories.splice(0)) {
+    NodeFS.rmSync(directory, { recursive: true, force: true });
+  }
+});
+
+const makeHome = () => {
+  const home = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-ssh-launch-"));
+  directories.push(home);
+  NodeFS.mkdirSync(NodePath.join(home, ".t3", "runtime"), { recursive: true });
+  NodeFS.mkdirSync(NodePath.join(home, ".t3", "userdata"), { recursive: true });
+  const serverScript = NodePath.join(home, "fake-server.cjs");
+  NodeFS.writeFileSync(serverScript, FAKE_SERVER);
+  return { home, serverScript };
+};
+
+const writeUpdateMarker = (home: string, options?: { readonly stale: true }) => {
+  const marker = NodePath.join(home, ".t3", "runtime", DESKTOP_UPDATE_RESTART_MARKER_FILE);
+  NodeFS.writeFileSync(marker, "");
+  if (options?.stale) {
+    const longAgo = new Date("2020-01-01T00:00:00Z");
+    NodeFS.utimesSync(marker, longAgo, longAgo);
+  }
+};
+
+const freePort = () =>
+  new Promise<number>((resolve) => {
+    const server = NodeNet.createServer();
+    server.listen(0, "127.0.0.1", () => {
+      const { port } = server.address() as NodeNet.AddressInfo;
+      server.close(() => resolve(port));
+    });
+  });
+
+const startDesktopBackend = (home: string, serverScript: string, port: number, delayMs: number) => {
+  const child = NodeChildProcess.spawn(process.execPath, [serverScript, "--port", String(port)], {
+    env: {
+      ...process.env,
+      T3_FAKE_DELAY_MS: String(delayMs),
+      T3_FAKE_RUNTIME_FILE: NodePath.join(home, ".t3", "userdata", "server-runtime.json"),
+    },
+    stdio: "ignore",
+  });
+  if (child.pid !== undefined) pids.push(child.pid);
+};
+
+const runLaunch = async (home: string, serverScript: string) => {
+  const result = await new Promise<{ readonly status: number | null; readonly stdout: string }>(
+    (resolve) => {
+      const child = NodeChildProcess.spawn("sh", ["-s", "--", STATE_KEY], {
+        env: { PATH: process.env.PATH, HOME: home },
+      });
+      let stdout = "";
+      child.stdout.on("data", (chunk) => (stdout += chunk));
+      child.on("close", (status) => resolve({ status, stdout }));
+      child.stdin.end(buildRemoteLaunchScript({ nodeScriptPath: serverScript }));
+    },
+  );
+  const pidFile = NodePath.join(home, ".t3", "ssh-launch", STATE_KEY, "pid");
+  if (NodeFS.existsSync(pidFile)) pids.push(Number(NodeFS.readFileSync(pidFile, "utf8")));
+  return result;
+};
+
+describe("remote launch script", () => {
+  it("waits for the desktop app's updated server instead of launching one", async () => {
+    const { home, serverScript } = makeHome();
+    const desktopPort = await freePort();
+    writeUpdateMarker(home);
+    startDesktopBackend(home, serverScript, desktopPort, 3_000);
+
+    const result = await runLaunch(home, serverScript);
+
+    expect(result.status).toBe(0);
+    expect(JSON.parse(result.stdout.trim())).toEqual({
+      remotePort: desktopPort,
+      serverKind: "external",
+    });
+  });
+
+  it("launches its own server when the update marker is stale", async () => {
+    const { home, serverScript } = makeHome();
+    const desktopPort = await freePort();
+    writeUpdateMarker(home, { stale: true });
+    // A launcher that wrongly waited would adopt this server once it is up.
+    startDesktopBackend(home, serverScript, desktopPort, 3_000);
+
+    const result = await runLaunch(home, serverScript);
+
+    expect(result.status).toBe(0);
+    expect(JSON.parse(result.stdout.trim()).serverKind).toBe("managed");
+  });
+});

packages/ssh/src/tunnel.ts

@@ -1,6 +1,8 @@
-import type {
-  DesktopSshEnvironmentBootstrap,
-  DesktopSshEnvironmentTarget,
+import {
+  DESKTOP_UPDATE_RESTART_MARKER_FILE,
+  DESKTOP_UPDATE_RESTART_MARKER_TTL_MS,
+  type DesktopSshEnvironmentBootstrap,
+  type DesktopSshEnvironmentTarget,
 } from "@t3tools/contracts";
 import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness";
 import { cliReleaseDownloadBaseUrl } from "@t3tools/shared/cliRelease";
@@ -62,6 +64,9 @@ const REMOTE_ARCHIVE_DOWNLOAD_SECONDS = 240;
 const REMOTE_ARCHIVE_LOCK_WAIT_SECONDS = 360;
 const REMOTE_ARCHIVE_LAUNCH_TIMEOUT_MS = 900_000;
 const REMOTE_REUSE_READY_TIMEOUT_MS = 2_000;
+// The updated desktop app is back within seconds; this bounds the wait when
+// it is not, and stays well inside REMOTE_LAUNCH_TIMEOUT_MS.
+const REMOTE_DESKTOP_UPDATE_WAIT_SECONDS = 20;
 
 export interface RemoteT3RunnerOptions {
   readonly nodeScriptPath?: string | null;
@@ -540,6 +545,7 @@ STATE_KEY="$1"
 STATE_DIR="$HOME/.t3/ssh-launch/$STATE_KEY"
 DEFAULT_SERVER_HOME="$HOME/.t3"
 DEFAULT_RUNTIME_FILE="$DEFAULT_SERVER_HOME/userdata/server-runtime.json"
+DESKTOP_UPDATE_MARKER="$DEFAULT_SERVER_HOME/runtime/@@T3_DESKTOP_UPDATE_MARKER_FILE@@"
 PORT_FILE="$STATE_DIR/port"
 PID_FILE="$STATE_DIR/pid"
 MANAGED_FILE="$STATE_DIR/managed"
@@ -630,10 +636,34 @@ try {
 }
 NODE
 }
+desktop_update_restart_pending() {
+  if [ "$T3_ARCHIVE_MODE" = "1" ]; then
+    "$RUNNER_FILE" __ssh-helper file-fresh "$DESKTOP_UPDATE_MARKER" "@@T3_DESKTOP_UPDATE_MARKER_TTL_MS@@"
+    return
+  fi
+  node - "$DESKTOP_UPDATE_MARKER" "@@T3_DESKTOP_UPDATE_MARKER_TTL_MS@@" <<'NODE'
+const fs = require("node:fs");
+try {
+  process.exit(Date.now() - fs.statSync(process.argv[2]).mtimeMs < Number(process.argv[3]) ? 0 : 1);
+} catch {
+  process.exit(1);
+}
+NODE
+}
 REMOTE_PID="$(cat "$PID_FILE" 2>/dev/null || true)"
 REMOTE_PORT="$(cat "$PORT_FILE" 2>/dev/null || true)"
 REMOTE_MANAGED="$(cat "$MANAGED_FILE" 2>/dev/null || true)"
 DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
+# A desktop app installing an update stops its server and starts the new
+# version on the default home seconds later. A server launched into that gap
+# owns the home, so the updated app cannot start. Wait for the app's server
+# instead and let the adopt path below take it.
+DESKTOP_UPDATE_WAITED=0
+while [ -z "$DEFAULT_RUNTIME_INFO" ] && [ "$DESKTOP_UPDATE_WAITED" -lt @@T3_DESKTOP_UPDATE_WAIT_SECONDS@@ ] && desktop_update_restart_pending; do
+  sleep 1
+  DESKTOP_UPDATE_WAITED=$((DESKTOP_UPDATE_WAITED + 1))
+  DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port 2>/dev/null || true)"
+done
 DEFAULT_RUNTIME_PID=""
 DEFAULT_REMOTE_PORT=""
 ADOPTED_PID=""
@@ -836,7 +866,7 @@ export function buildRemoteNodeEnvScript(input?: RemoteT3RunnerOptions): string
   );
 }
 
-function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
+export function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
   return applyScriptPlaceholders(REMOTE_LAUNCH_SCRIPT, {
     T3_ARCHIVE_MODE: isNodeScriptRunner(input) ? "0" : "1",
     T3_NODE_ENV_SCRIPT: buildRemoteNodeEnvScript(input),
@@ -848,6 +878,9 @@ function buildRemoteLaunchScript(input?: RemoteT3RunnerOptions): string {
     T3_READY_TIMEOUT_MS: String(REMOTE_READY_TIMEOUT_MS),
     T3_REUSE_READY_TIMEOUT_MS: String(REMOTE_REUSE_READY_TIMEOUT_MS),
     T3_READY_PROBE_TIMEOUT_MS: String(SSH_READY_PROBE_TIMEOUT_MS),
+    T3_DESKTOP_UPDATE_MARKER_FILE: DESKTOP_UPDATE_RESTART_MARKER_FILE,
+    T3_DESKTOP_UPDATE_MARKER_TTL_MS: String(DESKTOP_UPDATE_RESTART_MARKER_TTL_MS),
+    T3_DESKTOP_UPDATE_WAIT_SECONDS: String(REMOTE_DESKTOP_UPDATE_WAIT_SECONDS),
   });
 }