fix(desktop): wait for the server's database upgrade instead of reporting a failed start

apps/desktop/src/backend/DesktopBackendManager.test.ts

@@ -5,6 +5,10 @@ import {
   type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
   DesktopTelemetryControlMessage,
 } from "@t3tools/contracts";
+import {
+  DATABASE_MIGRATIONS_FINISHED_MESSAGE,
+  DATABASE_MIGRATIONS_STARTED_MESSAGE,
+} from "@t3tools/shared/databaseMigrationLog";
 import { assert, describe, it } from "@effect/vitest";
 import * as Deferred from "effect/Deferred";
 import * as Duration from "effect/Duration";
@@ -688,12 +692,21 @@ describe("DesktopBackendManager", () => {
     Effect.scoped(
       Effect.gen(function* () {
         const reports = yield* Queue.unbounded<DesktopBackendManager.BackendStartupFailure>();
-        const startInstance = (status: Ref.Ref<number>, onReady?: Effect.Effect<void>) =>
+        const startInstance = (
+          status: Ref.Ref<number>,
+          onReady?: Effect.Effect<void>,
+          stdout = "",
+        ) =>
           makeTestInstance({
             spawnerLayer: Layer.succeed(
               ChildProcessSpawner.ChildProcessSpawner,
               ChildProcessSpawner.make(() =>
-                Effect.succeed(makeProcess({ exitCode: Effect.never })),
+                Effect.succeed(
+                  makeProcess({
+                    exitCode: Effect.never,
+                    stdout: Stream.make(new TextEncoder().encode(stdout)),
+                  }),
+                ),
               ),
             ),
             httpClientLayer: layerHttpClient((request) =>
@@ -712,7 +725,24 @@ describe("DesktopBackendManager", () => {
         yield* Deferred.await(ready);
         assert.isTrue(Option.isNone(yield* Queue.poll(reports)));
 
-        yield* startInstance(yield* Ref.make(503));
+        const migratingStatus = yield* Ref.make(503);
+        const migrated = yield* Deferred.make<void>();
+        yield* startInstance(
+          migratingStatus,
+          Deferred.succeed(migrated, undefined).pipe(Effect.asVoid),
+          `INFO ${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n`,
+        );
+        yield* TestClock.adjust(Duration.minutes(10));
+        assert.isTrue(Option.isNone(yield* Queue.poll(reports)));
+        yield* Ref.set(migratingStatus, 200);
+        yield* TestClock.adjust(Duration.millis(100));
+        yield* Deferred.await(migrated);
+
+        yield* startInstance(
+          yield* Ref.make(503),
+          undefined,
+          `${DATABASE_MIGRATIONS_STARTED_MESSAGE}\n${DATABASE_MIGRATIONS_FINISHED_MESSAGE}\n`,
+        );
         yield* TestClock.adjust(Duration.seconds(179));
         assert.isTrue(Option.isNone(yield* Queue.poll(reports)));
         yield* TestClock.adjust(Duration.seconds(1));

apps/desktop/src/backend/DesktopBackendManager.ts

@@ -25,6 +25,7 @@ import {
   DesktopTelemetryControlMessage,
   type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue,
 } from "@t3tools/contracts";
+import { isRunningDatabaseMigrations } from "@t3tools/shared/databaseMigrationLog";
 import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness";
 
 import * as DesktopObservability from "../app/DesktopObservability.ts";
@@ -971,6 +972,12 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
           }),
           onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")(
             function* (error) {
+              if (isRunningDatabaseMigrations(yield* Ref.get(outputTail))) {
+                yield* logInstanceWarning("backend is still migrating its database; waiting", {
+                  error: error.message,
+                });
+                return;
+              }
               yield* logInstanceWarning("backend readiness check failed during bootstrap", {
                 error: error.message,
               });

apps/server/src/persistence/Migrations.ts

@@ -1,6 +1,10 @@
 import * as Migrator from "effect/sql/Migrator";
 import * as Effect from "effect/Effect";
 import * as SqlClient from "effect/sql/SqlClient";
+import {
+  DATABASE_MIGRATIONS_FINISHED_MESSAGE,
+  DATABASE_MIGRATIONS_STARTED_MESSAGE,
+} from "@t3tools/shared/databaseMigrationLog";
 import { reconcileV2PreviewMigration } from "./reconcileV2PreviewMigration.ts";
 
 import Migration0001 from "./Migrations/001_OrchestrationEvents.ts";
@@ -256,6 +260,8 @@ export interface RunMigrationsOptions {
 export const runMigrations = Effect.fn("runMigrations")(function* ({
   toMigrationInclusive,
 }: RunMigrationsOptions = {}) {
+  // The desktop waits past its readiness budget while this is the server's last output.
+  if (yield* hasPendingMigrations()) yield* Effect.log(DATABASE_MIGRATIONS_STARTED_MESSAGE);
   const includesV2 =
     toMigrationInclusive === undefined || toMigrationInclusive >= FIRST_V2_MIGRATION_ID;
   const previewMigrations = includesV2 ? yield* reconcileV2PreviewMigration() : [];
@@ -269,7 +275,7 @@ export const runMigrations = Effect.fn("runMigrations")(function* ({
   const migrations = executedMigrations.map(([id, name]) => `${id}_${name}`);
   yield* migrations.length === 0
     ? Effect.logDebug("Database schema is current")
-    : Effect.log("Migrations ran successfully").pipe(Effect.annotateLogs({ migrations }));
+    : Effect.log(DATABASE_MIGRATIONS_FINISHED_MESSAGE).pipe(Effect.annotateLogs({ migrations }));
 
   const sql = yield* SqlClient.SqlClient;
   const recorded = yield* sql<{

packages/shared/package.json

@@ -367,6 +367,10 @@
       "types": "./src/testing/symlinks.ts",
       "import": "./src/testing/symlinks.ts"
     },
+    "./databaseMigrationLog": {
+      "types": "./src/databaseMigrationLog.ts",
+      "import": "./src/databaseMigrationLog.ts"
+    },
     "./httpReadiness": {
       "types": "./src/httpReadiness.ts",
       "import": "./src/httpReadiness.ts"

packages/shared/src/databaseMigrationLog.ts

@@ -0,0 +1,11 @@
+// The server logs these around pending migrations; the desktop reads them from its output.
+// Migrations run synchronously, so a long one blocks the server from writing anything else.
+export const DATABASE_MIGRATIONS_STARTED_MESSAGE = "Running database migrations";
+export const DATABASE_MIGRATIONS_FINISHED_MESSAGE = "Migrations ran successfully";
+
+export function isRunningDatabaseMigrations(output: string): boolean {
+  return (
+    output.lastIndexOf(DATABASE_MIGRATIONS_STARTED_MESSAGE) >
+    output.lastIndexOf(DATABASE_MIGRATIONS_FINISHED_MESSAGE)
+  );
+}