diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index b69fa6e4bb299..d48bae628ef7c 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -38,10 +38,17 @@ if ( }) ) { // The launcher owns shutdown until its child exits, so it bypasses the Effect CLI runtime. - // The service unit passes no arguments; flags and help go through the full CLI. - if (process.argv[2] === "__service-launcher" && process.argv.length === 3) { - const { runServiceLauncherCli } = await import("./cli/serviceLauncher.ts"); - await runServiceLauncherCli(); + // Service units pass only launcher flags; global flags and help go through the full CLI. + if (process.argv[2] === "__service-launcher") { + const { parseServiceLauncherArgs, runServiceLauncherCli } = + await import("./cli/serviceLauncher.ts"); + const launcherOptions = parseServiceLauncherArgs(process.argv.slice(3)); + if (launcherOptions !== undefined) { + await runServiceLauncherCli(launcherOptions); + } else { + const { runCliCommand } = await loadCliMain(); + await runCliCommand(); + } } else { const { runCliCommand } = await loadCliMain(); await runCliCommand(); diff --git a/apps/server/src/binCli.ts b/apps/server/src/binCli.ts index 995b207e7c781..c0b14805334d1 100644 --- a/apps/server/src/binCli.ts +++ b/apps/server/src/binCli.ts @@ -5,7 +5,8 @@ import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { Argument, Command } from "effect/cli"; +import * as Option from "effect/Option"; +import { Argument, Command, Flag } from "effect/cli"; import * as CliError from "effect/cli/CliError"; import * as NetService from "@t3tools/shared/Net"; @@ -97,11 +98,24 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => updateCommand, installCommand, uninstallCommand, - // bin.ts runs the bare service invocation; this keeps flags and help working like other commands. - Command.make("__service-launcher").pipe( + // bin.ts runs the service invocation directly; this keeps global flags and help working. + Command.make("__service-launcher", { + baseDir: Flag.String("base-dir").pipe(Flag.optional), + logFile: Flag.String("log-file").pipe(Flag.optional), + serviceUnit: Flag.String("service-unit").pipe(Flag.optional), + }).pipe( Command.unlisted, // The launcher owns shutdown until its child exits; `runMain` would interrupt that shutdown. - Command.withHandler(() => Effect.sync(() => void runServiceLauncherCli())), + Command.withHandler((flags) => + Effect.sync( + () => + void runServiceLauncherCli({ + baseDir: Option.getOrUndefined(flags.baseDir), + logFile: Option.getOrUndefined(flags.logFile), + serviceUnit: Option.getOrUndefined(flags.serviceUnit), + }), + ), + ), ), claudeHistoryCommand, speechWorkerCommand, diff --git a/apps/server/src/cli/server.ts b/apps/server/src/cli/server.ts index 0e90979439ae3..c0930f4cb70b3 100644 --- a/apps/server/src/cli/server.ts +++ b/apps/server/src/cli/server.ts @@ -6,6 +6,7 @@ import * as Schema from "effect/Schema"; import { Command, GlobalFlag } from "effect/cli"; import * as CliError from "effect/cli/CliError"; +import * as ServiceLauncherClient from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; import { awaitLauncherLoss } from "../launcherWatch.ts"; import * as DatabaseMaintenance from "../persistence/DatabaseMaintenance.ts"; @@ -30,31 +31,43 @@ const runServerCommand = Effect.fn("server.runServerCommand")(function* ( readonly rejectRunningServer?: boolean; }, ) { - // Before config resolution, which claims a port and creates directories. - const { runServer } = yield* Effect.promise(() => import("../server.ts")); - const logLevel = yield* GlobalFlag.LogLevel; - const selectAutomaticPort = yield* AutomaticPort.make.pipe(Effect.provide(ProcessRunner.layer)); - const config = yield* resolveServerConfig(flags, logLevel, { ...options, selectAutomaticPort }); - const run = runServer.pipe( - Effect.provideService(ServerConfig.ServerConfig, config), - // Settings' Reclaim stops the server; repair once the database is closed, before the restart. - Effect.ensuring( - DatabaseMaintenance.runRequestedReclaim(config.dbPath).pipe( - Effect.provide(DatabaseMaintenance.layer), - ), - ), - ); - if (config.mode !== "desktop" || config.desktopBootstrapToken === undefined) return yield* run; - yield* Effect.sync(() => { - process.stdout.on("error", ignoreClosedLauncherPipe); - process.stderr.on("error", ignoreClosedLauncherPipe); - }); return yield* Effect.raceFirst( - run, - awaitLauncherLoss().pipe( - // End like SIGTERM does (interrupt-only exit, so the runtime forces process exit). - Effect.andThen(Effect.interrupt), - ), + Effect.gen(function* () { + // Before config resolution, which claims a port and creates directories. + const { runServer } = yield* Effect.promise(() => import("../server.ts")); + const logLevel = yield* GlobalFlag.LogLevel; + const selectAutomaticPort = yield* AutomaticPort.make.pipe( + Effect.provide(ProcessRunner.layer), + ); + const config = yield* resolveServerConfig(flags, logLevel, { + ...options, + selectAutomaticPort, + }); + const run = runServer.pipe( + Effect.provideService(ServerConfig.ServerConfig, config), + // Settings' Reclaim stops the server; repair once the database is closed, before the restart. + Effect.ensuring( + DatabaseMaintenance.runRequestedReclaim(config.dbPath).pipe( + Effect.provide(DatabaseMaintenance.layer), + ), + ), + ); + if (config.mode !== "desktop" || config.desktopBootstrapToken === undefined) { + return yield* run; + } + yield* Effect.sync(() => { + process.stdout.on("error", ignoreClosedLauncherPipe); + process.stderr.on("error", ignoreClosedLauncherPipe); + }); + return yield* Effect.raceFirst( + run, + awaitLauncherLoss().pipe( + // End like SIGTERM does (interrupt-only exit, so the runtime forces process exit). + Effect.andThen(Effect.interrupt), + ), + ); + }), + ServiceLauncherClient.awaitServiceLauncherStop().pipe(Effect.andThen(Effect.interrupt)), ); }); diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index ce43154116b8e..f6c5e1d5acb9a 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -154,7 +154,7 @@ function formatServiceStatus( protection?: ServiceProtection.ServiceProtectionState, ): string { if (!status.supported) { - return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd"; + return "T3 Code service\n Status: unavailable on this machine\n Supported on: Linux with systemd, macOS with launchd, Windows with Task Scheduler"; } if (!status.installed) { return "T3 Code service\n Status: not installed\n Next: Run `t3 service install`."; @@ -296,6 +296,7 @@ const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags). Effect.gen(function* () { const service = yield* BootService.BootService; const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; const { baseDir } = yield* ServerConfig.ServerConfig; // An unreadable unit stays removable; only a unit known to serve another home is kept. const installedBaseDir = yield* service.status.pipe( @@ -304,7 +305,7 @@ const serviceUninstallCommand = Command.make("uninstall", projectLocationFlags). ); if ( installedBaseDir !== undefined && - path.resolve(installedBaseDir) !== path.resolve(baseDir) + !BootService.sameBootServiceBaseDir(path, platform, installedBaseDir, baseDir) ) { yield* Console.log( `The T3 Code service serves ${installedBaseDir}, so it was not removed. Pass \`${BootService.formatBootServiceBaseDirOption(installedBaseDir)}\` to remove it.`, diff --git a/apps/server/src/cli/serviceLauncher.ts b/apps/server/src/cli/serviceLauncher.ts index 252659fd25489..4d16e59c055fe 100644 --- a/apps/server/src/cli/serviceLauncher.ts +++ b/apps/server/src/cli/serviceLauncher.ts @@ -1,8 +1,26 @@ -import { main as runServiceLauncher } from "../serviceLauncher.ts"; +import { type LauncherOptions, main as runServiceLauncher } from "../serviceLauncher.ts"; -export async function runServiceLauncherCli(): Promise { +/** Parses the Task Scheduler launcher flags without loading the full CLI graph. */ +export function parseServiceLauncherArgs(args: ReadonlyArray): LauncherOptions | undefined { + if (args.length % 2 !== 0) return undefined; + let baseDir: string | undefined; + let logFile: string | undefined; + let serviceUnit: string | undefined; + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]; + const value = args[index + 1]; + if (value === undefined) return undefined; + if (flag === "--base-dir") baseDir = value; + else if (flag === "--log-file") logFile = value; + else if (flag === "--service-unit") serviceUnit = value; + else return undefined; + } + return { baseDir, logFile, serviceUnit }; +} + +export async function runServiceLauncherCli(options?: LauncherOptions): Promise { try { - await runServiceLauncher(); + await runServiceLauncher(options); } catch (cause) { const message = cause instanceof Error ? cause.message : String(cause); process.stderr.write(`[service-launcher] ${message}\n`); diff --git a/apps/server/src/cli/uninstall.test.ts b/apps/server/src/cli/uninstall.test.ts index 4dcdc59e30a78..c04084d19581f 100644 --- a/apps/server/src/cli/uninstall.test.ts +++ b/apps/server/src/cli/uninstall.test.ts @@ -6,9 +6,93 @@ import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as HostProcess from "@t3tools/shared/HostProcess"; -import { findOwnedLauncher, findRecordedLaunchers, spawnWindowsRemoval } from "./uninstall.ts"; +import * as BootService from "../cloud/bootService.ts"; +import { + findOwnedLauncher, + findRecordedLaunchers, + runUninstall, + spawnWindowsRemoval, +} from "./uninstall.ts"; it.layer(NodeServices.layer)("t3 uninstall launcher", (it) => { + it.effect( + "recovers an orphaned background service and moves unrecovered database backups into userdata", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs + .makeTempDirectoryScoped({ prefix: "t3-uninstall-run-" }) + .pipe(Effect.flatMap(fs.realPath)); + const runtimeDir = path.join(root, "runtime"); + yield* fs.makeDirectory(path.join(runtimeDir, "versions", "1.0.0"), { recursive: true }); + yield* fs.writeFileString(path.join(runtimeDir, "versions", "1.0.0", "t3"), ""); + yield* fs.makeDirectory(path.join(runtimeDir, "db-backup", "pending-1"), { + recursive: true, + }); + yield* fs.writeFileString( + path.join(runtimeDir, "db-backup", "pending-1", "database"), + "saved-pre-trial", + ); + yield* fs.makeDirectory(path.join(runtimeDir, "db-backup", "partial.staging"), { + recursive: true, + }); + + let uninstallCalls = 0; + const bootService = BootService.BootService.of({ + install: () => Effect.die("unexpected install"), + restart: Effect.die("unexpected restart"), + withStopped: (operation) => operation, + uninstall: Effect.sync(() => { + uninstallCalls += 1; + return true; + }), + status: Effect.succeed({ + supported: true, + installed: false, + current: false, + unitPath: path.join(root, "unit", "t3code.service"), + logPath: path.join(root, "logs", "boot-service.log"), + }), + }); + + yield* runUninstall({ baseDir: root, assumeYes: true }).pipe( + Effect.provideService(BootService.BootService, bootService), + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, {}), + Effect.provideService(HostProcess.IsExecutable, false), + ); + + assert.equal(uninstallCalls, 1); + assert.isFalse(yield* fs.exists(runtimeDir)); + assert.equal( + yield* fs.readFileString( + path.join(root, "userdata", "db-backup", "pending-1", "database"), + ), + "saved-pre-trial", + ); + assert.isFalse( + yield* fs.exists(path.join(root, "userdata", "db-backup", "partial.staging")), + ); + + // If `runtime/db-backup` cannot be enumerated for a non-NotFound reason (e.g. ENOTDIR), + // uninstall aborts before deleting `runtimeDir`. + yield* fs.makeDirectory(path.join(runtimeDir, "versions", "1.0.0"), { recursive: true }); + yield* fs.writeFileString(path.join(runtimeDir, "versions", "1.0.0", "t3"), ""); + yield* fs.writeFileString(path.join(runtimeDir, "db-backup"), "not-a-directory"); + const failedError = yield* runUninstall({ baseDir: root, assumeYes: true }).pipe( + Effect.provideService(BootService.BootService, bootService), + Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Environment, {}), + Effect.provideService(HostProcess.IsExecutable, false), + Effect.flip, + ); + assert.equal(failedError._tag, "CliUninstallError"); + assert.include(failedError.message, "Could not inspect pending update database backups"); + assert.isTrue(yield* fs.exists(path.join(runtimeDir, "versions", "1.0.0", "t3"))); + }).pipe(Effect.scoped), + ); + it.effect("claims only a launcher that points into this home's runtime tree", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/server/src/cli/uninstall.ts b/apps/server/src/cli/uninstall.ts index 5cf6f0c5e64cc..6456ce5863bf9 100644 --- a/apps/server/src/cli/uninstall.ts +++ b/apps/server/src/cli/uninstall.ts @@ -32,6 +32,7 @@ export class CliUninstallError extends Schema.TaggedError()( export interface UninstallPlan { readonly service: boolean; + readonly canRecoverOrphanService: boolean; readonly versionsDir: string; readonly launchers: ReadonlyArray; readonly runtimeDir: string | undefined; @@ -144,17 +145,20 @@ const planUninstall = Effect.fn("cli.uninstall.plan")(function* (input: { }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const platform = yield* HostProcess.Platform; const service = yield* BootService.BootService; const status = yield* service.status; const servesThisHome = status.installedBaseDir !== undefined && - path.resolve(status.installedBaseDir) === path.resolve(input.baseDir); + BootService.sameBootServiceBaseDir(path, platform, status.installedBaseDir, input.baseDir); const versionsDir = pinnedRuntimeVersionsDir(path, input.baseDir); const runtimeDir = path.dirname(versionsDir); const launchedAs = (yield* HostProcess.IsExecutable) ? yield* resolveLauncherPath : undefined; const launchedFrom = yield* findOwnedLauncher({ launchedAs, versionsDir }); + const runtimeExists = yield* fs.exists(runtimeDir).pipe(Effect.orElseSucceed(() => false)); const plan: UninstallPlan = { service: status.supported && status.installed && servesThisHome, + canRecoverOrphanService: status.supported && !status.installed && runtimeExists, versionsDir, launchers: [ ...new Set([ @@ -162,9 +166,7 @@ const planUninstall = Effect.fn("cli.uninstall.plan")(function* (input: { ...(yield* findRecordedLaunchers(versionsDir)), ]), ], - runtimeDir: (yield* fs.exists(runtimeDir).pipe(Effect.orElseSucceed(() => false))) - ? runtimeDir - : undefined, + runtimeDir: runtimeExists ? runtimeDir : undefined, userdataDir: path.join(input.baseDir, "userdata"), }; return plan; @@ -194,7 +196,52 @@ export const uninstallCommand = Command.make("uninstall", { ), ); -const runUninstall = Effect.fn("cli.uninstall.run")(function* (input: { +const preserveUnrecoveredDatabaseBackups = Effect.fn("cli.uninstall.preserve_db_backups")( + function* (input: { readonly runtimeDir: string; readonly userdataDir: string }) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const backupRoot = path.join(input.runtimeDir, "db-backup"); + const entries = yield* fs.readDirectory(backupRoot).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed([] as Array), + ), + Effect.mapError( + () => + new CliUninstallError({ + reason: `Could not inspect pending update database backups under ${backupRoot}.`, + }), + ), + ); + const preserved = entries.filter((entry) => !entry.endsWith(".staging")); + if (preserved.length === 0) return; + const destinationRoot = path.join(input.userdataDir, "db-backup"); + yield* fs.makeDirectory(destinationRoot, { recursive: true }).pipe( + Effect.mapError( + () => + new CliUninstallError({ + reason: `Could not preserve pending update database backups under ${destinationRoot}.`, + }), + ), + ); + for (const entry of preserved) { + const source = path.join(backupRoot, entry); + const destination = path.join(destinationRoot, entry); + yield* fs.remove(destination, { recursive: true, force: true }).pipe( + Effect.andThen(fs.rename(source, destination)), + Effect.mapError( + () => + new CliUninstallError({ + reason: `Could not preserve pending update database backup ${source} under ${destination}.`, + }), + ), + ); + yield* Console.log(`Preserved interrupted update database backup at ${destination}.`); + } + }, +); + +export const runUninstall = Effect.fn("cli.uninstall.run")(function* (input: { readonly baseDir: string; readonly assumeYes: boolean; }) { @@ -243,6 +290,11 @@ const runUninstall = Effect.fn("cli.uninstall.run")(function* (input: { if (plan.service) { yield* service.uninstall; yield* Console.log("Removed the background service."); + } else if (plan.canRecoverOrphanService) { + const stoppedOrphan = yield* service.uninstall; + if (stoppedOrphan) { + yield* Console.log("Removed the background service."); + } } for (const launcher of plan.launchers) { // The prompt can wait a long time; never delete a file that replaced the launcher meanwhile. @@ -260,6 +312,10 @@ const runUninstall = Effect.fn("cli.uninstall.run")(function* (input: { yield* Console.log(`Removed ${launcher}.`); } if (plan.runtimeDir !== undefined) { + yield* preserveUnrecoveredDatabaseBackups({ + runtimeDir: plan.runtimeDir, + userdataDir: plan.userdataDir, + }); // Windows refuses to unlink this running executable; a detached shell removes it after exit. if (platform === "win32") { const runtimeDir = plan.runtimeDir; diff --git a/apps/server/src/cli/update.ts b/apps/server/src/cli/update.ts index 99bf380f1c0dd..b7117b85f8b06 100644 --- a/apps/server/src/cli/update.ts +++ b/apps/server/src/cli/update.ts @@ -420,7 +420,7 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { // Service names are per user, so another home's unit belongs to a different install. const servesThisHome = status.installedBaseDir !== undefined && - path.resolve(status.installedBaseDir) === path.resolve(input.baseDir); + BootService.sameBootServiceBaseDir(path, platform, status.installedBaseDir, input.baseDir); const serviceInstalled = status.supported && status.installed && servesThisHome; const foreground = yield* findForegroundServer({ serverRuntimeStatePath: input.serverRuntimeStatePath, @@ -596,7 +596,7 @@ export const runUpdate = Effect.fn("cli.update.run")(function* (input: { } if (servicePlan?.handedOff === true) { yield* Console.log( - ` Background service switching to ${targetVersion} from a separate systemd unit, since this session runs inside it and disconnects when it restarts. Result: ${servicePlan.logPath}`, + ` Background service switching to ${targetVersion} outside this session, since this session runs inside it and disconnects when it restarts. Result: ${servicePlan.logPath}`, ); } else if (servicePlan !== undefined && restartService) { yield* Console.log(` Background service restarted on ${targetVersion}`); diff --git a/apps/server/src/cliCompileCache.test.ts b/apps/server/src/cliCompileCache.test.ts index b58e7969999d3..e43cbd6781f4d 100644 --- a/apps/server/src/cliCompileCache.test.ts +++ b/apps/server/src/cliCompileCache.test.ts @@ -65,6 +65,16 @@ it.live("loads the CLI with its cache settings only when the command needs it", expect(launcher.exitCode).toBe(1); expect(launcher.stderr).toContain("[service-launcher] T3CODE_HOME is required"); expect(launcher.stderr).not.toMatch(/LOADED_CLI_MAIN|CACHE=/); + const windowsLauncher = yield* run({}, [ + "__service-launcher", + "--base-dir", + "", + "--service-unit", + "t3code.service", + ]); + expect(windowsLauncher.exitCode).toBe(1); + expect(windowsLauncher.stderr).toContain("[service-launcher] T3CODE_HOME is required"); + expect(windowsLauncher.stderr).not.toMatch(/LOADED_CLI_MAIN|CACHE=/); const launcherHelp = yield* run({ T3CODE_HOME: "" }, ["__service-launcher", "--help"]); expect(launcherHelp.exitCode).toBe(0); expect(launcherHelp.stderr).toContain("LOADED_CLI_MAIN"); diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 841aa78f6c879..8240d5f779b8c 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1,21 +1,35 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off -- Cross-process launcher contention uses real filesystem leases and IPC barriers. import * as NodeChildProcess from "node:child_process"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { expect, it } from "@effect/vitest"; +import { describe, expect, it } from "@effect/vitest"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import type * as Scope from "effect/Scope"; import { CliError } from "effect/cli"; import { HttpClient } from "effect/http"; import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; import { vi } from "vite-plus/test"; import * as ProcessRunner from "../processRunner.ts"; +import { + acquireServiceAccountLock, + type ControlAclHooks, + Launcher, + probeServiceLauncher, + readServiceState, + serviceAccountControlRoot, + serviceControlDir, + serviceTaskName, + serviceTaskNamePath, + windowsSystem32Binary, + writeServiceState, +} from "../serviceLauncher.ts"; import * as BootService from "./bootService.ts"; import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; import { @@ -35,7 +49,10 @@ vi.mock("node:fs", async (importOriginal) => { try { return actual.renameSync(...args); } catch (error) { - if (["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) { + const code = (error as NodeJS.ErrnoException).code ?? ""; + const existingTargetOnWindows = + (code === "EPERM" || code === "EACCES") && actual.existsSync(args[1]); + if (["EEXIST", "ENOTEMPTY"].includes(code) || existingTargetOnWindows) { leaseObserver.contended(String(args[1])); } throw error; @@ -44,7 +61,18 @@ vi.mock("node:fs", async (importOriginal) => { }; }); -it("reads the served T3 home back out of a rendered unit or plist", () => { +const trustedTestAcl: ControlAclHooks = { + restrictPathAcl: async () => undefined, + verifyPathAcl: async () => true, +}; + +const windowsTaskOptions = { + userId: "PC\\theo", + homeDir: "C:\\Users\\theo", + conhostPath: "C:\\Windows\\System32\\conhost.exe", +}; + +it("reads the served T3 home back out of a rendered unit, plist, or task", () => { const plan = (baseDir: string) => ({ program: [`${baseDir}/runtime/versions/1.2.3/t3`, "__service-launcher"], baseDir, @@ -68,9 +96,52 @@ it("reads the served T3 home back out of a rendered unit or plist", () => { }), ), ).toBe("/Users/theo/a&b"); + expect( + BootService.bootServiceBaseDirOf( + BootService.renderBootServiceTask(plan('C:\\Users\\theo\\T3 "a&b"\\'), windowsTaskOptions), + ), + ).toBe('C:\\Users\\theo\\T3 "a&b"\\'); expect(BootService.bootServiceBaseDirOf("[Service]\nExecStart=/x\n")).toBeUndefined(); }); +it("parses Task Scheduler /Query /XML exports with XML entities, multiline whitespace, and unquoted arguments", () => { + const exportedXml = [ + '', + '', + " ", + " ", + " C:\\Windows\\System32\\conhost.exe", + " ", + " --headless "C:\\Users\\theo\\T3 'Code' & Tools\\runtime\\versions\\1.2.3\\t3.exe"", + " __service-launcher --base-dir", + " "C:\\Users\\theo\\T3 & Home\\\\" --log-file C:\\Users\\theo\\boot.log", + " ", + " ", + " ", + "", + ].join("\r\n"); + + expect(BootService.bootServiceProgramOf(exportedXml)).toBe( + "C:\\Users\\theo\\T3 'Code' & Tools\\runtime\\versions\\1.2.3\\t3.exe", + ); + expect(BootService.bootServiceBaseDirOf(exportedXml)).toBe("C:\\Users\\theo\\T3 & Home\\"); + + const unquotedXml = [ + "", + " ", + " ", + " C:\\Windows\\System32\\conhost.exe", + " --headless C:\\t3\\runtime\\versions\\1.2.3\\t3.exe __service-launcher --base-dir=C:\\Users\\theo\\.t3", + " ", + " ", + "", + ].join("\n"); + expect(BootService.bootServiceProgramOf(unquotedXml)).toBe( + "C:\\t3\\runtime\\versions\\1.2.3\\t3.exe", + ); + expect(BootService.bootServiceBaseDirOf(unquotedXml)).toBe("C:\\Users\\theo\\.t3"); +}); + it("protects the server unit from agents under CPU and memory pressure", () => { const unit = BootService.renderBootServiceUnit({ program: ["/home/theo/.t3/runtime/versions/1.2.3/t3", "__service-launcher"], @@ -98,6 +169,12 @@ it.each([ (plan: BootService.BootServicePlan) => BootService.renderBootServicePlist(plan, { homeDir: "/Users/theo", environmentPath: "" }), ], + [ + "a task", + 'C:\\Users\\theo\\T3 "a&b" \\', + (plan: BootService.BootServicePlan) => + BootService.renderBootServiceTask(plan, windowsTaskOptions), + ], ] as const)("reads the launcher back out of %s", (_, baseDir, render) => { const program = `${baseDir}/runtime/versions/1.2.3/t3`; expect( @@ -119,6 +196,10 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-test-" }); + const localAppData = path.join(home, "AppData", "Local"); + const taskName = serviceTaskName("PC", "theo"); + const controlDirFor = (targetBaseDir: string) => + serviceControlDir(localAppData, targetBaseDir, { caseInsensitive: platform === "win32" }); const managerUnitDir = path.join( managerXdgConfigHome !== undefined && path.isAbsolute(managerXdgConfigHome) ? managerXdgConfigHome @@ -133,20 +214,31 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( yield* fs.writeFileString(runtime.entryPath, "#!/bin/sh\n"); yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); + const schtasksExe = windowsSystem32Binary("schtasks.exe", { SystemRoot: "C:\\Windows" }); + const activeLaunchers = new Set(); const commands: string[] = []; const invocations: ProcessRunner.ProcessRunInput[] = []; const timeouts = new Map(); const control: { failCommand: string | undefined; - stateAfterStop?: string; + failAllSchtasks?: boolean | undefined; + failTaskQuery?: boolean | undefined; + queryHresultOverride?: number | undefined; + windowsLauncherFlags?: boolean | undefined; + stopLaunchersOnTaskEnd?: boolean | undefined; + stateAfterStop?: string | undefined; + registeredTaskXml?: string | undefined; + onBeforeWriteUnit?: (() => Effect.Effect) | undefined; linger: string; enabled: boolean; active: boolean; + taskRegistered: boolean; } = { failCommand: undefined, linger: "yes", enabled: true, active: true, + taskRegistered: false, }; const runner = ProcessRunner.ProcessRunner.of({ run: Effect.fn("test.run_boot_service_command")(function* ( @@ -156,20 +248,104 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( commands.push(command); invocations.push(input); timeouts.set(command, input.timeout); - const failed = command === control.failCommand; + const isSchtasks = + input.command === "schtasks" || input.command.toLowerCase().endsWith("\\schtasks.exe"); + const isTaskXmlQuery = + input.command.toLowerCase().endsWith("\\powershell.exe") && + input.args.some((arg) => arg.includes("Schedule.Service")); + const failed = + command === control.failCommand || + (control.failAllSchtasks === true && (isSchtasks || isTaskXmlQuery)) || + (control.failTaskQuery === true && isTaskXmlQuery); if (!failed && command === "loginctl enable-linger --no-ask-password 501") control.linger = "yes"; - if (!failed && command === "systemctl --user enable t3code.service") control.enabled = true; + if (!failed && isSchtasks && input.args[0] === "/Create") { + const xmlIndex = input.args.indexOf("/XML"); + const xmlPath = xmlIndex >= 0 ? input.args[xmlIndex + 1] : undefined; + const xml = + xmlPath === undefined + ? "" + : yield* fs.readFileString(xmlPath).pipe(Effect.orElseSucceed(() => "")); + control.taskRegistered = true; + control.enabled = !/\s*false\s*<\/Enabled>/i.test(xml); + control.registeredTaskXml = xml; + } + if (!failed && isSchtasks && input.args[0] === "/Delete") { + control.taskRegistered = false; + control.registeredTaskXml = undefined; + } + const setTaskXmlEnabled = (xml: string | undefined, enabled: boolean) => { + if (xml === undefined) return undefined; + const tag = `${enabled}`; + return /\s*(?:true|false)\s*<\/Enabled>/i.test(xml) + ? xml.replace(/\s*(?:true|false)\s*<\/Enabled>/i, tag) + : xml.replace(/<\/Settings>/i, ` ${tag}\n `); + }; + if ( + !failed && + (command === "systemctl --user enable t3code.service" || + (isSchtasks && input.args.includes("/ENABLE"))) + ) { + control.enabled = true; + control.registeredTaskXml = setTaskXmlEnabled(control.registeredTaskXml, true); + } + if (!failed && isSchtasks && input.args.includes("/DISABLE")) { + control.enabled = false; + control.registeredTaskXml = setTaskXmlEnabled(control.registeredTaskXml, false); + } + if ( + !failed && + control.stopLaunchersOnTaskEnd === true && + isSchtasks && + input.args[0] === "/End" + ) { + yield* Effect.promise(() => + Promise.all(Array.from(activeLaunchers, (launcher) => launcher.stop("SIGTERM"))), + ); + } if (!failed && command === "systemctl --user restart t3code.service") control.active = true; // A timed-out stop may still stop the service. if (command === "systemctl --user stop t3code.service") control.active = false; if ( control.stateAfterStop !== undefined && (command === "systemctl --user stop t3code.service" || - command.startsWith("launchctl bootout --wait ")) + command.startsWith("launchctl bootout --wait ") || + (isSchtasks && input.args[0] === "/End")) ) { yield* fs.writeFileString(statePath, control.stateAfterStop).pipe(Effect.orDie); } + if ( + control.onBeforeWriteUnit !== undefined && + (command === "systemctl --user stop t3code.service" || + (isSchtasks && input.args[0] === "/End")) + ) { + yield* control.onBeforeWriteUnit(); + } + const exitCode = isTaskXmlQuery + ? failed + ? 1 + : !control.taskRegistered + ? 2 + : 0 + : isSchtasks && + input.args[0] === "/Query" && + input.args.includes("/HRESULT") && + control.queryHresultOverride !== undefined + ? control.queryHresultOverride + : failed + ? 1 + : isSchtasks && + (input.args[0] === "/Delete" || input.args[0] === "/Change") && + !control.taskRegistered + ? 1 + : isSchtasks && input.args[0] === "/Query" && !control.taskRegistered + ? input.args.includes("/HRESULT") + ? 0x80070002 + : 1 + : (input.args[1] === "is-active" && !control.active) || + (input.args[1] === "is-enabled" && !control.enabled) + ? 1 + : 0; return { stdout: input.command === "busctl" @@ -183,23 +359,24 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( "/etc/systemd/user", ], }) - : input.args[0] === "--version" - ? `t3 v${/versions\/([^/]+)\//.exec(input.command)?.[1] ?? "1.2.3"}\n` - : input.command === "loginctl" && input.args[0] === "show-user" - ? `${control.linger}\n` - : input.args[1] === "is-enabled" - ? control.enabled - ? "enabled\n" - : "disabled\n" - : "", + : isTaskXmlQuery && !failed && control.taskRegistered + ? (control.registeredTaskXml ?? + `${control.enabled}\n`) + : input.args[0] === "__service-launcher" && input.args[1] === "--help" + ? control.windowsLauncherFlags === false + ? "t3 __service-launcher\n" + : "--base-dir --log-file --service-unit\n" + : input.args[0] === "--version" + ? `t3 v${/versions[\\/]([^\\/]+)[\\/]/.exec(input.command)?.[1] ?? "1.2.3"}\n` + : input.command === "loginctl" && input.args[0] === "show-user" + ? `${control.linger}\n` + : input.args[1] === "is-enabled" + ? control.enabled + ? "enabled\n" + : "disabled\n" + : "", stderr: "", - code: ChildProcessSpawner.ExitCode( - failed || - (input.args[1] === "is-active" && !control.active) || - (input.args[1] === "is-enabled" && !control.enabled) - ? 1 - : 0, - ), + code: ChildProcessSpawner.ExitCode(exitCode), timedOut: false, stdoutTruncated: false, stderrTruncated: false, @@ -214,6 +391,9 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( serviceBaseDir = baseDir, insideService = false, hostExecPath = "/usr/bin/t3", + fallbackReleaseTimeoutMs?: number, + envOverrides: Readonly> = {}, + aclOverride: ControlAclHooks = trustedTestAcl, ) => Effect.gen(function* () { const paths = pinnedRuntimePaths(path, serviceBaseDir, cliVersion, platform); @@ -224,7 +404,11 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( baseDir: serviceBaseDir, logsDir: path.join(serviceBaseDir, "userdata", "logs"), cliVersion, - host: { execPath: hostExecPath }, + host: { + execPath: hostExecPath, + acl: aclOverride, + ...(fallbackReleaseTimeoutMs === undefined ? {} : { fallbackReleaseTimeoutMs }), + }, }); }).pipe( Effect.provideService(ProcessRunner.ProcessRunner, runner), @@ -242,20 +426,98 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( env: { HOME: home, ...(xdgConfigHome === undefined ? {} : { XDG_CONFIG_HOME: xdgConfigHome }), + USERPROFILE: home, + LOCALAPPDATA: localAppData, + USERDOMAIN: "PC", + USERNAME: "theo", ...(environmentPath === undefined || environmentPath === "" ? {} : { PATH: environmentPath }), - ...(insideService ? { T3_BOOT_SERVICE_UNIT: "t3code.service" } : {}), + ...(insideService + ? { + T3_BOOT_SERVICE_UNIT: + platform === "win32" + ? BootService.BOOT_SERVICE_TASK_FILE + : platform === "darwin" + ? BootService.BOOT_SERVICE_PLIST_FILE + : BootService.BOOT_SERVICE_UNIT_FILE, + } + : {}), + ...envOverrides, }, }), ), ), ), ); + const spawnLauncher = (version: string, acceptUpdate = false) => + Effect.gen(function* () { + const dbPath = path.join(baseDir, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); + yield* fs.writeFileString(dbPath, "db"); + const childScript = [ + 'process.on("message", (m) => { if (m.type === "stop" || m.type === "update-accepted") process.exit(0); });', + `process.send({ type: "request-update", targetVersion: "9.9.9", dbPath: ${JSON.stringify(dbPath)} });`, + "setInterval(() => {}, 10000);", + ].join("\n"); + for (const candidate of [version, "9.9.9"]) { + const hostBin = pinnedRuntimePaths( + path, + baseDir, + candidate, + path.sep === "\\" ? "win32" : "linux", + ); + yield* fs.makeDirectory(hostBin.versionDir, { recursive: true }); + yield* fs.writeFileString(hostBin.entryPath, childScript, { mode: 0o755 }); + yield* fs.writeFileString(hostBin.sentinelPath, `${candidate}\n`); + } + const ready = Promise.withResolvers(); + const state = yield* Effect.promise(() => readServiceState(statePath)); + const controlDir = controlDirFor(baseDir); + const launcher = new Launcher(baseDir, state, { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + spawnCommand: (entryPath) => { + if (acceptUpdate && entryPath.includes("9.9.9")) { + ready.resolve(); + } + return { command: process.execPath, args: [entryPath] }; + }, + backupSpaceProbe: { + readFreeBytes: () => { + if (!acceptUpdate) { + ready.resolve(); + } + return Promise.resolve(acceptUpdate ? Number.MAX_SAFE_INTEGER : 0); + }, + canClone: () => Promise.resolve(false), + }, + }); + activeLaunchers.add(launcher); + const running = launcher.run().finally(() => { + activeLaunchers.delete(launcher); + }); + yield* Effect.addFinalizer(() => + Effect.promise(() => + launcher + .stop("SIGTERM") + .then(() => running) + .catch(() => undefined), + ), + ); + yield* Effect.promise(() => Promise.race([ready.promise, running])); + return { launcher, running, controlDir }; + }); const service = yield* makeService(); return { service, makeService, + spawnLauncher, + controlDirFor, + localAppData, + taskName, + schtasksExe, fs, home, baseDir, @@ -994,6 +1256,12 @@ process.disconnect(); "systemctl --user daemon-reload", "systemctl --user restart t3code.service", ]); + + // `restart` runs `stop` before registering recovery so a failed stop does not immediately restart. + commands.length = 0; + control.failCommand = "systemctl --user stop t3code.service"; + expect((yield* service.restart.pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(commands).not.toContain("systemctl --user restart t3code.service"); }), ); @@ -1066,6 +1334,378 @@ process.disconnect(); }), ); + it.effect( + "installs, defers disabled installs, refuses post-stop downgrades, and uninstalls on Windows", + () => + Effect.gen(function* () { + const { + service, + makeService, + taskName, + schtasksExe, + control, + fs, + statePath, + commands, + runtime, + } = yield* makeHarness("win32"); + const path = yield* Path.Path; + const plan = yield* service.install(); + const task = yield* fs.readFileString(plan.unitPath); + const nonQuerySchtasks = () => + commands.filter( + (command) => + command.startsWith(`${schtasksExe} `) && + !command.startsWith(`${schtasksExe} /Query `), + ); + + expect(plan.unitPath).toBe( + path.join(path.dirname(plan.baseDir), "AppData", "Local", "t3code", "t3code-task.xml"), + ); + expect(BootService.bootServiceProgramOf(task)).toBe(runtime.entryPath); + expect(BootService.bootServiceBaseDirOf(task)).toBe(plan.baseDir); + expect(task).toContain("PC\\theo"); + expect(task).toContain("IgnoreNew"); + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }); + expect(nonQuerySchtasks()).toEqual([ + `${schtasksExe} /Create /XML ${plan.unitPath} /TN ${taskName} /F`, + `${schtasksExe} /Run /TN ${taskName}`, + ]); + expect(yield* service.status).toMatchObject({ + installed: true, + current: false, + problems: ["service-stopped"], + }); + + // Deferred install preserves an enabled task's state, and registers `false` + // directly in one `schtasks /Create` call when the task is disabled so the minute trigger never fires. + const newer = yield* makeService(undefined, "1.2.4"); + commands.length = 0; + yield* newer.install({ start: false }); + expect(control.enabled).toBe(true); + control.enabled = false; + control.registeredTaskXml = control.registeredTaskXml?.replace( + /<\/Settings>/i, + " false\n ", + ); + const latest = yield* makeService(undefined, "1.2.5"); + commands.length = 0; + yield* latest.install({ start: false }); + const deferredSchtasks = nonQuerySchtasks(); + expect(deferredSchtasks).toHaveLength(1); + expect(deferredSchtasks[0]).toMatch( + new RegExp( + `^${schtasksExe.replaceAll("\\", "\\\\")} /Create /XML .+ /TN ${taskName} /F$`, + ), + ); + expect(control.enabled).toBe(false); + expect(BootService.bootServiceProgramOf(yield* fs.readFileString(plan.unitPath))).toBe( + pinnedRuntimePaths(path, plan.baseDir, "1.2.5", "win32").entryPath, + ); + + // Failed /Run on a disabled+stopped task restores `false`. + control.failCommand = `${schtasksExe} /Run /TN ${taskName}`; + expect((yield* latest.install().pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(control.enabled).toBe(false); + control.failCommand = undefined; + + // Ordinary install checks state under the control lock after stop so a newer staged version wins. + control.stateAfterStop = `{"protocol":${SERVICE_LAUNCHER_PROTOCOL},"activeVersion":"1.2.9"}`; + expect(yield* service.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceDowngradeRefusedError", + installedVersion: "1.2.9", + targetVersion: "1.2.3", + }); + expect(control.enabled).toBe(false); + control.stateAfterStop = undefined; + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }), + ); + + // Uninstall preserves the XML mirror when the task stays registered or Task Scheduler fails, + // and removes it when `/Query /HRESULT` confirms the task is absent (0x80070002). + control.failCommand = `${schtasksExe} /Delete /TN ${taskName} /F`; + expect((yield* service.uninstall.pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(yield* fs.exists(plan.unitPath)).toBe(true); + control.failCommand = undefined; + + control.taskRegistered = false; + control.failAllSchtasks = true; + expect((yield* service.uninstall.pipe(Effect.flip))._tag).toBe("BootServiceCommandError"); + expect(yield* fs.exists(plan.unitPath)).toBe(true); + control.failAllSchtasks = false; + + commands.length = 0; + expect(yield* service.uninstall).toBe(true); + expect(commands.filter((command) => command.startsWith(`${schtasksExe} `))).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /Query /TN ${taskName} /HRESULT`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Delete /TN ${taskName} /F`, + `${schtasksExe} /Query /TN ${taskName} /HRESULT`, + ]); + expect((yield* service.status).installed).toBe(false); + }), + ); + + it.effect( + "aborts Windows mutations when disabling or querying fails, protects stale mirrors, and round-trips non-ASCII homes", + () => + Effect.gen(function* () { + const { service, makeService, taskName, schtasksExe, control, fs, home, commands } = + yield* makeHarness("win32"); + const path = yield* Path.Path; + const plan = yield* service.install(); + const nonQuerySchtasks = () => + commands.filter( + (command) => + command.startsWith(`${schtasksExe} `) && + !command.startsWith(`${schtasksExe} /Query `), + ); + + // A real `/DISABLE` failure on an existing task aborts before stopping or running recovery. + control.failCommand = `${schtasksExe} /Change /TN ${taskName} /DISABLE`; + commands.length = 0; + expect(yield* service.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "disabling the scheduled task", + }); + expect(commands.filter((command) => command.startsWith(`${schtasksExe} `))).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /Query /TN ${taskName} /HRESULT`, + ]); + + commands.length = 0; + expect(yield* service.restart.pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "disabling the scheduled task", + }); + expect(commands.filter((command) => command.startsWith(`${schtasksExe} `))).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /Query /TN ${taskName} /HRESULT`, + ]); + control.failCommand = undefined; + + // If `Schedule.Service` fails, `install`, `restart`, `withStopped`, and `uninstall` fail + // closed instead of falling back to a stale mirror or authorizing a blind overwrite. + const foreignHome = path.join(home, ".t3-stale-mirror"); + yield* fs.writeFileString( + plan.unitPath, + BootService.renderBootServiceTask( + { + program: [`${foreignHome}/runtime/versions/1.2.3/t3.exe`, "__service-launcher"], + baseDir: foreignHome, + logPath: `${foreignHome}/userdata/logs/boot-service.log`, + unitPath: plan.unitPath, + }, + windowsTaskOptions, + ), + ); + // With Task Scheduler healthy, live `registeredTaskXml` (`plan.baseDir`) wins over the + // stale foreign local mirror, and `restart` activates the registered task directly rather + // than overwriting Task Scheduler from the stale mirror via `/Create /XML`. + commands.length = 0; + expect(yield* service.restart).toBe("restarted"); + expect(nonQuerySchtasks()).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Change /TN ${taskName} /ENABLE`, + `${schtasksExe} /Run /TN ${taskName}`, + ]); + + control.failTaskQuery = true; + expect(yield* service.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "inspecting the scheduled task", + }); + expect(yield* service.restart.pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "inspecting the scheduled task", + }); + expect(yield* service.withStopped(Effect.void).pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "inspecting the scheduled task", + }); + expect(yield* service.uninstall.pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "inspecting the scheduled task", + }); + control.failTaskQuery = false; + + // Even when the local XML mirror is deleted, `Schedule.Service` preserves cross-home + // protection, mirrorless `status` fails closed when the query fails, and mirrorless + // `restart` re-enables and starts the registered task directly. + yield* fs.remove(plan.unitPath); + control.failTaskQuery = true; + expect(yield* service.status.pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "inspecting the scheduled task", + }); + control.failTaskQuery = false; + expect((yield* service.status).installed).toBe(true); + commands.length = 0; + expect(yield* service.restart).toBe("restarted"); + expect(nonQuerySchtasks()).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Change /TN ${taskName} /ENABLE`, + `${schtasksExe} /Run /TN ${taskName}`, + ]); + + const insideMirrorless = yield* makeService(undefined, "1.2.3", undefined, true); + commands.length = 0; + expect(yield* insideMirrorless.restart).toBe("queued"); + expect(nonQuerySchtasks()).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /ENABLE`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Run /TN ${taskName}`, + ]); + expect(yield* insideMirrorless.withStopped(Effect.void).pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceRunningInsideServiceError", + }); + + const other = yield* makeService(undefined, "1.2.3", path.join(home, ".t3-other")); + expect((yield* other.status).otherBaseDir).toBe(plan.baseDir); + expect((yield* other.install().pipe(Effect.flip))._tag).toBe( + "BootServiceOtherBaseDirError", + ); + expect((yield* other.uninstall.pipe(Effect.flip))._tag).toBe( + "BootServiceOtherBaseDirError", + ); + + // Non-ASCII Windows T3 home paths survive `Schedule.Service` UTF-8 task XML queries intact. + expect(yield* service.uninstall).toBe(true); + const unicodeHome = path.join(home, "José-Łukasz-用户", ".t3"); + const unicodeService = yield* makeService(undefined, "1.2.3", unicodeHome); + yield* unicodeService.install(); + expect(yield* unicodeService.status).toMatchObject({ + installed: true, + installedBaseDir: unicodeHome, + }); + expect((yield* unicodeService.status).otherBaseDir).toBeUndefined(); + yield* unicodeService.install({ allowDowngrade: true }); + expect(yield* unicodeService.uninstall).toBe(true); + }), + ); + + it.effect( + "persists the Windows task name across PC renames, fails closed on unrecognized task XML, and treats transient query errors as unhealthy", + () => + Effect.gen(function* () { + const { + service, + makeService, + localAppData, + taskName, + schtasksExe, + control, + fs, + home, + commands, + } = yield* makeHarness("win32"); + const path = yield* Path.Path; + const persistedTaskPath = serviceTaskNamePath(localAppData); + const plan = yield* service.install(); + expect((yield* fs.readFileString(persistedTaskPath)).trim()).toBe(taskName); + + // Transient query failure when the local mirror exists reports `service-unhealthy` instead + // of misclassifying an enabled task as `service-disabled`. + control.failTaskQuery = true; + expect((yield* service.status).problems).toContain("service-unhealthy"); + expect((yield* service.status).problems).not.toContain("service-disabled"); + control.failTaskQuery = false; + + // Live registered `` wins over a stale local XML mirror, and an `` block + // without `--base-dir` fails closed instead of silently overwriting a foreign task. + const otherHome = path.join(home, ".t3-live"); + const otherService = yield* makeService(undefined, "1.2.3", otherHome); + control.registeredTaskXml = BootService.renderBootServiceTask( + { + program: [`${otherHome}/runtime/versions/1.2.3/t3.exe`, "__service-launcher"], + baseDir: otherHome, + logPath: `${otherHome}/userdata/logs/boot-service.log`, + unitPath: plan.unitPath, + }, + windowsTaskOptions, + ); + expect((yield* service.status).otherBaseDir).toBe(otherHome); + expect((yield* service.install().pipe(Effect.flip))._tag).toBe( + "BootServiceOtherBaseDirError", + ); + + control.registeredTaskXml = [ + "", + " true", + " C:\\custom\\daemon.exe--serve", + "", + ].join("\n"); + const unrecognized = yield* otherService.install().pipe(Effect.flip); + expect(unrecognized).toMatchObject({ + _tag: "BootServiceOtherBaseDirError", + unrecognizedTaskName: taskName, + }); + expect(unrecognized.message).toContain( + `Windows scheduled task '${taskName}' has an unrecognized configuration`, + ); + expect((yield* otherService.status).problems).toContain("service-unhealthy"); + expect((yield* otherService.status).installedBaseDir).toBeUndefined(); + + // Restore the task XML for `plan.baseDir` and simulate a workgroup PC rename (`USERDOMAIN`). + control.registeredTaskXml = yield* fs.readFileString(plan.unitPath); + const afterRename = yield* makeService( + undefined, + "1.2.3", + plan.baseDir, + false, + "/usr/bin/t3", + undefined, + { USERDOMAIN: "RENAMED-PC" }, + ); + expect((yield* afterRename.status).installed).toBe(true); + commands.length = 0; + expect(yield* afterRename.uninstall).toBe(true); + expect(commands).toContain(`${schtasksExe} /Delete /TN ${taskName} /F`); + expect(yield* fs.exists(persistedTaskPath)).toBe(false); + }), + ); + + it.effect( + "restores an installed service when install is interrupted after stopping and removes unreadable POSIX units on uninstall", + () => + Effect.gen(function* () { + const { service, fs, commands, control } = yield* makeHarness(); + const plan = yield* service.install(); + + // Fiber interruption after stopping the service still triggers `Effect.onExit` restoration. + const stoppedGate = Promise.withResolvers(); + const holdAfterStop = Promise.withResolvers(); + control.onBeforeWriteUnit = () => + Effect.promise(async () => { + stoppedGate.resolve(); + await holdAfterStop.promise; + }); + commands.length = 0; + const installFiber = yield* Effect.forkChild(service.install()); + yield* Effect.promise(() => stoppedGate.promise); + expect(control.active).toBe(false); + yield* Fiber.interrupt(installFiber); + control.onBeforeWriteUnit = undefined; + expect(control.active).toBe(true); + expect(commands).toContain("systemctl --user restart t3code.service"); + + // An unreadable (`0o000`) unit file still uninstalls cleanly when the directory is writable. + yield* fs.chmod(plan.unitPath, 0o000); + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(plan.unitPath)).toBe(false); + }), + ); + // regression: #8173 keep provider CLIs available in the macOS service it.effect("reconstructs a launch agent search path when the installer has no PATH", () => Effect.gen(function* () { @@ -1104,7 +1744,7 @@ process.disconnect(); // regression: #17532 it.effect("puts the runtime an update installs on the launch agent PATH", () => Effect.gen(function* () { - const { makeService, fs, baseDir, runtime } = yield* makeHarness("darwin"); + const { makeService, baseDir, runtime } = yield* makeHarness("darwin"); const path = yield* Path.Path; const previous = runtime.versionDir; const installed = pinnedRuntimePaths(path, baseDir, "1.2.4", "darwin").versionDir; @@ -1118,7 +1758,7 @@ process.disconnect(); ); const plan = yield* updated.install(); - expect(yield* fs.readFileString(plan.unitPath)).toContain( + expect(yield* FileSystem.FileSystem.use((fs) => fs.readFileString(plan.unitPath))).toContain( ` PATH\n /usr/bin:${installed}:/bin:/opt/homebrew/bin:/usr/local/bin:/usr/sbin:/sbin`, ); }), @@ -1178,3 +1818,334 @@ process.disconnect(); }), ); }); + +const runLiveNode = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("boot service Windows control", () => { + it.live( + "reports unhealthy control locks, falls back to ending the task on stop, and fails closed if wedged", + () => + runLiveNode( + Effect.gen(function* () { + const { + service, + makeService, + spawnLauncher, + taskName, + schtasksExe, + control, + fs, + commands, + } = yield* makeHarness("win32"); + const path = yield* Path.Path; + const plan = yield* service.install(); + const live = yield* spawnLauncher("1.2.3", false); + + expect(yield* service.status).toMatchObject({ + installed: true, + current: true, + problems: [], + }); + + // Incompatible older runtimes lacking Windows launcher flags fail before stopping the service. + control.windowsLauncherFlags = false; + expect(yield* service.install({ allowDowngrade: true }).pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "verifying Windows service launcher support", + }); + expect(yield* Effect.promise(() => probeServiceLauncher(live.controlDir))).toBe( + "running", + ); + control.windowsLauncherFlags = true; + + // An occupied lock whose token cannot authenticate reports `service-unhealthy`, not healthy. + control.enabled = false; + if (control.registeredTaskXml !== undefined) { + control.registeredTaskXml = /\s*(?:true|false)\s*<\/Enabled>/i.test( + control.registeredTaskXml, + ) + ? control.registeredTaskXml.replace( + /\s*(?:true|false)\s*<\/Enabled>/i, + "false", + ) + : control.registeredTaskXml.replace( + /<\/Settings>/i, + " false\n ", + ); + } + const tokenPath = path.join(live.controlDir, "control.token"); + yield* fs.writeFileString(tokenPath, "corrupt\n"); + expect(yield* service.status).toMatchObject({ + current: false, + problems: ["service-disabled", "service-unhealthy"], + }); + + // If `schtasks /End` does not release a wedged lock holder, stop fails closed. + const differentlyCasedHome = path.join(path.dirname(plan.baseDir), ".T3"); + const fastFallback = yield* makeService( + undefined, + "1.2.3", + differentlyCasedHome, + false, + "/usr/bin/t3", + 100, + ); + expect((yield* fastFallback.status).otherBaseDir).toBeUndefined(); + let ranWhileWedged = false; + expect( + yield* fastFallback + .withStopped( + Effect.sync(() => { + ranWhileWedged = true; + }), + ) + .pipe(Effect.flip), + ).toMatchObject({ + _tag: "BootServiceCommandError", + step: "stopping the installed service", + }); + expect(ranWhileWedged).toBe(false); + + // When `schtasks /End` closes the task session and releases the lock, `withStopped` + // proceeds and restores the disabled state afterwards. + control.stopLaunchersOnTaskEnd = true; + commands.length = 0; + yield* fastFallback.withStopped( + Effect.gen(function* () { + expect(yield* Effect.promise(() => probeServiceLauncher(live.controlDir))).toBe( + "stopped", + ); + }), + ); + yield* Effect.promise(() => live.running); + expect(control.enabled).toBe(false); + expect( + commands.filter( + (command) => + command.startsWith(`${schtasksExe} `) && + !command.startsWith(`${schtasksExe} /Query `), + ), + ).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Change /TN ${taskName} /ENABLE`, + `${schtasksExe} /Run /TN ${taskName}`, + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + ]); + }), + ), + ); + + it.live( + "serializes concurrent Windows installs, restarts, and withStopped on the account lock", + () => + runLiveNode( + Effect.gen(function* () { + const { makeService, localAppData, home, commands } = yield* makeHarness("win32"); + const path = yield* Path.Path; + const firstHome = path.join(home, ".t3-first"); + const first = yield* makeService(undefined, "1.2.3", firstHome); + const second = yield* makeService(undefined, "1.2.3", path.join(home, ".t3-second")); + const capture = (effect: Effect.Effect) => + effect.pipe( + Effect.map((value) => ({ ok: true as const, value })), + Effect.catch((error) => Effect.succeed({ ok: false as const, error })), + ); + + const outcomes = yield* Effect.all( + [capture(first.install()), capture(second.install())], + { + concurrency: "unbounded", + }, + ); + const successes = outcomes.filter((outcome) => outcome.ok); + const failures = outcomes.filter((outcome) => !outcome.ok); + expect(successes).toHaveLength(1); + expect(failures).toHaveLength(1); + expect(failures[0]?.error._tag).toBe("BootServiceOtherBaseDirError"); + + // Ensure `firstHome` is the installed home, then prove both `restart` and `withStopped` + // block at `account-task.sqlite` before issuing any `schtasks` command while the lock is held. + const installedHome = successes[0]?.value.baseDir ?? firstHome; + let attemptedAccountLock = Promise.withResolvers(); + const contender = yield* makeService( + undefined, + "1.2.3", + installedHome, + false, + "/usr/bin/t3", + undefined, + {}, + { + restrictPathAcl: async () => undefined, + verifyPathAcl: async (targetPath) => { + if (targetPath.endsWith("account-task.sqlite")) { + queueMicrotask(() => attemptedAccountLock.resolve()); + } + return true; + }, + }, + ); + + const accountRoot = serviceAccountControlRoot(localAppData); + const lockForRestart = yield* Effect.promise(() => + acquireServiceAccountLock(accountRoot, { acl: trustedTestAcl }), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => lockForRestart.close())); + commands.length = 0; + const restartFiber = yield* Effect.forkScoped(contender.restart); + yield* Effect.promise(() => attemptedAccountLock.promise); + expect(commands).toEqual([]); + lockForRestart.close(); + expect(yield* Fiber.join(restartFiber)).toBe("restarted"); + + attemptedAccountLock = Promise.withResolvers(); + const lockForWithStopped = yield* Effect.promise(() => + acquireServiceAccountLock(accountRoot, { acl: trustedTestAcl }), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => lockForWithStopped.close())); + commands.length = 0; + const withStoppedFiber = yield* Effect.forkScoped( + contender.withStopped(Effect.succeed("repair-finished")), + ); + yield* Effect.promise(() => attemptedAccountLock.promise); + expect(commands).toEqual([]); + lockForWithStopped.close(); + expect(yield* Fiber.join(withStoppedFiber)).toBe("repair-finished"); + }), + ), + ); + + it.live( + "hands in-service install, restart, and uninstall to Task Scheduler and the control channel", + () => + runLiveNode( + Effect.gen(function* () { + const { + service, + spawnLauncher, + taskName, + schtasksExe, + control, + fs, + statePath, + commands, + makeService, + } = yield* makeHarness("win32"); + const path = yield* Path.Path; + const plan = yield* service.install(); + const inside = yield* makeService(undefined, "1.2.4", undefined, true); + const nonQuerySchtasks = () => + commands.filter( + (command) => + command.startsWith(`${schtasksExe} `) && + !command.startsWith(`${schtasksExe} /Query `), + ); + const handoff = [ + `${schtasksExe} /Create /XML ${plan.unitPath} /TN ${taskName} /F`, + `${schtasksExe} /End /TN ${taskName}`, + `${schtasksExe} /Run /TN ${taskName}`, + ]; + + // A remote update accepted over IPC blocks an in-service install serialized over control. + const pending = yield* spawnLauncher("1.2.3", true); + expect(yield* inside.install().pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceUpdatePendingError", + }); + yield* Effect.promise(() => pending.launcher.stop("SIGTERM")); + yield* Effect.promise(() => pending.running); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + }), + ); + + const active = yield* spawnLauncher("1.2.3", false); + commands.length = 0; + expect(yield* inside.install()).toMatchObject({ handedOff: true, logPath: plan.logPath }); + yield* Effect.promise(() => active.running); + expect(nonQuerySchtasks()).toEqual(handoff); + expect(parseServiceState(yield* fs.readFileString(statePath))?.activeVersion).toBe( + "1.2.4", + ); + expect( + yield* fs.readFileString(`${plan.baseDir}/runtime/${SERVICE_RESTART_PENDING_FILE}`), + ).toBe("1.2.4\n"); + + const restarted = yield* spawnLauncher("1.2.4", false); + commands.length = 0; + expect(yield* inside.restart).toBe("queued"); + yield* Effect.promise(() => restarted.running); + expect(nonQuerySchtasks()).toEqual(handoff); + + // In-service uninstall fails closed BEFORE deleting the task or mirror if the control + // channel is unhealthy (for example, a corrupt token), then succeeds once healthy. + const uninstalled = yield* spawnLauncher("1.2.4", false); + const tokenPath = path.join(uninstalled.controlDir, "control.token"); + const validToken = yield* fs.readFileString(tokenPath); + yield* fs.writeFileString(tokenPath, "corrupt\n"); + commands.length = 0; + expect(yield* inside.uninstall.pipe(Effect.flip)).toMatchObject({ + _tag: "BootServiceCommandError", + step: "stopping the installed service", + }); + expect(control.taskRegistered).toBe(true); + expect(yield* fs.exists(plan.unitPath)).toBe(true); + expect(nonQuerySchtasks()).toEqual([]); + + yield* fs.writeFileString(tokenPath, validToken); + commands.length = 0; + expect(yield* inside.uninstall).toBe(true); + yield* Effect.promise(() => uninstalled.running); + expect(yield* fs.exists(plan.unitPath)).toBe(false); + expect((yield* service.status).installed).toBe(false); + expect(nonQuerySchtasks()).toEqual([ + `${schtasksExe} /Change /TN ${taskName} /DISABLE`, + `${schtasksExe} /Delete /TN ${taskName} /F`, + ]); + + // If a previous in-service uninstall already removed the scheduled task and mirror while + // the launcher remained alive with a pending trial, retrying `uninstall` finishes stopping + // the launcher and recovers the pre-trial database backup. + const dbPath = path.join(plan.baseDir, "userdata", "state.sqlite"); + const orphanedLauncher = yield* spawnLauncher("1.2.4", true); + expect((yield* Effect.promise(() => readServiceState(statePath))).update?.status).toBe( + "pending", + ); + yield* fs.writeFileString(dbPath, "corrupted-during-trial"); + expect(yield* inside.uninstall).toBe(true); + yield* Effect.promise(() => orphanedLauncher.running); + expect(yield* fs.readFileString(dbPath)).toBe("db"); + expect((yield* Effect.promise(() => readServiceState(statePath))).update?.status).toBe( + "failed", + ); + expect(yield* inside.uninstall).toBe(false); + + // Reinstalling while uninstalled with an unrecovered pending update fails closed and + // surfaces the PowerShell foreground recovery command for the installed version. + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.4", + update: { + id: "u-win", + fromVersion: "1.2.4", + targetVersion: "9.9.9", + dbPath, + status: "pending", + }, + }), + ); + const pendingError = yield* service.install().pipe(Effect.flip); + expect(pendingError).toMatchObject({ + _tag: "BootServiceUpdatePendingError", + }); + expect(pendingError.message).toContain( + `__service-launcher --base-dir '${plan.baseDir.replaceAll("'", "''")}'`, + ); + }), + ), + ); +}); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index e7de29ad8953d..6a206ca0202f8 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -17,6 +17,25 @@ import { CLI_RELEASE_BASE_URL_ENV } from "@t3tools/shared/cliRelease"; import * as ProcessRunner from "../processRunner.ts"; import { acquireServiceSelectionLock } from "./serviceSelectionLock.ts"; +import { + WINDOWS_QUERY_REGISTERED_TASK_SCRIPT, + acquireServiceAccountLock, + isValidServiceTaskName, + probeServiceLauncher, + queryServiceLauncherId, + recoverServiceLauncherUninstall, + requestServiceLauncherStop, + serviceAccountControlRoot, + serviceControlDir, + serviceTaskName, + serviceTaskNamePath, + signalServiceLauncherStop, + stageServiceLauncherInstall, + waitForServiceLauncherRelease, + windowsPowerShellBinary, + windowsSystem32Binary, + type ControlAclHooks, +} from "../serviceLauncher.ts"; import { ensurePinnedRuntimeInstalled, launcherOwnsVersionsDir, @@ -31,17 +50,16 @@ import { SERVICE_STATE_FILE, SERVICE_UNIT_ENV, compareExactServiceVersions, + hasWindowsTaskEnvironmentExpansion, serviceUnitBaseDirOf as bootServiceBaseDirOf, + serviceUnitProgramOf as bootServiceProgramOf, parseServiceState, serviceStateActiveVersion, serviceStateHasPendingUpdate, type ServiceState, } from "./serviceProtocol.ts"; -export { - serviceUnitBaseDirOf as bootServiceBaseDirOf, - serviceUnitProgramOf as bootServiceProgramOf, -} from "./serviceProtocol.ts"; +export { bootServiceBaseDirOf, bootServiceProgramOf }; const BOOT_SERVICE_NAME = "t3code"; export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; @@ -50,6 +68,13 @@ export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; export const BOOT_SERVICE_LAUNCHD_LABEL = "com.t3tools.t3code.service"; export const BOOT_SERVICE_PLIST_FILE = `${BOOT_SERVICE_LAUNCHD_LABEL}.plist`; export const BOOT_SERVICE_LOG_FILE = "boot-service.log"; +/** Task Scheduler keeps its own copy; this one lets status and repair read the task back. */ +export const BOOT_SERVICE_TASK_FILE = `${BOOT_SERVICE_NAME}-task.xml`; +const BOOT_SERVICE_UNIT_VALUES = new Set([ + BOOT_SERVICE_UNIT_FILE, + BOOT_SERVICE_PLIST_FILE, + BOOT_SERVICE_TASK_FILE, +]); /** systemd expands `%` specifiers, including in unquoted append-log paths. */ function escapeSystemdSpecifiers(value: string): string { @@ -63,6 +88,10 @@ function quoteSystemdValue(value: string): string { : escaped; } +/** Escape embedded quotes and double trailing backslashes per CommandLineToArgvW rules. */ +const quoteWindowsArgument = (value: string) => + `"${value.replaceAll(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/, "$1$1")}"`; + export interface BootServicePlan { readonly program: ReadonlyArray; readonly baseDir: string; @@ -168,22 +197,99 @@ export function renderBootServicePlist( ].join("\n"); } -export interface BootServiceStep { +/** + * Task Scheduler can neither set variables nor redirect output, so the launcher takes them as + * flags. A console program started by a task opens a window unless a headless console hosts it. + * No XML declaration: schtasks rejects `encoding="UTF-8"` but reads undeclared XML as UTF-8. + */ +export function renderBootServiceTask( + plan: BootServicePlan, + options: { + readonly userId: string; + readonly homeDir: string; + readonly conhostPath: string; + readonly enabled?: boolean; + }, +): string { + const launcherArguments = [ + ...plan.program, + "--base-dir", + plan.baseDir, + "--log-file", + plan.logPath, + "--service-unit", + BOOT_SERVICE_TASK_FILE, + ] + .map(quoteWindowsArgument) + .join(" "); + // conhost reports exit code 0 whatever the launcher returns, so restart-on-failure never + // fires. A repeating trigger with IgnoreNew starts the task again within a minute instead. + // Priority 5 is normal; the default 7 runs below it. + return [ + ``, + ` `, + ` T3 Code server`, + ` `, + ` `, + ` `, + ` ${escapeXmlText(options.userId)}`, + ` `, + ` `, + ` 2000-01-01T00:00:00`, + ` `, + ` PT1M`, + ` `, + ` `, + ` `, + ` `, + ` `, + ` ${escapeXmlText(options.userId)}`, + ` InteractiveToken`, + ` LeastPrivilege`, + ` `, + ` `, + ` `, + ` IgnoreNew`, + ` false`, + ` false`, + ...(options.enabled === false ? [` false`] : []), + ` PT0S`, + ` 5`, + ` `, + ` `, + ` `, + ` ${escapeXmlText(options.conhostPath)}`, + ` --headless ${escapeXmlText(launcherArguments)}`, + ` ${escapeXmlText(options.homeDir)}`, + ` `, + ` `, + ``, + ``, + ].join("\n"); +} + +export type BootServiceStep = { readonly step: string; - readonly command: string; - readonly args: ReadonlyArray; /** Non-zero exit is logged and ignored. */ readonly optional?: boolean; readonly timeout?: Duration.Input; -} +} & ( + | { readonly command: string; readonly args: ReadonlyArray } + /** A step no command can do, such as waiting on the Windows launcher's control channel. */ + | { readonly run: Effect.Effect } +); /** Must exceed systemd TimeoutStopSec and launchd ExitTimeOut (both 90s) to avoid shutdown races. */ const STOP_STEP_TIMEOUT = Duration.seconds(120); +/** Covers both passes of `stopWindowsLauncher` (up to 95s graceful stop + 15s fallback release per pass). */ +const WINDOWS_STOP_STEP_TIMEOUT = Duration.seconds(240); export interface BootServiceManager { - readonly kind: "systemd" | "launchd"; + readonly kind: "systemd" | "launchd" | "schtasks"; readonly unitPath: string; readonly render: (plan: BootServicePlan) => string; + readonly prepareStop: ReadonlyArray; + readonly finishStop: ReadonlyArray; readonly stop: ReadonlyArray; readonly activate: ReadonlyArray; readonly restart: ReadonlyArray; @@ -210,18 +316,21 @@ function systemdManager(input: { args: ["--user", "enable", BOOT_SERVICE_UNIT_FILE], }, ]; + const stop: ReadonlyArray = [ + { + step: "stopping the installed service", + command: "systemctl", + args: ["--user", "stop", BOOT_SERVICE_UNIT_FILE], + timeout: STOP_STEP_TIMEOUT, + }, + ]; return { kind: "systemd", unitPath, render: renderBootServiceUnit, - stop: [ - { - step: "stopping the installed service", - command: "systemctl", - args: ["--user", "stop", BOOT_SERVICE_UNIT_FILE], - timeout: STOP_STEP_TIMEOUT, - }, - ], + prepareStop: [], + finishStop: stop, + stop, activate: [ ...prepareActivation, { @@ -277,6 +386,16 @@ function launchdManager(input: { ); const domainTarget = `gui/${input.uid}`; const serviceTarget = `${domainTarget}/${BOOT_SERVICE_LAUNCHD_LABEL}`; + // bootout --wait is undocumented but waits through ExitTimeOut, preventing bootstrap EIO races. + const stop: ReadonlyArray = [ + { + step: "stopping the installed launch agent", + command: "launchctl", + args: ["bootout", "--wait", serviceTarget], + optional: true, + timeout: STOP_STEP_TIMEOUT, + }, + ]; // Unloaded jobs may reject bootout/enable. Bootstrap starts RunAtLoad/KeepAlive jobs; kickstart would restart them. return { kind: "launchd", @@ -286,16 +405,9 @@ function launchdManager(input: { homeDir: input.homeDir, environmentPath: input.environmentPath, }), - // bootout --wait is undocumented but waits through ExitTimeOut, preventing bootstrap EIO races. - stop: [ - { - step: "stopping the installed launch agent", - command: "launchctl", - args: ["bootout", "--wait", serviceTarget], - optional: true, - timeout: STOP_STEP_TIMEOUT, - }, - ], + prepareStop: [], + finishStop: stop, + stop, activate: [ // A persisted `launchctl disable` override refuses bootstrap; clear it. { @@ -331,6 +443,91 @@ function launchdManager(input: { }; } +export interface WindowsSession { + readonly localAppData: string; + /** `DOMAIN\\user`, the account whose logon starts the task. */ + readonly userId: string; + readonly taskName: string; + readonly systemRoot: string; +} + +function schtasksManager(input: { + readonly path: Path.Path; + readonly homeDir: string; + readonly session: WindowsSession; + /** Disables the scheduled task unless it is already absent, failing if Task Scheduler rejects it. */ + readonly disableTask: Effect.Effect; + /** Asks the installed launcher to stop and waits for it; Task Scheduler cannot. */ + readonly stopLauncher: Effect.Effect; + /** Removes the scheduled task unless it is already absent, failing if removal leaves it registered. */ + readonly deleteTask: Effect.Effect; +}): BootServiceManager { + const unitPath = input.path.join(input.session.localAppData, "t3code", BOOT_SERVICE_TASK_FILE); + const schtasksBin = windowsSystem32Binary("schtasks.exe", { + SystemRoot: input.session.systemRoot, + }); + const conhostBin = windowsSystem32Binary("conhost.exe", { + SystemRoot: input.session.systemRoot, + }); + const task = ["/TN", input.session.taskName]; + const register = { + step: "registering the scheduled task", + command: schtasksBin, + args: ["/Create", "/XML", unitPath, ...task, "/F"], + }; + const run = { step: "starting the service", command: schtasksBin, args: ["/Run", ...task] }; + // Disabled first, or the repeating trigger restarts the task while it should stay stopped. + const prepareStop: ReadonlyArray = [ + { step: "disabling the scheduled task", run: input.disableTask }, + ]; + const finishStop: ReadonlyArray = [ + { + step: "stopping the installed service", + run: input.stopLauncher, + timeout: WINDOWS_STOP_STEP_TIMEOUT, + }, + ]; + const stop: ReadonlyArray = [...prepareStop, ...finishStop]; + return { + kind: "schtasks", + unitPath, + render: (plan) => + renderBootServiceTask(plan, { + userId: input.session.userId, + homeDir: input.homeDir, + conhostPath: conhostBin, + }), + prepareStop, + finishStop, + stop, + // Registration also enables the task. + activate: [register, run], + restart: [ + { + step: "enabling the scheduled task", + command: schtasksBin, + args: ["/Change", ...task, "/ENABLE"], + }, + { ...run, step: "restarting the service after a failed update" }, + ], + // `IgnoreNew` ignores `/Run` while the previous console host is active, so `/End` runs first: + // the launcher sees the console close and stops gracefully while the replacement waits for the + // per-home control lock, and the `PT1M` trigger heals if console close interrupts `/Run`. + handoff: [ + register, + { + step: "ending the scheduled task", + command: schtasksBin, + args: ["/End", ...task], + optional: true, + }, + { ...run, step: "restarting the service" }, + ], + deactivate: [...stop, { step: "removing the scheduled task", run: input.deleteTask }], + finalize: [], + }; +} + function selectBootServiceManager(input: { readonly platform: NodeJS.Platform; readonly homeDir: string; @@ -339,10 +536,19 @@ function selectBootServiceManager(input: { readonly uid: number | undefined; readonly path: Path.Path; readonly environmentPath: string; + readonly windows?: { + readonly session: WindowsSession; + readonly disableTask: Effect.Effect; + readonly stopLauncher: Effect.Effect; + readonly deleteTask: Effect.Effect; + }; }): BootServiceManager | undefined { if (input.homeDir === "") { return undefined; } + if (input.platform === "win32" && input.windows !== undefined) { + return schtasksManager({ path: input.path, homeDir: input.homeDir, ...input.windows }); + } if (input.platform === "linux") { return systemdManager({ path: input.path, @@ -409,7 +615,7 @@ export class BootServiceUnsupportedError extends Schema.TaggedError `--base-dir '${baseDir.replaceAll("'", `'"'"'`)}'`; +export function sameBootServiceBaseDir( + path: Path.Path, + platform: NodeJS.Platform, + left: string, + right: string, +): boolean { + const normalize = (target: string) => { + const resolved = path.resolve(target); + return platform === "win32" ? resolved.toLowerCase() : resolved; + }; + return normalize(left) === normalize(right); +} + export class BootServiceOtherBaseDirError extends Schema.TaggedError()( "BootServiceOtherBaseDirError", { installedBaseDir: Schema.String, baseDir: Schema.String, + unrecognizedTaskName: Schema.optional(Schema.String), }, ) { override get message(): string { + if (this.unrecognizedTaskName !== undefined) { + return `The Windows scheduled task '${this.unrecognizedTaskName}' has an unrecognized configuration, so T3 Code did not overwrite or remove it. Inspect it with \`schtasks /Query /TN "${this.unrecognizedTaskName}" /XML\`, or remove it with \`schtasks /Delete /TN "${this.unrecognizedTaskName}" /F\` and retry.`; + } return `The T3 Code service on this machine serves ${this.installedBaseDir}, not ${this.baseDir}. Pass ${formatBootServiceBaseDirOption(this.installedBaseDir)} to manage it, or run \`t3 service uninstall ${formatBootServiceBaseDirOption(this.installedBaseDir)}\` first to move it; both data directories are kept.`; } @@ -519,9 +745,23 @@ export class BootServiceOtherBaseDirError extends Schema.TaggedError()( + "BootServiceRunningInsideServiceError", + {}, +) { + override get message(): string { + return "This command must stop the T3 Code background service, so it cannot run from a terminal inside that service. Run it from an external shell on the host machine instead."; + } + + override get [CliError.userMessage]() { + return this.message; + } +} + export type BootServiceError = | BootServiceUnsupportedError | BootServiceOtherBaseDirError + | BootServiceRunningInsideServiceError | BootServiceCommandError | BootServiceInstallError | BootServicePrerequisiteError @@ -570,8 +810,17 @@ export class BootService extends Context.Service< export interface BootServiceHost { readonly execPath: string; + readonly controlDir?: (baseDir: string) => string; + readonly accountControlRoot?: string; + readonly acl?: ControlAclHooks; + readonly fallbackReleaseTimeoutMs?: number; } +type RegisteredWindowsTaskSnapshot = + | { readonly state: "present"; readonly xml: string } + | { readonly state: "absent" } + | { readonly state: "failed" }; + export const make = Effect.fn("cloud.boot_service.make")(function* (input: { readonly baseDir: string; readonly logsDir: string; @@ -586,16 +835,22 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { const releaseBaseUrl = Option.getOrUndefined( yield* Config.String(CLI_RELEASE_BASE_URL_ENV).pipe(Config.option), ); - const homeDir = yield* Config.String("HOME").pipe(Config.withDefault("")); + const homeDir = yield* Config.String(platform === "win32" ? "USERPROFILE" : "HOME").pipe( + Config.withDefault(""), + ); const xdgConfigHome = yield* Config.String("XDG_CONFIG_HOME").pipe(Config.withDefault("")); const installerPath = yield* Config.String("PATH").pipe(Config.withDefault("")); - const runningInsideService = - (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))) === - BOOT_SERVICE_UNIT_FILE; + const runningInsideService = BOOT_SERVICE_UNIT_VALUES.has( + (yield* Config.String(SERVICE_UNIT_ENV).pipe(Config.withDefault(""))).trim(), + ); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; const host = input.host ?? { execPath: hostExecPath }; + const probe = (command: string, args: ReadonlyArray) => + runner.run({ command, args, timeout: Duration.seconds(5) }).pipe(Effect.option); + const succeeded = (result: Option.Option) => + Option.isSome(result) && result.value.code === 0; const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion, platform); const versionsDir = pinnedRuntimeVersionsDir(path, input.baseDir); const xmlSafeInstallerDirectories = installerPath.split(":").filter( @@ -625,7 +880,299 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ), ).join(":"); + const statOptionalPath = (targetPath: string) => + fs.stat(targetPath).pipe( + Effect.asSome, + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + ); + const readOptionalFileString = (targetPath: string) => + fs.readFileString(targetPath).pipe( + Effect.asSome, + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + Effect.mapError((cause) => new BootServiceInstallError({ cause })), + ); + const readOptionalUnitFileString = Effect.fn("boot_service.read_optional_unit_file")(function* ( + targetPath: string, + ) { + const info = yield* statOptionalPath(targetPath); + if (Option.isNone(info) || info.value.type === "Directory") { + return Option.none(); + } + return yield* readOptionalFileString(targetPath); + }); + const systemdUnitDir = platform === "linux" ? yield* readSystemdUnitDirectory() : undefined; + const windowsVariable = (name: string) => + platform === "win32" ? Config.String(name).pipe(Config.withDefault("")) : Config.succeed(""); + const windowsDomain = (yield* windowsVariable("USERDOMAIN")).trim(); + const windowsUser = (yield* windowsVariable("USERNAME")).trim(); + const configuredLocalAppData = (yield* windowsVariable("LOCALAPPDATA")).trim(); + const windowsLocalAppData = + configuredLocalAppData !== "" + ? configuredLocalAppData + : platform === "win32" && homeDir.trim() !== "" + ? path.join(homeDir.trim(), "AppData", "Local") + : ""; + const taskNamePath = + platform === "win32" && windowsLocalAppData !== "" + ? serviceTaskNamePath(windowsLocalAppData) + : ""; + const persistedTaskName = + taskNamePath === "" + ? Option.none() + : yield* readOptionalFileString(taskNamePath).pipe( + Effect.map( + Option.flatMap((raw) => { + const trimmed = raw.trim(); + return isValidServiceTaskName(trimmed) ? Option.some(trimmed) : Option.none(); + }), + ), + Effect.orElseSucceed(() => Option.none()), + ); + const windowsSession: WindowsSession = { + localAppData: windowsLocalAppData, + userId: windowsDomain === "" ? windowsUser : `${windowsDomain}\\${windowsUser}`, + taskName: Option.getOrElse(persistedTaskName, () => + serviceTaskName(windowsDomain, windowsUser), + ), + systemRoot: (yield* windowsVariable("SystemRoot")).trim() || "C:\\Windows", + }; + const windowsBinEnv = { SystemRoot: windowsSession.systemRoot }; + const schtasksCommand = windowsSystem32Binary("schtasks.exe", windowsBinEnv); + const windowsPowerShellCommand = windowsPowerShellBinary(windowsBinEnv); + const resolveControlDir = + host.controlDir ?? + ((targetBaseDir: string) => + serviceControlDir(windowsSession.localAppData, targetBaseDir, { + caseInsensitive: platform === "win32", + })); + const accountControlRoot = + host.accountControlRoot ?? serviceAccountControlRoot(windowsSession.localAppData); + // Win32 HRESULTs returned by `schtasks /Query /HRESULT` when the task or folder does not exist. + const SCHED_TASK_NOT_FOUND_HRESULTS = new Set([0x80070002, 0x80070003]); + const windowsTaskHasActions = (contents: string) => /, + registeredTaskXml: string | undefined, + ): Option.Option => + registeredTaskXml !== undefined && windowsTaskHasActions(registeredTaskXml) + ? Option.some(registeredTaskXml) + : Option.isSome(localUnit) + ? localUnit + : Option.fromNullishOr(registeredTaskXml); + const windowsUnitPath = + windowsSession.localAppData === "" + ? "" + : path.join(windowsSession.localAppData, "t3code", BOOT_SERVICE_TASK_FILE); + const queryRegisteredWindowsTask = Effect.fn("cloud.boot_service.query_windows_task")( + function* (): Effect.fn.Return { + const queried = yield* runner + .run({ + command: windowsPowerShellCommand, + args: [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + WINDOWS_QUERY_REGISTERED_TASK_SCRIPT, + ], + env: { T3_TASK_NAME: windowsSession.taskName }, + timeout: Duration.seconds(15), + }) + .pipe(Effect.option); + if (Option.isNone(queried)) return { state: "failed" }; + if ( + queried.value.code === 0 && + !queried.value.stdoutInvalidUtf8 && + !queried.value.stdoutTruncated + ) { + return { state: "present", xml: queried.value.stdout }; + } + if (Number(queried.value.code) === 2) { + return { state: "absent" }; + } + return { state: "failed" }; + }, + ); + // The unit is per user, so the launcher to stop is the one serving the installed unit's home. + const launcherBaseDir = (unit: Option.Option) => + Option.getOrUndefined(Option.flatMapNullishOr(unit, bootServiceBaseDirOf)) ?? input.baseDir; + const disableWindowsTask = Effect.fn("cloud.boot_service.disable_windows_task")(function* () { + const runDisable = Effect.gen(function* () { + const disabled = yield* runner + .run({ + command: schtasksCommand, + args: ["/Change", "/TN", windowsSession.taskName, "/DISABLE"], + }) + .pipe( + Effect.mapError( + (cause) => new BootServiceCommandError({ step: "disabling the scheduled task", cause }), + ), + ); + if (disabled.code === 0) return; + const queried = yield* runner + .run({ + command: schtasksCommand, + args: ["/Query", "/TN", windowsSession.taskName, "/HRESULT"], + timeout: Duration.seconds(5), + }) + .pipe( + Effect.mapError( + (cause) => new BootServiceCommandError({ step: "disabling the scheduled task", cause }), + ), + ); + if (!SCHED_TASK_NOT_FOUND_HRESULTS.has(Number(queried.code) >>> 0)) { + return yield* new BootServiceCommandError({ + step: "disabling the scheduled task", + exitCode: Number(disabled.code), + stdoutLength: disabled.stdout.length, + stderrLength: disabled.stderr.length, + }); + } + }); + if (windowsUnitPath === "") { + return yield* runDisable; + } + // Serialize with `Launcher.#repointServiceUnit` (`reloadWindowsTaskUnit`), releasing before `stopWindowsLauncher`. + return yield* Effect.acquireUseRelease( + Effect.tryPromise({ + try: () => acquireServiceSelectionLock(windowsUnitPath), + catch: (cause) => + new BootServiceCommandError({ step: "disabling the scheduled task", cause }), + }), + () => runDisable, + (release) => Effect.sync(release), + ); + }); + const stopWindowsLauncher = Effect.fn("cloud.boot_service.stop_windows_launcher")(function* () { + const [localUnit, queriedTask] = yield* Effect.all( + [ + readOptionalUnitFileString(unitPath).pipe( + Effect.mapError( + (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + ), + ), + queryRegisteredWindowsTask(), + ], + { concurrency: "unbounded" }, + ); + if (queriedTask.state === "failed") { + return yield* new BootServiceCommandError({ step: "stopping the installed service" }); + } + const registeredTaskXml = queriedTask.state === "present" ? queriedTask.xml : undefined; + const unit = selectEffectiveWindowsUnit(localUnit, registeredTaskXml); + const targetBaseDir = launcherBaseDir(unit); + const controlDir = resolveControlDir(targetBaseDir); + for (let pass = 0; pass < 2; pass++) { + const fallbackToTaskEnd = yield* Effect.tryPromise({ + try: async (signal) => { + try { + await requestServiceLauncherStop(controlDir, { + signal, + acl: host.acl, + }); + return false; + } catch (cause) { + if (signal.aborted) throw cause; + return true; + } + }, + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + // Agents the server left behind keep its console host, and so the task, running. + // When control-channel stop failed (for example, a lost or untrusted token), ending the + // task closes conhost stdin so the launcher exits and releases its lock. + if (pass === 0 || fallbackToTaskEnd) { + yield* runner + .run({ command: schtasksCommand, args: ["/End", "/TN", windowsSession.taskName] }) + .pipe(Effect.ignore); + } + if (fallbackToTaskEnd) { + yield* Effect.tryPromise({ + try: (signal) => + waitForServiceLauncherRelease(controlDir, { + timeoutMs: host.fallbackReleaseTimeoutMs ?? 15_000, + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + } + const remaining = yield* Effect.tryPromise({ + try: (signal) => + probeServiceLauncher(controlDir, { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + if (remaining === "stopped") return; + if (pass === 1) { + return yield* new BootServiceCommandError({ + step: "stopping the installed service", + }); + } + } + }); + const deleteWindowsTask = Effect.fn("cloud.boot_service.delete_windows_task")(function* () { + const deleted = yield* runner + .run({ + command: schtasksCommand, + args: ["/Delete", "/TN", windowsSession.taskName, "/F"], + }) + .pipe( + Effect.mapError( + (cause) => new BootServiceCommandError({ step: "removing the scheduled task", cause }), + ), + ); + if (deleted.code === 0) return; + // Succeed only when Task Scheduler specifically confirms the task is absent; preserve the + // local XML mirror when access errors or an unavailable Task Scheduler prevent verification. + const queried = yield* runner + .run({ + command: schtasksCommand, + args: ["/Query", "/TN", windowsSession.taskName, "/HRESULT"], + timeout: Duration.seconds(5), + }) + .pipe( + Effect.mapError( + (cause) => new BootServiceCommandError({ step: "removing the scheduled task", cause }), + ), + ); + if (!SCHED_TASK_NOT_FOUND_HRESULTS.has(Number(queried.code) >>> 0)) { + return yield* new BootServiceCommandError({ + step: "removing the scheduled task", + exitCode: Number(deleted.code), + stdoutLength: deleted.stdout.length, + stderrLength: deleted.stderr.length, + }); + } + }); + const signalPreviousWindowsLauncher = Effect.fn( + "cloud.boot_service.signal_previous_windows_launcher", + )(function* (launcherId: string | undefined) { + if (launcherId === undefined) return; + yield* Effect.tryPromise({ + try: (signal) => + signalServiceLauncherStop(resolveControlDir(input.baseDir), launcherId, { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }).pipe(Effect.ignore); + }); const detectedManager = selectBootServiceManager({ platform, homeDir, @@ -634,18 +1181,115 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { uid, path, environmentPath, + ...(windowsSession.localAppData === "" || windowsUser === "" + ? {} + : { + windows: { + session: windowsSession, + disableTask: disableWindowsTask(), + stopLauncher: stopWindowsLauncher(), + deleteTask: deleteWindowsTask(), + }, + }), }); const unitPath = detectedManager?.unitPath ?? ""; + const isUnrecognizedWindowsTask = (contents: string): boolean => + detectedManager?.kind === "schtasks" && + windowsTaskHasActions(contents) && + bootServiceBaseDirOf(contents) === undefined; + const readInstalledUnitSnapshot = Effect.fn("cloud.boot_service.read_installed_unit_snapshot")( + function* () { + if (unitPath === "") { + return { + localUnit: Option.none(), + queriedTask: { state: "absent" } as RegisteredWindowsTaskSnapshot, + registeredTaskXml: undefined, + effectiveUnit: Option.none(), + }; + } + if (detectedManager?.kind !== "schtasks") { + const localUnit = yield* readOptionalUnitFileString(unitPath); + return { + localUnit, + queriedTask: { state: "absent" } as RegisteredWindowsTaskSnapshot, + registeredTaskXml: undefined, + effectiveUnit: localUnit, + }; + } + const [localUnit, queriedTask] = yield* Effect.all( + [readOptionalUnitFileString(unitPath), queryRegisteredWindowsTask()], + { concurrency: "unbounded" }, + ); + const registeredTaskXml = queriedTask.state === "present" ? queriedTask.xml : undefined; + const effectiveUnit = selectEffectiveWindowsUnit(localUnit, registeredTaskXml); + return { + localUnit, + queriedTask, + registeredTaskXml, + effectiveUnit, + }; + }, + ); + const withWindowsAccountLock = ( + manager: BootServiceManager, + effect: Effect.Effect, + mapLockError: (cause: unknown) => E, + ): Effect.Effect => + manager.kind !== "schtasks" + ? effect + : Effect.scoped( + Effect.acquireRelease( + Effect.tryPromise({ + try: (signal) => + acquireServiceAccountLock(accountControlRoot, { + signal, + acl: host.acl, + }), + catch: mapLockError, + }), + (lock) => Effect.sync(() => lock.close()), + ).pipe(Effect.andThen(effect)), + ); const logPath = path.join(input.logsDir, BOOT_SERVICE_LOG_FILE); const statePath = path.join(input.baseDir, "runtime", SERVICE_STATE_FILE); const restartPendingPath = path.join(input.baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); + const sameBaseDir = (left: string, right: string) => + sameBootServiceBaseDir(path, platform, left, right); const otherBaseDirOf = (unit: string) => { const installedBaseDir = bootServiceBaseDirOf(unit); - return installedBaseDir !== undefined && - path.resolve(installedBaseDir) !== path.resolve(input.baseDir) + return installedBaseDir !== undefined && !sameBaseDir(installedBaseDir, input.baseDir) ? installedBaseDir : undefined; }; + const guardWindowsSnapshot = (snapshot: { + readonly queriedTask: RegisteredWindowsTaskSnapshot; + readonly effectiveUnit: Option.Option; + }): Effect.Effect => { + if (detectedManager?.kind === "schtasks" && snapshot.queriedTask.state === "failed") { + return Effect.fail(new BootServiceCommandError({ step: "inspecting the scheduled task" })); + } + if (Option.isSome(snapshot.effectiveUnit)) { + if (isUnrecognizedWindowsTask(snapshot.effectiveUnit.value)) { + return Effect.fail( + new BootServiceOtherBaseDirError({ + installedBaseDir: "", + baseDir: input.baseDir, + unrecognizedTaskName: windowsSession.taskName, + }), + ); + } + const otherBaseDir = otherBaseDirOf(snapshot.effectiveUnit.value); + if (otherBaseDir !== undefined) { + return Effect.fail( + new BootServiceOtherBaseDirError({ + installedBaseDir: otherBaseDir, + baseDir: input.baseDir, + }), + ); + } + } + return Effect.void; + }; const writeDurably = (filePath: string, contents: string) => Effect.scoped( Effect.gen(function* () { @@ -712,21 +1356,26 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { Effect.forEach( steps, (entry) => { - const run = runStep( - entry.step, - entry.command, - entry.args, - entry.timeout === undefined ? undefined : { timeout: entry.timeout }, - ); + const run = + "run" in entry + ? entry.run.pipe( + Effect.timeoutOrElse({ + duration: entry.timeout ?? Duration.seconds(60), + orElse: () => Effect.fail(new BootServiceCommandError({ step: entry.step })), + }), + Effect.tapError(logFailure), + ) + : runStep( + entry.step, + entry.command, + entry.args, + entry.timeout === undefined ? undefined : { timeout: entry.timeout }, + ); return entry.optional === true ? run.pipe(Effect.ignore) : run.pipe(Effect.asVoid); }, { discard: true }, ); - const probe = (command: string, args: ReadonlyArray) => - runner.run({ command, args, timeout: Duration.seconds(5) }).pipe(Effect.option); - const succeeded = (result: Option.Option) => - Option.isSome(result) && result.value.code === 0; const lingerArgs = [ "show-user", ...(uid === undefined ? [] : [String(uid)]), @@ -793,20 +1442,98 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { (release) => Effect.sync(release), ); + // Running means the launcher answers authenticated control requests while holding its lock: + // a task can stay running after the launcher exits while orphaned agents hold conhost open, + // and an occupied lock that fails authentication reports `service-unhealthy` rather than healthy. + const readSchtasksProblems = Effect.fn("cloud.boot_service.read_schtasks_problems")(function* ( + unit: Option.Option, + prequeriedTask?: RegisteredWindowsTaskSnapshot, + ) { + const targetBaseDir = launcherBaseDir(unit); + const controlDir = resolveControlDir(targetBaseDir); + const [task, launcherState] = yield* Effect.all( + [ + prequeriedTask === undefined + ? queryRegisteredWindowsTask() + : Effect.succeed(prequeriedTask), + Effect.tryPromise({ + try: (signal) => + probeServiceLauncher(controlDir, { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "inspecting the installed service", cause }), + }).pipe(Effect.orElseSucceed(() => "unhealthy" as const)), + ], + { concurrency: "unbounded" }, + ); + const problems: BootServiceProblem[] = []; + if (task.state === "present") { + const settings = /[\s\S]*?<\/Settings>/.exec(task.xml)?.[0] ?? ""; + if (/\s*false\s*<\/Enabled>/i.test(settings)) { + problems.push("service-disabled"); + } + } else if (task.state === "absent") { + problems.push("service-disabled"); + } + if (launcherState === "stopped") problems.push("service-stopped"); + else if (launcherState === "unhealthy") problems.push("service-unhealthy"); + if ( + (task.state === "failed" || (Option.isSome(unit) && isUnrecognizedWindowsTask(unit.value))) && + !problems.includes("service-unhealthy") + ) { + problems.push("service-unhealthy"); + } + return problems; + }); + + const disableScheduledTaskStep: BootServiceStep = { + step: "keeping the scheduled task disabled", + run: disableWindowsTask(), + }; + + const schtasksRestoreSteps = ( + manager: BootServiceManager, + problems: ReadonlyArray, + options?: { readonly mayHaveEnabled?: boolean }, + ): ReadonlyArray => { + const wasEnabled = !problems.includes("service-disabled"); + if (wasEnabled) return manager.restart; + if (problems.includes("service-stopped")) { + return options?.mayHaveEnabled === true ? [disableScheduledTaskStep] : []; + } + return [ + { + step: "restarting the service after a failed update", + run: runSteps(manager.restart).pipe( + Effect.ensuring(disableWindowsTask().pipe(Effect.ignore)), + ), + }, + ]; + }; + const install = Effect.fn("cloud.boot_service.install")(function* (options?: { readonly allowDowngrade?: boolean; readonly start?: boolean; }) { const manager = yield* requireManager; + if ( + manager.kind === "schtasks" && + [input.baseDir, logPath, runtimePaths.entryPath, homeDir].some( + hasWindowsTaskEnvironmentExpansion, + ) + ) { + return yield* new BootServiceInstallError({ + cause: new Error( + "Windows Task Scheduler expands '%NAME%' segments in task action paths without an escape syntax; choose paths without '%...%' segments.", + ), + }); + } // The unit is per user. Rewriting one that serves another home moves the // service onto this home's data, which looks like every thread vanished. - const existingUnit = yield* fs.readFileString(unitPath).pipe(Effect.option); - const installedBaseDir = Option.isSome(existingUnit) - ? otherBaseDirOf(existingUnit.value) - : undefined; - if (installedBaseDir !== undefined) { - return yield* new BootServiceOtherBaseDirError({ installedBaseDir, baseDir: input.baseDir }); - } + const initialSnapshot = yield* readInstalledUnitSnapshot(); + yield* guardWindowsSnapshot(initialSnapshot); yield* fs .makeDirectory(input.logsDir, { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); @@ -867,100 +1594,224 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { : new BootServiceInstallError({ cause: error }), ), ); - const installed = yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // Stop outside the selection lock: the launcher may be waiting to acquire it. - const start = options?.start !== false; - if (installed && start && runningInsideService && manager.kind === "systemd") { - // Stopping the unit from inside it kills this process before the start. - // A transient unit runs the same install from outside, stop first, so it - // still serialises with a remote update the launcher may be starting. - yield* runStep("handing the install to a transient unit", "systemd-run", [ - "--user", - "--collect", - "--quiet", - // Transient properties reach systemd literally, unlike unit-file specifiers. - `--property=StandardOutput=append:${logPath}`, - `--property=StandardError=append:${logPath}`, - `--setenv=${SERVICE_UNIT_ENV}=`, - runtimePaths.entryPath, - "service", - "install", - "--base-dir", - // systemd expands $VAR in arguments (not in the executable path). - input.baseDir.replaceAll("$", () => "$$"), - ...(options?.allowDowngrade === true ? ["--allow-downgrade"] : []), - ]); - return { ...plan, handedOff: true }; - } - if (installed && start) { - yield* runSteps(manager.stop); + if (manager.kind === "schtasks") { + // Pre-Windows-service builds pass `--version` yet accept only bare `__service-launcher`; + // verify flag support before stopping a working service or repointing its scheduled task. + yield* runner + .run({ + command: pinnedRuntimeCommand(runtimePaths).command, + args: [...pinnedRuntimeCommand(runtimePaths).args, "__service-launcher", "--help"], + timeout: Duration.seconds(30), + }) + .pipe( + Effect.mapError( + (cause) => + new BootServiceCommandError({ + step: "verifying Windows service launcher support", + cause, + }), + ), + Effect.filterOrFail( + (result) => + result.code === 0 && + result.stdout.includes("--base-dir") && + result.stdout.includes("--log-file") && + result.stdout.includes("--service-unit"), + (result) => + new BootServiceCommandError({ + step: "verifying Windows service launcher support", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ), + Effect.tapError(logFailure), + ); } - // State first, then unit. The launcher acquires only one at a time, and a - // failure to lock the unit must never prevent its durable update commit. - yield* withSelectionLock( - statePath, - withSelectionLock( - unitPath, - Effect.gen(function* () { - const currentlyInstalled = yield* fs.exists(unitPath); - const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); - if (Option.isSome(previousStateText)) { - const installedVersion = serviceStateActiveVersion(previousStateText.value); - if (serviceStateHasPendingUpdate(previousStateText.value)) { - const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`; + return yield* withWindowsAccountLock( + manager, + Effect.gen(function* () { + const existingSnapshot = yield* readInstalledUnitSnapshot(); + yield* guardWindowsSnapshot(existingSnapshot); + const installed = + Option.isSome(existingSnapshot.localUnit) || + Option.isSome(existingSnapshot.effectiveUnit); + // Stop outside the selection lock: the launcher may be waiting to acquire it. + const start = options?.start !== false; + if (installed && start && runningInsideService && manager.kind === "systemd") { + // Stopping the unit from inside it kills this process before the start. + // A transient unit runs the same install from outside, stop first, so it + // still serialises with a remote update the launcher may be starting. + yield* runStep("handing the install to a transient unit", "systemd-run", [ + "--user", + "--collect", + "--quiet", + // Transient properties reach systemd literally, unlike unit-file specifiers. + `--property=StandardOutput=append:${logPath}`, + `--property=StandardError=append:${logPath}`, + `--setenv=${SERVICE_UNIT_ENV}=`, + runtimePaths.entryPath, + "service", + "install", + "--base-dir", + // systemd expands $VAR in arguments (not in the executable path). + input.baseDir.replaceAll("$", () => "$$"), + ...(options?.allowDowngrade === true ? ["--allow-downgrade"] : []), + ]); + return { ...plan, handedOff: true }; + } + // Task Scheduler restarts the task itself, so the files are written as for a deferred + // restart while this launcher keeps running, then the restart is handed to the manager. + const handOff = installed && start && runningInsideService && manager.kind === "schtasks"; + const schtasksProblems = + installed && !handOff && manager.kind === "schtasks" + ? yield* readSchtasksProblems( + existingSnapshot.effectiveUnit, + existingSnapshot.queriedTask, + ) + : undefined; + let canRestoreWindows = false; + + yield* Effect.gen(function* () { + if (installed && start && !handOff) { + if (schtasksProblems !== undefined) { + yield* runSteps(manager.prepareStop); + canRestoreWindows = true; + yield* runSteps(manager.finishStop); + } else { + yield* runSteps(manager.stop); + } + } + let stagedLauncherId: string | undefined; + if (manager.kind === "schtasks") { + const staged = yield* Effect.tryPromise({ + try: (signal) => + stageServiceLauncherInstall({ + controlDir: resolveControlDir(input.baseDir), + baseDir: input.baseDir, + version: input.cliVersion, + signal, + acl: host.acl, + ...(options?.allowDowngrade === undefined + ? {} + : { allowDowngrade: options.allowDowngrade }), + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + if (staged.status === "update-pending") { + const psQuote = (word: string) => `'${word.replaceAll("'", "''")}'`; return yield* new BootServiceUpdatePendingError( - !currentlyInstalled && installedVersion !== undefined + !installed && staged.installedVersion !== undefined ? { - recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`, + recoveryCommand: `& ${psQuote(pinnedRuntimePaths(path, input.baseDir, staged.installedVersion, platform).entryPath)} __service-launcher --base-dir ${psQuote(input.baseDir)}`, } : {}, ); } - // Read the final version after stopping the launcher; a remote update may finish after the status check. - if ( - installedVersion !== undefined && - options?.allowDowngrade !== true && - compareExactServiceVersions(input.cliVersion, installedVersion) < 0 - ) { + if (staged.status === "downgrade-refused") { return yield* new BootServiceDowngradeRefusedError({ - installedVersion, + installedVersion: staged.installedVersion, targetVersion: input.cliVersion, }); } + stagedLauncherId = staged.launcherId; + // State first, then unit. The launcher acquires only one at a time, and a + // failure to lock the unit must never prevent its durable update commit. + yield* withSelectionLock( + statePath, + withSelectionLock( + unitPath, + Effect.gen(function* () { + if (!start || handOff) { + const written = yield* fs.readFileString(statePath).pipe(Effect.option); + if ( + Option.isSome(written) && + (serviceStateHasPendingUpdate(written.value) || + serviceStateActiveVersion(written.value) !== input.cliVersion) + ) { + return yield* new BootServiceUpdatePendingError(); + } + } + yield* fs + .makeDirectory(path.dirname(unitPath), { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (taskNamePath !== "") { + yield* writeDurably(taskNamePath, `${windowsSession.taskName}\n`); + } + yield* writeDurably(unitPath, manager.render(plan)); + }), + ), + ); + } else { + // State first, then unit. The launcher acquires only one at a time, and a + // failure to lock the unit must never prevent its durable update commit. + yield* withSelectionLock( + statePath, + withSelectionLock( + unitPath, + Effect.gen(function* () { + const currentlyInstalled = yield* fs.exists(unitPath); + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if (Option.isSome(previousStateText)) { + const installedVersion = serviceStateActiveVersion(previousStateText.value); + if (serviceStateHasPendingUpdate(previousStateText.value)) { + const quote = (word: string) => `'${word.replaceAll("'", "'\\''")}'`; + return yield* new BootServiceUpdatePendingError( + !currentlyInstalled && installedVersion !== undefined + ? { + recoveryCommand: `T3CODE_HOME=${quote(input.baseDir)} ${quote(pinnedRuntimePaths(path, input.baseDir, installedVersion, platform).entryPath)} __service-launcher`, + } + : {}, + ); + } + // Read the final version after stopping the launcher; a remote update may finish after the status check. + if ( + installedVersion !== undefined && + options?.allowDowngrade !== true && + compareExactServiceVersions(input.cliVersion, installedVersion) < 0 + ) { + return yield* new BootServiceDowngradeRefusedError({ + installedVersion, + targetVersion: input.cliVersion, + }); + } + } + yield* fs + .makeDirectory(path.dirname(unitPath), { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + // Files can match the new version even when activation fails; keep that install retryable. + yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { + mode: 0o600, + }); + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { + const written = yield* fs.readFileString(statePath); + if (serviceStateActiveVersion(written) !== input.cliVersion) { + return yield* new BootServiceUpdatePendingError(); + } + } + yield* writeDurably(unitPath, manager.render(plan)); + }), + ), + ); } - yield* fs - .makeDirectory(path.dirname(unitPath), { recursive: true }) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - // Files can match the new version even when activation fails; keep that install retryable. - yield* fs.writeFileString(restartPendingPath, `${input.cliVersion}\n`, { mode: 0o600 }); - yield* writeDurably( - statePath, - // @effect-diagnostics-next-line globalJsonInEffect:off - fixed launcher-owned document. - `${JSON.stringify( - { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: input.cliVersion, - } satisfies ServiceState, - null, - 2, - )}\n`, - ); - if (!start && (currentlyInstalled || Option.isSome(previousStateText))) { - const written = yield* fs.readFileString(statePath); - if (serviceStateActiveVersion(written) !== input.cliVersion) { - return yield* new BootServiceUpdatePendingError(); - } - } - yield* writeDurably(unitPath, manager.render(plan)); - }), - ), - ).pipe( - Effect.andThen( - Effect.gen(function* () { - if (start) { + + if (handOff) { + yield* runSteps(manager.handoff ?? []); + yield* signalPreviousWindowsLauncher(stagedLauncherId); + } else if (start) { yield* runSteps(manager.activate); yield* withSelectionLock( statePath, @@ -972,126 +1823,501 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } }), ); + } else if (manager.kind === "schtasks") { + // Task Scheduler keeps its own copy of the task action; register `false` + // directly when the task must stay disabled so the minute trigger never fires in between. + if (!installed || schtasksProblems?.includes("service-disabled") === true) { + yield* Effect.scoped( + Effect.gen(function* () { + const latestUnit = Option.getOrElse( + yield* fs.readFileString(unitPath).pipe(Effect.option), + () => manager.render(plan), + ); + const disabledXml = /\s*false\s*<\/Enabled>/i.test(latestUnit) + ? latestUnit + : latestUnit.replace( + /<\/Settings>/i, + " false\n ", + ); + const disabledXmlPath = yield* fs + .makeTempFileScoped({ + directory: path.dirname(unitPath), + prefix: ".service-task-", + }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + yield* fs + .writeFileString(disabledXmlPath, disabledXml, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + yield* runStep("registering the scheduled task", schtasksCommand, [ + "/Create", + "/XML", + disabledXmlPath, + "/TN", + windowsSession.taskName, + "/F", + ]); + }), + ); + } else { + yield* runStep("registering the scheduled task", schtasksCommand, [ + "/Create", + "/XML", + unitPath, + "/TN", + windowsSession.taskName, + "/F", + ]); + } } - }), - ), - Effect.mapError((cause) => - cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, - ), - Effect.tapError(() => - installed && start ? runSteps(manager.restart).pipe(Effect.ignore) : Effect.void, - ), + }).pipe( + Effect.mapError((cause) => + cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, + ), + Effect.onExit((exit) => + Exit.isFailure(exit) && + installed && + start && + !handOff && + (schtasksProblems === undefined || canRestoreWindows) + ? runSteps( + schtasksProblems === undefined + ? manager.restart + : schtasksRestoreSteps(manager, schtasksProblems, { mayHaveEnabled: true }), + ).pipe(Effect.ignore) + : Effect.void, + ), + ); + return handOff ? { ...plan, handedOff: true } : plan; + }), + (cause) => new BootServiceInstallError({ cause }), ); - return plan; }); const restart: BootService["Service"]["restart"] = Effect.gen(function* () { const manager = yield* requireManager; - const unit = yield* fs.readFileString(unitPath).pipe(Effect.option); - if (Option.isNone(unit)) return "skipped" as const; - const installedBaseDir = bootServiceBaseDirOf(unit.value); - if ( - installedBaseDir === undefined || - path.resolve(installedBaseDir) !== path.resolve(input.baseDir) - ) { - return "skipped" as const; - } - // Writes no state, and the launcher lets an update transition in flight - // finish before the queued stop tears it down. - if (runningInsideService && manager.handoff !== undefined) { - yield* runSteps(manager.handoff); - return "queued" as const; - } - yield* runSteps(manager.stop); - yield* runSteps(manager.activate).pipe( - Effect.tapError(() => runSteps(manager.restart).pipe(Effect.ignore)), + return yield* withWindowsAccountLock( + manager, + Effect.gen(function* () { + const snapshot = yield* readInstalledUnitSnapshot(); + if (manager.kind === "schtasks" && snapshot.queriedTask.state === "failed") { + return yield* new BootServiceCommandError({ step: "inspecting the scheduled task" }); + } + if (Option.isNone(snapshot.effectiveUnit)) return "skipped" as const; + const installedBaseDir = bootServiceBaseDirOf(snapshot.effectiveUnit.value); + if (installedBaseDir === undefined || !sameBaseDir(installedBaseDir, input.baseDir)) { + return "skipped" as const; + } + const localUnitMatchesEffective = + Option.isSome(snapshot.localUnit) && + (() => { + const localBaseDir = bootServiceBaseDirOf(snapshot.localUnit.value); + const localProgram = bootServiceProgramOf(snapshot.localUnit.value); + if ( + localBaseDir === undefined || + localProgram === undefined || + !sameBaseDir(localBaseDir, input.baseDir) + ) { + return false; + } + if ( + snapshot.registeredTaskXml !== undefined && + windowsTaskHasActions(snapshot.registeredTaskXml) + ) { + return localProgram === bootServiceProgramOf(snapshot.registeredTaskXml); + } + return true; + })(); + const directWindowsTaskSteps = manager.kind === "schtasks" && !localUnitMatchesEffective; + // Writes no state, and the launcher lets an update transition in flight + // finish before the queued stop tears it down. + if (runningInsideService && manager.handoff !== undefined) { + const previousLauncherId = + manager.kind === "schtasks" + ? yield* Effect.tryPromise({ + try: (signal) => + queryServiceLauncherId(resolveControlDir(input.baseDir), { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ + step: "inspecting the installed service", + cause, + }), + }).pipe(Effect.orElseSucceed(() => undefined)) + : undefined; + const handoffSteps: ReadonlyArray = directWindowsTaskSteps + ? [ + { + step: "enabling the scheduled task", + command: schtasksCommand, + args: ["/Change", "/TN", windowsSession.taskName, "/ENABLE"], + }, + { + step: "ending the scheduled task", + command: schtasksCommand, + args: ["/End", "/TN", windowsSession.taskName], + optional: true, + }, + { + step: "restarting the service", + command: schtasksCommand, + args: ["/Run", "/TN", windowsSession.taskName], + }, + ] + : manager.handoff; + yield* runSteps(handoffSteps); + yield* signalPreviousWindowsLauncher(previousLauncherId); + return "queued" as const; + } + if (manager.kind === "schtasks") { + const schtasksProblems = yield* readSchtasksProblems( + snapshot.effectiveUnit, + snapshot.queriedTask, + ); + const activateSteps: ReadonlyArray = directWindowsTaskSteps + ? [ + { + step: "enabling the scheduled task", + command: schtasksCommand, + args: ["/Change", "/TN", windowsSession.taskName, "/ENABLE"], + }, + { + step: "starting the service", + command: schtasksCommand, + args: ["/Run", "/TN", windowsSession.taskName], + }, + ] + : manager.activate; + yield* runSteps(manager.prepareStop); + yield* Effect.gen(function* () { + yield* runSteps(manager.finishStop); + yield* runSteps(activateSteps); + }).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? runSteps( + schtasksRestoreSteps(manager, schtasksProblems, { mayHaveEnabled: true }), + ).pipe(Effect.ignore) + : Effect.void, + ), + ); + } else { + yield* runSteps(manager.stop); + yield* runSteps(manager.activate).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? runSteps(manager.restart).pipe(Effect.ignore) : Effect.void, + ), + ); + } + yield* fs + .remove(restartPendingPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + return "restarted" as const; + }), + (cause) => new BootServiceInstallError({ cause }), ); - yield* fs.remove(restartPendingPath, { force: true }); - return "restarted" as const; - }).pipe( - Effect.mapError((cause) => - cause._tag === "PlatformError" ? new BootServiceInstallError({ cause }) : cause, - ), - Effect.withSpan("cloud.boot_service.restart"), - ); + }).pipe(Effect.withSpan("cloud.boot_service.restart")); const withStopped = Effect.fn("cloud.boot_service.with_stopped")(function* ( operation: Effect.Effect, ): Effect.fn.Return { if (detectedManager === undefined) return yield* operation; const manager = detectedManager; - const unit = yield* fs.readFileString(unitPath).pipe(Effect.option); - if (Option.isNone(unit)) return yield* operation; - const installedBaseDir = bootServiceBaseDirOf(unit.value); - if ( - installedBaseDir === undefined || - path.resolve(installedBaseDir) !== path.resolve(input.baseDir) - ) { - return yield* operation; - } - // `is-active` misses crash-looping systemd units; launchd can be loaded without a pid. - // Always stop; restore only if previously active, enabled, or loaded, - // so disabled crash loops stay stopped. - const restore = - manager.kind === "systemd" - ? yield* Effect.map( - Effect.all( - [ - probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]), - probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]), - ], - { concurrency: "unbounded" }, - ), - ([enabled, active]) => succeeded(active) || succeeded(enabled), - ) - : succeeded( - yield* probe("launchctl", ["print", `gui/${uid}/${BOOT_SERVICE_LAUNCHD_LABEL}`]), - ); - return yield* Effect.acquireUseRelease( - // Require every stop before repair when restoring, so the server releases the database. - // Otherwise stops are best-effort; launchd rejects bootout of an unloaded job. - // A failed stop may still stop the service, so restoration runs either way. - runSteps(manager.stop.map((step) => ({ ...step, optional: !restore }))).pipe(Effect.exit), - (stopped): Effect.Effect => - Exit.isSuccess(stopped) ? operation : Effect.failCause(stopped.cause), - (_, exit) => { - if (!restore) return Effect.void; - // Start only: `activate` would also enable a unit the user disabled. - // When the operation itself failed, its error is the one worth reporting. - const start = runSteps(manager.restart); - return Exit.isFailure(exit) ? Effect.ignore(start) : start; - }, + return yield* withWindowsAccountLock( + manager, + Effect.gen(function* () { + const snapshot = yield* readInstalledUnitSnapshot(); + if (manager.kind === "schtasks" && snapshot.queriedTask.state === "failed") { + return yield* new BootServiceCommandError({ step: "inspecting the scheduled task" }); + } + if (Option.isNone(snapshot.effectiveUnit)) return yield* operation; + const installedBaseDir = bootServiceBaseDirOf(snapshot.effectiveUnit.value); + if (installedBaseDir === undefined || !sameBaseDir(installedBaseDir, input.baseDir)) { + return yield* operation; + } + if (runningInsideService) { + return yield* new BootServiceRunningInsideServiceError(); + } + // `is-active` misses crash-looping systemd units; launchd can be loaded without a pid. + // Always stop; restore only if previously active, enabled, or loaded, + // so disabled crash loops stay stopped. + const schtasksProblems = + manager.kind === "schtasks" + ? yield* readSchtasksProblems(snapshot.effectiveUnit, snapshot.queriedTask) + : undefined; + const restoreSteps = + schtasksProblems === undefined + ? manager.restart + : schtasksRestoreSteps(manager, schtasksProblems); + const restore = + manager.kind === "systemd" + ? yield* Effect.map( + Effect.all( + [ + probe("systemctl", ["--user", "is-enabled", BOOT_SERVICE_UNIT_FILE]), + probe("systemctl", ["--user", "is-active", BOOT_SERVICE_UNIT_FILE]), + ], + { concurrency: "unbounded" }, + ), + ([enabled, active]) => succeeded(active) || succeeded(enabled), + ) + : schtasksProblems !== undefined + ? restoreSteps.length > 0 + : succeeded( + yield* probe("launchctl", ["print", `gui/${uid}/${BOOT_SERVICE_LAUNCHD_LABEL}`]), + ); + return yield* Effect.acquireUseRelease( + // Require every stop before repair when restoring, so the server releases the database. + // Otherwise stops are best-effort; launchd rejects bootout of an unloaded job. + // A failed stop may still stop the service, so restoration runs either way. + runSteps(manager.stop.map((step) => ({ ...step, optional: !restore }))).pipe(Effect.exit), + (stopped): Effect.Effect => + Exit.isSuccess(stopped) ? operation : Effect.failCause(stopped.cause), + (_, exit) => { + if (!restore) return Effect.void; + // Start only: `activate` would also enable a unit the user disabled. + // When the operation itself failed, its error is the one worth reporting. + const start = runSteps(restoreSteps); + return Exit.isFailure(exit) ? Effect.ignore(start) : start; + }, + ); + }), + (cause) => new BootServiceInstallError({ cause }), ); }); + const signalCurrentWindowsLauncherForUninstall = (controlDir: string) => + Effect.tryPromise({ + try: async (signal) => { + const signaled = await signalServiceLauncherStop(controlDir, undefined, { + signal, + acl: host.acl, + uninstall: true, + }); + if (signaled) return; + const remaining = await probeServiceLauncher(controlDir, { + signal, + acl: host.acl, + }); + if (remaining !== "stopped") { + throw new Error("Could not signal the running service launcher to stop."); + } + }, + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { const manager = yield* requireManager; - if ( - !(yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) - ) - return false; - yield* runSteps(manager.deactivate); - yield* fs - .remove(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - yield* runSteps(manager.finalize); - return true; + return yield* withWindowsAccountLock( + manager, + Effect.gen(function* () { + if (manager.kind !== "schtasks") { + const unitInfo = yield* statOptionalPath(unitPath); + if (Option.isNone(unitInfo) || unitInfo.value.type === "Directory") { + if (Option.isNone(unitInfo)) { + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + signal, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + } + return false; + } + const readableUnit = yield* fs.readFileString(unitPath).pipe(Effect.option); + const otherBaseDir = Option.isSome(readableUnit) + ? otherBaseDirOf(readableUnit.value) + : undefined; + if (otherBaseDir !== undefined) { + return yield* new BootServiceOtherBaseDirError({ + installedBaseDir: otherBaseDir, + baseDir: input.baseDir, + }); + } + yield* runSteps(manager.deactivate); + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + signal, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + yield* withSelectionLock( + unitPath, + fs + .remove(unitPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))), + ); + yield* runSteps(manager.finalize); + return true; + } + + const [localUnitInfo, queriedTask] = yield* Effect.all( + [statOptionalPath(unitPath), queryRegisteredWindowsTask()], + { concurrency: "unbounded" }, + ); + const localUnit = + Option.isSome(localUnitInfo) && localUnitInfo.value.type !== "Directory" + ? yield* fs.readFileString(unitPath).pipe(Effect.option) + : Option.none(); + const registeredTaskXml = queriedTask.state === "present" ? queriedTask.xml : undefined; + const effectiveUnit = selectEffectiveWindowsUnit(localUnit, registeredTaskXml); + yield* guardWindowsSnapshot({ queriedTask, effectiveUnit }); + + const hasTaskOrMirror = + queriedTask.state === "present" || + (Option.isSome(localUnitInfo) && localUnitInfo.value.type !== "Directory"); + const controlDir = resolveControlDir(input.baseDir); + const removePersistedTaskName = + taskNamePath === "" + ? Effect.void + : fs + .remove(taskNamePath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + if (!hasTaskOrMirror) { + const launcherState = yield* Effect.tryPromise({ + try: (signal) => + probeServiceLauncher(controlDir, { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + if (launcherState === "stopped") { + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + controlDir, + signal, + acl: host.acl, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + yield* removePersistedTaskName; + return false; + } + if (runningInsideService) { + if (launcherState === "unhealthy") { + return yield* new BootServiceCommandError({ + step: "stopping the installed service", + }); + } + yield* signalCurrentWindowsLauncherForUninstall(controlDir); + } else { + yield* stopWindowsLauncher(); + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + controlDir, + signal, + acl: host.acl, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + } + yield* removePersistedTaskName; + return true; + } + + if (runningInsideService) { + const launcherState = yield* Effect.tryPromise({ + try: (signal) => + probeServiceLauncher(controlDir, { + signal, + acl: host.acl, + }), + catch: (cause) => + new BootServiceCommandError({ step: "stopping the installed service", cause }), + }); + if (launcherState === "unhealthy") { + return yield* new BootServiceCommandError({ + step: "stopping the installed service", + }); + } + // Remove the local mirror under its selection lease before `/Delete` so an in-flight + // `#repointServiceUnit` finishes first and any later one sees ENOENT before signaling stop. + yield* runSteps([{ step: "disabling the scheduled task", run: disableWindowsTask() }]); + yield* withSelectionLock( + unitPath, + fs + .remove(unitPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))), + ); + yield* runSteps([{ step: "removing the scheduled task", run: deleteWindowsTask() }]); + if (launcherState === "running") { + yield* signalCurrentWindowsLauncherForUninstall(controlDir); + } else { + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + controlDir, + signal, + acl: host.acl, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + } + yield* removePersistedTaskName; + return true; + } + + yield* runSteps(manager.deactivate); + yield* Effect.tryPromise({ + try: (signal) => + recoverServiceLauncherUninstall({ + baseDir: input.baseDir, + controlDir, + signal, + acl: host.acl, + }), + catch: (cause) => new BootServiceInstallError({ cause }), + }); + yield* withSelectionLock( + unitPath, + fs + .remove(unitPath, { force: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))), + ); + yield* removePersistedTaskName; + yield* runSteps(manager.finalize); + return true; + }), + (cause) => new BootServiceInstallError({ cause }), + ); }).pipe(Effect.withSpan("cloud.boot_service.uninstall")); const status: BootService["Service"]["status"] = Effect.gen(function* () { if (detectedManager === undefined) { return { supported: false, installed: false, current: false, unitPath, logPath }; } - if (!(yield* fs.exists(unitPath))) { + const snapshot = yield* readInstalledUnitSnapshot(); + if ( + detectedManager.kind === "schtasks" && + snapshot.queriedTask.state === "failed" && + Option.isNone(snapshot.effectiveUnit) + ) { + return yield* new BootServiceCommandError({ step: "inspecting the scheduled task" }); + } + if (Option.isNone(snapshot.effectiveUnit)) { return { supported: true, installed: false, current: false, unitPath, logPath }; } - const [unit, runtimeEntryExists, runtimeSentinel, stateText] = yield* Effect.all([ - fs.readFileString(unitPath), - fs.exists(runtimePaths.entryPath), - fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), - fs.readFileString(statePath).pipe(Effect.option), + const unit = snapshot.effectiveUnit.value; + const [runtimeEntryExists, runtimeSentinel, stateText] = yield* Effect.all([ + fs + .exists(runtimePaths.entryPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))), + readOptionalFileString(runtimePaths.sentinelPath), + readOptionalFileString(statePath), ]); const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; const installedVersion = Option.isSome(stateText) @@ -1104,8 +2330,25 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ? contents.replace(/(PATH<\/key>\n\s*)[^<]*(<\/string>)/, "$1$2") : contents; const problems: BootServiceProblem[] = - detectedManager.kind === "systemd" ? [...(yield* readSystemdProblems(true))] : []; - if (yield* fs.exists(restartPendingPath)) problems.push("restart-pending"); + detectedManager.kind === "systemd" + ? [...(yield* readSystemdProblems(true))] + : detectedManager.kind === "schtasks" + ? [...(yield* readSchtasksProblems(Option.some(unit), snapshot.queriedTask))] + : []; + if ( + yield* fs + .exists(restartPendingPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))) + ) { + problems.push("restart-pending"); + } + const registeredWindowsTaskCurrent = + detectedManager.kind !== "schtasks" || + snapshot.registeredTaskXml === undefined || + !windowsTaskHasActions(snapshot.registeredTaskXml) || + (installedBaseDir !== undefined && + sameBaseDir(installedBaseDir, input.baseDir) && + bootServiceProgramOf(snapshot.registeredTaskXml) === runtimePaths.entryPath); return { supported: true, installed: true, @@ -1115,7 +2358,9 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { problems, current: problems.length === 0 && - normalizeUnit(unit) === normalizeUnit(detectedManager.render(plan)) && + Option.isSome(snapshot.localUnit) && + normalizeUnit(snapshot.localUnit.value) === normalizeUnit(detectedManager.render(plan)) && + registeredWindowsTaskCurrent && runtimeEntryExists && Option.isSome(runtimeSentinel) && runtimeSentinel.value.trim() === input.cliVersion && @@ -1124,10 +2369,7 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { unitPath, logPath, }; - }).pipe( - Effect.mapError((cause) => new BootServiceInstallError({ cause })), - Effect.withSpan("cloud.boot_service.status"), - ); + }).pipe(Effect.withSpan("cloud.boot_service.status")); return BootService.of({ install, restart, withStopped, uninstall, status }); }); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 6be4288ce35e4..dc24840487277 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -1,5 +1,6 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; +import { cliArchiveFileName } from "@t3tools/shared/cliRelease"; import type { ManagedPolicyDocument } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; @@ -25,8 +26,10 @@ import * as ServerSelfUpdate from "./selfUpdate.ts"; interface HarnessOptions { readonly mode?: "web" | "desktop"; + readonly platform?: NodeJS.Platform; readonly managed?: boolean; readonly preflight?: "ready" | "blocked"; + readonly launcherHelpStdout?: string; readonly preflightDuration?: Duration.Input; readonly preflightStarted?: Deferred.Deferred; readonly beforeDownload?: Effect.Effect; @@ -48,7 +51,13 @@ const releaseHttpClient = (order: string[], urls: string[], beforeDownload?: Eff ).join(""); return HttpClientResponse.fromWeb( request, - new Response(`${hex} t3-1.1.0-linux-x64.tar.gz\n`), + new Response( + [ + `${hex} ${cliArchiveFileName("1.1.0", "linux-x64")}`, + `${hex} ${cliArchiveFileName("1.1.0", "win32-x64")}`, + "", + ].join("\n"), + ), ); } if (beforeDownload) yield* beforeDownload; @@ -62,17 +71,23 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const platform = options.platform ?? "linux"; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); const order: string[] = []; const urls: string[] = []; const runner = ProcessRunner.ProcessRunner.of({ run: (input) => Effect.gen(function* () { - if (input.command === "tar") { + if (input.command === "tar" || input.command.toLowerCase().endsWith("tar.exe")) { order.push("extract"); const stagingDir = input.args[input.args.indexOf("-C") + 1]; if (stagingDir === undefined) return yield* Effect.die("missing tar target"); - yield* fs.writeFileString(path.join(stagingDir, "t3"), "#!/bin/sh\n").pipe(Effect.orDie); + yield* fs + .writeFileString( + path.join(stagingDir, platform === "win32" ? "t3.exe" : "t3"), + "#!/bin/sh\n", + ) + .pipe(Effect.orDie); return { stdout: "", stderr: "", @@ -84,6 +99,21 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( stderrInvalidUtf8: false, }; } + if (input.args.includes("__service-launcher") && input.args.includes("--help")) { + order.push("launcher-help"); + return { + stdout: + options.launcherHelpStdout ?? + "Options:\n --base-dir \n --log-file \n --service-unit \n", + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, + }; + } if (options.preflightStarted) yield* Deferred.succeed(options.preflightStarted, undefined); if (options.preflightDuration !== undefined) { // Like the real runner, fail once the caller's timeout elapses. @@ -148,7 +178,7 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( HttpClient.HttpClient, options.httpClient ?? releaseHttpClient(order, urls, options.beforeDownload), ), - Effect.provideService(HostProcess.Platform, "linux"), + Effect.provideService(HostProcess.Platform, platform), Effect.provideService(HostProcess.Architecture, "x64"), Effect.provide( Layer.merge( @@ -211,6 +241,25 @@ it.layer(NodeServices.layer)("server self update", (it) => { }), ); + it.effect("requires Windows __service-launcher CLI flags before accepting a remote update", () => + Effect.gen(function* () { + const legacy = yield* makeHarness({ + platform: "win32", + launcherHelpStdout: "Options:\n --help\n", + }); + expect( + (yield* legacy.selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason, + ).toBe("t3@1.1.0 does not support the Windows background service launcher."); + expect(legacy.order).toEqual(["download", "extract", "preflight", "launcher-help"]); + + const modern = yield* makeHarness({ platform: "win32" }); + expect((yield* modern.selfUpdate.update({ targetVersion: "1.1.0" })).updateId).toBe( + "launcher-id", + ); + expect(modern.order).toEqual(["download", "extract", "preflight", "launcher-help", "accept"]); + }), + ); + it.effect("refuses every update when the managed policy turns updates off", () => Effect.gen(function* () { const policy: ManagedPolicyDocument = { diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 5e299964138da..cce856875c529 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -270,14 +270,45 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { }), ); } - return preflight.status === "ready" - ? Effect.void - : Effect.fail( - new PinnedRuntimePreflightBlockedError({ - version: targetVersion, - reason: preflight.reason, - }), - ); + if (preflight.status !== "ready") { + return Effect.fail( + new PinnedRuntimePreflightBlockedError({ + version: targetVersion, + reason: preflight.reason, + }), + ); + } + if (platform !== "win32") return Effect.void; + // Pre-Windows-service builds pass protocol-4 preflight yet lack the Task Scheduler + // launcher flags needed after unit repointing at the next sign-in. + return runner + .run({ + command: pinnedRuntimeCommand(runtime).command, + args: [...pinnedRuntimeCommand(runtime).args, "__service-launcher", "--help"], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "verifying Windows service launcher support", + cause, + }), + ), + Effect.flatMap((helpResult) => + helpResult.code === 0 && + helpResult.stdout.includes("--base-dir") && + helpResult.stdout.includes("--log-file") && + helpResult.stdout.includes("--service-unit") + ? Effect.void + : Effect.fail( + new PinnedRuntimePreflightBlockedError({ + version: targetVersion, + reason: `t3@${targetVersion} does not support the Windows background service launcher.`, + }), + ), + ), + ); }, ), ), diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index f60b8bb9cf886..587d93be3289e 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -12,13 +12,15 @@ import { import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; class FakeLauncherProcess { - readonly connected: boolean; + connected: boolean; + readonly ipc: boolean; readonly env: Record; readonly sent: ServiceLauncherChildMessage[] = []; readonly #listeners = new Map) => void>>(); - constructor(context: unknown, connected = true) { + constructor(context: unknown, connected = true, ipc = connected) { this.connected = connected; + this.ipc = ipc; this.env = { [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }; } @@ -41,6 +43,11 @@ class FakeLauncherProcess { emit(message: ServiceLauncherParentMessage) { for (const listener of this.#listeners.get("message") ?? []) listener(message); } + + disconnect() { + this.connected = false; + for (const listener of this.#listeners.get("disconnect") ?? []) listener(); + } } const makeClient = (host: FakeLauncherProcess, currentVersion: string) => @@ -120,6 +127,79 @@ it.effect("preserves a launcher rejection as a distinct error", () => }), ); +it.effect("stops the server when the launcher asks over IPC or disconnects", () => + Effect.gen(function* () { + const context = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + childVersion: "1.0.0", + }; + const host = new FakeLauncherProcess(context); + const stopped = yield* Effect.forkChild( + ServiceLauncherClient.awaitServiceLauncherStop({ currentVersion: "1.0.0" }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, host), + Effect.provideService(HostProcess.Environment, host.env), + ), + { startImmediately: true }, + ); + host.emit({ type: "stop" }); + yield* Fiber.join(stopped); + + const crashedHost = new FakeLauncherProcess(context); + const crashed = yield* Effect.forkChild( + ServiceLauncherClient.awaitServiceLauncherStop({ currentVersion: "1.0.0" }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, crashedHost), + Effect.provideService(HostProcess.Environment, crashedHost.env), + ), + { startImmediately: true }, + ); + crashedHost.disconnect(); + yield* Fiber.join(crashed); + + const preDisconnectedHost = new FakeLauncherProcess(context, false, true); + yield* ServiceLauncherClient.awaitServiceLauncherStop({ currentVersion: "1.0.0" }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, preDisconnectedHost), + Effect.provideService(HostProcess.Environment, preDisconnectedHost.env), + ); + + // Mirrors the default `ServiceLauncherHostProcess` reference: resolved while `process.connected` + // is true, then IPC disconnects before `awaitServiceLauncherStop` or `requestUpdate` subscribes. + let liveConnected = true; + const getterBackedHost: (typeof ServiceLauncherClient.ServiceLauncherHostProcess)["Service"] = { + get connected() { + return liveConnected; + }, + ipc: true, + send: (_message: ServiceLauncherChildMessage, callback?: (error: Error | null) => void) => { + callback?.(null); + return false; + }, + on: () => undefined, + off: () => undefined, + }; + const client = yield* ServiceLauncherClient.make({ currentVersion: "1.0.0" }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, getterBackedHost), + Effect.provideService(HostProcess.Environment, { + [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context), + }), + ); + liveConnected = false; + yield* ServiceLauncherClient.awaitServiceLauncherStop({ currentVersion: "1.0.0" }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, getterBackedHost), + Effect.provideService(HostProcess.Environment, { + [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context), + }), + ); + expect( + yield* client + .requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }) + .pipe(Effect.flip), + ).toMatchObject({ + _tag: "ServiceLauncherClientError", + operation: "disconnect", + }); + }), +); + it.effect("rejects contradictory trial context instead of leaving activation closed", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts index 6952c0085c392..5f20b0905a161 100644 --- a/apps/server/src/cloud/serviceLauncherClient.ts +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -3,6 +3,7 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import packageJson from "../../package.json" with { type: "json" }; @@ -60,6 +61,7 @@ export class ServiceLauncherRejectedError extends Schema.TaggedError void, @@ -74,11 +76,16 @@ interface ServiceLauncherProcess { ) => void; } +const initialProcessHasIpc = typeof process.send === "function"; + export const ServiceLauncherHostProcess = Context.Reference( "t3/cloud/serviceLauncherHostProcess", { defaultValue: () => ({ - connected: process.connected && process.send !== undefined, + get connected() { + return process.connected === true && typeof process.send === "function"; + }, + ipc: initialProcessHasIpc, send: (message, callback) => { if (process.send === undefined) return false; return callback === undefined ? process.send(message) : process.send(message, callback); @@ -113,8 +120,9 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup" const host = yield* ServiceLauncherHostProcess; const environment = yield* HostProcess.Environment; const currentVersion = options?.currentVersion ?? packageJson.version; + const hasIpc = host.ipc ?? host.connected; // IPC channels are not inherited; without one, launcher context belongs to an ancestor. - const rawContext = host.connected ? environment[SERVICE_LAUNCHER_CONTEXT_ENV] : undefined; + const rawContext = hasIpc ? environment[SERVICE_LAUNCHER_CONTEXT_ENV] : undefined; const context = rawContext === undefined ? undefined : decodeServiceLauncherContext(rawContext); if (rawContext !== undefined && context === undefined) { @@ -137,6 +145,36 @@ export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_clie }, ); +/** Completes when the launcher asks this server to stop or disconnects, including during startup. */ +export const awaitServiceLauncherStop = Effect.fn("cloud.service_launcher_client.await_stop")( + function* (options?: { readonly currentVersion?: string }) { + const startup = yield* resolveStartup(options).pipe(Effect.option); + if (Option.isNone(startup) || !startup.value.managed) return yield* Effect.never; + const { host } = startup.value; + if (!host.connected) return; + return yield* Effect.callback((resume) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + host.off("message", onMessage); + host.off("disconnect", finish); + resume(Effect.void); + }; + const onMessage = (...args: ReadonlyArray) => { + if (decodeServiceLauncherParentMessage(args[0])?.type === "stop") finish(); + }; + host.on("message", onMessage); + host.on("disconnect", finish); + if (!host.connected) finish(); + return Effect.sync(() => { + host.off("message", onMessage); + host.off("disconnect", finish); + }); + }); + }, +); + export const make = Effect.fn("cloud.service_launcher_client.make")(function* (options?: { readonly currentVersion?: string; }) { @@ -151,6 +189,10 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o resume(Effect.fail(new ServiceLauncherClientError({ operation: "unmanaged" }))); return; } + if (!host.connected) { + resume(Effect.fail(new ServiceLauncherClientError({ operation: "disconnect" }))); + return; + } let settled = false; const cleanup = () => { diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 8958c945aab84..3224f047a957a 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -14,6 +14,8 @@ export const SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; /** Marks a repointed unit whose service still runs the old version; launcher startup clears it. */ export const SERVICE_RESTART_PENDING_FILE = ".restart-pending"; +/** Durable record of the last server-owner lock holder and whether it was a managed update trial. */ +export const SERVER_OWNER_LEASE_FILE = ".server-owner-lease.json"; const unquoteSystemdValue = (raw: string) => (raw.startsWith('"') && raw.endsWith('"') @@ -21,24 +23,125 @@ const unquoteSystemdValue = (raw: string) => : raw ).replaceAll("%%", "%"); +const XML_NAMED_ENTITIES: Readonly> = { + quot: '"', + apos: "'", + lt: "<", + gt: ">", + amp: "&", +}; + const unescapeXmlText = (value: string) => - value.replaceAll("<", "<").replaceAll(">", ">").replaceAll("&", "&"); + value.replaceAll( + /&(?:(quot|apos|lt|gt|amp)|#(\d+)|#x([0-9a-fA-F]+));/g, + (match, named?: string, dec?: string, hex?: string) => { + if (named !== undefined) return XML_NAMED_ENTITIES[named] ?? match; + const codePoint = + dec !== undefined ? Number.parseInt(dec, 10) : Number.parseInt(hex ?? "", 16); + return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff + ? String.fromCodePoint(codePoint) + : match; + }, + ); + +// Task Scheduler passes `` fields through `ExpandEnvironmentStringsW` with no escape syntax. +export const hasWindowsTaskEnvironmentExpansion = (value: string): boolean => + /%[^%\r\n]+%/.test(value); + +function parseWindowsCommandLine(commandLine: string): ReadonlyArray { + const args: string[] = []; + let index = 0; + while (index < commandLine.length) { + while (index < commandLine.length && /\s/.test(commandLine[index]!)) { + index += 1; + } + if (index >= commandLine.length) break; + let current = ""; + let inQuotes = false; + while (index < commandLine.length) { + const char = commandLine[index]!; + if (char === "\\") { + let slashes = 0; + while (index < commandLine.length && commandLine[index] === "\\") { + slashes += 1; + index += 1; + } + if (commandLine[index] === '"') { + current += "\\".repeat(Math.floor(slashes / 2)); + if (slashes % 2 === 0) { + if (inQuotes && commandLine[index + 1] === '"') { + current += '"'; + index += 2; + } else { + inQuotes = !inQuotes; + index += 1; + } + } else { + current += '"'; + index += 1; + } + } else { + current += "\\".repeat(slashes); + } + continue; + } + if (char === '"') { + if (inQuotes && commandLine[index + 1] === '"') { + current += '"'; + index += 2; + } else { + inQuotes = !inQuotes; + index += 1; + } + continue; + } + if (!inQuotes && /\s/.test(char)) break; + current += char; + index += 1; + } + args.push(current); + } + return args; +} + +function readTaskArguments(contents: string): ReadonlyArray | undefined { + const actionsScope = /]*>([\s\S]*?)<\/Actions>/i.exec(contents)?.[1] ?? contents; + const match = /([\s\S]*?)<\/Arguments>/i.exec(actionsScope)?.[1]; + return match === undefined ? undefined : parseWindowsCommandLine(unescapeXmlText(match)); +} export function serviceUnitBaseDirOf(contents: string): string | undefined { const systemd = /^Environment=T3CODE_HOME=(.*)$/m.exec(contents)?.[1]; if (systemd !== undefined) return unquoteSystemdValue(systemd.trim()); const plist = /T3CODE_HOME<\/key>\s*([^<]*)<\/string>/.exec(contents)?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); + if (plist !== undefined) return unescapeXmlText(plist); + const taskArgs = readTaskArguments(contents); + if (taskArgs === undefined) return undefined; + const flagIndex = taskArgs.indexOf("--base-dir"); + const rawBaseDir = + flagIndex >= 0 && flagIndex + 1 < taskArgs.length + ? taskArgs[flagIndex + 1] + : taskArgs.find((arg) => arg.startsWith("--base-dir="))?.slice("--base-dir=".length); + return rawBaseDir !== undefined && !hasWindowsTaskEnvironmentExpansion(rawBaseDir) + ? rawBaseDir + : undefined; } -/** The executable an installed unit or plist starts: the launcher's runtime. */ +/** The executable an installed unit, plist, or task starts: the launcher's runtime. */ export function serviceUnitProgramOf(contents: string): string | undefined { const systemd = /^ExecStart=("(?:\\.|[^"\\])*"|\S+)/m.exec(contents)?.[1]; if (systemd !== undefined) return unquoteSystemdValue(systemd); const plist = /ProgramArguments<\/key>\s*\s*([^<]*)<\/string>/.exec( contents, )?.[1]; - return plist === undefined ? undefined : unescapeXmlText(plist); + if (plist !== undefined) return unescapeXmlText(plist); + const taskArgs = readTaskArguments(contents); + if (taskArgs === undefined) return undefined; + const headlessIndex = taskArgs.indexOf("--headless"); + const program = headlessIndex >= 0 ? taskArgs[headlessIndex + 1] : undefined; + return program !== undefined && !hasWindowsTaskEnvironmentExpansion(program) + ? program + : undefined; } export interface PendingServiceUpdate { @@ -86,6 +189,10 @@ export type ServiceLauncherParentMessage = | { readonly type: "committed"; readonly updateId: string; + } + | { + /** Shut down as on SIGTERM, which Windows cannot deliver. */ + readonly type: "stop"; }; const SEMVER_NUMBER = "(?:0|[1-9]\\d*)"; @@ -97,6 +204,13 @@ const EXACT_SERVICE_VERSION = new RegExp( export const isExactServiceVersion = (version: string): boolean => EXACT_SERVICE_VERSION.test(version); +export const isValidServiceUpdateId = (id: string): boolean => + /^[A-Za-z0-9._-]{1,128}$/.test(id) && id !== "." && id !== ".."; + +const isAbsoluteServiceDatabasePath = (dbPath: string): boolean => + !dbPath.includes("\0") && + (dbPath.startsWith("/") || dbPath.startsWith("\\\\") || /^[A-Za-z]:[\\/]/.test(dbPath)); + const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); @@ -105,7 +219,7 @@ function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { const { id, fromVersion, targetVersion, status } = value; if ( typeof id !== "string" || - id.trim() === "" || + !isValidServiceUpdateId(id) || typeof fromVersion !== "string" || !isExactServiceVersion(fromVersion) || typeof targetVersion !== "string" || @@ -114,7 +228,7 @@ function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { return undefined; } if (status === "pending") { - return typeof value.dbPath === "string" && value.dbPath.trim() !== "" + return typeof value.dbPath === "string" && isAbsoluteServiceDatabasePath(value.dbPath) ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status } : undefined; } @@ -284,6 +398,7 @@ export function decodeServiceLauncherParentMessage( if (value.type === "update-accepted" && typeof value.updateId === "string") { return { type: value.type, updateId: value.updateId }; } + if (value.type === "stop") return { type: value.type }; return value.type === "committed" && typeof value.updateId === "string" ? { type: value.type, updateId: value.updateId } : undefined; diff --git a/apps/server/src/cloud/serviceSelectionLock.ts b/apps/server/src/cloud/serviceSelectionLock.ts index 4cbd123c22f7e..96c6751a2a552 100644 --- a/apps/server/src/cloud/serviceSelectionLock.ts +++ b/apps/server/src/cloud/serviceSelectionLock.ts @@ -21,7 +21,11 @@ export async function acquireServiceSelectionLock(selectionPath: string): Promis try { NodeFS.rmdirSync(lock); } catch (error) { - if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes((error as NodeJS.ErrnoException).code ?? "")) + if ( + !["ENOENT", "ENOTEMPTY", "EEXIST", "EBUSY"].includes( + (error as NodeJS.ErrnoException).code ?? "", + ) + ) throw error; } }; @@ -37,14 +41,24 @@ export async function acquireServiceSelectionLock(selectionPath: string): Promis removeEmptyLock(); }; } catch (error) { - if (!["EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) - throw error; + const code = (error as NodeJS.ErrnoException).code ?? ""; + const windowsContention = + (code === "EPERM" || code === "EACCES") && NodeFS.existsSync(lock); + if (!["EEXIST", "ENOTEMPTY"].includes(code) && !windowsContention) throw error; } let owners: string[]; try { owners = NodeFS.readdirSync(lock); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + if (Date.now() >= deadline) + throw new Error( + `Service selection is locked at ${lock}. Retry when the other operation finishes.`, + { cause: error }, + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + continue; + } throw error; } if (owners.length === 1) { diff --git a/apps/server/src/serverOwnership.ts b/apps/server/src/serverOwnership.ts index 8644b7fd65c47..360b3509c0bc8 100644 --- a/apps/server/src/serverOwnership.ts +++ b/apps/server/src/serverOwnership.ts @@ -5,6 +5,7 @@ import * as NodeSqlite from "node:sqlite"; import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -21,10 +22,12 @@ import { } from "./cloud/bootService.ts"; import { ServiceLauncherHostProcess } from "./cloud/serviceLauncherClient.ts"; import { + SERVER_OWNER_LEASE_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, SERVICE_RESTART_PENDING_FILE, SERVICE_STATE_FILE, compareExactServiceVersions, + decodeServiceLauncherContext, serviceStateActiveVersion, } from "./cloud/serviceProtocol.ts"; import * as ProcessRunner from "./processRunner.ts"; @@ -73,6 +76,15 @@ const isSqliteBusy = (cause: unknown) => (("errcode" in cause && cause.errcode === 5) || ("code" in cause && cause.code === "SQLITE_BUSY")); +const persistServerOwnerLease = ( + filePath: string, + lease: { readonly ownerId: string; readonly pendingUpdateId?: string }, +) => + writeFileStringAtomically({ + filePath, + contents: `${JSON.stringify(lease)}\n`, + }); + // Pre-lock servers publish only runtime state; require start-time proof of PID reuse before treating it as stale. const legacyOwnerIsLive = Effect.fn("legacyOwnerIsLive")(function* ( state: PersistedServerRuntimeState, @@ -256,6 +268,17 @@ export const acquireServerOwnership = Effect.fn("acquireServerOwnership")(functi if (olderServiceHome !== undefined) { return yield* new OlderBootServiceError({ baseDir: olderServiceHome }); } + const rawLauncherContext = (yield* HostProcess.Environment)[SERVICE_LAUNCHER_CONTEXT_ENV]; + const launcherContext = + (yield* ServiceLauncherHostProcess).connected && rawLauncherContext !== undefined + ? decodeServiceLauncherContext(rawLauncherContext) + : undefined; + const pendingUpdateId = + launcherContext?.update?.status === "pending" ? launcherContext.update.id : undefined; + yield* persistServerOwnerLease(NodePath.join(stateDir, SERVER_OWNER_LEASE_FILE), { + ownerId, + ...(pendingUpdateId === undefined ? {} : { pendingUpdateId }), + }).pipe(Effect.mapError((cause) => new ServerOwnershipError({ stateDir, cause }))); yield* Effect.gen(function* () { let lost = lock.replacedDuringAcquire; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 3304211d9e7c6..7c2a614b01a9f 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1,5 +1,5 @@ // @effect-diagnostics-next-line nodeBuiltinImport:off - observes the children the launcher spawns with Node built-ins. -import type * as NodeChildProcess from "node:child_process"; +import * as NodeChildProcess from "node:child_process"; // @effect-diagnostics-next-line nodeBuiltinImport:off - the free-space probe is a plain Promise callback. import * as NodeFS from "node:fs"; // @effect-diagnostics-next-line nodeBuiltinImport:off - models the service manager reading disk through the launcher's Promise callback. @@ -7,38 +7,79 @@ import * as NodeFSP from "node:fs/promises"; import * as NodeSqlite from "node:sqlite"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; +import { assert, describe, it } from "@effect/vitest"; import { SERVER_EXIT_CODE_STATE_DIR_OWNED } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; +import type * as Scope from "effect/Scope"; import { vi } from "vite-plus/test"; -import { renderBootServicePlist, renderBootServiceUnit } from "./cloud/bootService.ts"; +import { + renderBootServicePlist, + renderBootServiceTask, + renderBootServiceUnit, +} from "./cloud/bootService.ts"; import { databaseBackupBlocker, + isPrivateWindowsControlSddl, + isServiceControlLauncherMismatchError, + isServiceControlUnauthorizedError, Launcher, + main, + probeControlLockHeld, + probeServiceLauncher, + queryServiceLauncherId, readServiceEnvironment, + recoverServiceLauncherUninstall, type BackupSpaceProbe, + type ControlAclHooks, readServiceState, + requestServiceLauncherStop, + serviceControlDir, + serviceTaskName, + serviceTaskNamePath, + signalServiceLauncherStop, + stageServiceLauncherInstall, + waitForServiceLauncherRelease, + windowsSystem32Binary, writeServiceState, } from "./serviceLauncher.ts"; +import { ServiceLauncherClientError } from "./cloud/serviceLauncherClient.ts"; import { compareExactServiceVersions, decodeServiceState, isExactServiceVersion, + SERVER_OWNER_LEASE_FILE, SERVICE_ENV_FILE, SERVICE_LAUNCHER_PROTOCOL, SERVICE_RESTART_PENDING_FILE, + SERVICE_STOP_MARKER_FILE, + serviceUnitBaseDirOf, } from "./cloud/serviceProtocol.ts"; +const trustedTestAcl: ControlAclHooks = { + restrictPathAcl: async () => undefined, + verifyPathAcl: async () => true, +}; + const spawnedChildren = vi.hoisted((): Array => []); vi.mock("node:child_process", async (importOriginal) => { const actual = await importOriginal(); const spawn = (...args: Parameters) => { - const child = actual.spawn(...args); + const [command, spawnArgs, spawnOptions] = args; + const child = + (command.endsWith("\\t3.exe") || command.endsWith("/t3.exe")) && Array.isArray(spawnArgs) + ? actual.spawn( + process.execPath, + [command, ...(spawnArgs as ReadonlyArray)], + (spawnOptions as NodeChildProcess.SpawnOptions | undefined) ?? {}, + ) + : actual.spawn(...args); spawnedChildren.push(child); return child; }; @@ -112,13 +153,164 @@ it("rejects contradictory service state", () => { }, }), ); + + assert.isUndefined( + decodeServiceState({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "../escape", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: "/tmp/state.sqlite", + status: "pending", + }, + }), + ); + + assert.isUndefined( + decodeServiceState({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-relative", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: "relative/state.sqlite", + status: "pending", + }, + }), + ); +}); + +it("unescapes Windows task XML entities in a single pass and rejects environment expansion", () => { + const literalAmpDir = "C:\\Users\\R&D\\.t3"; + const rendered = renderBootServiceTask( + { + program: [`${literalAmpDir}\\runtime\\versions\\1.0.0\\t3.exe`, "__service-launcher"], + baseDir: literalAmpDir, + logPath: `${literalAmpDir}\\logs\\boot-service.log`, + unitPath: `${literalAmpDir}\\unit\\t3code.xml`, + }, + { + homeDir: "C:\\Users\\R&D", + userId: "PC\\theo", + conhostPath: "C:\\Windows\\System32\\conhost.exe", + }, + ); + assert.equal(serviceUnitBaseDirOf(rendered), literalAmpDir); + assert.equal( + serviceUnitBaseDirOf(rendered.replaceAll("&", "&")), + literalAmpDir, + "numeric &amp; emitted by Task Scheduler must decode to literal &, not &", + ); + assert.isUndefined( + serviceUnitBaseDirOf( + '--headless -- "C:\\t3.exe" __service-launcher --base-dir "%LOCALAPPDATA%\\t3"', + ), + ); +}); + +it("disambiguates Windows task names, System32 paths, and account-private control directories", () => { + assert.notEqual(serviceTaskName("a-b", "c"), serviceTaskName("a", "b-c")); + assert.notEqual(serviceTaskName("", "a-b"), serviceTaskName("a", "b")); + assert.equal(serviceTaskName("PC", "Theo").slice(-8), serviceTaskName("pc", "theo").slice(-8)); + assert.equal( + windowsSystem32Binary("schtasks.exe", { SystemRoot: "D:\\Win" }), + "D:\\Win\\System32\\schtasks.exe", + ); + assert.equal( + serviceControlDir("/local", "/Homes/Theo/.T3", { caseInsensitive: true }), + serviceControlDir("/local", "/homes/theo/.t3", { caseInsensitive: true }), + ); + assert.notEqual( + serviceControlDir("/local", "/Homes/Theo/.T3", { caseInsensitive: false }), + serviceControlDir("/local", "/homes/theo/.t3", { caseInsensitive: false }), + ); +}); + +it("verifies owner-bound, protected Windows SDDL descriptors for files and directories", () => { + const userSid = "S-1-5-21-100-200-300-1001"; + const otherSid = "S-1-5-21-100-200-300-9999"; + assert.isTrue( + isPrivateWindowsControlSddl( + `O:${userSid}G:${userSid}D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;${userSid})S:AI(ML;;NW;;;HI)`, + userSid, + ), + ); + assert.isTrue( + isPrivateWindowsControlSddl( + `O:BAG:BAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;${userSid})`, + userSid, + { isDirectory: true }, + ), + ); + assert.isFalse( + isPrivateWindowsControlSddl( + `O:${userSid}D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;${userSid})`, + userSid, + { isDirectory: true }, + ), + "directory user ACE without OI+CI inheritance flags must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl( + `O:${otherSid}G:${otherSid}D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;${userSid})`, + userSid, + ), + "foreign file owner must be rejected even when DACL grants only trusted SIDs", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`O:${userSid}D:P(A;;FA;;;OW)(A;;FA;;;${userSid})`, userSid), + "OWNER RIGHTS (OW) trustee must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`O:${userSid}D:P(A;;FA;;;S-1-3-4)(A;;FA;;;${userSid})`, userSid), + "OWNER RIGHTS (S-1-3-4) trustee must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`D:P(A;;FA;;;SY)(A;;FA;;;BA)(A;;FA;;;${userSid})`, userSid), + "SDDL missing an O: owner section must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`O:${userSid}D:AI(A;;FA;;;SY)(A;;FA;;;${userSid})`, userSid), + "inherited DACL without D:P must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`O:${userSid}D:PAI(A;;FA;;;BU)(A;;FA;;;${userSid})`, userSid), + "Users group allow ACE must be rejected", + ); + assert.isFalse( + isPrivateWindowsControlSddl(`O:${userSid}D:PAI(A;;FA;;;SY)(A;;FA;;;BA)`, userSid), + "DACL missing an explicit current-user full-control ACE must be rejected", + ); + const localAdminSid = "S-1-5-21-100-200-300-500"; + assert.isTrue( + isPrivateWindowsControlSddl( + "O:LAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LA)", + localAdminSid, + { isDirectory: true, allowLocalAdminAlias: true }, + ), + "built-in local Administrator SDDL alias LA must be accepted when SID matches local LA", + ); + assert.isFalse( + isPrivateWindowsControlSddl( + "O:LAD:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;LA)", + userSid, + { isDirectory: true, allowLocalAdminAlias: false }, + ), + "LA alias must be rejected for non-Administrator accounts", + ); }); -const unitPlan = (root: string, version: string) => ({ - program: [`${root}/runtime/versions/${version}/t3`, "__service-launcher"], +const unitPlan = (root: string, version: string, unitFile = "t3code.service") => ({ + program: [ + `${root}/runtime/versions/${version}/${unitFile.endsWith(".xml") ? "t3.exe" : "t3"}`, + "__service-launcher", + ], baseDir: root, logPath: `${root}/logs/boot-service.log`, - unitPath: `${root}/unit/t3code.service`, + unitPath: `${root}/unit/${unitFile}`, }); const writeFakeRuntime = ( @@ -128,9 +320,12 @@ const writeFakeRuntime = ( childSource: string, ) => Effect.gen(function* () { - const entryPath = path.join(versionDir, "t3"); + const entryPath = path.join(versionDir, path.sep === "\\" ? "t3.exe" : "t3"); yield* fs.makeDirectory(versionDir, { recursive: true }); - yield* fs.writeFileString(entryPath, `#!${process.execPath}\n${childSource}`); + yield* fs.writeFileString( + entryPath, + path.sep === "\\" ? childSource : `#!${process.execPath}\n${childSource}`, + ); yield* fs.chmod(entryPath, 0o755); yield* fs.writeFileString( path.join(versionDir, ".install-complete"), @@ -145,9 +340,10 @@ const fakeServiceUnit = ( root: string, version: string, render = renderBootServiceUnit, + unitFile = "t3code.service", ) => Effect.gen(function* () { - const plan = unitPlan(root, version); + const plan = unitPlan(root, version, unitFile); yield* fs.makeDirectory(path.dirname(plan.unitPath), { recursive: true }); const contents = render(plan); yield* fs.writeFileString(plan.unitPath, contents); @@ -165,236 +361,253 @@ const fakeServiceUnit = ( return unit; }); -it.layer(NodeServices.layer)("service state persistence", (it) => { - it.effect( +const runNode = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("service state persistence", () => { + it.live( "loads optional service.env literally without changing inherited or managed variables", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-env-" }); - const inherited = Object.freeze({ - PATH: "inherited-path", - T3CODE_HOME: root, - T3_BOOT_SERVICE_UNIT: "managed-unit", - T3_SERVICE_LAUNCHER_CONTEXT: "managed-context", - T3CODE_PORT: "ambient", - }); - assert.deepEqual( - yield* Effect.promise(() => readServiceEnvironment(root, inherited)), - inherited, - ); - yield* fs.writeFileString( - path.join(root, SERVICE_ENV_FILE), - [ - "# service configuration", - "export T3CODE_PORT=1234 # inline comment", - 'T3CODE_TOKEN="literal $HOME ${USER} $(whoami) # token"', - 'MULTILINE="first\nsecond"', - "EMPTY=", - "PATH=/bad", - "Path=/bad", - "T3CODE_HOME=/bad", - "t3code_home=/bad", - "T3_BOOT_SERVICE_UNIT=bad", - "t3_boot_service_unit=bad", - "T3_SERVICE_LAUNCHER_CONTEXT=bad", - "t3_service_launcher_context=bad", - ].join("\n"), - ); - assert.deepEqual(yield* Effect.promise(() => readServiceEnvironment(root, inherited)), { - ...inherited, - T3CODE_PORT: "1234", - T3CODE_TOKEN: "literal $HOME ${USER} $(whoami) # token", - MULTILINE: "first\nsecond", - EMPTY: "", - }); - }), - ); - - it.effect("a fresh launcher clears a restart deferred by t3 update", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-restart-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const restartPending = path.join(root, "runtime", SERVICE_RESTART_PENDING_FILE); - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", "1.0.0"), - "setInterval(() => {}, 1_000);\n", - ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const run = () => + runNode( Effect.gen(function* () { - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-env-" }); + const inherited = Object.freeze({ + PATH: "inherited-path", + T3CODE_HOME: root, + T3_BOOT_SERVICE_UNIT: "managed-unit", + T3_SERVICE_LAUNCHER_CONTEXT: "managed-context", + T3CODE_PORT: "ambient", + }); + assert.deepEqual( + yield* Effect.promise(() => readServiceEnvironment(root, inherited)), + inherited, ); - const running = launcher.run(); - yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); - }); - - yield* fs.writeFileString(restartPending, "1.0.1\n"); - yield* run(); - assert.isTrue(yield* fs.exists(restartPending)); - - yield* fs.writeFileString(restartPending, "1.0.0\n"); - yield* run(); - assert.isFalse(yield* fs.exists(restartPending)); - }), + yield* fs.writeFileString( + path.join(root, SERVICE_ENV_FILE), + [ + "# service configuration", + "export T3CODE_PORT=1234 # inline comment", + 'T3CODE_TOKEN="literal $HOME ${USER} $(whoami) # token"', + 'MULTILINE="first\nsecond"', + "EMPTY=", + "PATH=/bad", + "Path=/bad", + "T3CODE_HOME=/bad", + "t3code_home=/bad", + "T3_BOOT_SERVICE_UNIT=bad", + "t3_boot_service_unit=bad", + "T3_SERVICE_LAUNCHER_CONTEXT=bad", + "t3_service_launcher_context=bad", + ].join("\n"), + ); + assert.deepEqual(yield* Effect.promise(() => readServiceEnvironment(root, inherited)), { + ...inherited, + T3CODE_PORT: "1234", + T3CODE_TOKEN: "literal $HOME ${USER} $(whoami) # token", + MULTILINE: "first\nsecond", + EMPTY: "", + }); + }), + ), ); - it.effect.each([ - { unitVersion: "1.0.0", replacement: "an intentional downgrade" }, - { unitVersion: "1.2.0", replacement: "another T3 home" }, - ])( - "recovers a rolled-back update with unit $unitVersion and preserves $replacement", - ({ unitVersion, replacement }) => + it.live("a fresh launcher clears a restart deferred by t3 update", () => + runNode( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-repoint-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-restart-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const restartPending = path.join(root, "runtime", SERVICE_RESTART_PENDING_FILE); yield* writeFakeRuntime( fs, path, - path.join(root, "runtime", "versions", "1.1.0"), + path.join(root, "runtime", "versions", "1.0.0"), "setInterval(() => {}, 1_000);\n", ); yield* Effect.promise(() => writeServiceState(statePath, { protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.1.0", - update: { - id: "update-1", - fromVersion: "1.1.0", - targetVersion: "1.2.0", - status: "rolled-back", - }, + activeVersion: "1.0.0", }), ); - const unit = yield* fakeServiceUnit(fs, path, root, unitVersion); - const expected = renderBootServiceUnit( - unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion), - ); - const createLauncher = Effect.gen(function* () { - return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), { - serviceUnit: unit, - }); - }); - const run = (launcher: Launcher) => + const run = () => Effect.gen(function* () { + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); const running = launcher.run(); yield* Effect.promise(() => launcher.stop("SIGTERM")); yield* Effect.promise(() => running); }); - yield* run(yield* createLauncher); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.loadedContents, expected); + yield* fs.writeFileString(restartPending, "1.0.1\n"); + yield* run(); + assert.isTrue(yield* fs.exists(restartPending)); - yield* run(yield* createLauncher); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.loadedContents, expected); + yield* fs.writeFileString(restartPending, "1.0.0\n"); + yield* run(); + assert.isFalse(yield* fs.exists(restartPending)); + }), + ), + ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.1.0", - update: { - id: "update-2", - fromVersion: "1.0.0", - targetVersion: "1.1.0", - status: "committed", - }, - }), - ); - const staleLauncher = yield* createLauncher; - // A local install can replace the selection or service owner without stopping this launcher. - const selectedRoot = - replacement === "another T3 home" ? path.join(root, "other-home") : root; + it.live.each([ + { unitVersion: "1.0.0", replacement: "an intentional downgrade" }, + { unitVersion: "1.2.0", replacement: "another T3 home" }, + ])( + "recovers a rolled-back update with unit $unitVersion and preserves $replacement", + ({ unitVersion, replacement }) => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "t3-service-launcher-repoint-", + }); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.1.0"), + "setInterval(() => {}, 1_000);\n", + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + status: "rolled-back", + }, + }), + ); + const unit = yield* fakeServiceUnit(fs, path, root, unitVersion); + const expected = renderBootServiceUnit( + unitPlan(root, unitVersion === "1.0.0" ? "1.1.0" : unitVersion), + ); + const createLauncher = Effect.gen(function* () { + return new Launcher(root, yield* Effect.promise(() => readServiceState(statePath)), { + serviceUnit: unit, + }); + }); + const run = (launcher: Launcher) => + Effect.gen(function* () { + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }); + + yield* run(yield* createLauncher); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + + yield* run(yield* createLauncher); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "update-2", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "committed", + }, + }), + ); + const staleLauncher = yield* createLauncher; + // A local install can replace the selection or service owner without stopping this launcher. + const selectedRoot = + replacement === "another T3 home" ? path.join(root, "other-home") : root; + yield* writeFakeRuntime( + fs, + path, + path.join(selectedRoot, "runtime", "versions", "1.0.0"), + "setInterval(() => {}, 1_000);\n", + ); + const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(selectedStatePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const selectedUnit = renderBootServiceUnit({ + ...unitPlan(selectedRoot, "1.0.0"), + unitPath: unit.path, + }); + yield* fs.writeFileString(unit.path, selectedUnit); + yield* Effect.promise(() => unit.reload()); + const oldHomeState = yield* fs.readFileString(statePath); + const selectedState = yield* fs.readFileString(selectedStatePath); + + yield* run(staleLauncher); + assert.equal(yield* fs.readFileString(statePath), oldHomeState); + assert.equal(yield* fs.readFileString(selectedStatePath), selectedState); + assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement); + assert.equal(unit.loadedContents, selectedUnit, replacement); + }), + ), + ); + + it.live("stays idle instead of exiting when another server owns the T3 home", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" }); + const statePath = path.join(root, "runtime", "service-state.json"); yield* writeFakeRuntime( fs, path, - path.join(selectedRoot, "runtime", "versions", "1.0.0"), - "setInterval(() => {}, 1_000);\n", + path.join(root, "runtime", "versions", "1.0.0"), + `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`, ); - const selectedStatePath = path.join(selectedRoot, "runtime", "service-state.json"); yield* Effect.promise(() => - writeServiceState(selectedStatePath, { + writeServiceState(statePath, { protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); - const selectedUnit = renderBootServiceUnit({ - ...unitPlan(selectedRoot, "1.0.0"), - unitPath: unit.path, + const refused = Promise.withResolvers(); + yield* Effect.acquireRelease( + Effect.sync(() => + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + if (String(chunk).includes("owns this T3 home")) refused.resolve(); + return true; + }), + ), + (spy) => Effect.sync(() => spy.mockRestore()), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + let completed = false; + const running = launcher.run().finally(() => { + completed = true; }); - yield* fs.writeFileString(unit.path, selectedUnit); - yield* Effect.promise(() => unit.reload()); - const oldHomeState = yield* fs.readFileString(statePath); - const selectedState = yield* fs.readFileString(selectedStatePath); - - yield* run(staleLauncher); - assert.equal(yield* fs.readFileString(statePath), oldHomeState); - assert.equal(yield* fs.readFileString(selectedStatePath), selectedState); - assert.equal(yield* fs.readFileString(unit.path), selectedUnit, replacement); - assert.equal(unit.loadedContents, selectedUnit, replacement); + yield* Effect.promise(() => refused.promise); + assert.isFalse(completed); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); }), - ); - - it.effect("stays idle instead of exiting when another server owns the T3 home", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-refused-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", "1.0.0"), - `process.exit(${SERVER_EXIT_CODE_STATE_DIR_OWNED});\n`, - ); - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const refused = Promise.withResolvers(); - yield* Effect.acquireRelease( - Effect.sync(() => - vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { - if (String(chunk).includes("owns this T3 home")) refused.resolve(); - return true; - }), - ), - (spy) => Effect.sync(() => spy.mockRestore()), - ); - const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); - let completed = false; - const running = launcher.run().finally(() => { - completed = true; - }); - yield* Effect.promise(() => refused.promise); - assert.isFalse(completed); - yield* Effect.promise(() => launcher.stop("SIGTERM")); - yield* Effect.promise(() => running); - }), + ), ); const commitUpdate = ( render = renderBootServiceUnit, missingUnit = false, legacyInstall = false, + unitFile = "t3code.service", ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -405,7 +618,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const activatedPath = path.join(root, "activated"); yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); yield* fs.writeFileString(databasePath, "before trial"); - const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render); + const unit = yield* fakeServiceUnit(fs, path, root, "1.0.0", render, unitFile); if (missingUnit) yield* fs.remove(unit.path); const childSource = ` import assert from "node:assert/strict"; @@ -489,74 +702,92 @@ if (context.update?.status === "pending") { return { fs, root, unit, recover }; }); - it.effect.each([ + it.live.each([ { name: "systemd after a protocol-3 local upgrade", render: renderBootServiceUnit, legacyInstall: true, + unitFile: "t3code.service", }, { name: "launchd", legacyInstall: false, + unitFile: "com.t3tools.t3code.service.plist", render: (plan: Parameters[0]) => renderBootServicePlist(plan, { homeDir: "/Users/test & ", environmentPath: "/usr/bin:/custom/runtime/versions/1.0.0", }), }, + { + name: "Windows Task Scheduler", + legacyInstall: false, + unitFile: "t3code-task.xml", + render: (plan: Parameters[0]) => + renderBootServiceTask(plan, { + userId: "PC\\theo", + homeDir: "C:\\Users\\theo & ", + conhostPath: "C:\\Windows\\System32\\conhost.exe", + }), + }, ])( "commits and repoints $name only after the trial reports prepared", - ({ render, legacyInstall }) => - Effect.gen(function* () { - const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall); - const expected = render(unitPlan(root, "1.1.0")); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.loadedContents, expected); - assert.equal(unit.reloads, 1); - }), + ({ render, legacyInstall, unitFile }) => + runNode( + Effect.gen(function* () { + const { fs, root, unit } = yield* commitUpdate(render, false, legacyInstall, unitFile); + const expected = render(unitPlan(root, "1.1.0", unitFile)); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 1); + }), + ), ); - it.effect( + it.live( "keeps a durable commit through unit failure and retries a failed reload on restart", () => - Effect.gen(function* () { - const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true); - assert.equal(unit.reloads, 0); - const original = renderBootServiceUnit(unitPlan(root, "1.0.0")); - const expected = renderBootServiceUnit(unitPlan(root, "1.1.0")); - assert.equal(unit.loadedContents, original); - yield* fs.writeFileString(unit.path, original); - unit.failReload = true; - yield* recover(); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.reloads, 1); - assert.equal(unit.loadedContents, original); + runNode( + Effect.gen(function* () { + const { fs, root, unit, recover } = yield* commitUpdate(renderBootServiceUnit, true); + assert.equal(unit.reloads, 0); + const original = renderBootServiceUnit(unitPlan(root, "1.0.0")); + const expected = renderBootServiceUnit(unitPlan(root, "1.1.0")); + assert.equal(unit.loadedContents, original); + yield* fs.writeFileString(unit.path, original); + unit.failReload = true; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.reloads, 1); + assert.equal(unit.loadedContents, original); - unit.failReload = false; - yield* recover(); - assert.equal(yield* fs.readFileString(unit.path), expected); - assert.equal(unit.loadedContents, expected); - assert.equal(unit.reloads, 2); - }), + unit.failReload = false; + yield* recover(); + assert.equal(yield* fs.readFileString(unit.path), expected); + assert.equal(unit.loadedContents, expected); + assert.equal(unit.reloads, 2); + }), + ), ); - it.effect("rolls back invalid trials with the startup environment", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - const observationsPath = path.join(root, "environment-observations.jsonl"); - const envPath = path.join(root, SERVICE_ENV_FILE); - yield* fs.writeFileString(envPath, "T3CODE_TEST_SERVICE_TOKEN=initial\n"); - vi.stubEnv("T3CODE_HOME", root); - vi.stubEnv("T3CODE_TEST_SERVICE_TOKEN", "inherited"); - yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "before trial"); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` + it.live("rolls back invalid trials with the startup environment", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const observationsPath = path.join(root, "environment-observations.jsonl"); + const envPath = path.join(root, SERVICE_ENV_FILE); + yield* fs.writeFileString(envPath, "T3CODE_TEST_SERVICE_TOKEN=initial\n"); + vi.stubEnv("T3CODE_HOME", root); + vi.stubEnv("T3CODE_TEST_SERVICE_TOKEN", "inherited"); + yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` import { appendFileSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); appendFileSync(${JSON.stringify(observationsPath)}, JSON.stringify({ @@ -573,87 +804,89 @@ if (context.update?.status === "pending") { process.exit(0); } `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0"); + const unit = yield* fakeServiceUnit(fs, path, root, "0.9.0"); - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { - serviceUnit: unit, - environment: yield* Effect.promise(() => readServiceEnvironment(root)), - }, - ); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + serviceUnit: unit, + environment: yield* Effect.promise(() => readServiceEnvironment(root)), + }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.0.0"); - assert.equal(state.update?.status, "rolled-back"); - assert.equal( - yield* fs.readFileString(unit.path), - renderBootServiceUnit(unitPlan(root, "1.0.0")), - ); - assert.equal(unit.reloads, 1); - assert.equal( - state.update?.status === "rolled-back" ? state.update.reason : undefined, - "invalid-prepared", - ); - const observations = (yield* fs.readFileString(observationsPath)) - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - assert.deepEqual(observations, [ - { token: "initial", update: "active" }, - { token: "initial", update: "pending" }, - { token: "initial", update: "rolled-back" }, - ]); - assert.equal((yield* HostProcess.Environment).T3CODE_TEST_SERVICE_TOKEN, "inherited"); - assert.equal(yield* fs.readFileString(envPath), "T3CODE_TEST_SERVICE_TOKEN=changed\n"); - }), + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal( + yield* fs.readFileString(unit.path), + renderBootServiceUnit(unitPlan(root, "1.0.0")), + ); + assert.equal(unit.reloads, 1); + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + "invalid-prepared", + ); + const observations = (yield* fs.readFileString(observationsPath)) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + assert.deepEqual(observations, [ + { token: "initial", update: "active" }, + { token: "initial", update: "pending" }, + { token: "initial", update: "rolled-back" }, + ]); + assert.equal((yield* HostProcess.Environment).T3CODE_TEST_SERVICE_TOKEN, "inherited"); + assert.equal(yield* fs.readFileString(envPath), "T3CODE_TEST_SERVICE_TOKEN=changed\n"); + }), + ), ); - it.effect.each(["file", "symlink", "dangling symlink"] as const)( + it.live.each(["file", "symlink", "dangling symlink"] as const)( "restores the database when a migrating trial exits, with the database a %s", (kind) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - const databaseFile = - kind === "file" ? databasePath : path.join(root, "data", "original.sqlite"); - const linkTarget = kind === "dangling symlink" ? "../original.sqlite" : databaseFile; - const original = "database before migration"; - yield* fs.makeDirectory(path.dirname(databaseFile), { recursive: true }); - if (kind === "dangling symlink") { - yield* fs.makeDirectory(path.join(root, "data", "sub")); - yield* fs.symlink(path.join(root, "data", "sub"), path.dirname(databasePath)); - } else { - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - } - yield* fs.writeFileString(databaseFile, original); - if (kind !== "file") yield* fs.symlink(linkTarget, databasePath); - const encodedDatabasePath = JSON.stringify(databasePath); - const childSource = ` + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const databaseFile = + kind === "file" ? databasePath : path.join(root, "data", "original.sqlite"); + const linkTarget = kind === "dangling symlink" ? "../original.sqlite" : databaseFile; + const original = "database before migration"; + yield* fs.makeDirectory(path.dirname(databaseFile), { recursive: true }); + if (kind === "dangling symlink") { + yield* fs.makeDirectory(path.join(root, "data", "sub")); + yield* fs.symlink(path.join(root, "data", "sub"), path.dirname(databasePath)); + } else { + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + } + yield* fs.writeFileString(databaseFile, original); + if (kind !== "file") yield* fs.symlink(linkTarget, databasePath); + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` import { realpathSync, unlinkSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { @@ -669,70 +902,72 @@ if (context.update?.status === "pending") { process.exit(0); } `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - ); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.0.0"); - assert.equal(state.update?.status, "rolled-back"); - assert.equal(yield* fs.readFileString(databaseFile), original); - if (kind !== "file") assert.equal(yield* fs.readLink(databasePath), linkTarget); - assert.isFalse(yield* fs.exists(path.join(root, "original.sqlite"))); - assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); - assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); - const updateId = state.update?.id; - assert.isDefined(updateId); - assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); - }), + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(databaseFile), original); + if (kind !== "file") assert.equal(yield* fs.readLink(databasePath), linkTarget); + assert.isFalse(yield* fs.exists(path.join(root, "original.sqlite"))); + assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); + assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); + const updateId = state.update?.id; + assert.isDefined(updateId); + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); + }), + ), ); - it.effect.each(["wal", "delete"] as const)( + it.live.each(["wal", "delete"] as const)( "backs up a %s-mode database (online when WAL) and restores its final state", (journalMode) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - const observationsPath = path.join(root, "observations.json"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - // SQLite keeps -wal beside the symlink's target, so the backup must restore it there. - yield* fs.symlink("actual.sqlite", databasePath); - const seed = new NodeSqlite.DatabaseSync(databasePath); - seed.exec(` + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-online-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const observationsPath = path.join(root, "observations.json"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + // SQLite keeps -wal beside the symlink's target, so the backup must restore it there. + yield* fs.symlink("actual.sqlite", databasePath); + const seed = new NodeSqlite.DatabaseSync(databasePath); + seed.exec(` PRAGMA journal_mode = ${journalMode}; CREATE TABLE events (id INTEGER PRIMARY KEY, payload TEXT NOT NULL); WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 500) INSERT INTO events (payload) SELECT 'before the update ' || i FROM n; `); - seed.close(); + seed.close(); - const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode }); - const childSource = ` + const encoded = JSON.stringify({ root, databasePath, observationsPath, journalMode }); + const childSource = ` import { readdirSync, writeFileSync } from "node:fs"; import { DatabaseSync } from "node:sqlite"; const paths = ${encoded}; @@ -751,7 +986,10 @@ if (context.update?.status === "pending") { db.exec("PRAGMA journal_mode = " + paths.journalMode + ";"); const insert = db.prepare("INSERT INTO events (payload) VALUES (?)"); const writer = setInterval(() => insert.run("written while serving"), 1); - process.once("SIGTERM", () => { + let stopping = false; + const finishServing = () => { + if (stopping) return; + stopping = true; clearInterval(writer); insert.run("last write before stopping"); const backupRoot = paths.root + "/runtime/db-backup"; @@ -761,71 +999,76 @@ if (context.update?.status === "pending") { })); db.close(); process.exit(0); + }; + process.once("SIGTERM", finishServing); + process.on("message", (message) => { + if (message?.type === "stop") finishServing(); }); process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }); } else { process.exit(0); } `; - for (const version of ["1.0.0", "1.1.0"]) { - yield* writeFakeRuntime( - fs, - path, - path.join(root, "runtime", "versions", version), - childSource, + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { - backupSpaceProbe: { canClone: async () => false }, - }, - ); - yield* Effect.promise(() => - launcher.run().then( - () => Promise.reject(new Error("launcher unexpectedly completed")), - () => Promise.resolve(), - ), - ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + backupSpaceProbe: { canClone: async () => false }, + }, + ); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.update?.status, "rolled-back"); - const observed = JSON.parse(yield* fs.readFileString(observationsPath)); - // A WAL database's main file is copied before the server is asked to stop; others fall back offline. - assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []); - assert.isAbove(observed.rows, 501); - // Only the online copy holds no -shm: the offline fallback copies the one the server left. - assert.deepEqual( - JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), - journalMode === "wal" ? ["database", "database-wal"] : ["database"], - ); - const restored = new NodeSqlite.DatabaseSync(databasePath); - try { - assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok"); - assert.equal( - restored.prepare("SELECT count(*) AS n FROM events").get()?.n, - observed.rows, + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.update?.status, "rolled-back"); + const observed = JSON.parse(yield* fs.readFileString(observationsPath)); + // A WAL database's main file is copied before the server is asked to stop; others fall back offline. + assert.deepEqual(observed.backupsAtStop, journalMode === "wal" ? [["database"]] : []); + assert.isAbove(observed.rows, 501); + // Only the online copy holds no -shm: the offline fallback copies the one the server left. + assert.deepEqual( + JSON.parse(yield* fs.readFileString(`${observationsPath}.trial`)), + journalMode === "wal" ? ["database", "database-wal"] : ["database"], ); - assert.equal( - restored - .prepare( - "SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'", - ) - .get()?.n, - 1, - ); - } finally { - restored.close(); - } - }), + const restored = new NodeSqlite.DatabaseSync(databasePath); + try { + assert.equal(restored.prepare("PRAGMA integrity_check").get()?.integrity_check, "ok"); + assert.equal( + restored.prepare("SELECT count(*) AS n FROM events").get()?.n, + observed.rows, + ); + assert.equal( + restored + .prepare( + "SELECT count(*) AS n FROM events WHERE payload = 'last write before stopping'", + ) + .get()?.n, + 1, + ); + } finally { + restored.close(); + } + }), + ), ); const requestRejectedUpdate = (backupSpaceProbe: Partial) => @@ -894,148 +1137,1338 @@ if (context.update?.status === "pending") { }; }); - it.effect("refuses a backup that cannot fit and leaves the server running", () => - Effect.gen(function* () { - const { reason, trialStarted, state } = yield* requestRejectedUpdate({ - readFreeBytes: () => Promise.resolve(0), - canClone: () => Promise.resolve(true), - }); - - assert.include(reason, "Not enough free disk space"); - assert.include(reason, "The server was not stopped"); - assert.isFalse(trialStarted); - assert.equal(state.activeVersion, "1.0.0"); - assert.isUndefined(state.update); - }), + it.live("refuses a backup that cannot fit and leaves the server running", () => + runNode( + Effect.gen(function* () { + const { reason, trialStarted, state } = yield* requestRejectedUpdate({ + readFreeBytes: () => Promise.resolve(0), + canClone: () => Promise.resolve(true), + }); + + assert.include(reason, "Not enough free disk space"); + assert.include(reason, "The server was not stopped"); + assert.isFalse(trialStarted); + assert.equal(state.activeVersion, "1.0.0"); + assert.isUndefined(state.update); + }), + ), ); - it.effect("refuses the update when free space cannot be checked", () => - Effect.gen(function* () { - const { reason, trialStarted, state } = yield* requestRejectedUpdate({ - readFreeBytes: () => Promise.reject(new Error("statfs unavailable")), - }); + it.live("refuses the update when free space cannot be checked", () => + runNode( + Effect.gen(function* () { + const { reason, trialStarted, state } = yield* requestRejectedUpdate({ + readFreeBytes: () => Promise.reject(new Error("statfs unavailable")), + }); - assert.include( - reason, - "Could not check that the database backup fits before updating: statfs unavailable", - ); - assert.isFalse(trialStarted); - assert.equal(state.activeVersion, "1.0.0"); - assert.isUndefined(state.update); - }), + assert.include( + reason, + "Could not check that the database backup fits before updating: statfs unavailable", + ); + assert.isFalse(trialStarted); + assert.equal(state.activeVersion, "1.0.0"); + assert.isUndefined(state.update); + }), + ), ); - it.effect("records no update when the server exits during the backup check", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-exit-" }); - const statePath = path.join(root, "runtime", "service-state.json"); - const databasePath = path.join(root, "userdata", "state.sqlite"); - yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); - yield* fs.writeFileString(databasePath, "database"); - const encoded = JSON.stringify({ databasePath }); - const childSource = ` -const paths = ${encoded}; -process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }, () => - process.exit(3), -); + it.live("stops the server over IPC with the stop marker set and ignores stale launcherIds", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-stop-" }); + const controlDir = serviceControlDir(path.join(root, "local"), root); + const statePath = path.join(root, "runtime", "service-state.json"); + const reportPath = path.join(root, "stopped-with-marker"); + const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE); + const childSource = ` +const fs = require("node:fs"); +const paths = ${JSON.stringify({ reportPath, markerPath })}; +process.on("message", (message) => { + if (message.type !== "stop") return; + fs.writeFileSync(paths.reportPath, String(fs.existsSync(paths.markerPath))); + process.exit(0); +}); setInterval(() => {}, 1_000); `; - for (const version of ["1.0.0", "1.1.0"]) { yield* writeFakeRuntime( fs, path, - path.join(root, "runtime", "versions", version), + path.join(root, "runtime", "versions", "1.0.0"), childSource, ); - } - yield* Effect.promise(() => - writeServiceState(statePath, { - protocol: SERVICE_LAUNCHER_PROTOCOL, - activeVersion: "1.0.0", - }), - ); - const launcher = new Launcher( - root, - yield* Effect.promise(() => readServiceState(statePath)), - { - backupSpaceProbe: { - readFreeBytes: () => { - const child = spawnedChildren.at(-1)!; - return new Promise((resolve) => { - if (!child.connected) resolve(Number.MAX_SAFE_INTEGER); - else child.once("disconnect", () => resolve(Number.MAX_SAFE_INTEGER)); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + yield* fs.makeDirectory(controlDir, { recursive: true }); + yield* fs.writeFileString(path.join(controlDir, "control.token"), "corrupt\n"); + + const acquired = Promise.withResolvers(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + onControlAcquired: () => acquired.resolve(), + }, + ); + const running = launcher.run(); + yield* Effect.promise(() => Promise.race([acquired.promise, running])); + + const activeId = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }), + ); + assert.isString(activeId); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl: trustedTestAcl })), + "running", + ); + + assert.isTrue( + yield* Effect.promise(() => + signalServiceLauncherStop(controlDir, "stale-launcher-id", { acl: trustedTestAcl }), + ), + ); + assert.equal( + yield* Effect.promise(() => queryServiceLauncherId(controlDir, { acl: trustedTestAcl })), + activeId, + ); + + assert.isTrue( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + yield* Effect.promise(() => running); + assert.isFalse(launcher.ownsControl); + assert.equal(yield* fs.readFileString(reportPath), "true"); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl: trustedTestAcl })), + "stopped", + ); + assert.isFalse( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + }), + ), + ); + + it.live("rotates untrusted control tokens and rejects untrusted directory or token ACLs", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-acl-" }); + const controlDir = serviceControlDir(path.join(root, "local"), root); + const tokenPath = path.join(controlDir, "control.token"); + const statePath = path.join(root, "runtime", "service-state.json"); + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + 'process.on("message", (m) => { if (m.type === "stop") process.exit(0); });\nsetInterval(() => {}, 1_000);\n', + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const preseededToken = "a".repeat(64); + yield* fs.makeDirectory(controlDir, { recursive: true }); + yield* fs.writeFileString(tokenPath, `${preseededToken}\n`); + + let clientTrustsDir = true; + let clientTrustsToken = false; + const restrictedDirs: Array = []; + const acl: ControlAclHooks = { + restrictPathAcl: async (targetPath, isDirectory) => { + if (isDirectory) { + restrictedDirs.push(targetPath); + clientTrustsDir = true; + } else if (path.basename(targetPath).startsWith(".control-token.")) { + clientTrustsToken = true; + } + }, + verifyPathAcl: async (targetPath, isDirectory) => { + if (isDirectory) { + return clientTrustsDir && restrictedDirs.includes(targetPath); + } + if (targetPath === tokenPath) return clientTrustsToken; + return true; + }, + }; + + const acquired = Promise.withResolvers(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl, + stopChildOverIpc: true, + onControlAcquired: () => acquired.resolve(), + }, + ); + const running = launcher.run(); + yield* Effect.promise(() => Promise.race([acquired.promise, running])); + + const rotatedToken = (yield* fs.readFileString(tokenPath)).trim(); + assert.notEqual(rotatedToken, preseededToken); + assert.include(restrictedDirs, controlDir); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl })), + "running", + ); + + clientTrustsDir = false; + const rejectedDir = yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl }).then( + () => undefined, + (error: unknown) => error, + ), + ); + assert.isTrue(isServiceControlUnauthorizedError(rejectedDir)); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl })), + "unhealthy", + ); + clientTrustsDir = true; + + clientTrustsToken = false; + const rejected = yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl }).then( + () => undefined, + (error: unknown) => error, + ), + ); + assert.isTrue(isServiceControlUnauthorizedError(rejected)); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl })), + "unhealthy", + ); + + clientTrustsToken = true; + assert.isTrue(yield* Effect.promise(() => requestServiceLauncherStop(controlDir, { acl }))); + yield* Effect.promise(() => running); + }), + ), + ); + + it.live( + "sends zero bytes to an invalid challenge, rejects forged or mismatched replies, and ignores unlocked stale endpoints", + () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-rogue-" }); + const controlDir = serviceControlDir(path.join(root, "local"), root); + const rogueScript = path.join(root, "rogue.mjs"); + const sharedToken = "b".repeat(64); + + yield* fs.makeDirectory(controlDir, { recursive: true }); + yield* fs.writeFileString(path.join(controlDir, "control.token"), `${sharedToken}\n`); + yield* fs.writeFileString( + rogueScript, + [ + 'import * as crypto from "node:crypto";', + 'import * as fs from "node:fs";', + 'import * as net from "node:net";', + 'import * as path from "node:path";', + 'import * as sqlite from "node:sqlite";', + "const controlDir = process.argv[2];", + `const token = ${JSON.stringify(sharedToken)};`, + 'const db = new sqlite.DatabaseSync(path.join(controlDir, "lock.sqlite"));', + 'db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;");', + "let lockReleased = false;", + "let unlockedConnections = 0;", + "let index = 0;", + "const server = net.createServer((socket) => {", + " if (lockReleased) { unlockedConnections += 1; socket.destroy(); return; }", + " const current = index++;", + " let bytes = 0;", + " if (current === 0) {", + ' socket.write(\'{"nonce":"invalid"}\\n\');', + ' socket.on("data", (chunk) => { bytes += chunk.length; });', + ' socket.on("close", () => { process.send({ type: "bad-closed", bytes }); });', + " return;", + " }", + ' const serverNonce = "0123456789abcdef0123456789abcdef";', + ' socket.write(JSON.stringify({ nonce: serverNonce }) + "\\n");', + ' let line = "";', + ' socket.setEncoding("utf8");', + ' socket.on("data", (chunk) => {', + " bytes += Buffer.byteLength(chunk);", + " line += chunk;", + ' if (line.includes("\\n")) {', + " if (current === 1) {", + ' process.send({ type: "forged-received", bytes });', + ' socket.write(JSON.stringify({ body: \'{"ok":true,"launcherId":"rogue"}\', mac: "0".repeat(64) }) + "\\n");', + " return;", + " }", + " const req = JSON.parse(line.trim());", + ' const body = JSON.stringify({ ok: true, launcherId: "other-instance" });', + ' const mac = crypto.createHmac("sha256", token).update(`res\\0${serverNonce}\\0${req.cnonce}\\0${body}`).digest("hex");', + ' socket.write(JSON.stringify({ body, mac }) + "\\n");', + " }", + " });", + "});", + 'server.listen(0, "127.0.0.1", () => {', + " const address = server.address();", + ' fs.writeFileSync(path.join(controlDir, "endpoint.json"), JSON.stringify({ port: address.port, launcherId: "rogue" }) + "\\n");', + ' process.send({ type: "ready" });', + "});", + 'process.on("message", (msg) => {', + ' if (msg.type === "release") {', + " lockReleased = true;", + " db.close();", + ' process.send({ type: "released" });', + ' } else if (msg.type === "exit") {', + ' server.close(() => process.send({ type: "exiting", unlockedConnections }, () => process.exit(0)));', + " }", + "});", + ].join("\n"), + ); + + const ready = Promise.withResolvers(); + const badClosed = Promise.withResolvers(); + let forgedBytes = 0; + const released = Promise.withResolvers(); + const exiting = Promise.withResolvers(); + const exited = Promise.withResolvers(); + + const rogue = yield* Effect.acquireRelease( + Effect.sync(() => { + const child = NodeChildProcess.spawn(process.execPath, [rogueScript, controlDir], { + stdio: ["ignore", "inherit", "inherit", "ipc"], + }); + child.on("message", (raw: unknown) => { + const msg = raw as { + readonly type: string; + readonly bytes?: number; + readonly unlockedConnections?: number; + }; + if (msg.type === "ready") ready.resolve(); + else if (msg.type === "bad-closed") badClosed.resolve(msg.bytes ?? -1); + else if (msg.type === "forged-received") forgedBytes = msg.bytes ?? 0; + else if (msg.type === "released") released.resolve(); + else if (msg.type === "exiting") exiting.resolve(msg.unlockedConnections ?? -1); }); + child.once("exit", () => exited.resolve()); + return child; + }), + (child) => + Effect.sync(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }), + ); + + yield* Effect.promise(() => ready.promise); + + const badChallengeError = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }).then( + () => undefined, + (error: unknown) => error as Error, + ), + ); + assert.include(badChallengeError?.message ?? "", "invalid control challenge"); + assert.equal(yield* Effect.promise(() => badClosed.promise), 0); + + const forgedReplyError = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }).then( + () => undefined, + (error: unknown) => error, + ), + ); + assert.isTrue(isServiceControlUnauthorizedError(forgedReplyError)); + assert.isAbove(forgedBytes, 0); + + const mismatchedIdError = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }).then( + () => undefined, + (error: unknown) => error, + ), + ); + assert.isTrue(isServiceControlLauncherMismatchError(mismatchedIdError)); + assert.isFalse(isServiceControlUnauthorizedError(mismatchedIdError)); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl: trustedTestAcl })), + "unhealthy", + ); + + const untrustedReleaseError = yield* Effect.promise(() => + waitForServiceLauncherRelease(controlDir, { + timeoutMs: 200, + acl: { + restrictPathAcl: async () => undefined, + verifyPathAcl: async () => false, + }, + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + assert.isTrue(isServiceControlUnauthorizedError(untrustedReleaseError)); + + // Releasing the SQLite lock while leaving endpoint.json + the TCP listener open simulates + // ephemeral port reuse after a crash: clients must see the unlocked state and never dial. + rogue.send({ type: "release" }); + yield* Effect.promise(() => released.promise); + assert.equal( + yield* Effect.promise(() => probeServiceLauncher(controlDir, { acl: trustedTestAcl })), + "stopped", + ); + assert.isFalse( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + + rogue.send({ type: "exit" }); + assert.equal(yield* Effect.promise(() => exiting.promise), 0); + yield* Effect.promise(() => exited.promise); + }), + ), + ); + + it.live( + "waits across multiple lock retries for a stopping launcher, reloads state, and spawns once", + () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-handoff-" }); + const controlDir = serviceControlDir(path.join(root, "local"), root); + const statePath = path.join(root, "runtime", "service-state.json"); + const spawnsLog = path.join(root, "spawns.log"); + const makeChildSource = (version: string) => ` +const fs = require("node:fs"); +fs.appendFileSync(${JSON.stringify(spawnsLog)}, ${JSON.stringify(`${version}\n`)}); +process.on("message", (message) => { + if (message.type === "stop") process.exit(0); +}); +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + makeChildSource(version), + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const initialState = yield* Effect.promise(() => readServiceState(statePath)); + const firstAcquired = Promise.withResolvers(); + const first = new Launcher(root, initialState, { + controlDir, + acl: trustedTestAcl, + controlListenRetryMs: 10, + stopChildOverIpc: true, + onControlAcquired: () => firstAcquired.resolve(), + }); + const firstRun = first.run(); + yield* Effect.promise(() => Promise.race([firstAcquired.promise, firstRun])); + + const firstId = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }), + ); + assert.isDefined(firstId); + assert.deepEqual( + yield* Effect.promise(() => + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.1.0", + acl: trustedTestAcl, + }), + ), + { status: "staged", launcherId: firstId }, + ); + + const retriedTwice = Promise.withResolvers(); + const secondAcquired = Promise.withResolvers(); + const second = new Launcher(root, initialState, { + controlDir, + acl: trustedTestAcl, + controlListenRetryMs: 10, + stopChildOverIpc: true, + onControlRetry: (attempt) => { + if (attempt >= 2) retriedTwice.resolve(); }, - }, - }, - ); - const failure = yield* Effect.promise(() => - launcher.run().then( - () => new Error("launcher unexpectedly completed"), - (cause: unknown) => cause as Error, - ), - ); + onControlAcquired: () => secondAcquired.resolve(), + }); + const secondRun = second.run(); + yield* Effect.promise(() => Promise.race([retriedTwice.promise, secondRun])); + assert.isFalse(second.ownsControl); - assert.include(failure.message, "Active child exited unexpectedly (3)"); - const state = yield* Effect.promise(() => readServiceState(statePath)); - assert.equal(state.activeVersion, "1.0.0"); - assert.isUndefined(state.update); - }), + assert.isTrue( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + yield* Effect.promise(() => firstRun); + yield* Effect.promise(() => Promise.race([secondAcquired.promise, secondRun])); + assert.isTrue(second.ownsControl); + + assert.isTrue( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + yield* Effect.promise(() => secondRun); + assert.deepEqual((yield* fs.readFileString(spawnsLog)).trim().split("\n"), [ + "1.0.0", + "1.1.0", + ]); + }), + ), ); - it.effect("reclaims partial backups left by killed updates before checking space", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" }); - const databasePath = path.join(root, "state.sqlite"); - yield* writeSparseFile(databasePath, 10 * 1024 ** 3); - const backupRoot = path.join(root, "runtime", "db-backup"); - const orphan = path.join(backupRoot, "earlier-update.staging"); - const completed = path.join(backupRoot, "completed-update"); - yield* fs.makeDirectory(orphan, { recursive: true }); - yield* fs.writeFileString(path.join(orphan, "database"), "partial copy"); - yield* fs.makeDirectory(completed); - yield* fs.writeFileString(path.join(completed, "database"), "complete copy"); - - // The orphan is what keeps this disk too full for a full copy. - const blocker = yield* Effect.promise(() => - databaseBackupBlocker(root, databasePath, { - readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3), - canClone: () => Promise.resolve(false), + it.live( + "cancels an aborted stage-install queued behind an in-flight transition and serializes offline staging", + () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-control-queue-" }); + const controlDir = serviceControlDir(path.join(root, "local"), root); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + const childSource = ` +process.on("message", (message) => { + if (message.type === "stop") process.exit(0); +}); +process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${JSON.stringify(databasePath)} }); +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0", "1.2.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const inUpdateCheck = Promise.withResolvers(); + const finishUpdateCheck = Promise.withResolvers(); + const stageQueued = Promise.withResolvers<{ readonly disconnected: Promise }>(); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + onControlStageQueued: ({ version, disconnected }) => { + if (version === "1.2.0") stageQueued.resolve({ disconnected }); + }, + backupSpaceProbe: { + readFreeBytes: () => { + inUpdateCheck.resolve(); + return finishUpdateCheck.promise; + }, + canClone: () => Promise.resolve(false), + }, + }, + ); + const running = launcher.run(); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + finishUpdateCheck.resolve(0); + await launcher.stop("SIGTERM"); + await running.catch(() => undefined); + }), + ); + yield* Effect.promise(() => Promise.race([inUpdateCheck.promise, running])); + const activeId = yield* Effect.promise(() => + queryServiceLauncherId(controlDir, { acl: trustedTestAcl }), + ); + assert.isDefined(activeId); + + const abortedStageFiber = yield* Effect.forkChild( + Effect.tryPromise({ + try: (signal) => + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.2.0", + acl: trustedTestAcl, + signal, + }), + catch: (cause) => new ServiceLauncherClientError({ operation: "send", cause }), + }), + ); + const { disconnected } = yield* Effect.promise(() => stageQueued.promise); + yield* Fiber.interrupt(abortedStageFiber); + const abortedExit = yield* Fiber.await(abortedStageFiber); + assert.isTrue(Exit.isFailure(abortedExit)); + yield* Effect.promise(() => disconnected); + + // Return 0 free bytes so the update check rejects without setting `update: pending`; if the + // aborted `1.2.0` stage had still executed, staging `1.1.0` without allowDowngrade would fail. + finishUpdateCheck.resolve(0); + assert.deepEqual( + yield* Effect.promise(() => + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.1.0", + acl: trustedTestAcl, + }), + ), + { status: "staged", launcherId: activeId }, + ); + assert.equal( + (yield* Effect.promise(() => readServiceState(statePath))).activeVersion, + "1.1.0", + ); + + assert.isTrue( + yield* Effect.promise(() => + requestServiceLauncherStop(controlDir, { acl: trustedTestAcl }), + ), + ); + yield* Effect.promise(() => running); + + const concurrentOffline = yield* Effect.promise(() => + Promise.all([ + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.2.0", + acl: trustedTestAcl, + }), + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.2.0", + acl: trustedTestAcl, + }), + ]), + ); + assert.deepEqual(concurrentOffline, [{ status: "staged" }, { status: "staged" }]); + + // Offline `stageServiceLauncherInstall` refuses while a remote update is pending and + // surfaces `installedVersion` without touching the database or backup; `recoverServiceLauncherUninstall` + // requires exclusive `server-owner.sqlite` ownership before restoring a pre-trial backup. + yield* fs.remove(path.join(root, "runtime", "stopped"), { force: true }); + const backupDir = path.join(root, "runtime", "db-backup", "abandoned-update"); + yield* fs.makeDirectory(backupDir, { recursive: true }); + yield* fs.writeFileString(path.join(backupDir, "database"), "pre-trial-database"); + yield* fs.writeFileString(databasePath, "corrupted-trial-database"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "abandoned-update", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + assert.deepEqual( + yield* Effect.promise(() => + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.2.0", + acl: trustedTestAcl, + }), + ), + { status: "update-pending", installedVersion: "1.1.0" }, + ); + assert.equal(yield* fs.readFileString(databasePath), "corrupted-trial-database"); + assert.isTrue(yield* fs.exists(backupDir)); + + const ownerLockPath = path.join(path.dirname(databasePath), "server-owner.sqlite"); + const heldOwnerLock = new NodeSqlite.DatabaseSync(ownerLockPath); + heldOwnerLock.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + try { + yield* Effect.promise(() => + recoverServiceLauncherUninstall({ + baseDir: root, + controlDir, + acl: trustedTestAcl, + }), + ); + assert.equal(yield* fs.readFileString(databasePath), "corrupted-trial-database"); + const ownedRecoveryState = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(ownedRecoveryState.update?.status, "failed"); + assert.equal( + ownedRecoveryState.update?.status === "failed" + ? ownedRecoveryState.update.reason + : undefined, + "state-dir-owned", + ); + assert.isTrue(yield* fs.exists(backupDir)); + } finally { + heldOwnerLock.close(); + } + + // If a launcher stopped cleanly during a pending trial (recording `stopped-db-digest.json`) + // and a foreground server later saved new work before uninstall recovery, recovery preserves + // the foreground database AND keeps the backup instead of overwriting or deleting either copy. + const postStopBackupDir = path.join(root, "runtime", "db-backup", "post-stop-update"); + yield* fs.makeDirectory(postStopBackupDir, { recursive: true }); + yield* fs.writeFileString(path.join(postStopBackupDir, "database"), "pre-trial-database"); + yield* fs.writeFileString(databasePath, "trial-database-at-stop"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "post-stop-update", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + const stoppingPending = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + onControlAcquired: () => { + void stoppingPending.stop("SIGTERM"); + }, + }, + ); + yield* Effect.promise(() => stoppingPending.run()); + yield* fs.writeFileString(databasePath, "foreground-work-saved-after-stop"); + yield* Effect.promise(() => + recoverServiceLauncherUninstall({ + baseDir: root, + controlDir, + acl: trustedTestAcl, + }), + ); + assert.equal(yield* fs.readFileString(databasePath), "foreground-work-saved-after-stop"); + assert.isTrue(yield* fs.exists(postStopBackupDir)); + assert.deepEqual( + yield* Effect.promise(() => + stageServiceLauncherInstall({ + controlDir, + baseDir: root, + version: "1.2.0", + acl: trustedTestAcl, + }), + ), + { status: "staged" }, + ); + + // Without positive stop evidence (`stopped-db-digest.json` or `.restore-pending`), such as + // after a launcher crash where a foreground server may have saved work and exited, + // `recoverServiceLauncherUninstall` refuses to overwrite `databasePath` and preserves the backup. + const crashedBackupDir = path.join(root, "runtime", "db-backup", "crashed-update"); + yield* fs.makeDirectory(crashedBackupDir, { recursive: true }); + yield* fs.writeFileString(path.join(crashedBackupDir, "database"), "clean-pre-trial"); + yield* fs.writeFileString(databasePath, "interrupted-trial"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "crashed-update", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + yield* Effect.promise(() => + recoverServiceLauncherUninstall({ + baseDir: root, + controlDir, + acl: trustedTestAcl, + }), + ); + assert.equal(yield* fs.readFileString(databasePath), "interrupted-trial"); + assert.isTrue(yield* fs.exists(crashedBackupDir)); + + // When the launcher stopped cleanly first (`stopped-db-digest.json` matches), offline + // `recoverServiceLauncherUninstall` restores the pre-trial backup and cleans up `backupDir`. + const uninstallBackupDir = path.join(root, "runtime", "db-backup", "uninstall-update"); + yield* fs.makeDirectory(uninstallBackupDir, { recursive: true }); + yield* fs.writeFileString(path.join(uninstallBackupDir, "database"), "clean-pre-trial"); + yield* fs.writeFileString(databasePath, "interrupted-trial"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "uninstall-update", + fromVersion: "1.1.0", + targetVersion: "1.2.0", + dbPath: databasePath, + status: "pending", + }, + }), + ); + const cleanlyStoppedPending = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + onControlAcquired: () => { + void cleanlyStoppedPending.stop("SIGTERM"); + }, + }, + ); + yield* Effect.promise(() => cleanlyStoppedPending.run()); + yield* Effect.promise(() => + recoverServiceLauncherUninstall({ + baseDir: root, + controlDir, + acl: trustedTestAcl, + }), + ); + assert.equal(yield* fs.readFileString(databasePath), "clean-pre-trial"); + assert.equal( + (yield* Effect.promise(() => readServiceState(statePath))).update?.status, + "failed", + ); + assert.isFalse(yield* fs.exists(uninstallBackupDir)); + + // Startup `#recover()` refuses to overwrite a database whose `.server-owner-lease.json` + // shows a foreground server took ownership after the pending trial crashed. + const leaseConflictBackupDir = path.join(root, "runtime", "db-backup", "lease-conflict"); + yield* fs.makeDirectory(leaseConflictBackupDir, { recursive: true }); + yield* fs.writeFileString(path.join(leaseConflictBackupDir, "database"), "old-pre-trial"); + yield* fs.writeFileString(databasePath, "foreground-work-after-crash"); + yield* fs.writeFileString( + path.join(path.dirname(databasePath), SERVER_OWNER_LEASE_FILE), + `${JSON.stringify({ ownerId: "foreground-server" })}\n`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "lease-conflict", + fromVersion: "1.1.0", + targetVersion: "9.9.9", + dbPath: databasePath, + status: "pending", + }, + }), + ); + const recoveringAfterForeground = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + onControlAcquired: () => { + void recoveringAfterForeground.stop("SIGTERM"); + }, + }, + ); + yield* Effect.promise(() => recoveringAfterForeground.run()); + assert.equal(yield* fs.readFileString(databasePath), "foreground-work-after-crash"); + assert.isTrue(yield* fs.exists(leaseConflictBackupDir)); + yield* fs.remove(path.join(path.dirname(databasePath), SERVER_OWNER_LEASE_FILE), { + force: true, + }); + + // Live in-service `signalServiceLauncherStop(..., { uninstall: true })` restores the + // pre-trial SQLite database even when the managed trial child checkpoints WAL writes on shutdown. + const walDbPath = path.join(root, "userdata", "wal-trial.sqlite"); + const preTrialDb = new NodeSqlite.DatabaseSync(walDbPath); + preTrialDb.exec( + "PRAGMA journal_mode = WAL; CREATE TABLE notes (value TEXT); INSERT INTO notes VALUES ('clean-pre-trial');", + ); + preTrialDb.close(); + const liveUninstallBackupDir = path.join(root, "runtime", "db-backup", "live-uninstall"); + yield* fs.makeDirectory(liveUninstallBackupDir, { recursive: true }); + yield* fs.copyFile(walDbPath, path.join(liveUninstallBackupDir, "database")); + if (yield* fs.exists(`${walDbPath}-wal`)) { + yield* fs.copyFile(`${walDbPath}-wal`, path.join(liveUninstallBackupDir, "wal")); + } + const walTrialVersionDir = path.join(root, "runtime", "versions", "1.3.0"); + const walChildReady = Promise.withResolvers(); + yield* writeFakeRuntime( + fs, + path, + walTrialVersionDir, + ` +const { DatabaseSync } = require("node:sqlite"); +const db = new DatabaseSync(${JSON.stringify(walDbPath)}); +db.exec("PRAGMA journal_mode = WAL; INSERT INTO notes VALUES ('trial-running');"); +process.on("message", (message) => { + if (message?.type === "stop") { + db.exec("INSERT INTO notes VALUES ('trial-shutdown-flush'); PRAGMA wal_checkpoint(TRUNCATE);"); + db.close(); + process.exit(0); + } +}); +process.send({ type: "wal-trial-ready" }); +`, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.1.0", + update: { + id: "live-uninstall", + fromVersion: "1.1.0", + targetVersion: "1.3.0", + dbPath: walDbPath, + status: "pending", + }, + }), + ); + const liveUninstallLauncher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + controlDir, + acl: trustedTestAcl, + stopChildOverIpc: true, + spawnCommand: (entryPath) => { + queueMicrotask(() => { + const spawned = spawnedChildren.at(-1); + spawned?.on("message", (message) => { + if ( + (message as { readonly type?: string } | null)?.type === "wal-trial-ready" + ) { + walChildReady.resolve(); + } + }); + }); + return { command: entryPath, args: ["serve"] }; + }, + }, + ); + const liveUninstallRun = liveUninstallLauncher.run(); + yield* Effect.promise(() => walChildReady.promise); + assert.isTrue( + yield* Effect.promise(() => + signalServiceLauncherStop(controlDir, undefined, { + acl: trustedTestAcl, + uninstall: true, + }), + ), + ); + yield* Effect.promise(() => liveUninstallRun); + const verifyRestored = new NodeSqlite.DatabaseSync(walDbPath); + try { + assert.deepEqual(verifyRestored.prepare("SELECT value FROM notes").all(), [ + { value: "clean-pre-trial" }, + ]); + assert.deepEqual(verifyRestored.prepare("PRAGMA integrity_check").get(), { + integrity_check: "ok", + }); + } finally { + verifyRestored.close(); + } + assert.isFalse(yield* fs.exists(liveUninstallBackupDir)); }), - ); + ), + ); - assert.isUndefined(blocker); - assert.isFalse(yield* fs.exists(orphan)); - assert.isTrue(yield* fs.exists(path.join(completed, "database"))); - }), + it.live( + "ends the persisted Windows scheduled task while holding the control lock when the active server crashes", + () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-win-crash-" }); + const userProfile = path.join(root, "profile"); + const localAppData = path.join(userProfile, "AppData", "Local"); + const persistedTask = serviceTaskName("OLD-PC", "theo"); + const taskFile = serviceTaskNamePath(localAppData); + yield* fs.makeDirectory(path.dirname(taskFile), { recursive: true }); + yield* fs.writeFileString(taskFile, `${persistedTask}\n`); + + const statePath = path.join(root, "runtime", "service-state.json"); + const logFile = path.join(root, "logs", "boot-service.log"); + const childSource = ` +process.on("message", (message) => { + if (message?.type === "crash") process.exit(5); + if (message?.type === "stop") process.exit(0); +}); +process.send({ type: "test-ready" }); +`; + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", "1.0.0"), + childSource, + ); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const activeChild = Promise.withResolvers(); + const endedTasks: Array<{ readonly taskName: string; readonly lockHeld: boolean }> = []; + const controlDir = serviceControlDir(localAppData, root, { caseInsensitive: true }); + + // Blank `LOCALAPPDATA` falls back to `USERPROFILE\AppData\Local`, and a waiting + // non-owner failing on lock timeout must NOT end the active owner's scheduled task. + const activeMain = main({ + baseDir: root, + logFile, + serviceUnit: "t3code.service", + platform: "win32", + environment: { + ...process.env, + LOCALAPPDATA: " ", + USERPROFILE: userProfile, + USERDOMAIN: "RENAMED-PC", + USERNAME: "theo", + }, + acl: trustedTestAcl, + spawnCommand: (entryPath) => { + queueMicrotask(() => { + const spawned = spawnedChildren.at(-1); + spawned?.on("message", (message) => { + if ((message as { readonly type?: string } | null)?.type === "test-ready") { + activeChild.resolve(spawned); + } + }); + }); + return { command: entryPath, args: ["serve"] }; + }, + endScheduledTask: (taskName) => { + endedTasks.push({ + taskName, + lockHeld: probeControlLockHeld(controlDir), + }); + }, + }); + yield* Effect.addFinalizer(() => + Effect.promise(async () => { + spawnedChildren.at(-1)?.kill("SIGKILL"); + await activeMain.catch(() => undefined); + }), + ); + + const runningChild = yield* Effect.promise(() => + Promise.race([ + activeChild.promise, + activeMain.then(() => Promise.reject(new Error("launcher exited early"))), + ]), + ); + + const contenderFailure = yield* Effect.promise(() => + main({ + baseDir: root, + serviceUnit: "t3code.service", + platform: "win32", + environment: { + ...process.env, + LOCALAPPDATA: localAppData, + USERDOMAIN: "RENAMED-PC", + USERNAME: "theo", + }, + acl: trustedTestAcl, + controlListenRetryMs: 1, + endScheduledTask: (taskName) => { + endedTasks.push({ taskName: `contender:${taskName}`, lockHeld: false }); + }, + }).then( + () => undefined, + (cause: unknown) => cause as Error, + ), + ); + assert.include(contenderFailure?.message ?? "", "Timed out waiting to acquire"); + assert.deepEqual(endedTasks, []); + + runningChild.send({ type: "crash" }); + const activeFailure = yield* Effect.promise(() => + activeMain.then( + () => undefined, + (cause: unknown) => cause as Error, + ), + ); + assert.include(activeFailure?.message ?? "", "Active child exited unexpectedly (5)"); + assert.deepEqual(endedTasks, [{ taskName: persistedTask, lockHeld: true }]); + assert.include( + yield* fs.readFileString(logFile), + "[service-launcher] Active child exited unexpectedly (5)", + ); + + // Even if writing the fatal log fails (for example ENOSPC on `/dev/full`), `endScheduledTask` still runs. + endedTasks.length = 0; + const nextChild = Promise.withResolvers(); + const enospcMain = main({ + baseDir: root, + logFile: NodeFS.existsSync("/dev/full") ? "/dev/full" : logFile, + serviceUnit: "t3code.service", + platform: "win32", + environment: { + ...process.env, + LOCALAPPDATA: localAppData, + USERDOMAIN: "RENAMED-PC", + USERNAME: "theo", + }, + acl: trustedTestAcl, + spawnCommand: (entryPath) => { + queueMicrotask(() => { + const spawned = spawnedChildren.at(-1); + spawned?.on("message", (message) => { + if ((message as { readonly type?: string } | null)?.type === "test-ready") { + nextChild.resolve(spawned); + } + }); + }); + return { command: entryPath, args: ["serve"] }; + }, + endScheduledTask: (taskName) => { + endedTasks.push({ + taskName, + lockHeld: probeControlLockHeld(controlDir), + }); + }, + }); + const secondChild = yield* Effect.promise(() => + Promise.race([ + nextChild.promise, + enospcMain.then(() => Promise.reject(new Error("launcher exited early"))), + ]), + ); + secondChild.send({ type: "crash" }); + const enospcFailure = yield* Effect.promise(() => + enospcMain.then( + () => undefined, + (cause: unknown) => cause as Error, + ), + ); + assert.include(enospcFailure?.message ?? "", "Active child exited unexpectedly (5)"); + assert.deepEqual(endedTasks, [{ taskName: persistedTask, lockHeld: true }]); + }), + ), ); - it.effect("checks database backup space against clone or full-copy needs", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" }); - const databasePath = path.join(root, "state.sqlite"); - yield* writeSparseFile(databasePath, 10 * 1024 ** 3); - const check = (free: number, cloneable: boolean) => - Effect.promise(() => + it.live("records no update when the server exits during the backup check", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-exit-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "database"); + const encoded = JSON.stringify({ databasePath }); + const childSource = ` +const paths = ${encoded}; +process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: paths.databasePath }, () => + process.exit(3), +); +setInterval(() => {}, 1_000); +`; + for (const version of ["1.0.0", "1.1.0"]) { + yield* writeFakeRuntime( + fs, + path, + path.join(root, "runtime", "versions", version), + childSource, + ); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + const launcher = new Launcher( + root, + yield* Effect.promise(() => readServiceState(statePath)), + { + backupSpaceProbe: { + readFreeBytes: () => { + const child = spawnedChildren.at(-1)!; + return new Promise((resolve) => { + if (!child.connected) resolve(Number.MAX_SAFE_INTEGER); + else child.once("disconnect", () => resolve(Number.MAX_SAFE_INTEGER)); + }); + }, + }, + }, + ); + const failure = yield* Effect.promise(() => + launcher.run().then( + () => new Error("launcher unexpectedly completed"), + (cause: unknown) => cause as Error, + ), + ); + + assert.include(failure.message, "Active child exited unexpectedly (3)"); + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.isUndefined(state.update); + }), + ), + ); + + it.effect("reclaims partial backups left by killed updates before checking space", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-orphan-" }); + const databasePath = path.join(root, "state.sqlite"); + yield* writeSparseFile(databasePath, 10 * 1024 ** 3); + const backupRoot = path.join(root, "runtime", "db-backup"); + const orphan = path.join(backupRoot, "earlier-update.staging"); + const completed = path.join(backupRoot, "completed-update"); + yield* fs.makeDirectory(orphan, { recursive: true }); + yield* fs.writeFileString(path.join(orphan, "database"), "partial copy"); + yield* fs.makeDirectory(completed); + yield* fs.writeFileString(path.join(completed, "database"), "complete copy"); + + // The orphan is what keeps this disk too full for a full copy. + const blocker = yield* Effect.promise(() => databaseBackupBlocker(root, databasePath, { - readFreeBytes: () => Promise.resolve(free), - canClone: () => Promise.resolve(cloneable), + readFreeBytes: () => Promise.resolve((NodeFS.existsSync(orphan) ? 8 : 11) * 1024 ** 3), + canClone: () => Promise.resolve(false), }), ); - assert.isUndefined(yield* check(11 * 1024 ** 3, false)); - assert.isUndefined(yield* check(1.5 * 1024 ** 3, true)); - assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free"); - const blocker = yield* check(8 * 1024 ** 3, false); - assert.include(blocker ?? "", "Not enough free disk space"); - assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free"); - assert.include(blocker ?? "", "The server was not stopped"); - }), + assert.isUndefined(blocker); + assert.isFalse(yield* fs.exists(orphan)); + assert.isTrue(yield* fs.exists(path.join(completed, "database"))); + }), + ), + ); + + it.effect("checks database backup space against clone or full-copy needs", () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-blocker-" }); + const databasePath = path.join(root, "state.sqlite"); + yield* writeSparseFile(databasePath, 10 * 1024 ** 3); + const check = (free: number, cloneable: boolean) => + Effect.promise(() => + databaseBackupBlocker(root, databasePath, { + readFreeBytes: () => Promise.resolve(free), + canClone: () => Promise.resolve(cloneable), + }), + ); + + assert.isUndefined(yield* check(11 * 1024 ** 3, false)); + assert.isUndefined(yield* check(1.5 * 1024 ** 3, true)); + assert.include((yield* check(0.5 * 1024 ** 3, true)) ?? "", "1.0 GB needed, 0.5 GB free"); + const blocker = yield* check(8 * 1024 ** 3, false); + assert.include(blocker ?? "", "Not enough free disk space"); + assert.include(blocker ?? "", "10.8 GB needed, 8.0 GB free"); + assert.include(blocker ?? "", "The server was not stopped"); + }), + ), + ); + + it.effect( + "accepts Windows UNC database paths and recovers missing databases through a symlinked home", + () => + runNode( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + + const uncDecoded = decodeServiceState({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "unc-update", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: "\\\\server\\share\\t3\\userdata\\statev2.sqlite", + status: "pending", + }, + }); + assert.equal(uncDecoded?.update?.status, "pending"); + + const outer = yield* fs.makeTempDirectoryScoped({ + prefix: "t3-service-launcher-symlink-home-", + }); + const realHome = path.join(outer, "real-home"); + const symlinkHome = path.join(outer, "symlink-home"); + yield* fs.makeDirectory(path.join(realHome, "userdata"), { recursive: true }); + yield* fs.symlink(realHome, symlinkHome); + + const symlinkDbPath = path.join(symlinkHome, "userdata", "statev2.sqlite"); + const backupDir = path.join(symlinkHome, "runtime", "db-backup", "interrupted-restore"); + yield* fs.makeDirectory(backupDir, { recursive: true }); + yield* fs.writeFileString(path.join(backupDir, "database"), "restored-through-symlink"); + yield* fs.writeFileString(path.join(backupDir, ".restore-pending"), ""); + + const statePath = path.join(symlinkHome, "runtime", "service-state.json"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "interrupted-restore", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + dbPath: symlinkDbPath, + status: "pending", + }, + }), + ); + + const controlDir = path.join(outer, "control"); + yield* fs.makeDirectory(controlDir, { recursive: true }); + yield* Effect.promise(() => + recoverServiceLauncherUninstall({ + baseDir: symlinkHome, + controlDir, + acl: trustedTestAcl, + }), + ); + + assert.equal( + yield* fs.readFileString(path.join(realHome, "userdata", "statev2.sqlite")), + "restored-through-symlink", + ); + const recoveredState = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(recoveredState.update?.status, "failed"); + }), + ), ); }); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index e6d3431f39b3b..c7aa49a758088 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -5,7 +5,9 @@ import * as NodeChildProcess from "node:child_process"; import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; +import * as NodeNet from "node:net"; import * as NodePath from "node:path"; +import * as NodeSqlite from "node:sqlite"; import * as NodeUtil from "node:util"; import { acquireServiceSelectionLock } from "./cloud/serviceSelectionLock.ts"; @@ -21,7 +23,9 @@ import { compareExactServiceVersions, decodeServiceLauncherChildMessage, isExactServiceVersion, + isValidServiceUpdateId, parseServiceState, + SERVER_OWNER_LEASE_FILE, serviceUnitBaseDirOf, SERVICE_ENV_FILE, SERVICE_LAUNCHER_CONTEXT_ENV, @@ -30,6 +34,8 @@ import { SERVICE_RESTART_PENDING_FILE, SERVICE_STOP_MARKER_FILE, SERVICE_UNIT_ENV, + serviceStateActiveVersion, + serviceStateHasPendingUpdate, } from "./cloud/serviceProtocol.ts"; const HANDOFF_DELAY_MS = 2_000; @@ -39,8 +45,20 @@ const HANDOFF_DELAY_MS = 2_000; */ const PREPARED_TIMEOUT_MS = 30 * 60_000; const TERMINATE_GRACE_MS = 5_000; +// A replacement started by a handoff waits up to two minutes for the stopping launcher. +const CONTROL_LISTEN_RETRY_MS = 500; +const CONTROL_LISTEN_ATTEMPTS = 240; +const CONTROL_HANDSHAKE_TIMEOUT_MS = 5_000; +const CONTROL_QUERY_TIMEOUT_MS = 5_000; +// Leave 30s inside BootService STOP_STEP_TIMEOUT (120s) for fallback task termination + lock release. +const CONTROL_STOP_REQUEST_TIMEOUT_MS = 90_000; +const CONTROL_MUTATION_TIMEOUT_MS = 120_000; +const MAX_CONTROL_MESSAGE_BYTES = 16_384; +const MAX_UNAUTHENTICATED_CONTROL_SOCKETS = 16; +const MAX_TOTAL_CONTROL_SOCKETS = 64; // Match contracts' SERVER_EXIT_CODE_STATE_DIR_OWNED without depending on Effect. const STATE_DIR_OWNED_EXIT_CODE = 78; +const IS_WINDOWS = NodePath.sep === "\\"; type TerminalStatus = "committed" | "rolled-back" | "failed"; type ChildRole = "active" | "trial"; @@ -53,8 +71,7 @@ interface ManagedChild { const runtimePaths = (baseDir: string, version: string) => { const versionDir = NodePath.join(baseDir, "runtime", "versions", version); - // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. - const executableName = process.platform === "win32" ? "t3.exe" : "t3"; + const executableName = IS_WINDOWS ? "t3.exe" : "t3"; return { versionDir, entryPath: NodePath.join(versionDir, executableName), @@ -62,9 +79,9 @@ const runtimePaths = (baseDir: string, version: string) => { }; }; -const runtimeSpawnArguments = (paths: ReturnType) => ({ - command: paths.entryPath, - args: ["serve"], +const runtimeSpawnArguments = (entryPath: string) => ({ + command: entryPath, + args: ["serve"] as ReadonlyArray, }); /** @@ -75,14 +92,155 @@ const runtimeSpawnArguments = (paths: ReturnType) => ({ */ export interface ServiceUnit { readonly path: string; - /** systemd keeps a loaded copy of the unit; launchd re-reads the plist at bootstrap. */ + /** systemd and Task Scheduler keep a loaded copy of the definition; launchd re-reads the plist at bootstrap. */ readonly reload: () => Promise; } -function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): ServiceUnit | undefined { +export const WINDOWS_QUERY_REGISTERED_TASK_SCRIPT = [ + "$ErrorActionPreference = 'Stop'", + "$service = New-Object -ComObject Schedule.Service", + "$service.Connect()", + "try {", + " $task = $service.GetFolder('\\').GetTask($env:T3_TASK_NAME)", + " $bytes = [System.Text.Encoding]::UTF8.GetBytes([string]$task.Xml)", + " $stdout = [Console]::OpenStandardOutput()", + " $stdout.Write($bytes, 0, $bytes.Length)", + " $stdout.Flush()", + "} catch {", + " $err = $_.Exception", + " while ($null -ne $err.InnerException) { $err = $err.InnerException }", + " $hr = $err.HResult", + " if ($err -is [System.IO.FileNotFoundException] -or $err -is [System.IO.DirectoryNotFoundException] -or $hr -eq -2147024894 -or $hr -eq -2147024893) {", + " exit 2", + " }", + " throw", + "}", +].join("; "); + +const sameInstalledBaseDir = (left: string, right: string, caseInsensitive: boolean): boolean => { + if (caseInsensitive) { + const looksWindowsDrive = /^[A-Za-z]:[\\/]/.test(left) || /^[A-Za-z]:[\\/]/.test(right); + const resolvePath = looksWindowsDrive ? NodePath.win32.resolve : NodePath.resolve; + return resolvePath(left).toLowerCase() === resolvePath(right).toLowerCase(); + } + return NodePath.resolve(left) === NodePath.resolve(right); +}; + +async function reloadWindowsTaskUnit(input: { + readonly unitPath: string; + readonly baseDir: string; + readonly localAppData: string; + readonly userName: string; + readonly env: NodeJS.ProcessEnv; +}): Promise { + const taskName = resolvePersistedServiceTaskName( + input.localAppData, + serviceTaskName(input.env.USERDOMAIN ?? "", input.userName), + ); + let registeredXml: string; + try { + const queried = await execFileAsync( + windowsPowerShellBinary(input.env), + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + WINDOWS_QUERY_REGISTERED_TASK_SCRIPT, + ], + { + env: { ...input.env, T3_TASK_NAME: taskName }, + windowsHide: true, + timeout: 15_000, + }, + ); + registeredXml = queried.stdout; + } catch (error) { + if ((error as { readonly code?: unknown }).code === 2) return; + throw error; + } + const registeredBaseDir = serviceUnitBaseDirOf(registeredXml); + if ( + registeredBaseDir === undefined || + !sameInstalledBaseDir(registeredBaseDir, input.baseDir, true) + ) { + return; + } + const settings = /[\s\S]*?<\/Settings>/i.exec(registeredXml)?.[0] ?? ""; + const registeredDisabled = /\s*false\s*<\/Enabled>/i.test(settings); + const unitContents = await NodeFSP.readFile(input.unitPath, "utf8"); + const unitSettings = /[\s\S]*?<\/Settings>/i.exec(unitContents)?.[0] ?? ""; + const unitAlreadyDisabled = /\s*false\s*<\/Enabled>/i.test(unitSettings); + let tempDisabledPath: string | undefined; + try { + let xmlPath = input.unitPath; + if (registeredDisabled && !unitAlreadyDisabled && /<\/Settings>/i.test(unitContents)) { + tempDisabledPath = NodePath.join( + NodePath.dirname(input.unitPath), + `.service-task-${process.pid}.${NodeCrypto.randomUUID()}.xml`, + ); + await NodeFSP.writeFile( + tempDisabledPath, + unitContents.replace(/<\/Settings>/i, " false\n "), + { mode: 0o600 }, + ); + xmlPath = tempDisabledPath; + } + await execFileAsync( + windowsSystem32Binary("schtasks.exe", input.env), + ["/Create", "/XML", xmlPath, "/TN", taskName, "/F"], + { + env: input.env, + windowsHide: true, + timeout: 30_000, + }, + ); + } finally { + if (tempDisabledPath !== undefined) { + await NodeFSP.rm(tempDisabledPath, { force: true }).catch(() => undefined); + } + } +} + +function serviceUnitFromEnvironment( + env: NodeJS.ProcessEnv = process.env, + options?: { + readonly isWindows?: boolean | undefined; + readonly localAppData?: string | undefined; + readonly baseDir?: string | undefined; + }, +): ServiceUnit | undefined { const unitName = env[SERVICE_UNIT_ENV]?.trim(); + if ( + !unitName || + unitName === "." || + unitName === ".." || + unitName.includes("/") || + unitName.includes("\\") + ) { + return undefined; + } + if (options?.isWindows === true || unitName.endsWith(".xml")) { + const localAppData = (options?.localAppData ?? env.LOCALAPPDATA ?? "").trim(); + const userName = (env.USERNAME ?? "").trim(); + const baseDir = (options?.baseDir ?? env.T3CODE_HOME ?? "").trim(); + if (!localAppData || !userName || !baseDir) return undefined; + const taskFile = unitName.endsWith(".xml") ? unitName : "t3code-task.xml"; + const unitPath = NodePath.join(localAppData, "t3code", taskFile); + return { + path: unitPath, + reload: () => + reloadWindowsTaskUnit({ + unitPath, + baseDir, + localAppData, + userName, + env, + }), + }; + } const home = env.HOME?.trim(); - if (!unitName || !home) return undefined; + if (!home) return undefined; if (unitName.endsWith(".plist")) { return { path: NodePath.join(home, "Library", "LaunchAgents", unitName), @@ -113,12 +271,13 @@ function serviceUnitFromEnvironment(env: NodeJS.ProcessEnv = process.env): Servi }; } -// systemd's ExecStart value, or the plist string before the __service-launcher argument. +// systemd's ExecStart value, the plist string, or the Windows task action before __service-launcher. // Only the version segment changes, so quoted or escaped paths stay as rendered. // The launch agent's PATH names a versions directory without the executable; leave it. const SERVICE_UNIT_PROGRAMS = [ /^(ExecStart=.*?[\\/]runtime[\\/]versions[\\/])([^\\/\s"]+)([\\/]t3)(?=["\s])/m, /([^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<]+)([\\/]t3<\/string>\s*__service-launcher<\/string>)/, + /(--headless\s+(?:"|")[^<]*?[\\/]runtime[\\/]versions[\\/])([^\\/<"&]+)([\\/]t3(?:\.exe)?(?:"|")\s+(?:"|")?__service-launcher\b)/i, ] as const; /** @@ -142,8 +301,37 @@ function repointServiceUnit(contents: string, version: string): string { const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const; const RESTORE_MARKER = ".restore-pending"; -const databaseBackupDir = (baseDir: string, updateId: string) => - NodePath.join(baseDir, "runtime", "db-backup", updateId); +const databaseBackupDir = (baseDir: string, updateId: string) => { + if (!isValidServiceUpdateId(updateId)) { + throw new Error(`Invalid service update identifier '${updateId}'.`); + } + return NodePath.join(baseDir, "runtime", "db-backup", updateId); +}; + +function resolvePathForContainment(baseDir: string, candidate: string) { + const useWin32 = + IS_WINDOWS || + baseDir.startsWith("\\\\") || + candidate.startsWith("\\\\") || + /^[A-Za-z]:[\\/]/.test(baseDir) || + /^[A-Za-z]:[\\/]/.test(candidate); + const pathMod = useWin32 ? NodePath.win32 : NodePath.posix; + const normalizeCase = (value: string) => (useWin32 ? value.toLowerCase() : value); + if (!pathMod.isAbsolute(baseDir) || !pathMod.isAbsolute(candidate)) return undefined; + const resolvedBase = normalizeCase(pathMod.resolve(baseDir)); + const resolvedCandidate = normalizeCase(pathMod.resolve(candidate)); + const rel = pathMod.relative(resolvedBase, resolvedCandidate); + if (rel === "" || rel.startsWith("..") || pathMod.isAbsolute(rel)) return undefined; + const segments = rel.split(/[\\/]+/).filter(Boolean); + if (segments.length === 0 || segments.includes("..")) return undefined; + return segments; +} + +function isDatabasePathInsideBaseDir(baseDir: string, dbPath: string): boolean { + if (dbPath.includes("\0") || baseDir.includes("\0")) return false; + const segments = resolvePathForContainment(baseDir, dbPath); + return segments !== undefined && segments.length >= 1 && segments[0] !== "runtime"; +} const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) => NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`); @@ -333,11 +521,59 @@ export async function databaseBackupBlocker( // A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks. async function copyWithoutOpening(source: string, destination: string): Promise { - // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime. - if (process.platform === "win32") return NodeFSP.copyFile(source, destination); + if (IS_WINDOWS) return NodeFSP.copyFile(source, destination); if (!(await runCp([source, destination]))) throw new Error(`cp ${source} failed.`); } +const STATE_DIR_OWNED_DURING_BACKUP_CODE = "ERR_T3_STATE_DIR_OWNED_DURING_BACKUP"; + +const createStateDirOwnedDuringBackupError = (): NodeJS.ErrnoException => { + const error: NodeJS.ErrnoException = new Error( + "Another T3 Code server owns this T3 home while backing up the database.", + ); + error.code = STATE_DIR_OWNED_DURING_BACKUP_CODE; + return error; +}; + +const isStateDirOwnedDuringBackupError = (cause: unknown): boolean => + (cause as NodeJS.ErrnoException | undefined)?.code === STATE_DIR_OWNED_DURING_BACKUP_CODE; + +async function writeLauncherServerOwnerLease( + dbPath: string, + pendingUpdateId: string, +): Promise { + const stateDir = NodePath.dirname(dbPath); + await NodeFSP.mkdir(stateDir, { recursive: true, mode: 0o700 }); + await replaceFileDurably( + NodePath.join(stateDir, SERVER_OWNER_LEASE_FILE), + `${JSON.stringify({ ownerId: "service-launcher", pendingUpdateId })}\n`, + ); +} + +async function hasConflictingServerOwnerLease( + dbPath: string, + pendingUpdateId: string, +): Promise { + let raw: string; + try { + raw = await NodeFSP.readFile( + NodePath.join(NodePath.dirname(dbPath), SERVER_OWNER_LEASE_FILE), + "utf8", + ); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return false; + return true; + } + try { + const parsed = JSON.parse(raw) as { readonly pendingUpdateId?: unknown } | null; + return ( + typeof parsed !== "object" || parsed === null || parsed.pendingUpdateId !== pendingUpdateId + ); + } catch { + return true; + } +} + // Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL, // which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint. // `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate, @@ -346,26 +582,28 @@ async function startOnlineDatabaseBackup( baseDir: string, pending: PendingServiceUpdate, probe: Partial = {}, + log: (message: string) => void = (message) => process.stderr.write(message), ): Promise<(() => Promise) | undefined> { + if (!(await isDatabaseRestoreTargetInsideBaseDir(baseDir, pending.dbPath))) { + return undefined; + } const backupDir = databaseBackupDir(baseDir, pending.id); if (await pathExists(backupDir)) return undefined; const { canClone } = { ...filesystemBackupSpaceProbe, ...probe }; if (await canClone(pending.dbPath, NodePath.dirname(backupDir)).catch(() => true)) return undefined; - const NodeSqlite = await import("node:sqlite").catch(() => undefined); - if (NodeSqlite === undefined) return undefined; const stagingDir = `${backupDir}.staging`; const discardStaging = () => NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); const fallBack = async (cause: unknown) => { await discardStaging(); - process.stderr.write( + log( `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${cause instanceof Error ? cause.message : String(cause)}\n`, ); return undefined; }; - let snapshot: InstanceType | undefined; + let snapshot: NodeSqlite.DatabaseSync | undefined; // SQLite keeps -wal beside the resolved file, not beside a symlink to it. let dbPath = pending.dbPath; const release = () => { @@ -377,7 +615,11 @@ async function startOnlineDatabaseBackup( try { await discardStaging(); await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + const realBaseDir = await NodeFSP.realpath(baseDir).catch(() => baseDir); dbPath = await NodeFSP.realpath(pending.dbPath); + if (resolvePathForContainment(realBaseDir, dbPath) === undefined) { + throw new Error("Resolved database path escapes the T3 home."); + } snapshot = new NodeSqlite.DatabaseSync(dbPath, { readOnly: true }); const mode = String(snapshot.prepare("PRAGMA journal_mode").get()?.journal_mode); if (mode.toLowerCase() !== "wal") throw new Error(`journal_mode is ${mode}, not wal.`); @@ -391,6 +633,12 @@ async function startOnlineDatabaseBackup( return fallBack(cause); } return async () => { + const ownerLock = tryAcquireServerOwnerLock(pending.dbPath); + if (ownerLock === undefined) { + release(); + await discardStaging(); + throw createStateDirOwnedDuringBackupError(); + } try { const wal = `${dbPath}-wal`; if (!(await pathExists(wal))) @@ -401,22 +649,32 @@ async function startOnlineDatabaseBackup( release(); await NodeFSP.rename(stagingDir, backupDir); await syncDirectory(NodePath.dirname(backupDir)); + await writeLauncherServerOwnerLease(pending.dbPath, pending.id); } catch (cause) { release(); await fallBack(cause); + } finally { + ownerLock.close(); } }; } // Preserve completed backups; restarted trials may already have changed the live database. async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { + if (!(await isDatabaseRestoreTargetInsideBaseDir(baseDir, pending.dbPath))) { + throw new Error("Refusing to back up a database path outside the T3 home."); + } const backupDir = databaseBackupDir(baseDir, pending.id); if (await pathExists(backupDir)) return; + const ownerLock = tryAcquireServerOwnerLock(pending.dbPath); + if (ownerLock === undefined) { + throw createStateDirOwnedDuringBackupError(); + } const stagingDir = `${backupDir}.staging`; - await NodeFSP.rm(stagingDir, { recursive: true, force: true }); - await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); try { + await NodeFSP.rm(stagingDir, { recursive: true, force: true }); + await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); for (const suffix of DB_FILE_SUFFIXES) { const source = `${pending.dbPath}${suffix}`; if (suffix !== "" && !(await pathExists(source))) continue; @@ -426,9 +684,12 @@ async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate } await NodeFSP.rename(stagingDir, backupDir); await syncDirectory(NodePath.dirname(backupDir)); + await writeLauncherServerOwnerLease(pending.dbPath, pending.id); } catch (cause) { await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); throw cause; + } finally { + ownerLock.close(); } } @@ -452,33 +713,64 @@ async function markDatabaseRestorePending(backupDir: string): Promise { } } -// Resolve symlink targets even if a crashed cp -c left the final target missing. -async function restoreTarget(path: string): Promise { +// Resolve symlink targets and missing leaves under symlinked ancestors. +async function restoreTarget(path: string, depth = 0): Promise { let current = path; - for (let hops = 0; hops < 40; hops++) { + for (let hops = depth; hops < 40; hops++) { try { return await NodeFSP.realpath(current); } catch (cause) { - if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause; + const code = (cause as NodeJS.ErrnoException).code; + if (code !== "ENOENT" && code !== "ENOTDIR") throw cause; } const link = await NodeFSP.readlink(current).catch((cause: unknown) => { const code = (cause as NodeJS.ErrnoException).code; - if (code === "ENOENT" || code === "EINVAL") return undefined; + if (code === "ENOENT" || code === "EINVAL" || code === "ENOTDIR") return undefined; throw cause; }); - if (link === undefined) return current; - // Do not normalize .. before following directory symlinks; match filesystem resolution. - current = NodePath.isAbsolute(link) - ? link - : `${await NodeFSP.realpath(NodePath.dirname(current))}${NodePath.sep}${link}`; + if (link !== undefined) { + // Do not normalize .. before following directory symlinks; match filesystem resolution. + const resolvedParent = await restoreTarget(NodePath.dirname(current), hops + 1); + current = NodePath.isAbsolute(link) ? link : `${resolvedParent}${NodePath.sep}${link}`; + continue; + } + const parent = NodePath.dirname(current); + if (parent === current) return current; + const resolvedParent = await restoreTarget(parent, hops + 1); + return NodePath.join(resolvedParent, NodePath.basename(current)); } throw new Error(`Too many symbolic links resolving ${path}.`); } +async function isDatabaseRestoreTargetInsideBaseDir( + baseDir: string, + dbPath: string, +): Promise { + if (!isDatabasePathInsideBaseDir(baseDir, dbPath)) return false; + try { + const realBaseDir = await restoreTarget(baseDir); + const mainTarget = await restoreTarget(dbPath); + if (resolvePathForContainment(realBaseDir, mainTarget) === undefined) return false; + for (const suffix of DB_FILE_SUFFIXES) { + if (suffix === "") continue; + for (const candidate of [`${dbPath}${suffix}`, `${mainTarget}${suffix}`]) { + const target = await restoreTarget(candidate); + if (resolvePathForContainment(realBaseDir, target) === undefined) return false; + } + } + return true; + } catch { + return false; + } +} + async function restoreDatabaseBackup( baseDir: string, pending: PendingServiceUpdate, ): Promise { + if (!(await isDatabaseRestoreTargetInsideBaseDir(baseDir, pending.dbPath))) { + throw new Error("Refusing to restore a database path outside the T3 home."); + } const backupDir = databaseBackupDir(baseDir, pending.id); if (!(await pathExists(backupDir))) return; @@ -592,196 +884,2183 @@ function waitForExit(child: NodeChildProcess.ChildProcess): Promise { async function terminateChild( child: NodeChildProcess.ChildProcess, - signal: NodeJS.Signals = "SIGTERM", + signal: NodeJS.Signals, + overIpc: boolean, ): Promise { if (child.exitCode !== null || child.signalCode !== null) return; - child.kill(signal); + let resend: NodeJS.Timeout | undefined; + if (overIpc) { + await sendMessage(child, { type: "stop" }).catch(() => child.kill(signal)); + // A child stopped while importing its entrypoint may not have attached its IPC listener yet. + resend = setInterval(() => { + if (child.exitCode === null && child.signalCode === null && child.connected) { + void sendMessage(child, { type: "stop" }).catch(() => undefined); + } + }, 200); + } else { + child.kill(signal); + } const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS); try { await waitForExit(child); } finally { + if (resend !== undefined) clearInterval(resend); clearTimeout(force); } } -const stopMarkerPath = (baseDir: string) => - NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); -const restartPendingPath = (baseDir: string) => - NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); +const sanitizeTaskNamePart = (value: string) => value.trim().replaceAll(/[\\/:*?"<>|]+/g, "_"); + +/** Task Scheduler names are machine-wide, so each Windows account runs its own collision-free task. */ +export const serviceTaskName = (domain: string, userName: string) => { + const trimmedDomain = domain.trim(); + const trimmedUser = userName.trim(); + const safeDomain = sanitizeTaskNamePart(trimmedDomain); + const safeUser = sanitizeTaskNamePart(trimmedUser); + const label = (safeDomain === "" ? safeUser : `${safeDomain}-${safeUser}`).slice(0, 64); + const suffix = NodeCrypto.createHash("sha256") + .update(`${trimmedDomain.toLowerCase()}\0${trimmedUser.toLowerCase()}`) + .digest("hex") + .slice(0, 8); + return `t3code-${label}-${suffix}`; +}; -export class Launcher { - readonly #baseDir: string; - readonly #statePath: string; - #state: ServiceState; - #child: ManagedChild | null = null; - #timer: NodeJS.Timeout | undefined; - #transitions: Promise = Promise.resolve(); - #stopRequested = false; - #stopping = false; - #done = false; - readonly #completion = Promise.withResolvers(); - readonly #backupSpaceProbe: Partial | undefined; - readonly #serviceUnit: ServiceUnit | undefined; - readonly #environment: Readonly; +const BOOT_SERVICE_TASK_NAME_FILE = "t3code-task-name"; - constructor( - baseDir: string, - state: ServiceState, - options: { - readonly backupSpaceProbe?: Partial; - readonly serviceUnit?: ServiceUnit; - readonly environment?: Readonly; - } = {}, - ) { - this.#baseDir = baseDir; - this.#backupSpaceProbe = options.backupSpaceProbe; - this.#serviceUnit = options.serviceUnit; - this.#environment = { ...(options.environment ?? process.env) }; - this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); - this.#state = state; - } +export const isValidServiceTaskName = (value: string): boolean => + /^t3code-[^\r\n/\\:*?"<>|]{1,80}$/.test(value); - async run(): Promise { - const onSigterm = () => void this.stop("SIGTERM"); - const onSigint = () => void this.stop("SIGINT"); - process.once("SIGTERM", onSigterm); - process.once("SIGINT", onSigint); - try { - this.#enqueue(() => this.#recover()); - await this.#completion.promise; - } finally { - process.off("SIGTERM", onSigterm); - process.off("SIGINT", onSigint); - } - } +export const serviceTaskNamePath = (localAppData: string) => + NodePath.join(localAppData, "t3code", BOOT_SERVICE_TASK_NAME_FILE); - #enqueue(transition: () => Promise): void { - this.#transitions = this.#transitions - .then(transition, transition) - .catch((cause: unknown) => - this.#fatal(cause instanceof Error ? cause : new Error(String(cause))), - ); +function resolvePersistedServiceTaskName(localAppData: string, fallbackTaskName: string): string { + try { + const persisted = NodeFS.readFileSync(serviceTaskNamePath(localAppData), "utf8").trim(); + return isValidServiceTaskName(persisted) ? persisted : fallbackTaskName; + } catch { + return fallbackTaskName; } +} - async #fatal(error: Error): Promise { - if (this.#done) return; - this.#done = true; - this.#stopping = true; - this.#clearTimer(); - const child = this.#child?.process; - this.#child = null; - if (child !== undefined) await terminateChild(child); - this.#completion.reject(error); - } +export const serviceAccountControlRoot = (localAppData: string) => + NodePath.join(localAppData, "t3code", "control"); - async stop(signal: NodeJS.Signals): Promise { - // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer. - try { - NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); - } catch {} - if (this.#stopRequested || this.#stopping) { - await this.#completion.promise.catch(() => undefined); - return; - } - this.#stopRequested = true; - this.#clearTimer(); - this.#enqueue(async () => { - // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. - this.#stopping = true; - const child = this.#child?.process; - this.#child = null; - if (child !== undefined) await terminateChild(child, signal); - this.#done = true; - this.#completion.resolve(); - }); - await this.#completion.promise.catch(() => undefined); - } +/** Stored under user-private `%LOCALAPPDATA%` rather than `baseDir`, which may be shared. */ +export const serviceControlDir = ( + localAppData: string, + baseDir: string, + options?: { readonly caseInsensitive?: boolean | undefined }, +) => { + const resolved = NodePath.resolve(baseDir); + const normalized = (options?.caseInsensitive ?? IS_WINDOWS) ? resolved.toLowerCase() : resolved; + const id = NodeCrypto.createHash("sha256").update(normalized).digest("hex").slice(0, 24); + return NodePath.join(serviceAccountControlRoot(localAppData), id); +}; - #clearTimer(): void { - clearTimeout(this.#timer); - this.#timer = undefined; - } +export interface ControlAclHooks { + readonly restrictPathAcl: (targetPath: string, isDirectory: boolean) => Promise; + readonly verifyPathAcl: (targetPath: string, isDirectory: boolean) => Promise; + readonly verifyPathsAcl?: + | (( + targets: ReadonlyArray<{ readonly path: string; readonly isDirectory: boolean }>, + ) => Promise) + | undefined; +} - async #selectState(next: ServiceState): Promise { - const release = await acquireServiceSelectionLock(this.#statePath); - try { - const selected = await readServiceState(this.#statePath); - // Local installs replace the selection and clear its update record. An old - // launcher must not restore its own selection when asked to update again. - if ( - selected.activeVersion !== this.#state.activeVersion || - selected.update?.id !== this.#state.update?.id || - selected.update?.status !== this.#state.update?.status - ) { - throw new Error("Service selection changed. Restart the service before updating."); - } - await writeServiceState(this.#statePath, next); - this.#state = next; - } finally { - release(); - } - } +export function windowsSystem32Binary( + name: string, + environment: Readonly = process.env, +): string { + const systemRoot = (environment.SystemRoot ?? "").trim() || "C:\\Windows"; + return NodePath.win32.join(systemRoot, "System32", name); +} - async #recover(): Promise { - // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. - await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); - const restartPending = restartPendingPath(this.#baseDir); - try { - const release = await acquireServiceSelectionLock(this.#statePath); - try { - const selected = await readServiceState(this.#statePath); - const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch( - () => undefined, - ); - const awaited = awaitedVersion?.trim(); - if ( - selected.activeVersion === this.#state.activeVersion && - selected.update?.id === this.#state.update?.id && - selected.update?.status === this.#state.update?.status && - awaited !== undefined && - isExactServiceVersion(awaited) && - compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 - ) { - await NodeFSP.rm(restartPending, { force: true }); +export function windowsPowerShellBinary( + environment: Readonly = process.env, +): string { + const systemRoot = (environment.SystemRoot ?? "").trim() || "C:\\Windows"; + return NodePath.win32.join(systemRoot, "System32", "WindowsPowerShell", "v1.0", "powershell.exe"); +} + +/** + * Verifies a full Windows security descriptor SDDL (`O:...D:...`) is owned by `SYSTEM`, + * `Administrators`, or the current user SID, is protected from inheritance (`D:P`), and grants + * access only to `SYSTEM`, `Administrators`, and the current user SID (with `OI`+`CI` on directories). + */ +export function isPrivateWindowsControlSddl( + sddlText: string, + userSid: string, + options?: { + readonly isDirectory?: boolean | undefined; + readonly allowLocalAdminAlias?: boolean | undefined; + }, +): boolean { + const expectedUser = userSid.trim().toUpperCase(); + if (!/^S-1-[0-9-]+$/.test(expectedUser)) return false; + const allowLocalAdminAlias = options?.allowLocalAdminAlias === true; + const allowedTrustees = new Set([ + "SY", + "S-1-5-18", + "BA", + "S-1-5-32-544", + expectedUser, + ...(allowLocalAdminAlias ? ["LA"] : []), + ]); + const isExpectedUserTrustee = (trustee: string) => + trustee === expectedUser || (allowLocalAdminAlias && trustee === "LA"); + const trimmed = sddlText.replaceAll("\0", "").trim(); + if (trimmed === "") return false; + const sections = Array.from(trimmed.matchAll(/([OGDS]):(.*?)(?=[OGDS]:|$)/gi)); + if (sections.length === 0 || sections.map((section) => section[0]).join("") !== trimmed) { + return false; + } + let ownerOk = false; + let daclOk = false; + for (const section of sections) { + const tag = section[1]!.toUpperCase(); + const body = section[2]!.trim(); + if (tag === "O") { + if (ownerOk || !allowedTrustees.has(body.toUpperCase())) return false; + ownerOk = true; + } else if (tag === "D") { + if (daclOk) return false; + const match = /^([A-Z]*)((?:\([^()]+\))+)$/i.exec(body); + if (match === null) return false; + const flags = (match[1] ?? "").toUpperCase(); + const aceList = match[2] ?? ""; + if (!flags.includes("P")) return false; + const aces = Array.from(aceList.matchAll(/\(([^()]+)\)/g), (aceMatch) => aceMatch[1]!); + if (aces.length === 0) return false; + let hasUserFullControl = false; + for (const ace of aces) { + const parts = ace.split(";"); + if (parts.length !== 6) return false; + const aceType = (parts[0] ?? "").trim().toUpperCase(); + const aceFlags = (parts[1] ?? "").trim().toUpperCase(); + const rights = (parts[2] ?? "").trim().toUpperCase(); + const trustee = (parts[5] ?? "").trim().toUpperCase(); + if (aceType !== "A" || !allowedTrustees.has(trustee)) return false; + if (isExpectedUserTrustee(trustee)) { + if ( + options?.isDirectory === true && + (!aceFlags.includes("OI") || !aceFlags.includes("CI")) + ) { + return false; + } + if (rights === "FA" || rights === "GA" || rights === "0X1F01FF") { + hasUserFullControl = true; + } } - } finally { - release(); } - } catch (error) { - process.stderr.write( - `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`, - ); + if (!hasUserFullControl) return false; + daclOk = true; } - const update = this.#state.update; - if (update?.status !== "pending") { - if (update !== undefined) { - await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); - } - try { - await this.#startChild(this.#state.activeVersion, "active", update); - } finally { - if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion); + } + return ownerOk && daclOk; +} + +const execFileAsync = NodeUtil.promisify(NodeChildProcess.execFile); +let cachedWindowsIdentityPromise: + | Promise<{ readonly sid: string; readonly allowLocalAdminAlias: boolean }> + | undefined; + +function readCurrentWindowsIdentity(): Promise<{ + readonly sid: string; + readonly allowLocalAdminAlias: boolean; +}> { + cachedWindowsIdentityPromise ??= execFileAsync( + windowsPowerShellBinary(), + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'; [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $sid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value; $la = try { ([System.Security.Principal.SecurityIdentifier]::new('LA')).Value } catch { '' }; [Console]::Out.Write(\"$sid`n$la\")", + ], + { + windowsHide: true, + timeout: 10_000, + }, + ) + .then(({ stdout }) => { + const [rawSid = "", rawLocalAdminSid = ""] = stdout.split(/\r?\n/); + const sid = rawSid.trim(); + const localAdminSid = rawLocalAdminSid.trim(); + if (!/^S-1-[0-9-]+$/i.test(sid)) { + throw new Error("Could not resolve current Windows user SID."); } - return; + return { + sid, + allowLocalAdminAlias: + localAdminSid !== "" && sid.toUpperCase() === localAdminSid.toUpperCase(), + }; + }) + .catch((cause) => { + cachedWindowsIdentityPromise = undefined; + throw cause; + }); + return cachedWindowsIdentityPromise; +} + +const defaultWindowsControlAclHooks: ControlAclHooks = { + async restrictPathAcl(targetPath, isDirectory) { + const { sid, allowLocalAdminAlias } = await readCurrentWindowsIdentity(); + const aceFlags = isDirectory ? "OICI" : ""; + const dacl = `D:P(A;${aceFlags};FA;;;SY)(A;${aceFlags};FA;;;BA)(A;${aceFlags};FA;;;${sid})`; + const { stdout } = await execFileAsync( + windowsPowerShellBinary(), + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'; [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $acl = Get-Acl -LiteralPath $env:T3_ACL_TARGET; $acl.SetSecurityDescriptorSddlForm($env:T3_ACL_SDDL, [System.Security.AccessControl.AccessControlSections]::Access); Set-Acl -LiteralPath $env:T3_ACL_TARGET -AclObject $acl; [Console]::Out.Write((Get-Acl -LiteralPath $env:T3_ACL_TARGET).Sddl)", + ], + { + env: { ...process.env, T3_ACL_TARGET: targetPath, T3_ACL_SDDL: dacl }, + windowsHide: true, + timeout: 15_000, + }, + ); + if (!isPrivateWindowsControlSddl(stdout, sid, { isDirectory, allowLocalAdminAlias })) { + throw createControlRejectedError("unauthorized"); } - if (await databaseRestorePending(this.#baseDir, update)) { - await this.#returnToPrevious(update, "failed", "rollback-interrupted"); - return; + }, + async verifyPathAcl(targetPath, isDirectory) { + const { sid, allowLocalAdminAlias } = await readCurrentWindowsIdentity(); + try { + const { stdout } = await execFileAsync( + windowsPowerShellBinary(), + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'; [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); [Console]::Out.Write((Get-Acl -LiteralPath $env:T3_ACL_TARGET).Sddl)", + ], + { + env: { ...process.env, T3_ACL_TARGET: targetPath }, + windowsHide: true, + timeout: 15_000, + }, + ); + return isPrivateWindowsControlSddl(stdout, sid, { isDirectory, allowLocalAdminAlias }); + } catch { + return false; } - if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { - await this.#returnToPrevious(update, "failed", "target-runtime-missing"); - return; + }, + async verifyPathsAcl(targets) { + if (targets.length === 0) return true; + if (targets.length === 1) { + return defaultWindowsControlAclHooks.verifyPathAcl(targets[0]!.path, targets[0]!.isDirectory); } - await this.#startTrial(update); - } - - async #startTrial(pending: PendingServiceUpdate): Promise { + const { sid, allowLocalAdminAlias } = await readCurrentWindowsIdentity(); try { - await backupDatabaseOnce(this.#baseDir, pending); + const { stdout } = await execFileAsync( + windowsPowerShellBinary(), + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + "$ErrorActionPreference = 'Stop'; [Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false); $paths = @(ConvertFrom-Json -InputObject $env:T3_ACL_TARGETS); $sddls = foreach ($p in $paths) { (Get-Acl -LiteralPath ([string]$p)).Sddl }; [Console]::Out.Write(($sddls -join \"`n\"))", + ], + { + env: { ...process.env, T3_ACL_TARGETS: JSON.stringify(targets.map((item) => item.path)) }, + windowsHide: true, + timeout: 15_000, + }, + ); + const lines = stdout.split(/\r?\n/); + if (lines.length !== targets.length) return false; + return targets.every((target, index) => + isPrivateWindowsControlSddl(lines[index] ?? "", sid, { + isDirectory: target.isDirectory, + allowLocalAdminAlias, + }), + ); } catch { + return false; + } + }, +}; + +const resolveControlAclHooks = (override?: ControlAclHooks): ControlAclHooks | undefined => + override ?? (IS_WINDOWS ? defaultWindowsControlAclHooks : undefined); + +const isMissingControl = (error: unknown) => { + const code = (error as NodeJS.ErrnoException).code; + return code === "ENOENT" || code === "ECONNREFUSED"; +}; + +const isSqliteBusy = (cause: unknown) => + cause instanceof Error && + (("errcode" in cause && cause.errcode === 5) || + ("code" in cause && cause.code === "SQLITE_BUSY")); + +const CONTROL_REJECTED_UNAUTHORIZED_CODE = "ERR_T3_CONTROL_UNAUTHORIZED"; +const CONTROL_REJECTED_CODE = "ERR_T3_CONTROL_REJECTED"; +const CONTROL_CLOSED_EARLY_CODE = "ERR_T3_CONTROL_CLOSED_EARLY"; +const CONTROL_LAUNCHER_MISMATCH_CODE = "ERR_T3_CONTROL_LAUNCHER_MISMATCH"; +const CONTROL_ABORTED_CODE = "ABORT_ERR"; + +function createControlError(message: string, code: string): NodeJS.ErrnoException { + const error: NodeJS.ErrnoException = new Error(message); + error.code = code; + return error; +} + +const createControlRejectedError = (reason: string) => + createControlError( + `Service launcher rejected control request (${reason}).`, + reason === "unauthorized" ? CONTROL_REJECTED_UNAUTHORIZED_CODE : CONTROL_REJECTED_CODE, + ); + +const createControlClosedEarlyError = () => + createControlError( + "Service launcher closed control connection before replying.", + CONTROL_CLOSED_EARLY_CODE, + ); + +const createControlLauncherMismatchError = () => + createControlError( + "Service launcher control reply did not match the published launcher instance.", + CONTROL_LAUNCHER_MISMATCH_CODE, + ); + +const createControlAbortedError = () => + createControlError("Service launcher control operation was aborted.", CONTROL_ABORTED_CODE); + +const isTransientControlClose = (error: unknown) => { + const code = (error as NodeJS.ErrnoException).code; + return code === CONTROL_CLOSED_EARLY_CODE || code === "ECONNRESET" || code === "EPIPE"; +}; + +export const isServiceControlUnauthorizedError = (error: unknown): boolean => + (error as NodeJS.ErrnoException | undefined)?.code === CONTROL_REJECTED_UNAUTHORIZED_CODE; + +export const isServiceControlLauncherMismatchError = (error: unknown): boolean => + (error as NodeJS.ErrnoException | undefined)?.code === CONTROL_LAUNCHER_MISMATCH_CODE; + +const stopMarkerPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); +const restartPendingPath = (baseDir: string) => + NodePath.join(baseDir, "runtime", SERVICE_RESTART_PENDING_FILE); +const controlLockPath = (controlDir: string) => NodePath.join(controlDir, "lock.sqlite"); +const controlTokenPath = (controlDir: string) => NodePath.join(controlDir, "control.token"); +const controlEndpointPath = (controlDir: string) => NodePath.join(controlDir, "endpoint.json"); + +async function waitForControlDelay(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted === true) throw createControlAbortedError(); + await new Promise((resolve, reject) => { + const onAbort = () => { + clearTimeout(timer); + reject(createControlAbortedError()); + }; + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +/** + * Exclusive SQLite transaction lock (`BEGIN EXCLUSIVE`), matching `server-owner.sqlite`: the OS + * releases the file lock automatically if the holder crashes or is terminated, and the file is + * never unlinked. + */ +function tryAcquireSqliteExclusiveLock( + lockPath: string, + fileMode = 0o600, +): { readonly close: () => void } | undefined { + NodeFS.mkdirSync(NodePath.dirname(lockPath), { recursive: true, mode: 0o700 }); + if (!NodeFS.existsSync(lockPath)) { + NodeFS.closeSync(NodeFS.openSync(lockPath, "a", fileMode)); + } + const db = new NodeSqlite.DatabaseSync(lockPath); + try { + db.exec("PRAGMA busy_timeout = 0; BEGIN EXCLUSIVE;"); + return { + close: () => { + try { + db.close(); + } catch { + // Already closed. + } + }, + }; + } catch (cause) { + try { + db.close(); + } catch { + // Ignore close error after failed BEGIN EXCLUSIVE. + } + if (isSqliteBusy(cause)) return undefined; + throw cause; + } +} + +/** Acquires the same `server-owner.sqlite` lock held by foreground and service servers. */ +function tryAcquireServerOwnerLock(dbPath: string): { readonly close: () => void } | undefined { + return tryAcquireSqliteExclusiveLock( + NodePath.join(NodePath.dirname(dbPath), "server-owner.sqlite"), + 0o644, + ); +} + +export function probeControlLockHeld(controlDir: string): boolean { + const lockPath = controlLockPath(controlDir); + if (!NodeFS.existsSync(lockPath)) return false; + const lock = tryAcquireSqliteExclusiveLock(lockPath); + if (lock === undefined) return true; + lock.close(); + return false; +} + +async function acquireSqliteExclusiveLock( + lockPath: string, + options: { + readonly timeoutMs: number; + readonly retryMs: number; + readonly signal?: AbortSignal | undefined; + readonly onRetry?: ((attempt: number) => void) | undefined; + }, +): Promise<{ readonly close: () => void }> { + const deadline = performance.now() + Math.max(0, options.timeoutMs); + for (let attempt = 1; ; attempt++) { + if (options.signal?.aborted === true) throw createControlAbortedError(); + const lock = tryAcquireSqliteExclusiveLock(lockPath); + if (lock !== undefined) return lock; + options.onRetry?.(attempt); + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) break; + await waitForControlDelay(Math.min(options.retryMs, remainingMs), options.signal); + } + throw new Error("Timed out waiting to acquire the service launcher control lock."); +} + +async function verifyControlPathsAcl( + targets: ReadonlyArray<{ readonly path: string; readonly isDirectory: boolean }>, + acl?: ControlAclHooks, +): Promise { + if (acl === undefined || targets.length === 0) return true; + if (acl.verifyPathsAcl !== undefined) { + return acl.verifyPathsAcl(targets); + } + for (const target of targets) { + if (!(await acl.verifyPathAcl(target.path, target.isDirectory))) return false; + } + return true; +} + +async function ensurePrivateControlLockReady( + directoryPath: string, + lockFileName: string, + acl?: ControlAclHooks, +): Promise { + await NodeFSP.mkdir(directoryPath, { recursive: true, mode: 0o700 }); + if (acl !== undefined && !(await acl.verifyPathAcl(directoryPath, true))) { + await acl.restrictPathAcl(directoryPath, true); + } + const lockPath = NodePath.join(directoryPath, lockFileName); + if (!NodeFS.existsSync(lockPath)) { + NodeFS.closeSync(NodeFS.openSync(lockPath, "a", 0o600)); + } + if (acl !== undefined && !(await acl.verifyPathAcl(lockPath, false))) { + await acl.restrictPathAcl(lockPath, false); + } + return lockPath; +} + +async function verifyControlLockAcl(controlDir: string, acl?: ControlAclHooks): Promise { + if (acl === undefined || !NodeFS.existsSync(controlDir)) return true; + const lockPath = controlLockPath(controlDir); + return verifyControlPathsAcl( + [ + { path: controlDir, isDirectory: true }, + ...(NodeFS.existsSync(lockPath) ? [{ path: lockPath, isDirectory: false }] : []), + ], + acl, + ); +} + +export async function acquireServiceAccountLock( + accountControlRoot: string, + options: { + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; + } = {}, +): Promise<{ readonly close: () => void }> { + const acl = resolveControlAclHooks(options.acl); + const lockPath = await ensurePrivateControlLockReady( + accountControlRoot, + "account-task.sqlite", + acl, + ); + return acquireSqliteExclusiveLock(lockPath, { + timeoutMs: options.timeoutMs ?? CONTROL_MUTATION_TIMEOUT_MS, + retryMs: 50, + signal: options.signal, + }); +} + +async function readVerifiedControlToken( + controlDir: string, + acl: ControlAclHooks | undefined, + alreadyVerifiedAcl = false, +): Promise { + const targetPath = controlTokenPath(controlDir); + if (!NodeFS.existsSync(targetPath)) return undefined; + if (!alreadyVerifiedAcl && acl !== undefined && !(await acl.verifyPathAcl(targetPath, false))) { + return undefined; + } + try { + const token = (await NodeFSP.readFile(targetPath, "utf8")).trim(); + if (!/^[0-9a-f]{64}$/.test(token)) return undefined; + return token; + } catch (cause) { + const code = (cause as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "EACCES" || code === "EPERM") return undefined; + throw cause; + } +} + +async function ensureControlToken(controlDir: string, acl?: ControlAclHooks): Promise { + await NodeFSP.mkdir(controlDir, { recursive: true, mode: 0o700 }); + if (acl !== undefined && !(await acl.verifyPathAcl(controlDir, true))) { + await acl.restrictPathAcl(controlDir, true); + } + const existing = await readVerifiedControlToken(controlDir, acl); + if (existing !== undefined) return existing; + const targetPath = controlTokenPath(controlDir); + const tempPath = NodePath.join( + controlDir, + `.control-token.${process.pid}.${NodeCrypto.randomUUID()}`, + ); + const token = NodeCrypto.randomBytes(32).toString("hex"); + let handle: NodeFSP.FileHandle | undefined; + try { + // Created inside the verified `OICI` private directory before writing secret bytes. + handle = await NodeFSP.open(tempPath, "wx", 0o600); + await handle.writeFile(`${token}\n`, "utf8"); + await handle.sync(); + await handle.close(); + handle = undefined; + if (acl !== undefined) { + await acl.restrictPathAcl(tempPath, false); + } + try { + await NodeFSP.rename(tempPath, targetPath); + } catch (cause) { + const code = (cause as NodeJS.ErrnoException).code; + if (code !== "EACCES" && code !== "EPERM") throw cause; + await NodeFSP.rm(targetPath, { force: true }); + await NodeFSP.rename(tempPath, targetPath); + } + await syncDirectory(controlDir); + if (acl !== undefined && !(await acl.verifyPathAcl(targetPath, false))) { + throw new Error("Could not verify owner-only Windows ACL on service control token."); + } + return token; + } finally { + await handle?.close().catch(() => undefined); + await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); + } +} + +interface ControlEndpoint { + readonly port: number; + readonly launcherId: string; +} + +async function readControlEndpoint(controlDir: string): Promise { + try { + const raw = await NodeFSP.readFile(controlEndpointPath(controlDir), "utf8"); + const parsed: unknown = JSON.parse(raw); + if ( + typeof parsed !== "object" || + parsed === null || + Array.isArray(parsed) || + typeof (parsed as { readonly port?: unknown }).port !== "number" || + !Number.isInteger((parsed as { readonly port: number }).port) || + (parsed as { readonly port: number }).port < 1 || + (parsed as { readonly port: number }).port > 65_535 || + typeof (parsed as { readonly launcherId?: unknown }).launcherId !== "string" + ) { + throw createControlError("Invalid service launcher control endpoint.", CONTROL_REJECTED_CODE); + } + return parsed as ControlEndpoint; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw cause; + } +} + +async function writeControlEndpoint( + controlDir: string, + endpoint: ControlEndpoint, + acl?: ControlAclHooks, +): Promise { + const targetPath = controlEndpointPath(controlDir); + const tempPath = NodePath.join( + controlDir, + `.endpoint.${process.pid}.${NodeCrypto.randomUUID()}.tmp`, + ); + try { + await NodeFSP.writeFile(tempPath, `${JSON.stringify(endpoint)}\n`, { mode: 0o600 }); + if (acl !== undefined) { + await acl.restrictPathAcl(tempPath, false); + } + await NodeFSP.rename(tempPath, targetPath); + } finally { + await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); + } +} + +const computeControlMac = ( + token: string, + domain: "req" | "res", + serverNonce: string, + clientNonce: string, + body: string, +) => + NodeCrypto.createHmac("sha256", token) + .update(`${domain}\0${serverNonce}\0${clientNonce}\0${body}`) + .digest("hex"); + +function hexDigestsMatch(expectedHex: string, provided: unknown): boolean { + if (typeof provided !== "string" || provided.length !== expectedHex.length) return false; + const expectedBytes = Buffer.from(expectedHex, "utf8"); + const providedBytes = Buffer.from(provided, "utf8"); + return ( + expectedBytes.length === providedBytes.length && + NodeCrypto.timingSafeEqual(expectedBytes, providedBytes) + ); +} + +export type ControlStageInstallResult = + | { readonly status: "staged"; readonly launcherId?: string } + | { readonly status: "update-pending"; readonly installedVersion?: string } + | { readonly status: "downgrade-refused"; readonly installedVersion: string }; + +type LauncherStageInstallResult = ControlStageInstallResult | { readonly status: "not-running" }; + +type ControlReply = + | { + readonly ok: true; + readonly launcherId: string; + readonly result?: LauncherStageInstallResult; + } + | { + readonly ok: false; + readonly error: string; + }; + +/** + * Connects over loopback TCP (`127.0.0.1`) rather than a Windows named pipe so connecting and + * writing never expose a named-pipe security token to `ImpersonateNamedPipeClient`, and binds + * both request and response MACs to fresh per-connection server and client nonces. + */ +async function sendControlRequestToPort( + port: number, + token: string | undefined, + payload: Record, + options: { + readonly expectedLauncherId?: string | undefined; + readonly waitForClose?: boolean | undefined; + readonly timeoutMs: number; + readonly signal?: AbortSignal | undefined; + }, +): Promise { + if (options.signal?.aborted === true) throw createControlAbortedError(); + if (token === undefined) throw createControlRejectedError("unauthorized"); + return new Promise((resolve, reject) => { + let connected = false; + let buffer = ""; + let serverNonce: string | undefined; + let clientNonce: string | undefined; + let reply: ControlReply | undefined; + let failure: Error | undefined; + const failWith = (error: Error) => { + if (failure === undefined) failure = error; + socket.destroy(); + }; + const socket = NodeNet.createConnection({ host: "127.0.0.1", port }, () => { + connected = true; + }); + const onAbort = () => { + failWith(createControlAbortedError()); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + const deadline = setTimeout(() => { + failWith(new Error("Timed out waiting for the service launcher control channel.")); + }, options.timeoutMs); + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => { + if (reply !== undefined || failure !== undefined) return; + if (buffer.length + chunk.length > MAX_CONTROL_MESSAGE_BYTES) { + failWith(new Error("Service launcher control message exceeded maximum size.")); + return; + } + buffer += chunk; + for (;;) { + const newline = buffer.indexOf("\n"); + if (newline === -1) return; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (serverNonce === undefined) { + try { + const challenge = JSON.parse(line) as { readonly nonce?: unknown }; + if (typeof challenge.nonce !== "string" || !/^[0-9a-f]{32}$/.test(challenge.nonce)) { + failWith(new Error("Service launcher sent an invalid control challenge.")); + return; + } + serverNonce = challenge.nonce; + clientNonce = NodeCrypto.randomBytes(16).toString("hex"); + const body = JSON.stringify(payload); + const mac = computeControlMac(token, "req", serverNonce, clientNonce, body); + // Not end(): when waiting for exit, a half-close closes the socket before shutdown completes. + socket.write( + `${JSON.stringify({ snonce: serverNonce, cnonce: clientNonce, payload: body, mac })}\n`, + ); + } catch (cause) { + failWith(cause instanceof Error ? cause : new Error(String(cause))); + return; + } + continue; + } + try { + const envelope = JSON.parse(line) as { + readonly body?: unknown; + readonly mac?: unknown; + }; + if ( + clientNonce === undefined || + typeof envelope.body !== "string" || + !hexDigestsMatch( + computeControlMac(token, "res", serverNonce, clientNonce, envelope.body), + envelope.mac, + ) + ) { + failWith(createControlRejectedError("unauthorized")); + return; + } + const parsed = JSON.parse(envelope.body) as ControlReply; + if (parsed.ok === false) { + failWith(createControlRejectedError(parsed.error)); + return; + } + if ( + options.expectedLauncherId !== undefined && + parsed.launcherId !== options.expectedLauncherId + ) { + failWith(createControlLauncherMismatchError()); + return; + } + reply = parsed; + if (options.waitForClose !== true) { + clearTimeout(deadline); + socket.destroy(); + resolve(parsed); + } + return; + } catch (cause) { + failWith(cause instanceof Error ? cause : new Error(String(cause))); + return; + } + } + }); + socket.on("error", (error) => { + if (!connected) { + clearTimeout(deadline); + options.signal?.removeEventListener("abort", onAbort); + if (failure !== undefined) reject(failure); + else if (isMissingControl(error)) resolve(undefined); + else reject(error); + } else if (failure === undefined && reply === undefined) { + failure = error; + } + }); + socket.once("close", () => { + clearTimeout(deadline); + options.signal?.removeEventListener("abort", onAbort); + if (failure !== undefined) { + reject(failure); + return; + } + if (!connected) return; + if (reply !== undefined) { + resolve(reply); + } else { + reject(createControlClosedEarlyError()); + } + }); + }); +} + +export interface ControlRequestOptions { + readonly waitForClose?: boolean | undefined; + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; +} + +async function readVerifiedEndpointAndToken( + controlDir: string, + acl?: ControlAclHooks, +): Promise<{ readonly endpoint: ControlEndpoint; readonly token: string | undefined } | undefined> { + const endpointPath = controlEndpointPath(controlDir); + if (!NodeFS.existsSync(endpointPath)) return undefined; + const tokenPath = controlTokenPath(controlDir); + if (!NodeFS.existsSync(tokenPath)) { + throw createControlRejectedError("unauthorized"); + } + if ( + !(await verifyControlPathsAcl( + [ + { path: endpointPath, isDirectory: false }, + { path: tokenPath, isDirectory: false }, + ], + acl, + )) + ) { + throw createControlRejectedError("unauthorized"); + } + const endpoint = await readControlEndpoint(controlDir); + if (endpoint === undefined) return undefined; + const token = await readVerifiedControlToken(controlDir, acl, true); + return { endpoint, token }; +} + +async function sendControlRequest( + controlDir: string, + payload: Record, + options: ControlRequestOptions & { readonly skipLockAclVerify?: boolean | undefined } = {}, +): Promise { + if (options.signal?.aborted === true) throw createControlAbortedError(); + const acl = resolveControlAclHooks(options.acl); + if (options.skipLockAclVerify !== true && !(await verifyControlLockAcl(controlDir, acl))) { + throw createControlRejectedError("unauthorized"); + } + if (!probeControlLockHeld(controlDir)) return undefined; + const verified = await readVerifiedEndpointAndToken(controlDir, acl); + if (verified === undefined) return undefined; + const timeoutMs = + options.timeoutMs ?? + (options.waitForClose === true + ? CONTROL_STOP_REQUEST_TIMEOUT_MS + : payload.type === "stage-install" + ? CONTROL_MUTATION_TIMEOUT_MS + : CONTROL_QUERY_TIMEOUT_MS); + try { + return await sendControlRequestToPort(verified.endpoint.port, verified.token, payload, { + expectedLauncherId: verified.endpoint.launcherId, + waitForClose: options.waitForClose, + timeoutMs, + signal: options.signal, + }); + } catch (error) { + // Home-scoped requests (`payload.launcherId === undefined`) intentionally retry once onto a + // replacement launcher that acquired this home's lock during handoff; instance-targeted + // `signalServiceLauncherStop` carries `launcherId`, which a replacement safely ignores. + if ( + (!isTransientControlClose(error) && !isServiceControlLauncherMismatchError(error)) || + !probeControlLockHeld(controlDir) + ) { + throw error; + } + if (!(await verifyControlLockAcl(controlDir, acl))) { + throw createControlRejectedError("unauthorized"); + } + const retryVerified = await readVerifiedEndpointAndToken(controlDir, acl); + if (retryVerified === undefined) return undefined; + return sendControlRequestToPort(retryVerified.endpoint.port, retryVerified.token, payload, { + expectedLauncherId: retryVerified.endpoint.launcherId, + waitForClose: options.waitForClose, + timeoutMs, + signal: options.signal, + }); + } +} + +/** Waits until no process holds the per-home control lock. */ +export async function waitForServiceLauncherRelease( + controlDir: string, + options: { + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; + } = {}, +): Promise { + const acl = resolveControlAclHooks(options.acl); + if (!(await verifyControlLockAcl(controlDir, acl))) { + throw createControlRejectedError("unauthorized"); + } + const timeoutMs = options.timeoutMs ?? CONTROL_MUTATION_TIMEOUT_MS; + const deadline = performance.now() + Math.max(0, timeoutMs); + for (;;) { + if (options.signal?.aborted === true) throw createControlAbortedError(); + if (!probeControlLockHeld(controlDir)) return; + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) break; + await waitForControlDelay(Math.min(50, remainingMs), options.signal); + } + throw new Error("Timed out waiting for the service launcher to release its control lock."); +} + +/** Asks the launcher to stop and resolves once its control lock is released; false when none was running. */ +export async function requestServiceLauncherStop( + controlDir: string, + options: Omit & { + readonly publishTimeoutMs?: number | undefined; + } = {}, +): Promise { + const acl = resolveControlAclHooks(options.acl); + if (!(await verifyControlLockAcl(controlDir, acl))) { + throw createControlRejectedError("unauthorized"); + } + if (!probeControlLockHeld(controlDir)) return false; + const publishDeadline = performance.now() + Math.max(0, options.publishTimeoutMs ?? 5_000); + for (;;) { + const reply = await sendControlRequest( + controlDir, + { type: "stop", wait: true }, + { ...options, acl, waitForClose: true, skipLockAclVerify: true }, + ); + if (reply !== undefined) { + await waitForServiceLauncherRelease(controlDir, { + timeoutMs: 5_000, + signal: options.signal, + acl, + }); + return true; + } + if (!probeControlLockHeld(controlDir)) return true; + const remainingMs = publishDeadline - performance.now(); + if (remainingMs <= 0) break; + await waitForControlDelay(Math.min(50, remainingMs), options.signal); + } + throw new Error("Timed out waiting for the starting service launcher to publish its endpoint."); +} + +/** Enqueues a graceful launcher stop without waiting for process exit; targets a specific instance when given. */ +export async function signalServiceLauncherStop( + controlDir: string, + launcherId?: string, + options: Omit & { + readonly uninstall?: boolean | undefined; + } = {}, +): Promise { + const reply = await sendControlRequest( + controlDir, + { + type: "stop", + wait: false, + ...(launcherId === undefined ? {} : { launcherId }), + ...(options.uninstall === true ? { uninstall: true } : {}), + }, + options, + ); + return reply !== undefined; +} + +/** Reads the active launcher instance identifier, or undefined when none is listening. */ +export async function queryServiceLauncherId( + controlDir: string, + options: Omit = {}, +): Promise { + const reply = await sendControlRequest(controlDir, { type: "identify" }, options); + return reply?.ok === true ? reply.launcherId : undefined; +} + +const STOPPED_DB_DIGEST_FILE = "stopped-db-digest.json"; +const MAX_FULL_FINGERPRINT_BYTES = 64 * 1024 * 1024; +const FINGERPRINT_CHUNK_BYTES = 64 * 1024; + +interface StoppedDatabaseFileFingerprint { + readonly suffix: string; + readonly exists: boolean; + readonly size?: number | undefined; + readonly mtimeNs?: string | undefined; + readonly sha256?: string | undefined; +} + +async function fingerprintDatabaseFiles( + dbPath: string, +): Promise> { + const mainTarget = await restoreTarget(dbPath).catch(() => dbPath); + const entries: StoppedDatabaseFileFingerprint[] = []; + const chunk = Buffer.allocUnsafe(FINGERPRINT_CHUNK_BYTES); + for (const suffix of DB_FILE_SUFFIXES) { + const livePath = suffix === "" ? mainTarget : `${mainTarget}${suffix}`; + let handle: NodeFSP.FileHandle | undefined; + try { + const stat = await NodeFSP.stat(livePath, { bigint: true }); + handle = await NodeFSP.open(livePath, "r"); + const size = Number(stat.size); + const hash = NodeCrypto.createHash("sha256"); + if (size <= MAX_FULL_FINGERPRINT_BYTES) { + let offset = 0; + while (offset < size) { + const toRead = Math.min(FINGERPRINT_CHUNK_BYTES, size - offset); + const { bytesRead } = await handle.read(chunk, 0, toRead, offset); + if (bytesRead === 0) break; + hash.update(chunk.subarray(0, bytesRead)); + offset += bytesRead; + } + } else { + const headLen = Math.min(FINGERPRINT_CHUNK_BYTES, size); + const tailOffset = Math.max(headLen, size - FINGERPRINT_CHUNK_BYTES); + const tailLen = Math.max(0, size - tailOffset); + if (headLen > 0) { + const { bytesRead } = await handle.read(chunk, 0, headLen, 0); + hash.update(chunk.subarray(0, bytesRead)); + } + if (tailLen > 0) { + const { bytesRead } = await handle.read(chunk, 0, tailLen, tailOffset); + hash.update(chunk.subarray(0, bytesRead)); + } + } + entries.push({ + suffix, + exists: true, + size, + mtimeNs: stat.mtimeNs.toString(), + sha256: hash.digest("hex"), + }); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") { + entries.push({ suffix, exists: false }); + } else { + throw cause; + } + } finally { + await handle?.close().catch(() => undefined); + } + } + return entries; +} + +async function recordStoppedPendingDatabaseFingerprint( + baseDir: string, + pending: PendingServiceUpdate, +): Promise { + if ( + !isValidServiceUpdateId(pending.id) || + !(await isDatabaseRestoreTargetInsideBaseDir(baseDir, pending.dbPath)) + ) { + return; + } + const backupDir = databaseBackupDir(baseDir, pending.id); + if (!(await pathExists(backupDir))) return; + const files = await fingerprintDatabaseFiles(pending.dbPath); + await replaceFileDurably( + NodePath.join(backupDir, STOPPED_DB_DIGEST_FILE), + `${JSON.stringify({ updateId: pending.id, files })}\n`, + ); +} + +async function stoppedDigestMatchesCurrentDatabase( + baseDir: string, + pending: PendingServiceUpdate, + options?: { readonly requireDigest?: boolean | undefined }, +): Promise { + const backupDir = databaseBackupDir(baseDir, pending.id); + const digestPath = NodePath.join(backupDir, STOPPED_DB_DIGEST_FILE); + let rawDigest: string; + try { + rawDigest = await NodeFSP.readFile(digestPath, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") { + return options?.requireDigest !== true; + } + return false; + } + try { + const parsed = JSON.parse(rawDigest) as { + readonly updateId?: unknown; + readonly files?: unknown; + } | null; + if ( + typeof parsed !== "object" || + parsed === null || + parsed.updateId !== pending.id || + !Array.isArray(parsed.files) + ) { + return false; + } + const current = await fingerprintDatabaseFiles(pending.dbPath); + return JSON.stringify(current) === JSON.stringify(parsed.files); + } catch { + return false; + } +} + +async function canSafelyRestoreStoppedTrialBackup( + baseDir: string, + pending: PendingServiceUpdate, +): Promise { + if ( + !isValidServiceUpdateId(pending.id) || + !(await isDatabaseRestoreTargetInsideBaseDir(baseDir, pending.dbPath)) + ) { + return false; + } + if (await hasConflictingServerOwnerLease(pending.dbPath, pending.id)) { + return false; + } + if (await databaseRestorePending(baseDir, pending)) { + return true; + } + return stoppedDigestMatchesCurrentDatabase(baseDir, pending, { requireDigest: true }); +} + +async function acquirePendingDatabaseOwnerLock( + baseDir: string, + pending: PendingServiceUpdate, +): Promise<{ readonly acquired: true; readonly close: () => void } | { readonly acquired: false }> { + const ownerLockPath = NodePath.join(NodePath.dirname(pending.dbPath), "server-owner.sqlite"); + if ( + !(await pathExists(databaseBackupDir(baseDir, pending.id))) && + !NodeFS.existsSync(ownerLockPath) + ) { + return { acquired: true, close: () => undefined }; + } + const lock = tryAcquireServerOwnerLock(pending.dbPath); + return lock === undefined ? { acquired: false } : { acquired: true, close: lock.close }; +} + +/** + * Recovers an interrupted trial update after `service uninstall` stops the launcher so releasing + * the home to `t3 serve` or desktop mode starts from a consistent database when stop history is + * proven clean, while preserving both copies whenever foreground work or a crash makes it ambiguous. + */ +export async function recoverServiceLauncherUninstall(input: { + readonly baseDir: string; + readonly controlDir?: string | undefined; + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; +}): Promise { + const acl = resolveControlAclHooks(input.acl); + if (input.controlDir !== undefined && !(await verifyControlLockAcl(input.controlDir, acl))) { + throw createControlRejectedError("unauthorized"); + } + const statePath = NodePath.join(input.baseDir, "runtime", SERVICE_STATE_FILE); + const lockHeld = input.controlDir !== undefined && probeControlLockHeld(input.controlDir); + if (!lockHeld) { + let initialStateText: string | undefined; + try { + initialStateText = await NodeFSP.readFile(statePath, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return; + throw cause; + } + if (parseServiceState(initialStateText)?.update?.status !== "pending") return; + } + const runUnderControlLock = async (fn: () => Promise): Promise => { + if (input.controlDir === undefined) return fn(); + const lockPath = await ensurePrivateControlLockReady(input.controlDir, "lock.sqlite", acl); + const lock = await acquireSqliteExclusiveLock(lockPath, { + timeoutMs: input.timeoutMs ?? CONTROL_MUTATION_TIMEOUT_MS, + retryMs: 50, + signal: input.signal, + }); + try { + return await fn(); + } finally { + lock.close(); + } + }; + await runUnderControlLock(async () => { + const release = await acquireServiceSelectionLock(statePath); + try { + let stateText: string; + try { + stateText = await NodeFSP.readFile(statePath, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return; + throw cause; + } + const parsed = parseServiceState(stateText); + if (parsed?.update?.status !== "pending") return; + const pending = parsed.update; + const validTarget = + isValidServiceUpdateId(pending.id) && + (await isDatabaseRestoreTargetInsideBaseDir(input.baseDir, pending.dbPath)); + if (!validTarget) { + await writeServiceState(statePath, { + ...parsed, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "service-uninstalled" }), + }); + await NodeFSP.rm(stopMarkerPath(input.baseDir), { force: true }).catch(() => undefined); + return; + } + const serverOwnerLock = await acquirePendingDatabaseOwnerLock(input.baseDir, pending); + if (!serverOwnerLock.acquired) { + await writeServiceState(statePath, { + ...parsed, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), + }); + return; + } + try { + const hasBackup = await pathExists(databaseBackupDir(input.baseDir, pending.id)); + const restored = + hasBackup && (await canSafelyRestoreStoppedTrialBackup(input.baseDir, pending)); + if (restored) { + await restoreDatabaseBackup(input.baseDir, pending); + } + await writeServiceState(statePath, { + ...parsed, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "service-uninstalled" }), + }); + await NodeFSP.rm(stopMarkerPath(input.baseDir), { force: true }).catch(() => undefined); + if (restored) { + await discardDatabaseBackup(input.baseDir, pending.id).catch(() => undefined); + } + } finally { + serverOwnerLock.close(); + } + } finally { + release(); + } + }); +} + +function throwIfControlAborted(signal?: AbortSignal): void { + if (signal?.aborted === true) throw createControlAbortedError(); +} + +async function restorePreviousRestartPending( + targetPath: string, + previousContents: string | undefined, +): Promise { + if (previousContents === undefined) { + await NodeFSP.rm(targetPath, { force: true }).catch(() => undefined); + } else { + await NodeFSP.writeFile(targetPath, previousContents, { mode: 0o600 }).catch(() => undefined); + } +} + +async function stageOfflineInstall( + baseDir: string, + version: string, + allowDowngrade: boolean, + signal?: AbortSignal, +): Promise { + const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + const release = await acquireServiceSelectionLock(statePath); + try { + let previousStateText: string | undefined; + try { + previousStateText = await NodeFSP.readFile(statePath, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause; + } + if (previousStateText !== undefined) { + const installedVersion = serviceStateActiveVersion(previousStateText); + if (serviceStateHasPendingUpdate(previousStateText)) { + return { + status: "update-pending", + ...(installedVersion === undefined ? {} : { installedVersion }), + }; + } + if ( + installedVersion !== undefined && + !allowDowngrade && + compareExactServiceVersions(version, installedVersion) < 0 + ) { + return { status: "downgrade-refused", installedVersion }; + } + } + const restartPending = restartPendingPath(baseDir); + const previousRestartPending = await NodeFSP.readFile(restartPending, "utf8").catch( + () => undefined, + ); + await NodeFSP.mkdir(NodePath.join(baseDir, "runtime"), { recursive: true, mode: 0o700 }); + throwIfControlAborted(signal); + await NodeFSP.writeFile(restartPending, `${version}\n`, { mode: 0o600 }); + if (signal?.aborted === true) { + await restorePreviousRestartPending(restartPending, previousRestartPending); + throw createControlAbortedError(); + } + await writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: version, + }); + return { status: "staged" }; + } finally { + release(); + } +} + +/** + * Atomically checks and stages a CLI service install through the running launcher's transition + * queue, or while holding the per-home SQLite control lock when no launcher is running. + */ +export async function stageServiceLauncherInstall(input: { + readonly controlDir: string; + readonly baseDir: string; + readonly version: string; + readonly allowDowngrade?: boolean | undefined; + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; +}): Promise { + if (!isExactServiceVersion(input.version)) { + throw new Error(`Invalid target service version '${input.version}'.`); + } + const allowDowngrade = input.allowDowngrade === true; + const acl = resolveControlAclHooks(input.acl); + const timeoutMs = input.timeoutMs ?? CONTROL_MUTATION_TIMEOUT_MS; + const deadline = performance.now() + Math.max(0, timeoutMs); + const lockPath = await ensurePrivateControlLockReady(input.controlDir, "lock.sqlite", acl); + for (;;) { + if (input.signal?.aborted === true) throw createControlAbortedError(); + const lock = tryAcquireSqliteExclusiveLock(lockPath); + if (lock !== undefined) { + try { + await NodeFSP.rm(controlEndpointPath(input.controlDir), { force: true }).catch( + () => undefined, + ); + return await stageOfflineInstall( + input.baseDir, + input.version, + allowDowngrade, + input.signal, + ); + } finally { + lock.close(); + } + } + const remainingBeforeSendMs = deadline - performance.now(); + if (remainingBeforeSendMs <= 0) break; + let reply: ControlReply | undefined; + try { + reply = await sendControlRequest( + input.controlDir, + { + type: "stage-install", + version: input.version, + allowDowngrade, + }, + { + timeoutMs: remainingBeforeSendMs, + signal: input.signal, + acl, + skipLockAclVerify: true, + }, + ); + } catch (error) { + if (!isTransientControlClose(error) && !isMissingControl(error)) throw error; + const remainingAfterErrorMs = deadline - performance.now(); + if (remainingAfterErrorMs <= 0) break; + await waitForControlDelay(Math.min(50, remainingAfterErrorMs), input.signal); + continue; + } + if (reply?.ok === true && reply.result !== undefined && reply.result.status !== "not-running") { + return reply.result; + } + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) break; + await waitForControlDelay(Math.min(50, remainingMs), input.signal); + } + throw new Error("Timed out acquiring service launcher control lock to stage install."); +} + +export type ServiceLauncherProbeStatus = "running" | "unhealthy" | "stopped"; + +/** + * Distinguishes an authenticated running launcher (`"running"`), a stopped launcher (`"stopped"`), + * and an occupied or unauthenticated control lock (`"unhealthy"`). + */ +export async function probeServiceLauncher( + controlDir: string, + options: { + readonly timeoutMs?: number | undefined; + readonly signal?: AbortSignal | undefined; + readonly acl?: ControlAclHooks | undefined; + } = {}, +): Promise { + const acl = resolveControlAclHooks(options.acl); + try { + if (!(await verifyControlLockAcl(controlDir, acl))) return "unhealthy"; + } catch { + return "unhealthy"; + } + const deadline = performance.now() + Math.max(0, options.timeoutMs ?? 1_500); + for (;;) { + let lockHeld: boolean; + try { + lockHeld = probeControlLockHeld(controlDir); + } catch { + return "unhealthy"; + } + if (!lockHeld) return "stopped"; + try { + const reply = await sendControlRequest( + controlDir, + { type: "identify" }, + { + timeoutMs: Math.min( + CONTROL_QUERY_TIMEOUT_MS, + Math.max(250, deadline - performance.now()), + ), + signal: options.signal, + acl, + skipLockAclVerify: true, + }, + ); + if (reply?.ok === true && reply.launcherId !== undefined) return "running"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === CONTROL_ABORTED_CODE) throw error; + try { + return probeControlLockHeld(controlDir) ? "unhealthy" : "stopped"; + } catch { + return "unhealthy"; + } + } + const remainingMs = deadline - performance.now(); + if (remainingMs <= 0) break; + await waitForControlDelay(Math.min(100, remainingMs), options.signal); + } + try { + return probeControlLockHeld(controlDir) ? "unhealthy" : "stopped"; + } catch { + return "unhealthy"; + } +} + +export class Launcher { + readonly #instanceId = NodeCrypto.randomUUID(); + readonly #baseDir: string; + readonly #statePath: string; + #state: ServiceState; + #child: ManagedChild | null = null; + #managedPendingTrialId: string | undefined; + #timer: NodeJS.Timeout | undefined; + #transitions: Promise = Promise.resolve(); + #stopRequested = false; + #stopping = false; + #uninstalling = false; + #done = false; + #ownsControl: boolean; + readonly #stopAbort = new AbortController(); + readonly #completion = Promise.withResolvers(); + readonly #backupSpaceProbe: Partial | undefined; + readonly #serviceUnit: ServiceUnit | undefined; + readonly #environment: Readonly; + readonly #output: number | undefined; + readonly #controlDir: string | undefined; + readonly #controlListenRetryMs: number; + readonly #acl: ControlAclHooks | undefined; + readonly #onControlAcquired: (() => void) | undefined; + readonly #onControlRetry: ((attempt: number) => void) | undefined; + readonly #onControlStageQueued: + | ((info: { readonly version: string; readonly disconnected: Promise }) => void) + | undefined; + readonly #onFatalControlOwnerExit: ((error: unknown) => void) | undefined; + readonly #stopChildOverIpc: boolean; + readonly #spawnCommand: (entryPath: string) => { + readonly command: string; + readonly args: ReadonlyArray; + }; + + constructor( + baseDir: string, + state: ServiceState, + options: { + readonly backupSpaceProbe?: Partial; + readonly serviceUnit?: ServiceUnit; + readonly environment?: Readonly; + /** Launcher and server output, for managers that cannot redirect it. */ + readonly output?: number; + /** Account-private directory holding the SQLite lock, token, and loopback TCP endpoint. */ + readonly controlDir?: string; + readonly controlListenRetryMs?: number; + readonly acl?: ControlAclHooks; + readonly onControlAcquired?: () => void; + readonly onControlRetry?: (attempt: number) => void; + readonly onControlStageQueued?: (info: { + readonly version: string; + readonly disconnected: Promise; + }) => void; + /** Invoked while still holding the control lock when the active launcher fails fatally. */ + readonly onFatalControlOwnerExit?: (error: unknown) => void; + /** Windows `kill()` is TerminateProcess, which skips the server's shutdown finalizers. */ + readonly stopChildOverIpc?: boolean; + readonly spawnCommand?: (entryPath: string) => { + readonly command: string; + readonly args: ReadonlyArray; + }; + } = {}, + ) { + this.#baseDir = baseDir; + this.#backupSpaceProbe = options.backupSpaceProbe; + this.#serviceUnit = options.serviceUnit; + this.#environment = { ...(options.environment ?? process.env) }; + this.#output = options.output; + this.#controlDir = options.controlDir; + this.#controlListenRetryMs = options.controlListenRetryMs ?? CONTROL_LISTEN_RETRY_MS; + this.#acl = resolveControlAclHooks(options.acl); + this.#onControlAcquired = options.onControlAcquired; + this.#onControlRetry = options.onControlRetry; + this.#onControlStageQueued = options.onControlStageQueued; + this.#onFatalControlOwnerExit = options.onFatalControlOwnerExit; + this.#ownsControl = options.controlDir === undefined; + this.#stopChildOverIpc = options.stopChildOverIpc ?? IS_WINDOWS; + this.#spawnCommand = options.spawnCommand ?? runtimeSpawnArguments; + this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + this.#state = state; + } + + get ownsControl(): boolean { + return this.#ownsControl; + } + + async run(): Promise { + const onSigterm = () => void this.stop("SIGTERM"); + const onSigint = () => void this.stop("SIGINT"); + process.once("SIGTERM", onSigterm); + process.once("SIGINT", onSigint); + try { + const control = + this.#controlDir === undefined + ? undefined + : await this.#listenForControl( + this.#controlDir, + (publishReady, didPublish) => { + // Enqueued before `endpoint.json` is written so any request arriving on the newly + // published loopback port runs strictly after persisted state reload + recovery. + this.#enqueue(async () => { + await publishReady; + if (!didPublish() || this.#stopRequested) return; + this.#state = await readServiceState(this.#statePath); + await this.#recover(); + }); + }, + () => this.#onControlAcquired?.(), + ); + try { + if (this.#controlDir === undefined && !this.#stopRequested) { + this.#enqueue(() => this.#recover()); + } + await this.#completion.promise; + } catch (error) { + if (this.#ownsControl) { + try { + this.#onFatalControlOwnerExit?.(error); + } catch { + // Crash cleanup must not mask the original launcher failure. + } + } + throw error; + } finally { + await control?.close(); + } + } finally { + process.off("SIGTERM", onSigterm); + process.off("SIGINT", onSigint); + } + } + + /** Acquires the per-home SQLite lock, verifies the private token, and binds a loopback port once. */ + async #listenForControl( + controlDir: string, + beforePublish: (publishReady: Promise, didPublish: () => boolean) => void, + onPublished: () => void, + ): Promise<{ readonly close: () => Promise } | undefined> { + if (this.#stopRequested) return undefined; + const lockPath = await ensurePrivateControlLockReady(controlDir, "lock.sqlite", this.#acl); + let lock: { readonly close: () => void }; + try { + lock = await acquireSqliteExclusiveLock(lockPath, { + timeoutMs: CONTROL_LISTEN_ATTEMPTS * this.#controlListenRetryMs, + retryMs: this.#controlListenRetryMs, + signal: this.#stopAbort.signal, + onRetry: this.#onControlRetry, + }); + } catch (error) { + if (this.#stopRequested && (error as NodeJS.ErrnoException).code === CONTROL_ABORTED_CODE) { + return undefined; + } + throw error; + } + if (this.#stopRequested) { + lock.close(); + return undefined; + } + const sockets = new Set(); + const unauthenticatedSockets = new Set(); + let server: NodeNet.Server | undefined; + try { + NodeFS.rmSync(controlEndpointPath(controlDir), { force: true }); + const controlToken = await ensureControlToken(controlDir, this.#acl); + server = NodeNet.createServer((socket) => { + if (unauthenticatedSockets.size >= MAX_UNAUTHENTICATED_CONTROL_SOCKETS) { + const oldest = unauthenticatedSockets.values().next().value; + oldest?.destroy(); + } + if (sockets.size >= MAX_TOTAL_CONTROL_SOCKETS) { + socket.destroy(); + return; + } + sockets.add(socket); + unauthenticatedSockets.add(socket); + const handshakeDeadline = setTimeout(() => { + socket.destroy(); + }, CONTROL_HANDSHAKE_TIMEOUT_MS); + handshakeDeadline.unref(); + const markAuthenticated = () => { + clearTimeout(handshakeDeadline); + unauthenticatedSockets.delete(socket); + }; + socket.once("close", () => { + clearTimeout(handshakeDeadline); + unauthenticatedSockets.delete(socket); + sockets.delete(socket); + }); + socket.on("error", () => undefined); + socket.setEncoding("utf8"); + const serverNonce = NodeCrypto.randomBytes(16).toString("hex"); + let buffer = ""; + let handled = false; + socket.on("data", (chunk: string) => { + if (handled) return; + if (buffer.length + chunk.length > MAX_CONTROL_MESSAGE_BYTES) { + socket.destroy(); + return; + } + buffer += chunk; + const newline = buffer.indexOf("\n"); + if (newline === -1) return; + handled = true; + const line = buffer.slice(0, newline); + buffer = ""; + void this.#handleControlLine(socket, controlToken, serverNonce, line, markAuthenticated); + }); + socket.write(`${JSON.stringify({ nonce: serverNonce })}\n`); + }); + const activeServer = server; + await new Promise((resolve, reject) => { + activeServer.once("error", reject); + activeServer.listen(0, "127.0.0.1", () => { + activeServer.off("error", reject); + resolve(); + }); + }); + activeServer.on("error", () => undefined); + const address = activeServer.address(); + if (address === null || typeof address !== "object") { + throw new Error("Could not resolve loopback control port."); + } + let published = false; + const publishGate = Promise.withResolvers(); + this.#ownsControl = true; + beforePublish(publishGate.promise, () => published); + try { + await writeControlEndpoint( + controlDir, + { + port: address.port, + launcherId: this.#instanceId, + }, + this.#acl, + ); + published = true; + } finally { + publishGate.resolve(); + } + if (!this.#stopRequested) { + onPublished(); + } + return { + close: async () => { + await NodeFSP.rm(controlEndpointPath(controlDir), { force: true }).catch(() => undefined); + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => activeServer.close(() => resolve())); + this.#ownsControl = false; + lock.close(); + }, + }; + } catch (cause) { + this.#ownsControl = false; + await NodeFSP.rm(controlEndpointPath(controlDir), { force: true }).catch(() => undefined); + for (const socket of sockets) socket.destroy(); + if (server !== undefined) { + const closingServer = server; + await new Promise((resolve) => closingServer.close(() => resolve())); + } + lock.close(); + throw cause; + } + } + + async #handleControlLine( + socket: NodeNet.Socket, + controlToken: string, + serverNonce: string, + line: string, + markAuthenticated: () => void, + ): Promise { + const sendSigned = (reply: ControlReply, closeAfterWrite: boolean, macClientNonce: string) => { + const body = JSON.stringify(reply); + const mac = computeControlMac(controlToken, "res", serverNonce, macClientNonce, body); + const wire = `${JSON.stringify({ ok: reply.ok, body, mac })}\n`; + if (closeAfterWrite) socket.end(wire); + else socket.write(wire); + }; + let providedServerNonce: string; + let clientNonce: string; + let payloadBody: string; + let providedMac: unknown; + try { + const envelope: unknown = JSON.parse(line); + if ( + typeof envelope !== "object" || + envelope === null || + Array.isArray(envelope) || + typeof (envelope as { readonly snonce?: unknown }).snonce !== "string" || + typeof (envelope as { readonly cnonce?: unknown }).cnonce !== "string" || + !/^[0-9a-f]{32}$/.test((envelope as { readonly cnonce: string }).cnonce) || + typeof (envelope as { readonly payload?: unknown }).payload !== "string" + ) { + sendSigned({ ok: false, error: "invalid-request" }, true, ""); + return; + } + providedServerNonce = (envelope as { readonly snonce: string }).snonce; + clientNonce = (envelope as { readonly cnonce: string }).cnonce; + payloadBody = (envelope as { readonly payload: string }).payload; + providedMac = (envelope as { readonly mac?: unknown }).mac; + } catch { + sendSigned({ ok: false, error: "invalid-json" }, true, ""); + return; + } + if ( + providedServerNonce !== serverNonce || + !hexDigestsMatch( + computeControlMac(controlToken, "req", serverNonce, clientNonce, payloadBody), + providedMac, + ) + ) { + sendSigned({ ok: false, error: "unauthorized" }, true, clientNonce); + return; + } + let request: Record; + try { + const parsed: unknown = JSON.parse(payloadBody); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + sendSigned({ ok: false, error: "invalid-request" }, true, clientNonce); + return; + } + request = parsed as Record; + } catch { + sendSigned({ ok: false, error: "invalid-json" }, true, clientNonce); + return; + } + markAuthenticated(); + if (request.type === "identify") { + sendSigned({ ok: true, launcherId: this.#instanceId }, true, clientNonce); + return; + } + if (request.type === "stop") { + const targeted = typeof request.launcherId === "string" ? request.launcherId : undefined; + if (targeted !== undefined && targeted !== this.#instanceId) { + sendSigned({ ok: true, launcherId: this.#instanceId }, true, clientNonce); + return; + } + if (request.uninstall === true) { + this.#uninstalling = true; + } + if (request.wait === true) { + sendSigned({ ok: true, launcherId: this.#instanceId }, false, clientNonce); + void this.stop("SIGTERM"); + return; + } + void this.stop("SIGTERM"); + sendSigned({ ok: true, launcherId: this.#instanceId }, true, clientNonce); + return; + } + if (request.type === "stage-install" && typeof request.version === "string") { + const version = request.version; + const allowDowngrade = request.allowDowngrade === true; + let clientDisconnected = socket.destroyed; + const disconnected = Promise.withResolvers(); + if (clientDisconnected) { + disconnected.resolve(); + } else { + socket.once("close", () => { + clientDisconnected = true; + disconnected.resolve(); + }); + } + const isClientDisconnected = () => clientDisconnected || socket.destroyed; + const outcome = Promise.withResolvers(); + this.#enqueue(async () => { + if (isClientDisconnected()) { + outcome.resolve(undefined); + return; + } + try { + outcome.resolve(await this.#stageInstall(version, allowDowngrade, isClientDisconnected)); + } catch (cause) { + outcome.reject(cause); + } + }); + this.#onControlStageQueued?.({ version, disconnected: disconnected.promise }); + try { + const result = await outcome.promise; + if (result !== undefined && !isClientDisconnected()) { + sendSigned({ ok: true, launcherId: this.#instanceId, result }, true, clientNonce); + } + } catch { + if (!isClientDisconnected()) { + sendSigned({ ok: false, error: "stage-failed" }, true, clientNonce); + } + } + return; + } + sendSigned({ ok: false, error: "unknown-command" }, true, clientNonce); + } + + async #stageInstall( + version: string, + allowDowngrade: boolean, + isClientDisconnected: () => boolean, + ): Promise { + if (!isExactServiceVersion(version)) { + throw new Error(`Invalid target service version '${version}'.`); + } + if (this.#stopping || this.#done) { + return { status: "not-running" }; + } + const release = await acquireServiceSelectionLock(this.#statePath); + try { + let previousStateText: string | undefined; + try { + previousStateText = await NodeFSP.readFile(this.#statePath, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause; + } + const diskVersion = + previousStateText === undefined ? undefined : serviceStateActiveVersion(previousStateText); + const effectiveVersion = diskVersion ?? this.#state.activeVersion; + if ( + this.#state.update?.status === "pending" || + (previousStateText !== undefined && serviceStateHasPendingUpdate(previousStateText)) + ) { + return { + status: "update-pending", + ...(effectiveVersion === undefined ? {} : { installedVersion: effectiveVersion }), + }; + } + if (!allowDowngrade && compareExactServiceVersions(version, effectiveVersion) < 0) { + return { status: "downgrade-refused", installedVersion: effectiveVersion }; + } + const next: ServiceState = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: version, + }; + const restartPending = restartPendingPath(this.#baseDir); + const previousRestartPending = await NodeFSP.readFile(restartPending, "utf8").catch( + () => undefined, + ); + await NodeFSP.mkdir(NodePath.join(this.#baseDir, "runtime"), { + recursive: true, + mode: 0o700, + }); + if (isClientDisconnected()) return undefined; + await NodeFSP.writeFile(restartPending, `${version}\n`, { mode: 0o600 }); + if (isClientDisconnected()) { + await restorePreviousRestartPending(restartPending, previousRestartPending); + return undefined; + } + await writeServiceState(this.#statePath, next); + return { status: "staged", launcherId: this.#instanceId }; + } finally { + release(); + } + } + + #terminate(child: NodeChildProcess.ChildProcess, signal: NodeJS.Signals = "SIGTERM") { + return terminateChild(child, signal, this.#stopChildOverIpc); + } + + #log(line: string): void { + try { + if (this.#output === undefined) process.stderr.write(line); + else NodeFS.writeSync(this.#output, line); + } catch { + // Log write failures must not terminate a running launcher. + } + } + + #enqueue(transition: () => Promise): void { + this.#transitions = this.#transitions + .then(transition, transition) + .catch((cause: unknown) => + this.#fatal(cause instanceof Error ? cause : new Error(String(cause))), + ); + } + + async #fatal(error: Error): Promise { + if (this.#done) return; + this.#done = true; + this.#stopping = true; + this.#clearTimer(); + const child = this.#child?.process; + this.#child = null; + if (child !== undefined) await this.#terminate(child); + this.#completion.reject(error); + } + + async #assertSelectionMatchesState(): Promise { + const selected = await readServiceState(this.#statePath); + // Local installs replace the selection and clear its update record. An old + // launcher must not restore its own selection when asked to update again. + if ( + selected.activeVersion !== this.#state.activeVersion || + selected.update?.id !== this.#state.update?.id || + selected.update?.status !== this.#state.update?.status + ) { + throw new Error("Service selection changed. Restart the service before updating."); + } + } + + async #recoverPendingUpdateOnUninstall(pending: PendingServiceUpdate): Promise { + const managedByThisLauncher = this.#managedPendingTrialId === pending.id; + this.#managedPendingTrialId = undefined; + const release = await acquireServiceSelectionLock(this.#statePath); + try { + await this.#assertSelectionMatchesState(); + const validTarget = + isValidServiceUpdateId(pending.id) && + (await isDatabaseRestoreTargetInsideBaseDir(this.#baseDir, pending.dbPath)); + if (!validTarget) { + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "service-uninstalled" }), + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); + return; + } + const serverOwnerLock = await acquirePendingDatabaseOwnerLock(this.#baseDir, pending); + if (!serverOwnerLock.acquired) { + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + return; + } + try { + const hasBackup = await pathExists(databaseBackupDir(this.#baseDir, pending.id)); + const restored = + hasBackup && + (managedByThisLauncher + ? !(await hasConflictingServerOwnerLease(pending.dbPath, pending.id)) + : await canSafelyRestoreStoppedTrialBackup(this.#baseDir, pending)); + if (restored) { + await restoreDatabaseBackup(this.#baseDir, pending); + } + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: terminalUpdate({ pending, status: "failed", reason: "service-uninstalled" }), + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); + if (restored) { + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + } + } finally { + serverOwnerLock.close(); + } + } finally { + release(); + } + } + + async stop(signal: NodeJS.Signals): Promise { + // Only the lock owner writes the stop marker; a waiting replacement must not mark the active server's home. + if (this.#ownsControl) { + // Write synchronously on signal receipt; a terminating child needs the marker in its finalizer. + try { + NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); + } catch {} + } + this.#stopAbort.abort(); + if (this.#stopRequested || this.#stopping) { + await this.#completion.promise.catch(() => undefined); + return; + } + this.#stopRequested = true; + this.#clearTimer(); + this.#enqueue(async () => { + // Let in-progress updates start replacements first; replacements observe the marker and own pre-activation tunnel cleanup. + this.#stopping = true; + const child = this.#child?.process; + this.#child = null; + if (child !== undefined) await this.#terminate(child, signal); + if (this.#ownsControl) { + const pending = this.#state.update?.status === "pending" ? this.#state.update : undefined; + if (this.#uninstalling && pending !== undefined) { + await this.#recoverPendingUpdateOnUninstall(pending); + } else if (pending !== undefined) { + const validTarget = + isValidServiceUpdateId(pending.id) && + (await isDatabaseRestoreTargetInsideBaseDir(this.#baseDir, pending.dbPath)); + const serverOwnerLock = validTarget + ? await acquirePendingDatabaseOwnerLock(this.#baseDir, pending) + : { acquired: false as const }; + try { + NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); + } catch {} + if (serverOwnerLock.acquired) { + try { + if (!(await hasConflictingServerOwnerLease(pending.dbPath, pending.id))) { + await recordStoppedPendingDatabaseFingerprint(this.#baseDir, pending).catch( + () => undefined, + ); + } + } finally { + serverOwnerLock.close(); + } + } + } else { + try { + NodeFS.writeFileSync(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }); + } catch {} + } + } + this.#done = true; + this.#completion.resolve(); + }); + await this.#completion.promise.catch(() => undefined); + } + + #clearTimer(): void { + clearTimeout(this.#timer); + this.#timer = undefined; + } + + async #selectState(next: ServiceState): Promise { + const release = await acquireServiceSelectionLock(this.#statePath); + try { + await this.#assertSelectionMatchesState(); + await writeServiceState(this.#statePath, next); + this.#state = next; + } finally { + release(); + } + } + + async #recover(): Promise { + // Clear old stop markers to preserve future tunnels; clear restart-pending only for the awaited version to avoid CLI races. + await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); + const restartPending = restartPendingPath(this.#baseDir); + try { + const release = await acquireServiceSelectionLock(this.#statePath); + try { + const selected = await readServiceState(this.#statePath); + const awaitedVersion = await NodeFSP.readFile(restartPending, "utf8").catch( + () => undefined, + ); + const awaited = awaitedVersion?.trim(); + if ( + selected.activeVersion === this.#state.activeVersion && + selected.update?.id === this.#state.update?.id && + selected.update?.status === this.#state.update?.status && + awaited !== undefined && + isExactServiceVersion(awaited) && + compareExactServiceVersions(awaited, this.#state.activeVersion) <= 0 + ) { + await NodeFSP.rm(restartPending, { force: true }); + } + } finally { + release(); + } + } catch (error) { + this.#log( + `[service-launcher] Could not clear ${restartPending}: ${error instanceof Error ? error.message : String(error)}\n`, + ); + } + const update = this.#state.update; + if (update?.status !== "pending") { + if (update?.status === "committed" && isValidServiceUpdateId(update.id)) { + await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); + } + try { + await this.#startChild(this.#state.activeVersion, "active", update); + } finally { + if (update !== undefined) await this.#repointServiceUnit(this.#state.activeVersion); + } + return; + } + if (!isValidServiceUpdateId(update.id)) { + await this.#returnToPrevious(update, "failed", "invalid-update", undefined, { + startupRecovery: true, + }); + return; + } + if (await databaseRestorePending(this.#baseDir, update)) { + await this.#returnToPrevious(update, "failed", "rollback-interrupted", undefined, { + startupRecovery: true, + }); + return; + } + if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { + await this.#returnToPrevious(update, "failed", "target-runtime-missing", undefined, { + startupRecovery: true, + }); + return; + } + await this.#startTrial(update); + } + + async #startTrial(pending: PendingServiceUpdate): Promise { + if ( + !isValidServiceUpdateId(pending.id) || + !(await isDatabaseRestoreTargetInsideBaseDir(this.#baseDir, pending.dbPath)) + ) { + await this.#returnToPrevious(pending, "failed", "db-backup-failed"); + return; + } + const existingBackup = await pathExists(databaseBackupDir(this.#baseDir, pending.id)); + if (existingBackup && this.#managedPendingTrialId !== pending.id) { + if ( + (await hasConflictingServerOwnerLease(pending.dbPath, pending.id)) || + !(await stoppedDigestMatchesCurrentDatabase(this.#baseDir, pending)) + ) { + await this.#returnToPrevious(pending, "failed", "live-db-modified", undefined, { + startupRecovery: true, + }); + return; + } + } + try { + await backupDatabaseOnce(this.#baseDir, pending); + } catch (cause) { + if (isStateDirOwnedDuringBackupError(cause)) { + await this.#idleWhileStateDirOwned(); + return; + } await this.#returnToPrevious(pending, "failed", "db-backup-failed"); return; } + this.#managedPendingTrialId = pending.id; try { await this.#startChild(pending.targetVersion, "trial", pending); } catch { @@ -801,10 +3080,13 @@ export class Launcher { childVersion: version, ...(update === undefined ? {} : { update }), }; - const spawnArguments = runtimeSpawnArguments(paths); + const spawnArguments = this.#spawnCommand(paths.entryPath); const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, { env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }, - stdio: ["inherit", "inherit", "inherit", "ipc"], + stdio: + this.#output === undefined + ? ["inherit", "inherit", "inherit", "ipc"] + : ["ignore", this.#output, this.#output, "ipc"], }); await new Promise((resolve, reject) => { const onError = (error: Error) => reject(error); @@ -816,7 +3098,7 @@ export class Launcher { }); }); if (this.#stopping) { - await terminateChild(child); + await this.#terminate(child); return; } @@ -877,8 +3159,11 @@ export class Launcher { await reject("Remote updates must select a newer server version."); return; } - if (!NodePath.isAbsolute(message.dbPath)) { - await reject("The requested database path is not absolute."); + if ( + !NodePath.isAbsolute(message.dbPath) || + !isDatabasePathInsideBaseDir(this.#baseDir, message.dbPath) + ) { + await reject("The requested database path must be inside the T3 home."); return; } if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { @@ -924,15 +3209,25 @@ export class Launcher { return; } this.#timer = undefined; + this.#managedPendingTrialId = pending.id; // Anything thrown outside its own fallback still leaves the offline backup to run. const finishOnlineBackup = await startOnlineDatabaseBackup( this.#baseDir, pending, this.#backupSpaceProbe, + (message) => this.#log(message), ).catch(() => undefined); this.#child = null; - await terminateChild(child.process); - await finishOnlineBackup?.(); + await this.#terminate(child.process); + try { + await finishOnlineBackup?.(); + } catch (cause) { + if (isStateDirOwnedDuringBackupError(cause)) { + await this.#idleWhileStateDirOwned(); + return; + } + throw cause; + } // A stop during a minutes-long copy should not start a trial; restart resumes the pending update. if (this.#stopRequested) return; await this.#startTrial(pending); @@ -953,6 +3248,7 @@ export class Launcher { throw new Error("Trial child reported prepared for an unexpected update."); } this.#clearTimer(); + this.#managedPendingTrialId = undefined; const committed = terminalUpdate({ pending, status: "committed" }); const next: ServiceState = { ...this.#state, @@ -990,7 +3286,11 @@ export class Launcher { const installedBaseDir = serviceUnitBaseDirOf(contents); if ( installedBaseDir === undefined || - NodePath.resolve(installedBaseDir) !== NodePath.resolve(this.#baseDir) + !sameInstalledBaseDir( + installedBaseDir, + this.#baseDir, + IS_WINDOWS || unit.path.endsWith(".xml"), + ) ) { return; } @@ -1002,7 +3302,7 @@ export class Launcher { release(); } } catch (error) { - process.stderr.write( + this.#log( `[service-launcher] Could not point ${unit.path} at t3@${version}: ${error instanceof Error ? error.message : String(error)}\n`, ); } @@ -1051,9 +3351,18 @@ export class Launcher { throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); } - // Stay idle to avoid supervisor restart loops; never restore backups over another owner's possible writes. + #enterOwnedStateDirIdle(): void { + // Signal listeners alone do not keep Node alive. + this.#timer = setInterval(() => {}, 2_147_483_647); + this.#log( + "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n", + ); + } + + // Stay idle to avoid supervisor restart loops; preserve backups rather than overwriting or discarding data. async #idleWhileStateDirOwned(): Promise { this.#clearTimer(); + this.#managedPendingTrialId = undefined; const pending = this.#state.update; if (pending?.status === "pending") { const next: ServiceState = { @@ -1062,14 +3371,9 @@ export class Launcher { update: terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }), }; await this.#selectState(next); - await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#repointServiceUnit(next.activeVersion); } - // Signal listeners alone do not keep Node alive. - this.#timer = setInterval(() => {}, 2_147_483_647); - process.stderr.write( - "[service-launcher] Another T3 Code server owns this T3 home. Stop it, then restart this service.\n", - ); + this.#enterOwnedStateDirIdle(); } async #returnToPrevious( @@ -1077,20 +3381,73 @@ export class Launcher { status: "rolled-back" | "failed", reason: string, child?: ManagedChild, + options?: { readonly startupRecovery?: boolean | undefined }, ): Promise { + const managedByThisLauncher = + this.#managedPendingTrialId === pending.id && options?.startupRecovery !== true; + this.#managedPendingTrialId = undefined; if (child !== undefined) { this.#child = null; - await terminateChild(child.process); + await this.#terminate(child.process); + } + const release = await acquireServiceSelectionLock(this.#statePath); + let enteredOwnedIdle = false; + let next: ServiceState; + let outcome: Exclude; + let restored = false; + try { + await this.#assertSelectionMatchesState(); + const validTarget = + isValidServiceUpdateId(pending.id) && + (await isDatabaseRestoreTargetInsideBaseDir(this.#baseDir, pending.dbPath)); + const serverOwnerLock = validTarget + ? await acquirePendingDatabaseOwnerLock(this.#baseDir, pending) + : { acquired: true as const, close: () => undefined }; + if (!serverOwnerLock.acquired) { + outcome = terminalUpdate({ pending, status: "failed", reason: "state-dir-owned" }); + next = { + ...this.#state, + activeVersion: pending.fromVersion, + update: outcome, + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + enteredOwnedIdle = true; + } else { + try { + const hasBackup = + validTarget && (await pathExists(databaseBackupDir(this.#baseDir, pending.id))); + restored = + hasBackup && + (managedByThisLauncher + ? !(await hasConflictingServerOwnerLease(pending.dbPath, pending.id)) + : await canSafelyRestoreStoppedTrialBackup(this.#baseDir, pending)); + if (restored) { + await restoreDatabaseBackup(this.#baseDir, pending); + } + outcome = terminalUpdate({ pending, status, reason }); + next = { + ...this.#state, + activeVersion: pending.fromVersion, + update: outcome, + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + if (restored) { + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); + } + } finally { + serverOwnerLock.close(); + } + } + } finally { + release(); + } + if (enteredOwnedIdle) { + await this.#repointServiceUnit(next.activeVersion); + this.#enterOwnedStateDirIdle(); + return; } - await restoreDatabaseBackup(this.#baseDir, pending); - const outcome = terminalUpdate({ pending, status, reason }); - const next: ServiceState = { - ...this.#state, - activeVersion: pending.fromVersion, - update: outcome, - }; - await this.#selectState(next); - await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); try { await this.#startChild(next.activeVersion, "active", outcome); } finally { @@ -1099,18 +3456,142 @@ export class Launcher { } } -export async function main(): Promise { - const baseDir = process.env.T3CODE_HOME?.trim(); +/** Task Scheduler cannot set variables or redirect output, so a Windows task passes them as flags. */ +export interface LauncherOptions { + readonly baseDir?: string | undefined; + readonly logFile?: string | undefined; + readonly serviceUnit?: string | undefined; + readonly platform?: NodeJS.Platform | undefined; + readonly environment?: Readonly | undefined; + readonly acl?: ControlAclHooks | undefined; + readonly controlListenRetryMs?: number | undefined; + readonly spawnCommand?: (entryPath: string) => { + readonly command: string; + readonly args: ReadonlyArray; + }; + readonly endScheduledTask?: ((taskName: string) => void) | undefined; +} + +export async function main(options: LauncherOptions = {}): Promise { + const hostEnv = options.environment ?? process.env; + const isWindows = options.platform !== undefined ? options.platform === "win32" : IS_WINDOWS; + const baseDir = (options.baseDir ?? hostEnv.T3CODE_HOME)?.trim(); if (baseDir === undefined || baseDir === "") { throw new Error("T3CODE_HOME is required by the T3 Code service launcher."); } - const environment = await readServiceEnvironment(baseDir); - const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); - const state = await readServiceState(statePath); - const serviceUnit = serviceUnitFromEnvironment(); - await new Launcher( - baseDir, - state, - serviceUnit === undefined ? { environment } : { environment, serviceUnit }, - ).run(); + if (options.logFile !== undefined) { + NodeFS.mkdirSync(NodePath.dirname(options.logFile), { recursive: true, mode: 0o700 }); + } + const output = + options.logFile === undefined ? undefined : NodeFS.openSync(options.logFile, "a", 0o600); + let loggedFatalError = false; + const writeFatalError = (cause: unknown) => { + if (loggedFatalError || output === undefined) return; + loggedFatalError = true; + try { + NodeFS.writeSync( + output, + `[service-launcher] ${cause instanceof Error ? cause.message : String(cause)}\n`, + ); + } catch { + // Log write failures (for example ENOSPC) must not suppress crash cleanup or mask the error. + } + }; + try { + const managerEnvironment: NodeJS.ProcessEnv = { + ...hostEnv, + T3CODE_HOME: baseDir, + ...(options.serviceUnit === undefined ? {} : { [SERVICE_UNIT_ENV]: options.serviceUnit }), + }; + const environment = await readServiceEnvironment(baseDir, managerEnvironment); + const configuredLocalAppData = (hostEnv.LOCALAPPDATA ?? "").trim(); + const userProfile = (hostEnv.USERPROFILE ?? "").trim(); + const localAppData = + configuredLocalAppData !== "" + ? configuredLocalAppData + : userProfile === "" + ? "" + : NodePath.join(userProfile, "AppData", "Local"); + if (isWindows && localAppData === "") { + throw new Error("LOCALAPPDATA is required by the T3 Code Windows service launcher."); + } + const userName = (hostEnv.USERNAME ?? "").trim(); + const endScheduledTask = + options.endScheduledTask ?? + ((taskName: string) => { + NodeChildProcess.spawnSync( + windowsSystem32Binary("schtasks.exe", hostEnv), + ["/End", "/TN", taskName], + { + stdio: "ignore", + windowsHide: true, + }, + ); + }); + const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + const state = await readServiceState(statePath); + const serviceUnit = serviceUnitFromEnvironment(managerEnvironment, { + isWindows, + localAppData, + baseDir, + }); + const activeLauncher = new Launcher(baseDir, state, { + environment, + ...(serviceUnit === undefined ? {} : { serviceUnit }), + ...(output === undefined ? {} : { output }), + ...(options.acl === undefined ? {} : { acl: options.acl }), + ...(options.controlListenRetryMs === undefined + ? {} + : { controlListenRetryMs: options.controlListenRetryMs }), + ...(options.spawnCommand === undefined ? {} : { spawnCommand: options.spawnCommand }), + ...(isWindows + ? { + controlDir: serviceControlDir(localAppData, baseDir, { caseInsensitive: true }), + stopChildOverIpc: true, + ...(options.serviceUnit !== undefined && userName !== "" + ? { + // Runs while `lock.sqlite` is still held so a replacement launcher cannot own + // the task yet, and flushes the fatal log line before `schtasks /End` kills conhost. + onFatalControlOwnerExit: (cause) => { + try { + writeFatalError(cause); + } finally { + const taskName = resolvePersistedServiceTaskName( + localAppData, + serviceTaskName(hostEnv.USERDOMAIN ?? "", userName), + ); + endScheduledTask(taskName); + } + }, + } + : {}), + } + : {}), + }); + // Ending a scheduled task kills its console host without a signal; the console read fails instead. + const watchConsole = isWindows && process.stdin.isTTY === true; + const onConsoleLost = () => void activeLauncher.stop("SIGTERM"); + if (watchConsole) { + process.stdin.on("error", () => undefined); + process.stdin.once("close", onConsoleLost); + process.stdin.resume(); + } + try { + await activeLauncher.run(); + } finally { + if (watchConsole) { + process.stdin.off("close", onConsoleLost); + process.stdin.destroy(); + } + } + } catch (cause) { + writeFatalError(cause); + throw cause; + } finally { + if (output !== undefined) { + try { + NodeFS.closeSync(output); + } catch {} + } + } } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index e0f1d286a1608..bf65ee0ae63a9 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -1,7 +1,7 @@ # Server updates A [stable launcher](../../apps/server/src/serviceLauncher.ts) owns the runtime -selected by systemd or launchd. It is the only runtime writer of durable service +selected by systemd, launchd, or Windows Task Scheduler. It is the only runtime writer of durable service state. Server children request updates over inherited IPC; they never rewrite their service definition or select their own replacement. Local service commands may replace the launcher and state, optionally deferring the restart. Foreground @@ -25,9 +25,9 @@ After a commit or rollback the launcher points the unit at the selected runtime reloads the manager, so the next service start runs that version's launcher; the protocol check is what makes that safe. Each start after a finished update retries this, because a previous reload may have failed, and leaves a unit naming a newer runtime or another -T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload) -and launchd (before bootstrap) can respawn the old launcher, which the new runtime -refuses to run under. +T3 home alone. Until a deferred local upgrade restarts, systemd (before daemon-reload), +launchd (before bootstrap), and Task Scheduler (before `/Create`) can respawn the old launcher, +which the new runtime refuses to run under. ## Commit boundary diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 51a16878d2bbc..88011fcac84d6 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -1,7 +1,7 @@ # Running T3 Code in the background -On Linux and macOS, T3 Code can run as a service for your user so you do not need -to keep a terminal open. +On Linux, macOS, and Windows, T3 Code can run as a service for your user so you +do not need to keep a terminal open. ## Manage the service @@ -92,11 +92,12 @@ Mac logged in and awake for unattended remote access. Installing over SSH while nobody is logged in at the Mac's screen can fail at the final start step; the service is still installed and will start at the next login. -Windows background services are not supported. To keep a Windows host available -through T3 Connect, leave the desktop app running with its local environment -enabled, or keep `t3 serve` running in a terminal. Minimize the desktop app rather -than closing its last window, which stops its local server. Linking T3 Connect -does not configure automatic startup at Windows sign-in. +Windows starts the service when you log in and stops it when you sign out. It +runs without a window as a `t3code---` task in Task Scheduler, and Task +Scheduler starts it again within a minute if it exits. Lock the screen instead +of signing out, and keep the PC awake, for unattended remote access. Installing +over SSH while nobody is signed in at the PC can fail at the final start step; +the service is still installed and will start at the next sign-in. `t3 up` installs the service when you set up T3 Connect, but the two are managed separately. Signing out of T3 Connect does not stop or uninstall the service. @@ -143,12 +144,12 @@ with sudo; running T3 Code as root creates a separate installation and Connect identity. Without administrator access, run `t3 serve` in a terminal and keep that session open. -| Status problem | Next step | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `linger-unavailable` | Run `loginctl show-user "$(id -un)" --property=Linger` and check that systemd-logind is available. | -| `user-manager-unavailable` | Run `systemctl --user status` in a login session for the service user; check your distribution's systemd user-session support. | -| `service-disabled` or `service-stopped` | Read the log and `systemctl --user status t3code.service`, then use the repair command printed by T3 Code. | -| `restart-pending` | A newer version is installed but the service still runs the previous one. Run `t3 service restart`. | +| Status problem | Next step | +| ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `linger-unavailable` | Run `loginctl show-user "$(id -un)" --property=Linger` and check that systemd-logind is available. | +| `user-manager-unavailable` | Run `systemctl --user status` in a login session for the service user; check your distribution's systemd user-session support. | +| `service-disabled`, `service-stopped`, or `service-unhealthy` | Read the log and, on Linux, `systemctl --user status t3code.service`, then use the repair command printed by T3 Code. | +| `restart-pending` | A newer version is installed but the service still runs the previous one. Run `t3 service restart`. | On macOS, check **System Settings → General → Login Items** if the service no longer starts at login. If agent work cannot access Desktop, Documents, or diff --git a/docs/user/install.md b/docs/user/install.md index 7b28781e69612..6c5ca51fd2a9f 100644 --- a/docs/user/install.md +++ b/docs/user/install.md @@ -237,6 +237,6 @@ For provider-specific setup and accounts, see [Codex](./providers-codex.md), - [Working with threads](./thread-sidebar.md): start tasks and organize parallel work. - [Permission modes](./permission-modes.md): choose when agents ask before acting. - [Remote access](./remote-access.md): connect from another device. -- [Running in the background](./background-service.md): keep a Linux or macOS host available. +- [Running in the background](./background-service.md): keep a Linux, macOS, or Windows host available. - [Updating T3 Code](./updating.md): update the app and connected servers. - [Managed policy](./managed-policy.md): configure T3 Code for everyone on a machine. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 9635a8af2a0a7..5abedb45f1c93 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -16,16 +16,15 @@ t3 up ``` It asks how your other devices reach the machine: T3 Connect, the local network, -or both. Choose T3 Connect and follow the sign-in instructions. On Linux and -macOS, it then runs T3 Code as a [background service](./background-service.md). +or both. Choose T3 Connect and follow the sign-in instructions. On Linux, +macOS, and Windows, it then runs T3 Code as a [background service](./background-service.md). `t3 connect` is the same as `t3 up --connect`. Run `t3 up` again to change your choice, and `t3 status` to see it. -The background service is not available on Windows, so there `t3 up` prints the -`t3 serve` command to keep running in a terminal. You can also keep the desktop -app running with its local environment enabled. Closing the desktop app's last -window stops its local server. Saving your sign-in alone does not start a server -or make the machine reachable. +Instead of the CLI background service, you can also keep the desktop app running +with its local environment enabled. Closing the desktop app's last window stops +its local server. Saving your sign-in alone does not start a server or make the +machine reachable. On your other device, sign in to the same T3 Connect account and choose the environment. On the app.t3.codes home page with no computer added yet, signing