fix(server): host sleep no longer idle-releases provider sessions on wake

apps/server/src/orchestration-v2/ProviderSessionManager.test.ts

@@ -23,6 +23,7 @@ import {
   TurnItemId,
 } from "@t3tools/contracts";
 import * as Cause from "effect/Cause";
+import * as Clock from "effect/Clock";
 import * as DateTime from "effect/DateTime";
 import * as Deferred from "effect/Deferred";
 import * as Effect from "effect/Effect";
@@ -3615,6 +3616,120 @@ it.effect(
     }),
 );
 
+const HOST_SLEEP_IDLE_MS = 30 * 60_000;
+
+// Models host sleep starting awakeMs after open and returns the idle deadline (1-based, in awake idle
+// windows) at which the session closed. Where the timer clock counts sleep, the timer fires on resume
+// far past its due time; where it pauses, only wall time jumps.
+const idleDeadlineAcrossHostSleep = (input: {
+  readonly timerCountsSleep: boolean;
+  readonly awakeMs: number;
+  readonly sleepMs: number;
+  readonly pinnedForMs?: number;
+}) =>
+  Effect.gen(function* () {
+    const state = yield* Ref.make(emptyState);
+    const closes = yield* Queue.unbounded<string>();
+    const testClock = yield* TestClock.testClockWith(Effect.succeed);
+    let sleptMs = 0;
+    const sleptNanos = () => BigInt(sleptMs) * BigInt(1_000_000);
+    const monotonicSleptNanos = () => (input.timerCountsSleep ? sleptNanos() : BigInt(0));
+    const hostClock: TestClock.TestClock = {
+      ...testClock,
+      currentTimeMillisUnsafe: () => testClock.currentTimeMillisUnsafe() + sleptMs,
+      currentTimeMillis: Effect.sync(() => testClock.currentTimeMillisUnsafe() + sleptMs),
+      currentTimeNanosUnsafe: () => testClock.currentTimeNanosUnsafe() + sleptNanos(),
+      currentTimeNanos: Effect.sync(() => testClock.currentTimeNanosUnsafe() + sleptNanos()),
+      monotonicTimeNanosUnsafe: () => testClock.monotonicTimeNanosUnsafe() + monotonicSleptNanos(),
+      monotonicTimeNanos: Effect.sync(
+        () => testClock.monotonicTimeNanosUnsafe() + monotonicSleptNanos(),
+      ),
+    };
+    const effect = Effect.gen(function* () {
+      const eventSink = yield* EventSink.EventSinkV2;
+      const idAllocator = yield* IdAllocator.IdAllocatorV2;
+      const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
+      const now = yield* DateTime.now;
+      const threadId = ThreadId.make("thread-provider-session-manager-host-sleep");
+      const providerSessionId = yield* idAllocator.allocate.providerSession({
+        providerInstanceId: modelSelection.instanceId,
+        threadId,
+      });
+      yield* eventSink.write({
+        events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
+      });
+      yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy });
+
+      yield* TestClock.adjust(input.awakeMs);
+      const untilDeadlineMs = HOST_SLEEP_IDLE_MS - (input.awakeMs % HOST_SLEEP_IDLE_MS);
+      // A counting timer spends part of the sleep reaching its due time, then fires on resume.
+      sleptMs = input.timerCountsSleep ? input.sleepMs - untilDeadlineMs : input.sleepMs;
+      yield* TestClock.adjust(untilDeadlineMs);
+      let deadline = Math.ceil(input.awakeMs / HOST_SLEEP_IDLE_MS);
+      while (yield* manager.isLive(providerSessionId)) {
+        deadline += 1;
+        yield* TestClock.adjust(HOST_SLEEP_IDLE_MS);
+      }
+      assert.equal(yield* Queue.take(closes), "close");
+      return deadline;
+    });
+
+    return yield* effect.pipe(
+      Effect.provide(
+        layerTest({
+          state,
+          unloads: closes,
+          idleTimeoutMs: HOST_SLEEP_IDLE_MS,
+          ...(input.pinnedForMs === undefined
+            ? {}
+            : { maxIdlePinMs: input.pinnedForMs, hasPendingBackgroundWork: Effect.succeed(true) }),
+        }),
+      ),
+      Effect.provideService(Clock.Clock, hostClock),
+    );
+  });
+
+// regression: #11805 sessions were released right after the computer woke from sleep
+it.effect("ProviderSessionManagerV2 does not count host sleep toward idle or pin budgets", () =>
+  Effect.gen(function* () {
+    for (const { label, expectedDeadline, ...scenario } of [
+      {
+        label: "timer counting sleep",
+        timerCountsSleep: true,
+        awakeMs: 5 * 60_000,
+        sleepMs: 2 * 60 * 60_000,
+        expectedDeadline: 2,
+      },
+      {
+        // Overshoots the deadline by under a minute, so deadline lateness alone misses it.
+        label: "short sleep across the deadline",
+        timerCountsSleep: true,
+        awakeMs: 1_000,
+        sleepMs: HOST_SLEEP_IDLE_MS + 30_000,
+        expectedDeadline: 2,
+      },
+      {
+        label: "timer paused during sleep",
+        timerCountsSleep: false,
+        awakeMs: 5 * 60_000,
+        sleepMs: 2 * 60 * 60_000,
+        expectedDeadline: 1,
+      },
+      {
+        // Pinned at the first deadline, then asleep longer than the whole pin budget.
+        label: "pending work pinned across sleep",
+        timerCountsSleep: false,
+        awakeMs: 35 * 60_000,
+        sleepMs: 8 * 60 * 60_000,
+        pinnedForMs: 2 * HOST_SLEEP_IDLE_MS,
+        expectedDeadline: 3,
+      },
+    ]) {
+      assert.equal(yield* idleDeadlineAcrossHostSleep(scenario), expectedDeadline, label);
+    }
+  }),
+);
+
 it.effect("ProviderSessionManagerV2 releases sessions when provider event streams fail", () =>
   Effect.gen(function* () {
     const state = yield* Ref.make(emptyState);

apps/server/src/orchestration-v2/ProviderSessionManager.ts

@@ -88,6 +88,22 @@ const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
   "running",
   "waiting",
 ]);
+const AWAKE_SLICE_MS = 60 * 1000;
+const SUSPEND_TOLERANCE_MS = 10 * 1000;
+
+// Where the timer clock counts host sleep, a timer fires on wake far past due. Waiting in short slices
+// and dropping any slice that overran its tolerance keeps suspended time out of the count, whatever its
+// length. Where both clocks pause, no slice overruns. Untraced: every provider event re-arms the timer.
+const sleepWhileAwake = Effect.fnUntraced(function* (ms: number): Effect.fn.Return<void> {
+  let remainingMs = ms;
+  while (remainingMs > 0) {
+    const sliceMs = Math.min(remainingMs, AWAKE_SLICE_MS);
+    const startedAtNanos = yield* Clock.monotonicTimeNanos;
+    yield* Effect.sleep(Duration.millis(sliceMs));
+    const elapsedMs = Number((yield* Clock.monotonicTimeNanos) - startedAtNanos) / 1_000_000;
+    if (elapsedMs <= sliceMs + SUSPEND_TOLERANCE_MS) remainingMs -= sliceMs;
+  }
+});
 
 /** Matches planDelegatedCompletionDelivery: true while a delegated task's result still wakes the thread. */
 export const delegatedTaskWillWake = (records: {
@@ -291,7 +307,7 @@ interface LiveSessionEntry {
   readonly idleGeneration: number;
   readonly busyTurns: ReadonlySet<string>;
   readonly idleFiber: Fiber.Fiber<void, never> | null;
-  readonly pinnedSinceMs: number | null;
+  readonly pinnedForMs: number | null;
   readonly idleThreadUnloads: ReadonlyMap<ThreadId, IdleThreadUnload>;
 }
 
@@ -1365,9 +1381,9 @@ export const layerWithOptions = (
               ),
             );
           if (hasPendingWork) {
-            const now = yield* Clock.currentTimeMillis;
-            const pinnedSinceMs = entry.pinnedSinceMs ?? now;
-            if (now - pinnedSinceMs < maxIdlePinMs) {
+            // Counted in awake idle windows, so host sleep does not spend the pin budget.
+            const pinnedForMs = entry.pinnedForMs ?? 0;
+            if (pinnedForMs < maxIdlePinMs) {
               const shouldContinuePin = yield* Ref.modify(sessions, (latest) => {
                 const latestEntry = latest.get(key);
                 if (
@@ -1379,23 +1395,23 @@ export const layerWithOptions = (
                   return [false, latest] as const;
                 }
                 const updated = new Map(latest);
-                updated.set(key, { ...latestEntry, pinnedSinceMs });
+                updated.set(key, { ...latestEntry, pinnedForMs: pinnedForMs + idleTimeoutMs });
                 return [true, updated] as const;
               });
               if (!shouldContinuePin) {
                 return;
               }
               yield* Effect.logInfo("orchestration-v2.driver-session.idle-release-deferred", {
                 providerSessionId: input.providerSessionId,
-                pinnedForMs: now - pinnedSinceMs,
+                pinnedForMs,
               });
               // Recheck on this fiber; scheduleIdleReleaseInternal would interrupt it and self-deadlock.
-              yield* Effect.sleep(Duration.millis(idleTimeoutMs));
+              yield* sleepWhileAwake(idleTimeoutMs);
               return yield* releaseIfStillIdle(input);
             }
             yield* Effect.logWarning("orchestration-v2.driver-session.idle-release-pin-expired", {
               providerSessionId: input.providerSessionId,
-              pinnedForMs: now - pinnedSinceMs,
+              pinnedForMs,
             });
           }
           // Closing skips the threads' own idle unloads, so archive delivered children first.
@@ -1459,7 +1475,7 @@ export const layerWithOptions = (
 
         yield* cancelIdleFiber(entry.idleFiber);
         const generation = entry.idleGeneration + 1;
-        const idleFiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe(
+        const idleFiber = yield* sleepWhileAwake(idleTimeoutMs).pipe(
           Effect.andThen(releaseIfStillIdle({ providerSessionId, generation })),
           Effect.forkIn(layerScope),
         );
@@ -1473,7 +1489,7 @@ export const layerWithOptions = (
             ...latestEntry,
             idleGeneration: generation,
             idleFiber,
-            ...(options?.providerActivity === true ? { pinnedSinceMs: null } : {}),
+            ...(options?.providerActivity === true ? { pinnedForMs: null } : {}),
           });
           return updated;
         });
@@ -1704,7 +1720,7 @@ export const layerWithOptions = (
                 ...entry,
                 busyTurns: new Set(entry.busyTurns).add(turnKey),
                 idleFiber: null,
-                pinnedSinceMs: null,
+                pinnedForMs: null,
               });
               return [{ idleFiber: entry.idleFiber }, updated] as const;
             });
@@ -1812,7 +1828,7 @@ export const layerWithOptions = (
           if (tracked === undefined) return;
           const [threadId, pending] = tracked;
           const generation = pending.generation + 1;
-          const fiber = yield* Effect.sleep(Duration.millis(idleTimeoutMs)).pipe(
+          const fiber = yield* sleepWhileAwake(idleTimeoutMs).pipe(
             Effect.andThen(unloadIdleThread({ providerSessionId, threadId, generation })),
             Effect.forkIn(layerScope),
           );
@@ -1959,7 +1975,7 @@ export const layerWithOptions = (
           );
           // Wait outside the lock so the next attach is not blocked.
           if (outcome === "deferred" && input.archiveOnly !== true) {
-            yield* Effect.sleep(Duration.millis(idleTimeoutMs));
+            yield* sleepWhileAwake(idleTimeoutMs);
             return yield* unloadIdleThread(input);
           }
         },
@@ -2961,7 +2977,7 @@ export const layerWithOptions = (
                 idleGeneration: 0,
                 busyTurns: new Set(),
                 idleFiber: null,
-                pinnedSinceMs: null,
+                pinnedForMs: null,
                 idleThreadUnloads: new Map(),
               };
               yield* Ref.update(sessions, (current) => {