[muse] fix(server): bound managed-tunnel recovery retry window to 10 minutes

apps/server/src/cloud/http.test.ts

@@ -61,6 +61,7 @@ import {
   registerManagedCloudTunnelRecovery,
   releaseManagedTunnelOnShutdown,
   startManagedCloudTunnelIfOriginConfirmed,
+  activateManagedTunnelWithRetry,
 } from "./http.ts";
 import {
   managedTunnelStartupAction,
@@ -1260,3 +1261,68 @@ describe("link proof provider kinds", () => {
     expect(linkProofScopes(proofRequest("manual"))).toEqual(["agent_activity_notifications"]);
   });
 });
+
+describe("activateManagedTunnelWithRetry", () => {
+  it.effect("gives up after the retry window instead of spinning forever", () =>
+    Effect.gen(function* () {
+      const configJson = `{"providerKind":"cloudflare_tunnel","connectorToken":"token","tunnelId":"tunnel"}`;
+      const values = new Map<string, Uint8Array>([
+        [CLOUD_ENDPOINT_RUNTIME_CONFIG, new TextEncoder().encode(configJson)],
+      ]);
+      const applyConfigCalls: Array<unknown> = [];
+      const secrets: ServerSecretStore.ServerSecretStore["Service"] = {
+        get: (name: string) => {
+          const value = values.get(name);
+          return value === undefined ? Effect.succeedNone : Effect.succeedSome(value);
+        },
+        set: (name: string, value: Uint8Array) =>
+          Effect.sync(() => {
+            values.set(name, value);
+          }),
+        create: unusedSecretStoreOperation,
+        getOrCreateRandom: unusedSecretStoreOperation,
+        remove: unusedSecretStoreOperation,
+      };
+      const endpointRuntime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"] = {
+        applyConfig: (config) =>
+          Effect.sync(() => {
+            applyConfigCalls.push(config);
+            return {
+              status: "failed" as const,
+              providerKind: "cloudflare_tunnel" as const,
+              failure: "spawn-failed" as const,
+              reason: "connector spawn failed",
+            };
+          }),
+        recoveryRequests: Stream.empty,
+        requestRecovery: () => Effect.void,
+        withLinkStateLock: (effect) => effect,
+      };
+      const dependencies = { secrets, endpointRuntime };
+      const fiber = yield* Effect.forkScoped(
+        activateManagedTunnelWithRetry(
+          dependencies,
+          {
+            config: {
+              providerKind: "cloudflare_tunnel",
+              connectorToken: "token",
+              tunnelId: "tunnel",
+            },
+            configJson,
+            origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
+          },
+          true,
+        ),
+      );
+      // Past the 10-minute retry window: the request must have failed with
+      // the last activation error, releasing the sequential recovery queue.
+      // On base (unbounded retry) the fiber is still running here.
+      yield* TestClock.adjust("11 minutes");
+      // pollUnsafe: in Effect v4 there is no Effect-returning Fiber.poll.
+      expect(fiber.pollUnsafe() !== undefined).toBe(true);
+      const error = yield* Fiber.join(fiber).pipe(Effect.flip);
+      expect(error).toMatchObject({ _tag: "EnvironmentCloudEndpointUnavailableError" });
+      expect(applyConfigCalls.length).toBeGreaterThan(1);
+    }),
+  );
+});

apps/server/src/cloud/http.ts

@@ -71,6 +71,7 @@ import * as ServerConfig from "../config.ts";
 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
 import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts";
 import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
+import { MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW } from "./managedTunnelStartup.ts";
 import {
   SERVICE_STATE_FILE,
   SERVICE_STOP_MARKER_FILE,
@@ -517,7 +518,7 @@ function managedEndpointRuntimeConfigsMatch(
 }
 
 const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* (
-  dependencies: CloudHttpDependencies,
+  dependencies: Pick<CloudHttpDependencies, "secrets" | "endpointRuntime">,
   input: {
     readonly config: RelayManagedEndpointRuntimeConfig;
     readonly configJson: string;
@@ -547,8 +548,10 @@ const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel
   );
 });
 
-const activateManagedTunnelWithRetry = (
-  dependencies: CloudHttpDependencies,
+// Exported for tests: the retry window is what keeps one poisoned recovery
+// request from occupying the sequential recovery queue forever.
+export const activateManagedTunnelWithRetry = (
+  dependencies: Pick<CloudHttpDependencies, "secrets" | "endpointRuntime">,
   input: {
     readonly config: RelayManagedEndpointRuntimeConfig;
     readonly configJson: string;
@@ -565,11 +568,15 @@ const activateManagedTunnelWithRetry = (
             ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus(
               error.endpointRuntimeStatus,
             ),
+          // Bounded like the registration retry window: a persistently
+          // failing connector (binary missing, spawn broken) must release
+          // the recovery queue so newer requests can be processed.
           schedule: Schedule.exponential("1 second").pipe(
             Schedule.modifyDelay(({ duration }) =>
               Effect.succeed(Duration.min(duration, Duration.seconds(30))),
             ),
             Schedule.jittered,
+            Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }),
           ),
         }),
       )

apps/server/src/cloud/managedTunnelStartup.ts

@@ -36,7 +36,7 @@ export function managedTunnelStartupAction(input: {
 
 // After this window the host can start its stored connector config while
 // registration keeps retrying to reconcile the origin when the relay returns.
-const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
+export const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10);
 
 export const retryManagedTunnelRegistration = <A, E, R>(
   registration: Effect.Effect<A, E, R>,