From f6ac40bb373f66b04fd92d1d23e341bbb3b26774 Mon Sep 17 00:00:00 2001 From: Kevin Rajan Date: Sun, 27 Sep 2026 22:55:39 -0500 Subject: [PATCH] [muse] fix(server): bound managed-tunnel recovery retry window to 10 minutes activateManagedTunnelWithRetry retried retryable connector failures (not-installed, spawn-failed) with an unbounded exponential schedule. server.ts drains recovery requests sequentially, so one persistently failing connector occupied the consumer forever and starved every later recovery request. Bound the inner schedule with Schedule.upTo using the existing 10-minute MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW, so a poisoned request fails after the window and the queue keeps draining. Also narrows activateManagedTunnel/activateManagedTunnelWithRetry to the secrets + endpointRuntime dependencies they actually use, and exports the window constant and the retry helper (the helper is exported for tests). Authored with AI assistance (Muse, Meta's Muse Spark) under the contributor's direction, as documented in the PR description. --- apps/server/src/cloud/http.test.ts | 66 +++++++++++++++++++ apps/server/src/cloud/http.ts | 13 +++- apps/server/src/cloud/managedTunnelStartup.ts | 2 +- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/apps/server/src/cloud/http.test.ts b/apps/server/src/cloud/http.test.ts index 1d6b78e56cefc..0ff46fbfde590 100644 --- a/apps/server/src/cloud/http.test.ts +++ b/apps/server/src/cloud/http.test.ts @@ -61,6 +61,7 @@ import { registerManagedCloudTunnelRecovery, releaseManagedTunnelOnShutdown, startManagedCloudTunnelIfOriginConfirmed, + activateManagedTunnelWithRetry, } from "./http.ts"; import { managedTunnelStartupAction, @@ -1260,3 +1261,68 @@ describe("link proof provider kinds", () => { expect(linkProofScopes(proofRequest("manual"))).toEqual(["agent_activity_notifications"]); }); }); + +describe("activateManagedTunnelWithRetry", () => { + it.effect("gives up after the retry window instead of spinning forever", () => + Effect.gen(function* () { + const configJson = `{"providerKind":"cloudflare_tunnel","connectorToken":"token","tunnelId":"tunnel"}`; + const values = new Map([ + [CLOUD_ENDPOINT_RUNTIME_CONFIG, new TextEncoder().encode(configJson)], + ]); + const applyConfigCalls: Array = []; + const secrets: ServerSecretStore.ServerSecretStore["Service"] = { + get: (name: string) => { + const value = values.get(name); + return value === undefined ? Effect.succeedNone : Effect.succeedSome(value); + }, + set: (name: string, value: Uint8Array) => + Effect.sync(() => { + values.set(name, value); + }), + create: unusedSecretStoreOperation, + getOrCreateRandom: unusedSecretStoreOperation, + remove: unusedSecretStoreOperation, + }; + const endpointRuntime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"] = { + applyConfig: (config) => + Effect.sync(() => { + applyConfigCalls.push(config); + return { + status: "failed" as const, + providerKind: "cloudflare_tunnel" as const, + failure: "spawn-failed" as const, + reason: "connector spawn failed", + }; + }), + recoveryRequests: Stream.empty, + requestRecovery: () => Effect.void, + withLinkStateLock: (effect) => effect, + }; + const dependencies = { secrets, endpointRuntime }; + const fiber = yield* Effect.forkScoped( + activateManagedTunnelWithRetry( + dependencies, + { + config: { + providerKind: "cloudflare_tunnel", + connectorToken: "token", + tunnelId: "tunnel", + }, + configJson, + origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 }, + }, + true, + ), + ); + // Past the 10-minute retry window: the request must have failed with + // the last activation error, releasing the sequential recovery queue. + // On base (unbounded retry) the fiber is still running here. + yield* TestClock.adjust("11 minutes"); + // pollUnsafe: in Effect v4 there is no Effect-returning Fiber.poll. + expect(fiber.pollUnsafe() !== undefined).toBe(true); + const error = yield* Fiber.join(fiber).pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "EnvironmentCloudEndpointUnavailableError" }); + expect(applyConfigCalls.length).toBeGreaterThan(1); + }), + ); +}); diff --git a/apps/server/src/cloud/http.ts b/apps/server/src/cloud/http.ts index a7e3de8c72831..620a099d6f75f 100644 --- a/apps/server/src/cloud/http.ts +++ b/apps/server/src/cloud/http.ts @@ -71,6 +71,7 @@ import * as ServerConfig from "../config.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as AgentAwarenessRelay from "../relay/AgentAwarenessRelay.ts"; import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; +import { MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW } from "./managedTunnelStartup.ts"; import { SERVICE_STATE_FILE, SERVICE_STOP_MARKER_FILE, @@ -517,7 +518,7 @@ function managedEndpointRuntimeConfigsMatch( } const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel")(function* ( - dependencies: CloudHttpDependencies, + dependencies: Pick, input: { readonly config: RelayManagedEndpointRuntimeConfig; readonly configJson: string; @@ -547,8 +548,10 @@ const activateManagedTunnel = Effect.fn("environment.cloud.activateManagedTunnel ); }); -const activateManagedTunnelWithRetry = ( - dependencies: CloudHttpDependencies, +// Exported for tests: the retry window is what keeps one poisoned recovery +// request from occupying the sequential recovery queue forever. +export const activateManagedTunnelWithRetry = ( + dependencies: Pick, input: { readonly config: RelayManagedEndpointRuntimeConfig; readonly configJson: string; @@ -565,11 +568,15 @@ const activateManagedTunnelWithRetry = ( ManagedEndpointRuntime.isRetryableManagedEndpointRuntimeStatus( error.endpointRuntimeStatus, ), + // Bounded like the registration retry window: a persistently + // failing connector (binary missing, spawn broken) must release + // the recovery queue so newer requests can be processed. schedule: Schedule.exponential("1 second").pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(30))), ), Schedule.jittered, + Schedule.upTo({ duration: MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW }), ), }), ) diff --git a/apps/server/src/cloud/managedTunnelStartup.ts b/apps/server/src/cloud/managedTunnelStartup.ts index 511d5545a0ac9..192532a53bc46 100644 --- a/apps/server/src/cloud/managedTunnelStartup.ts +++ b/apps/server/src/cloud/managedTunnelStartup.ts @@ -36,7 +36,7 @@ export function managedTunnelStartupAction(input: { // After this window the host can start its stored connector config while // registration keeps retrying to reconcile the origin when the relay returns. -const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10); +export const MANAGED_TUNNEL_REGISTRATION_RETRY_WINDOW = Duration.minutes(10); export const retryManagedTunnelRegistration = ( registration: Effect.Effect,