feat(server): continue a delegated child its provider cut off before failing the task

apps/server/src/mcp/OrchestratorMcpService.ts

@@ -2221,9 +2221,22 @@ const make = Effect.gen(function* () {
           } satisfies OrchestratorMcpTaskCancelResult;
         }
         const child = yield* loadProjection(current.childThreadId);
+        // A child whose cut-off run waits to be continued (after a restart or a
+        // passing provider failure) has no live run, yet Stop still ends the wait.
+        const awaitingContinuation =
+          ThreadManagementService.latestActiveRun(child) === undefined &&
+          child.runs.some((run) => run.status === "failed" || run.status === "cancelled") &&
+          parentTask !== undefined &&
+          (yield* threadManagement
+            .delegatedTaskResultPending(
+              current.childThreadId,
+              delegatedTaskRun(child, parentTask)?.delegatedTaskId,
+            )
+            .pipe(Effect.mapError(threadManagementFailure)));
         if (
           current.workState !== "waiting_for_children" &&
-          ThreadManagementService.latestActiveRun(child) === undefined
+          ThreadManagementService.latestActiveRun(child) === undefined &&
+          !awaitingContinuation
         ) {
           return yield* failure(
             "task_not_cancellable",

apps/server/src/orchestration-v2/DelegatedProviderRetry.ts

@@ -0,0 +1,57 @@
+import type { OrchestrationV2ProviderFailure, OrchestrationV2Run, RunId } from "@t3tools/contracts";
+import {
+  providerFailureVerdict,
+  providerRetryWaitMs,
+} from "@t3tools/shared/providerFailureVerdict";
+import * as DateTime from "effect/DateTime";
+
+/**
+ * A delegated child whose turn a passing provider failure cut off is continued in place, with the same
+ * `delegatedTaskId`, after 1, 2, 4 and 8 minutes. While the continuation is due, the task and the parent's wake are
+ * held as they are for a restart continuation, so the parent hears once: the result of the turn that finally ends.
+ */
+export const providerRetryEffectId = (runId: RunId) => `effect:provider-retry:${runId}`;
+export const providerRetryCommandId = (runId: RunId) => `command:provider-retry:${runId}`;
+export const providerRetryReleaseEffectId = (runId: RunId) =>
+  `effect:provider-retry-release:${runId}`;
+export const providerRetryMessageId = (runId: RunId) => `message:provider-retry:${runId}`;
+
+export interface ProviderRetryPlan {
+  /** This run's place in the chain of cut-off turns in a row, from 1. */
+  readonly cuts: number;
+  readonly reason: string;
+  readonly availableAt: DateTime.Utc;
+}
+
+/**
+ * When to continue a failed run of a delegated task, or null when its failure is final: lasting or unrecognised, the
+ * fifth cut in a row, or a run Stop already reached.
+ */
+export function providerRetryPlan(input: {
+  readonly run: OrchestrationV2Run;
+  readonly runs: ReadonlyArray<OrchestrationV2Run>;
+  readonly failure: Pick<OrchestrationV2ProviderFailure, "class" | "code" | "message"> | null;
+  readonly stopped: boolean;
+  readonly now: DateTime.Utc;
+}): ProviderRetryPlan | null {
+  const { run, runs, failure } = input;
+  if (run.status !== "failed" || failure === null || input.stopped) return null;
+  const verdict = providerFailureVerdict(failure);
+  let cuts = 1;
+  const seen = new Set<RunId>([run.id]);
+  for (
+    let previousId = run.providerRetryOfRunId;
+    previousId !== undefined && !seen.has(previousId);
+    previousId = runs.find((candidate) => candidate.id === previousId)?.providerRetryOfRunId
+  ) {
+    seen.add(previousId);
+    cuts += 1;
+  }
+  const waitMs = providerRetryWaitMs(verdict, cuts);
+  if (waitMs === null) return null;
+  return {
+    cuts,
+    reason: verdict.reason,
+    availableAt: DateTime.add(run.completedAt ?? input.now, { milliseconds: waitMs }),
+  };
+}

apps/server/src/orchestration-v2/EffectOutbox.ts

@@ -33,6 +33,14 @@ export const OrchestrationEffectRequestV2 = Schema.Union([
     type: Schema.Literal("provider-runtime.continue"),
     sourceRunId: RunId,
   }),
+  Schema.Struct({
+    type: Schema.Literal("provider-runtime.retry"),
+    sourceRunId: RunId,
+  }),
+  Schema.Struct({
+    type: Schema.Literal("provider-runtime.retry-release"),
+    settledRunId: RunId,
+  }),
   Schema.Struct({
     type: Schema.Literal("provider-session.detach"),
     providerSessionId: ProviderSessionId,
@@ -119,6 +127,8 @@ export type OrchestrationEffectRequestV2 = typeof OrchestrationEffectRequestV2.T
 
 export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [
   "provider-runtime.continue",
+  "provider-runtime.retry",
+  "provider-runtime.retry-release",
   "provider-session.detach",
   "provider-thread.rollback",
   "checkpoint.capture",

apps/server/src/orchestration-v2/EffectWorker.ts

@@ -7,7 +7,9 @@ import * as Effect from "effect/Effect";
 import * as Exit from "effect/Exit";
 import * as Layer from "effect/Layer";
 import * as Metric from "effect/Metric";
+import * as FileSystem from "effect/FileSystem";
 import * as Option from "effect/Option";
+import * as Path from "effect/Path";
 import * as Schema from "effect/Schema";
 
 import {
@@ -27,7 +29,16 @@ import * as RuntimeRequestService from "./RuntimeRequestService.ts";
 import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
 import * as ThreadManagementService from "./ThreadManagementService.ts";
 import * as ServerSettings from "../serverSettings.ts";
+import * as ServerConfig from "../config.ts";
 import { continueRestartedRun } from "./RestartContinuation.ts";
+import {
+  continueAfterProviderFailure,
+  releaseAfterProviderRetry,
+  type ResumeGuard,
+  workspaceResumeGuard,
+} from "./ProviderRetryContinuation.ts";
+import * as ProcessRunner from "../processRunner.ts";
+import * as ProjectStore from "./ProjectStore.ts";
 
 export class OrchestrationEffectExecutionError extends Schema.TaggedError<OrchestrationEffectExecutionError>()(
   "OrchestrationEffectExecutionError",
@@ -97,6 +108,21 @@ export const layerExecutor: Layer.Layer<
       yield* ThreadTitleRegenerationService.ThreadTitleRegenerationService;
     const threads = yield* ThreadManagementService.ThreadManagementService;
     const settings = yield* ServerSettings.ServerSettingsService;
+    // Optional so executors built without process access still continue
+    // provider-cut runs, only without the workspace's record of their steps.
+    const guardServices = Option.all({
+      processRunner: yield* Effect.serviceOption(ProcessRunner.ProcessRunner),
+      fileSystem: yield* Effect.serviceOption(FileSystem.FileSystem),
+      path: yield* Effect.serviceOption(Path.Path),
+      projects: yield* Effect.serviceOption(ProjectStore.ProjectStoreV2),
+    });
+    const serverConfig = yield* Effect.serviceOption(ServerConfig.ServerConfig);
+    const resumeGuard: ResumeGuard | undefined = Option.isSome(guardServices)
+      ? workspaceResumeGuard({
+          ...guardServices.value,
+          stateDbPath: Option.getOrUndefined(serverConfig)?.dbPath,
+        })
+      : undefined;
     return OrchestrationEffectExecutorV2.of({
       execute: (effect, options) => {
         const willRetry = options?.willRetry ?? false;
@@ -121,6 +147,40 @@ export const layerExecutor: Layer.Layer<
               ),
             );
           }
+          case "provider-runtime.retry": {
+            const sourceRunId = effect.request.sourceRunId;
+            return continueAfterProviderFailure({
+              threadId: effect.threadId,
+              sourceRunId,
+              ...(resumeGuard === undefined
+                ? {}
+                : { listCompletedOutwardSteps: resumeGuard.listCompletedOutwardSteps }),
+            }).pipe(
+              Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
+              Effect.tapError(() =>
+                willRetry
+                  ? Effect.void
+                  : threads.recoverDelegatedTask(effect.threadId, sourceRunId),
+              ),
+              Effect.mapError(
+                (cause) =>
+                  new OrchestrationEffectExecutionError({
+                    effectId: effect.id,
+                    effectType: effect.request.type,
+                    cause,
+                  }),
+              ),
+            );
+          }
+          case "provider-runtime.retry-release":
+            return releaseAfterProviderRetry({
+              threadId: effect.threadId,
+              ...(resumeGuard === undefined
+                ? {}
+                : { releaseOutwardSteps: resumeGuard.releaseOutwardSteps }),
+            }).pipe(
+              Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
+            );
           case "provider-session.detach":
             return providerSessions
               .detach({