Skip to content

Commit 31a5741

Browse files
committed
fix(reactor): flush accepted rejections and timeouts during shutdown
1 parent a702539 commit 31a5741

4 files changed

Lines changed: 237 additions & 4 deletions

File tree

‎e2e/reactor-approval-reload.test.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,92 @@ describe("integration: an ask settled without running, then reloading the agent
593593
);
594594
});
595595

596+
// The loop is busy on a second message's inference when the rejection is
597+
// accepted, so the close's abort is served first and only the shutdown flush
598+
// writes the rejection's result. The reactor leaves the approval record
599+
// behind; the session store must close it on load.
600+
describe("integration: a rejection the reactor accepted but never ran, then reopening the agent", () => {
601+
test.serial(
602+
"the reopened agent loads the denial once and no parked call, and the old deadline adds no result",
603+
async () => {
604+
const clock = holdLongTimers();
605+
cleanups.push(() => clock.restore());
606+
const ctx = deferredApprovalGate();
607+
const sessionOpts = {
608+
permissionGate: ctx.gate,
609+
authorize: createReactorAuthorize(ctx.gate),
610+
pinToolSetAtBuild: true,
611+
};
612+
const live = { session: await openIntegrationSession(sessionOpts) };
613+
cleanups.push(() => closeIntegrationSession(live.session));
614+
live.session.harness.scenario.replyOnce("anthropic", {
615+
toolCalls: [
616+
{ name: "run_shell", args: { command: CLOSED_PORT_PROBE } },
617+
],
618+
});
619+
live.session.harness.scenario.stall();
620+
621+
const turn = await runUntilSuspended(
622+
live.session,
623+
"Probe the local port.",
624+
);
625+
const suspended = turn.result;
626+
if (suspended.type !== "suspended") throw new Error("did not park");
627+
const parked = (await live.session.storage.load()).pendingOperations.find(
628+
(op) => op.correlationId === suspended.correlationId,
629+
);
630+
const callId = parked?.suspendedCall?.id;
631+
const deadline = parked?.timeoutAt;
632+
if (callId === undefined || deadline === undefined) {
633+
throw new Error("parked call has no persisted call id or deadline");
634+
}
635+
636+
void live.session.agent.send("Still there?").catch(() => undefined);
637+
await waitFor(
638+
() => live.session.harness.scenario.matchedRequests().length === 2,
639+
"second message's inference",
640+
);
641+
const resume = createApprovalResume({
642+
getAgent: () => live.session.agent,
643+
gate: ctx.gate,
644+
resolveParkedCallId: (id) =>
645+
resolveParkedCallIdFromStore(live.session.storage, id),
646+
});
647+
const handling = resume.handle(suspended);
648+
await waitFor(ctx.asked, "approval prompt");
649+
ctx.reject();
650+
expect(await handling).toBe(true);
651+
await waitFor(
652+
() => turn.events.some((e) => e.type === "message.correlated"),
653+
"rejection accepted",
654+
);
655+
656+
live.session = await rebuildIntegrationSession(live.session, sessionOpts);
657+
const { session } = live;
658+
expect((await session.storage.load()).pendingOperations).toEqual([]);
659+
const settled = toolResultsFor(await session.agent.history(), callId);
660+
expect(settled).toHaveLength(1);
661+
expect(settled.join("")).toContain("denied by approver");
662+
663+
const reloaded = {
664+
session,
665+
events: collect(session),
666+
callId,
667+
deadline,
668+
clock,
669+
};
670+
await passDeadline(reloaded, deadline + FRESH_WINDOW_PASSED_MS);
671+
expect(reloaded.events.filter((e) => e.type === "reactor.error")).toEqual(
672+
[],
673+
);
674+
expect(toolResultsFor(await session.agent.history(), callId)).toEqual(
675+
settled,
676+
);
677+
},
678+
20_000,
679+
);
680+
});
681+
596682
// Reloads at the run's end, where the TUI's deferred reload fires.
597683
describe("integration: an ask settled without running, then reloading the agent at the run's end", () => {
598684
const runEnded = (

‎vendor/intx-inference/PATCHES.md‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -522,12 +522,23 @@ Upstream needs no equivalent while it commits every tool cycle. A staged
522522
compact skips the write in both places; a parked approval cannot coincide
523523
with one, since the suspend terminal commits and clears it.
524524

525+
`initiateShutdown` appends any `resume.tool_result` still queued behind the
526+
abort (a decision the reactor accepted and removed from its pending set, but
527+
never processed, since the loop serves `abort` first), before
528+
`gates.shutdown()` clears the gate timers. It then writes turns once, only
529+
when it appended something, inside a `try/catch` that reports a non-fatal
530+
`reactor.error`, so a failing store cannot skip `reactor.done` or leak the
531+
workdir lock. Approved calls (`resume.execute_tools`) are not flushed: one
532+
that never ran stays parked on disk rather than gaining a result for a
533+
tool that did not run. A flushed result closes the approval under the same
534+
store dependency as the `resume.tool_result` write.
535+
525536
**Disposition:** Promotion candidate. Performance coalescing — collapses K
526537
git commits per agentic turn to one deferred commit while preserving
527-
rebuild-from-disk durability. The `resume.tool_result` write relies on the
528-
store closing an answered call's approval, which upstream stores do not do,
529-
so promoting it also needs the reactor to clear that pending operation when
530-
it writes the result.
538+
rebuild-from-disk durability. The `resume.tool_result` write and the
539+
shutdown flush rely on the store closing an answered call's approval, which
540+
upstream stores do not do, so promoting them also needs the reactor to clear
541+
that pending operation when it writes the result.
531542
**Removal path:** Upstream PR replacing the `executeTools` checkpoint with a
532543
writeTurns-only segments touch and deferring the cycle commit to cycle
533544
terminals.

‎vendor/intx-inference/src/reactor.test.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5494,6 +5494,109 @@ describe("createReactor — approval resume error result", () => {
54945494
await waitForEvent(events, (e) => e.type === "connector.reply");
54955495
expect(parkedCallResultTexts(cell.turns)).toHaveLength(1);
54965496
});
5497+
5498+
// The loop must be busy when the decision is accepted, or it acts on it
5499+
// on the next turn of the loop before an abort can be queued. A second
5500+
// message's held inference keeps it busy; the abort queued behind that
5501+
// inference is dequeued ahead of the accepted decision.
5502+
async function decideThenAbortBehindHeldInference(
5503+
contextStore: ContextStore,
5504+
beforeAbort: () => void = () => {},
5505+
decision: (correlationId: string) => InboundMessage = rejection,
5506+
) {
5507+
const held = heldContinuationRunner();
5508+
const handle = createTestReactor({
5509+
contextStore,
5510+
director: createDefaultDirector("test agent", []),
5511+
inferenceRunner: held.runner,
5512+
beforeToolExtensions: [askExtension()],
5513+
});
5514+
const { reactor, events, waitFor } = handle;
5515+
5516+
reactor.start();
5517+
reactor.deliver(makeInboundMessage());
5518+
const blocked = await waitFor("reactor.gate.blocked");
5519+
if (blocked.type !== "reactor.gate.blocked") throw new Error("unreachable");
5520+
const correlationId = blocked.data.correlationId;
5521+
if (correlationId === undefined) throw new Error("expected correlationId");
5522+
5523+
reactor.deliver(makeInboundMessage());
5524+
await held.continuationStarted;
5525+
reactor.deliver(decision(correlationId));
5526+
await waitForEvent(events, (e) => e.type === "message.correlated");
5527+
5528+
beforeAbort();
5529+
reactor.abort("user_disconnect");
5530+
held.release();
5531+
await waitFor("reactor.done");
5532+
return handle;
5533+
}
5534+
5535+
test("an accepted rejection still queued at shutdown reaches the store", async () => {
5536+
const cell: PersistedCell = {
5537+
turns: [],
5538+
pendingOperations: [],
5539+
tokenUsage: emptyUsage(),
5540+
};
5541+
5542+
await decideThenAbortBehindHeldInference(makePersistingContextStore(cell));
5543+
5544+
expect(parkedCallResultTexts(cell.turns)).toEqual([
5545+
"denied by approver: too risky",
5546+
]);
5547+
});
5548+
5549+
test("an accepted approval still queued at shutdown stays parked with no result", async () => {
5550+
const cell: PersistedCell = {
5551+
turns: [],
5552+
pendingOperations: [],
5553+
tokenUsage: emptyUsage(),
5554+
};
5555+
5556+
await decideThenAbortBehindHeldInference(
5557+
makePersistingContextStore(cell),
5558+
undefined,
5559+
(correlationId) => makeApprovalMessage(correlationId),
5560+
);
5561+
5562+
// The host handed the approval over and the reactor accepted it, but the
5563+
// call never ran: only a rebuilt agent's host can settle it now.
5564+
expect(parkedCallResultTexts(cell.turns)).toEqual([]);
5565+
expect(cell.pendingOperations.map((op) => op.suspendedCall?.id)).toEqual([
5566+
"call-ask",
5567+
]);
5568+
});
5569+
5570+
test("a store that fails the shutdown write is reported without blocking shutdown", async () => {
5571+
const cell: PersistedCell = {
5572+
turns: [],
5573+
pendingOperations: [],
5574+
tokenUsage: emptyUsage(),
5575+
};
5576+
const persisting = makePersistingContextStore(cell);
5577+
let failWrites = false;
5578+
const contextStore: ContextStore = {
5579+
...persisting,
5580+
async writeTurns(turns) {
5581+
if (failWrites) throw new Error("disk full");
5582+
await persisting.writeTurns(turns);
5583+
},
5584+
};
5585+
5586+
const { events } = await decideThenAbortBehindHeldInference(
5587+
contextStore,
5588+
() => {
5589+
failWrites = true;
5590+
},
5591+
);
5592+
5593+
const errors = events.flatMap((e) =>
5594+
e.type === "reactor.error" ? [e.data] : [],
5595+
);
5596+
expect(errors).toHaveLength(1);
5597+
expect(errors[0]?.fatal).toBe(false);
5598+
expect(errors[0]?.error).toContain("disk full");
5599+
});
54975600
});
54985601

54995602
// ---------------------------------------------------------------------------

‎vendor/intx-inference/src/reactor.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1886,12 +1886,45 @@ export function createReactor(config: ReactorConfig): Reactor {
18861886
shutdownStarted = true;
18871887

18881888
abortOperations();
1889+
1890+
// A rejection or timeout the reactor accepted but had not yet processed
1891+
// when the abort overtook it has already left the pending set, so its
1892+
// result line is the only trace of the decision, and closes the approval
1893+
// on disk only in a store that drops an answered call's pending
1894+
// operation, as the resume.tool_result write above relies on. Taken before
1895+
// gates.shutdown(), which clears every gate timer, so no timeout can
1896+
// enqueue another after it; isLiveApproval refuses decisions from here.
1897+
//
1898+
// Locally patched — see vendor/intx-inference/PATCHES.md#reactor-ts-tool-cycle-turns-only
1899+
const acceptedResults = queue.flatMap((event) =>
1900+
event.type === "resume.tool_result" ? [event.result] : [],
1901+
);
1902+
if (stateManager !== null) {
1903+
for (const result of acceptedResults) {
1904+
stateManager.appendTurn(createToolResultTurn([result]));
1905+
}
1906+
}
1907+
18891908
gates.shutdown();
18901909

18911910
if (stateManager !== null) {
18921911
stateManager.setGatesSnapshot([]);
18931912
}
18941913

1914+
// Shutdown also runs after fatal errors, where the store may be what
1915+
// failed; a throw here must not skip reactor.done or the lock release.
1916+
if (acceptedResults.length > 0) {
1917+
try {
1918+
await writeTurnsIfChanged();
1919+
} catch (cause) {
1920+
logger.error`shutdown turn write failed: ${cause}`;
1921+
emitError(
1922+
`shutdown turn write failed: ${cause instanceof Error ? cause.message : String(cause)}`,
1923+
false,
1924+
);
1925+
}
1926+
}
1927+
18951928
if (inFlight.size > 0) {
18961929
const deadline = new Promise<void>((resolve) =>
18971930
setTimeout(resolve, shutdownTimeoutMs),

0 commit comments

Comments
 (0)