feat: clients show when the host is short on memory and which agents use the most

apps/mobile/src/connection/runtime.ts

@@ -36,7 +36,11 @@ type ConnectionLayerSource =
 
 const layerProvidedClientConnection = layerSnapshotLoader.pipe(
   Layer.provideMerge(
-    Connection.layerWithOptions({ usageLimitSources: true, usageLimitsCommand: true }),
+    Connection.layerWithOptions({
+      usageLimitSources: true,
+      memoryPressure: true,
+      usageLimitsCommand: true,
+    }),
   ),
   Layer.provideMerge(
     Layer.mergeAll(

apps/mobile/src/features/home/HomeScreen.tsx

@@ -84,6 +84,7 @@ import { createSwipeRowActivation } from "./swipe-row-activation";
 import { SwipeableScrollGateProvider, useSwipeableScrollGate } from "./thread-swipe-actions";
 import { useFabScroll } from "./FabScrollContext";
 import { useWorkspaceTabs } from "../workspace/WorkspaceTabsContext";
+import { MemoryPressureBanner } from "./MemoryPressureBanner";
 import { ThreadListFilters } from "./ThreadListFilters";
 
 interface HomeScreenProps {
@@ -707,6 +708,7 @@ export function HomeScreen(props: HomeScreenProps) {
   const v2ListHeader = (
     <>
       {listHeader}
+      <MemoryPressureBanner />
       {Platform.OS === "ios" && workspace ? (
         <ThreadListFilters
           gutter={primaryColumn ? 8 : 0}

apps/mobile/src/features/home/MemoryPressureBanner.tsx

@@ -0,0 +1,136 @@
+import { useAtomValue } from "@effect/atom-react";
+import { scopeThreadRef } from "@t3tools/client-runtime/environment";
+import { isAtomCommandInterrupted } from "@t3tools/client-runtime/state/runtime";
+import type { EnvironmentId, HostMemoryPressureAgent } from "@t3tools/contracts";
+import { useState } from "react";
+import { Pressable, View } from "react-native";
+
+import { AppText } from "../../components/ui/AppText";
+import { SymbolView } from "../../components/ui/AppSymbol";
+import * as Dialog from "../../components/ui/Dialog";
+import { useQuietThreadShell, useServerConfigs } from "../../state/entities";
+import { orchestrationEnvironment } from "../../state/orchestration";
+import { useAtomCommand } from "../../state/use-atom-command";
+import { useOpenLinkedThread } from "../threads/useOpenLinkedThread";
+
+const GiB = 1024 ** 3;
+
+/** Agents throttled for memory stop answering, so name the heaviest ones while it lasts. */
+export function MemoryPressureBanner() {
+  const configs = useServerConfigs();
+  const severe = [...configs].flatMap(([environmentId, config]) =>
+    config.memoryPressure?.level === "severe"
+      ? [{ environmentId, config, pressure: config.memoryPressure }]
+      : [],
+  );
+  if (severe.length === 0) return null;
+  return (
+    <View className="gap-2 px-4 pb-2">
+      {severe.map(({ environmentId, config, pressure }) => (
+        <View
+          key={environmentId}
+          accessibilityRole="summary"
+          className="gap-2 rounded-2xl border border-border bg-card px-3.5 py-3"
+        >
+          <View className="flex-row items-center gap-2">
+            <SymbolView
+              name="exclamationmark.triangle"
+              size={14}
+              tintColorClassName="accent-warning-foreground"
+            />
+            <AppText textRole="footnote" className="min-w-0 flex-1 font-t3-medium text-foreground">
+              {configs.size > 1 ? `${config.environment.label} is low on memory` : "Low on memory"}
+            </AppText>
+          </View>
+          <AppText textRole="caption" className="text-muted-foreground">
+            Agents are being slowed down and may stop responding. Stopping the largest one helps the
+            rest.
+          </AppText>
+          {pressure.heaviestAgents.map((agent, index) => (
+            <MemoryPressureAgentRow
+              key={agent.threadId ?? `${agent.agent}:${index}`}
+              environmentId={environmentId}
+              agent={agent}
+              canForceStop={config.environment.capabilities.threadForceStop === true}
+            />
+          ))}
+        </View>
+      ))}
+    </View>
+  );
+}
+
+function MemoryPressureAgentRow(props: {
+  readonly environmentId: EnvironmentId;
+  readonly agent: HostMemoryPressureAgent;
+  readonly canForceStop: boolean;
+}) {
+  const threadId = props.agent.threadId;
+  const thread = useQuietThreadShell(
+    threadId === null ? null : scopeThreadRef(props.environmentId, threadId),
+  );
+  const openThread = useOpenLinkedThread();
+  const permitted = useAtomValue(
+    orchestrationEnvironment.forceStopThread.permissionAtom(props.environmentId),
+  );
+  const forceStopThread = useAtomCommand(orchestrationEnvironment.forceStopThread, {
+    reportFailure: false,
+  });
+  const [stopping, setStopping] = useState(false);
+  const name =
+    thread?.title ?? (threadId === null ? `${props.agent.agent} (shared)` : props.agent.agent);
+  const forceStop = async (target: NonNullable<typeof threadId>) => {
+    setStopping(true);
+    const result = await forceStopThread({
+      environmentId: props.environmentId,
+      input: { threadId: target },
+    });
+    setStopping(false);
+    if (result._tag === "Failure" && !isAtomCommandInterrupted(result)) {
+      Dialog.alert("Could not force stop the agent");
+    }
+  };
+  return (
+    <View className="flex-row items-center gap-3">
+      <Pressable
+        accessibilityRole={threadId === null ? undefined : "link"}
+        disabled={threadId === null}
+        onPress={() =>
+          threadId === null
+            ? undefined
+            : openThread({ environmentId: props.environmentId, threadId })
+        }
+        className="min-w-0 flex-1 active:opacity-70"
+      >
+        <AppText textRole="footnote" numberOfLines={1} className="text-foreground">
+          {name}
+        </AppText>
+      </Pressable>
+      <AppText textRole="footnote" className="text-muted-foreground tabular-nums">
+        {(props.agent.memoryBytes / GiB).toFixed(1)} GB
+      </AppText>
+      {threadId !== null && props.canForceStop && permitted ? (
+        <Pressable
+          accessibilityRole="button"
+          accessibilityLabel={`Force stop ${name}`}
+          disabled={stopping}
+          hitSlop={8}
+          onPress={() =>
+            Dialog.confirm({
+              title: "Force stop the agent?",
+              message: `Force stop kills ${name}'s processes and ends its turn.`,
+              confirmText: "Force stop",
+              destructive: true,
+              onConfirm: () => void forceStop(threadId),
+            })
+          }
+          className={`active:opacity-70 ${stopping ? "opacity-50" : ""}`}
+        >
+          <AppText textRole="footnote" className="font-t3-medium text-danger-foreground">
+            {stopping ? "Stopping..." : "Force stop"}
+          </AppText>
+        </Pressable>
+      ) : null}
+    </View>
+  );
+}

apps/mobile/src/state/server.ts

@@ -12,6 +12,7 @@ import { environmentSession } from "./session";
 export const serverEnvironment = createServerEnvironmentAtoms(connectionAtomRuntime, {
   initialConfigValueAtom: environmentSession.initialConfigValueAtom,
   usageLimitSources: true,
+  memoryPressure: true,
   usageLimitsCommand: true,
 });
 export const describeOutdatedServer = createOutdatedServerDescriptionCommand(connectionAtomRuntime);

apps/server/src/environment/ServerEnvironment.ts

@@ -241,6 +241,7 @@ export const make = Effect.gen(function* () {
       projectProviderInstanceOverrides: true,
       threadSnooze: true,
       threadForceStop: true,
+      memoryPressure: true,
       environmentThemes: true,
       usageLimitSources: true,
       providerWorkspaceSnapshotsOnDemand: true,

apps/server/src/process/agentScope.test.ts

@@ -110,6 +110,8 @@ const withFakeSystemd = <A, E>(
         ...fileSystem,
         readFileString: (path, encoding) =>
           fileSystem.readFileString(path.replace("/sys/fs/cgroup", cgroupRoot), encoding),
+        readDirectory: (path, options) =>
+          fileSystem.readDirectory(path.replace("/sys/fs/cgroup", cgroupRoot), options),
         writeFileString: (path, data, options) =>
           fileSystem.writeFileString(path.replace("/sys/fs/cgroup", cgroupRoot), data, options),
       }),
@@ -245,6 +247,37 @@ describe("AgentScope service", () => {
       ),
   );
 
+  it.effect("reports each live agent scope's cgroup with the thread that owns it", () =>
+    withFakeSystemd(({ scope, units, cgroupRoot, stateDir }) =>
+      Effect.gen(function* () {
+        expect(yield* scope.controlGroups).toBeUndefined();
+        units.set("app-t3code-agents.slice", {
+          activeState: "active",
+          result: "success",
+          controlGroup: "/agents",
+        });
+        const agent = yield* scope.wrap({
+          command: "agent",
+          args: [],
+          name: "claude",
+          threadId: "a",
+        });
+        const codex = yield* launchCodex(scope, "b");
+        const owned = NodePath.join("/agents", ownedAgentSlice(stateDir));
+        for (const unit of [unitOf(agent), unitOf(codex)]) {
+          NodeFS.mkdirSync(NodePath.join(cgroupRoot, owned, unit), { recursive: true });
+        }
+
+        const groups = yield* scope.controlGroups;
+        expect(groups?.slice).toBe("/agents");
+        expect(groups?.scopes.toSorted((a, b) => a.agent.localeCompare(b.agent))).toEqual([
+          { path: NodePath.join(owned, unitOf(agent)), threadId: "a", agent: "claude" },
+          { path: NodePath.join(owned, unitOf(codex)), threadId: undefined, agent: "codex" },
+        ]);
+      }),
+    ),
+  );
+
   it.effect("keeps a running scope while hundreds of other threads launch", () =>
     withFakeSystemd(({ scope, units }) =>
       Effect.gen(function* () {

apps/server/src/process/agentScope.ts

@@ -3,6 +3,7 @@ import * as NodeCrypto from "node:crypto";
 import * as NodeOS from "node:os";
 
 import {
+  type AgentControlGroups,
   AgentScope,
   agentScopeCommand,
   type AgentScopeShape,
@@ -376,6 +377,43 @@ export const make = Effect.gen(function* () {
     return killed._tag === "Some" && killed.value.code === 0;
   });
 
+  let sliceControlGroup: string | undefined;
+  const controlGroups: AgentScopeShape["controlGroups"] = Effect.gen(function* () {
+    const scope = yield* systemd;
+    if (scope === undefined) return undefined;
+    if (sliceControlGroup === undefined) {
+      const shown = yield* run(scope.systemctl, [
+        "--user",
+        "show",
+        AGENT_SLICE,
+        "--property=ControlGroup",
+      ]);
+      const controlGroup =
+        shown._tag === "Some" ? parseKeyValues(shown.value.stdout).get("ControlGroup") : undefined;
+      // Empty until the first agent starts the slice.
+      if (controlGroup === undefined || controlGroup.length === 0) return undefined;
+      sliceControlGroup = controlGroup;
+    }
+    const ownedControlGroup = path.join(sliceControlGroup, ownedSlice);
+    const entries = yield* fileSystem
+      .readDirectory(path.join("/sys/fs/cgroup", ownedControlGroup))
+      .pipe(Effect.orElseSucceed((): ReadonlyArray<string> => []));
+    const threadByUnit = new Map(
+      [...scopeByThread].flatMap(([threadId, tracked]) =>
+        tracked.shared ? [] : [[tracked.unit, threadId] as const],
+      ),
+    );
+    return {
+      slice: sliceControlGroup,
+      scopes: entries.flatMap((unit) => {
+        const agent = /^t3code-(.+)-[0-9a-f]{8}\.scope$/.exec(unit)?.[1];
+        return agent === undefined
+          ? []
+          : [{ path: path.join(ownedControlGroup, unit), threadId: threadByUnit.get(unit), agent }];
+      }),
+    } satisfies AgentControlGroups;
+  });
+
   const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) {
     if (!scopeByThread.has(threadId)) return;
     const scope = yield* systemd;
@@ -423,6 +461,7 @@ export const make = Effect.gen(function* () {
     oomKilled,
     stop,
     kill,
+    controlGroups,
     clear,
   } satisfies AgentScopeShape;
 });