Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/test-layout/layout-codex-collision.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"codex-quota-persistence.test.ts": "codex-integration",
"codex-stored-main-duplicate.test.ts": "codex-integration",
"codex-pool-caller-fallback-policy.test.ts": "codex-integration",
"codex-windows-current-sid.test.ts": "codex-integration"
Expand Down
11 changes: 10 additions & 1 deletion src/codex/quota.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const QUOTA_CACHE_FILENAME = "codex-quota-cache.json";
/** Keep last-known bars across restarts; WHAM still refreshes on TTL in live/prime paths. */
const QUOTA_DISK_MAX_AGE_MS = 6 * 60 * 60_000;
const QUOTA_PERSIST_DEBOUNCE_MS = 250;
/** Match the reset ledger's maximum deferral under sustained quota observations. */
const QUOTA_PERSIST_MAX_DEFERRAL_MS = 1_000;

type QuotaDiskFile = {
version: 1;
Expand All @@ -39,6 +41,7 @@ type MainPolicyQuotaObservation = Omit<StoredAccountQuota, "updatedAt"> & { shor
let mainPolicyQuota: MainPolicyQuota | null = null;
let diskHydrated = false;
let persistTimer: ReturnType<typeof setTimeout> | null = null;
let persistStartedAt: number | null = null;

const MONTHLY_WINDOW_MIN_SECONDS = 28 * 24 * 60 * 60;
/**
Expand Down Expand Up @@ -778,9 +781,14 @@ function hydrateAccountQuotasFromDisk(): void {
}

function schedulePersistAccountQuotas(): void {
const now = performance.now();
persistStartedAt ??= now;
const delay = Math.max(0, Math.min(QUOTA_PERSIST_DEBOUNCE_MS,
QUOTA_PERSIST_MAX_DEFERRAL_MS - (now - persistStartedAt)));
if (persistTimer) clearTimeout(persistTimer);
persistTimer = setTimeout(() => {
persistTimer = null;
persistStartedAt = null;
try {
const quotas: Record<string, StoredAccountQuota> = {};
for (const [accountId, quota] of accountQuota.entries()) {
Expand All @@ -796,7 +804,7 @@ function schedulePersistAccountQuotas(): void {
} catch {
// Best-effort persistence only.
}
}, QUOTA_PERSIST_DEBOUNCE_MS);
}, delay);
}

export function getAccountQuota(accountId: string): StoredAccountQuota | null {
Expand Down Expand Up @@ -861,6 +869,7 @@ export function clearAccountQuota(accountId?: string): void {
clearTimeout(persistTimer);
persistTimer = null;
}
persistStartedAt = null;
try {
const path = join(getConfigDir(), QUOTA_CACHE_FILENAME);
if (existsSync(path)) unlinkSync(path);
Expand Down
16 changes: 12 additions & 4 deletions src/providers/account-quota-disk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const FILENAME = "provider-account-quota-cache.json";
*/
const DISK_MAX_AGE_MS = 6 * 60 * 60_000;
const PERSIST_DEBOUNCE_MS = 250;
/** Match the reset ledger's maximum deferral under sustained quota observations. */
const MAX_PERSIST_DEFERRAL_MS = 1_000;

type DiskFile = {
version: 1;
Expand All @@ -38,6 +40,7 @@ type DiskFile = {
};

let persistTimer: ReturnType<typeof setTimeout> | null = null;
let persistStartedAt: number | null = null;

/** Read the snapshot. Returns an empty map for a missing, corrupt or stale file. */
export function readPersistedAccountQuotas(now = Date.now()): Map<string, ProviderQuota> {
Expand Down Expand Up @@ -86,14 +89,19 @@ export function readPersistedKiroVerdicts(now = Date.now()): Map<string, KiroPer
return result;
}

/** Write the snapshot, debounced. Best-effort: a failed write is not an error. */
/** Debounce snapshots without starving writes under load. Failures remain best-effort. */
export function schedulePersistAccountQuotas(
rows: () => Iterable<[string, ProviderQuota | KiroPersistedQuota]>,
verdicts: () => Iterable<[string, KiroPersistedVerdict]> = () => [],
): void {
const now = performance.now();
persistStartedAt ??= now;
const delay = Math.max(0, Math.min(PERSIST_DEBOUNCE_MS,
MAX_PERSIST_DEFERRAL_MS - (now - persistStartedAt)));
if (persistTimer) clearTimeout(persistTimer);
persistTimer = setTimeout(() => {
persistTimer = null;
persistStartedAt = null;
try {
const out: Record<string, ProviderQuota> = {};
for (const [key, quota] of rows()) out[key] = quota;
Expand All @@ -106,12 +114,12 @@ export function schedulePersistAccountQuotas(
} catch {
// Best-effort persistence only.
}
}, PERSIST_DEBOUNCE_MS);
}, delay);
}

/** Test seam: drop any pending write so a suite cannot leak one into the next file. */
export function cancelPendingAccountQuotaPersist(): void {
if (!persistTimer) return;
clearTimeout(persistTimer);
if (persistTimer) clearTimeout(persistTimer);
persistTimer = null;
persistStartedAt = null;
}
3 changes: 3 additions & 0 deletions structure/INDEX.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ Per-vendor contracts and the adapter authority that constructs them.

| Doc | Scope |
| --- | --- |
| [`quota-snapshots.md`](quota-snapshots.md) | Bounded write deferral and cancellation for Codex and provider account quota snapshots. |
| [`providers/generic-account-pool.md`](providers/generic-account-pool.md) | Opt-in conversation ownership, request-local selection receipts and bounded release for generic OAuth pools. |
| [`providers/anthropic-progress.md`](providers/anthropic-progress.md) | Opus 5.5 progress display, visible commentary, and exact signed-block replay. |
| [`providers/anthropic-account-thresholds.md`](providers/anthropic-account-thresholds.md) | Account-owned usage thresholds, inheritance, routing boundaries and durable policy changes. |
Expand Down Expand Up @@ -214,6 +215,7 @@ A source area can be described by more than one doc, because these docs are orga
| `src/codex/catalog/vendor-context.ts` | [`catalog-vendor-context.md`](catalog-vendor-context.md) |
| `src/codex/cli-update-plan.ts` | [`codex-cli-update.md`](codex-cli-update.md) |
| `src/codex/provider-routing.ts` | [`codex-provider-routing.md`](codex-provider-routing.md) |
| `src/codex/quota.ts` | [`quota-snapshots.md`](quota-snapshots.md) |
| `src/codex/runtime.ts` | [`catalog-effort-clamp.md`](catalog-effort-clamp.md) |
| `src/codex/signed-in-identity.ts` | [`providers/openai-signed-in-preservation.md`](providers/openai-signed-in-preservation.md) |
| `src/combos/` | [`runtime.md`](runtime.md)<br>[`providers-and-adapters.md`](providers-and-adapters.md)<br>[`providers/jev-decision.md`](providers/jev-decision.md)<br>[`fallback-policy.md`](fallback-policy.md) |
Expand Down Expand Up @@ -254,6 +256,7 @@ A source area can be described by more than one doc, because these docs are orga
| `src/plugins/` | [`ops/plugins.md`](ops/plugins.md) |
| `src/protocols/` | [`data-planes/protocol-paths.md`](data-planes/protocol-paths.md) |
| `src/providers/` | [`runtime.md`](runtime.md)<br>[`subagents.md`](subagents.md)<br>[`transports/inventory.md`](transports/inventory.md)<br>[`providers-and-adapters.md`](providers-and-adapters.md)<br>[`providers/anthropic-account-pool.md`](providers/anthropic-account-pool.md)<br>[`providers/xai-grok.md`](providers/xai-grok.md) |
| `src/providers/account-quota-disk.ts` | [`quota-snapshots.md`](quota-snapshots.md) |
| `src/providers/anthropic-billing-ownership.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) |
| `src/providers/anthropic-billing-recovery.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) |
| `src/providers/anthropic-billing-refusal.ts` | [`providers/anthropic-key-billing.md`](providers/anthropic-key-billing.md) |
Expand Down
7 changes: 7 additions & 0 deletions structure/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@
}
],
"docs": [
{
"path": "quota-snapshots.md",
"tier": 4,
"title": "Account Quota Snapshots",
"scope": "Bounded write deferral and cancellation for Codex and provider account quota snapshots.",
"documents": ["src/codex/quota.ts", "src/providers/account-quota-disk.ts"]
},
{
"path": "providers/generic-account-pool.md",
"tier": 4,
Expand Down
25 changes: 25 additions & 0 deletions structure/quota-snapshots.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Account Quota Snapshots

`src/codex/quota.ts` and `src/providers/account-quota-disk.ts` persist last-known account
quotas for restart recovery. The Codex snapshot also carries bounded observation history
and separately identity-bound main-policy evidence; provider snapshots include Kiro's
identity-bound exhaustion verdicts. Their existing hydration and expiry rules govern readers.

## Bounded write deferral

Both writers coalesce updates with a 250 ms idle debounce and a one-second maximum
deferral from the first pending update, measured on the monotonic clock. Continued
observations cannot postpone the scheduled write beyond that deadline; event-loop delays
and disk failures remain best-effort. This uses the same maximum deferral as the quota
reset ledger in `src/quota/reset-seen-store.ts` without making producer calls write synchronously.

The timer snapshots the latest accepted state and resolves the configured home when it
runs. Completion, failure, and cancellation reset the deadline. Clearing all Codex quotas
cancels its pending snapshot; clearing one account persists the remaining rows with the
existing deadline. Provider cancellation discards the pending row and verdict callbacks.
Atomic replacement and account identity checks stay with the existing writer and producer.

`tests/codex-integration/codex-quota-persistence.test.ts` and
`tests/providers/provider-account-quota-persistence.test.ts` check real disk contents under
controlled timers: idle debounce, consecutive sustained-update windows, cancellation and
home reset, removed rows, latest provider callbacks, and recovery after a failed write.
109 changes: 109 additions & 0 deletions tests/codex-integration/codex-quota-persistence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { afterEach, beforeEach, describe, expect, jest, spyOn, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { clearAccountQuota, flushQuotaObservationsForTests, setAccountQuotaFromParsed } from "../../src/codex/quota";
import { removeTreeWithRetry } from "../helpers/remove-tree";

const previousHome = process.env.OPENCODEX_HOME;
const FILE = "codex-quota-cache.json";
let homes: string[];
let home: string;
let elapsed: number;
let restoreClock: () => void;

function advance(ms: number): void {
elapsed += ms;
jest.advanceTimersByTime(ms);
}

function snapshot(): Record<string, { weeklyPercent: number }> {
return JSON.parse(readFileSync(join(home, FILE), "utf8")).quotas;
}

beforeEach(async () => {
home = mkdtempSync(join(tmpdir(), "ocx-codex-quota-persist-"));
homes = [home];
process.env.OPENCODEX_HOME = home;
clearAccountQuota();
await flushQuotaObservationsForTests();
jest.useFakeTimers();
elapsed = 0;
const clock = spyOn(performance, "now").mockImplementation(() => elapsed);
restoreClock = () => clock.mockRestore();
});

afterEach(async () => {
clearAccountQuota();
await flushQuotaObservationsForTests();
restoreClock();
jest.clearAllTimers();
jest.useRealTimers();
if (previousHome === undefined) delete process.env.OPENCODEX_HOME;
else process.env.OPENCODEX_HOME = previousHome;
for (const path of homes) removeTreeWithRetry(path);
});

describe("Codex quota snapshot persistence", () => {
test("a short burst keeps the 250 ms idle debounce and writes the latest quota", () => {
setAccountQuotaFromParsed("pool-a", { weeklyPercent: 10 });
advance(100);
setAccountQuotaFromParsed("pool-a", { weeklyPercent: 20 });
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
advance(1);
expect(snapshot()["pool-a"]?.weeklyPercent).toBe(20);
});

test("sustained updates persist the latest quota at each one-second deadline", () => {
for (let batch = 0; batch < 2; batch++) {
for (let update = 1; update <= 10; update++) {
setAccountQuotaFromParsed("pool-a", { weeklyPercent: batch * 10 + update });
advance(100);
}
expect(snapshot()["pool-a"]?.weeklyPercent).toBe((batch + 1) * 10);
}
});

test("clearing one account keeps the pending deadline and excludes its row", () => {
setAccountQuotaFromParsed("removed", { weeklyPercent: 75 });
for (let update = 1; update <= 9; update++) {
setAccountQuotaFromParsed("kept", { weeklyPercent: update });
advance(100);
}
clearAccountQuota("removed");
advance(100);
expect(snapshot()).toEqual({ kept: expect.objectContaining({ weeklyPercent: 9 }) });
});

test("clearing all accounts cancels the old write and resets the deadline for a new home", () => {
const oldHome = home;
for (let update = 0; update < 9; update++) {
setAccountQuotaFromParsed("old-account", { weeklyPercent: update });
advance(100);
}
clearAccountQuota();
home = mkdtempSync(join(tmpdir(), "ocx-codex-quota-next-"));
homes.push(home);
process.env.OPENCODEX_HOME = home;
setAccountQuotaFromParsed("new-account", { weeklyPercent: 15 });
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
expect(existsSync(join(oldHome, FILE))).toBe(false);
advance(1);
expect(snapshot()).toEqual({ "new-account": expect.objectContaining({ weeklyPercent: 15 }) });
});

test("a failed write remains best-effort and the next update gets a fresh debounce", () => {
mkdirSync(join(home, FILE));
setAccountQuotaFromParsed("pool-a", { weeklyPercent: 10 });
expect(() => advance(250)).not.toThrow();
rmdirSync(join(home, FILE));
advance(1_000);
setAccountQuotaFromParsed("pool-a", { weeklyPercent: 20 });
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
advance(1);
expect(snapshot()["pool-a"]?.weeklyPercent).toBe(20);
});
});
1 change: 1 addition & 0 deletions tests/fixtures/test-layout-expected.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{ "workbuddy-quota.test.ts": "providers", "workbuddy-login.test.ts": "oauth", "workbuddy-adapter.test.ts": "adapters", "workbuddy-provider.test.ts": "providers", "link-tunnel-history.test.ts": "clients", "link-readiness.test.ts": "clients", "link-diagnostics.test.ts": "clients", "combo-description.test.ts": "routing", "combo-description-catalog.test.ts": "codex-integration", "catalog-auto-review-override.test.ts": "codex-integration", "management-auto-review-settings.test.ts": "server", "management-auto-review-auth.test.ts": "server", "prefer-model-alias-ids.test.ts": "server", "responses-mimo-envelope.test.ts": "responses", "responses-call-id-linkage.test.ts": "responses", "responses-phase-inference.test.ts": "responses",
"service-proxy-env.test.ts": "service",
"codex-quota-persistence.test.ts": "codex-integration",
"bridge-terminal-refusal.test.ts": "adapters",
"responses-replay-metadata-principal.test.ts": "responses",
"responses-combo-replay-principal.test.ts": "responses",
Expand Down
87 changes: 85 additions & 2 deletions tests/providers/provider-account-quota-persistence.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
import { afterEach, beforeEach, describe, expect, jest, spyOn, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
cancelPendingAccountQuotaPersist,
readPersistedAccountQuotas,
readPersistedKiroVerdicts,
schedulePersistAccountQuotas,
} from "../../src/providers/account-quota-disk";
import type { ProviderQuota } from "../../src/providers/quota-types";
Expand Down Expand Up @@ -105,3 +106,85 @@ describe("provider account quota persistence", () => {
expect(existsSync(join(home, FILE))).toBe(false);
});
});

describe("provider quota snapshot deadlines", () => {
let elapsed: number;
let restoreClock: () => void;
const advance = (ms: number) => {
elapsed += ms;
jest.advanceTimersByTime(ms);
};

beforeEach(() => {
jest.useFakeTimers();
elapsed = 0;
const clock = spyOn(performance, "now").mockImplementation(() => elapsed);
restoreClock = () => clock.mockRestore();
});

afterEach(() => {
cancelPendingAccountQuotaPersist();
restoreClock();
jest.clearAllTimers();
jest.useRealTimers();
});

test("a short burst keeps the 250 ms idle debounce", () => {
schedulePersistAccountQuotas(() => [[KEY, quota(10)]]);
advance(100);
schedulePersistAccountQuotas(() => [[KEY, quota(20)]]);
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
advance(1);
expect(readPersistedAccountQuotas().get(KEY)?.monthlyPercent).toBe(20);
});

test("sustained updates persist the latest row and verdict callbacks every second", () => {
const key = "kiro\u0000acct-a";
const identity = "a".repeat(64);
for (let batch = 0; batch < 2; batch++) {
for (let update = 1; update <= 10; update++) {
const percent = batch * 10 + update;
schedulePersistAccountQuotas(() => [[key, { ...quota(percent), identity }]],
() => [[key, { exhausted: percent >= 20, identity, observedAt: Date.now() }]]);
advance(100);
}
expect(readPersistedAccountQuotas().get(key)?.monthlyPercent).toBe((batch + 1) * 10);
expect(readPersistedKiroVerdicts().get(key)?.exhausted).toBe(batch === 1);
}
});

test("cancellation resets the deadline before the next home schedules a write", () => {
const oldHome = home;
for (let update = 0; update < 9; update++) {
schedulePersistAccountQuotas(() => [[KEY, quota(update)]]);
advance(100);
}
cancelPendingAccountQuotaPersist();
home = mkdtempSync(join(tmpdir(), "ocx-quota-persist-next-"));
process.env.OPENCODEX_HOME = home;
try {
schedulePersistAccountQuotas(() => [["cursor\u0000new-account", quota(15)]]);
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
expect(existsSync(join(oldHome, FILE))).toBe(false);
advance(1);
expect([...readPersistedAccountQuotas().keys()]).toEqual(["cursor\u0000new-account"]);
} finally {
removeTreeWithRetry(oldHome);
}
});

test("a failed write remains best-effort and the next update gets a fresh debounce", () => {
mkdirSync(join(home, FILE));
schedulePersistAccountQuotas(() => [[KEY, quota(10)]]);
expect(() => advance(250)).not.toThrow();
rmdirSync(join(home, FILE));
advance(1_000);
schedulePersistAccountQuotas(() => [[KEY, quota(20)]]);
advance(249);
expect(existsSync(join(home, FILE))).toBe(false);
advance(1);
expect(readPersistedAccountQuotas().get(KEY)?.monthlyPercent).toBe(20);
});
});
Loading