From 308845770be513968a021cf8233ff16db47ab7ce Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:13:41 -0700 Subject: [PATCH 1/7] feat: bound audit_log details payloads --- backend/src/db/audit-log.ts | 42 ++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/backend/src/db/audit-log.ts b/backend/src/db/audit-log.ts index 23cd6555f..f690b0951 100644 --- a/backend/src/db/audit-log.ts +++ b/backend/src/db/audit-log.ts @@ -19,11 +19,51 @@ export interface AuditLogEntry { export const AUDIT_LOG_INSERT_SQL = `INSERT INTO audit_log (user_id, action, resource_type, resource_id, details) VALUES (?, ?, ?, ?, ?)`; +/** + * Upper bound, in UTF-8 bytes, on an audit row's `details` payload (#1189). + * + * The D1 backup renders each row as one INSERT statement, and D1 refuses to + * execute a statement over ~100 KB on restore (SQLITE_TOOBIG, #1188) -- so an + * unbounded `details` column is a row that can be written but never restored. + * 16 KB leaves a wide margin under that limit even after the backup's SQL + * quote-escaping (worst case doubles the payload), while being far above any + * legitimate summary payload: an audit row carries flags, counts, and a + * pointer to the artifact that owns the detail, not the detail itself. + */ +export const AUDIT_DETAILS_MAX_BYTES = 16_384; + +/** Characters of an oversized payload kept in the truncation marker -- enough + * to expose a JSON payload's leading summary fields for identification. */ +const AUDIT_DETAILS_HEAD_CHARS = 2_000; + +/** + * Bound a `details` payload to {@link AUDIT_DETAILS_MAX_BYTES}: a payload at + * or under the limit passes through untouched; an oversized one is replaced + * by a small JSON marker recording that it was dropped, its original size, + * and its head. This is deliberately lossy -- by the time a payload is this + * large it is per-file detail that belongs in the artifact that owns it + * (see #1189), and preserving it here is what made the backup unrestorable. + */ +export function boundAuditDetails(details: string | null): string | null { + if (details === null) return null; + const bytes = new TextEncoder().encode(details).byteLength; + if (bytes <= AUDIT_DETAILS_MAX_BYTES) return details; + return JSON.stringify({ + audit_details_truncated: true, + original_bytes: bytes, + head: details.slice(0, AUDIT_DETAILS_HEAD_CHARS), + }); +} + /** * Positional params for AUDIT_LOG_INSERT_SQL. Exported separately so the * behavioral test can run the production SQL + marshaling against bun:sqlite * (same approach as user-tombstone.ts, whose test uses a different driver API * than D1). + * + * `details` is bounded here, in the shared marshaling, rather than at call + * sites: every write through auditLogStatement inherits the bound, so the + * next unbounded payload shape cannot silently recreate #1188. */ export function auditLogParams( entry: AuditLogEntry, @@ -33,7 +73,7 @@ export function auditLogParams( entry.action, entry.resourceType ?? null, entry.resourceId ?? null, - entry.details ?? null, + boundAuditDetails(entry.details ?? null), ]; } From e84de00459640aef09b7e9011f52f6c960530b36 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:13:45 -0700 Subject: [PATCH 2/7] feat: verify audits store counts and pointer --- backend/src/routes/admin/imports.ts | 39 +++++++++++++++---- backend/src/services/import-integrity.ts | 48 ++++++++++++++++++++++++ backend/src/services/import-recovery.ts | 13 +++++-- backend/src/services/import-retry.ts | 28 +++++++++++--- 4 files changed, 111 insertions(+), 17 deletions(-) diff --git a/backend/src/routes/admin/imports.ts b/backend/src/routes/admin/imports.ts index 7262d1791..a04d612c6 100644 --- a/backend/src/routes/admin/imports.ts +++ b/backend/src/routes/admin/imports.ts @@ -17,7 +17,8 @@ import { stampDatasetIntegrity } from "../../services/dataset-metadata-columns"; import { deleteDatasetCascade } from "../../services/deletion"; import { type GitHubRepo, createRepository, deleteRepository } from "../../services/github"; import { getDatasetsToken } from "../../services/github-auth"; -import { verifyDatasetVersionS3 } from "../../services/import-integrity"; +import type { DatasetVersionIntegrityResult } from "../../services/import-integrity"; +import { integrityAuditSummary, verifyDatasetVersionS3 } from "../../services/import-integrity"; import { IMPORT_STATUSES } from "../../services/import-recovery"; import { recoverRow } from "../../services/import-retry"; import { hasRole } from "../../types/bindings"; @@ -74,6 +75,35 @@ export function isValidSourceIdForDataset( return rule ? rule(datasetId, sourceId) === true : false; } +/** + * The `import_verify_forced` audit write, as a bound statement (#1189). + * + * `details` is the bounded {@link integrityAuditSummary} -- counts and a + * pointer to `.nemar/availability-report.json` -- never + * `JSON.stringify(verified)`, whose inlined key arrays are what made the D1 + * backup unrestorable (#1188). + * + * Extracted from the verify route body so a real-D1 test can drive the + * write: the route itself cannot be exercised past verifyDatasetVersionS3 + * without live AWS credentials (its S3 listing hardcodes a + * `*.s3.*.amazonaws.com` host; see imports-verify-route.test.ts), and the + * no-mocks policy rules out faking that call. Same seam discipline as + * stampDatasetIntegrity. + */ +export function importVerifyAuditStatement( + db: D1Database, + userId: number, + datasetId: string, + verified: DatasetVersionIntegrityResult, +): D1PreparedStatement { + return auditLogStatement(db, { + userId, + action: "import_verify_forced", + resourceId: datasetId, + details: JSON.stringify(integrityAuditSummary(verified)), + }); +} + export function registerImportRoutes(admin: AdminRouter): void { // ─── Import external dataset ──────────────────────────────────────────────── @@ -513,12 +543,7 @@ export function registerImportRoutes(admin: AdminRouter): void { ); } - await auditLogStatement(c.env.DB, { - userId: c.get("user").id, - action: "import_verify_forced", - resourceId: datasetId, - details: JSON.stringify(verified), - }).run(); + await importVerifyAuditStatement(c.env.DB, c.get("user").id, datasetId, verified).run(); // Explicit pick, not a spread: verifyDatasetVersionS3's runtime result // carries extra bytesPresent/declaredBytes/declaredFiles/version fields diff --git a/backend/src/services/import-integrity.ts b/backend/src/services/import-integrity.ts index 43770ae95..ba94d9821 100644 --- a/backend/src/services/import-integrity.ts +++ b/backend/src/services/import-integrity.ts @@ -155,6 +155,54 @@ export interface DatasetVersionIntegrityResult extends ImportIntegrityResult { version: string | null; } +/** + * Bounded audit summary of a {@link DatasetVersionIntegrityResult} (#1189). + * + * Flags, counts, and a pointer -- never the key arrays. Audit rows used to + * inline `missingKeys`/`zeroByteKeys` in full via JSON.stringify(verified), + * which scaled row size with the dataset's file count (largest production + * row: 12,397 keys, 1.15 MB) and made the hourly D1 backup unrestorable + * (#1188: a row's single-INSERT backup statement exceeded D1's ~100 KB + * statement limit). The per-file detail lives in the artifact that owns it, + * `.nemar/availability-report.json` on the dataset repo's `main` + * (services/availability-report.ts), which is richer than the arrays were: + * keyed by manifest path rather than annex key (two paths can share one + * key), with a per-entry reason (`zero_byte` / `absent`), versioned in git. + * + * An explicit pick, not a spread, for the same reason the verify route's + * response is (routes/admin/imports.ts): a field added to the result type + * later must not silently start being persisted -- least of all a new + * unbounded one. + * + * Migration 0074 rewrites pre-existing array-carrying audit rows into this + * same shape (plus a `compacted_by` marker); keep the two in sync. + */ +export function integrityAuditSummary(verified: DatasetVersionIntegrityResult): { + complete: boolean; + expectedCount: number; + presentCount: number; + bytesPresent: number; + declaredBytes: number; + declaredFiles: number; + version: string | null; + missing_count: number; + zero_byte_count: number; + detail_ref: string; +} { + return { + complete: verified.complete, + expectedCount: verified.expectedCount, + presentCount: verified.presentCount, + bytesPresent: verified.bytesPresent, + declaredBytes: verified.declaredBytes, + declaredFiles: verified.declaredFiles, + version: verified.version, + missing_count: verified.missingKeys.length, + zero_byte_count: verified.zeroByteKeys.length, + detail_ref: ".nemar/availability-report.json", + }; +} + /** A resolved, parsed manifest -- version and files travel together so * "we have files but no version" (or vice versa) is unrepresentable. */ export interface ResolvedManifest { diff --git a/backend/src/services/import-recovery.ts b/backend/src/services/import-recovery.ts index 6ef2039ff..9f108d629 100644 --- a/backend/src/services/import-recovery.ts +++ b/backend/src/services/import-recovery.ts @@ -9,6 +9,7 @@ * without DB/network. */ +import { auditLogStatement } from "../db/audit-log.js"; import { SYSTEM_USER_ID } from "../lib/constants.js"; import type { Bindings } from "../types/bindings.js"; import { deleteDatasetCascade } from "./deletion.js"; @@ -134,10 +135,14 @@ async function writeAudit( details: unknown, ): Promise { try { - await db - .prepare("INSERT INTO audit_log (action, resource_id, details) VALUES (?, ?, ?)") - .bind(action, datasetId, JSON.stringify(details)) - .run(); + // Through auditLogStatement so these writes inherit the shared details + // size bound (#1189). userId null = system-initiated. + await auditLogStatement(db, { + userId: null, + action, + resourceId: datasetId, + details: JSON.stringify(details), + }).run(); } catch (err) { console.error(`[import-recovery] audit_log write failed for ${datasetId}:`, err); } diff --git a/backend/src/services/import-retry.ts b/backend/src/services/import-retry.ts index f0e10d11e..2dbd6865d 100644 --- a/backend/src/services/import-retry.ts +++ b/backend/src/services/import-retry.ts @@ -24,6 +24,7 @@ * logic is exhaustively unit-tested without D1 or GitHub. */ +import { auditLogStatement } from "../db/audit-log.js"; import type { Bindings } from "../types/bindings.js"; import { parseSqliteUtc } from "./auto-import.js"; import { stampDatasetIntegrity } from "./dataset-metadata-columns.js"; @@ -32,7 +33,7 @@ import { isNonProductionEnv } from "./environment.js"; import { getDatasetsToken } from "./github-auth.js"; import { triggerOpenNeuroOnboard } from "./github.js"; import type { DatasetVersionIntegrityResult } from "./import-integrity.js"; -import { verifyDatasetVersionS3 } from "./import-integrity.js"; +import { integrityAuditSummary, verifyDatasetVersionS3 } from "./import-integrity.js"; import { OPENNEURO_UPSTREAM_MARKER } from "./import-recovery.js"; /** ~2 weeks: the window an incomplete/failed import is retried before an @@ -244,10 +245,16 @@ async function writeAudit( details: unknown, ): Promise { try { - await db - .prepare("INSERT INTO audit_log (action, resource_id, details) VALUES (?, ?, ?)") - .bind(action, datasetId, JSON.stringify(details)) - .run(); + // Through auditLogStatement, not a hand-rolled INSERT, so these writes + // inherit the shared details size bound (#1189) -- the retry engine is + // one of the two writers of integrity-result payloads that made the + // backup unrestorable (#1188). userId null = system-initiated. + await auditLogStatement(db, { + userId: null, + action, + resourceId: datasetId, + details: JSON.stringify(details), + }).run(); } catch (err) { console.error(`[import-retry] audit_log write failed for ${datasetId}:`, err); } @@ -341,7 +348,16 @@ export async function reclassifyCompleteRows( `[import-retry] reclassify-to-incomplete UPDATE matched 0 rows for ${row.dataset_id}`, ); } - await writeAudit(env.DB, "import_reclassified_incomplete", row.dataset_id, verified); + // The bounded summary (#1189), never the raw `verified`: its inlined + // key arrays are the same unbounded shape that made import_verify_forced + // rows unrestorable (#1188). The per-path detail belongs to the + // availability report the summary points at. + await writeAudit( + env.DB, + "import_reclassified_incomplete", + row.dataset_id, + integrityAuditSummary(verified), + ); reclassified++; } From 8a579e68183c191847a18aa2ba6c90896bb7a047 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:13:49 -0700 Subject: [PATCH 3/7] feat: zarr failures column stores summary --- backend/src/routes/callbacks/zarr-ready.ts | 31 ++++++++++------- test/zarr-failure-tracking.test.ts | 40 +++++++++++++++------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/backend/src/routes/callbacks/zarr-ready.ts b/backend/src/routes/callbacks/zarr-ready.ts index 6accb06ac..f6660d4d2 100644 --- a/backend/src/routes/callbacks/zarr-ready.ts +++ b/backend/src/routes/callbacks/zarr-ready.ts @@ -69,6 +69,16 @@ interface ZarrReadyBody { * `zarr_failed_at`, so a clean run clears the failure detail and the dashboard * can sort/filter recent failures. Defensive against missing/garbage fields so * the always-200 callback contract holds. + * + * `dataFailuresJson` is a bounded summary -- an entry count plus a pointer -- + * never the entries themselves (#1189). This column used to store the full + * sanitized `data_failures` array (largest production row: 877 entries, + * 178 KB), which pushed `datasets` rows past D1's ~100 KB statement limit in + * the single-INSERT-per-row backup and made it unrestorable (#1188). The + * per-entry detail (path/code/reason) is owned by the published Zarr index's + * `failures` list at `/zarr/index.json` in the serving bucket + * (scripts/zarr/generate_zarr.py writes it; purge_non_raw_stores.py rewrites + * it), which is what `detail_ref` names. */ export function zarrFailureColumns(body: { errors?: number; @@ -94,23 +104,18 @@ export function zarrFailureColumns(body: { // Typed data failures are a SUBSET of total errors; clamp so a converter bug // (or a missing failure_count) can never render "3 data failures of 1 error". const failureCount = Math.min(rawFailureCount, errors); - // Store only the known fields, length-capped: the values are display-only on - // the dashboard and `datasets` is heavily queried, so a malformed/huge - // data_failures item must not bloat the row's TEXT column. - const sanitized = dataFailures.map((item) => { - if (typeof item !== "object" || item === null) return {}; - const i = item as Record; - return { - ...(typeof i.path === "string" ? { path: i.path.slice(0, 512) } : {}), - ...(typeof i.code === "string" ? { code: i.code.slice(0, 64) } : {}), - ...(typeof i.reason === "string" ? { reason: i.reason.slice(0, 256) } : {}), - }; - }); return { errors, failureCount, deterministic: body.deterministic === true ? 1 : 0, - dataFailuresJson: sanitized.length > 0 ? JSON.stringify(sanitized) : null, + // `count` is the reported entry count of the index's failures list, + // deliberately unclamped -- the clamped display count is the separate + // zarr_failure_count column (failureCount above). Row size is now + // independent of how many recordings failed. + dataFailuresJson: + dataFailures.length > 0 + ? JSON.stringify({ count: dataFailures.length, detail_ref: "zarr/index.json" }) + : null, hadErrors: errors > 0, }; } diff --git a/test/zarr-failure-tracking.test.ts b/test/zarr-failure-tracking.test.ts index 8034ffec4..ac3d90a91 100644 --- a/test/zarr-failure-tracking.test.ts +++ b/test/zarr-failure-tracking.test.ts @@ -1,6 +1,9 @@ /** - * #774: /webhooks/zarr-ready persists per-dataset conversion-failure detail so - * the observability dashboard can render a live failures table. + * #774: /webhooks/zarr-ready persists per-dataset conversion-failure state + * for the observability dashboard. Since #1189 the `zarr_data_failures` + * column carries a bounded summary (entry count + a pointer to the Zarr + * index's `failures` list), never the per-entry detail -- the inlined array + * is what made the D1 backup unrestorable (#1188). * * Two parts, no mocks: * - zarrFailureColumns(): the pure derivation of the failure columns from the @@ -28,7 +31,10 @@ describe("zarrFailureColumns (#774)", () => { }); }); - test("partial run: typed data failures -> JSON + counts, not yet terminal", () => { + test("partial run: typed data failures -> bounded summary + counts, not yet terminal", () => { + // #1189: the column stores a count and a pointer to the Zarr index's + // `failures` list, never the entries themselves (the inlined array is + // what made the D1 backup unrestorable, #1188). const df = [{ path: "sub-01_meg.fif", code: "maxshield", reason: "needs MaxFilter" }]; const f = zarrFailureColumns({ errors: 3, @@ -39,7 +45,10 @@ describe("zarrFailureColumns (#774)", () => { expect(f.errors).toBe(3); expect(f.failureCount).toBe(1); expect(f.deterministic).toBe(0); - expect(JSON.parse(f.dataFailuresJson as string)).toEqual(df); + expect(JSON.parse(f.dataFailuresJson as string)).toEqual({ + count: 1, + detail_ref: "zarr/index.json", + }); expect(f.hadErrors).toBe(true); }); @@ -85,20 +94,24 @@ describe("zarrFailureColumns (#774)", () => { ).toBe(1); }); - test("data_failures items are sanitized to known fields + length-capped", () => { + test("stored payload size is independent of the entries (counts + pointer only)", () => { + // Successor to the old per-entry sanitize/length-cap test: since #1189 + // nothing from an entry is stored at all, so a huge or malformed item + // cannot bloat the row -- only the entry count and the pointer land. const f = zarrFailureColumns({ - errors: 2, + errors: 3, data_failures: [ { path: "p", code: "c", reason: "r", junk: { huge: "x".repeat(99999) } }, "not-an-object", { code: "x".repeat(200) }, ], }); - const parsed = JSON.parse(f.dataFailuresJson as string); - expect(parsed).toHaveLength(3); - expect(parsed[0]).toEqual({ path: "p", code: "c", reason: "r" }); // junk dropped - expect(parsed[1]).toEqual({}); // non-object -> empty - expect(parsed[2].code.length).toBe(64); // capped + expect(JSON.parse(f.dataFailuresJson as string)).toEqual({ + count: 3, + detail_ref: "zarr/index.json", + }); + expect((f.dataFailuresJson as string).length).toBeLessThan(100); + expect(f.dataFailuresJson).not.toContain("x".repeat(64)); }); }); @@ -189,7 +202,10 @@ describe("migration 0046 + zarr-ready handler SQL", () => { expect(r.zarr_store_count).toBe(5); expect(r.zarr_errors).toBe(2); expect(r.zarr_deterministic).toBe(0); - expect(JSON.parse(r.zarr_data_failures as string)).toHaveLength(1); + expect(JSON.parse(r.zarr_data_failures as string)).toEqual({ + count: 1, + detail_ref: "zarr/index.json", + }); expect(r.zarr_failed_at).not.toBeNull(); }); From da453616dc2c6a6d72a276e9dcd0f354d19d1cde Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:13:53 -0700 Subject: [PATCH 4/7] feat: compact stored payloads in 0074 --- .../0074_compact_oversized_payloads.sql | 76 ++++ ...mpact-oversized-payloads-migration.test.ts | 377 ++++++++++++++++++ 2 files changed, 453 insertions(+) create mode 100644 backend/src/db/migrations/0074_compact_oversized_payloads.sql create mode 100644 backend/test/compact-oversized-payloads-migration.test.ts diff --git a/backend/src/db/migrations/0074_compact_oversized_payloads.sql b/backend/src/db/migrations/0074_compact_oversized_payloads.sql new file mode 100644 index 000000000..89fc04b21 --- /dev/null +++ b/backend/src/db/migrations/0074_compact_oversized_payloads.sql @@ -0,0 +1,76 @@ +-- Compact the two unbounded per-file payloads already stored in D1 (#1189, +-- fixes #1188): rewrite them into the counts-and-pointer summaries the write +-- paths now produce, so no `audit_log` or `datasets` row renders a backup +-- INSERT statement over D1's ~100 KB statement limit. Restoring the hourly +-- backup aborted with SQLITE_TOOBIG on 15 such statements: 11 audit rows +-- whose `details` inlined a full integrity-check `missingKeys`/`zeroByteKeys` +-- key list (largest: 12,397 keys, 1.15 MB), and 4 datasets rows whose +-- `zarr_data_failures` inlined the per-recording failure array (largest: 877 +-- entries, 178 KB). +-- +-- Counts are DERIVED from the arrays being dropped (json_array_length), not +-- zeroed: the summary must answer "how many" even though "which" now lives in +-- the artifact that owns the detail -- `.nemar/availability-report.json` on +-- the dataset repo for missing content, the published Zarr index's `failures` +-- list (`/zarr/index.json` in the serving bucket) for conversion +-- failures. Both are richer than the copies dropped here. +-- +-- Each rewrite carries 'compacted_by' = 'migration_0074' so a reader can tell +-- a migrated record from one the write paths produced natively. +-- +-- Scoped by payload SHAPE, not by action/status: BOTH integrity key arrays +-- must be present for the audit rewrite (that pair is the +-- DatasetVersionIntegrityResult signature, written by import_verify_forced +-- AND import_reclassified_incomplete), and `zarr_data_failures` must be a +-- top-level JSON array. Rows that are NULL, non-JSON, or already compact +-- match neither predicate and are left byte-identical. +-- +-- The CASE wrapper in each WHERE is load-bearing: json_type() throws on +-- malformed JSON rather than returning NULL, and SQLite does not guarantee +-- left-to-right evaluation of AND operands, so `json_valid(x) AND +-- json_type(x, ...)` could still evaluate json_type first. CASE guarantees +-- the json_valid gate runs before any json_type call. +-- +-- json(col -> '$.key') rather than json_extract(col, '$.key') for the copied +-- scalar fields: json_extract collapses JSON true/false to 1/0, while `->` +-- returns the JSON subvalue verbatim and json() re-embeds it, so `complete` +-- stays a boolean and a null `version` stays null -- byte-compatible with +-- what JSON.stringify(integrityAuditSummary(...)) writes. +-- +-- Naturally idempotent: a rewritten payload no longer carries the arrays, so +-- a re-run matches zero rows. + +UPDATE audit_log +SET details = json_object( + 'complete', json(details -> '$.complete'), + 'expectedCount', json(details -> '$.expectedCount'), + 'presentCount', json(details -> '$.presentCount'), + 'bytesPresent', json(details -> '$.bytesPresent'), + 'declaredBytes', json(details -> '$.declaredBytes'), + 'declaredFiles', json(details -> '$.declaredFiles'), + 'version', json(details -> '$.version'), + 'missing_count', json_array_length(details, '$.missingKeys'), + 'zero_byte_count', json_array_length(details, '$.zeroByteKeys'), + 'detail_ref', '.nemar/availability-report.json', + 'compacted_by', 'migration_0074' +) +WHERE CASE + WHEN details IS NOT NULL AND json_valid(details) + THEN json_type(details, '$.missingKeys') = 'array' + AND json_type(details, '$.zeroByteKeys') = 'array' + ELSE 0 +END; + +-- No datasets trigger fires here: datasets_fts_au and datasets_embed_dirty_au +-- are UPDATE OF triggers on the search/embedding columns only. +UPDATE datasets +SET zarr_data_failures = json_object( + 'count', json_array_length(zarr_data_failures), + 'detail_ref', 'zarr/index.json', + 'compacted_by', 'migration_0074' +) +WHERE CASE + WHEN zarr_data_failures IS NOT NULL AND json_valid(zarr_data_failures) + THEN json_type(zarr_data_failures) = 'array' + ELSE 0 +END; diff --git a/backend/test/compact-oversized-payloads-migration.test.ts b/backend/test/compact-oversized-payloads-migration.test.ts new file mode 100644 index 000000000..6df89d526 --- /dev/null +++ b/backend/test/compact-oversized-payloads-migration.test.ts @@ -0,0 +1,377 @@ +/** + * Migration 0074_compact_oversized_payloads (#1189, fixes #1188): rewrite the + * stored unbounded per-file payloads -- `audit_log.details` integrity-check + * key arrays and `datasets.zarr_data_failures` entry arrays -- into the + * counts-and-pointer summaries the write paths now produce, so no row renders + * a backup INSERT statement over D1's ~100 KB limit (SQLITE_TOOBIG aborted a + * real restore partway). + * + * Real engine, no mocks: every migration before 0074 is applied to a real + * bun:sqlite database, rows are seeded at the production worst case (12,397 + * missing keys / 877 zarr failure entries) plus every leave-alone class + * (already compact, NULL, non-JSON, array-carrying but not the integrity + * shape), then 0074 is applied and the outcome asserted -- including that + * derived counts SURVIVE and that a rendered single-row INSERT stays under + * the 95,000-byte acceptance bound that makes restore-remote.sh work again. + */ + +import { Database } from "bun:sqlite"; +import { beforeEach, describe, expect, test } from "bun:test"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { + type DatasetVersionIntegrityResult, + integrityAuditSummary, +} from "../src/services/import-integrity"; + +const MIGRATIONS_DIR = join(import.meta.dir, "../src/db/migrations"); +const COMPACT_FILE = "0074_compact_oversized_payloads.sql"; + +/** + * The acceptance bound (#1189): after 0074, no audit_log or datasets row may + * render a backup statement larger than this. Comfortably under D1's ~100 KB + * statement limit, which the restore hit at 15 statements (#1188). + */ +const MAX_STATEMENT_BYTES = 95_000; + +function migrationFiles(): string[] { + return readdirSync(MIGRATIONS_DIR) + .filter((f) => f.endsWith(".sql")) + .sort(); +} + +/** Apply every migration strictly before 0074 (the known-good chain). */ +function dbBefore0074(): Database { + const db = new Database(":memory:"); + for (const f of migrationFiles().filter((f) => f < COMPACT_FILE)) { + db.exec(readFileSync(join(MIGRATIONS_DIR, f), "utf-8")); + } + return db; +} + +/** + * Statement-at-a-time application, same as + * reclaim-column-budget-migration.test.ts (see there for the verified + * failure mode): bun:sqlite's exec() on a multi-statement string swallows + * any statement error except the last one's, so exec()ing the whole file + * could let a failing first UPDATE pass silently. D1 applies statements + * individually and fails loudly; this models that. 0074 has no triggers, + * but the splitter keeps the CREATE TRIGGER handling so it stays reusable. + */ +function splitStatements(sql: string): string[] { + const statements: string[] = []; + let current: string[] = []; + let inTrigger = false; + for (const line of sql.split("\n")) { + const stripped = line.trim(); + if (stripped === "" || stripped.startsWith("--")) continue; + current.push(line); + if (/^CREATE TRIGGER/i.test(stripped)) inTrigger = true; + const ends = inTrigger ? /^END;$/i.test(stripped) : stripped.endsWith(";"); + if (ends) { + statements.push(current.join("\n")); + current = []; + inTrigger = false; + } + } + if (current.length > 0) statements.push(current.join("\n")); + return statements; +} + +function apply0074(db: Database): void { + for (const stmt of splitStatements(readFileSync(join(MIGRATIONS_DIR, COMPACT_FILE), "utf-8"))) { + db.exec(stmt); + } +} + +const byteLength = (s: string) => new TextEncoder().encode(s).byteLength; + +/** Synthetic annex keys shaped like production's (`SHA256E-s--.`). */ +function annexKeys(n: number): string[] { + const keys: string[] = []; + for (let i = 0; i < n; i++) { + keys.push(`SHA256E-s${1000000 + i}--${String(i).padStart(8, "0").repeat(8)}.edf`); + } + return keys; +} + +/** The production worst case (#1188): 12,397 missing keys, ~1.1 MB inlined. */ +function bigVerified(): DatasetVersionIntegrityResult { + const missingKeys = annexKeys(12397); + return { + complete: false, + missingKeys, + zeroByteKeys: missingKeys.slice(0, 41), + expectedCount: 12400, + presentCount: 3, + bytesPresent: 123456, + declaredBytes: 987654321, + declaredFiles: 12500, + version: "2.0.0", + }; +} + +/** Null-version variant: the migration must carry `null` through as JSON + * null, not the string "null" and not 0. */ +function smallVerified(): DatasetVersionIntegrityResult { + return { + complete: false, + missingKeys: annexKeys(500), + zeroByteKeys: [], + expectedCount: 500, + presentCount: 0, + bytesPresent: 0, + declaredBytes: 4200, + declaredFiles: 500, + version: null, + }; +} + +/** 877 entries, the largest production `zarr_data_failures` row (#1188). */ +function zarrFailureArray(n: number): string { + const out: { path: string; code: string; reason: string }[] = []; + for (let i = 0; i < n; i++) { + out.push({ + path: `sub-${String(i).padStart(4, "0")}/ses-01/eeg/sub-${String(i).padStart(4, "0")}_ses-01_task-rest_run-01_eeg.bdf`, + code: "mixed_sample_rates", + reason: "EDF/BDF channels declare differing sample rates (#737)", + }); + } + return JSON.stringify(out); +} + +// Leave-alone payloads, asserted byte-identical after the migration. +const NATIVE_SUMMARY = JSON.stringify( + integrityAuditSummary({ + complete: true, + missingKeys: [], + zeroByteKeys: [], + expectedCount: 10, + presentCount: 10, + bytesPresent: 100, + declaredBytes: 100, + declaredFiles: 10, + version: "1.0.0", + }), +); +const NON_JSON_DETAILS = "plain text log line, not JSON at all"; +const UNRELATED_ARRAY_DETAILS = JSON.stringify({ + dataset_ids: ["on000002", "on000003"], + updated: 2, +}); +// Carries ONE of the two integrity arrays: not the +// DatasetVersionIntegrityResult signature (that pair travels together), so +// the shape-scoped rewrite must not fabricate a summary with null counts +// out of some other action's payload. +const SINGLE_ARRAY_DETAILS = JSON.stringify({ missingKeys: ["SHA256E-s1--a.edf"], note: "other" }); +const NATIVE_ZARR_SUMMARY = JSON.stringify({ count: 3, detail_ref: "zarr/index.json" }); +const NON_JSON_ZARR = "corrupted, not json"; + +function seedAudit( + db: Database, + action: string, + resourceId: string | null, + details: string | null, +) { + db.query("INSERT INTO audit_log (action, resource_id, details) VALUES (?, ?, ?)").run( + action, + resourceId, + details, + ); +} + +function seed(db: Database): void { + db.exec(` + INSERT INTO users (username, email, password_hash, status, role, email_verified) + VALUES ('compactowner', 'compactowner@example.org', 'x', 'approved', 'user', 1); + `); + + seedAudit(db, "import_verify_forced", "on004952", JSON.stringify(bigVerified())); + seedAudit(db, "import_reclassified_incomplete", "on005170", JSON.stringify(smallVerified())); + seedAudit(db, "import_verify_forced", "on000001", NATIVE_SUMMARY); + seedAudit(db, "user_login", null, null); + seedAudit(db, "legacy_note", null, NON_JSON_DETAILS); + seedAudit(db, "import_dispatch_cooldown", "on000002,on000003", UNRELATED_ARRAY_DETAILS); + seedAudit(db, "some_other_action", "on000004", SINGLE_ARRAY_DETAILS); + + const owner = db.query<{ id: number }, []>("SELECT id FROM users LIMIT 1").get(); + if (!owner) throw new Error("seed: owner insert failed"); + const insertDataset = db.query( + `INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility, + zarr_data_failures, readme, enrichment_json) + VALUES (?, ?, ?, 'active', 'public', ?, ?, ?)`, + ); + // Realistic bulk in the OTHER text columns (#1188 measured readme <= 8 KB, + // enrichment_json 2-4 KB on the oversized rows) so the statement-size pin + // below bounds a whole realistic row, not a bare one. + insertDataset.run( + "on007523", + "Worst case zarr failures", + owner.id, + zarrFailureArray(877), + "R".repeat(8000), + JSON.stringify({ abstract: "e".repeat(4000) }), + ); + insertDataset.run("on007688", "Null failures", owner.id, null, null, null); + insertDataset.run("on001111", "Native summary", owner.id, NATIVE_ZARR_SUMMARY, null, null); + insertDataset.run("on002222", "Non-JSON failures", owner.id, NON_JSON_ZARR, null, null); +} + +/** + * Render a row as the backup does -- one INSERT with inline literals, + * single quotes doubled. A local model of the statement `wrangler d1 export + * --table` emits (the exporter itself lives in nemarOrg/nemar-db-backup): + * this pins OUR shapes' statement size, it does not test their exporter. + */ +function renderInsert(db: Database, table: string, where: string, param: string): string { + const cols = (db.query(`PRAGMA table_info(${table})`).all() as { name: string }[]).map( + (c) => c.name, + ); + const row = db + .query(`SELECT ${cols.map((c) => `"${c}"`).join(", ")} FROM ${table} WHERE ${where}`) + .get(param) as Record; + if (!row) throw new Error(`renderInsert: no ${table} row for ${param}`); + const literal = (v: unknown): string => { + if (v === null) return "NULL"; + if (typeof v === "number" || typeof v === "bigint") return String(v); + return `'${String(v).replaceAll("'", "''")}'`; + }; + const values = cols.map((c) => literal(row[c])); + return `INSERT INTO ${table}(${cols.join(",")}) VALUES(${values.join(",")});`; +} + +const auditDetails = (db: Database, resourceId: string): string | null => + ( + db.query("SELECT details FROM audit_log WHERE resource_id = ?").get(resourceId) as { + details: string | null; + } + ).details; + +const zarrFailures = (db: Database, datasetId: string): string | null => + ( + db.query("SELECT zarr_data_failures FROM datasets WHERE dataset_id = ?").get(datasetId) as { + zarr_data_failures: string | null; + } + ).zarr_data_failures; + +describe("migration 0074: compact oversized payloads", () => { + let db: Database; + + beforeEach(() => { + db = dbBefore0074(); + seed(db); + }); + + test("integrity-array audit payloads are rewritten to the write-path summary, counts derived", () => { + // The fixtures genuinely reach the boundary the migration exists for. + expect( + byteLength(renderInsert(db, "audit_log", "resource_id = ?", "on004952")), + ).toBeGreaterThan(MAX_STATEMENT_BYTES); + + apply0074(db); + + const big = auditDetails(db, "on004952") as string; + expect(byteLength(big)).toBeLessThan(400); + // The exact shape the write path (integrityAuditSummary) now produces, + // plus the migration marker. Counts are DERIVED from the dropped arrays + // (12,397 / 41), `complete` stays boolean false (not 0), scalars ride + // through. toEqual is strict about false-vs-0 and null-vs-0. + expect(JSON.parse(big)).toEqual({ + ...integrityAuditSummary(bigVerified()), + compacted_by: "migration_0074", + }); + expect(JSON.parse(big).missing_count).toBe(12397); + expect(JSON.parse(big).zero_byte_count).toBe(41); + expect(big).not.toContain("missingKeys"); + + // Shape-scoped, not action-scoped: the reclassification sweep wrote the + // same integrity shape under a different action. A null version must + // survive as JSON null. + const small = auditDetails(db, "on005170") as string; + expect(JSON.parse(small)).toEqual({ + ...integrityAuditSummary(smallVerified()), + compacted_by: "migration_0074", + }); + expect(JSON.parse(small).version).toBeNull(); + expect(JSON.parse(small).missing_count).toBe(500); + expect(JSON.parse(small).zero_byte_count).toBe(0); + }); + + test("already-compact, NULL, non-JSON, and non-integrity payloads are byte-identical", () => { + apply0074(db); + expect(auditDetails(db, "on000001")).toBe(NATIVE_SUMMARY); + expect( + ( + db.query("SELECT details FROM audit_log WHERE action = 'user_login'").get() as { + details: string | null; + } + ).details, + ).toBeNull(); + expect( + ( + db.query("SELECT details FROM audit_log WHERE action = 'legacy_note'").get() as { + details: string | null; + } + ).details, + ).toBe(NON_JSON_DETAILS); + expect(auditDetails(db, "on000002,on000003")).toBe(UNRELATED_ARRAY_DETAILS); + expect(auditDetails(db, "on000004")).toBe(SINGLE_ARRAY_DETAILS); + }); + + test("zarr_data_failures arrays become count + pointer; leave-alone rows untouched", () => { + expect(byteLength(renderInsert(db, "datasets", "dataset_id = ?", "on007523"))).toBeGreaterThan( + MAX_STATEMENT_BYTES, + ); + + apply0074(db); + + const compacted = zarrFailures(db, "on007523") as string; + expect(byteLength(compacted)).toBeLessThan(120); + expect(JSON.parse(compacted)).toEqual({ + count: 877, + detail_ref: "zarr/index.json", + compacted_by: "migration_0074", + }); + + expect(zarrFailures(db, "on007688")).toBeNull(); + expect(zarrFailures(db, "on001111")).toBe(NATIVE_ZARR_SUMMARY); + expect(zarrFailures(db, "on002222")).toBe(NON_JSON_ZARR); + }); + + test("a compacted record is distinguishable from a natively-written one", () => { + apply0074(db); + expect(JSON.parse(auditDetails(db, "on004952") as string).compacted_by).toBe("migration_0074"); + expect(JSON.parse(auditDetails(db, "on000001") as string).compacted_by).toBeUndefined(); + expect(JSON.parse(zarrFailures(db, "on007523") as string).compacted_by).toBe("migration_0074"); + expect(JSON.parse(zarrFailures(db, "on001111") as string).compacted_by).toBeUndefined(); + }); + + test("re-running 0074 is a no-op (rewritten rows no longer match)", () => { + apply0074(db); + const first = auditDetails(db, "on004952"); + const firstZarr = zarrFailures(db, "on007523"); + apply0074(db); + expect(auditDetails(db, "on004952")).toBe(first as string); + expect(zarrFailures(db, "on007523")).toBe(firstZarr as string); + }); + + test("acceptance pin (#1188): no rendered backup statement exceeds 95,000 bytes", () => { + apply0074(db); + const auditIds = (db.query("SELECT id FROM audit_log").all() as { id: number }[]).map((r) => + String(r.id), + ); + for (const id of auditIds) { + expect(byteLength(renderInsert(db, "audit_log", "id = ?", id))).toBeLessThan( + MAX_STATEMENT_BYTES, + ); + } + const datasetIds = ( + db.query("SELECT dataset_id FROM datasets").all() as { dataset_id: string }[] + ).map((r) => r.dataset_id); + for (const id of datasetIds) { + expect(byteLength(renderInsert(db, "datasets", "dataset_id = ?", id))).toBeLessThan( + MAX_STATEMENT_BYTES, + ); + } + }); +}); From 05181f5eddde957c268936dd1a3fb79dcced5d26 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:13:57 -0700 Subject: [PATCH 5/7] test: cover bounded audit payload writes --- backend/test/bounded-audit-payloads.test.ts | 278 ++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 backend/test/bounded-audit-payloads.test.ts diff --git a/backend/test/bounded-audit-payloads.test.ts b/backend/test/bounded-audit-payloads.test.ts new file mode 100644 index 000000000..6b1b4d99a --- /dev/null +++ b/backend/test/bounded-audit-payloads.test.ts @@ -0,0 +1,278 @@ +/** + * Write-path bounds for audit and catalog payloads (#1189, fixes #1188). + * + * The D1 backup renders one INSERT per row and D1 refuses a statement over + * ~100 KB on restore, so any row column that inlines a per-file list is a + * row that can be written but never restored. These tests pin the three + * write-path defenses, against the real schema (freshDb applies every + * migration) with production SQL -- no mocks: + * + * 1. auditLogStatement bounds EVERY `details` payload (the shared + * marshaling in db/audit-log.ts) -- oversized ones become a small + * truncation marker. + * 2. importVerifyAuditStatement writes the counts-and-pointer summary, + * never the integrity result's key arrays. Driven as a function rather + * than through POST /admin/imports/:id/verify because the route cannot + * be exercised past verifyDatasetVersionS3 without live AWS credentials + * (its S3 listing hardcodes a `*.s3.*.amazonaws.com` host; see + * imports-verify-route.test.ts, which documents the same constraint) -- + * the route -> function call link is the uncovered residue, the same + * seam accepted for stampDatasetIntegrity. + * 3. The zarr-ready callback stores a count + pointer in + * `zarr_data_failures`, driven through the REAL Hono handler. + */ + +import type { Database } from "bun:sqlite"; +import { beforeEach, describe, expect, test } from "bun:test"; +import { Hono } from "hono"; +import { AUDIT_DETAILS_MAX_BYTES, auditLogStatement } from "../src/db/audit-log"; +import { importVerifyAuditStatement } from "../src/routes/admin/imports"; +import { registerZarrReadyRoutes } from "../src/routes/callbacks/zarr-ready"; +import type { DatasetVersionIntegrityResult } from "../src/services/import-integrity"; +import type { Bindings } from "../src/types/bindings"; +import { freshDb, realD1 } from "./helpers/d1"; + +/** + * Real constant-time comparison for the Workers-only + * `crypto.subtle.timingSafeEqual` the zarr-ready handler's token check needs + * (bun's runtime lacks it). Same non-mock platform shim as + * zarr-pool-breaks.test.ts, guarded so whichever file loads first in the + * shared bun test process installs it once; the wrong-token 401 there proves + * the check stays live. + */ +const subtle = crypto.subtle as SubtleCrypto & { + timingSafeEqual?: (a: ArrayBufferView, b: ArrayBufferView) => boolean; +}; +if (typeof subtle.timingSafeEqual !== "function") { + subtle.timingSafeEqual = (a: ArrayBufferView, b: ArrayBufferView): boolean => { + const x = new Uint8Array(a.buffer, a.byteOffset, a.byteLength); + const y = new Uint8Array(b.buffer, b.byteOffset, b.byteLength); + if (x.length !== y.length) return false; + let diff = 0; + for (let i = 0; i < x.length; i++) diff |= (x[i] as number) ^ (y[i] as number); + return diff === 0; + }; +} + +const byteLength = (s: string) => new TextEncoder().encode(s).byteLength; + +/** Synthetic annex keys shaped like production's (`SHA256E-s--.`). */ +function annexKeys(n: number): string[] { + const keys: string[] = []; + for (let i = 0; i < n; i++) { + keys.push(`SHA256E-s${1000000 + i}--${String(i).padStart(8, "0").repeat(8)}.edf`); + } + return keys; +} + +/** The production worst case (#1188): 12,397 missing keys, ~1.1 MB inlined. */ +function worstCaseVerified(): DatasetVersionIntegrityResult { + const missingKeys = annexKeys(12397); + return { + complete: false, + missingKeys, + zeroByteKeys: missingKeys.slice(0, 41), + expectedCount: 12400, + presentCount: 3, + bytesPresent: 123456, + declaredBytes: 987654321, + declaredFiles: 12500, + version: "2.0.0", + }; +} + +let db: Database; + +beforeEach(() => { + db = freshDb(); +}); + +function auditRow(): { user_id: number | null; action: string; details: string | null } { + const row = db + .query("SELECT user_id, action, resource_id, details FROM audit_log ORDER BY id DESC LIMIT 1") + .get() as { user_id: number | null; action: string; details: string | null } | null; + if (!row) throw new Error("no audit_log row inserted"); + return row; +} + +describe("auditLogStatement details bound (#1189)", () => { + test("an over-limit payload is stored as a small marker, not inlined", async () => { + const oversized = "x".repeat(AUDIT_DETAILS_MAX_BYTES + 1); // exactly 1 byte over + await auditLogStatement(realD1(db), { + userId: null, + action: "guard_probe", + details: oversized, + }).run(); + + const stored = auditRow().details as string; + expect(byteLength(stored)).toBeLessThan(AUDIT_DETAILS_MAX_BYTES); + const marker = JSON.parse(stored); + expect(marker.audit_details_truncated).toBe(true); + expect(marker.original_bytes).toBe(AUDIT_DETAILS_MAX_BYTES + 1); + expect(marker.head).toBe("x".repeat(2000)); + }); + + test("a payload exactly at the limit passes through unchanged", async () => { + const atLimit = "y".repeat(AUDIT_DETAILS_MAX_BYTES); + await auditLogStatement(realD1(db), { + userId: null, + action: "guard_probe", + details: atLimit, + }).run(); + expect(auditRow().details).toBe(atLimit); + }); + + test("the bound is measured in UTF-8 bytes, not JS characters", async () => { + // 2 bytes per char: under the limit in characters, over it in bytes. A + // `.length` (UTF-16 chars) implementation would pass this through and + // the stored row could still exceed the restore budget. + const multibyte = "é".repeat(AUDIT_DETAILS_MAX_BYTES / 2 + 1); + await auditLogStatement(realD1(db), { + userId: null, + action: "guard_probe", + details: multibyte, + }).run(); + const marker = JSON.parse(auditRow().details as string); + expect(marker.audit_details_truncated).toBe(true); + expect(marker.original_bytes).toBe(AUDIT_DETAILS_MAX_BYTES + 2); + }); + + test("null details stays null", async () => { + await auditLogStatement(realD1(db), { userId: null, action: "guard_probe" }).run(); + expect(auditRow().details).toBeNull(); + }); +}); + +describe("import verify audit write (#1189)", () => { + test("stores counts and the availability-report pointer, never the key arrays", async () => { + const verified = worstCaseVerified(); + // The fixture genuinely reaches the boundary the bound exists for. + expect(byteLength(JSON.stringify(verified))).toBeGreaterThan(1_000_000); + + await importVerifyAuditStatement(realD1(db), 7, "on004952", verified).run(); + + const row = db + .query("SELECT user_id, action, resource_id, details FROM audit_log ORDER BY id DESC LIMIT 1") + .get() as { user_id: number; action: string; resource_id: string; details: string }; + expect(row.user_id).toBe(7); + expect(row.action).toBe("import_verify_forced"); + expect(row.resource_id).toBe("on004952"); + + expect(byteLength(row.details)).toBeLessThan(1_000); + // Independent literal expectation (not integrityAuditSummary(), which + // would be circular): counts derived from the arrays, pointer to the + // artifact that owns the per-path detail, scalars copied through. + expect(JSON.parse(row.details)).toEqual({ + complete: false, + expectedCount: 12400, + presentCount: 3, + bytesPresent: 123456, + declaredBytes: 987654321, + declaredFiles: 12500, + version: "2.0.0", + missing_count: 12397, + zero_byte_count: 41, + detail_ref: ".nemar/availability-report.json", + }); + expect(row.details).not.toContain("missingKeys"); + expect(row.details).not.toContain("SHA256E-"); + }); +}); + +describe("zarr-ready callback zarr_data_failures (#1189, real handler)", () => { + const TOKEN = "bounded-payloads-webhook-token"; + const DATASET = "on007523"; + let app: Hono<{ Bindings: Bindings }>; + + function post(body: Record): Promise { + return app.request( + "/zarr-ready", + { + method: "POST", + headers: { "Content-Type": "application/json", "X-Webhook-Token": TOKEN }, + body: JSON.stringify(body), + }, + { DB: realD1(db), NEMAR_WEBHOOK_TOKEN: TOKEN, ENVIRONMENT: "test" } as Bindings, + ); + } + + const row = () => + db.query("SELECT * FROM datasets WHERE dataset_id = ?").get(DATASET) as Record; + + beforeEach(() => { + app = new Hono<{ Bindings: Bindings }>(); + registerZarrReadyRoutes(app); + db.query( + `INSERT INTO users (username, email, password_hash, status, role, email_verified) + VALUES ('bapowner', 'bapowner@example.org', 'x', 'approved', 'user', 1)`, + ).run(); + const owner = db + .query<{ id: number }, []>("SELECT id FROM users WHERE username='bapowner'") + .get(); + if (!owner) throw new Error("seed: owner insert failed"); + db.query( + `INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility) + VALUES (?, 'Bounded payloads fixture', ?, 'active', 'public')`, + ).run(DATASET, owner.id); + }); + + /** The production worst case (#1188): 877 failure entries, 178 KB in one row. */ + function failures(n: number): { path: string; code: string; reason: string }[] { + const out: { path: string; code: string; reason: string }[] = []; + for (let i = 0; i < n; i++) { + out.push({ + path: `sub-${String(i).padStart(4, "0")}/ses-01/eeg/sub-${String(i).padStart(4, "0")}_ses-01_task-rest_run-01_eeg.bdf`, + code: "mixed_sample_rates", + reason: "EDF/BDF channels declare differing sample rates (#737)", + }); + } + return out; + } + + test("a failed run stores a count and pointer, independent of entry count", async () => { + const df = failures(877); + expect(byteLength(JSON.stringify(df))).toBeGreaterThan(95_000); // fixture reaches the boundary + const res = await post({ + dataset_id: DATASET, + status: "failed", + errors: 900, + failure_count: 877, + deterministic: true, + data_failures: df, + }); + expect(res.status).toBe(200); + + const stored = row().zarr_data_failures as string; + expect(byteLength(stored)).toBeLessThan(100); + expect(JSON.parse(stored)).toEqual({ count: 877, detail_ref: "zarr/index.json" }); + expect(stored).not.toContain("sub-0000"); + expect(row().zarr_failure_count).toBe(877); + }); + + test("a partial ready run stores the summary and stamps zarr_failed_at", async () => { + const res = await post({ + dataset_id: DATASET, + status: "ready", + store_count: 5, + errors: 2, + failure_count: 2, + data_failures: failures(2), + }); + expect(res.status).toBe(200); + expect(JSON.parse(row().zarr_data_failures as string)).toEqual({ + count: 2, + detail_ref: "zarr/index.json", + }); + expect(row().zarr_failed_at).not.toBeNull(); + }); + + test("a clean ready run still clears the failure detail", async () => { + await post({ dataset_id: DATASET, status: "failed", errors: 3, data_failures: failures(3) }); + expect(row().zarr_data_failures).not.toBeNull(); + + const res = await post({ dataset_id: DATASET, status: "ready", store_count: 9, errors: 0 }); + expect(res.status).toBe(200); + expect(row().zarr_data_failures).toBeNull(); + expect(row().zarr_failed_at).toBeNull(); + }); +}); From f8fab9b71e76fa2ba4e0cf5b1b3a34299d98a9c0 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:15:16 -0700 Subject: [PATCH 6/7] docs: ADR 0036 and restore statement note --- ...-counts-and-pointers-not-per-file-lists.md | 89 +++++++++++++++++++ .context/decisions/README.md | 1 + .context/systems-inventory.md | 9 ++ 3 files changed, 99 insertions(+) create mode 100644 .context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md diff --git a/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md b/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md new file mode 100644 index 000000000..4b4d4b6bb --- /dev/null +++ b/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md @@ -0,0 +1,89 @@ +# ADR 0036: Operational rows carry counts and pointers, not per-file lists + +**Status:** accepted +**Date:** 2026-09-02 +**Owner:** Seyed Yahya Shirazi + +## Context + +The hourly production D1 backup could not be restored (#1188): +`restore-remote.sh` aborted with `SQLITE_TOOBIG` on 15 single-row INSERT +statements over D1's ~100 KB statement limit. +Every offender was a row column that inlined an unbounded per-file list, +so row size scaled with a dataset's file count: +`audit_log.details` on forced import verifications inlined the full +`missingKeys`/`zeroByteKeys` arrays (largest: 12,397 annex keys, 1.15 MB, +~88% of the entire audit log), +and `datasets.zarr_data_failures` inlined the per-recording conversion +failure array (largest: 877 entries, 178 KB; +~29% of all `datasets` data). +In both cases the inlined detail duplicated a richer artifact that already +existed outside D1: +`.nemar/availability-report.json` on the dataset repo +(path-keyed with a per-entry reason, versioned in git) +and the published Zarr index's `failures` list +(`/zarr/index.json` in the serving bucket). + +## Decision + +A row in an operational table (audit log, catalog bookkeeping) carries +**flags, counts, and a pointer** — +booleans and totals that answer "how many" and "did it complete", +plus a `detail_ref` naming the artifact that owns the per-file detail. +The detail itself is never inlined, and never merely truncated: +a truncated list is still the wrong data in the wrong place, +and silently discards everything past the cut. + +Concretely (#1189): +integrity audit writes store `integrityAuditSummary()` +(services/import-integrity.ts) instead of the raw result; +`zarr_data_failures` stores `{count, detail_ref}`; +migration 0074 rewrote the pre-existing rows into the same summaries, +deriving the counts from the arrays it dropped; +and `auditLogParams` (db/audit-log.ts) bounds every `details` payload at +`AUDIT_DETAILS_MAX_BYTES`, +replacing an oversized one with a truncation marker, +so the next unbounded payload shape cannot recreate the failure. + +## Consequences + +- Row size — and therefore backup-statement size — is independent of how + many files a dataset has, which is exactly the condition that makes a + restore work (a restore is only as good as its largest statement). +- Answering "which files" requires following the pointer to the owning + artifact rather than querying D1. + That artifact is richer than the dropped copy was, + but it is a second hop, and for the availability report it requires repo + access. +- Compacted historical rows carry `compacted_by: "migration_0074"`; + their per-key lists are gone from D1. + The counts survive, and the reports/indexes hold the current detail — + though not the historical per-run snapshots the audit rows once froze. +- Any new column or audit payload that wants to store a per-item list must + instead store counts plus a pointer, + or argue its way past the write-time bound in an ADR superseding this one. +- Hand-rolled `INSERT INTO audit_log` call sites predating #903 bypass the + bound until they converge on `auditLogStatement`; + the two import services were converged in #1189, + the rest write small fixed-shape payloads today. + +## Alternatives considered + +- **Truncate the arrays to N entries:** keeps wrong data in the wrong + place, silently discards entries N+1 onward, and leaves row size coupled + to a cap instead of to the row's actual job. Rejected by design review. +- **Fix it in the backup format** (bound statements, chunked UPDATEs, + parameterized loads): treats the symptom; every D1 reader and the + dashboard still pays for megabyte rows, and the duplicate detail still + drifts from the owning artifact. Kept only as a defense-in-depth idea for + the backup repo. +- **A separate detail table:** normalizes the duplicate instead of removing + it; the artifact outside D1 remains the source of truth, so D1 would hold + a second copy either way. + +## Receipts + +- #1188 — the restore failure and its statement-size measurements. +- #1189 — the fix: summaries, write-time bound, migration 0074. +- ADR 0004 — the backup this exists to keep restorable. +- ADR 0034 — the same instinct applied to column count rather than row size. diff --git a/.context/decisions/README.md b/.context/decisions/README.md index ef24f3aee..9126bca2a 100644 --- a/.context/decisions/README.md +++ b/.context/decisions/README.md @@ -69,6 +69,7 @@ invisible one — which is why it is checked rather than trusted. - [ADR 0033](0033-the-zarr-queue-stamps-the-engine-that-converted-each-dataset.md) - The Zarr queue stamps the engine that converted each dataset; pre-existing rows are declared current - [ADR 0034](0034-datasets-stays-one-table-under-a-column-budget.md) - `datasets` stays one table, under an enforced column budget - [ADR 0035](0035-sweep-stamps-live-in-one-json-column.md) - Sweep bookkeeping stamps live in one JSON column +- [ADR 0036](0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md) - Operational rows carry counts and pointers, not per-file lists ## Backfill note (2026-07-31) diff --git a/.context/systems-inventory.md b/.context/systems-inventory.md index e92069a88..26e3425bb 100644 --- a/.context/systems-inventory.md +++ b/.context/systems-inventory.md @@ -236,3 +236,12 @@ Both recovery tools live in the backup repo, not here: Verifies sha256 and row counts, and refuses production without `--force-prod`. - `scripts/run-local.sh --nemar-cli ` — loads a real snapshot into a local miniflare D1 and runs `wrangler dev` against it. + +**A restore is only as good as its largest statement.** The backup renders one +INSERT per row and D1 refuses any statement over ~100 KB on restore +(`SQLITE_TOOBIG`), so a single oversized row makes the whole backup +unrestorable — and nothing at backup time detects it (#1188: 15 such +statements, found only by rehearsing a real restore). Row payloads are +therefore bounded at the write path (ADR 0036: counts and pointers, never +inline per-file lists; `AUDIT_DETAILS_MAX_BYTES` in `backend/src/db/audit-log.ts`), +and migration 0074 compacted the rows that had already outgrown the limit. From 079c305ed6f0a1229d8890da5586d5e57441a723 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 03:42:18 -0700 Subject: [PATCH 7/7] fix: bound draft-delete audit payload --- ...-counts-and-pointers-not-per-file-lists.md | 30 +++++++++++++++-- backend/src/routes/datasets/draft-delete.ts | 30 ++++++++--------- ...mpact-oversized-payloads-migration.test.ts | 33 +++++++++++++++++++ 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md b/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md index 4b4d4b6bb..80dcbaf18 100644 --- a/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md +++ b/.context/decisions/0036-operational-rows-carry-counts-and-pointers-not-per-file-lists.md @@ -63,9 +63,33 @@ so the next unbounded payload shape cannot recreate the failure. instead store counts plus a pointer, or argue its way past the write-time bound in an ADR superseding this one. - Hand-rolled `INSERT INTO audit_log` call sites predating #903 bypass the - bound until they converge on `auditLogStatement`; - the two import services were converged in #1189, - the rest write small fixed-shape payloads today. + bound until they converge on `auditLogStatement`. + Three were converged in #1189: the two import services, + and the owner draft-delete route (`routes/datasets/draft-delete.ts`), + which embedded `steps.s3.failed` from `deleteDatasetCascade` -- + and `deleteObjects` pushes EVERY key of a failed batch into that array, + so a single S3 outage during an owner delete could reproduce + exactly the unbounded shape this ADR exists to prevent. + It was found in review, after an earlier draft of this ADR asserted + the remaining sites were all small and fixed-shape; that claim was wrong, + which is itself the argument for the write-time bound rather than + a per-call-site audit. + The sites still hand-rolled today (`index.ts` scheduled_cleanup, capped at + `MAX_DELETIONS_PER_RUN`; `auth.ts`; `publication.ts`; `collaborators.ts`; + `upload.ts`; `dataset-search.ts`; `auto-import.ts`) + were each checked and write small fixed-shape payloads, + but that is a property of today's code, not a guarantee -- + converging them is the durable fix. + +- `zarr_data_failures` changes shape on the wire, from an array to + `{count, detail_ref}`. It is not declared in `shared/contract/dataset.ts`, + and nothing in this repo reads it, but the detail route serves it + incidentally through `SELECT d.*` (`routes/datasets/catalog.ts`), + so it IS wire-visible. The observability dashboard in `nemarOrg/website` + is the known consumer and lives outside this repo; + a reader doing `.length` or `.map()` on it will break. + Tracked as a heads-up against that repo rather than blocked here, + because the count it now carries is the number that dashboard actually shows. ## Alternatives considered diff --git a/backend/src/routes/datasets/draft-delete.ts b/backend/src/routes/datasets/draft-delete.ts index 069d4ce35..bcf46a751 100644 --- a/backend/src/routes/datasets/draft-delete.ts +++ b/backend/src/routes/datasets/draft-delete.ts @@ -6,6 +6,7 @@ * intentional changes are import paths and the register-function wrapper. */ +import { auditLogStatement } from "../../db/audit-log"; import { authMiddleware } from "../../middleware/auth"; import { deleteDatasetCascade } from "../../services/deletion"; import { hasRole } from "../../types/bindings"; @@ -109,21 +110,20 @@ export function registerDraftDeleteRoutes(datasetRoutes: DatasetsRouter): void { // Audit log (best-effort). try { - await db - .prepare("INSERT INTO audit_log (action, user_id, details) VALUES (?, ?, ?)") - .bind( - "dataset_deleted", - currentUser.id, - JSON.stringify({ - dataset_id: datasetId, - dataset_name: dataset.name, - owner_user_id: dataset.owner_user_id, - via: "owner_draft_delete", - steps: result.steps, - warnings: result.warnings, - }), - ) - .run(); + await auditLogStatement(db, { + userId: currentUser.id, + action: "dataset_deleted", + resourceType: "dataset", + resourceId: datasetId, + details: JSON.stringify({ + dataset_id: datasetId, + dataset_name: dataset.name, + owner_user_id: dataset.owner_user_id, + via: "owner_draft_delete", + steps: result.steps, + warnings: result.warnings, + }), + }).run(); } catch (err) { console.error("Failed to write owner-delete audit log:", err); result.warnings.push("Audit log write failed"); diff --git a/backend/test/compact-oversized-payloads-migration.test.ts b/backend/test/compact-oversized-payloads-migration.test.ts index 6df89d526..bb90489b9 100644 --- a/backend/test/compact-oversized-payloads-migration.test.ts +++ b/backend/test/compact-oversized-payloads-migration.test.ts @@ -154,7 +154,24 @@ const NATIVE_SUMMARY = JSON.stringify( version: "1.0.0", }), ); +// Production carries 5 audit rows whose `details` is not valid JSON at all. +// The migration guards these with CASE rather than an AND chain, because +// json_type() RAISES on malformed JSON and SQLite guarantees short-circuit +// evaluation for CASE but not for AND operands. +// +// LIMIT OF THIS FIXTURE, stated rather than implied (#1190 review): replacing +// the migration's CASE guards with a plain AND chain leaves these tests GREEN, +// because SQLite's planner happens to evaluate this particular non-indexed +// single-table predicate left to right. So these cases prove the shipped +// migration leaves malformed rows alone; they do NOT prove the CASE guard is +// what makes that true, and they would not catch a refactor that dropped it. +// The guard is correct and load-bearing on the documented semantics; treat its +// removal as a real change even if CI stays green. const NON_JSON_DETAILS = "plain text log line, not JSON at all"; +// Malformed in three different ways, so the leave-alone path is exercised +// against more than one failure shape. +const TRUNCATED_JSON_DETAILS = '{"complete":false,"missingKeys":["a","b"'; +const TRAILING_COMMA_JSON_DETAILS = '{"complete":false,"missingKeys":["a",]}'; const UNRELATED_ARRAY_DETAILS = JSON.stringify({ dataset_ids: ["on000002", "on000003"], updated: 2, @@ -191,6 +208,8 @@ function seed(db: Database): void { seedAudit(db, "import_verify_forced", "on000001", NATIVE_SUMMARY); seedAudit(db, "user_login", null, null); seedAudit(db, "legacy_note", null, NON_JSON_DETAILS); + seedAudit(db, "legacy_truncated", null, TRUNCATED_JSON_DETAILS); + seedAudit(db, "legacy_trailing_comma", null, TRAILING_COMMA_JSON_DETAILS); seedAudit(db, "import_dispatch_cooldown", "on000002,on000003", UNRELATED_ARRAY_DETAILS); seedAudit(db, "some_other_action", "on000004", SINGLE_ARRAY_DETAILS); @@ -314,6 +333,20 @@ describe("migration 0074: compact oversized payloads", () => { } ).details, ).toBe(NON_JSON_DETAILS); + // Two further malformed shapes: the migration must leave each byte-for-byte + // rather than nulling or partially rewriting it. + for (const [action, original] of [ + ["legacy_truncated", TRUNCATED_JSON_DETAILS], + ["legacy_trailing_comma", TRAILING_COMMA_JSON_DETAILS], + ] as const) { + expect( + ( + db.query("SELECT details FROM audit_log WHERE action = ?").get(action) as { + details: string | null; + } + ).details, + ).toBe(original); + } expect(auditDetails(db, "on000002,on000003")).toBe(UNRELATED_ARRAY_DETAILS); expect(auditDetails(db, "on000004")).toBe(SINGLE_ARRAY_DETAILS); });