From 91290bbc11343cc65b0b7e0672c4935c532f97f5 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Sat, 22 Aug 2026 23:41:45 -0700 Subject: [PATCH] Surface dataset withdrawal in the CLI --- backend/src/routes/admin/imports.ts | 18 ++- backend/test/imports-withdrawn-count.test.ts | 130 +++++++++++++++++++ src/commands/admin.ts | 8 ++ src/commands/dataset.ts | 39 +++++- src/lib/api/admin.ts | 7 + src/lib/api/datasets.ts | 12 ++ 6 files changed, 209 insertions(+), 5 deletions(-) create mode 100644 backend/test/imports-withdrawn-count.test.ts diff --git a/backend/src/routes/admin/imports.ts b/backend/src/routes/admin/imports.ts index 813b1e417..7262d1791 100644 --- a/backend/src/routes/admin/imports.ts +++ b/backend/src/routes/admin/imports.ts @@ -319,7 +319,23 @@ export function registerImportRoutes(admin: AdminRouter): void { const by_status: Record = {}; for (const s of IMPORT_STATUSES) by_status[s] = 0; for (const row of counts.results ?? []) by_status[row.status] = row.n; - return c.json({ imports: results, total: results.length, by_status }); + // Withdrawal is orthogonal to import status, not another value of it: a + // withdrawn dataset imported fine and stays `complete` here. Reported as its + // own number so the summary stops implying that every `complete` import is + // a dataset you can still fetch (#1048). + const withdrawn = await db + .prepare( + `SELECT COUNT(*) AS n FROM import_jobs j + JOIN datasets d ON d.dataset_id = j.dataset_id + WHERE d.withdrawn_at IS NOT NULL`, + ) + .first<{ n: number }>(); + return c.json({ + imports: results, + total: results.length, + by_status, + withdrawn: withdrawn?.n ?? 0, + }); }); /** diff --git a/backend/test/imports-withdrawn-count.test.ts b/backend/test/imports-withdrawn-count.test.ts new file mode 100644 index 000000000..1d1a6483e --- /dev/null +++ b/backend/test/imports-withdrawn-count.test.ts @@ -0,0 +1,130 @@ +/** + * Real GET /admin/imports tests for the withdrawn count (#1048). + * + * A withdrawn dataset's import succeeded, so its `import_jobs` row stays + * `complete` forever. Counting it only there made `complete N` read as "N + * datasets you can fetch", which is how a withdrawal cohort got mistaken for a + * data-integrity incident in the first place. `withdrawn` is reported as its + * own number, alongside rather than inside `by_status`. + * + * Runs against a real D1 (bun:sqlite + the actual migrations), no mocks. + */ + +import type { Database } from "bun:sqlite"; +import { beforeEach, describe, expect, test } from "bun:test"; +import { Hono } from "hono"; +import { adminRoutes } from "../src/routes/admin"; +import { hashApiKey } from "../src/services/token"; +import type { Bindings, Variables } from "../src/types/bindings"; +import { freshDb, realD1 } from "./helpers/d1"; + +const ADMIN_KEY = "withdrawn-admin-key-0123456789abcdef0123456789ab"; + +let db: Database; +let app: Hono<{ Bindings: Bindings; Variables: Variables }>; +let ownerId: number; + +async function seedAdmin(): Promise { + db.run( + `INSERT INTO users (username, email, password_hash, status, role, email_verified) + VALUES ('wdadmin', 'wdadmin@example.org', 'x', 'approved', 'admin', 1)`, + ); + const u = db.query<{ id: number }, []>("SELECT id FROM users WHERE username='wdadmin'").get(); + if (!u) throw new Error("seed: admin insert failed"); + ownerId = u.id; + db.query("INSERT INTO tokens (user_id, api_key_hash, api_key_prefix) VALUES (?, ?, ?)").run( + u.id, + await hashApiKey(ADMIN_KEY), + ADMIN_KEY.slice(0, 8), + ); +} + +/** An imported dataset, optionally withdrawn. `status` is the import status. */ +function seedImportedDataset( + datasetId: string, + status: string, + withdrawnAt: string | null = null, +): void { + db.query( + `INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility, withdrawn_at, withdrawn_reason) + VALUES (?, ?, ?, 'active', 'public', ?, ?)`, + ).run( + datasetId, + `Dataset ${datasetId}`, + ownerId, + withdrawnAt, + withdrawnAt ? "source returns 403 for every client" : null, + ); + db.query( + `INSERT INTO import_jobs (dataset_id, source, source_id, stage, status) + VALUES (?, 'openneuro', ?, 'finalize', ?)`, + ).run(datasetId, datasetId.replace(/^on/, "ds"), status); +} + +function get(path: string): Promise { + return app.request(path, { headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, { + DB: realD1(db), + ENVIRONMENT: "development", + } as Bindings); +} + +beforeEach(async () => { + db = freshDb(); + app = new Hono<{ Bindings: Bindings; Variables: Variables }>(); + app.route("/admin", adminRoutes); + await seedAdmin(); +}); + +describe("GET /admin/imports withdrawn count", () => { + test("counts only datasets with withdrawn_at set", async () => { + seedImportedDataset("on000001", "complete"); + seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z"); + seedImportedDataset("on000003", "complete", "2026-07-02T00:00:00Z"); + + const body = await (await get("/admin/imports")).json(); + expect(body.withdrawn).toBe(2); + }); + + test("withdrawn rows stay inside by_status.complete rather than moving out of it", async () => { + seedImportedDataset("on000001", "complete"); + seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z"); + + const body = await (await get("/admin/imports")).json(); + // The import genuinely completed; withdrawal is a separate axis. If this + // ever drops to 1, the count moved instead of being reported alongside. + expect(body.by_status.complete).toBe(2); + expect(body.withdrawn).toBe(1); + }); + + test("is zero when nothing is withdrawn", async () => { + seedImportedDataset("on000001", "complete"); + seedImportedDataset("on000002", "failed"); + + const body = await (await get("/admin/imports")).json(); + expect(body.withdrawn).toBe(0); + }); + + test("stays fleet-wide when a status filter is applied", async () => { + // by_status is deliberately unfiltered so the summary line is not + // misleading under a filter; `withdrawn` sits on that same line. + seedImportedDataset("on000001", "failed"); + seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z"); + + const body = await (await get("/admin/imports?status=failed")).json(); + expect(body.imports.length).toBe(1); + expect(body.withdrawn).toBe(1); + }); + + test("a withdrawn dataset with no import job is not counted", async () => { + // The count is about imported datasets; a natively-uploaded one that was + // withdrawn has no import_jobs row and does not belong in this total. + db.query( + `INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility, withdrawn_at) + VALUES ('nm000001', 'Native', ?, 'active', 'public', '2026-07-01T00:00:00Z')`, + ).run(ownerId); + seedImportedDataset("on000001", "complete"); + + const body = await (await get("/admin/imports")).json(); + expect(body.withdrawn).toBe(0); + }); +}); diff --git a/src/commands/admin.ts b/src/commands/admin.ts index ac97c0804..b8395fd60 100644 --- a/src/commands/admin.ts +++ b/src/commands/admin.ts @@ -3450,6 +3450,14 @@ importCommand console.log( ` ${chalk.yellow(`in-flight ${(counts.preparing ?? 0) + (counts.copying ?? 0) + (counts.finalizing ?? 0)}`)} ${chalk.green(`complete ${counts.complete ?? 0}`)} ${chalk.magenta(`incomplete ${counts.incomplete ?? 0}`)} ${chalk.red(`failed ${counts.failed ?? 0}`)} ${chalk.red(`quarantined ${counts.quarantined ?? 0}`)} ${chalk.dim(`rolled_back ${counts.rolled_back ?? 0}`)}\n`, ); + // Withdrawn datasets are counted inside `complete` -- their import did + // succeed -- so say so rather than letting the total read as datasets + // that are still fetchable (#1048). + if (result.withdrawn) { + console.log( + ` ${chalk.red(`withdrawn ${result.withdrawn}`)} ${chalk.dim("(included in complete; content removed and DOI tombstoned)")}\n`, + ); + } if (rows.length === 0) { console.log(chalk.dim(" No import jobs match.")); diff --git a/src/commands/dataset.ts b/src/commands/dataset.ts index 1bf5defc4..5e8aa72a4 100644 --- a/src/commands/dataset.ts +++ b/src/commands/dataset.ts @@ -1422,6 +1422,17 @@ Examples: console.log(` Name: ${datasetInfo.name}`); console.log(` Owner: ${datasetInfo.owner_username}`); console.log(` Status: ${colorizeStatus(datasetInfo.status)}`); + // Withdrawal is a separate axis from `status`, which stays "active", so it + // has to be printed or the dataset reads as live. Directly under Status, + // because that is the line a reader takes their answer from (#1048). + if (datasetInfo.withdrawn_at) { + const reason = datasetInfo.withdrawn_reason + ? ` ${chalk.dim(`- ${datasetInfo.withdrawn_reason}`)}` + : ""; + console.log( + ` Withdrawn: ${chalk.red(new Date(datasetInfo.withdrawn_at).toLocaleDateString())}${reason}`, + ); + } console.log(` Created: ${new Date(datasetInfo.created_at).toLocaleDateString()}`); if (datasetInfo.description) { @@ -1433,20 +1444,40 @@ Examples: } if (datasetInfo.concept_doi) { - console.log(` DOI: https://doi.org/${datasetInfo.concept_doi}`); + // A withdrawn dataset's DOI resolves to a registrar tombstone, not the + // dataset. Printing it as a bare link invites the reader to follow it and + // conclude the dataset is fine. + const doiUrl = `https://doi.org/${datasetInfo.concept_doi}`; + console.log( + datasetInfo.withdrawn_at + ? ` DOI: ${chalk.dim(doiUrl)} ${chalk.red("(tombstoned)")}` + : ` DOI: ${doiUrl}`, + ); } // #970: surface-but-visible -- only shout when data is verified incomplete; // complete/not-yet-audited stay silent (mirrors the list table's Data column). if (datasetInfo.data_complete === 0) { + // A withdrawn dataset's content is 0-byte BY DESIGN, so the integrity + // sweep has nothing to find. Pointing an operator at it here is the exact + // false lead #1048 opened with: a withdrawal cohort read as a data loss + // incident because every surface described it as broken rather than gone. console.log( - ` Data: ${chalk.red("incomplete")} ${chalk.dim("(some files failed to upload; see `nemar admin data-integrity-sweep`)")}`, + datasetInfo.withdrawn_at + ? ` Data: ${chalk.dim("removed as part of the withdrawal")}` + : ` Data: ${chalk.red("incomplete")} ${chalk.dim("(some files failed to upload; see `nemar admin data-integrity-sweep`)")}`, ); } console.log(); - console.log(chalk.dim("To download this dataset:")); - console.log(chalk.dim(` nemar dataset download ${datasetId}`)); + if (datasetInfo.withdrawn_at) { + // Offering the download hint here would send the reader after content + // that was deliberately removed. + console.log(chalk.dim("This dataset was withdrawn; its content is no longer available.")); + } else { + console.log(chalk.dim("To download this dataset:")); + console.log(chalk.dim(` nemar dataset download ${datasetId}`)); + } }); /** diff --git a/src/lib/api/admin.ts b/src/lib/api/admin.ts index 81cca2ac4..ba3e56955 100644 --- a/src/lib/api/admin.ts +++ b/src/lib/api/admin.ts @@ -630,6 +630,13 @@ export interface ImportStatusResponse { imports: ImportJobRow[]; total: number; by_status: Record; + /** + * Imported datasets that have since been withdrawn. NOT a value of + * `by_status` -- withdrawal is orthogonal to the import lifecycle, and these + * rows are counted inside `by_status.complete` because their import did + * succeed. Optional: older backends omit it (#1048). + */ + withdrawn?: number; } export async function getImportStatus( diff --git a/src/lib/api/datasets.ts b/src/lib/api/datasets.ts index ca0bfd044..37c56d917 100644 --- a/src/lib/api/datasets.ts +++ b/src/lib/api/datasets.ts @@ -91,6 +91,18 @@ export interface Dataset { github_repo: string | null; concept_doi: string | null; created_at: string; + /** + * Withdrawal state (migration 0060). A withdrawn dataset had its concept DOI + * tombstoned at the registrar and its repo made private; its S3 content is + * gone by design. NULL/absent means not withdrawn. + * + * `status` stays "active" for a withdrawn dataset -- withdrawal is a separate + * axis from the lifecycle column, not a value of it -- so anything reporting + * on a dataset has to read this field to avoid presenting a tombstoned + * dataset as live (#1048). + */ + withdrawn_at?: string | null; + withdrawn_reason?: string | null; // Catalog-enriched fields (from nemar_catalog JOIN or catalog-only) modalities?: string; participants?: number;