Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion backend/src/routes/admin/imports.ts
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,23 @@ export function registerImportRoutes(admin: AdminRouter): void {
const by_status: Record<string, number> = {};
for (const s of IMPORT_STATUSES) by_status[s] = 0;
for (const row of counts.results ?? []) by_status[row.status] = row.n;
return c.json({ imports: results, total: results.length, by_status });
// Withdrawal is orthogonal to import status, not another value of it: a
// withdrawn dataset imported fine and stays `complete` here. Reported as its
// own number so the summary stops implying that every `complete` import is
// a dataset you can still fetch (#1048).
const withdrawn = await db
.prepare(
`SELECT COUNT(*) AS n FROM import_jobs j
JOIN datasets d ON d.dataset_id = j.dataset_id
WHERE d.withdrawn_at IS NOT NULL`,
)
.first<{ n: number }>();
return c.json({
imports: results,
total: results.length,
by_status,
withdrawn: withdrawn?.n ?? 0,
});
});

/**
Expand Down
130 changes: 130 additions & 0 deletions backend/test/imports-withdrawn-count.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/**
* Real GET /admin/imports tests for the withdrawn count (#1048).
*
* A withdrawn dataset's import succeeded, so its `import_jobs` row stays
* `complete` forever. Counting it only there made `complete N` read as "N
* datasets you can fetch", which is how a withdrawal cohort got mistaken for a
* data-integrity incident in the first place. `withdrawn` is reported as its
* own number, alongside rather than inside `by_status`.
*
* Runs against a real D1 (bun:sqlite + the actual migrations), no mocks.
*/

import type { Database } from "bun:sqlite";
import { beforeEach, describe, expect, test } from "bun:test";
import { Hono } from "hono";
import { adminRoutes } from "../src/routes/admin";
import { hashApiKey } from "../src/services/token";
import type { Bindings, Variables } from "../src/types/bindings";
import { freshDb, realD1 } from "./helpers/d1";

const ADMIN_KEY = "withdrawn-admin-key-0123456789abcdef0123456789ab";

let db: Database;
let app: Hono<{ Bindings: Bindings; Variables: Variables }>;
let ownerId: number;

async function seedAdmin(): Promise<void> {
db.run(
`INSERT INTO users (username, email, password_hash, status, role, email_verified)
VALUES ('wdadmin', 'wdadmin@example.org', 'x', 'approved', 'admin', 1)`,
);
const u = db.query<{ id: number }, []>("SELECT id FROM users WHERE username='wdadmin'").get();
if (!u) throw new Error("seed: admin insert failed");
ownerId = u.id;
db.query("INSERT INTO tokens (user_id, api_key_hash, api_key_prefix) VALUES (?, ?, ?)").run(
u.id,
await hashApiKey(ADMIN_KEY),
ADMIN_KEY.slice(0, 8),
);
}

/** An imported dataset, optionally withdrawn. `status` is the import status. */
function seedImportedDataset(
datasetId: string,
status: string,
withdrawnAt: string | null = null,
): void {
db.query(
`INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility, withdrawn_at, withdrawn_reason)
VALUES (?, ?, ?, 'active', 'public', ?, ?)`,
).run(
datasetId,
`Dataset ${datasetId}`,
ownerId,
withdrawnAt,
withdrawnAt ? "source returns 403 for every client" : null,
);
db.query(
`INSERT INTO import_jobs (dataset_id, source, source_id, stage, status)
VALUES (?, 'openneuro', ?, 'finalize', ?)`,
).run(datasetId, datasetId.replace(/^on/, "ds"), status);
}

function get(path: string): Promise<Response> {
return app.request(path, { headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, {
DB: realD1(db),
ENVIRONMENT: "development",
} as Bindings);
}

beforeEach(async () => {
db = freshDb();
app = new Hono<{ Bindings: Bindings; Variables: Variables }>();
app.route("/admin", adminRoutes);
await seedAdmin();
});

describe("GET /admin/imports withdrawn count", () => {
test("counts only datasets with withdrawn_at set", async () => {
seedImportedDataset("on000001", "complete");
seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z");
seedImportedDataset("on000003", "complete", "2026-07-02T00:00:00Z");

const body = await (await get("/admin/imports")).json();
expect(body.withdrawn).toBe(2);
});

test("withdrawn rows stay inside by_status.complete rather than moving out of it", async () => {
seedImportedDataset("on000001", "complete");
seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z");

const body = await (await get("/admin/imports")).json();
// The import genuinely completed; withdrawal is a separate axis. If this
// ever drops to 1, the count moved instead of being reported alongside.
expect(body.by_status.complete).toBe(2);
expect(body.withdrawn).toBe(1);
});

test("is zero when nothing is withdrawn", async () => {
seedImportedDataset("on000001", "complete");
seedImportedDataset("on000002", "failed");

const body = await (await get("/admin/imports")).json();
expect(body.withdrawn).toBe(0);
});

test("stays fleet-wide when a status filter is applied", async () => {
// by_status is deliberately unfiltered so the summary line is not
// misleading under a filter; `withdrawn` sits on that same line.
seedImportedDataset("on000001", "failed");
seedImportedDataset("on000002", "complete", "2026-07-01T00:00:00Z");

const body = await (await get("/admin/imports?status=failed")).json();
expect(body.imports.length).toBe(1);
expect(body.withdrawn).toBe(1);
});

test("a withdrawn dataset with no import job is not counted", async () => {
// The count is about imported datasets; a natively-uploaded one that was
// withdrawn has no import_jobs row and does not belong in this total.
db.query(
`INSERT INTO datasets (dataset_id, name, owner_user_id, status, visibility, withdrawn_at)
VALUES ('nm000001', 'Native', ?, 'active', 'public', '2026-07-01T00:00:00Z')`,
).run(ownerId);
seedImportedDataset("on000001", "complete");

const body = await (await get("/admin/imports")).json();
expect(body.withdrawn).toBe(0);
});
});
8 changes: 8 additions & 0 deletions src/commands/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3450,6 +3450,14 @@ importCommand
console.log(
` ${chalk.yellow(`in-flight ${(counts.preparing ?? 0) + (counts.copying ?? 0) + (counts.finalizing ?? 0)}`)} ${chalk.green(`complete ${counts.complete ?? 0}`)} ${chalk.magenta(`incomplete ${counts.incomplete ?? 0}`)} ${chalk.red(`failed ${counts.failed ?? 0}`)} ${chalk.red(`quarantined ${counts.quarantined ?? 0}`)} ${chalk.dim(`rolled_back ${counts.rolled_back ?? 0}`)}\n`,
);
// Withdrawn datasets are counted inside `complete` -- their import did
// succeed -- so say so rather than letting the total read as datasets
// that are still fetchable (#1048).
if (result.withdrawn) {
console.log(
` ${chalk.red(`withdrawn ${result.withdrawn}`)} ${chalk.dim("(included in complete; content removed and DOI tombstoned)")}\n`,
);
}

if (rows.length === 0) {
console.log(chalk.dim(" No import jobs match."));
Expand Down
39 changes: 35 additions & 4 deletions src/commands/dataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1422,6 +1422,17 @@ Examples:
console.log(` Name: ${datasetInfo.name}`);
console.log(` Owner: ${datasetInfo.owner_username}`);
console.log(` Status: ${colorizeStatus(datasetInfo.status)}`);
// Withdrawal is a separate axis from `status`, which stays "active", so it
// has to be printed or the dataset reads as live. Directly under Status,
// because that is the line a reader takes their answer from (#1048).
if (datasetInfo.withdrawn_at) {
const reason = datasetInfo.withdrawn_reason
? ` ${chalk.dim(`- ${datasetInfo.withdrawn_reason}`)}`
: "";
console.log(
` Withdrawn: ${chalk.red(new Date(datasetInfo.withdrawn_at).toLocaleDateString())}${reason}`,
);
}
console.log(` Created: ${new Date(datasetInfo.created_at).toLocaleDateString()}`);

if (datasetInfo.description) {
Expand All @@ -1433,20 +1444,40 @@ Examples:
}

if (datasetInfo.concept_doi) {
console.log(` DOI: https://doi.org/${datasetInfo.concept_doi}`);
// A withdrawn dataset's DOI resolves to a registrar tombstone, not the
// dataset. Printing it as a bare link invites the reader to follow it and
// conclude the dataset is fine.
const doiUrl = `https://doi.org/${datasetInfo.concept_doi}`;
console.log(
datasetInfo.withdrawn_at
? ` DOI: ${chalk.dim(doiUrl)} ${chalk.red("(tombstoned)")}`
: ` DOI: ${doiUrl}`,
);
}

// #970: surface-but-visible -- only shout when data is verified incomplete;
// complete/not-yet-audited stay silent (mirrors the list table's Data column).
if (datasetInfo.data_complete === 0) {
// A withdrawn dataset's content is 0-byte BY DESIGN, so the integrity
// sweep has nothing to find. Pointing an operator at it here is the exact
// false lead #1048 opened with: a withdrawal cohort read as a data loss
// incident because every surface described it as broken rather than gone.
console.log(
` Data: ${chalk.red("incomplete")} ${chalk.dim("(some files failed to upload; see `nemar admin data-integrity-sweep`)")}`,
datasetInfo.withdrawn_at
? ` Data: ${chalk.dim("removed as part of the withdrawal")}`
: ` Data: ${chalk.red("incomplete")} ${chalk.dim("(some files failed to upload; see `nemar admin data-integrity-sweep`)")}`,
);
}

console.log();
console.log(chalk.dim("To download this dataset:"));
console.log(chalk.dim(` nemar dataset download ${datasetId}`));
if (datasetInfo.withdrawn_at) {
// Offering the download hint here would send the reader after content
// that was deliberately removed.
console.log(chalk.dim("This dataset was withdrawn; its content is no longer available."));
} else {
console.log(chalk.dim("To download this dataset:"));
console.log(chalk.dim(` nemar dataset download ${datasetId}`));
}
});

/**
Expand Down
7 changes: 7 additions & 0 deletions src/lib/api/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -630,6 +630,13 @@ export interface ImportStatusResponse {
imports: ImportJobRow[];
total: number;
by_status: Record<string, number>;
/**
* Imported datasets that have since been withdrawn. NOT a value of
* `by_status` -- withdrawal is orthogonal to the import lifecycle, and these
* rows are counted inside `by_status.complete` because their import did
* succeed. Optional: older backends omit it (#1048).
*/
withdrawn?: number;
}

export async function getImportStatus(
Expand Down
12 changes: 12 additions & 0 deletions src/lib/api/datasets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,18 @@ export interface Dataset {
github_repo: string | null;
concept_doi: string | null;
created_at: string;
/**
* Withdrawal state (migration 0060). A withdrawn dataset had its concept DOI
* tombstoned at the registrar and its repo made private; its S3 content is
* gone by design. NULL/absent means not withdrawn.
*
* `status` stays "active" for a withdrawn dataset -- withdrawal is a separate
* axis from the lifecycle column, not a value of it -- so anything reporting
* on a dataset has to read this field to avoid presenting a tombstoned
* dataset as live (#1048).
*/
withdrawn_at?: string | null;
withdrawn_reason?: string | null;
// Catalog-enriched fields (from nemar_catalog JOIN or catalog-only)
modalities?: string;
participants?: number;
Expand Down
Loading