Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .context/phase5-cross-repo-owner-deploys.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,3 +63,29 @@ clone tool, not OpenNeuro onboarding), but worth folding into the same pass.
Deploy 1 + 2 before pointing the dev worker at the exemplar fleet (else dev manifests/archives/zarr
write to the prod bucket / call back to the prod worker). 3 + 4 can follow. None affect prod: absent
fields -> prod defaults.

## CONFIRMED EMPIRICALLY (epic #923 Phase 7, 2026-07-18)

The first real staging fleet run reproduced items 1, 2 and 4 above as concrete failures. They are
no longer theoretical; staging is functional without them, but two of its surfaces are not.

**Enrichment is broken on staging.** `run-enrichment.yml` hardcodes
`https://api.nemar.org/webhooks/llm-enrich` (line 149 as of this writing), so an enrichment run
dispatched for a staging exemplar calls the PRODUCTION worker, which correctly answers
`404 {"error":"Dataset not found"}` because the exemplar exists only in dev D1. Every exemplar's
`Run Enrichment` job therefore fails. This is the Phase 1 prod-safety gate behaving exactly as
designed (prod refuses to act on a dev-range dataset), but it means the central-workflow
enrichment path cannot be exercised on staging until `callback_base_url` lands. Note the CLI's
INLINE reindex is a separate path and does work, which is why exemplar metadata, modalities and
`has_hed` are populated correctly despite these failures.

**Archive generation would write to the production bucket.** `run-generate-archive.yml` still
hardcodes `s3://nemar` in its skip-check, build and cleanup steps. No staging archive has been
dispatched yet (no `Run Generate Archive` jobs appear in the run history for the exemplars), so
nothing has been written to prod, but this must be deployed before archive generation is enabled
for staging. Until then `data-test.nemar.org/<id>/` reports `archive: null` for every exemplar.

Deploy order is unchanged: items 1 and 2 first. Zarr serving on staging was validated separately
by copying derived artifacts directly (`zarr-test.nemar.org/<id>/zarr/index.json` served 20 stores
with 0 failures), so the serving gateway is proven; only the CONVERSION and ARCHIVE dispatch paths
depend on these workflow changes.
417 changes: 248 additions & 169 deletions backend/src/index.ts

Large diffs are not rendered by default.

13 changes: 13 additions & 0 deletions backend/src/services/archive-retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
* records state for the observability dashboard.
*/
import type { Bindings } from "../types/bindings.js";
import { isNonProductionEnv } from "./environment.js";
import { getDatasetsToken } from "./github-auth.js";
import { triggerArchiveGeneration } from "./github.js";

Expand Down Expand Up @@ -105,6 +106,18 @@ interface SweepRow {
* carries it forward (reset on 'ready', or the webhook's own retry on 'failed').
*/
export async function archiveRetrySweep(env: Bindings): Promise<void> {
// Production only (epic #923 Phase 7). The candidate query filters on
// archive_status alone, with no dataset-id prefix restriction, and dispatch
// targets the hardcoded nemarDatasets/.github central repo. On the dev/staging
// worker, whose D1 is a partial production mirror, that means running real
// Actions against real dataset repos. The daily cron already excludes this
// outside production; the guard is repeated here so a future caller inherits
// the same safety.
if (isNonProductionEnv(env)) {
console.log("[archive-retry-sweep] skipped (non-production)");
return;
}

let candidates: SweepRow[];
try {
const res = await env.DB.prepare(ARCHIVE_RETRY_SWEEP_QUERY)
Expand Down
19 changes: 19 additions & 0 deletions backend/src/services/datasetId.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,25 @@ export function isDevRangeDatasetId(id: string): boolean {
return DEV_SANDBOX_RANGE_RE.test(id);
}

// The dev partition splits again: xx090001-xx099899 is the EPHEMERAL band that
// the dev cleanup cron may delete, and xx099900-xx099999 is the curated
// exemplar fleet, which is never auto-deleted. Bounds are half-open
// [START, END) and safe to compare as strings because ids are fixed-width
// zero-padded (epic #923 Phase 7).
export const DEV_EPHEMERAL_BAND_START = "xx090001";
export const DEV_EPHEMERAL_BAND_END = "xx099900";

/**
* True when a dataset id is in the dev EPHEMERAL sandbox band, i.e. the only
* ids the non-production cleanup cron is allowed to delete. Excludes the
* exemplar fleet (xx099900+) and prod's sandbox band (<= xx089999).
*/
export function isDevEphemeralSandboxId(id: string): boolean {
return (
DEV_SANDBOX_RANGE_RE.test(id) && id >= DEV_EPHEMERAL_BAND_START && id < DEV_EPHEMERAL_BAND_END
);
}

// Test dataset IDs excluded from the gap-filling candidate pool.
// nm099999 would otherwise contribute candidate 100000 (99999+1).
const EXCLUDED_IDS = new Set(["nm099999"]);
Expand Down
13 changes: 12 additions & 1 deletion backend/src/services/doi-reconcile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import { datasetVersionLandingUrl } from "../../../shared/datacite-constants.js";
import type { Bindings } from "../types/bindings.js";
import { resolveEzidAuth } from "./doi.js";
import { resolveDatasetLandingBase } from "./environment.js";
import { isNonProductionEnv, resolveDatasetLandingBase } from "./environment.js";
import { getIdentifier, makePublic } from "./ezid.js";

/** Datasets inspected per sweep run. */
Expand Down Expand Up @@ -66,6 +66,17 @@ export async function reconcileReservedVersionDois(
env: Bindings,
nowMs = Date.now(),
): Promise<void> {
// Production only (epic #923 Phase 7). Sandbox-vs-production EZID auth is
// chosen from the DOI's own shoulder, not from ENVIRONMENT, so a real
// 10.82901 DOI on the prod-mirror dev D1 resolves to production credentials
// and makePublic is documented as permanent. The daily cron already excludes
// this outside production; the guard is repeated here so a future caller
// inherits the same safety.
if (isNonProductionEnv(env)) {
console.log("[doi-reconcile] skipped (non-production)");
return;
}

const WHERE = "latest_version_doi IS NOT NULL AND latest_version_doi != ''";
let rows: DoiRow[];
try {
Expand Down
16 changes: 16 additions & 0 deletions backend/src/services/publication-sweep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/

import type { Bindings } from "../types/bindings.js";
import { isNonProductionEnv } from "./environment.js";
import { getDatasetsToken } from "./github-auth.js";
import { getWorkflowRuns } from "./github.js";

Expand Down Expand Up @@ -81,6 +82,20 @@ export async function sweepBlockedBidsValidationRequests(
const db = env.DB;
const result: BlockedSweepResult = { scanned: 0, unblocked: 0, reblocked: 0, errors: 0 };

// Production only (epic #923 Phase 7). The candidate query filters on request
// status alone, with no dataset-id prefix restriction, so on the dev/staging
// worker (whose D1 is a partial production mirror) it would select REAL
// datasets' publication requests, read their real repos via the shared
// nemarDatasets installation token, and rewrite their status.
//
// Narrowed rather than disabled: staging genuinely needs this sweep, because
// an exemplar published while its BIDS validation is still running lands in
// exactly this 'blocked' state and would otherwise stay stuck forever. So
// outside production the candidate set is scoped to the dev range (xx09NNNN),
// which is the same fence the sandbox cleanup uses.
const devRangeOnly = isNonProductionEnv(env);
const scopeClause = devRangeOnly ? "AND pr.dataset_id LIKE 'xx09%'" : "";

const placeholders = BIDS_VALIDATION_BLOCK_REASONS.map(() => "?").join(", ");
// Guard the initial query so a D1 outage / schema drift surfaces as errors>0
// in the cron tally rather than an all-zero result indistinguishable from
Expand All @@ -101,6 +116,7 @@ export async function sweepBlockedBidsValidationRequests(
JOIN datasets d ON d.dataset_id = pr.dataset_id
WHERE pr.status = 'blocked'
AND pr.block_reason IN (${placeholders})
${scopeClause}
ORDER BY pr.updated_at ASC
LIMIT ?`,
)
Expand Down
250 changes: 250 additions & 0 deletions backend/test/cron-dev-safety.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,250 @@
/**
* Dev/staging cron safety tests (epic #923, Phase 7 / #930).
*
* Phase 7 enables a daily cron on the dev/staging worker for the first time.
* That worker's D1 is a partial PRODUCTION MIRROR (real nm* dataset rows, real
* user email addresses) and the worker holds a live RESEND_API_KEY, so the
* daily job set has to be narrowed before the trigger is safe to add.
*
* These tests pin the two guarantees that keep it safe:
* 1. Outside production the sandbox cleanup only ever selects ids in the dev
* EPHEMERAL band (xx090001-xx099899), never the curated exemplar fleet
* (xx099900+), never prod's sandbox band, never a real nm/on/ds row. This
* matters because deleteDatasetCascade's GitHub half is NOT env-scoped: a
* stray match deletes a real nemarDatasets repository.
* 2. archiveRetrySweep and reconcileReservedVersionDois refuse to run outside
* production even when called directly, so the guarantee does not depend on
* the cron's allowlist being the only caller.
* 3. The blocked-publication sweep is SCOPED rather than disabled outside
* production: staging needs it (an exemplar published while its BIDS
* validation is still running lands in 'blocked' and would otherwise stay
* stuck), so it narrows to the dev range instead of skipping.
*
* The band test runs the ACTUAL cleanup SQL from index.ts against real
* in-memory SQLite with the full migrated schema (no mocks): if the query in
* index.ts changes, this test has to change with it.
*/

import type { Database } from "bun:sqlite";
import { describe, expect, test } from "bun:test";
// The queries are imported from the production module, NOT retyped here, so a
// change to the real SQL cannot silently leave this test asserting stale text.
import {
NON_PROD_SANDBOX_CLEANUP_QUERY as NON_PROD_SANDBOX_QUERY,
PROD_SANDBOX_CLEANUP_QUERY as PROD_SANDBOX_QUERY,
} from "../src/index";
import { archiveRetrySweep } from "../src/services/archive-retry";
import {
DEV_EPHEMERAL_BAND_END,
DEV_EPHEMERAL_BAND_START,
isDevEphemeralSandboxId,
} from "../src/services/datasetId";
import { reconcileReservedVersionDois } from "../src/services/doi-reconcile";
import type { Bindings } from "../src/types/bindings";
import { freshDb, realD1 } from "./helpers/d1";

/** Seed an aged, active dataset row (old enough to be a cleanup candidate). */
function seedAged(db: Database, ids: string[], isExemplar = 0): void {
const stmt = db.query(
`INSERT INTO datasets (dataset_id, name, owner_user_id, status, is_exemplar, created_at)
VALUES (?, 'test', 1, 'active', ?, datetime('now', '-30 days'))`,
);
for (const id of ids) stmt.run(id, isExemplar);
}

describe("isDevEphemeralSandboxId", () => {
test("accepts the ephemeral band only", () => {
expect(isDevEphemeralSandboxId("xx090001")).toBe(true); // floor
expect(isDevEphemeralSandboxId("xx095000")).toBe(true);
expect(isDevEphemeralSandboxId("xx099899")).toBe(true); // last before exemplars
});

test("rejects the exemplar fleet band", () => {
expect(isDevEphemeralSandboxId("xx099900")).toBe(false); // first exemplar
expect(isDevEphemeralSandboxId("xx099950")).toBe(false);
expect(isDevEphemeralSandboxId("xx099999")).toBe(false); // last exemplar
});

test("rejects prod's sandbox band and the dev floor boundary", () => {
expect(isDevEphemeralSandboxId("xx090000")).toBe(false); // below floor
expect(isDevEphemeralSandboxId("xx089999")).toBe(false); // prod ceiling
expect(isDevEphemeralSandboxId("xx000001")).toBe(false);
});

test("rejects non-sandbox prefixes", () => {
expect(isDevEphemeralSandboxId("nm090001")).toBe(false);
expect(isDevEphemeralSandboxId("on090001")).toBe(false);
expect(isDevEphemeralSandboxId("ds090001")).toBe(false);
});

test("band constants are half-open and ordered", () => {
expect(DEV_EPHEMERAL_BAND_START < DEV_EPHEMERAL_BAND_END).toBe(true);
expect(isDevEphemeralSandboxId(DEV_EPHEMERAL_BAND_START)).toBe(true);
expect(isDevEphemeralSandboxId(DEV_EPHEMERAL_BAND_END)).toBe(false);
});
});

describe("non-production sandbox cleanup query", () => {
test("selects only ephemeral-band ids, sparing exemplars and real datasets", async () => {
const db = freshDb();
seedAged(db, [
"xx090001", // ephemeral, deletable
"xx095555", // ephemeral, deletable
"xx099899", // ephemeral, deletable (last)
"xx089999", // prod sandbox band, must survive
"xx000042", // prod sandbox band, must survive
]);
seedAged(db, ["xx099900", "xx099906"], 1); // exemplar fleet, must survive
// The rows that make the dev D1 dangerous: real production datasets.
seedAged(db, ["nm000155", "on003805", "ds000117"]);

const rows = await realD1(db)
.prepare(NON_PROD_SANDBOX_QUERY)
.bind(DEV_EPHEMERAL_BAND_START, DEV_EPHEMERAL_BAND_END, 10)
.all<{ dataset_id: string }>();

expect(rows.results.map((r) => r.dataset_id).sort()).toEqual([
"xx090001",
"xx095555",
"xx099899",
]);
});

test("spares an exemplar even if its is_exemplar flag were cleared", async () => {
// Belt and suspenders: the band alone must exclude the fleet, so a bad
// is_exemplar write cannot make a curated exemplar deletable.
const db = freshDb();
seedAged(db, ["xx099900", "xx099999"], 0);

const rows = await realD1(db)
.prepare(NON_PROD_SANDBOX_QUERY)
.bind(DEV_EPHEMERAL_BAND_START, DEV_EPHEMERAL_BAND_END, 10)
.all<{ dataset_id: string }>();

expect(rows.results).toEqual([]);
});

test("production query still sweeps the whole xx space", async () => {
// The narrowing must be non-production only; prod keeps its 14-day sweep
// across xx000001-xx089999 (prod never has exemplar rows).
const db = freshDb();
seedAged(db, ["xx000042", "xx089999"]);

const rows = await realD1(db)
.prepare(PROD_SANDBOX_QUERY)
.bind(10)
.all<{ dataset_id: string }>();

expect(rows.results.map((r) => r.dataset_id).sort()).toEqual(["xx000042", "xx089999"]);
});
});

describe("production-only daily jobs are environment-gated", () => {
// Both jobs swallow their own query errors, so "resolved without throwing" is
// NOT evidence the guard fired. Observe whether D1 was reached at all.
function probe(): { db: D1Database; touched: () => boolean } {
let reached = false;
const db = {
prepare() {
reached = true;
throw new Error("probe: candidate query reached");
},
} as unknown as D1Database;
return { db, touched: () => reached };
}

// Recognized non-production values: the job must not touch D1 at all.
for (const environment of ["development", "staging", "test"]) {
test(`archiveRetrySweep never queries D1 when ENVIRONMENT=${environment}`, async () => {
const p = probe();
await archiveRetrySweep({ ENVIRONMENT: environment, DB: p.db } as unknown as Bindings);
expect(p.touched()).toBe(false);
});

test(`reconcileReservedVersionDois never queries D1 when ENVIRONMENT=${environment}`, async () => {
const p = probe();
await reconcileReservedVersionDois({
ENVIRONMENT: environment,
DB: p.db,
} as unknown as Bindings);
expect(p.touched()).toBe(false);
});
}

// isNonProductionEnv is an allow-list and fails CLOSED: production, and any
// unset/unrecognized value, are treated as production so these backstops keep
// running rather than silently disabling themselves on a config typo.
for (const environment of ["production", "", undefined, "prod", "Production"]) {
test(`archiveRetrySweep still runs when ENVIRONMENT=${JSON.stringify(environment)}`, async () => {
const p = probe();
await archiveRetrySweep({ ENVIRONMENT: environment, DB: p.db } as unknown as Bindings);
expect(p.touched()).toBe(true);
});

test(`reconcileReservedVersionDois still runs when ENVIRONMENT=${JSON.stringify(environment)}`, async () => {
const p = probe();
await reconcileReservedVersionDois({
ENVIRONMENT: environment,
DB: p.db,
} as unknown as Bindings);
expect(p.touched()).toBe(true);
});
}
});

describe("blocked publication sweep scopes by dataset range, not by skipping", () => {
// This sweep must keep working on staging (an exemplar published while its
// BIDS validation is still running lands in 'blocked' and would otherwise
// stay stuck), while never touching the real datasets in the prod-mirror D1.
function seedBlocked(db: Database, ids: string[]): void {
const ds = db.query(
"INSERT INTO datasets (dataset_id, name, owner_user_id, github_repo) VALUES (?, 'test', 1, ?)",
);
const pr = db.query(
"INSERT INTO publication_requests (dataset_id, status, block_reason, requested_by) VALUES (?, 'blocked', 'bids_validation_pending', 1)",
);
for (const id of ids) {
ds.run(id, `nemarDatasets/${id}`);
pr.run(id);
}
}

/** The candidate query as built for each environment (see publication-sweep.ts). */
function candidateQuery(devRangeOnly: boolean): string {
return `SELECT pr.dataset_id
FROM publication_requests pr
JOIN datasets d ON d.dataset_id = pr.dataset_id
WHERE pr.status = 'blocked'
AND pr.block_reason IN ('bids_validation_pending')
${devRangeOnly ? "AND pr.dataset_id LIKE 'xx09%'" : ""}
ORDER BY pr.updated_at ASC`;
}

test("non-production sees only dev-range requests", async () => {
const db = freshDb();
seedBlocked(db, ["xx099900", "xx090001", "nm000155", "on003805"]);

const rows = await realD1(db)
.prepare(candidateQuery(true))
.bind()
.all<{ dataset_id: string }>();

expect(rows.results.map((r) => r.dataset_id).sort()).toEqual(["xx090001", "xx099900"]);
});

test("production still sees every blocked request", async () => {
const db = freshDb();
seedBlocked(db, ["xx099900", "nm000155", "on003805"]);

const rows = await realD1(db)
.prepare(candidateQuery(false))
.bind()
.all<{ dataset_id: string }>();

expect(rows.results.map((r) => r.dataset_id).sort()).toEqual([
"nm000155",
"on003805",
"xx099900",
]);
});
});
Loading
Loading