From 268738b0ad19bd7fc3dddaea3e6ed85a272e587d Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Fri, 26 Jun 2026 16:59:19 -0700 Subject: [PATCH 1/2] chore: remove legacy nemar.org dataexplorer couplings Retire both directions of the legacy nemar.org/dataexplorer coupling (epic #837 Phase 3): - Incoming (catalog pull): delete catalog-sync.yml, catalog-sync.ts, and the POST /admin/catalog/sync + GET /admin/catalog/status endpoints. This is what folded the ds shadow rows (#748). - Outgoing (datapipeline push): delete nemar-sync.ts + nemar-sync-reference.md. Split out the 4 BIDS-tree helpers into bids-tree.ts (still used by the metadata pipeline) and the slim delete-only client into legacy-purge.ts (for the Phase 4 one-time purge). - Callers: publish sync_nemar step -> logged no-op (kept in allSteps); webhook post-version-DOI sync -> refreshDatasetMetadata (keeps the metadata-columns refresh, drops the nemar.org push); reindex runDatasetSync -> refreshDatasetMetadata; remove POST /admin/datasets/:id/sync, GET /admin/sync/status, the nemar admin sync CLI + api methods, and NEMAR_USERNAME/NEMAR_PASSWORD bindings. - Delete 5 legacy-sync test files; relocate helper-test imports to bids-tree. nemar_sync_* columns + catalog_sync_log table + CF secrets are dropped in Phase 5. Tested: backend + CLI typecheck, biome, affected unit tests green. --- .github/workflows/catalog-sync.yml | 159 ----- backend/src/routes/admin.ts | 384 +--------- backend/src/routes/webhooks.ts | 248 +------ backend/src/services/bids-tree.ts | 94 +++ backend/src/services/catalog-sync.ts | 395 ----------- backend/src/services/citation-counts-sync.ts | 2 +- .../src/services/dataset-metadata-columns.ts | 8 +- backend/src/services/dataset-reindex.ts | 295 ++------ backend/src/services/enrich-dataset.ts | 2 +- backend/src/services/github.ts | 4 +- backend/src/services/legacy-purge.ts | 92 +++ backend/src/services/nemar-sync-reference.md | 187 ----- backend/src/services/nemar-sync.ts | 653 ------------------ backend/src/services/openneuro-discovery.ts | 5 +- backend/src/types/bindings.ts | 2 - src/commands/admin.ts | 90 --- src/lib/api.ts | 37 - test/catalog-contract.test.ts | 253 ------- test/catalog-fold-migration.test.ts | 559 --------------- test/catalog-phase3-reads.test.ts | 378 ---------- test/catalog-reembed.test.ts | 395 ----------- test/dataset-metadata-columns.test.ts | 6 +- test/dataset-reindex.test.ts | 13 +- .../manifest-callback-body-validation.test.ts | 3 +- test/version-doi-sync-trigger.test.ts | 91 --- 25 files changed, 305 insertions(+), 4050 deletions(-) delete mode 100644 .github/workflows/catalog-sync.yml create mode 100644 backend/src/services/bids-tree.ts delete mode 100644 backend/src/services/catalog-sync.ts create mode 100644 backend/src/services/legacy-purge.ts delete mode 100644 backend/src/services/nemar-sync-reference.md delete mode 100644 backend/src/services/nemar-sync.ts delete mode 100644 test/catalog-contract.test.ts delete mode 100644 test/catalog-fold-migration.test.ts delete mode 100644 test/catalog-phase3-reads.test.ts delete mode 100644 test/catalog-reembed.test.ts delete mode 100644 test/version-doi-sync-trigger.test.ts diff --git a/.github/workflows/catalog-sync.yml b/.github/workflows/catalog-sync.yml deleted file mode 100644 index 03edfd9f3..000000000 --- a/.github/workflows/catalog-sync.yml +++ /dev/null @@ -1,159 +0,0 @@ -name: Catalog Sync - -# Sync the nemar.org dataset catalog into D1 + Vectorize every 4 hours. -# The nemar.org API requires GET with a JSON body, which Cloudflare Workers -# rejects. This Action fetches the catalog via curl and POSTs records -# to the backend's /admin/catalog/sync endpoint. - -on: - schedule: - - cron: '0 */4 * * *' # Every 4 hours - workflow_dispatch: # Manual trigger - -env: - # Centralized config: update these when URLs or versions change. - # - # PROD_API_URL points at the workers.dev hostname rather than - # api.nemar.org. The custom domain sits behind Cloudflare's zone WAF, - # which 403s the multi-MB catalog POST from GitHub-runner IPs as a - # suspected bot. The workers.dev hostname bypasses zone WAF while - # still hitting the same Worker, D1, and Vectorize bindings. The - # /admin/catalog/sync route is already auth-gated by ADMIN_API_KEY, - # so this does not change the security posture. - PROD_API_URL: https://nemar-api.sccn-org.workers.dev - DEV_API_URL: https://nemar-api-dev.sccn-org.workers.dev - NEMAR_API_URL: https://nemar.org/api/dataexplorer/datapipeline - -jobs: - sync: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout (for package.json version) - uses: actions/checkout@v4 - with: - sparse-checkout: package.json - sparse-checkout-cone-mode: false - - - name: Read CLI version from package.json - id: version - run: | - VERSION=$(jq -r '.version' package.json) - echo "cli_version=$VERSION" >> $GITHUB_OUTPUT - echo "CLI version: $VERSION" - - - name: Fetch nemar.org catalog - id: fetch - run: | - # nemar.org requires GET with JSON body (non-standard). - # Some curl versions strip the body on GET; use Python urllib as fallback. - python3 << 'PYEOF' - import json, sys, subprocess, os - - # Try curl first (some versions support GET with body) - result = subprocess.run( - ["curl", "-s", "-X", "GET", - f"{os.environ.get('NEMAR_API_URL', 'https://nemar.org/api/dataexplorer/datapipeline')}/records", - "-H", "Content-Type: application/json", - "-d", '{"table_name":"dataexplorer_dataset","start":0,"limit":1000}'], - capture_output=True, text=True, timeout=60 - ) - - data = None - if result.stdout.strip(): - try: - data = json.loads(result.stdout) - except json.JSONDecodeError: - print(f"curl response not valid JSON, trying http.client...") - - # Fallback: use http.client which supports GET with body - if not data or not data.get("success"): - import http.client - conn = http.client.HTTPSConnection("nemar.org", timeout=60) - body = json.dumps({"table_name": "dataexplorer_dataset", "start": 0, "limit": 1000}) - conn.request("GET", "/api/dataexplorer/datapipeline/records", body=body, - headers={"Content-Type": "application/json", - "User-Agent": "nemar-cli/catalog-sync"}) - resp = conn.getresponse() - raw = resp.read().decode() - print(f"http.client response: status={resp.status}, length={len(raw)}") - if not raw.strip(): - print(f"::error::nemar.org returned empty response (HTTP {resp.status})") - print(f"Response headers: {dict(resp.getheaders())}") - sys.exit(1) - data = json.loads(raw) - conn.close() - - if not data.get("success"): - print(f"::error::nemar.org API returned unsuccessful response: {data.get('error', 'unknown')}") - sys.exit(1) - - entries = data.get("entries", {}) - records = [v for v in entries.values() if v.get("id")] - print(f"Fetched {data.get('total', 0)} records, {len(records)} valid") - - # Wrap in {"records": [...]} for the admin endpoint - with open("/tmp/catalog-records.json", "w") as f: - json.dump({"records": records}, f) - - # Write count to GITHUB_OUTPUT - import os - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write(f"count={len(records)}\n") - PYEOF - - - name: Push to production backend - if: fromJSON(steps.fetch.outputs.count) > 0 - env: - ADMIN_API_KEY: ${{ secrets.PROD_ADMIN_API_KEY }} - run: | - HTTP_CODE=$(curl -s -o /tmp/prod-response.json -w "%{http_code}" -X POST \ - "${{ env.PROD_API_URL }}/admin/catalog/sync" \ - -H "Authorization: Bearer $ADMIN_API_KEY" \ - -H "X-CLI-Version: ${{ steps.version.outputs.cli_version }}" \ - -H "Content-Type: application/json" \ - -d @/tmp/catalog-records.json \ - --max-time 120) - - RESPONSE=$(cat /tmp/prod-response.json) - echo "Production sync response (HTTP $HTTP_CODE):" - echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE" - - if [ "$HTTP_CODE" -ne 200 ]; then - echo "::error::Production backend returned HTTP $HTTP_CODE" - exit 1 - fi - - # #646: records_synced counts actual D1 row changes folded into the - # `datasets` source of truth (the nemar_catalog cache is gone). SQLite - # marks an `ON CONFLICT DO UPDATE` as changes=1 even when the values are - # unchanged, so a full re-sync of N existing rows returns N, not 0; a - # real sync therefore always folds >0 rows. - SYNCED=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('records_synced',0))") - echo "Synced: $SYNCED" - - if [ "$SYNCED" -eq 0 ]; then - echo "::error::Production sync reported 0 records synced" - exit 1 - fi - - - name: Push to dev backend - if: fromJSON(steps.fetch.outputs.count) > 0 - env: - ADMIN_API_KEY: ${{ secrets.TEST_ADMIN_API_KEY }} - run: | - HTTP_CODE=$(curl -s -o /tmp/dev-response.json -w "%{http_code}" -X POST \ - "${{ env.DEV_API_URL }}/admin/catalog/sync" \ - -H "Authorization: Bearer $ADMIN_API_KEY" \ - -H "X-CLI-Version: ${{ steps.version.outputs.cli_version }}" \ - -H "Content-Type: application/json" \ - -d @/tmp/catalog-records.json \ - --max-time 120) - - RESPONSE=$(cat /tmp/dev-response.json) - echo "Dev sync response (HTTP $HTTP_CODE):" - echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE" - - if [ "$HTTP_CODE" -ne 200 ]; then - echo "::warning::Dev backend returned HTTP $HTTP_CODE (non-fatal)" - fi diff --git a/backend/src/routes/admin.ts b/backend/src/routes/admin.ts index 3131aaeca..69aad9b8a 100644 --- a/backend/src/routes/admin.ts +++ b/backend/src/routes/admin.ts @@ -22,11 +22,6 @@ import { getBroadcastRecipients, sendBroadcast, } from "../services/broadcast"; -import { - type NemarCatalogRecord, - importCatalogRecords, - syncCatalog, -} from "../services/catalog-sync"; import { type DataCiteEnrichment, bidsToDataCite, @@ -38,7 +33,7 @@ import { DatasetReindexError, type ReindexFilter, buildReindexFilterQuery, - runDatasetSync, + refreshDatasetMetadata, runEnrichmentForDataset, } from "../services/dataset-reindex"; import { reembedDatasetVector } from "../services/dataset-search"; @@ -107,7 +102,6 @@ import { revokeUserIamAccess } from "../services/iam"; import { IMPORT_STATUSES } from "../services/import-recovery"; import { generateManifest } from "../services/manifest"; import { buildCoverageReport } from "../services/manifest-coverage"; -import { syncDatasetToNemar } from "../services/nemar-sync"; import { createNotice, deleteNotice, listAllNotices } from "../services/notices"; import { errorMessage, @@ -3501,7 +3495,7 @@ adminRoutes.post("/publish/:id/deny", zValidator("json", denySchema), async (c) * 10. upload_to_zenodo - Upload release archive to Zenodo * 11. publish_doi - Publish the Zenodo DOI (permanent and irreversible!) * 12. s3_lock - Apply S3 Object Lock (Governance mode) - * 13. sync_nemar - Sync metadata to nemar.org datapipeline (non-fatal) + * 13. sync_nemar - Disabled no-op (legacy nemar.org sync retired, #837) * 14. notify_user - Send publication confirmation email * * (Archive zip generation is NOT an orchestrator step -- the central @@ -4965,156 +4959,13 @@ adminRoutes.post("/publish/:id/approve", zValidator("json", approveSchema), asyn // per publish; it was removed (run-generate-archive.yml also gained an // S3 skip-if-exists guard as defense-in-depth). - // Step 14: Sync metadata to nemar.org datapipeline + // Step 14: nemar.org datapipeline sync — disabled (epic #837 retired the legacy + // dataexplorer coupling). Kept in allSteps + stepsToRun so existing + // publication_requests step records stay valid; the step is now a logged no-op + // (mirrors the disabled upload_to_zenodo step above). if (stepsToRun.includes("sync_nemar")) { - try { - await startStep("sync_nemar"); - - // OpenNeuro-imported datasets need alternate_id mapping before syncing - if (datasetId.startsWith("on")) { - console.log( - `[publish] Skipping nemar.org sync for OpenNeuro dataset ${datasetId} (alternate_id not yet supported)`, - ); - await updateProgress("sync_nemar"); - } else if (!c.env.NEMAR_USERNAME || !c.env.NEMAR_PASSWORD) { - console.warn("[publish] NEMAR_USERNAME/PASSWORD not configured; skipping nemar.org sync"); - await updateProgress("sync_nemar"); - } else { - const nemarUser = c.env.NEMAR_USERNAME; - const nemarPass = c.env.NEMAR_PASSWORD; - // Gather source data - const s3Cfg = getS3Config(c.env); - const tree = await getTreeAtRef(repoName, "main", pat); - const bidsFile = tree.find((f) => f.path === "dataset_description.json"); - let bidsDescription: Record = {}; - if (bidsFile) { - try { - bidsDescription = JSON.parse(await getBlobContent(repoName, bidsFile.sha, pat)); - } catch (err) { - console.warn(`[publish] Failed to parse dataset_description.json: ${err}`); - } - } - - const readmeFile = tree.find((f) => f.path === "README" || f.path === "README.md"); - const readme = readmeFile ? await getBlobContent(repoName, readmeFile.sha, pat) : ""; - - const nemarMetaFile = tree.find((f) => f.path === ".nemar/metadata.json"); - let nemarMeta = null; - if (nemarMetaFile) { - try { - const raw = JSON.parse(await getBlobContent(repoName, nemarMetaFile.sha, pat)); - const parsed = parseNemarMetadata(raw); - if (parsed && parsed.version === "2.0") nemarMeta = parsed; - } catch (err) { - console.warn(`[publish] Failed to parse .nemar/metadata.json: ${err}`); - } - } - - // Gather D1 data, S3 stats, repo info in parallel - const [latestVersion, updatedDoi, pubRequest, s3Stats, zipFileSize] = await Promise.all([ - db - .prepare( - "SELECT version, doi, created_at FROM dataset_versions WHERE dataset_id = ? ORDER BY created_at DESC LIMIT 1", - ) - .bind(datasetId) - .first<{ version: string; doi: string; created_at: string }>(), - db - .prepare("SELECT concept_doi, created_at FROM datasets WHERE dataset_id = ?") - .bind(datasetId) - .first<{ concept_doi: string | null; created_at: string | null }>(), - db - .prepare("SELECT approved_at FROM publication_requests WHERE id = ?") - .bind(requestId) - .first<{ approved_at: string | null }>(), - getDatasetS3Stats(s3Cfg, datasetId).catch((err) => { - console.warn(`[publish] S3 stats failed for ${datasetId}: ${err}`); - return { totalSize: 0, objectCount: 0 }; - }), - getArchiveSize(s3Cfg, datasetId).catch((err) => { - console.warn(`[publish] Archive size failed for ${datasetId}: ${err}`); - return 0; - }), - ]); - - // Try to read version manifest from S3 for accurate file sizes - let manifest = null; - if (latestVersion?.version) { - try { - const raw = await getManifest(s3Cfg, datasetId, latestVersion.version); - if (raw) { - try { - manifest = JSON.parse(raw); - } catch (parseErr) { - console.warn( - `[publish] Manifest JSON corrupted for ${datasetId} v${latestVersion.version}: ${parseErr}`, - ); - } - } - } catch (err) { - console.warn(`[publish] Failed to fetch manifest from S3 for ${datasetId}: ${err}`); - } - } - - const syncResult = await syncDatasetToNemar(nemarUser, nemarPass, { - datasetId, - bidsDescription, - nemarMetadata: nemarMeta, - readme, - tree, - conceptDoi: updatedDoi?.concept_doi || null, - latestVersionDoi: latestVersion?.doi || null, - latestVersion: latestVersion?.version || null, - versionCreatedAt: latestVersion?.created_at || null, - ownerUsername: dataset.owner_username, - createdAt: updatedDoi?.created_at || null, - publishDate: pubRequest?.approved_at || null, - repoName, - pat, - manifest, - s3Stats, - zipFileSize, - }); - - // Update sync tracking - await db - .prepare( - `UPDATE datasets SET nemar_sync_status = ?, nemar_sync_at = CASE WHEN ? = 'synced' THEN datetime('now') ELSE nemar_sync_at END, nemar_sync_error = ?, updated_at = datetime('now') WHERE dataset_id = ?`, - ) - .bind( - syncResult.synced ? "synced" : "failed", - syncResult.synced ? "synced" : "failed", - syncResult.errors.length ? syncResult.errors.join("; ") : null, - datasetId, - ) - .run(); - - if (!syncResult.synced) { - console.warn( - `[publish] nemar.org sync partially failed for ${datasetId}: ${syncResult.errors.join("; ")}`, - ); - } - - await updateProgress( - "sync_nemar", - syncResult.synced ? undefined : syncResult.errors.join("; "), - ); - } - } catch (err) { - const msg = errorMessage(err); - // Non-fatal: nemar.org sync failure should not block publication - console.warn(`[publish] nemar.org sync failed for ${datasetId} (non-fatal): ${msg}`); - try { - await db - .prepare( - "UPDATE datasets SET nemar_sync_status = 'failed', nemar_sync_error = ?, updated_at = datetime('now') WHERE dataset_id = ?", - ) - .bind(msg, datasetId) - .run(); - } catch (d1Err) { - console.warn(`[publish] Failed to update sync status in D1: ${d1Err}`); - } - await updateProgress("sync_nemar", msg); - } + console.log("[publish] nemar.org sync skipped (legacy datapipeline sync disabled)"); + await updateProgress("sync_nemar"); } // Step 15: Notify user (non-fatal — mirrors sync_nemar pattern) @@ -6024,41 +5875,8 @@ adminRoutes.post( // --------------------------------------------------------------------------- /** - * POST /admin/datasets/:id/sync - Manually sync a dataset to nemar.org - * - * Gathers metadata from D1/GitHub and pushes to nemar.org datapipeline API. - * Useful for backfilling datasets published before this feature existed. - */ -adminRoutes.post("/datasets/:id/sync", async (c) => { - const datasetId = c.req.param("id"); - - // Thin wrapper around runDatasetSync (epic #417 phase 3) so the admin - // and post-version-DOI sync paths share one implementation, and so this - // endpoint also populates the Phase 2 metadata columns. - try { - const result = await runDatasetSync(c.env, datasetId); - return c.json({ - dataset_id: datasetId, - synced: result.synced, - errors: result.errors, - metadata_columns_written: result.metadata_columns_written, - ...(result.metadata_columns_error && { - metadata_columns_error: result.metadata_columns_error, - }), - ...(result.nemar_sync_skipped && { nemar_sync_skipped: true }), - }); - } catch (err) { - if (err instanceof DatasetReindexError) { - return c.json({ error: err.message }, err.statusCode); - } - console.error(`[admin/sync] Unexpected error for ${datasetId}:`, err); - return c.json({ error: errorMessage(err) }, 500); - } -}); - -/** - * POST /admin/datasets/:id/reindex - Refresh enrichment + nemar.org sync + - * Phase 2 metadata columns for a single dataset (epic #417 phase 3). + * POST /admin/datasets/:id/reindex - Refresh enrichment + Phase 2 metadata + * columns for a single dataset (epic #417 phase 3). * * Body: { skip_enrichment?: boolean, skip_sync?: boolean, ref?: string } * @@ -6088,7 +5906,6 @@ adminRoutes.post("/datasets/:id/reindex", async (c) => { enrichment: { status: "ok" | "failed" | "skipped"; ref?: string; error?: string }; sync: { status: "ok" | "failed" | "skipped"; - errors?: string[]; metadata_columns_written?: boolean; metadata_columns_error?: string; }; @@ -6107,23 +5924,12 @@ adminRoutes.post("/datasets/:id/reindex", async (c) => { if (!skipSync) { try { - const sync = await runDatasetSync(c.env, datasetId); - // `nemar_sync_skipped` is set when the upstream push was intentionally - // suppressed (today: OpenNeuro datasets that have no alternate_id - // mapping). Map that to status="skipped" so callers can distinguish a - // real failure from a benign skip. Metadata columns are still written - // in the skip case, so we keep that field accurate. - const syncStatus: "ok" | "failed" | "skipped" = sync.nemar_sync_skipped - ? "skipped" - : sync.synced - ? "ok" - : "failed"; + const refreshed = await refreshDatasetMetadata(c.env, datasetId); result.sync = { - status: syncStatus, - errors: sync.errors, - metadata_columns_written: sync.metadata_columns_written, - ...(sync.metadata_columns_error && { - metadata_columns_error: sync.metadata_columns_error, + status: refreshed.metadata_columns_written ? "ok" : "failed", + metadata_columns_written: refreshed.metadata_columns_written, + ...(refreshed.metadata_columns_error && { + metadata_columns_error: refreshed.metadata_columns_error, }), }; } catch (err) { @@ -6232,14 +6038,13 @@ adminRoutes.post("/datasets/reindex/bulk", async (c) => { enrichment: { status: "ok" | "failed" | "skipped"; error?: string }; sync: { status: "ok" | "failed" | "skipped"; - errors?: string[]; metadata_columns_error?: string; }; }; const results: PerDataset[] = []; // Sequential to keep per-dataset failures isolated and respect upstream - // rate limits (nemar.org, OpenRouter, GitHub). The runtime budget for a + // rate limits (OpenRouter, GitHub). The runtime budget for a // Cloudflare Worker request is the limiting factor for very large batches; // operators should narrow the filter or split into multiple runs. for (const datasetId of datasetIds) { @@ -6254,20 +6059,19 @@ adminRoutes.post("/datasets/reindex/bulk", async (c) => { } if (!skipSync) { try { - const sync = await runDatasetSync(c.env, datasetId); + const refreshed = await refreshDatasetMetadata(c.env, datasetId); entry.sync = { - status: sync.synced ? "ok" : "failed", - errors: sync.errors, - ...(sync.metadata_columns_error && { - metadata_columns_error: sync.metadata_columns_error, + status: refreshed.metadata_columns_written ? "ok" : "failed", + ...(refreshed.metadata_columns_error && { + metadata_columns_error: refreshed.metadata_columns_error, }), }; } catch (err) { // Per-dataset failure must surface in server logs in addition to the // response body so an operator running with -i (or a CI job that // only checks HTTP status) still gets a trace. - console.error(`[admin/reindex/bulk] ${datasetId} sync threw:`, err); - entry.sync = { status: "failed", errors: [errorMessage(err)] }; + console.error(`[admin/reindex/bulk] ${datasetId} reindex threw:`, err); + entry.sync = { status: "failed" }; } } if (entry.enrichment.status === "failed") { @@ -6286,36 +6090,6 @@ adminRoutes.post("/datasets/reindex/bulk", async (c) => { }); }); -/** - * GET /admin/sync/status - List sync status for all published datasets - */ -adminRoutes.get("/sync/status", async (c) => { - const db = c.env.DB; - - const results = await db - .prepare( - `SELECT d.dataset_id, d.name, d.nemar_sync_status, d.nemar_sync_at, d.nemar_sync_error - FROM datasets d - WHERE d.visibility = 'public' OR d.concept_doi IS NOT NULL - ORDER BY d.nemar_sync_status IS NULL DESC, d.nemar_sync_status = 'failed' DESC, d.dataset_id`, - ) - .all<{ - dataset_id: string; - name: string; - nemar_sync_status: string | null; - nemar_sync_at: string | null; - nemar_sync_error: string | null; - }>(); - - return c.json({ - datasets: results.results, - total: results.results.length, - synced: results.results.filter((d) => d.nemar_sync_status === "synced").length, - failed: results.results.filter((d) => d.nemar_sync_status === "failed").length, - pending: results.results.filter((d) => d.nemar_sync_status === null).length, - }); -}); - // ============================================================================ // Import jobs (issue #754) - import state view + rollback/retry // ============================================================================ @@ -6456,120 +6230,6 @@ adminRoutes.post("/imports/:id/retry", async (c) => { return c.json({ ok: true, dataset_id: datasetId, status: "preparing" }); }); -// ============================================================================ -// Catalog Sync (nemar.org catalog -> D1) -// ============================================================================ - -/** - * POST /admin/catalog/sync - Fold pre-fetched catalog records into `datasets` - * - * The nemar.org API requires GET with a JSON body, which Workers' fetch() - * rejects. The GitHub Action fetches the catalog and POSTs records here. - * Accepts { records: NemarCatalogRecord[] } in the request body. New/changed - * rows are marked embedding_dirty=1 for the scheduled re-embed. - */ -adminRoutes.post("/catalog/sync", async (c) => { - const body = await c.req.json<{ records?: unknown[] }>(); - - if (body.records && Array.isArray(body.records) && body.records.length > 0) { - // Validate records before importing - const validRecords: NemarCatalogRecord[] = []; - const validationErrors: string[] = []; - for (const [i, raw] of (body.records as Array>).entries()) { - if (!raw.id || typeof raw.id !== "string") { - validationErrors.push(`Record ${i}: missing or invalid 'id'`); - continue; - } - if (!raw.name || typeof raw.name !== "string") { - validationErrors.push(`Record ${i} (${raw.id}): missing or invalid 'name'`); - continue; - } - validRecords.push(raw as unknown as NemarCatalogRecord); - } - if (validRecords.length === 0) { - return c.json( - { error: "No valid records in payload", validation_errors: validationErrors }, - 400, - ); - } - if (validationErrors.length > 0) { - console.warn( - `[catalog-sync] ${validationErrors.length} records failed validation, importing ${validRecords.length}`, - ); - } - - const result = await importCatalogRecords(c.env.DB, validRecords); - return c.json({ - records_synced: result.recordsSynced, - errors: result.errors, - duration_ms: result.durationMs, - }); - } - - // No records provided; try fetching directly (will fail in Workers due to GET+body) - const result = await syncCatalog(c.env.DB); - return c.json({ - records_synced: result.recordsSynced, - errors: result.errors, - duration_ms: result.durationMs, - }); -}); - -/** - * GET /admin/catalog/status - Show catalog sync history - */ -adminRoutes.get("/catalog/status", async (c) => { - const db = c.env.DB; - - try { - // #646: records_indexed is no longer written (vectors are re-embedded - // lazily by the embedding_dirty drain cron, not counted during sync), so - // it's dropped from the status projection. - const logs = await db - .prepare( - `SELECT id, started_at, completed_at, records_synced, errors, status - FROM catalog_sync_log - ORDER BY started_at DESC - LIMIT 10`, - ) - .all<{ - id: number; - started_at: string; - completed_at: string | null; - records_synced: number; - errors: string | null; - status: string; - }>(); - - // #646: the catalog is now the folded legacy rows in `datasets` (the - // sentinel-owned source-of-truth rows), not the dropped nemar_catalog table. - const catalogCount = await db - .prepare( - "SELECT COUNT(*) AS count FROM datasets WHERE owner_user_id = ? AND status = 'active'", - ) - .bind(SYSTEM_USER_ID) - .first<{ count: number }>(); - - return c.json({ - catalog_size: catalogCount?.count ?? 0, - recent_syncs: logs.results, - }); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - // Narrow the graceful path to a missing catalog_sync_log (the only - // optional table here). A missing `datasets` is a real infra failure and - // must surface as a 500, not a misleading "not initialized" 200. - if (msg.includes("no such table: catalog_sync_log")) { - return c.json({ - catalog_size: 0, - recent_syncs: [], - message: "catalog_sync_log not initialized", - }); - } - return c.json({ error: "Failed to query catalog status", details: msg }, 500); - } -}); - /** * POST /admin/vectorize/reindex-all - Re-embed datasets' vectors from the * `datasets` source of truth (#646 Phase 4). Fixes the stale-vector backlog. diff --git a/backend/src/routes/webhooks.ts b/backend/src/routes/webhooks.ts index 9da4b023f..299d8ac65 100644 --- a/backend/src/routes/webhooks.ts +++ b/backend/src/routes/webhooks.ts @@ -10,7 +10,7 @@ import { Hono } from "hono"; import { MAX_ARCHIVE_RETRIES, decideArchiveRetry } from "../services/archive-retry.js"; import { purgeCacheUrls, zarrPurgeTargets } from "../services/cloudflare.js"; -import { runDatasetSync } from "../services/dataset-reindex.js"; +import { refreshDatasetMetadata } from "../services/dataset-reindex.js"; import { isValidDatasetId } from "../services/datasetId.js"; import { createEzidVersionDoi, parseDoiProvider } from "../services/doi.js"; import { enrichDataset } from "../services/enrich-dataset.js"; @@ -79,47 +79,6 @@ export function validateEnrichmentRef(ref: unknown): string | null { return null; } -/** - * Decide whether to trigger an nemar.org sync after a version-DOI publish. - * - * Pure helper so the EZID and Zenodo paths share one rule and we can pin - * the matrix in unit tests without spinning up the webhook harness. - * - * Rules (mirrors the prior inline EZID logic): - * - missing NEMAR_USERNAME or NEMAR_PASSWORD: skip with "no_credentials" - * - OpenNeuro dataset (`on`-prefix): skip with "openneuro" (nemar.org - * pipeline doesn't yet accept alternate_id; see nemarOrg/nemar-cli#339) - * - Sandbox dataset (`xx`-prefix): skip with "sandbox" (blocked from - * publishing; syncing would write a false-alarm failed row) - * - missing DOI string: skip with "no_doi" (Zenodo only; EZID always - * returns a DOI on success) - * - otherwise: trigger - */ -export type NemarSyncDecision = - | { trigger: true } - | { trigger: false; reason: "no_credentials" | "openneuro" | "sandbox" | "no_doi" }; - -export function shouldSyncToNemarAfterVersionDoi(input: { - datasetId: string; - versionDoi: string | null | undefined; - nemarUsername: string | null | undefined; - nemarPassword: string | null | undefined; -}): NemarSyncDecision { - if (!input.nemarUsername || !input.nemarPassword) { - return { trigger: false, reason: "no_credentials" }; - } - if (input.datasetId.startsWith("on")) { - return { trigger: false, reason: "openneuro" }; - } - if (input.datasetId.startsWith("xx")) { - return { trigger: false, reason: "sandbox" }; - } - if (!input.versionDoi) { - return { trigger: false, reason: "no_doi" }; - } - return { trigger: true }; -} - /** * Returns true iff the central manifest workflow path (#557) is enabled * for this Worker. Reads `MANIFEST_VIA_CENTRAL_WORKFLOW` from env and @@ -1087,31 +1046,20 @@ async function handleEzidVersionDoiLegacy( }); // Sync to nemar.org in the background (non-fatal, non-blocking). - // Pass the freshly-minted DOI + version through as overrides so a D1 - // read-after-write race (background waitUntil firing before the new - // dataset_versions row replicates) doesn't drop the version DOI - // from the nemar.org payload. - // - // Under centralFlow, the manifest/summary/dataset_versions row don't - // exist yet at this point -- the /webhooks/manifest-ready handler - // owns those writes. Skip the inline sync; manifest-ready will fire - // sync after the row insert lands. + // Refresh D1 metadata columns from the new version's tree (the legacy + // nemar.org datapipeline sync was removed in epic #837). Background + + // non-fatal. Under centralFlow the dataset_versions row doesn't exist yet + // here -- the /webhooks/manifest-ready handler triggers the refresh after + // the row insert lands. if (!centralFlow) { - const ezidSyncDecision = shouldSyncToNemarAfterVersionDoi({ - datasetId: dataset.dataset_id, - versionDoi: result.doi, - nemarUsername: c.env.NEMAR_USERNAME, - nemarPassword: c.env.NEMAR_PASSWORD, - }); - if (ezidSyncDecision.trigger) { - c.executionCtx.waitUntil(syncToNemarAfterVersionDoi(c.env, dataset, version, result.doi)); - } else if (ezidSyncDecision.reason === "no_credentials") { - console.warn("[webhook] NEMAR_USERNAME/PASSWORD not configured; skipping nemar.org sync"); - } else if (ezidSyncDecision.reason === "openneuro") { - console.info( - `[webhook] Skipping nemar.org sync for OpenNeuro dataset ${dataset.dataset_id}`, - ); - } + c.executionCtx.waitUntil( + refreshDatasetMetadata(c.env, dataset.dataset_id).catch((err) => + console.error( + `[webhook] metadata refresh failed for ${dataset.dataset_id} (non-fatal):`, + err, + ), + ), + ); } return c.json({ @@ -1142,63 +1090,6 @@ async function handleEzidVersionDoiLegacy( } } -/** - * Background task: sync dataset metadata to nemar.org after a version DOI is published. - * Non-fatal; logs errors but never throws. - */ -async function syncToNemarAfterVersionDoi( - env: Bindings, - dataset: { - id: number; - dataset_id: string; - name: string; - github_repo: string | null; - concept_doi: string | null; - }, - version: string, - versionDoi: string, -): Promise { - // Delegated to runDatasetSync (epic #417 phase 3) so this path stays in - // step with the admin reindex endpoint. The helper handles tree+S3+ - // participants gathering, syncDatasetToNemar, nemar_sync_* fields, and the - // Phase 2 metadata columns + metadata_columns_error. - try { - const result = await runDatasetSync(env, dataset.dataset_id, { - versionOverride: version, - versionDoiOverride: versionDoi, - }); - if (result.synced) { - console.log(`[webhook] nemar.org sync succeeded for ${dataset.dataset_id} v${version}`); - } else { - console.warn( - `[webhook] nemar.org sync failed for ${dataset.dataset_id}: ${result.errors.join("; ")}`, - ); - } - if (result.metadata_columns_error) { - console.warn( - `[webhook] metadata_columns failed for ${dataset.dataset_id}: ${result.metadata_columns_error}`, - ); - } - } catch (err) { - console.error(`[webhook] nemar.org sync error for ${dataset.dataset_id} (non-fatal):`, err); - // Record both failures together: runDatasetSync threw before reaching - // either the nemar_sync UPDATE or the metadata-columns UPDATE, so set - // both so operators querying for stale state get a consistent view. - const msg = errorMessage(err); - try { - await env.DB.prepare( - "UPDATE datasets SET nemar_sync_status = 'failed', nemar_sync_error = ?, metadata_columns_error = ?, updated_at = datetime('now') WHERE dataset_id = ?", - ) - .bind(msg, `runDatasetSync threw before metadata-columns write: ${msg}`, dataset.dataset_id) - .run(); - } catch (d1Err) { - console.warn( - `[webhook] Failed to update sync status in D1 for ${dataset.dataset_id}: ${d1Err}`, - ); - } - } -} - /** * Handle Zenodo version DOI creation: downloads release, uploads to Zenodo, publishes, and generates version manifest. */ @@ -1407,44 +1298,19 @@ async function handleZenodoVersionDoi( } } - // Issue #339: keep nemar.org in step with the latest version DOI. - // Mirrors the EZID path: non-fatal, runs in the background via - // waitUntil, and skips OpenNeuro datasets (alternate_id not yet - // supported by the nemar.org pipeline). - // - // Under centralFlow, the manifest/summary/dataset_versions row don't - // exist yet at this point -- the /webhooks/manifest-ready handler - // owns those writes. Skip the inline sync; manifest-ready will fire - // sync after the row insert lands. + // Refresh D1 metadata columns from the new version's tree (the legacy + // nemar.org datapipeline sync was removed in epic #837). Background + + // non-fatal. Under centralFlow the dataset_versions row doesn't exist yet + // here -- manifest-ready triggers the refresh after the row insert lands. if (!centralFlow) { - const zenodoSyncDecision = shouldSyncToNemarAfterVersionDoi({ - datasetId: dataset.dataset_id, - versionDoi: published.doi ?? null, - nemarUsername: c.env.NEMAR_USERNAME, - nemarPassword: c.env.NEMAR_PASSWORD, - }); - if (zenodoSyncDecision.trigger) { - // trigger: true guarantees versionDoi is non-null (no_doi guard in predicate) - c.executionCtx.waitUntil( - syncToNemarAfterVersionDoi(c.env, dataset, version, published.doi!), - ); - } else { - if (zenodoSyncDecision.reason === "no_credentials") { - console.warn("[webhook] NEMAR_USERNAME/PASSWORD not configured; skipping nemar.org sync"); - } else if (zenodoSyncDecision.reason === "openneuro") { - console.info( - `[webhook] Skipping nemar.org sync for OpenNeuro dataset ${dataset.dataset_id}`, - ); - } else if (zenodoSyncDecision.reason === "sandbox") { - console.info( - `[webhook] Skipping nemar.org sync for sandbox dataset ${dataset.dataset_id}`, - ); - } else if (zenodoSyncDecision.reason === "no_doi") { - console.warn( - `[webhook] Skipping nemar.org sync for ${dataset.dataset_id}: Zenodo returned no DOI`, - ); - } - } + c.executionCtx.waitUntil( + refreshDatasetMetadata(c.env, dataset.dataset_id).catch((err) => + console.error( + `[webhook] metadata refresh failed for ${dataset.dataset_id} (non-fatal):`, + err, + ), + ), + ); } return c.json({ @@ -1693,57 +1559,19 @@ webhooks.post("/manifest-ready", async (c) => { console.error("[manifest-ready] manifest_jobs UPDATE to 'ready' failed:", err); } - // Issue #557: under centralFlow the EZID/Zenodo publish handlers - // intentionally skip the nemar.org sync because the - // dataset_versions row + manifest/summary on S3 don't exist yet - // when the DOI is minted. Fire the sync HERE, after the row insert - // lands, so the nemar.org payload sees the new version DOI. - // Non-fatal: log + carry on (legacy behavior). + // Issue #557: under centralFlow the dataset_versions row + manifest/summary + // on S3 don't exist when the DOI is minted, so refresh the D1 metadata + // columns HERE, after the row insert lands. (The legacy nemar.org sync was + // removed in epic #837.) Background + non-fatal. if (job.doi) { - try { - const dataset = await c.env.DB.prepare( - "SELECT id, dataset_id, name, github_repo, concept_doi FROM datasets WHERE dataset_id = ?", - ) - .bind(body.dataset_id) - .first<{ - id: number; - dataset_id: string; - name: string; - github_repo: string | null; - concept_doi: string | null; - }>(); - if (dataset) { - const syncDecision = shouldSyncToNemarAfterVersionDoi({ - datasetId: dataset.dataset_id, - versionDoi: job.doi, - nemarUsername: c.env.NEMAR_USERNAME, - nemarPassword: c.env.NEMAR_PASSWORD, - }); - if (syncDecision.trigger) { - c.executionCtx.waitUntil( - syncToNemarAfterVersionDoi(c.env, dataset, body.version, job.doi), - ); - } else if (syncDecision.reason === "no_credentials") { - console.warn( - "[manifest-ready] NEMAR_USERNAME/PASSWORD not configured; skipping nemar.org sync", - ); - } else if (syncDecision.reason === "openneuro") { - console.info( - `[manifest-ready] Skipping nemar.org sync for OpenNeuro dataset ${dataset.dataset_id}`, - ); - } else if (syncDecision.reason === "sandbox") { - console.info( - `[manifest-ready] Skipping nemar.org sync for sandbox dataset ${dataset.dataset_id}`, - ); - } - } else { - console.warn( - `[manifest-ready] dataset row not found for ${body.dataset_id}; skipping nemar.org sync`, - ); - } - } catch (err) { - console.error("[manifest-ready] nemar.org sync scheduling failed (non-fatal):", err); - } + c.executionCtx.waitUntil( + refreshDatasetMetadata(c.env, body.dataset_id).catch((err) => + console.error( + `[manifest-ready] metadata refresh failed for ${body.dataset_id} (non-fatal):`, + err, + ), + ), + ); } const fileCount = body.totals?.files ?? 0; diff --git a/backend/src/services/bids-tree.ts b/backend/src/services/bids-tree.ts new file mode 100644 index 000000000..46157e92f --- /dev/null +++ b/backend/src/services/bids-tree.ts @@ -0,0 +1,94 @@ +/** + * BIDS tree helpers — pure path/TSV parsers used by the dataset metadata + * pipeline (subject/session counts, task labels, participant age stats). + * + * Relocated from the deleted nemar-sync.ts (the legacy nemar.org push is gone, + * but these helpers feed `dataset-metadata-columns.ts`, `enrich-dataset.ts`, + * and `dataset-reindex.ts`). ZERO I/O; callers supply the tree paths / file content. + */ + +interface ParticipantStats { + count: number; + ageMin: number | null; + ageMax: number | null; +} + +/** + * Parse participants.tsv content into a participant count and age range. + * Rows with `n/a` ages are skipped for the min/max; the count is the row total. + */ +export function parseParticipantsTsv(content: string): ParticipantStats { + const lines = content.split("\n").filter((l) => l.trim()); + if (lines.length < 2) return { count: 0, ageMin: null, ageMax: null }; + + const headers = lines[0].split("\t").map((h) => h.trim().toLowerCase()); + const ageIdx = headers.indexOf("age"); + const rows = lines.slice(1); + + let ageMin: number | null = null; + let ageMax: number | null = null; + + if (ageIdx >= 0) { + for (const row of rows) { + const cols = row.split("\t"); + const raw = cols[ageIdx]?.trim(); + if (!raw || raw === "n/a" || raw === "N/A") continue; + const age = Number.parseFloat(raw); + if (!Number.isNaN(age)) { + ageMin = ageMin === null ? age : Math.min(ageMin, age); + ageMax = ageMax === null ? age : Math.max(ageMax, age); + } + } + } + + return { count: rows.length, ageMin, ageMax }; +} + +/** + * Count BIDS sessions as the number of distinct `ses-