diff --git a/AGENTS.md b/AGENTS.md index 646091c32..21852f07a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,10 +67,9 @@ The backend code (`ORG_NAME = "nemarDatasets"` in `backend/src/services/github.t ### Public Browser Sites Two user-facing dataset browsers exist during the cutover; **prefer ww2 for new references**. -- **`https://ww2.nemar.org`** — **current beta browser** (Astro SSR, lives in `nemarOrg/website`). Dataset pages at `ww2.nemar.org/dataset/`. Reads `api.nemar.org` and `data.nemar.org` directly; does not depend on the legacy datapipeline. -- **`https://nemar.org/dataexplorer/...`** — **legacy PHP site**. Backed by the `dataexplorer_*` tables on `nemar.org/api/dataexplorer/datapipeline`, which `nemar admin sync run` populates. Still public, slated for replacement by ww2. -- **Implication for on\* datasets:** `nemar admin sync` intentionally skips on\*-prefix datasets (alternate_id mapping pending, issue #339), so they render on ww2.nemar.org but the legacy `nemar.org/dataexplorer` detail page is empty until that ships. The archive zip download link in the legacy UI is driven by `zip_file_size` on the datapipeline row, so even an existing S3 archive won't surface there for on\* datasets. -- **Default in docs/comments:** when something says "the website," "the browser," or "the UI" without further qualification, mean ww2.nemar.org. Name `nemar.org` explicitly only when referring to the legacy dataexplorer, the datapipeline tables, or the `nemar.org` DOI landing target. +- **`https://ww2.nemar.org`** — **current dataset browser** (Astro SSR, lives in `nemarOrg/website`). Dataset pages at `ww2.nemar.org/dataset/`. Reads `api.nemar.org` and `data.nemar.org` directly. DOIs land on the canonical `https://nemar.org/dataset/` (versions `?v=v`), which a Cloudflare redirect on the `nemar.org` zone forwards to ww2. +- **`https://nemar.org/dataexplorer/...`** — **legacy PHP site, being retired.** As of epic #837 NEMAR no longer syncs to or from it: the outgoing datapipeline push and the incoming 4h catalog pull are removed, our `nm`/`on` records are purged from its `dataexplorer_*` tables, and the legacy `ds######` shadow rows are dropped from our D1. It may still serve its own OpenNeuro `ds` catalog independently. +- **Default in docs/comments:** when something says "the website," "the browser," or "the UI" without further qualification, mean ww2.nemar.org. Name `nemar.org` explicitly only when referring to the legacy dataexplorer or the `nemar.org/dataset/` DOI landing target. ### S3 Bucket Structure ``` @@ -115,7 +114,7 @@ Datasets are synced to SDSC Hallu (`ssh hallu`) for processing pipelines and dow - **Zip dir:** `/data/qumulo/openneuro/zip_files/` (downloadable archives from S3) - **Discovery:** Queries `GET /datasets` API, filters for `nm`-prefix only - **Manual run:** `ssh hallu /data/qumulo/openneuro/nemar-cli/scripts/hallu-sync.sh --dataset nm000132 --verbose` -- **Note:** This syncs data files. Legacy `nemar.org/dataexplorer` metadata is synced separately via `nemar admin sync run` or automatically on version DOI publish. +- **Note:** This syncs data files only. (The legacy `nemar.org/dataexplorer` metadata sync was retired in epic #837.) ## Environment Setup ```bash @@ -412,9 +411,7 @@ nemar admin revert # Revert dataset to previous version nemar admin make-public # Publish dataset (permanent) nemar admin delete-dataset # Delete dataset and all resources nemar admin bulk-delete # Delete multiple phantom/orphaned datasets (owner only) -nemar admin sync run # Sync dataset metadata to nemar.org (legacy dataexplorer) -nemar admin sync status # Show nemar.org sync status -nemar admin reindex # Refresh enrichment + nemar.org sync + D1 metadata columns +nemar admin reindex # Refresh enrichment + D1 metadata columns nemar admin reindex --all # Bulk; also --missing-metadata, --stale [--older-than N] nemar admin summary # summary.json coverage across published versions nemar admin email-preferences show # Show email notification preferences diff --git a/backend/src/db/migrations/0053_drop_legacy_sync_state.sql b/backend/src/db/migrations/0053_drop_legacy_sync_state.sql new file mode 100644 index 000000000..fdfc85a6c --- /dev/null +++ b/backend/src/db/migrations/0053_drop_legacy_sync_state.sql @@ -0,0 +1,17 @@ +-- Drop the legacy nemar.org datapipeline sync state (epic #837 Phase 5). +-- +-- The outgoing datapipeline push (nemar-sync.ts) and the incoming catalog pull +-- (catalog-sync.ts + /admin/catalog/sync) were removed in Phase 3, so the +-- nemar_sync_* columns (migration 0014) and the catalog_sync_log table +-- (migration 0018) are now dead and have no remaining readers/writers. +-- +-- DROP COLUMN is safe here: no index, trigger, FK, or the datasets_fts external +-- -content virtual table references these columns (datasets_fts indexes only +-- name/description/authors/tasks/modalities/readme). Validated on the SCCN dev +-- D1 mirror before prod. + +ALTER TABLE datasets DROP COLUMN nemar_sync_status; +ALTER TABLE datasets DROP COLUMN nemar_sync_at; +ALTER TABLE datasets DROP COLUMN nemar_sync_error; + +DROP TABLE IF EXISTS catalog_sync_log; diff --git a/backend/src/routes/datasets.ts b/backend/src/routes/datasets.ts index 6e330bb53..06ebab284 100644 --- a/backend/src/routes/datasets.ts +++ b/backend/src/routes/datasets.ts @@ -707,7 +707,7 @@ datasetRoutes.get("/", optionalAuthMiddleware, async (c) => { let query = ` SELECT d.dataset_id, d.name, d.description, d.status, d.visibility, d.github_repo, d.concept_doi, d.created_at, d.updated_at, - u.username AS owner_username, d.nemar_sync_status, + u.username AS owner_username, d.source, d.source_id, COALESCE(d.modalities, '') AS modalities, COALESCE(d.subject_count, 0) AS participants, @@ -754,7 +754,7 @@ datasetRoutes.get("/", optionalAuthMiddleware, async (c) => { let query = ` SELECT d.dataset_id, d.dataset_id AS id, d.name, d.description, d.status, d.visibility, d.github_repo, d.concept_doi, d.concept_doi AS doi, d.created_at, d.updated_at, - COALESCE(d.uploader, u.username) AS owner_username, d.nemar_sync_status, + COALESCE(d.uploader, u.username) AS owner_username, d.source, d.source_id, COALESCE(d.modalities, '') AS modalities, COALESCE(d.subject_count, 0) AS participants, diff --git a/src/commands/dataset.ts b/src/commands/dataset.ts index 240fa2938..7a20b8619 100644 --- a/src/commands/dataset.ts +++ b/src/commands/dataset.ts @@ -2331,16 +2331,6 @@ function renderDatasetTable( const subjects = dataset.participants ? String(dataset.participants) : "-"; const owner = dataset.owner_username || "-"; - // Discrepancy indicators: - // * = managed (on GitHub) but not synced to nemar.org - // italic/dim = catalog-only (on nemar.org but not on GitHub) - let statusIndicator = ""; - if (dataset.source_type === "managed" && !dataset.nemar_sync_status) { - statusIndicator = chalk.yellow("*"); // On GitHub, not synced - } else if (dataset.source_type === "managed" && dataset.nemar_sync_status === "failed") { - statusIndicator = chalk.red("!"); // Sync failed - } - const visLabel = dataset.visibility === "public" ? chalk.green("pub") : chalk.yellow("prv"); let idDisplay: string; @@ -2361,7 +2351,7 @@ function renderDatasetTable( modality.padEnd(modWidth), subjects.padEnd(subjWidth), owner.padEnd(ownerWidth), - `${visLabel} ${statusIndicator}`.trim(), + visLabel, ].join(" "); console.log(row); } diff --git a/src/lib/api.ts b/src/lib/api.ts index c8be6ad4e..00a8815b7 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -647,8 +647,6 @@ export interface Dataset { file_size_formatted?: string; /** 'managed' = in D1 datasets table, 'catalog' = nemar.org only */ source_type?: "managed" | "catalog"; - /** nemar.org sync status for managed datasets */ - nemar_sync_status?: string | null; /** DOI field from catalog (for catalog-only datasets) */ doi?: string | null; /** Import source (e.g. "openneuro") */ diff --git a/test/license-tier.test.ts b/test/license-tier.test.ts index edf289f69..53620cb05 100644 --- a/test/license-tier.test.ts +++ b/test/license-tier.test.ts @@ -118,7 +118,7 @@ CREATE TABLE datasets ( created_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')), is_sandbox INTEGER NOT NULL DEFAULT 0, visibility TEXT NOT NULL DEFAULT 'private' CHECK (visibility IN ('private','public')), - enrichment_json TEXT, source TEXT, source_id TEXT, nemar_sync_status TEXT, + enrichment_json TEXT, source TEXT, source_id TEXT, subject_count INTEGER, modalities TEXT, age_min REAL, age_max REAL, file_size INTEGER, total_files INTEGER, tasks TEXT, FOREIGN KEY (owner_user_id) REFERENCES users(id)