diff --git a/AGENTS.md b/AGENTS.md index 93916b6c7..cf0039ede 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -512,11 +512,11 @@ and what is historical. The entries worth knowing by name: | Group | Covers | |---|---| | `nemar auth` | login, signup, status/whoami, switch, logout, verification, SSH setup, key retrieval and regeneration | -| `nemar dataset` | validate, upload, download, status, list, search, release, update, clone, get, commit, push, drop, ci, manifest | +| `nemar dataset` | validate, upload, download, status (alias: view), list, search, release, update, clone, get, commit, push, drop, ci, manifest | | `nemar dataset publish` | request, status, resend | | `nemar dataset` (access) | request-access, access, invite, collaborators | | `nemar sandbox` | training run, status, reset — required before uploading | -| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, zarr-fidelity-sweep, doctor, summary, notice, email-preferences, e2e-test | +| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, recording-stats-sweep, signal-defaults-sweep, zarr-fidelity-sweep, doctor, summary, notice, email-preferences, e2e-test | | `nemar admin import*` | OpenNeuro import, status, rollback, retry, verify, recover (issue #754, epic #967) | | `nemar admin fleet` | drift, enforce, revalidate — governance across dataset repos (epic #713) | | `nemar admin exemplar` | create, status, remint-dois — the staging exemplar fleet | diff --git a/package.json b/package.json index f338437c1..320b4bde4 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ "files": [ "dist", "scripts/postinstall-check.mjs", + "scripts/recover-datasets.json", + "scripts/withdrawn-datasets.json", "README.md", "LICENSE" ], diff --git a/src/commands/admin.ts b/src/commands/admin.ts index 84671a9f4..ab3a49df3 100644 --- a/src/commands/admin.ts +++ b/src/commands/admin.ts @@ -26,7 +26,7 @@ import { join } from "node:path"; import chalk from "chalk"; import { Command } from "commander"; import inquirer from "inquirer"; -import ora from "ora"; +import ora, { type Ora } from "ora"; import { type AvailabilityReport, type AvailabilityReportResult, @@ -38,11 +38,13 @@ import { type DoctorScanResponse, type EmailPreferences, type HedSweepBatchResponse, + type RecordingStatsSweepBatchResponse, type ReindexBulkOptions, type ReindexBulkResponse, type ReindexFilter, type ReindexOptions, type ReindexResponse, + type SignalDefaultsSweepBatchResponse, type SummaryVersionCoverage, type ZarrFidelitySweepBatchResponse, addCi, @@ -76,6 +78,8 @@ import { listUsers, publishDataset, publishZarrCatalog, + recordingStatsSweep, + recordingStatsSweepReset, reindexBulk, reindexDataset, remintExemplarDois, @@ -85,6 +89,8 @@ import { revokeUser, rollbackImport, sendBroadcast, + signalDefaultsSweep, + signalDefaultsSweepReset, syncCi, updateDoi, updateEmailPreferences, @@ -2874,9 +2880,14 @@ adminCommand /** Default location of the checked-in withdrawal target list, resolved * relative to this source file rather than cwd (mirrors - * defaultExemplarFleetPath below). */ + * defaultExemplarFleetPath below). Shipped to npm (package.json "files"), + * same as defaultRecoverDatasetsPath below -- see that function's comment + * for why a single hardcoded depth breaks between a source checkout and + * the published, single-file dist/index.js bundle (#1049 review). */ function defaultWithdrawnDatasetsPath(): string { - return join(import.meta.dir, "..", "..", "scripts", "withdrawn-datasets.json"); + const fromSourceTree = join(import.meta.dir, "..", "..", "scripts", "withdrawn-datasets.json"); + if (existsSync(fromSourceTree)) return fromSourceTree; + return join(import.meta.dir, "..", "scripts", "withdrawn-datasets.json"); } function printTransitionResult( @@ -3616,9 +3627,23 @@ adminCommand.addCommand(importCommand); /** Default location of the checked-in recovery target list, resolved * relative to this source file rather than cwd (mirrors - * defaultWithdrawnDatasetsPath). */ + * defaultWithdrawnDatasetsPath, which shares this same shape). Shipped + * to npm (package.json "files") so `nemar admin recover` works from an + * installed build, not just a repo checkout -- the two layouts put this + * file at different depths from `import.meta.dir` (#1049): + * - source (`bun run src/index.ts`): this code lives at src/commands/, so + * the repo-root scripts/ dir is two levels up. + * - published (`dist/index.js`, a single-file bundle from `bun build`): + * this code lives at dist/, so the package's scripts/ dir is only ONE + * level up -- a hardcoded "two levels up" instead resolved to a path + * outside the installed package entirely and 404'd on every real run. + * Try the source-tree depth first (existsSync), then fall back to the + * bundled depth; a repo checkout always has the former, an npm install + * always has the latter, and only ever one of the two exists on disk. */ function defaultRecoverDatasetsPath(): string { - return join(import.meta.dir, "..", "..", "scripts", "recover-datasets.json"); + const fromSourceTree = join(import.meta.dir, "..", "..", "scripts", "recover-datasets.json"); + if (existsSync(fromSourceTree)) return fromSourceTree; + return join(import.meta.dir, "..", "scripts", "recover-datasets.json"); } const recoverCommand = new Command("recover").description( @@ -4902,6 +4927,58 @@ const availabilityReportCommand = new Command("availability-report") adminCommand.addCommand(availabilityReportCommand); +/** Batch response shape every `nemar admin *-sweep` command shares (PR + * #1223 review, suggestion 5). */ +interface SweepBatchLike { + remaining: number | null; + errors: { dataset_id: string; error: string }[]; +} + +/** + * Shared do-while loop for hed-sweep / recording-stats-sweep / + * signal-defaults-sweep: calls `runBatch()` until `remaining` reaches 0, + * paces GitHub-heavy batches, and bails with a warning if `remaining` + * stops strictly decreasing (persistent write errors leave rows + * unstamped). `onBatch` owns everything sweep-specific -- totals + * accumulation and the verbose/spinner text -- so this extraction changes + * no command's actual output. A `runBatch` throw is caught here exactly + * as each call site did before: `spinner.fail(failLabel)` + the error + * detail + `process.exit(1)`. + */ +async function runSweepBatchLoop( + spinner: Ora, + failLabel: string, + runBatch: () => Promise, + onBatch: (res: R, batch: number, remaining: number | null) => void, +): Promise<{ batches: number; remaining: number | null }> { + let batch = 0; + let remaining: number | null = null; + let prevRemaining: number | null = null; + try { + let res: R; + do { + res = await runBatch(); + batch++; + remaining = res.remaining; + onBatch(res, batch, remaining); + + if (remaining != null && prevRemaining != null && remaining >= prevRemaining) { + spinner.warn(`No progress (remaining stuck at ${remaining}); stopping - see errors`); + break; + } + prevRemaining = remaining; + + if ((remaining ?? 0) > 0) await sleep(1000); // pace the GitHub-heavy batches + } while ((remaining ?? 0) > 0); + spinner.stop(); + } catch (err) { + spinner.fail(failLabel); + console.error(chalk.red(errorDetail(err))); + process.exit(1); + } + return { batches: batch, remaining }; +} + const hedSweepCommand = new Command("hed-sweep").description( "Backfill HED detection (has_hed / hed_version) for existing datasets (#869)", ); @@ -4935,51 +5012,112 @@ hedSweepCommand const totals = { processed: 0, withHed: 0, withoutHed: 0, unknown: 0, errors: 0 }; const spinner = ora("Sweeping datasets for HED...").start(); - let batch = 0; - let remaining: number | null = null; - let prevRemaining: number | null = null; - try { - let res: HedSweepBatchResponse; - do { - res = await hedSweep({ limit }); - batch++; + const { batches: batch, remaining } = await runSweepBatchLoop( + spinner, + "HED sweep failed", + () => hedSweep({ limit }), + (res, batchNum, rem) => { totals.processed += res.processed; totals.withHed += res.withHed; totals.withoutHed += res.withoutHed; totals.unknown += res.unknown; totals.errors += res.errors.length; - remaining = res.remaining; if (options.verbose) { spinner.stop(); console.log( - ` [batch ${batch}] processed=${res.processed} hed=${res.withHed} no-hed=${res.withoutHed} unknown=${res.unknown} errors=${res.errors.length} remaining=${remaining ?? "?"}`, + ` [batch ${batchNum}] processed=${res.processed} hed=${res.withHed} no-hed=${res.withoutHed} unknown=${res.unknown} errors=${res.errors.length} remaining=${rem ?? "?"}`, ); for (const e of res.errors) console.log(` ${chalk.red(e.dataset_id)}: ${e.error}`); spinner.start("Sweeping datasets for HED..."); } else { - spinner.text = `Sweeping datasets for HED... ${totals.processed} processed, ${remaining ?? "?"} remaining`; + spinner.text = `Sweeping datasets for HED... ${totals.processed} processed, ${rem ?? "?"} remaining`; } + }, + ); - // Progress guard: hed_checked_at is stamped for every processed row, so - // `remaining` must strictly decrease. If it doesn't (e.g. persistent D1 - // write errors leave rows unstamped), bail instead of looping forever. - if (remaining != null && prevRemaining != null && remaining >= prevRemaining) { - spinner.warn(`No progress (remaining stuck at ${remaining}); stopping - see errors`); - break; - } - prevRemaining = remaining; + if (options.json) { + console.log(JSON.stringify({ ...totals, batches: batch, remaining }, null, 2)); + } else { + console.log(); + if (remaining === null) { + console.log( + chalk.yellow( + "Warning: backend returned a null remaining count; the sweep may be incomplete - check server logs.", + ), + ); + } + console.log( + chalk.cyan( + `Done in ${batch} batch(es): processed=${totals.processed} hed=${totals.withHed} no-hed=${totals.withoutHed} unknown=${totals.unknown} errors=${totals.errors} remaining=${remaining ?? "unknown"}`, + ), + ); + } + // Non-zero exit on errors OR an indeterminate (null) remaining, so a caller + // never reads a partial/uncertain sweep as success. + if (totals.errors > 0 || remaining === null) process.exit(1); + }, + ); - if ((remaining ?? 0) > 0) await sleep(1000); // pace the GitHub-heavy batches - } while ((remaining ?? 0) > 0); +adminCommand.addCommand(hedSweepCommand); - spinner.stop(); - } catch (err) { - spinner.fail("HED sweep failed"); - console.error(chalk.red(errorDetail(err))); - process.exit(1); +const recordingStatsSweepCommand = new Command("recording-stats-sweep").description( + "Backfill dataset-level recording duration/count/channel-range stats from each dataset's zarr index (migration 0070, #1146)", +); + +recordingStatsSweepCommand + .option("--limit ", "Datasets per batch (server clamps to [1,200])", "50") + .option("--reset", "Clear every stamped recording-stats row so it re-sweeps from scratch") + .option("--verbose", "Print per-batch progress") + .option("--json", "Output raw JSON instead of the human summary") + .action( + async (options: { limit?: string; reset?: boolean; verbose?: boolean; json?: boolean }) => { + if (!requireAuth()) return; + + if (options.reset) { + const spinner = ora("Resetting recording-stats sweep state...").start(); + try { + const r = await recordingStatsSweepReset(); + spinner.succeed( + `Cleared ${r.reset} stamped recording-stats row(s); eligible datasets are candidates again`, + ); + if (options.json) console.log(JSON.stringify(r, null, 2)); + } catch (err) { + spinner.fail("Reset failed"); + console.error(chalk.red(errorDetail(err))); + process.exit(1); + } + return; } + const limit = Number.parseInt(options.limit ?? "50", 10) || 50; + const totals = { processed: 0, measured: 0, unmeasured: 0, errors: 0 }; + const spinner = ora("Sweeping datasets for recording stats...").start(); + + const { batches: batch, remaining } = + await runSweepBatchLoop( + spinner, + "Recording-stats sweep failed", + () => recordingStatsSweep({ limit }), + (res, batchNum, rem) => { + totals.processed += res.processed; + totals.measured += res.measured; + totals.unmeasured += res.unmeasured; + totals.errors += res.errors.length; + + if (options.verbose) { + spinner.stop(); + console.log( + ` [batch ${batchNum}] processed=${res.processed} measured=${res.measured} unmeasured=${res.unmeasured} errors=${res.errors.length} remaining=${rem ?? "?"}`, + ); + for (const e of res.errors) console.log(` ${chalk.red(e.dataset_id)}: ${e.error}`); + spinner.start("Sweeping datasets for recording stats..."); + } else { + spinner.text = `Sweeping datasets for recording stats... ${totals.processed} processed, ${rem ?? "?"} remaining`; + } + }, + ); + if (options.json) { console.log(JSON.stringify({ ...totals, batches: batch, remaining }, null, 2)); } else { @@ -4993,7 +5131,7 @@ hedSweepCommand } console.log( chalk.cyan( - `Done in ${batch} batch(es): processed=${totals.processed} hed=${totals.withHed} no-hed=${totals.withoutHed} unknown=${totals.unknown} errors=${totals.errors} remaining=${remaining ?? "unknown"}`, + `Done in ${batch} batch(es): processed=${totals.processed} measured=${totals.measured} unmeasured=${totals.unmeasured} errors=${totals.errors} remaining=${remaining ?? "unknown"}`, ), ); } @@ -5003,7 +5141,89 @@ hedSweepCommand }, ); -adminCommand.addCommand(hedSweepCommand); +adminCommand.addCommand(recordingStatsSweepCommand); + +const signalDefaultsSweepCommand = new Command("signal-defaults-sweep").description( + "Backfill BIDS signal defaults (sampling frequency, power line frequency, EEG reference, placement scheme) from each dataset's exemplar sidecar (migrations 0072/0073, #1153)", +); + +signalDefaultsSweepCommand + .option("--limit ", "Datasets per batch (server clamps to [1,30])", "15") + .option("--reset", "Clear every stamped signal-defaults row so it re-sweeps from scratch") + .option("--verbose", "Print per-batch progress") + .option("--json", "Output raw JSON instead of the human summary") + .action( + async (options: { limit?: string; reset?: boolean; verbose?: boolean; json?: boolean }) => { + if (!requireAuth()) return; + + if (options.reset) { + const spinner = ora("Resetting signal-defaults sweep state...").start(); + try { + const r = await signalDefaultsSweepReset(); + spinner.succeed( + `Cleared ${r.reset} stamped signal-defaults row(s); eligible datasets are candidates again`, + ); + if (options.json) console.log(JSON.stringify(r, null, 2)); + } catch (err) { + spinner.fail("Reset failed"); + console.error(chalk.red(errorDetail(err))); + process.exit(1); + } + return; + } + + const limit = Number.parseInt(options.limit ?? "15", 10) || 15; + const totals = { processed: 0, populated: 0, noData: 0, errors: 0 }; + const spinner = ora("Sweeping datasets for signal defaults...").start(); + + const { batches: batch, remaining } = + await runSweepBatchLoop( + spinner, + "Signal-defaults sweep failed", + () => signalDefaultsSweep({ limit }), + (res, batchNum, rem) => { + totals.processed += res.processed; + totals.populated += res.populated; + totals.noData += res.noData; + totals.errors += res.errors.length; + + if (options.verbose) { + spinner.stop(); + console.log( + ` [batch ${batchNum}] processed=${res.processed} populated=${res.populated} no-data=${res.noData} errors=${res.errors.length} remaining=${rem ?? "?"}`, + ); + for (const e of res.errors) console.log(` ${chalk.red(e.dataset_id)}: ${e.error}`); + spinner.start("Sweeping datasets for signal defaults..."); + } else { + spinner.text = `Sweeping datasets for signal defaults... ${totals.processed} processed, ${rem ?? "?"} remaining`; + } + }, + ); + + if (options.json) { + console.log(JSON.stringify({ ...totals, batches: batch, remaining }, null, 2)); + } else { + console.log(); + if (remaining === null) { + console.log( + chalk.yellow( + "Warning: backend returned a null remaining count; the sweep may be incomplete - check server logs.", + ), + ); + } + console.log( + chalk.cyan( + `Done in ${batch} batch(es): processed=${totals.processed} populated=${totals.populated} no-data=${totals.noData} errors=${totals.errors} remaining=${remaining ?? "unknown"}`, + ), + ); + } + // Non-zero exit on errors OR an indeterminate (null) remaining, so a caller + // never reads a partial/uncertain sweep as success. + if (totals.errors > 0 || remaining === null) process.exit(1); + }, + ); + +adminCommand.addCommand(signalDefaultsSweepCommand); /** One data-integrity-sweep batch, as consumed by {@link runReauditLoop}. */ export interface ReauditBatchResult { diff --git a/src/commands/dataset.ts b/src/commands/dataset.ts index 5412f3879..9c59d0273 100644 --- a/src/commands/dataset.ts +++ b/src/commands/dataset.ts @@ -1391,7 +1391,8 @@ Examples: // Status command datasetCommand .command("status") - .description("Check status of a dataset") + .alias("view") + .description("Check status of a dataset (alias: view)") .argument("", "Dataset ID (e.g., nm000104)") .option("--json", "Output as JSON for scripting") .addHelpText( diff --git a/src/lib/api/admin.ts b/src/lib/api/admin.ts index 0d7f06132..a8f66c130 100644 --- a/src/lib/api/admin.ts +++ b/src/lib/api/admin.ts @@ -801,6 +801,84 @@ export async function hedSweepReset(): Promise { ); } +/** One batch of the recording-stats backfill sweep (migration 0070, epic + * #1144 Phase 2, issue #1146, `POST /admin/datasets/recording-stats-sweep`). */ +export interface RecordingStatsSweepBatchResponse { + processed: number; + /** Candidates whose zarr index yielded at least one measured recording. */ + measured: number; + /** Candidates whose zarr index existed but had no measured recordings. */ + unmeasured: number; + errors: { dataset_id: string; error: string }[]; + /** Datasets still unswept; 0 when done, null if the count query failed. */ + remaining: number | null; +} + +/** Response of `?reset=1`: count of stamped rows cleared back to unswept. */ +export interface RecordingStatsSweepResetResponse { + reset: number; +} + +/** Run one bounded recording-stats sweep batch (default 50, server-clamped to [1,200]). */ +export async function recordingStatsSweep(options?: { + limit?: number; +}): Promise { + const limit = options?.limit ?? 50; + return request( + `/admin/datasets/recording-stats-sweep?limit=${encodeURIComponent(String(limit))}`, + { method: "POST", headers: { "Content-Type": "application/json" } }, + true, + ); +} + +/** Clear every stamped recording-stats row so a corrected aggregator can re-sweep from scratch. */ +export async function recordingStatsSweepReset(): Promise { + return request( + "/admin/datasets/recording-stats-sweep?reset=1", + { method: "POST", headers: { "Content-Type": "application/json" } }, + true, + ); +} + +/** One batch of the signal-defaults backfill sweep (migrations 0072/0073, epic + * #1144 Phase 2b, issue #1153, `POST /admin/datasets/signal-defaults-sweep`). */ +export interface SignalDefaultsSweepBatchResponse { + processed: number; + /** Candidates whose probe found at least one usable sidecar key and wrote it. */ + populated: number; + /** Candidates whose probe completed but found nothing to write. */ + noData: number; + errors: { dataset_id: string; error: string }[]; + /** Datasets still unswept; 0 when done, null if the count query failed. */ + remaining: number | null; +} + +/** Response of `?reset=1`: count of stamped rows cleared back to unswept. */ +export interface SignalDefaultsSweepResetResponse { + reset: number; +} + +/** Run one bounded signal-defaults sweep batch (default 15, server-clamped to [1,30]). */ +export async function signalDefaultsSweep(options?: { + limit?: number; +}): Promise { + const limit = options?.limit ?? 15; + return request( + `/admin/datasets/signal-defaults-sweep?limit=${encodeURIComponent(String(limit))}`, + { method: "POST", headers: { "Content-Type": "application/json" } }, + true, + ); +} + +/** Clear every stamped signal-defaults row so a corrected probe can re-sweep from scratch. */ +export async function signalDefaultsSweepReset(): Promise { + return request( + "/admin/datasets/signal-defaults-sweep?reset=1", + { method: "POST", headers: { "Content-Type": "application/json" } }, + true, + ); +} + /** One batch of the data-integrity sweep (epic #967 Phase 3, #970, * `POST /admin/datasets/data-integrity-sweep`). */ export interface DataIntegritySweepBatchResponse { diff --git a/src/lib/upload/transfer.ts b/src/lib/upload/transfer.ts index 03bd825db..25c5c7de1 100644 --- a/src/lib/upload/transfer.ts +++ b/src/lib/upload/transfer.ts @@ -399,30 +399,67 @@ export async function uploadDataToS3( writeUploadProgress(absolutePath, progress); } - if (!isStepCompleted(progress, "s3_upload")) { - // Reconcile against ACTUAL git state, not just the progress file: the - // progress file can survive while .git does not match it (#884 review). - // Cheap index read; no content access. - let trackedPaths: Set; - try { - trackedPaths = await listTrackedPaths(absolutePath); - } catch (indexError) { - console.log(chalk.red(`Failed to read the git index: ${errorDetail(indexError)}`)); - console.log(chalk.dim(" Re-run the command to retry.")); - return FAIL; - } - const addTargets = computeAddTargets(filesToUpload, dataFiles, trackedPaths); - const untrackedCount = addTargets.length - filesToUpload.length; - if (untrackedCount > 0 && isStepCompleted(progress, "tracking")) { - clearStepCompleted(progress, "tracking"); - writeUploadProgress(absolutePath, progress); - console.log( - chalk.yellow( - ` ${untrackedCount} data files are missing from git despite recorded progress; re-tracking them`, - ), - ); - } + // Reconcile against ACTUAL git state, not just the progress file: the + // progress file can survive while .git does not match it (#884 review). + // Cheap index read; no content access -- runs even when "s3_upload" is + // already marked complete (#1070). A prior run's completion stamp must + // not hide data files added to the dataset since then: filesToUpload + // (computed by the caller from the current file list) can be non-empty + // even though "s3_upload" is stamped, and computeAddTargets can also + // surface git-untracked files the stamp knows nothing about. Either way, + // reopening the gate below is what lets a resumed upload actually pick + // those files up instead of silently skipping them forever. + let trackedPaths: Set; + try { + trackedPaths = await listTrackedPaths(absolutePath); + } catch (indexError) { + console.log(chalk.red(`Failed to read the git index: ${errorDetail(indexError)}`)); + console.log(chalk.dim(" Re-run the command to retry.")); + return FAIL; + } + const addTargets = computeAddTargets(filesToUpload, dataFiles, trackedPaths); + const untrackedCount = addTargets.length - filesToUpload.length; + const reopenTracking = untrackedCount > 0 && isStepCompleted(progress, "tracking"); + const reopenUpload = addTargets.length > 0 && isStepCompleted(progress, "s3_upload"); + if (reopenTracking) { + clearStepCompleted(progress, "tracking"); + } + if (reopenUpload) { + clearStepCompleted(progress, "s3_upload"); + // Reopening s3_upload for new content must also reopen the finalize + // steps that depend on it (Step 11 saveDatasetStep / Step 12 + // pushMetadata in commands/dataset.ts, upload/finalize.ts). Both are + // gated by their own persisted step and stay stamped complete from + // the PRIOR run, so without this a new file's git-annex pointer would + // reach S3 here but never be committed or pushed -- and + // printUploadSuccess still clears progress and reports "Upload + // complete!" with that pointer sitting only in the working tree + // (review finding, critical: a run interrupted between pushMetadata + // and printUploadSuccess -- e.g. mid deployCiStep -- leaves both + // stamped true on disk with progress never cleared). + clearStepCompleted(progress, "dataset_save"); + clearStepCompleted(progress, "github_push"); + } + // One combined write instead of one per reopened step. + if (reopenTracking || reopenUpload) { + writeUploadProgress(absolutePath, progress); + } + if (reopenTracking) { + console.log( + chalk.yellow( + ` ${untrackedCount} data files are missing from git despite recorded progress; re-tracking them`, + ), + ); + } + if (reopenUpload) { + console.log( + chalk.yellow( + ` ${addTargets.length} data file(s) still need upload despite a completed s3_upload step; resuming`, + ), + ); + } + if (!isStepCompleted(progress, "s3_upload")) { if (addTargets.length > 0) { // Track data files with git-annex before anything else: the add can // take hours on multi-TB datasets, so it must not eat into the 2h STS diff --git a/test/api-export-surface.unit.test.ts b/test/api-export-surface.unit.test.ts index 6c4dbaa8a..1daff47ad 100644 --- a/test/api-export-surface.unit.test.ts +++ b/test/api-export-surface.unit.test.ts @@ -121,6 +121,8 @@ const MODULE_EXPORTS: Record = { "publishDataset", "publishVersionDoi", "publishZarrCatalog", + "recordingStatsSweep", + "recordingStatsSweepReset", "reindexBulk", "reindexDataset", "remintExemplarDois", @@ -131,6 +133,8 @@ const MODULE_EXPORTS: Record = { "revokeUser", "rollbackImport", "sendBroadcast", + "signalDefaultsSweep", + "signalDefaultsSweepReset", "syncCi", "updateDoi", "updateEmailPreferences", @@ -166,6 +170,10 @@ const POST_SPLIT_ADDITIONS = [ "doctorScan", // #1130, CLI wrapper for POST /admin/doctor/scan "getFacets", // #1149, epic #1144 phase 5b: GET /datasets/facets for shell completion "publishZarrCatalog", // #1062, epic #1181 phase 2: POST /admin/zarr-catalog/publish + "recordingStatsSweep", // #1194, CLI wrapper for POST /admin/datasets/recording-stats-sweep + "recordingStatsSweepReset", // #1194, CLI wrapper for the recording-stats sweep reset + "signalDefaultsSweep", // #1194, CLI wrapper for POST /admin/datasets/signal-defaults-sweep + "signalDefaultsSweepReset", // #1194, CLI wrapper for the signal-defaults sweep reset "zarrFidelitySweep", // #1068, epic #1181 phase 8: POST /admin/datasets/zarr-fidelity-sweep ]; diff --git a/test/cli.test.ts b/test/cli.test.ts index 3fa6799e8..92df44189 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -686,6 +686,36 @@ describe("CLI Dataset Status", () => { expect(result.dataset_id).toBe("nm099999"); expect(result.name).toBeDefined(); }); + + test("nemar dataset view is an alias for dataset status", async () => { + const { stdout: viewHelp, exitCode: viewExit } = await runCli(["dataset", "view", "--help"]); + const { stdout: statusHelp, exitCode: statusExit } = await runCli([ + "dataset", + "status", + "--help", + ]); + + expect(viewExit).toBe(0); + expect(statusExit).toBe(0); + // Both resolve to the same command implementation, so their help text + // (description, options, examples) is byte-identical. + expect(viewHelp).toContain("Check status of a dataset"); + expect(viewHelp).toContain("--json"); + expect(viewHelp).toBe(statusHelp); + }); + + test("nemar dataset --help lists view as an alias of status", async () => { + const { stdout, exitCode } = await runCli(["dataset", "--help"]); + + expect(exitCode).toBe(0); + expect(stdout).toContain("status|view"); + }); + + test("nemar dataset view with non-existent dataset behaves like status", async () => { + const { stdout, exitCode } = await runCli(["dataset", "view", "nm099998"]); + + expect(stdout).toContain("not found"); + }); }); describe("CLI Dataset List", () => { diff --git a/test/curl-stream-copy.unit.test.ts b/test/curl-stream-copy.unit.test.ts index c10088ccf..e2f4902a2 100644 --- a/test/curl-stream-copy.unit.test.ts +++ b/test/curl-stream-copy.unit.test.ts @@ -63,7 +63,12 @@ beforeAll(() => { afterAll(() => { server.stop(true); - process.env.PATH = previousPath; + // Guarded restore (#1175): assigning `undefined` to a process.env key + // coerces to the literal string "undefined" instead of deleting it, which + // would poison PATH for every test running later in the same `bun test` + // process (test/ + backend/test/ share one process at the root). + if (previousPath === undefined) delete process.env.PATH; + else process.env.PATH = previousPath; rmSync(binDir, { recursive: true, force: true }); }); diff --git a/test/hed-sweep-cli.test.ts b/test/hed-sweep-cli.test.ts new file mode 100644 index 000000000..38769b4c6 --- /dev/null +++ b/test/hed-sweep-cli.test.ts @@ -0,0 +1,295 @@ +/** + * CLI `nemar admin hed-sweep` (#869), driven through the real entry point + * (`bun run src/index.ts admin hed-sweep ...`). + * + * PR #1223 review, suggestion 5: hed-sweep's batch-loop control flow (the + * do-while loop, progress guard, GitHub-pacing sleep, and the try/catch + * around it) is now shared with recording-stats-sweep and + * signal-defaults-sweep via `runSweepBatchLoop` in src/commands/admin.ts. + * hed-sweep itself had no CLI-level test asserting its exact output before + * that extraction, so this file closes that gap and doubles as the + * regression guard for the shared helper's hed-sweep call site -- mirrors + * test/recording-signal-sweeps-cli.test.ts's harness exactly: a real + * subprocess CLI invocation pointed at a real local HTTP server via + * TEST_API_URL (src/lib/api/client.ts#getApiUrl), no mocks. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "bun"; + +const CLI_ENTRY = join(import.meta.dir, "..", "src", "index.ts"); +const REPO_ROOT = join(import.meta.dir, ".."); + +interface CaptureServer { + url: string; + requests: URL[]; + stop: () => void; +} + +function startCaptureServer(responses: unknown[], status = 200): CaptureServer { + const requests: URL[] = []; + let i = 0; + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/notices") { + return new Response(JSON.stringify({ notices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.pathname === "/datasets/facets") { + return new Response(JSON.stringify({}), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + requests.push(url); + const body = responses[Math.min(i, responses.length - 1)]; + i++; + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }); + return { + url: `http://localhost:${server.port}`, + requests, + stop: () => server.stop(true), + }; +} + +let configDir: string; + +function seedAuthenticatedConfig(): void { + writeFileSync( + join(configDir, "config.json"), + JSON.stringify({ + activeAccount: "hedsweepcliuser", + accounts: { hedsweepcliuser: { apiKey: "test-admin-key" } }, + }), + ); +} + +beforeEach(() => { + configDir = mkdtempSync(join(tmpdir(), "nemar-hed-sweep-cli-")); +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); +}); + +async function runCli( + args: string[], + testApiUrl: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const env = { + ...process.env, + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: testApiUrl, + NEMAR_NO_UPDATE_CHECK: "1", + NO_COLOR: "1", + }; + env.FORCE_COLOR = undefined; + env.CLICOLOR_FORCE = undefined; + const proc = spawn({ + cmd: ["bun", "run", CLI_ENTRY, ...args], + cwd: REPO_ROOT, + env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const exitCode = await proc.exited; + return { stdout, stderr, exitCode }; +} + +const HED_SWEEP_DONE = { + processed: 1, + withHed: 1, + withoutHed: 0, + unknown: 0, + errors: [], + remaining: 0, +}; + +describe("nemar admin hed-sweep: auth", () => { + test("without a stored apiKey, requireAuth blocks before any request is sent", async () => { + const server = startCaptureServer([HED_SWEEP_DONE]); + try { + const result = await runCli(["admin", "hed-sweep"], server.url); + expect(result.stdout).toContain("Not authenticated"); + expect(server.requests.length).toBe(0); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin hed-sweep: request shape", () => { + test("--limit is forwarded as ?limit=N on the real request", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([HED_SWEEP_DONE]); + try { + const result = await runCli(["admin", "hed-sweep", "--limit", "10", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].pathname).toBe("/admin/datasets/hed-sweep"); + expect(server.requests[0].searchParams.get("limit")).toBe("10"); + } finally { + server.stop(); + } + }); + + test("without --limit, the CLI's own default (15) is sent", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([HED_SWEEP_DONE]); + try { + const result = await runCli(["admin", "hed-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.get("limit")).toBe("15"); + } finally { + server.stop(); + } + }); + + test("--reset sends ?reset=1 and does not fall through to a sweep batch", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([{ reset: 9 }]); + try { + const result = await runCli(["admin", "hed-sweep", "--reset", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].searchParams.get("reset")).toBe("1"); + expect(JSON.parse(result.stdout)).toEqual({ reset: 9 }); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin hed-sweep: loop and output", () => { + test("loops across batches until remaining reaches 0, then reports totals", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 4, withHed: 2, withoutHed: 1, unknown: 1, errors: [], remaining: 3 }, + { processed: 3, withHed: 1, withoutHed: 2, unknown: 0, errors: [], remaining: 0 }, + ]); + try { + const result = await runCli(["admin", "hed-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(2); + const parsed = JSON.parse(result.stdout); + expect(parsed.processed).toBe(7); + expect(parsed.withHed).toBe(3); + expect(parsed.withoutHed).toBe(3); + expect(parsed.unknown).toBe(1); + expect(parsed.batches).toBe(2); + expect(parsed.remaining).toBe(0); + } finally { + server.stop(); + } + }); + + test("the human summary reports processed/hed/no-hed/unknown/remaining", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([HED_SWEEP_DONE]); + try { + const result = await runCli(["admin", "hed-sweep"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("processed=1"); + expect(result.stdout).toContain("hed=1"); + expect(result.stdout).toContain("no-hed=0"); + expect(result.stdout).toContain("unknown=0"); + expect(result.stdout).toContain("remaining=0"); + } finally { + server.stop(); + } + }); + + // NOT covered here: a `--verbose` per-batch-line assertion. Investigating + // this test surfaced a PRE-EXISTING, unrelated bug (confirmed with a + // temporary debug patch, not committed): the root `program` also + // declares a global `.option("--verbose", ...)` (src/index.ts:54), which + // shadows every subcommand's own local `--verbose` (hed-sweep, + // recording-stats-sweep, signal-defaults-sweep, and at least two more -- + // admin.ts:3372, 4750) -- `options.verbose` in the action handler comes + // back `undefined` no matter what, so `--verbose` is currently a no-op + // on all of them. Predates this PR and this refactor; out of scope for + // PR #1223 review suggestion 5 (extracting runSweepBatchLoop without + // changing output) and left for a separate follow-up issue. + + test("a non-empty errors array exits non-zero", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { + processed: 1, + withHed: 0, + withoutHed: 0, + unknown: 1, + errors: [{ dataset_id: "nm000900", error: "github: simulated failure" }], + remaining: 0, + }, + ]); + try { + const result = await runCli(["admin", "hed-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("a null remaining count exits non-zero (an uncertain sweep is never reported as success)", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 1, withHed: 1, withoutHed: 0, unknown: 0, errors: [], remaining: null }, + ]); + try { + const result = await runCli(["admin", "hed-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("remaining stuck at the same value stops the loop instead of looping forever", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 0, withHed: 0, withoutHed: 0, unknown: 0, errors: [], remaining: 2 }, + { processed: 0, withHed: 0, withoutHed: 0, unknown: 0, errors: [], remaining: 2 }, + ]); + try { + const result = await runCli(["admin", "hed-sweep"], server.url); + // Mirrors recording-stats-sweep/signal-defaults-sweep: a stuck (but + // non-null, error-free) `remaining` breaks the loop with a warning + // rather than exiting non-zero. + expect(result.exitCode).toBe(0); + expect(result.stderr).toContain("No progress"); + expect(server.requests.length).toBe(2); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin hed-sweep: help", () => { + test("--help documents the reset flag", async () => { + const server = startCaptureServer([HED_SWEEP_DONE]); + try { + const result = await runCli(["admin", "hed-sweep", "--help"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("--limit"); + expect(result.stdout).toContain("--reset"); + expect(result.stdout).toContain("--json"); + } finally { + server.stop(); + } + }); +}); diff --git a/test/maintenance-client.test.ts b/test/maintenance-client.test.ts index 5bd6a50ee..f6600ac01 100644 --- a/test/maintenance-client.test.ts +++ b/test/maintenance-client.test.ts @@ -27,7 +27,12 @@ beforeAll(() => { afterAll(() => { server?.stop(true); - process.env.TEST_API_URL = previousTestApiUrl; + // Guarded restore (#1175): assigning `undefined` to a process.env key + // coerces to the literal string "undefined" instead of deleting it, which + // poisoned TEST_API_URL for every test running later in the same `bun + // test` process (test/ + backend/test/ share one process at the root). + if (previousTestApiUrl === undefined) delete process.env.TEST_API_URL; + else process.env.TEST_API_URL = previousTestApiUrl; }); function setResponse(status: number, body: Record): void { diff --git a/test/recording-signal-sweeps-cli.test.ts b/test/recording-signal-sweeps-cli.test.ts new file mode 100644 index 000000000..a32d23302 --- /dev/null +++ b/test/recording-signal-sweeps-cli.test.ts @@ -0,0 +1,449 @@ +/** + * CLI `nemar admin recording-stats-sweep` / `nemar admin signal-defaults-sweep` + * (issue #1194), driven through the real entry point (`bun run src/index.ts + * admin ... `). Mirrors test/zarr-fidelity-sweep-cli.test.ts's harness: a + * real subprocess CLI invocation pointed at a real local HTTP server via + * TEST_API_URL (src/lib/api/client.ts#getApiUrl) -- the recorded request is + * what the CLI's own `fetch()` actually sent, not a re-implementation of it. + * + * Both commands are admin-only, so the harness also seeds a real + * config.json ({activeAccount, accounts: {: {apiKey}}}) into an + * isolated NEMAR_CONFIG_DIR -- the same shape src/lib/config.ts's Conf + * store reads (cwd: NEMAR_CONFIG_DIR) -- so isAuthenticated() is genuinely + * true rather than stubbed. + * + * Unlike zarr-fidelity-sweep, both commands loop, calling the batch + * endpoint repeatedly until `remaining` reaches 0 (mirrors + * hedSweepCommand/availabilityReportCommand). The capture server below can + * be seeded with a queue of responses to exercise that loop for real. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "bun"; + +const CLI_ENTRY = join(import.meta.dir, "..", "src", "index.ts"); +const REPO_ROOT = join(import.meta.dir, ".."); + +interface CaptureServer { + url: string; + requests: URL[]; + stop: () => void; +} + +/** Serves each entry in `responses` in order (one per non-excluded POST); + * the last entry repeats if more requests arrive than responses provided. */ +function startCaptureServer(responses: unknown[], status = 200): CaptureServer { + const requests: URL[] = []; + let i = 0; + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + // Real traffic every invocation fires that is not what these tests + // check (mirrors zarr-fidelity-sweep-cli.test.ts's exclusions). + if (url.pathname === "/notices") { + return new Response(JSON.stringify({ notices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.pathname === "/datasets/facets") { + return new Response(JSON.stringify({}), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + requests.push(url); + const body = responses[Math.min(i, responses.length - 1)]; + i++; + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }); + return { + url: `http://localhost:${server.port}`, + requests, + stop: () => server.stop(true), + }; +} + +let configDir: string; + +function seedAuthenticatedConfig(): void { + writeFileSync( + join(configDir, "config.json"), + JSON.stringify({ + activeAccount: "sweepcliuser", + accounts: { sweepcliuser: { apiKey: "test-admin-key" } }, + }), + ); +} + +beforeEach(() => { + configDir = mkdtempSync(join(tmpdir(), "nemar-recording-signal-sweeps-cli-")); +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); +}); + +async function runCli( + args: string[], + testApiUrl: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const env = { + ...process.env, + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: testApiUrl, + NEMAR_NO_UPDATE_CHECK: "1", + NO_COLOR: "1", + }; + env.FORCE_COLOR = undefined; + env.CLICOLOR_FORCE = undefined; + const proc = spawn({ + cmd: ["bun", "run", CLI_ENTRY, ...args], + cwd: REPO_ROOT, + env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const exitCode = await proc.exited; + return { stdout, stderr, exitCode }; +} + +const RECORDING_STATS_DONE = { + processed: 1, + measured: 1, + unmeasured: 0, + errors: [], + remaining: 0, +}; + +const SIGNAL_DEFAULTS_DONE = { + processed: 1, + populated: 1, + noData: 0, + errors: [], + remaining: 0, +}; + +describe("nemar admin recording-stats-sweep: auth", () => { + test("without a stored apiKey, requireAuth blocks before any request is sent", async () => { + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli(["admin", "recording-stats-sweep"], server.url); + expect(result.stdout).toContain("Not authenticated"); + expect(server.requests.length).toBe(0); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin recording-stats-sweep: request shape", () => { + test("--limit is forwarded as ?limit=N on the real request", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli( + ["admin", "recording-stats-sweep", "--limit", "10", "--json"], + server.url, + ); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].pathname).toBe("/admin/datasets/recording-stats-sweep"); + expect(server.requests[0].searchParams.get("limit")).toBe("10"); + } finally { + server.stop(); + } + }); + + test("without --limit, the CLI's own default (50) is sent", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli(["admin", "recording-stats-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.get("limit")).toBe("50"); + } finally { + server.stop(); + } + }); + + test("--reset sends ?reset=1 and does not fall through to a sweep batch", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([{ reset: 7 }]); + try { + const result = await runCli( + ["admin", "recording-stats-sweep", "--reset", "--json"], + server.url, + ); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].searchParams.get("reset")).toBe("1"); + expect(JSON.parse(result.stdout)).toEqual({ reset: 7 }); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin recording-stats-sweep: loop and output", () => { + test("loops across batches until remaining reaches 0, then reports totals", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 5, measured: 3, unmeasured: 2, errors: [], remaining: 2 }, + { processed: 2, measured: 2, unmeasured: 0, errors: [], remaining: 0 }, + ]); + try { + const result = await runCli(["admin", "recording-stats-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(2); + const parsed = JSON.parse(result.stdout); + expect(parsed.processed).toBe(7); + expect(parsed.measured).toBe(5); + expect(parsed.unmeasured).toBe(2); + expect(parsed.batches).toBe(2); + expect(parsed.remaining).toBe(0); + } finally { + server.stop(); + } + }); + + test("the human summary reports processed/measured/unmeasured/remaining", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli(["admin", "recording-stats-sweep"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("processed=1"); + expect(result.stdout).toContain("measured=1"); + expect(result.stdout).toContain("unmeasured=0"); + expect(result.stdout).toContain("remaining=0"); + } finally { + server.stop(); + } + }); + + test("a non-empty errors array exits non-zero", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { + processed: 1, + measured: 0, + unmeasured: 0, + errors: [{ dataset_id: "nm000900", error: "s3: simulated timeout" }], + remaining: 0, + }, + ]); + try { + const result = await runCli(["admin", "recording-stats-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("a null remaining count exits non-zero (an uncertain sweep is never reported as success)", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 1, measured: 1, unmeasured: 0, errors: [], remaining: null }, + ]); + try { + const result = await runCli(["admin", "recording-stats-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("remaining stuck at the same value stops the loop instead of looping forever", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 0, measured: 0, unmeasured: 0, errors: [], remaining: 3 }, + { processed: 0, measured: 0, unmeasured: 0, errors: [], remaining: 3 }, + ]); + try { + const result = await runCli(["admin", "recording-stats-sweep"], server.url); + // Mirrors hedSweepCommand/availabilityReportCommand: a stuck (but + // non-null, error-free) `remaining` breaks the loop with a warning + // rather than exiting non-zero -- only a null `remaining` or a + // non-empty errors array trips the final exit-code guard. + expect(result.exitCode).toBe(0); + // The "No progress" warning is printed via ora's spinner, which + // writes to stderr, not stdout. + expect(result.stderr).toContain("No progress"); + // Stops after the second identical `remaining`, not the caller-side + // repeat-response fallback going on forever. + expect(server.requests.length).toBe(2); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin signal-defaults-sweep: auth", () => { + test("without a stored apiKey, requireAuth blocks before any request is sent", async () => { + const server = startCaptureServer([SIGNAL_DEFAULTS_DONE]); + try { + const result = await runCli(["admin", "signal-defaults-sweep"], server.url); + expect(result.stdout).toContain("Not authenticated"); + expect(server.requests.length).toBe(0); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin signal-defaults-sweep: request shape", () => { + test("--limit is forwarded as ?limit=N on the real request", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([SIGNAL_DEFAULTS_DONE]); + try { + const result = await runCli( + ["admin", "signal-defaults-sweep", "--limit", "5", "--json"], + server.url, + ); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].pathname).toBe("/admin/datasets/signal-defaults-sweep"); + expect(server.requests[0].searchParams.get("limit")).toBe("5"); + } finally { + server.stop(); + } + }); + + test("without --limit, the CLI's own default (15) is sent", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([SIGNAL_DEFAULTS_DONE]); + try { + const result = await runCli(["admin", "signal-defaults-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.get("limit")).toBe("15"); + } finally { + server.stop(); + } + }); + + test("--reset sends ?reset=1 and does not fall through to a sweep batch", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([{ reset: 4 }]); + try { + const result = await runCli( + ["admin", "signal-defaults-sweep", "--reset", "--json"], + server.url, + ); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].searchParams.get("reset")).toBe("1"); + expect(JSON.parse(result.stdout)).toEqual({ reset: 4 }); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin signal-defaults-sweep: loop and output", () => { + test("loops across batches until remaining reaches 0, then reports totals", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { processed: 3, populated: 1, noData: 2, errors: [], remaining: 1 }, + { processed: 1, populated: 0, noData: 1, errors: [], remaining: 0 }, + ]); + try { + const result = await runCli(["admin", "signal-defaults-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(2); + const parsed = JSON.parse(result.stdout); + expect(parsed.processed).toBe(4); + expect(parsed.populated).toBe(1); + expect(parsed.noData).toBe(3); + expect(parsed.batches).toBe(2); + expect(parsed.remaining).toBe(0); + } finally { + server.stop(); + } + }); + + test("the human summary reports processed/populated/no-data/remaining", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([SIGNAL_DEFAULTS_DONE]); + try { + const result = await runCli(["admin", "signal-defaults-sweep"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("processed=1"); + expect(result.stdout).toContain("populated=1"); + expect(result.stdout).toContain("no-data=0"); + expect(result.stdout).toContain("remaining=0"); + } finally { + server.stop(); + } + }); + + test("a non-empty errors array exits non-zero", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer([ + { + processed: 1, + populated: 0, + noData: 0, + errors: [{ dataset_id: "*", error: "github auth failed" }], + remaining: 0, + }, + ]); + try { + const result = await runCli(["admin", "signal-defaults-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin recording-stats-sweep / signal-defaults-sweep: help", () => { + test("recording-stats-sweep --help documents the reset flag", async () => { + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli(["admin", "recording-stats-sweep", "--help"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("--limit"); + expect(result.stdout).toContain("--reset"); + expect(result.stdout).toContain("--json"); + } finally { + server.stop(); + } + }); + + test("signal-defaults-sweep --help documents the reset flag", async () => { + const server = startCaptureServer([SIGNAL_DEFAULTS_DONE]); + try { + const result = await runCli(["admin", "signal-defaults-sweep", "--help"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("--limit"); + expect(result.stdout).toContain("--reset"); + expect(result.stdout).toContain("--json"); + } finally { + server.stop(); + } + }); + + test("nemar admin --help lists both new sweep commands", async () => { + const server = startCaptureServer([RECORDING_STATS_DONE]); + try { + const result = await runCli(["admin", "--help"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("recording-stats-sweep"); + expect(result.stdout).toContain("signal-defaults-sweep"); + } finally { + server.stop(); + } + }); +}); diff --git a/test/recover-datasets-packaging.test.ts b/test/recover-datasets-packaging.test.ts new file mode 100644 index 000000000..97cf30179 --- /dev/null +++ b/test/recover-datasets-packaging.test.ts @@ -0,0 +1,298 @@ +/** + * Issue #1049: `nemar admin recover` was broken in published npm builds + * because `scripts/recover-datasets.json` was never shipped (package.json + * "files" omitted it) AND `defaultRecoverDatasetsPath()` hardcoded a + * "two levels up from this source file" resolution that only holds for a + * repo checkout (src/commands/ -> src/ -> repo root); the published, + * single-file `dist/index.js` bundle sits only ONE level above the + * package's `scripts/` dir, so the old code resolved to a path outside + * the installed package entirely and 404'd on every real run. + * + * PR #1223 review: `nemar admin withdraw` (`defaultWithdrawnDatasetsPath`, + * `scripts/withdrawn-datasets.json`) shares the exact same shape and had + * the exact same bug, confirmed with `bun pm pack --dry-run` -- this file + * covers both siblings. + * + * Both halves are driven through their real entry points here, no mocks: + * - `bun pm pack` is the actual npm packaging tool, so the packed-file-list + * assertion is what a real `npm publish` would ship, not a re-read of + * package.json's "files" array. + * - the path-resolution assertions run the real `nemar admin recover` / + * `nemar admin withdraw` command as a subprocess, once from source + * (`bun run src/index.ts`) and once from a real `bun build` bundle + * placed at the same repo-relative layout npm installs it at + * (dist/index.js next to scripts/), so the fallback branch in + * defaultRecoverDatasetsPath() / defaultWithdrawnDatasetsPath() is + * exercised for real. + */ + +import { describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "bun"; +import rootPkg from "../package.json"; + +const REPO_ROOT = join(import.meta.dir, ".."); +const CLI_ENTRY = join(REPO_ROOT, "src", "index.ts"); +const DIST_ENTRY = join(REPO_ROOT, "dist", "index.js"); +const RECOVER_JSON = join(REPO_ROOT, "scripts", "recover-datasets.json"); +const WITHDRAWN_JSON = join(REPO_ROOT, "scripts", "withdrawn-datasets.json"); + +interface RunResult { + stdout: string; + stderr: string; + exitCode: number; +} + +async function run( + cmd: string[], + envOverrides?: Record, +): Promise { + const proc = spawn({ + cmd, + cwd: REPO_ROOT, + env: { ...process.env, NEMAR_NO_UPDATE_CHECK: "1", ...envOverrides }, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const exitCode = await proc.exited; + return { stdout, stderr, exitCode }; +} + +function seedAuthenticatedConfig(configDir: string): void { + writeFileSync( + join(configDir, "config.json"), + JSON.stringify({ + activeAccount: "recoverpkguser", + accounts: { recoverpkguser: { apiKey: "test-admin-key" } }, + }), + ); +} + +/** The dry-run recover path never calls the backend itself, but every real + * `nemar` invocation fires a `GET /notices` preAction hook first (mirrors + * test/zarr-fidelity-sweep-cli.test.ts's harness) -- point it at a local + * stub instead of the real production API so this stays a pure, offline + * test of path resolution. */ +function startNoticesStub(): { url: string; stop: () => void } { + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/notices") { + return new Response(JSON.stringify({ notices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return new Response("not found", { status: 404 }); + }, + }); + return { url: `http://localhost:${server.port}`, stop: () => server.stop(true) }; +} + +/** Same idea as startNoticesStub, but also answers `POST + * /admin/datasets/:id/withdraw` -- unlike recover's dry-run path, + * `nemar admin withdraw` (even with --execute omitted) calls the backend + * once per target, since the dry-run logic lives server-side. Recording + * each request lets a test tell "the file loaded and every target + * reached the network step" apart from "the command never got that + * far" (the exact #1049 failure mode: a broken default path exits with + * "Failed to load withdrawn-datasets file" before any request is sent). */ +function startWithdrawStub(): { + url: string; + withdrawRequests: string[]; + stop: () => void; +} { + const withdrawRequests: string[] = []; + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/notices") { + return new Response(JSON.stringify({ notices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + const withdrawMatch = url.pathname.match(/^\/admin\/datasets\/([^/]+)\/withdraw$/); + if (withdrawMatch && req.method === "POST") { + withdrawRequests.push(withdrawMatch[1]); + return new Response( + JSON.stringify({ resumed: false, visibility: { status: "ok" }, dois: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + } + return new Response("not found", { status: 404 }); + }, + }); + return { + url: `http://localhost:${server.port}`, + withdrawRequests, + stop: () => server.stop(true), + }; +} + +describe("npm package includes scripts/recover-datasets.json (#1049)", () => { + test("package.json 'files' lists it", () => { + expect(rootPkg.files).toContain("scripts/recover-datasets.json"); + }); + + test("the file actually exists on disk (a files entry alone proves nothing)", () => { + expect(existsSync(RECOVER_JSON)).toBe(true); + }); + + test("`bun pm pack` (the real npm packaging tool) actually includes it in the tarball", async () => { + const result = await run(["bun", "pm", "pack", "--dry-run", "--ignore-scripts", "--quiet"]); + expect(result.exitCode).toBe(0); + // --quiet still prints the tarball filename; run without --quiet once + // more captured to stderr/stdout isn't guaranteed stable across bun + // versions, so assert on a second, verbose invocation instead. + const verbose = await run(["bun", "pm", "pack", "--dry-run", "--ignore-scripts"]); + expect(verbose.exitCode).toBe(0); + expect(verbose.stdout).toContain("scripts/recover-datasets.json"); + }); +}); + +describe("nemar admin recover resolves its default file from source (#1049)", () => { + test("`bun run src/index.ts admin recover --all --json` loads the real 45-entry list", async () => { + const configDir = mkdtempSync(join(tmpdir(), "nemar-recover-src-")); + seedAuthenticatedConfig(configDir); + const notices = startNoticesStub(); + try { + const result = await run(["bun", "run", CLI_ENTRY, "admin", "recover", "--all", "--json"], { + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: notices.url, + }); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.dry_run).toBe(true); + expect(parsed.targets.length).toBe(45); + expect(parsed.targets).toContain("on002814"); + } finally { + notices.stop(); + rmSync(configDir, { recursive: true, force: true }); + } + }); +}); + +describe("nemar admin recover resolves its default file from a published-layout dist bundle (#1049)", () => { + test("a real `bun build` bundle at dist/index.js (sibling of scripts/, the npm-installed layout) finds the file", async () => { + // Build the actual single-file bundle at the same repo-relative path npm + // ships it at (dist/index.js next to scripts/) -- dist/ is gitignored so + // this doesn't touch tracked files. This is the exact layout the old + // "two levels up from import.meta.dir" resolution 404'd against, because + // from dist/ that resolution lands one directory ABOVE the installed + // package instead of at its scripts/ dir. + const build = await run([ + "bun", + "build", + CLI_ENTRY, + "--outdir", + join(REPO_ROOT, "dist"), + "--target", + "bun", + ]); + expect(build.exitCode).toBe(0); + expect(existsSync(DIST_ENTRY)).toBe(true); + + const configDir = mkdtempSync(join(tmpdir(), "nemar-recover-dist-")); + seedAuthenticatedConfig(configDir); + const notices = startNoticesStub(); + try { + const result = await run(["bun", DIST_ENTRY, "admin", "recover", "--all", "--json"], { + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: notices.url, + }); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout); + expect(parsed.dry_run).toBe(true); + expect(parsed.targets.length).toBe(45); + expect(parsed.targets).toContain("on002814"); + } finally { + notices.stop(); + rmSync(configDir, { recursive: true, force: true }); + } + }); +}); + +describe("npm package includes scripts/withdrawn-datasets.json (PR #1223 review)", () => { + test("package.json 'files' lists it", () => { + expect(rootPkg.files).toContain("scripts/withdrawn-datasets.json"); + }); + + test("the file actually exists on disk (a files entry alone proves nothing)", () => { + expect(existsSync(WITHDRAWN_JSON)).toBe(true); + }); + + test("`bun pm pack` (the real npm packaging tool) actually includes it in the tarball", async () => { + const result = await run(["bun", "pm", "pack", "--dry-run", "--ignore-scripts", "--quiet"]); + expect(result.exitCode).toBe(0); + const verbose = await run(["bun", "pm", "pack", "--dry-run", "--ignore-scripts"]); + expect(verbose.exitCode).toBe(0); + expect(verbose.stdout).toContain("scripts/withdrawn-datasets.json"); + }); +}); + +describe("nemar admin withdraw resolves its default file from source (PR #1223 review)", () => { + test("`bun run src/index.ts admin withdraw --all --json` loads the real 11-entry list", async () => { + const configDir = mkdtempSync(join(tmpdir(), "nemar-withdraw-src-")); + seedAuthenticatedConfig(configDir); + const stub = startWithdrawStub(); + try { + const result = await run(["bun", "run", CLI_ENTRY, "admin", "withdraw", "--all", "--json"], { + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: stub.url, + }); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.length).toBe(11); + // Every target reached the network step -- proof the file loaded, + // not just that the command exited 0. + expect(stub.withdrawRequests.length).toBe(11); + expect(stub.withdrawRequests).toContain("on004148"); + } finally { + stub.stop(); + rmSync(configDir, { recursive: true, force: true }); + } + }); +}); + +describe("nemar admin withdraw resolves its default file from a published-layout dist bundle (PR #1223 review)", () => { + test("a real `bun build` bundle at dist/index.js (sibling of scripts/, the npm-installed layout) finds the file", async () => { + const build = await run([ + "bun", + "build", + CLI_ENTRY, + "--outdir", + join(REPO_ROOT, "dist"), + "--target", + "bun", + ]); + expect(build.exitCode).toBe(0); + expect(existsSync(DIST_ENTRY)).toBe(true); + + const configDir = mkdtempSync(join(tmpdir(), "nemar-withdraw-dist-")); + seedAuthenticatedConfig(configDir); + const stub = startWithdrawStub(); + try { + const result = await run(["bun", DIST_ENTRY, "admin", "withdraw", "--all", "--json"], { + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: stub.url, + }); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout); + expect(Array.isArray(parsed)).toBe(true); + expect(parsed.length).toBe(11); + expect(stub.withdrawRequests.length).toBe(11); + } finally { + stub.stop(); + rmSync(configDir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/search-color.unit.test.ts b/test/search-color.unit.test.ts index 7351c14aa..bd9ba10c5 100644 --- a/test/search-color.unit.test.ts +++ b/test/search-color.unit.test.ts @@ -19,21 +19,12 @@ * order and would make every "no colour" assertion below vacuously true * for the wrong reason if left set. * - * TEST_API_URL is also explicitly cleared. `bun test` runs every file in - * one process, and test/maintenance-client.test.ts's `afterAll` restores it - * with `process.env.TEST_API_URL = previousTestApiUrl` -- unguarded, unlike - * test/publish-progress.test.ts's equivalent restore. Node/Bun coerce - * assigning `undefined` to a `process.env` key into the literal string - * `"undefined"` rather than deleting it, so once that file has run in this - * environment (no test/.env.test => TEST_API_URL was never set to begin - * with), `process.env.TEST_API_URL` is left holding `"undefined"` for every - * test that runs afterward in the same process -- a real, pre-existing - * test-isolation bug (confirmed by a 2-file repro: - * `bun test test/maintenance-client.test.ts test/search-color.unit.test.ts` - * fails 5/12 with `getApiUrl()` returning the string "undefined"; this file - * alone, or with test/api.test.ts + test/cli.test.ts instead, passes clean). - * Clearing it here is a workaround for THIS file rather than a fix for that - * one -- see the phase report for the recommended follow-up. + * TEST_API_URL is explicitly set to this file's own local stub server for + * every spawn, so a search subprocess never depends on whatever TEST_API_URL + * happens to be in the ambient/parent env. (It used to also work around a + * cross-file test-isolation bug in test/maintenance-client.test.ts's + * unguarded `afterAll` restore -- #1175 fixed that restore, so this is now + * just normal per-test isolation, not a workaround.) */ import { describe, expect, test } from "bun:test"; @@ -73,8 +64,7 @@ async function runSearch( FORCE_COLOR: undefined, CLICOLOR_FORCE: undefined, NO_COLOR: undefined, - // See the file-level comment: neutralises a pre-existing cross-file - // test-isolation bug rather than depending on it not having run yet. + // Point at this test's own local stub server rather than the ambient env. TEST_API_URL: stub.url, NEMAR_NO_UPDATE_CHECK: "1", ...envOverrides, diff --git a/test/upload-s3-resume.unit.test.ts b/test/upload-s3-resume.unit.test.ts new file mode 100644 index 000000000..e7310fc48 --- /dev/null +++ b/test/upload-s3-resume.unit.test.ts @@ -0,0 +1,358 @@ +/** + * Issue #1070: uploadDataToS3's git-state reconcile (listTrackedPaths / + * computeAddTargets, plus the "untracked despite recorded progress" + * repair) used to live ENTIRELY inside `if (!isStepCompleted(progress, + * "s3_upload"))`. Once a prior run stamped "s3_upload" complete, that + * whole block -- including the reconcile itself -- was skipped on every + * later invocation, so a data file added to the dataset after that point + * was silently never uploaded: the function took the "already completed + * (skipping)" branch and returned immediately without even looking at git + * state. + * + * Drives the real production entry point (uploadDataToS3, src/lib/upload/ + * transfer.ts) in-process against a real git-annex repo on disk -- no + * mocks. A hand-built UploadProgress fixture simulates a prior run that + * finished with one file uploaded and "s3_upload" stamped complete + * (exactly the shape `.nemar/upload-progress.json` has after a real + * successful upload); a second data file is then added to the working + * tree without going through any upload, matching a user re-running + * `nemar dataset upload` after adding files to an already-published + * dataset. + * + * The credentials endpoint is a real local server that always errors, + * so `requestUploadCredentials` fails fast and deterministically without + * ever reaching production or requiring live AWS credentials -- this test + * proves the RECONCILE runs and reopens the gate (git-annex tracking is a + * fully offline, git-verifiable side effect that happens strictly BEFORE + * the credential request in uploadDataToS3), not that a full S3 + * round-trip succeeds (that needs live AWS credentials and belongs to the + * e2e/sandbox suite, test/e2e-upload.test.ts). + */ + +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { + chmodSync, + existsSync, + mkdirSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { spawn } from "bun"; +import { configureLargefiles, gitAnnexAdd, initDataset } from "../src/lib/git-annex/init"; +import { + type UploadProgress, + clearUploadProgress, + initUploadProgress, + isStepCompleted, + markFileUploaded, + markStepCompleted, + readUploadProgress, +} from "../src/lib/upload-progress"; +import { saveDatasetStep } from "../src/lib/upload/finalize"; +import { type UploadFileEntry, uploadDataToS3 } from "../src/lib/upload/transfer"; +import type { DatasetInfo } from "../src/lib/upload/types"; + +const TMP_DIR = join(import.meta.dir, ".test-upload-s3-resume"); + +async function runCmd( + cmd: string[], + cwd?: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const proc = spawn({ cmd, cwd, stdout: "pipe", stderr: "pipe" }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const exitCode = await proc.exited; + return { stdout, stderr, exitCode }; +} + +// git-annex marks object files read-only; rm needs write+execute on dirs. +function chmodTreeWritable(dir: string): void { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + let st: ReturnType; + try { + st = statSync(full); + } catch { + continue; + } + if (st.isDirectory()) { + try { + chmodSync(full, 0o755); + } catch {} + chmodTreeWritable(full); + } else { + try { + chmodSync(full, 0o644); + } catch {} + } + } +} + +async function setRepoIdentity(dir: string): Promise { + await runCmd(["git", "config", "user.email", "test@test.com"], dir); + await runCmd(["git", "config", "user.name", "Test"], dir); +} + +/** Real dataset repo through the production init path (mirrors + * test/annex-add-targets.unit.test.ts's helper of the same shape). */ +async function newDatasetRepo(name: string): Promise { + const dir = join(TMP_DIR, `${name}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); + mkdirSync(dir, { recursive: true }); + const init = await initDataset(dir, { author: { name: "Test", email: "test@test.com" } }); + if (!init.success) { + throw new Error(`initDataset failed: ${init.error}`); + } + await setRepoIdentity(dir); + const largefiles = await configureLargefiles(dir); + if (!largefiles.success) { + throw new Error(`configureLargefiles failed: ${largefiles.error}`); + } + return dir; +} + +function writeDataFile(dir: string, relPath: string, content: string): void { + const full = join(dir, relPath); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content); +} + +async function annexedFiles(dir: string): Promise { + const { stdout, exitCode } = await runCmd(["git", "annex", "find", "--include", "*"], dir); + expect(exitCode).toBe(0); + return stdout.split("\n").filter(Boolean).sort(); +} + +function fakeDatasetInfo(id: string): DatasetInfo { + return { + dataset_id: id, + ssh_url: `git@github.com:nemarDatasets/${id}.git`, + s3_prefix: `${id}/objects`, + github_url: `https://github.com/nemarDatasets/${id}`, + upload_urls: {}, + s3_config: { bucket: "nemar-test", region: "us-east-2", public_url: "https://example.invalid" }, + }; +} + +/** Real local server standing in for the backend's upload-credentials + * endpoint: always errors, so requestUploadCredentials fails fast without + * ever reaching production or needing live AWS credentials. */ +function startFailingCredentialsServer(): { url: string; stop: () => void } { + const server = Bun.serve({ + port: 0, + fetch() { + return new Response(JSON.stringify({ error: "no credentials in this test" }), { + status: 503, + headers: { "content-type": "application/json" }, + }); + }, + }); + return { url: `http://localhost:${server.port}`, stop: () => server.stop(true) }; +} + +let previousTestApiUrl: string | undefined; + +beforeAll(async () => { + const probe = await runCmd(["git", "annex", "version"]); + if (probe.exitCode !== 0) { + throw new Error("git-annex is required for this test file"); + } + previousTestApiUrl = process.env.TEST_API_URL; +}); + +afterAll(() => { + // Guarded restore (#1175): never assign `undefined` directly. + if (previousTestApiUrl === undefined) delete process.env.TEST_API_URL; + else process.env.TEST_API_URL = previousTestApiUrl; + + if (existsSync(TMP_DIR)) { + chmodTreeWritable(TMP_DIR); + rmSync(TMP_DIR, { recursive: true, force: true }); + } +}); + +describe("uploadDataToS3 resume when s3_upload is already stamped complete (#1070)", () => { + test("a data file added after a completed run is git-annex tracked and reopens s3_upload, instead of being silently skipped", async () => { + const dir = await newDatasetRepo("resume-s3"); + const oldFile: UploadFileEntry = { path: "sub-01/eeg/a.edf", size: 4096, type: "data" }; + const newFile: UploadFileEntry = { path: "sub-02/eeg/b.edf", size: 4096, type: "data" }; + writeDataFile(dir, oldFile.path, "a".repeat(oldFile.size)); + + // --- Simulate a PRIOR successful run: oldFile tracked + uploaded, both + // "tracking" and "s3_upload" stamped complete -- the exact on-disk shape + // a real finished upload leaves in .nemar/upload-progress.json. + expect((await gitAnnexAdd(dir, [oldFile.path])).success).toBe(true); + const progress: UploadProgress = initUploadProgress(dir, "nm000900", [oldFile]); + markStepCompleted(progress, "tracking"); + markFileUploaded(progress, oldFile.path, { size: oldFile.size }); + markStepCompleted(progress, "s3_upload"); + expect(isStepCompleted(progress, "s3_upload")).toBe(true); + + // --- Now a second data file lands in the working tree without ever + // being uploaded -- e.g. the user added it and re-ran `nemar dataset + // upload`. Confirm the starting state: genuinely untracked by git. + writeDataFile(dir, newFile.path, "b".repeat(newFile.size)); + expect(await annexedFiles(dir)).toEqual([oldFile.path]); + + const dataFiles = [oldFile, newFile]; + // filesToUpload mirrors what the real CLI computes: oldFile is + // unchanged+uploaded (excluded), newFile has no progress record (included). + const filesToUpload = [newFile]; + + const credServer = startFailingCredentialsServer(); + process.env.TEST_API_URL = credServer.url; + let result: Awaited>; + try { + result = await uploadDataToS3( + dir, + { jobs: "3" }, + dataFiles, + filesToUpload, + progress, + fakeDatasetInfo("nm000900"), + ); + } finally { + credServer.stop(); + } + + // The credentials endpoint always errors, so the upload itself cannot + // finish -- this test is not asserting a full S3 round-trip. + expect(result.status).toBe("fail"); + + // But the fix's actual claim: the RECONCILE ran despite s3_upload having + // been stamped complete, and picked the new file up for tracking. Under + // the pre-#1070 code this whole block, including gitAnnexAdd, never ran + // at all -- the function returned "ok" immediately after printing + // "S3 upload already completed (skipping)", and newFile stayed untracked. + expect(await annexedFiles(dir)).toEqual([newFile.path, oldFile.path].sort()); + + // The stale "s3_upload" completion stamp must have been cleared -- both + // on the in-memory object the caller holds and on disk -- so a later + // successful run (once credentials work) does not need a second manual + // nudge to re-enter the upload path. + expect(isStepCompleted(progress, "s3_upload")).toBe(false); + expect(isStepCompleted(progress, "tracking")).toBe(true); + const onDisk = readUploadProgress(dir); + expect(onDisk).not.toBeNull(); + expect(isStepCompleted(onDisk as UploadProgress, "s3_upload")).toBe(false); + + clearUploadProgress(dir); + }); + + test("dataset_save and github_push are also reopened, so the new file's pointer actually gets committed (review finding, critical)", async () => { + const dir = await newDatasetRepo("resume-s3-finalize"); + const oldFile: UploadFileEntry = { path: "sub-01/eeg/a.edf", size: 4096, type: "data" }; + const newFile: UploadFileEntry = { path: "sub-02/eeg/b.edf", size: 4096, type: "data" }; + writeDataFile(dir, oldFile.path, "a".repeat(oldFile.size)); + const author = { name: "Test", email: "test@test.com" }; + + // --- Simulate a run that got all the way through pushMetadata (dataset.ts + // Step 12) but was interrupted before printUploadSuccess ever cleared + // progress (e.g. mid deployCiStep's network call, Step 12b): tracking, + // s3_upload, dataset_save AND github_push all stamped complete, and + // oldFile's pointer genuinely committed via the real saveDatasetStep. + expect((await gitAnnexAdd(dir, [oldFile.path])).success).toBe(true); + const progress: UploadProgress = initUploadProgress(dir, "nm000902", [oldFile]); + markStepCompleted(progress, "tracking"); + markFileUploaded(progress, oldFile.path, { size: oldFile.size }); + markStepCompleted(progress, "s3_upload"); + expect((await saveDatasetStep(dir, author, progress)).status).toBe("ok"); + expect(isStepCompleted(progress, "dataset_save")).toBe(true); + // No real GitHub remote in this harness; stamp github_push directly to + // reach the exact "both finalize steps already complete" prior state. + markStepCompleted(progress, "github_push"); + const commitsBefore = (await runCmd(["git", "log", "--oneline"], dir)).stdout + .trim() + .split("\n").length; + + // --- A second data file lands in the working tree after that point -- + // e.g. the user added it and re-ran `nemar dataset upload`. + writeDataFile(dir, newFile.path, "b".repeat(newFile.size)); + const dataFiles = [oldFile, newFile]; + const filesToUpload = [newFile]; + + const credServer = startFailingCredentialsServer(); + process.env.TEST_API_URL = credServer.url; + let result: Awaited>; + try { + result = await uploadDataToS3( + dir, + { jobs: "3" }, + dataFiles, + filesToUpload, + progress, + fakeDatasetInfo("nm000902"), + ); + } finally { + credServer.stop(); + } + // Credentials still fail deterministically; this is not asserting a + // full S3 round-trip (see the file-level comment). + expect(result.status).toBe("fail"); + + // THE FIX: dataset_save and github_push must be reopened alongside + // s3_upload/tracking. Before it, a resumed upload that picked up new + // content would eventually re-complete s3_upload but dataset.ts's Step + // 11/12 (saveDatasetStep/pushMetadata) would see dataset_save/ + // github_push still stamped true from the PRIOR run and skip entirely + // -- the new file's git-annex pointer would reach S3 but never be + // committed or pushed, and printUploadSuccess would still clear + // progress and report "Upload complete!". + expect(isStepCompleted(progress, "dataset_save")).toBe(false); + expect(isStepCompleted(progress, "github_push")).toBe(false); + const onDisk = readUploadProgress(dir); + expect(onDisk).not.toBeNull(); + expect(isStepCompleted(onDisk as UploadProgress, "dataset_save")).toBe(false); + expect(isStepCompleted(onDisk as UploadProgress, "github_push")).toBe(false); + + // --- Drive the REAL saveDatasetStep (dataset.ts Step 11) now that + // dataset_save is reopened, proving the new file's pointer actually + // gets committed rather than silently skipped. + expect((await saveDatasetStep(dir, author, progress)).status).toBe("ok"); + expect(isStepCompleted(progress, "dataset_save")).toBe(true); + const committed = await runCmd(["git", "show", "--stat", "HEAD"], dir); + expect(committed.stdout).toContain(newFile.path); + const commitsAfter = (await runCmd(["git", "log", "--oneline"], dir)).stdout + .trim() + .split("\n").length; + expect(commitsAfter).toBeGreaterThan(commitsBefore); + + clearUploadProgress(dir); + }); + + test("nothing new to upload: s3_upload stays complete and no git-annex add runs (fast path preserved)", async () => { + const dir = await newDatasetRepo("resume-s3-noop"); + const oldFile: UploadFileEntry = { path: "sub-01/eeg/a.edf", size: 4096, type: "data" }; + writeDataFile(dir, oldFile.path, "a".repeat(oldFile.size)); + expect((await gitAnnexAdd(dir, [oldFile.path])).success).toBe(true); + + const progress: UploadProgress = initUploadProgress(dir, "nm000901", [oldFile]); + markStepCompleted(progress, "tracking"); + markFileUploaded(progress, oldFile.path, { size: oldFile.size }); + markStepCompleted(progress, "s3_upload"); + + // No new files, nothing changed: filesToUpload is empty, exactly like a + // genuinely finished dataset being re-run with no local changes. + const dataFiles = [oldFile]; + const filesToUpload: UploadFileEntry[] = []; + + const result = await uploadDataToS3( + dir, + { jobs: "3" }, + dataFiles, + filesToUpload, + progress, + fakeDatasetInfo("nm000901"), + ); + + // The reconcile still runs (cheap, always-on per #1070), finds nothing + // to add, and the expensive path -- including any network call -- is + // never entered, so this succeeds with no credentials server at all. + expect(result.status).toBe("ok"); + expect(isStepCompleted(progress, "s3_upload")).toBe(true); + + clearUploadProgress(dir); + }); +});