Skip to content
4 changes: 2 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -512,11 +512,11 @@ and what is historical. The entries worth knowing by name:
| Group | Covers |
|---|---|
| `nemar auth` | login, signup, status/whoami, switch, logout, verification, SSH setup, key retrieval and regeneration |
| `nemar dataset` | validate, upload, download, status, list, search, release, update, clone, get, commit, push, drop, ci, manifest |
| `nemar dataset` | validate, upload, download, status (alias: view), list, search, release, update, clone, get, commit, push, drop, ci, manifest |
| `nemar dataset publish` | request, status, resend |
| `nemar dataset` (access) | request-access, access, invite, collaborators |
| `nemar sandbox` | training run, status, reset — required before uploading |
| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, zarr-fidelity-sweep, doctor, summary, notice, email-preferences, e2e-test |
| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, recording-stats-sweep, signal-defaults-sweep, zarr-fidelity-sweep, doctor, summary, notice, email-preferences, e2e-test |
| `nemar admin import*` | OpenNeuro import, status, rollback, retry, verify, recover (issue #754, epic #967) |
| `nemar admin fleet` | drift, enforce, revalidate — governance across dataset repos (epic #713) |
| `nemar admin exemplar` | create, status, remint-dois — the staging exemplar fleet |
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@
"files": [
"dist",
"scripts/postinstall-check.mjs",
"scripts/recover-datasets.json",
"scripts/withdrawn-datasets.json",
"README.md",
"LICENSE"
],
Expand Down
286 changes: 253 additions & 33 deletions src/commands/admin.ts

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion src/commands/dataset.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1391,7 +1391,8 @@ Examples:
// Status command
datasetCommand
.command("status")
.description("Check status of a dataset")
.alias("view")
.description("Check status of a dataset (alias: view)")
.argument("<dataset-id>", "Dataset ID (e.g., nm000104)")
.option("--json", "Output as JSON for scripting")
.addHelpText(
Expand Down
78 changes: 78 additions & 0 deletions src/lib/api/admin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,84 @@ export async function hedSweepReset(): Promise<HedSweepResetResponse> {
);
}

/** One batch of the recording-stats backfill sweep (migration 0070, epic
* #1144 Phase 2, issue #1146, `POST /admin/datasets/recording-stats-sweep`). */
export interface RecordingStatsSweepBatchResponse {
processed: number;
/** Candidates whose zarr index yielded at least one measured recording. */
measured: number;
/** Candidates whose zarr index existed but had no measured recordings. */
unmeasured: number;
errors: { dataset_id: string; error: string }[];
/** Datasets still unswept; 0 when done, null if the count query failed. */
remaining: number | null;
}

/** Response of `?reset=1`: count of stamped rows cleared back to unswept. */
export interface RecordingStatsSweepResetResponse {
reset: number;
}

/** Run one bounded recording-stats sweep batch (default 50, server-clamped to [1,200]). */
export async function recordingStatsSweep(options?: {
limit?: number;
}): Promise<RecordingStatsSweepBatchResponse> {
const limit = options?.limit ?? 50;
return request<RecordingStatsSweepBatchResponse>(
`/admin/datasets/recording-stats-sweep?limit=${encodeURIComponent(String(limit))}`,
{ method: "POST", headers: { "Content-Type": "application/json" } },
true,
);
}

/** Clear every stamped recording-stats row so a corrected aggregator can re-sweep from scratch. */
export async function recordingStatsSweepReset(): Promise<RecordingStatsSweepResetResponse> {
return request<RecordingStatsSweepResetResponse>(
"/admin/datasets/recording-stats-sweep?reset=1",
{ method: "POST", headers: { "Content-Type": "application/json" } },
true,
);
}

/** One batch of the signal-defaults backfill sweep (migrations 0072/0073, epic
* #1144 Phase 2b, issue #1153, `POST /admin/datasets/signal-defaults-sweep`). */
export interface SignalDefaultsSweepBatchResponse {
processed: number;
/** Candidates whose probe found at least one usable sidecar key and wrote it. */
populated: number;
/** Candidates whose probe completed but found nothing to write. */
noData: number;
errors: { dataset_id: string; error: string }[];
/** Datasets still unswept; 0 when done, null if the count query failed. */
remaining: number | null;
}

/** Response of `?reset=1`: count of stamped rows cleared back to unswept. */
export interface SignalDefaultsSweepResetResponse {
reset: number;
}

/** Run one bounded signal-defaults sweep batch (default 15, server-clamped to [1,30]). */
export async function signalDefaultsSweep(options?: {
limit?: number;
}): Promise<SignalDefaultsSweepBatchResponse> {
const limit = options?.limit ?? 15;
return request<SignalDefaultsSweepBatchResponse>(
`/admin/datasets/signal-defaults-sweep?limit=${encodeURIComponent(String(limit))}`,
{ method: "POST", headers: { "Content-Type": "application/json" } },
true,
);
}

/** Clear every stamped signal-defaults row so a corrected probe can re-sweep from scratch. */
export async function signalDefaultsSweepReset(): Promise<SignalDefaultsSweepResetResponse> {
return request<SignalDefaultsSweepResetResponse>(
"/admin/datasets/signal-defaults-sweep?reset=1",
{ method: "POST", headers: { "Content-Type": "application/json" } },
true,
);
}

/** One batch of the data-integrity sweep (epic #967 Phase 3, #970,
* `POST /admin/datasets/data-integrity-sweep`). */
export interface DataIntegritySweepBatchResponse {
Expand Down
83 changes: 60 additions & 23 deletions src/lib/upload/transfer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -399,30 +399,67 @@ export async function uploadDataToS3(
writeUploadProgress(absolutePath, progress);
}

if (!isStepCompleted(progress, "s3_upload")) {
// Reconcile against ACTUAL git state, not just the progress file: the
// progress file can survive while .git does not match it (#884 review).
// Cheap index read; no content access.
let trackedPaths: Set<string>;
try {
trackedPaths = await listTrackedPaths(absolutePath);
} catch (indexError) {
console.log(chalk.red(`Failed to read the git index: ${errorDetail(indexError)}`));
console.log(chalk.dim(" Re-run the command to retry."));
return FAIL;
}
const addTargets = computeAddTargets(filesToUpload, dataFiles, trackedPaths);
const untrackedCount = addTargets.length - filesToUpload.length;
if (untrackedCount > 0 && isStepCompleted(progress, "tracking")) {
clearStepCompleted(progress, "tracking");
writeUploadProgress(absolutePath, progress);
console.log(
chalk.yellow(
` ${untrackedCount} data files are missing from git despite recorded progress; re-tracking them`,
),
);
}
// Reconcile against ACTUAL git state, not just the progress file: the
// progress file can survive while .git does not match it (#884 review).
// Cheap index read; no content access -- runs even when "s3_upload" is
// already marked complete (#1070). A prior run's completion stamp must
// not hide data files added to the dataset since then: filesToUpload
// (computed by the caller from the current file list) can be non-empty
// even though "s3_upload" is stamped, and computeAddTargets can also
// surface git-untracked files the stamp knows nothing about. Either way,
// reopening the gate below is what lets a resumed upload actually pick
// those files up instead of silently skipping them forever.
let trackedPaths: Set<string>;
try {
trackedPaths = await listTrackedPaths(absolutePath);
} catch (indexError) {
console.log(chalk.red(`Failed to read the git index: ${errorDetail(indexError)}`));
console.log(chalk.dim(" Re-run the command to retry."));
return FAIL;
}
const addTargets = computeAddTargets(filesToUpload, dataFiles, trackedPaths);
const untrackedCount = addTargets.length - filesToUpload.length;
const reopenTracking = untrackedCount > 0 && isStepCompleted(progress, "tracking");
const reopenUpload = addTargets.length > 0 && isStepCompleted(progress, "s3_upload");
if (reopenTracking) {
clearStepCompleted(progress, "tracking");
}
if (reopenUpload) {
clearStepCompleted(progress, "s3_upload");
// Reopening s3_upload for new content must also reopen the finalize
// steps that depend on it (Step 11 saveDatasetStep / Step 12
// pushMetadata in commands/dataset.ts, upload/finalize.ts). Both are
// gated by their own persisted step and stay stamped complete from
// the PRIOR run, so without this a new file's git-annex pointer would
// reach S3 here but never be committed or pushed -- and
// printUploadSuccess still clears progress and reports "Upload
// complete!" with that pointer sitting only in the working tree
// (review finding, critical: a run interrupted between pushMetadata
// and printUploadSuccess -- e.g. mid deployCiStep -- leaves both
// stamped true on disk with progress never cleared).
clearStepCompleted(progress, "dataset_save");
clearStepCompleted(progress, "github_push");
}
// One combined write instead of one per reopened step.
if (reopenTracking || reopenUpload) {
writeUploadProgress(absolutePath, progress);
}
if (reopenTracking) {
console.log(
chalk.yellow(
` ${untrackedCount} data files are missing from git despite recorded progress; re-tracking them`,
),
);
}
if (reopenUpload) {
console.log(
chalk.yellow(
` ${addTargets.length} data file(s) still need upload despite a completed s3_upload step; resuming`,
),
);
}

if (!isStepCompleted(progress, "s3_upload")) {
if (addTargets.length > 0) {
// Track data files with git-annex before anything else: the add can
// take hours on multi-TB datasets, so it must not eat into the 2h STS
Expand Down
8 changes: 8 additions & 0 deletions test/api-export-surface.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ const MODULE_EXPORTS: Record<string, string[]> = {
"publishDataset",
"publishVersionDoi",
"publishZarrCatalog",
"recordingStatsSweep",
"recordingStatsSweepReset",
"reindexBulk",
"reindexDataset",
"remintExemplarDois",
Expand All @@ -131,6 +133,8 @@ const MODULE_EXPORTS: Record<string, string[]> = {
"revokeUser",
"rollbackImport",
"sendBroadcast",
"signalDefaultsSweep",
"signalDefaultsSweepReset",
"syncCi",
"updateDoi",
"updateEmailPreferences",
Expand Down Expand Up @@ -166,6 +170,10 @@ const POST_SPLIT_ADDITIONS = [
"doctorScan", // #1130, CLI wrapper for POST /admin/doctor/scan
"getFacets", // #1149, epic #1144 phase 5b: GET /datasets/facets for shell completion
"publishZarrCatalog", // #1062, epic #1181 phase 2: POST /admin/zarr-catalog/publish
"recordingStatsSweep", // #1194, CLI wrapper for POST /admin/datasets/recording-stats-sweep
"recordingStatsSweepReset", // #1194, CLI wrapper for the recording-stats sweep reset
"signalDefaultsSweep", // #1194, CLI wrapper for POST /admin/datasets/signal-defaults-sweep
"signalDefaultsSweepReset", // #1194, CLI wrapper for the signal-defaults sweep reset
"zarrFidelitySweep", // #1068, epic #1181 phase 8: POST /admin/datasets/zarr-fidelity-sweep
];

Expand Down
30 changes: 30 additions & 0 deletions test/cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -686,6 +686,36 @@ describe("CLI Dataset Status", () => {
expect(result.dataset_id).toBe("nm099999");
expect(result.name).toBeDefined();
});

test("nemar dataset view is an alias for dataset status", async () => {
const { stdout: viewHelp, exitCode: viewExit } = await runCli(["dataset", "view", "--help"]);
const { stdout: statusHelp, exitCode: statusExit } = await runCli([
"dataset",
"status",
"--help",
]);

expect(viewExit).toBe(0);
expect(statusExit).toBe(0);
// Both resolve to the same command implementation, so their help text
// (description, options, examples) is byte-identical.
expect(viewHelp).toContain("Check status of a dataset");
expect(viewHelp).toContain("--json");
expect(viewHelp).toBe(statusHelp);
});

test("nemar dataset --help lists view as an alias of status", async () => {
const { stdout, exitCode } = await runCli(["dataset", "--help"]);

expect(exitCode).toBe(0);
expect(stdout).toContain("status|view");
});

test("nemar dataset view with non-existent dataset behaves like status", async () => {
const { stdout, exitCode } = await runCli(["dataset", "view", "nm099998"]);

expect(stdout).toContain("not found");
});
});

describe("CLI Dataset List", () => {
Expand Down
7 changes: 6 additions & 1 deletion test/curl-stream-copy.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,12 @@ beforeAll(() => {

afterAll(() => {
server.stop(true);
process.env.PATH = previousPath;
// Guarded restore (#1175): assigning `undefined` to a process.env key
// coerces to the literal string "undefined" instead of deleting it, which
// would poison PATH for every test running later in the same `bun test`
// process (test/ + backend/test/ share one process at the root).
if (previousPath === undefined) delete process.env.PATH;
else process.env.PATH = previousPath;
rmSync(binDir, { recursive: true, force: true });
});

Expand Down
Loading
Loading