From bf1ae5678e9331ed0604be25e4d6c49d90636177 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:30:09 -0700 Subject: [PATCH 01/15] feat: zarr fidelity verification sweep service --- backend/src/services/channel-montage.ts | 19 + backend/src/services/zarr-fidelity-sweep.ts | 763 ++++++++++++++++++++ backend/test/zarr-fidelity-sweep.test.ts | 663 +++++++++++++++++ 3 files changed, 1445 insertions(+) create mode 100644 backend/src/services/zarr-fidelity-sweep.ts create mode 100644 backend/test/zarr-fidelity-sweep.test.ts diff --git a/backend/src/services/channel-montage.ts b/backend/src/services/channel-montage.ts index aa060f435..61fbf7f9e 100644 --- a/backend/src/services/channel-montage.ts +++ b/backend/src/services/channel-montage.ts @@ -152,6 +152,25 @@ export function parsePowerLineFrequency(content: string): number | null { return v === 50 || v === 60 ? v : null; } +/** + * Read `RecordingDuration` (seconds) from a `*_eeg.json`/`*_ieeg.json`/ + * `*_meg.json`/`*_emg.json` sidecar for the zarr fidelity sweep (issue + * #1068, epic #1181 phase 8) -- ground truth for `duration_mismatch`. + * Same shape as parseSamplingFrequency: null on absent/invalid JSON, a + * non-number, a non-finite value, or <= 0 (a recording cannot have zero or + * negative duration). + */ +export function parseRecordingDuration(content: string): number | null { + let obj: Record; + try { + obj = JSON.parse(content) as Record; + } catch { + return null; + } + const v = obj.RecordingDuration; + return typeof v === "number" && Number.isFinite(v) && v > 0 ? v : null; +} + /** * Reject BIDS's "not applicable" placeholder and blank strings. Shared by * the two free-text sidecar parsers below; not exported -- an external diff --git a/backend/src/services/zarr-fidelity-sweep.ts b/backend/src/services/zarr-fidelity-sweep.ts new file mode 100644 index 000000000..e0d1a99c5 --- /dev/null +++ b/backend/src/services/zarr-fidelity-sweep.ts @@ -0,0 +1,763 @@ +/** + * Zarr fidelity verification sweep (issue #1068, epic #1181 phase 8). + * + * The converter's own `channel_count_mismatch` gate (scripts/zarr/generate_zarr.py) + * withholds an unfaithful store AT CONVERSION TIME, but nothing re-checks a + * store already published, and `has_zarr` (dataset-filters.ts) means only + * "converted" -- not "agrees with the dataset's own BIDS metadata". This + * sweep is the standing verification gate #1068 asked for: it re-derives + * ground truth per sampled recording from the dataset's own `channels.tsv` + * and modality sidecar (`*_eeg.json` / `*_ieeg.json` / `*_meg.json` / + * `*_emg.json`) and compares it against what the published index.json + * claims, writing a per-dataset verdict into `sweep_stamps` (ADR 0034/0035 + * -- no new column). `has_zarr` itself is UNCHANGED (dataset-filters.ts); + * `has_zarr_verified` is the new, stricter filter this sweep feeds. + * + * WHY RAW.GITHUBUSERCONTENT.COM, NOT THE GITHUB API. `channels.tsv` and + * every `*_.json` sidecar are metadata extensions that ADR 0031 + * guarantees NEVER annex (`*_motion.tsv` is the one exception, irrelevant + * here) -- they are always plain git blobs. Fetching them by RAW CONTENT URL + * (`raw.githubusercontent.com////`) is therefore + * sufficient, and is a plain unauthenticated public-CDN GET: no GitHub App + * installation token, no PAT, no shared API rate limit, and it 404s outright + * for a private repo. That is a materially different exposure than + * `signal-defaults-sweep.ts`'s `getBidsTreeStats`, which is PROD-ONLY + * specifically because it spends the shared GitHub App/PAT reading the + * shared `nemarDatasets` org (see that sweep's cron-wiring comment in + * index.ts). This sweep touches only S3 (the index) and this same public, + * credential-free content host, so -- like `recording-stats-sweep.ts` -- + * it is safe on the non-production cron: AGENTS.md's dev-cron fence exists + * to stop a job from emailing a real user, dispatching GitHub WORK (a + * mutation, or an authenticated read that spends the shared org's quota) + * against `nemarDatasets`, or mutating a real DOI/prod-bucket object; this + * sweep does none of those. + * + * NEAREST-FIRST, NOT A FULL BIDS-INHERITANCE WALK. generate_zarr.py's own + * `expected_channel_count_for` / `power_line_frequency_for` resolve the + * winning sidecar against the repo's COMPLETE HEAD file list (an ancestor + * directory, most-specific-entities-first). That list isn't available here + * without either a full git tree walk (the GitHub API exposure this module + * exists to avoid) or the version manifest (versioned by published tag, not + * by the store's own `source_commit`). `bidsSidecarCandidates` below is the + * brief's sanctioned fallback: a small, fixed, nearest-first candidate list + * (recording directory, subject directory, dataset root) that covers the + * placements real BIDS datasets actually use, tried in order and cached per + * dataset run so repeated subject-/root-level defaults cost one fetch each. + * `MAX_SIDECAR_FETCHES_PER_DATASET` is a hard safety net on top of that + * cache so one dataset with nothing but misses can't blow the Worker's + * subrequest budget. + * + * Modelled on recording-stats-sweep.ts's shape: candidate SQL, a bounded + * per-invocation cap, three-way per-dataset error handling (throw/d1-error + * keeps the row a candidate; a real verdict stamps it), and NOTHING else + * written outside `sweep_stamps`. + */ + +import { AwsClient } from "aws4fetch"; +import { auditLogStatement } from "../db/audit-log.js"; +import type { Bindings } from "../types/bindings.js"; +import { + parseChannelsTsv, + parseRecordingDuration, + parseSamplingFrequency, +} from "./channel-montage.js"; +import { ORG_NAME, sampleEvenly } from "./github.js"; +import type { PresignedUrlOptions } from "./s3.js"; + +/** Per-modality serving-rate cap the converter applies (generate_zarr.py's + * `MODALITY_RATES`, mirrored here -- kept in sync deliberately rather than + * imported, since this is Python and this is the TypeScript verification + * side reading the SAME published fact back out of the index). Keys are + * UPPERCASE to match a `group.modality` value uppercased at lookup time. */ +export const ZARR_FIDELITY_MODALITY_RATE_CAPS: Record = { + EEG: 250, + MEG: 250, + IEEG: 1000, + EMG: 1000, +}; + +/** Bounded sample size (decision 1): every store when the dataset has 40 or + * fewer, else 40 spread evenly by path order. */ +export const ZARR_FIDELITY_MAX_SAMPLE_STORES = 40; + +/** Hard cap on `zarr_verify_examples`: at most this many {path, code} + * entries, and the cap below also bounds the serialized byte size. */ +export const ZARR_FIDELITY_MAX_EXAMPLES = 20; + +/** Hard byte cap on the serialized `zarr_verify_examples` JSON array. */ +export const ZARR_FIDELITY_MAX_EXAMPLES_BYTES = 4096; + +/** Safety net on top of the per-dataset resolution cache: at most this many + * sidecar candidate fetches (across every sampled store) per dataset per + * run, so a dataset with nothing but 404s can't consume an unbounded + * subrequest budget. Sized generously above the common case (3 candidates + * x 2 suffix-kinds x a modest number of distinct directories among the + * sample, after subject-/root-level candidates are cached across stores). */ +export const ZARR_FIDELITY_MAX_SIDECAR_FETCHES = 150; + +/** Default / max datasets per invocation (decision 1). */ +export const ZARR_FIDELITY_SWEEP_DEFAULT = 25; +export const ZARR_FIDELITY_SWEEP_MAX = 100; + +const GITHUB_RAW_ORIGIN = "https://raw.githubusercontent.com"; + +export type ZarrFidelityVerdict = "verified" | "failed" | "unverifiable"; + +export interface ZarrFidelityMismatchExample { + path: string; + code: "channel_count_mismatch" | "duration_mismatch" | "rate_mismatch"; +} + +/** + * Candidates (decision 1): converted-with-stores datasets never verified, OR + * verified against a commit that is no longer the dataset's current + * `zarr_source_commit` -- a re-conversion re-arms verification. `github_repo + * IS NOT NULL` is a defensive narrowing beyond the brief's literal predicate + * (not a change to it): a row with no repo has nothing this sweep could ever + * fetch a sidecar from, so it would only ever resolve `unverifiable` -- + * excluding it here just avoids wasted candidate slots, same reasoning + * channel-montage-sweep / signal-defaults-sweep already apply to their own + * candidate sets. + */ +export const ZARR_FIDELITY_SWEEP_CANDIDATE_SQL = `SELECT dataset_id, github_repo FROM datasets + WHERE zarr_status = 'ready' + AND zarr_store_count > 0 + AND github_repo IS NOT NULL + AND ( + json_extract(sweep_stamps, '$.zarr_verified_at') IS NULL + OR json_extract(sweep_stamps, '$.zarr_verified_commit') != zarr_source_commit + ) + ORDER BY dataset_id + LIMIT ?`; + +export const ZARR_FIDELITY_SWEEP_REMAINING_SQL = `SELECT COUNT(*) AS n FROM datasets + WHERE zarr_status = 'ready' + AND zarr_store_count > 0 + AND github_repo IS NOT NULL + AND ( + json_extract(sweep_stamps, '$.zarr_verified_at') IS NULL + OR json_extract(sweep_stamps, '$.zarr_verified_commit') != zarr_source_commit + )`; + +/** + * The per-candidate stamp write, exported so a test drives the exact SQL + * text (`.rules/testing.md`: never hand-copy). Writes ONLY `sweep_stamps` + * (decision 1 / ADR 0034) -- no other `datasets` column changes, on any + * verdict. `json(?)` wraps the examples parameter so it lands as a nested + * JSON array, not an escaped string (`json_set(x, '$.k', ?)` with a raw TEXT + * bind would store the literal characters `[...]`, unreadable by + * `json_extract('$.k[0]')`). Bind order: commit, status, examples-json, + * sampled, checked, dataset_id. + */ +export const ZARR_FIDELITY_SWEEP_STAMP_SQL = `UPDATE datasets + SET sweep_stamps = json_set( + COALESCE(sweep_stamps, '{}'), + '$.zarr_verified_at', datetime('now'), + '$.zarr_verified_commit', ?, + '$.zarr_verify_status', ?, + '$.zarr_verify_examples', json(?), + '$.zarr_verify_sampled', ?, + '$.zarr_verify_checked', ? + ) + WHERE dataset_id = ?`; + +/** `PresignedUrlOptions` plus a TEST-ONLY origin override, same idiom as + * `zarr-catalog.ts`'s `ZarrCatalogS3Options` -- a test points signing at a + * local `Bun.serve()` receiver instead of mocking `fetch`. */ +export interface ZarrFidelityS3Options extends PresignedUrlOptions { + endpointUrl?: string; +} + +/** Shape of a single channel-group entry this module reads out of a zarr + * index store (biosigIO's `store_metadata` contract, unchanged between + * index format v1 and v3 per the phase 7 brief -- only NEW top-level/store + * fields were added in v3, so reading only these four keys and ignoring + * everything else is tolerant of both, mirroring s3.ts's + * `ZarrIndexGroupJson`/`ZarrIndexStoreJson` version-agnostic subset + * approach). */ +export interface ZarrFidelityGroupJson { + n_channels?: unknown; + duration_s?: unknown; + rate?: unknown; + modality?: unknown; +} + +export interface ZarrFidelityStoreJson { + path?: unknown; + zarr?: unknown; + groups?: unknown; +} + +/** Shape of the parsed `/zarr/index.json` this module reads (subset, + * v1/v3-tolerant -- see the module doc). */ +export interface ZarrFidelityIndexJson { + source_commit?: unknown; + store_count?: unknown; + stores?: unknown; +} + +function toFiniteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +/** 40 lowercase hex characters: the phase 7 invariant for a valid, fetchable + * git ref. An index whose `source_commit` fails this (absent, empty, or the + * pre-phase-7 empty-string bug) has nothing this sweep can check a sample + * against. */ +const FULL_COMMIT_RE = /^[0-9a-f]{40}$/i; + +/** + * Nearest-first BIDS sidecar candidate paths for one recording (decision 1's + * "reuse the existing sidecar resolution helper if one exists, otherwise + * implement nearest-first resolution" -- see the module doc for why this is + * a bounded heuristic rather than generate_zarr.py's full HEAD-file-list + * walk). `suffix` is the sidecar's own trailing name, e.g. `"channels.tsv"` + * or `"eeg.json"`. + * + * Three placements, nearest first, deduplicated: + * 1. the recording's own directory, full BIDS entities (a per-recording + * override); + * 2. the subject directory (session entity included when present), a + * dataset's most common "shared default" placement; + * 3. the dataset root, bare (the dataset-wide default). + * + * Exported for direct unit testing. + */ +export function bidsSidecarCandidates(recordingPath: string, suffix: string): string[] { + const parts = recordingPath.split("/").filter((p) => p.length > 0); + if (parts.length === 0) return [suffix]; + + const filename = parts[parts.length - 1]; + const dir = parts.slice(0, -1).join("/"); + const subjectDir = parts[0].startsWith("sub-") ? parts[0] : null; + + const dot = filename.lastIndexOf("."); + const stem = dot > 0 ? filename.slice(0, dot) : filename; + // Every underscore-token except the last (the suffix, e.g. "eeg") that + // looks like a BIDS entity (`key-value`), in filename order. + const entityTokens = stem + .split("_") + .slice(0, -1) + .filter((t) => t.includes("-")); + const subjectEntity = entityTokens.find((t) => t.startsWith("sub-")) ?? null; + const sessionEntity = entityTokens.find((t) => t.startsWith("ses-")) ?? null; + + const candidates: string[] = []; + const add = (candidateDir: string, entities: string[]): void => { + const name = entities.length > 0 ? `${entities.join("_")}_${suffix}` : suffix; + const path = candidateDir ? `${candidateDir}/${name}` : name; + if (!candidates.includes(path)) candidates.push(path); + }; + + add(dir, entityTokens); + if (subjectDir) { + const subjectEntities = [subjectEntity, sessionEntity].filter((e): e is string => e !== null); + add(subjectDir, subjectEntities); + } + add("", []); + + return candidates; +} + +/** Total channels a store serves, summed across its groups -- mirrors + * generate_zarr.py's `store_total_channels` exactly (the SAME rule the + * converter's own conversion-time gate uses), so a post-hoc disagreement + * here reflects real drift, not a differently-derived total. */ +export function zarrFidelityStoreChannelTotal(groups: readonly ZarrFidelityGroupJson[]): number { + return groups.reduce((sum, g) => sum + (toFiniteNumber(g.n_channels) ?? 0), 0); +} + +/** Longest `duration_s` across a set of groups (MAX within a store, mirroring + * s3.ts's `aggregateRecordingStats` rule for the identical shape), or null + * when none of them measured a duration. */ +export function zarrFidelityStoreDuration(groups: readonly ZarrFidelityGroupJson[]): number | null { + let max: number | null = null; + for (const g of groups) { + const d = toFiniteNumber(g.duration_s); + if (d === null) continue; + max = max === null ? d : Math.max(max, d); + } + return max; +} + +function indexObjectUrl(options: ZarrFidelityS3Options, datasetId: string): string { + const origin = ( + options.endpointUrl ?? `https://${options.bucket}.s3.${options.region}.amazonaws.com` + ).replace(/\/+$/, ""); + const key = `${datasetId}/zarr/index.json`; + const encodedKey = key + .split("/") + .map((seg) => encodeURIComponent(seg)) + .join("/"); + return `${origin}/${encodedKey}`; +} + +/** + * Signed GET of a dataset's `index.json`, own implementation rather than + * `s3.ts`'s `getZarrIndex` (which this module must not edit, and which does + * not expose the full `stores[]`/`groups[]` shape this sweep needs) -- + * mirrors that function's 404/403-as-absent, other-non-2xx-throws contract. + * `fetchImpl` is the DI seam a test substitutes with a local receiver. + */ +async function fetchFidelityIndex( + options: ZarrFidelityS3Options, + datasetId: string, + fetchImpl: typeof fetch, +): Promise { + const aws = new AwsClient({ + accessKeyId: options.accessKeyId, + secretAccessKey: options.secretAccessKey, + region: options.region, + service: "s3", + }); + const signed = await aws.sign(indexObjectUrl(options, datasetId), { method: "GET" }); + const response = await fetchImpl(signed); + if (response.status === 404 || response.status === 403) return null; + if (!response.ok) { + throw new Error( + `zarr-fidelity-sweep: HTTP ${response.status} fetching index.json for ${datasetId}`, + ); + } + return (await response.json()) as ZarrFidelityIndexJson; +} + +/** + * One candidate fetch against the public, credential-free content host (see + * the module doc for why this is dev-safe). A 404 is the expected "not at + * this candidate" signal; any other non-2xx or network error is logged and + * treated the same as a miss for resolution purposes -- fail-open, mirroring + * generate_zarr.py's `expected_channel_count_for`, which turns the gate off + * for a recording rather than treating an unreadable ground truth as a + * mismatch. + */ +async function fetchSidecarCandidate( + base: string, + repo: string, + commit: string, + path: string, + fetchImpl: typeof fetch, +): Promise { + const encoded = path + .split("/") + .map((seg) => encodeURIComponent(seg)) + .join("/"); + const url = `${base}/${ORG_NAME}/${repo}/${commit}/${encoded}`; + let response: Response; + try { + response = await fetchImpl(url); + } catch (err) { + console.warn( + `[zarr-fidelity-sweep] network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`, + ); + return null; + } + if (response.status === 404) return null; + if (!response.ok) { + console.warn(`[zarr-fidelity-sweep] non-2xx ${response.status} fetching ${url}`); + return null; + } + return response.text(); +} + +/** Per-dataset-run resolution state: a content cache (path -> content, or + * null for a confirmed miss) shared across every sampled store so a + * subject-/root-level default is fetched at most once, and a hard fetch + * budget as the safety net described in the module doc. */ +interface FidelityRunContext { + cache: Map; + budget: { remaining: number }; + githubRawBase: string; + repo: string; + commit: string; + fetchImpl: typeof fetch; +} + +async function resolveSidecar( + ctx: FidelityRunContext, + recordingPath: string, + suffix: string, +): Promise<{ path: string; content: string } | null> { + for (const candidate of bidsSidecarCandidates(recordingPath, suffix)) { + if (ctx.cache.has(candidate)) { + const cached = ctx.cache.get(candidate) ?? null; + if (cached !== null) return { path: candidate, content: cached }; + continue; + } + if (ctx.budget.remaining <= 0) return null; + ctx.budget.remaining--; + const content = await fetchSidecarCandidate( + ctx.githubRawBase, + ctx.repo, + ctx.commit, + candidate, + ctx.fetchImpl, + ); + ctx.cache.set(candidate, content); + if (content !== null) return { path: candidate, content }; + } + return null; +} + +/** + * Sample selection (decision 1): every store when the dataset has at most + * {@link ZARR_FIDELITY_MAX_SAMPLE_STORES}, else that many spread evenly by + * path order (`sampleEvenly`, reused from bids-tree.ts) plus every store + * whose group has `n_channels === 1` -- a single-channel recording is the + * shape most likely to reveal a truncation bug (biosigio#110's own + * signature), so it is never left to chance by the even spread. Exported + * for direct unit testing. + */ +export function zarrFidelitySelectSample( + stores: readonly ZarrFidelityStoreJson[], +): (ZarrFidelityStoreJson & { path: string })[] { + const withPath = stores.filter( + (s): s is ZarrFidelityStoreJson & { path: string } => typeof s.path === "string", + ); + const sorted = [...withPath].sort((a, b) => a.path.localeCompare(b.path)); + const spread = sampleEvenly(sorted, ZARR_FIDELITY_MAX_SAMPLE_STORES); + const forced = sorted.filter((s) => { + const groups = Array.isArray(s.groups) ? (s.groups as ZarrFidelityGroupJson[]) : []; + return groups.some((g) => toFiniteNumber(g.n_channels) === 1); + }); + + const seen = new Set(); + const sample: (ZarrFidelityStoreJson & { path: string })[] = []; + for (const s of [...spread, ...forced]) { + const key = `${s.path}::${typeof s.zarr === "string" ? s.zarr : ""}`; + if (seen.has(key)) continue; + seen.add(key); + sample.push(s); + } + return sample; +} + +/** + * Verify one sampled store against its own ground truth. Returns whether any + * ground truth was reachable at all (`checked`) and the mismatches found (a + * store can be `checked` with zero mismatches). + */ +async function verifyStore( + store: ZarrFidelityStoreJson & { path: string }, + ctx: FidelityRunContext, +): Promise<{ checked: boolean; mismatches: ZarrFidelityMismatchExample[] }> { + const mismatches: ZarrFidelityMismatchExample[] = []; + let checked = false; + const groups = (Array.isArray(store.groups) ? store.groups : []) as ZarrFidelityGroupJson[]; + + // Channel count: total store channels vs. channels.tsv's data-row count. + const channelsHit = await resolveSidecar(ctx, store.path, "channels.tsv"); + if (channelsHit) { + const parsed = parseChannelsTsv(channelsHit.content); + if (parsed) { + checked = true; + const total = zarrFidelityStoreChannelTotal(groups); + if (total < parsed.count) { + mismatches.push({ path: store.path, code: "channel_count_mismatch" }); + } + } + } + + // Duration + rate, resolved once per distinct modality among this store's + // groups (a mixed-modality store is rare, but keeps this correct if one + // ever exists). + const byModality = new Map(); + for (const g of groups) { + const modality = typeof g.modality === "string" ? g.modality.toLowerCase() : null; + if (!modality) continue; + const list = byModality.get(modality) ?? []; + list.push(g); + byModality.set(modality, list); + } + + for (const [modality, modalityGroups] of byModality) { + const sidecarHit = await resolveSidecar(ctx, store.path, `${modality}.json`); + if (!sidecarHit) continue; + checked = true; + + const recordingDuration = parseRecordingDuration(sidecarHit.content); + if (recordingDuration !== null) { + const storeDuration = zarrFidelityStoreDuration(modalityGroups); + if (storeDuration !== null && Math.abs(storeDuration - recordingDuration) > 1) { + mismatches.push({ path: store.path, code: "duration_mismatch" }); + } + } + + const samplingFrequency = parseSamplingFrequency(sidecarHit.content); + const cap = ZARR_FIDELITY_MODALITY_RATE_CAPS[modality.toUpperCase()]; + if (samplingFrequency !== null && cap !== undefined) { + const expectedRate = Math.min(samplingFrequency, cap); + for (const g of modalityGroups) { + const rate = toFiniteNumber(g.rate); + if (rate === null) continue; + if (Math.abs(rate - expectedRate) > 0.5) { + mismatches.push({ path: store.path, code: "rate_mismatch" }); + break; // one example per store per code is enough + } + } + } + } + + return { checked, mismatches }; +} + +/** Append to a bounded examples array, respecting both the entry-count cap + * and the serialized-byte cap (decision 1: "hard cap 20 entries and 4 KB"). + * Exported for direct unit testing. */ +export function pushZarrFidelityExample( + examples: ZarrFidelityMismatchExample[], + entry: ZarrFidelityMismatchExample, +): void { + if (examples.length >= ZARR_FIDELITY_MAX_EXAMPLES) return; + const candidate = [...examples, entry]; + const bytes = new TextEncoder().encode(JSON.stringify(candidate)).length; + if (bytes > ZARR_FIDELITY_MAX_EXAMPLES_BYTES) return; + examples.push(entry); +} + +interface DatasetVerificationOutcome { + /** null means "could not even produce a verdict" -- an infra error the + * caller should record and leave the row an untouched candidate. */ + status: ZarrFidelityVerdict | null; + commit: string | null; + sampled: number; + checked: number; + examples: ZarrFidelityMismatchExample[]; + error?: string; +} + +async function verifyDataset( + datasetId: string, + repo: string, + s3Options: ZarrFidelityS3Options, + githubRawBase: string, + fetchIndexImpl: typeof fetch, + fetchSidecarImpl: typeof fetch, +): Promise { + let index: ZarrFidelityIndexJson | null; + try { + index = await fetchFidelityIndex(s3Options, datasetId, fetchIndexImpl); + } catch (err) { + return { + status: null, + commit: null, + sampled: 0, + checked: 0, + examples: [], + error: `s3: ${err instanceof Error ? err.message : String(err)}`, + }; + } + if (!index) { + return { + status: null, + commit: null, + sampled: 0, + checked: 0, + examples: [], + error: "zarr_status=ready but index.json is absent", + }; + } + + const commit = + typeof index.source_commit === "string" && FULL_COMMIT_RE.test(index.source_commit) + ? index.source_commit + : null; + const stores = (Array.isArray(index.stores) ? index.stores : []) as ZarrFidelityStoreJson[]; + const sample = zarrFidelitySelectSample(stores); + + if (!commit) { + // No fetchable ref -- ADR 0005: report the gap, don't fake a verdict. + return { + status: "unverifiable", + commit: null, + sampled: sample.length, + checked: 0, + examples: [], + }; + } + + const ctx: FidelityRunContext = { + cache: new Map(), + budget: { remaining: ZARR_FIDELITY_MAX_SIDECAR_FETCHES }, + githubRawBase, + repo, + commit, + fetchImpl: fetchSidecarImpl, + }; + + let checkedCount = 0; + const examples: ZarrFidelityMismatchExample[] = []; + for (const store of sample) { + const { checked, mismatches } = await verifyStore(store, ctx); + if (checked) checkedCount++; + for (const m of mismatches) pushZarrFidelityExample(examples, m); + } + + const status: ZarrFidelityVerdict = + checkedCount === 0 ? "unverifiable" : examples.length > 0 ? "failed" : "verified"; + return { status, commit, sampled: sample.length, checked: checkedCount, examples }; +} + +export interface ZarrFidelityDatasetResult { + dataset_id: string; + verdict: ZarrFidelityVerdict; + sampled: number; + checked: number; + examples: ZarrFidelityMismatchExample[]; +} + +export interface ZarrFidelitySweepResult { + processed: number; + verified: number; + failed: number; + unverifiable: number; + /** Per-dataset outcomes for every candidate a verdict was actually reached + * for (decision 3: "include the dataset in the sweep's response ... so + * the observability dashboard can pick it up"). Datasets that errored + * before a verdict was reached are in `errors` instead, not here. */ + results: ZarrFidelityDatasetResult[]; + errors: { dataset_id: string; error: string }[]; + /** Candidates still unverified after this run; null if the count query + * failed. */ + remaining: number | null; +} + +/** + * Run one bounded pass of the zarr fidelity sweep (decision 1): fetch up to + * `limit` candidates' full index, sample a bounded set of stores, verify + * each against its own ground truth, and stamp a verdict. Shared by + * `POST /admin/datasets/zarr-fidelity-sweep` and the daily cron (both prod + * and non-prod -- see the module doc for why this is dev-safe) so the two + * callers can never drift. + * + * A dataset whose index can't even be fetched (S3 infra error, or + * `zarr_status='ready'` but index.json absent) is recorded in `errors` and + * left completely untouched -- no stamp write, stays a candidate for the + * next run -- exactly recording-stats-sweep's "throw -> stays a candidate" + * handling for the identical shape of failure. + * + * Test-only DI seams (`s3Options.endpointUrl`, `githubRawBase`, + * `fetchIndexImpl`, `fetchSidecarImpl`) mirror `zarr-catalog.ts`'s + * `endpointUrl` / `runRecordingStatsSweep`'s `fetchIndex` idiom: every real + * caller omits them, so production always resolves the real S3 host and the + * real `raw.githubusercontent.com`. + * + * Throws only if the candidate query itself fails. Per-dataset failures are + * collected into `errors`, never thrown. + */ +export async function runZarrFidelitySweep( + env: Bindings, + opts?: { + limit?: number; + s3Options?: { endpointUrl?: string }; + githubRawBase?: string; + fetchIndexImpl?: typeof fetch; + fetchSidecarImpl?: typeof fetch; + }, +): Promise { + const requested = opts?.limit ?? ZARR_FIDELITY_SWEEP_DEFAULT; + const limit = Math.min(Math.max(requested, 1), ZARR_FIDELITY_SWEEP_MAX); + + const rows = await env.DB.prepare(ZARR_FIDELITY_SWEEP_CANDIDATE_SQL) + .bind(limit) + .all<{ dataset_id: string; github_repo: string }>(); + const candidates = rows.results ?? []; + + const s3Options: ZarrFidelityS3Options = { + bucket: env.S3_BUCKET, + region: env.AWS_REGION, + accessKeyId: env.AWS_ACCESS_KEY_ID, + secretAccessKey: env.AWS_SECRET_ACCESS_KEY, + endpointUrl: opts?.s3Options?.endpointUrl, + }; + const githubRawBase = opts?.githubRawBase ?? GITHUB_RAW_ORIGIN; + const fetchIndexImpl = opts?.fetchIndexImpl ?? fetch; + const fetchSidecarImpl = opts?.fetchSidecarImpl ?? fetch; + + let verified = 0; + let failed = 0; + let unverifiable = 0; + const results: ZarrFidelityDatasetResult[] = []; + const errors: { dataset_id: string; error: string }[] = []; + + for (const { dataset_id, github_repo } of candidates) { + const repo = github_repo.split("/")[1] ?? github_repo; + const outcome = await verifyDataset( + dataset_id, + repo, + s3Options, + githubRawBase, + fetchIndexImpl, + fetchSidecarImpl, + ); + + if (outcome.status === null) { + errors.push({ dataset_id, error: outcome.error ?? "zarr-fidelity-sweep: unknown error" }); + continue; + } + + try { + await env.DB.prepare(ZARR_FIDELITY_SWEEP_STAMP_SQL) + .bind( + outcome.commit, + outcome.status, + JSON.stringify(outcome.examples), + outcome.sampled, + outcome.checked, + dataset_id, + ) + .run(); + } catch (err) { + errors.push({ + dataset_id, + error: `d1 write: ${err instanceof Error ? err.message : String(err)}`, + }); + continue; + } + + if (outcome.status === "verified") verified++; + else if (outcome.status === "failed") failed++; + else unverifiable++; + results.push({ + dataset_id, + verdict: outcome.status, + sampled: outcome.sampled, + checked: outcome.checked, + examples: outcome.examples, + }); + + if (outcome.status === "failed") { + // Escalation (decision 3): audit row + the dataset already rides along + // in `results` above for the response/cron log line. Best-effort -- + // an audit-write failure must not undo the verdict just stamped. + try { + await auditLogStatement(env.DB, { + userId: null, + action: "zarr_fidelity_failed", + resourceType: "dataset", + resourceId: dataset_id, + details: JSON.stringify({ + sampled: outcome.sampled, + checked: outcome.checked, + examples: outcome.examples, + }), + }).run(); + } catch (err) { + console.error(`[zarr-fidelity-sweep] audit log write failed for ${dataset_id}:`, err); + } + } + } + + const remainingRow = await env.DB.prepare(ZARR_FIDELITY_SWEEP_REMAINING_SQL) + .first<{ n: number }>() + .catch(() => null); + + return { + processed: candidates.length, + verified, + failed, + unverifiable, + results, + errors, + remaining: remainingRow?.n ?? null, + }; +} diff --git a/backend/test/zarr-fidelity-sweep.test.ts b/backend/test/zarr-fidelity-sweep.test.ts new file mode 100644 index 000000000..a4907e112 --- /dev/null +++ b/backend/test/zarr-fidelity-sweep.test.ts @@ -0,0 +1,663 @@ +/** + * Real behavioral tests for `runZarrFidelitySweep` (issue #1068, epic #1181 + * phase 8) -- the entry point BOTH real callers + * (`POST /admin/datasets/zarr-fidelity-sweep` and the daily cron) use. + * + * Drives the REAL function against a real D1 (bun:sqlite behind realD1), + * substituting the two true network boundaries -- the S3 index GET and the + * raw.githubusercontent.com sidecar GET -- with ONE real local `Bun.serve()` + * receiver, via the `s3Options.endpointUrl` / `githubRawBase` DI seams + * (`zarr-catalog.ts`'s `endpointUrl` idiom). Everything else -- the + * candidate query, sample selection, nearest-first resolution, the mismatch + * rules, the stamp write, the audit row -- runs for real. + * + * Per `.rules/testing.md`: never hand-copy a SQL statement -- the assertions + * below read the stamped `sweep_stamps` JSON back with `json_extract` + * against the REAL column the sweep wrote, not a re-implemented predicate. + */ + +import type { Database } from "bun:sqlite"; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import type { Server } from "bun"; +import { + type ZarrFidelityGroupJson, + bidsSidecarCandidates, + pushZarrFidelityExample, + runZarrFidelitySweep, + zarrFidelitySelectSample, + zarrFidelityStoreChannelTotal, + zarrFidelityStoreDuration, +} from "../src/services/zarr-fidelity-sweep"; +import type { Bindings } from "../src/types/bindings"; +import { freshDb, realD1 } from "./helpers/d1"; + +const ORG_NAME = "nemarDatasets"; +const COMMIT_A = "a".repeat(40); +const COMMIT_B = "b".repeat(40); + +// --------------------------------------------------------------------------- +// Pure helpers +// --------------------------------------------------------------------------- + +describe("bidsSidecarCandidates: nearest-first, bounded, deduplicated", () => { + test("recording directory, subject directory, dataset root -- in that order", () => { + const candidates = bidsSidecarCandidates( + "sub-01/ses-01/eeg/sub-01_ses-01_task-rest_run-01_eeg.set", + "channels.tsv", + ); + expect(candidates).toEqual([ + "sub-01/ses-01/eeg/sub-01_ses-01_task-rest_run-01_channels.tsv", + "sub-01/sub-01_ses-01_channels.tsv", + "channels.tsv", + ]); + }); + + test("no subject entity and no dashed entity tokens: recording dir collapses to a bare name", () => { + // "file" has no "-", so it is not treated as a BIDS entity -- the + // recording-directory candidate is the bare suffix in that directory, + // and there is no subject-directory candidate at all (no "sub-*" path + // segment), leaving just the recording dir and the dataset root. + const candidates = bidsSidecarCandidates("weird/path/file_eeg.set", "channels.tsv"); + expect(candidates).toEqual(["weird/path/channels.tsv", "channels.tsv"]); + }); + + test("a root-level recording (no directory) still yields the bare root candidate", () => { + const candidates = bidsSidecarCandidates("sub-01_task-rest_eeg.set", "channels.tsv"); + expect(candidates).toContain("channels.tsv"); + expect(candidates[candidates.length - 1]).toBe("channels.tsv"); + }); + + test("works identically for a modality sidecar suffix", () => { + const candidates = bidsSidecarCandidates("sub-02/eeg/sub-02_task-x_eeg.set", "eeg.json"); + expect(candidates).toEqual([ + "sub-02/eeg/sub-02_task-x_eeg.json", + "sub-02/sub-02_eeg.json", + "eeg.json", + ]); + }); +}); + +describe("zarrFidelityStoreChannelTotal: sum across groups", () => { + test("sums n_channels across groups, treating missing/invalid as 0", () => { + const groups: ZarrFidelityGroupJson[] = [ + { n_channels: 10 }, + { n_channels: 9 }, + { n_channels: undefined }, + { n_channels: "not a number" }, + ]; + expect(zarrFidelityStoreChannelTotal(groups)).toBe(19); + }); + + test("zero groups -> zero total", () => { + expect(zarrFidelityStoreChannelTotal([])).toBe(0); + }); +}); + +describe("zarrFidelityStoreDuration: max across groups", () => { + test("takes the max duration_s, never the sum", () => { + const groups: ZarrFidelityGroupJson[] = [{ duration_s: 60 }, { duration_s: 120 }]; + expect(zarrFidelityStoreDuration(groups)).toBe(120); + }); + + test("null when no group measured a duration", () => { + expect(zarrFidelityStoreDuration([{ duration_s: undefined }])).toBeNull(); + }); +}); + +describe("zarrFidelitySelectSample: decision 1's sampling rule", () => { + function store(path: string, nChannels?: number) { + return { + path, + zarr: `${path}.zarr`, + groups: nChannels != null ? [{ n_channels: nChannels }] : [], + }; + } + + test("40 or fewer stores: every store is sampled", () => { + const stores = Array.from({ length: 40 }, (_, i) => store(`s${String(i).padStart(3, "0")}`)); + const sample = zarrFidelitySelectSample(stores); + expect(sample.length).toBe(40); + }); + + test("more than 40: spread evenly to 40, PLUS every n_channels===1 store outside the spread", () => { + // 100 stores; only store #50 has n_channels === 1, positioned to be + // unlikely to land in an even 40-of-100 spread by construction. + const stores = Array.from({ length: 100 }, (_, i) => + store(`s${String(i).padStart(3, "0")}`, i === 50 ? 1 : 8), + ); + const sample = zarrFidelitySelectSample(stores); + expect(sample.length).toBeGreaterThanOrEqual(40); + expect(sample.some((s) => s.path === "s050")).toBe(true); + }); + + test("a store with no string path is excluded", () => { + const stores = [store("s000"), { path: undefined, groups: [] }]; + const sample = zarrFidelitySelectSample(stores); + expect(sample.map((s) => s.path)).toEqual(["s000"]); + }); +}); + +describe("pushZarrFidelityExample: bounded to 20 entries / 4 KB", () => { + test("stops appending past 20 entries", () => { + const examples: { path: string; code: "channel_count_mismatch" }[] = []; + for (let i = 0; i < 30; i++) { + pushZarrFidelityExample(examples, { path: `p${i}`, code: "channel_count_mismatch" }); + } + expect(examples.length).toBe(20); + expect(examples[0].path).toBe("p0"); + expect(examples[19].path).toBe("p19"); + }); + + test("stops appending once the serialized array would exceed 4 KB, even under 20 entries", () => { + const examples: { path: string; code: "channel_count_mismatch" }[] = []; + const longPath = "x".repeat(500); + for (let i = 0; i < 20; i++) { + pushZarrFidelityExample(examples, { + path: `${longPath}${i}`, + code: "channel_count_mismatch", + }); + } + expect(examples.length).toBeLessThan(20); + expect(new TextEncoder().encode(JSON.stringify(examples)).length).toBeLessThanOrEqual(4096); + }); +}); + +// --------------------------------------------------------------------------- +// Real-engine fixture server: one local Bun.serve() answering both the +// signed S3 index GET (path `//zarr/index.json`) and the public +// raw.githubusercontent.com sidecar GET (path +// `////`). +// --------------------------------------------------------------------------- + +let server: Server; +const indexFixtures = new Map(); +const sidecarFixtures = new Map(); + +beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + const parts = url.pathname + .split("/") + .filter((p) => p.length > 0) + .map((p) => decodeURIComponent(p)); + + if ( + parts.length >= 3 && + parts[parts.length - 2] === "zarr" && + parts[parts.length - 1] === "index.json" + ) { + const datasetId = parts[0]; + const body = indexFixtures.get(datasetId); + if (body === undefined) return new Response("not found", { status: 404 }); + return new Response(JSON.stringify(body), { status: 200 }); + } + + if (parts[0] === ORG_NAME) { + const [, repo, commit, ...rest] = parts; + const key = `${repo}/${commit}/${rest.join("/")}`; + const content = sidecarFixtures.get(key); + if (content === undefined) return new Response("not found", { status: 404 }); + return new Response(content, { status: 200 }); + } + + return new Response("not found", { status: 404 }); + }, + }); +}); + +afterAll(() => { + server.stop(true); +}); + +let db: Database; + +function env(): Bindings { + return { + DB: realD1(db), + S3_BUCKET: "test-bucket", + AWS_REGION: "us-east-1", + AWS_ACCESS_KEY_ID: "test", + AWS_SECRET_ACCESS_KEY: "test", + ENVIRONMENT: "test", + } as Bindings; +} + +function runOpts() { + return { + s3Options: { endpointUrl: `http://127.0.0.1:${server.port}` }, + githubRawBase: `http://127.0.0.1:${server.port}`, + }; +} + +/** Minimal, valid `datasets` row -- zarr-ready, converted, with a repo. */ +function seedDataset( + id: string, + repo: string, + overrides: Record = {}, +): void { + const merged: Record = { + owner_user_id: -1, + name: id, + visibility: "public", + status: "active", + is_sandbox: 0, + zarr_status: "ready", + zarr_store_count: 1, + zarr_source_commit: COMMIT_A, + github_repo: `${ORG_NAME}/${repo}`, + ...overrides, + }; + const keys = Object.keys(merged); + db.query( + `INSERT INTO datasets (dataset_id, ${keys.join(", ")}) VALUES (?, ${keys.map(() => "?").join(", ")})`, + ).run(id, ...(keys.map((k) => merged[k]) as never[])); +} + +function row(id: string): Record { + return db + .query( + `SELECT *, + json_extract(sweep_stamps, '$.zarr_verify_status') AS zarr_verify_status, + json_extract(sweep_stamps, '$.zarr_verified_at') AS zarr_verified_at, + json_extract(sweep_stamps, '$.zarr_verified_commit') AS zarr_verified_commit, + json_extract(sweep_stamps, '$.zarr_verify_examples') AS zarr_verify_examples_raw, + json_extract(sweep_stamps, '$.zarr_verify_sampled') AS zarr_verify_sampled, + json_extract(sweep_stamps, '$.zarr_verify_checked') AS zarr_verify_checked + FROM datasets WHERE dataset_id = ?`, + ) + .get(id) as Record; +} + +function channelsTsv(n: number): string { + const lines = ["name\ttype"]; + for (let i = 0; i < n; i++) lines.push(`Ch${i}\tEEG`); + return lines.join("\n"); +} + +function eegJson(samplingFrequency: number, recordingDuration: number): string { + return JSON.stringify({ + SamplingFrequency: samplingFrequency, + RecordingDuration: recordingDuration, + }); +} + +beforeEach(() => { + db = freshDb(); + indexFixtures.clear(); + sidecarFixtures.clear(); +}); + +describe("runZarrFidelitySweep: per-dataset verdicts", () => { + test("VERIFIED: channel count, duration, and rate all agree with ground truth", () => { + const id = "on800001"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "sub-01_task-rest_eeg.zarr", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + return runZarrFidelitySweep(env(), runOpts()).then((result) => { + expect(result.errors).toEqual([]); + expect(result.verified).toBe(1); + expect(result.failed).toBe(0); + expect(result.unverifiable).toBe(0); + expect(result.results).toEqual([ + { dataset_id: id, verdict: "verified", sampled: 1, checked: 1, examples: [] }, + ]); + + const r = row(id); + expect(r.zarr_verify_status).toBe("verified"); + expect(r.zarr_verified_commit).toBe(COMMIT_A); + expect(r.zarr_verify_sampled).toBe(1); + expect(r.zarr_verify_checked).toBe(1); + expect(JSON.parse(r.zarr_verify_examples_raw as string)).toEqual([]); + expect(r.zarr_verified_at).not.toBeNull(); + // Untouched columns: only sweep_stamps changed. + expect(r.name).toBe(id); + expect(r.zarr_store_count).toBe(1); + }); + }); + + test("FAILED (channel_count_mismatch): store total channels fall short of channels.tsv", async () => { + const id = "on800002"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 10, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.failed).toBe(1); + expect(result.results[0].examples).toEqual([ + { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "channel_count_mismatch" }, + ]); + + const r = row(id); + expect(r.zarr_verify_status).toBe("failed"); + expect(JSON.parse(r.zarr_verify_examples_raw as string)).toEqual([ + { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "channel_count_mismatch" }, + ]); + + // An audit_log row was written for the failure (decision 3). + const audit = db + .query( + "SELECT action, resource_id, details FROM audit_log WHERE action = 'zarr_fidelity_failed'", + ) + .get() as { action: string; resource_id: string; details: string } | null; + expect(audit).not.toBeNull(); + expect(audit?.resource_id).toBe(id); + expect(JSON.parse(audit?.details ?? "{}").examples).toEqual([ + { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "channel_count_mismatch" }, + ]); + }); + + test("FAILED (duration_mismatch): store duration disagrees with RecordingDuration by more than 1s", async () => { + const id = "on800003"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 100, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + // RecordingDuration 120 vs the store's duration_s 100 -- a 20s gap. + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.failed).toBe(1); + expect(result.results[0].examples).toEqual([ + { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "duration_mismatch" }, + ]); + }); + + test("FAILED (rate_mismatch): the served rate disagrees with SamplingFrequency after the modality cap", async () => { + const id = "on800004"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + // SamplingFrequency 500 capped at EEG's 250 -> expected rate 250, + // but the store serves 500 (as if the cap were never applied). + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 500 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(500, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.failed).toBe(1); + expect(result.results[0].examples).toEqual([ + { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "rate_mismatch" }, + ]); + }); + + test("UNVERIFIABLE: no sidecar is reachable at any candidate path", async () => { + const id = "on800005"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + // No sidecarFixtures registered at all -- every candidate 404s. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.unverifiable).toBe(1); + expect(result.results[0]).toEqual({ + dataset_id: id, + verdict: "unverifiable", + sampled: 1, + checked: 0, + examples: [], + }); + const r = row(id); + expect(r.zarr_verify_status).toBe("unverifiable"); + expect(r.zarr_verify_checked).toBe(0); + }); + + test("v1/v3 index tolerance: unrelated v3-only top-level and store fields do not break parsing", async () => { + const id = "on800006"; + seedDataset(id, id); + indexFixtures.set(id, { + format: "nemar-zarr-index", + format_version: 3, + source_commit: COMMIT_A, + contract_base: "https://zarr.nemar.org/on800006/zarr/", + data_base: "https://bucket.s3.region.amazonaws.com/on800006/zarr/", + discovered_count: 1, + store_count: 1, + failure_count: 0, + pending_count: 0, + failures: [], + pending: [], + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + source_tree: "raw", + derived: false, + n_events: 4, + groups: [ + { + modality: "eeg", + n_channels: 19, + duration_s: 120, + rate: 250, + n_view_levels: 3, + chunk_samples: 1000, + }, + ], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + }); + + test("bounded examples: 25 mismatching stores cap zarr_verify_examples at 20 entries", async () => { + const id = "on800007"; + seedDataset(id, id, { zarr_store_count: 25 }); + const stores = Array.from({ length: 25 }, (_, i) => { + const p = `sub-${String(i).padStart(2, "0")}`; + return { + path: `${p}/eeg/${p}_task-x_eeg.set`, + zarr: `${p}.zarr`, + groups: [{ modality: "eeg", n_channels: 1, duration_s: 10, rate: 250 }], + }; + }); + indexFixtures.set(id, { source_commit: COMMIT_A, store_count: 25, stores }); + for (let i = 0; i < 25; i++) { + const p = `sub-${String(i).padStart(2, "0")}`; + // 5 channels declared, store only serves 1 -- every store mismatches. + sidecarFixtures.set(`${id}/${COMMIT_A}/${p}/eeg/${p}_task-x_channels.tsv`, channelsTsv(5)); + sidecarFixtures.set(`${id}/${COMMIT_A}/${p}/eeg/${p}_task-x_eeg.json`, eegJson(250, 10)); + } + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.failed).toBe(1); + expect(result.results[0].sampled).toBe(25); + expect(result.results[0].checked).toBe(25); + expect(result.results[0].examples.length).toBe(20); + + const r = row(id); + const examples = JSON.parse(r.zarr_verify_examples_raw as string) as unknown[]; + expect(examples.length).toBe(20); + }); + + test("a re-conversion (zarr_source_commit changes) re-arms verification", async () => { + const id = "on800008"; + seedDataset(id, id); + const buildIndex = (commit: string) => ({ + source_commit: commit, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + indexFixtures.set(id, buildIndex(COMMIT_A)); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const first = await runZarrFidelitySweep(env(), runOpts()); + expect(first.processed).toBe(1); + expect(row(id).zarr_verified_commit).toBe(COMMIT_A); + + // Not re-selected while the store's commit is unchanged. + const second = await runZarrFidelitySweep(env(), runOpts()); + expect(second.processed).toBe(0); + + // Simulate a re-conversion: the dataset's zarr_source_commit advances, + // and the published index now reflects the new commit. + db.query("UPDATE datasets SET zarr_source_commit = ? WHERE dataset_id = ?").run(COMMIT_B, id); + indexFixtures.set(id, buildIndex(COMMIT_B)); + sidecarFixtures.set( + `${id}/${COMMIT_B}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_B}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const third = await runZarrFidelitySweep(env(), runOpts()); + expect(third.processed).toBe(1); + expect(third.verified).toBe(1); + expect(row(id).zarr_verified_commit).toBe(COMMIT_B); + }); + + test("index.json absent (zarr_status='ready' but nothing at the S3 key) is an error, not a stamp", async () => { + const id = "on800009"; + seedDataset(id, id); + // No indexFixtures entry -- the fixture server 404s. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.errors).toEqual([ + { dataset_id: id, error: "zarr_status=ready but index.json is absent" }, + ]); + const r = row(id); + expect(r.zarr_verify_status).toBeNull(); + }); + + test("an index with no fetchable source_commit is unverifiable, not a crash", async () => { + const id = "on800010"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: "", // pre-phase-7 empty-string bug + store_count: 1, + stores: [{ path: "sub-01/eeg/sub-01_task-rest_eeg.set", zarr: "z", groups: [] }], + }); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.unverifiable).toBe(1); + expect(row(id).zarr_verify_status).toBe("unverifiable"); + }); + + test("candidates are clamped to ZARR_FIDELITY_SWEEP_MAX and remaining reflects real state", async () => { + for (let i = 0; i < 3; i++) { + const id = `on80101${i}`; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + } + const result = await runZarrFidelitySweep(env(), { ...runOpts(), limit: 1 }); + expect(result.processed).toBe(1); + expect(result.remaining).toBe(2); + }); +}); From 8d78713f832388686cfc33192d7fe58b510082ce Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:30:21 -0700 Subject: [PATCH 02/15] feat: admin route for zarr fidelity sweep --- backend/src/routes/admin/index.ts | 2 + .../src/routes/admin/zarr-fidelity-sweep.ts | 60 ++++ .../test/zarr-fidelity-sweep-route.test.ts | 261 ++++++++++++++++++ test/admin-route-inventory.unit.test.ts | 5 +- 4 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 backend/src/routes/admin/zarr-fidelity-sweep.ts create mode 100644 backend/test/zarr-fidelity-sweep-route.test.ts diff --git a/backend/src/routes/admin/index.ts b/backend/src/routes/admin/index.ts index d284aa7fb..bd3346c01 100644 --- a/backend/src/routes/admin/index.ts +++ b/backend/src/routes/admin/index.ts @@ -20,6 +20,7 @@ import type { AdminRouter } from "./shared"; import { registerUsersRoutes } from "./users"; import { registerWithdrawRoutes } from "./withdraw"; import { registerZarrCatalogRoutes } from "./zarr-catalog"; +import { registerZarrFidelitySweepRoutes } from "./zarr-fidelity-sweep"; export const adminRoutes: AdminRouter = new Hono(); @@ -37,3 +38,4 @@ registerExemplarRoutes(adminRoutes); registerNoticeRoutes(adminRoutes); registerWithdrawRoutes(adminRoutes); registerZarrCatalogRoutes(adminRoutes); +registerZarrFidelitySweepRoutes(adminRoutes); diff --git a/backend/src/routes/admin/zarr-fidelity-sweep.ts b/backend/src/routes/admin/zarr-fidelity-sweep.ts new file mode 100644 index 000000000..110f8fb0f --- /dev/null +++ b/backend/src/routes/admin/zarr-fidelity-sweep.ts @@ -0,0 +1,60 @@ +/** + * Admin route: on-demand Zarr fidelity verification sweep (issue #1068, + * epic #1181 phase 8). + * + * Same shape as `routes/admin/zarr-catalog.ts` (issue #1062, phase 2): the + * daily cron (backend/src/index.ts) is the regular driver, and this route + * exists for an operator who wants an immediate batch without waiting for + * the next tick. One implementation (`runZarrFidelitySweep`, + * services/zarr-fidelity-sweep.ts) shared by both callers so they cannot + * drift. + */ + +import { + ZARR_FIDELITY_SWEEP_DEFAULT, + ZARR_FIDELITY_SWEEP_MAX, + runZarrFidelitySweep, +} from "../../services/zarr-fidelity-sweep"; +import type { AdminRouter } from "./shared"; + +/** + * `deps.sweep` defaults to the real `runZarrFidelitySweep` and exists so + * `backend/test/zarr-fidelity-sweep-route.test.ts` can register this exact + * route against a fresh app with the S3/GitHub-raw boundaries substituted + * (real local `Bun.serve()` receivers) -- the same DI-seam idiom + * `registerZarrCatalogRoutes` uses for `publishZarrCatalog`. + * `registerZarrFidelitySweepRoutes(adminRoutes)` (routes/admin/index.ts) + * omits it, so production always resolves the default. + */ +export function registerZarrFidelitySweepRoutes( + admin: AdminRouter, + deps: { sweep?: typeof runZarrFidelitySweep } = {}, +): void { + const sweep = deps.sweep ?? runZarrFidelitySweep; + + /** + * POST /admin/datasets/zarr-fidelity-sweep?limit=N + * + * Bounded batch (default {@link ZARR_FIDELITY_SWEEP_DEFAULT}, max + * {@link ZARR_FIDELITY_SWEEP_MAX}) of the fidelity sweep. Fails loud with a + * 500 on a candidate-query error (e.g. migration 0073 not applied); + * per-dataset failures are collected in the response body's `errors` + * instead of throwing (see `runZarrFidelitySweep`'s doc comment). + */ + admin.post("/datasets/zarr-fidelity-sweep", async (c) => { + const limitRaw = Number.parseInt(c.req.query("limit") || "", 10); + const limit = Number.isFinite(limitRaw) ? limitRaw : undefined; + + try { + const result = await sweep(c.env, { limit }); + return c.json(result); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error("[zarr-fidelity-sweep] candidate query failed:", msg); + return c.json( + { error: `Failed to query sweep candidates (is migration 0073 applied?): ${msg}` }, + 500, + ); + } + }); +} diff --git a/backend/test/zarr-fidelity-sweep-route.test.ts b/backend/test/zarr-fidelity-sweep-route.test.ts new file mode 100644 index 000000000..2e8084587 --- /dev/null +++ b/backend/test/zarr-fidelity-sweep-route.test.ts @@ -0,0 +1,261 @@ +/** + * Real `POST /admin/datasets/zarr-fidelity-sweep` route tests (issue #1068, + * epic #1181 phase 8). + * + * Mirrors `zarr-catalog-publish-route.test.ts`: real engine (bun:sqlite + * behind realD1, the real auth/admin middleware with seeded tokens, real + * route dispatch via Hono `app.request()`). The success path injects + * `runZarrFidelitySweep` with its `s3Options.endpointUrl` / + * `githubRawBase` test seams pointed at a real local `Bun.serve()` + * receiver -- the same DI-seam idiom `registerZarrCatalogRoutes` uses for + * `publishZarrCatalog` -- with the real auth/admin middleware stack still + * attached. + */ + +import type { Database } from "bun:sqlite"; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import type { Server } from "bun"; +import { Hono } from "hono"; +import { adminMiddleware, authMiddleware } from "../src/middleware/auth"; +import { registerZarrFidelitySweepRoutes } from "../src/routes/admin/zarr-fidelity-sweep"; +import { hashApiKey } from "../src/services/token"; +import { runZarrFidelitySweep } from "../src/services/zarr-fidelity-sweep"; +import type { Bindings, Variables } from "../src/types/bindings"; +import { freshDb, realD1 } from "./helpers/d1"; + +const ADMIN_KEY = "zfs-admin-key-0123456789abcdef0123456789abcdef01"; +const MEMBER_KEY = "zfs-member-key-0123456789abcdef0123456789abcdef0"; +const COMMIT_A = "a".repeat(40); +const ORG_NAME = "nemarDatasets"; + +type App = Hono<{ Bindings: Bindings; Variables: Variables }>; + +function newApp(deps: Parameters[1] = {}): App { + const app: App = new Hono(); + app.use("*", authMiddleware); + app.use("*", adminMiddleware); + registerZarrFidelitySweepRoutes(app, deps); + return app; +} + +async function seedUsers(db: Database): Promise { + db.run( + `INSERT INTO users (username, email, password_hash, status, role, email_verified) + VALUES ('zfsadmin', 'zfsadmin@example.org', 'x', 'approved', 'admin', 1)`, + ); + const admin = db + .query<{ id: number }, []>("SELECT id FROM users WHERE username='zfsadmin'") + .get(); + if (!admin) throw new Error("seed: admin insert failed"); + db.query("INSERT INTO tokens (user_id, api_key_hash, api_key_prefix) VALUES (?, ?, ?)").run( + admin.id, + await hashApiKey(ADMIN_KEY), + ADMIN_KEY.slice(0, 8), + ); + + db.run( + `INSERT INTO users (username, email, password_hash, status, role, email_verified) + VALUES ('zfsmember', 'zfsmember@example.org', 'x', 'approved', 'member', 1)`, + ); + const member = db + .query<{ id: number }, []>("SELECT id FROM users WHERE username='zfsmember'") + .get(); + if (!member) throw new Error("seed: member insert failed"); + db.query("INSERT INTO tokens (user_id, api_key_hash, api_key_prefix) VALUES (?, ?, ?)").run( + member.id, + await hashApiKey(MEMBER_KEY), + MEMBER_KEY.slice(0, 8), + ); +} + +function env(db: Database): Bindings { + return { + DB: realD1(db), + ENVIRONMENT: "development", + S3_BUCKET: "test-bucket", + AWS_REGION: "us-east-1", + AWS_ACCESS_KEY_ID: "test", + AWS_SECRET_ACCESS_KEY: "test", + } as Bindings; +} + +describe("POST /admin/datasets/zarr-fidelity-sweep: auth", () => { + let db: Database; + let app: App; + + beforeEach(async () => { + db = freshDb(); + app = newApp(); + await seedUsers(db); + }); + + test("unauthenticated (no Authorization header) is rejected with 401", async () => { + const res = await app.request("/datasets/zarr-fidelity-sweep", { method: "POST" }, env(db)); + expect(res.status).toBe(401); + }); + + test("a non-admin (member) token is rejected with 403", async () => { + const res = await app.request( + "/datasets/zarr-fidelity-sweep", + { method: "POST", headers: { Authorization: `Bearer ${MEMBER_KEY}` } }, + env(db), + ); + expect(res.status).toBe(403); + }); +}); + +describe("POST /admin/datasets/zarr-fidelity-sweep: candidate query failure becomes a 500", () => { + let db: Database; + let app: App; + + beforeEach(async () => { + db = freshDb(); + app = newApp(); + await seedUsers(db); + }); + + test("a throwing sweep dependency surfaces as a 500 carrying the message", async () => { + const throwingApp = newApp({ + sweep: async () => { + throw new Error("simulated candidate query failure"); + }, + }); + const res = await throwingApp.request( + "/datasets/zarr-fidelity-sweep", + { method: "POST", headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, + env(db), + ); + expect(res.status).toBe(500); + const body = (await res.json()) as { error: string }; + expect(body.error).toContain("simulated candidate query failure"); + }); +}); + +describe("POST /admin/datasets/zarr-fidelity-sweep: success against real S3/raw-content receivers", () => { + let db: Database; + let server: Server; + const indexFixtures = new Map(); + const sidecarFixtures = new Map(); + + beforeAll(() => { + server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + const parts = url.pathname + .split("/") + .filter((p) => p.length > 0) + .map((p) => decodeURIComponent(p)); + if ( + parts.length >= 3 && + parts[parts.length - 2] === "zarr" && + parts[parts.length - 1] === "index.json" + ) { + const body = indexFixtures.get(parts[0]); + if (body === undefined) return new Response("not found", { status: 404 }); + return new Response(JSON.stringify(body), { status: 200 }); + } + if (parts[0] === ORG_NAME) { + const [, repo, commit, ...rest] = parts; + const content = sidecarFixtures.get(`${repo}/${commit}/${rest.join("/")}`); + if (content === undefined) return new Response("not found", { status: 404 }); + return new Response(content, { status: 200 }); + } + return new Response("not found", { status: 404 }); + }, + }); + }); + + afterAll(() => { + server.stop(true); + }); + + beforeEach(async () => { + db = freshDb(); + indexFixtures.clear(); + sidecarFixtures.clear(); + await seedUsers(db); + }); + + test("returns the real per-dataset verdict, and it is actually persisted to D1", async () => { + const id = "on810001"; + db.query( + `INSERT INTO datasets + (dataset_id, owner_user_id, name, visibility, status, is_sandbox, + zarr_status, zarr_store_count, zarr_source_commit, github_repo) + VALUES (?, -1, ?, 'public', 'active', 0, 'ready', 1, ?, ?)`, + ).run(id, id, COMMIT_A, `${ORG_NAME}/${id}`); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + `name\ttype\n${Array.from({ length: 19 }, (_, i) => `Ch${i}\tEEG`).join("\n")}`, + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + JSON.stringify({ SamplingFrequency: 250, RecordingDuration: 120 }), + ); + + const app = newApp({ + sweep: (env, opts) => + runZarrFidelitySweep(env, { + ...opts, + s3Options: { endpointUrl: `http://127.0.0.1:${server.port}` }, + githubRawBase: `http://127.0.0.1:${server.port}`, + }), + }); + const res = await app.request( + "/datasets/zarr-fidelity-sweep?limit=5", + { method: "POST", headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, + env(db), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { + processed: number; + verified: number; + results: { dataset_id: string; verdict: string }[]; + }; + expect(body.processed).toBe(1); + expect(body.verified).toBe(1); + expect(body.results).toEqual([ + { dataset_id: id, verdict: "verified", sampled: 1, checked: 1, examples: [] }, + ]); + + const row = db + .query( + "SELECT json_extract(sweep_stamps, '$.zarr_verify_status') AS s FROM datasets WHERE dataset_id = ?", + ) + .get(id) as { s: string }; + expect(row.s).toBe("verified"); + }); + + test("an empty candidate set still returns a well-formed 200", async () => { + const app = newApp({ + sweep: (env, opts) => + runZarrFidelitySweep(env, { + ...opts, + s3Options: { endpointUrl: `http://127.0.0.1:${server.port}` }, + githubRawBase: `http://127.0.0.1:${server.port}`, + }), + }); + const res = await app.request( + "/datasets/zarr-fidelity-sweep", + { method: "POST", headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, + env(db), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { processed: number; results: unknown[]; remaining: number }; + expect(body.processed).toBe(0); + expect(body.results).toEqual([]); + expect(body.remaining).toBe(0); + }); +}); diff --git a/test/admin-route-inventory.unit.test.ts b/test/admin-route-inventory.unit.test.ts index 292479502..0721b6847 100644 --- a/test/admin-route-inventory.unit.test.ts +++ b/test/admin-route-inventory.unit.test.ts @@ -113,6 +113,9 @@ const EXPECTED_ENTRIES: Record = { // Zarr catalog (issue #1062, epic #1181 phase 2) "POST /zarr-catalog/publish": 1, + + // Zarr fidelity verification sweep (issue #1068, epic #1181 phase 8) + "POST /datasets/zarr-fidelity-sweep": 1, }; describe("admin route inventory", () => { @@ -126,7 +129,7 @@ describe("admin route inventory", () => { }); test("entry total is pinned", () => { - expect(adminRoutes.routes.length).toBe(95); + expect(adminRoutes.routes.length).toBe(96); }); // The count pin above can't see a SWAP of the two router-level middleware From 9efb1e7563ff7feaf8154b4fe64bf3287ab58214 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:30:32 -0700 Subject: [PATCH 03/15] feat: wire fidelity sweep into daily cron --- backend/src/index.ts | 36 ++++++++++++++++++++- backend/test/cron-sweep-wiring.test.ts | 43 ++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 46670433c..c860bc771 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -65,6 +65,7 @@ import { warningStageForDaysLeft, } from "./services/staleness"; import { publishZarrCatalog } from "./services/zarr-catalog"; +import { runZarrFidelitySweep } from "./services/zarr-fidelity-sweep"; import type { Bindings, Variables } from "./types/bindings"; // Create the API app with all routes @@ -293,10 +294,17 @@ export const PROD_SANDBOX_CLEANUP_QUERY = * against the shared `nemarDatasets` org, or mutate a real DOI or the PROD * bucket -- `publishZarrCatalog` (services/zarr-catalog.ts) qualifies * because it only ever reads/writes the CALLING env's own D1 and S3 bucket. + * `runZarrFidelitySweep` (services/zarr-fidelity-sweep.ts, issue #1068, + * epic #1181 phase 8) qualifies for the same reason PLUS one more: its + * dataset-repo reads go through the public, credential-free + * `raw.githubusercontent.com` content host, never the GitHub API/App/PAT -- + * see that module's doc comment for why that is a materially different, + * dev-safe exposure from `signal-defaults-sweep.ts`'s GitHub-API-based + * `getBidsTreeStats` (which IS prod-only, in the guarded block below). * `backend/test/cron-sweep-wiring.test.ts` pins that every name here is * both actually called and called outside the guarded block. */ -export const DEV_CRON_ALLOWLIST = ["publishZarrCatalog"] as const; +export const DEV_CRON_ALLOWLIST = ["publishZarrCatalog", "runZarrFidelitySweep"] as const; async function scheduledCleanup(env: Bindings): Promise { const db = env.DB; @@ -982,5 +990,31 @@ export default { ), ), ); + // Issue #1068 (epic #1181 phase 8): the standing Zarr fidelity + // verification sweep. ALLOWED ON THE NON-PROD CRON -- listed in + // DEV_CRON_ALLOWLIST above: it only reads this env's own S3 bucket plus + // the public, credential-free raw.githubusercontent.com content host, + // and writes only this env's own D1 -- no email, no GitHub App/PAT + // dispatch against the shared nemarDatasets org, no DOI mutation. + ctx.waitUntil( + runZarrFidelitySweep(env) + .then((r) => { + const failedIds = r.results + .filter((d) => d.verdict === "failed") + .map((d) => d.dataset_id); + console.log( + `[zarr-fidelity-sweep] processed=${r.processed} verified=${r.verified} failed=${r.failed} unverifiable=${r.unverifiable} errors=${r.errors.length} remaining=${r.remaining ?? "?"}${failedIds.length > 0 ? ` failedDatasets=${failedIds.join(",")}` : ""}`, + ); + for (const e of r.errors) { + console.error(`[zarr-fidelity-sweep] ${e.dataset_id}: ${e.error}`); + } + }) + .catch((err) => + console.error( + "[zarr-fidelity-sweep] sweep failed:", + err instanceof Error ? (err.stack ?? err.message) : err, + ), + ), + ); }, }; diff --git a/backend/test/cron-sweep-wiring.test.ts b/backend/test/cron-sweep-wiring.test.ts index 6ac445a61..c08ba3f82 100644 --- a/backend/test/cron-sweep-wiring.test.ts +++ b/backend/test/cron-sweep-wiring.test.ts @@ -84,6 +84,7 @@ const SWEEP_WIRING: Record=1 check above. + */ +describe("runZarrFidelitySweep is wired into the daily cron (outside the sweep framework)", () => { + const NAME = "runZarrFidelitySweep"; + + test("is declared in the DEV_CRON_ALLOWLIST named constant in index.ts", () => { + expect(allCode).toContain("DEV_CRON_ALLOWLIST"); + expect(allCode).toContain(`"${NAME}"`); + }); + + test("is called at least once in scheduled()", () => { + expect(callCount(allCode, NAME)).toBeGreaterThanOrEqual(1); + }); + + test("is wrapped in ctx.waitUntil", () => { + expect(isScheduled(allCode, NAME)).toBe(true); + }); + + test("is called OUTSIDE the prod-only block -- it also runs on the dev cron", () => { + expect(callCount(prodOnlyCode, NAME)).toBe(0); + expect(callCount(allCode, NAME)).toBeGreaterThan(callCount(prodOnlyCode, NAME)); + }); + + test("its ctx.waitUntil chain includes a .catch (a rejection must never crash scheduled())", () => { + const idx = allCode.indexOf(`${NAME}(env)`); + expect(idx).toBeGreaterThanOrEqual(0); + // A wider window than publishZarrCatalog's: this call's `.then()` builds + // a summary line with the failed-dataset ids (decision 3), so there is + // more code between the call and its `.catch(`. + const window = allCode.slice(idx, idx + 2000); + expect(window).toContain(".catch("); + }); +}); From 78034bdd9f1532436cc7ea552e38ce0d76fa486c Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:30:43 -0700 Subject: [PATCH 04/15] feat: add has_zarr_verified filter --- backend/src/routes/datasets/catalog.ts | 25 ++++++- backend/src/services/dataset-filters.ts | 19 +++++ backend/src/services/dataset-search.ts | 1 + backend/test/catalog-has-zarr.test.ts | 96 +++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 2 deletions(-) diff --git a/backend/src/routes/datasets/catalog.ts b/backend/src/routes/datasets/catalog.ts index 90aadbaaf..3570d7d96 100644 --- a/backend/src/routes/datasets/catalog.ts +++ b/backend/src/routes/datasets/catalog.ts @@ -77,6 +77,12 @@ const FACET_PROJECTION_COLUMNS = `d.subject_count, d.zarr_failure_count, d.zarr_deterministic, d.zarr_failed_at, + -- Issue #1068 (epic #1181 phase 8): the standing fidelity + -- verification sweep's verdict, derived from the JSON + -- sweep_stamps column (ADR 0034/0035 -- no new column). + -- Null until the sweep reaches this dataset. + json_extract(d.sweep_stamps, '$.zarr_verify_status') AS zarr_verify_status, + json_extract(d.sweep_stamps, '$.zarr_verified_at') AS zarr_verified_at, d.total_recording_duration, d.recording_duration_min, d.recording_duration_max, @@ -318,7 +324,8 @@ export function parseMinScore(c: FilterQueryContext): number { /** * Parse every filter query param -- the legacy bespoke filters (nine as of - * epic #1144 phase 3, #1147, D6; has_zarr added by issue #1062) plus the + * epic #1144 phase 3, #1147, D6; has_zarr added by issue #1062; + * has_zarr_verified added by issue #1068, epic #1181 phase 8) plus the * full facet table -- into one * `DatasetFilterOptions`, shared by both catalog endpoints so they can no * longer drift apart on which filters they honour. Before this, `GET @@ -354,6 +361,13 @@ export function parseFilterQuery( // review, item 8: documented here because it is easy to assume `=false` // means "exclude zarr datasets" when it silently means "no filter". const hasZarr = c.req.query("has_zarr") === "1" || c.req.query("has_zarr") === "true"; + // #1068 (epic #1181 phase 8): same `1`/`true` convention as has_zarr, and + // deliberately a SEPARATE param rather than a stricter value for + // `has_zarr` -- see dataset-filters.ts's `hasZarrVerified` doc comment for + // why `has_zarr`'s existing meaning must not narrow under callers already + // relying on it. + const hasZarrVerified = + c.req.query("has_zarr_verified") === "1" || c.req.query("has_zarr_verified") === "true"; // #970: same `1`/`true` convention. const dataComplete = c.req.query("data_complete") === "1" || c.req.query("data_complete") === "true"; @@ -374,6 +388,7 @@ export function parseFilterQuery( hasDoi, hasHed, hasZarr, + hasZarrVerified, dataComplete, recent, licenseTiers, @@ -627,7 +642,8 @@ export function registerCatalogRoutes(datasetRoutes: DatasetsRouter): void { * - Admin: all datasets (including private managed datasets) * * Filter params: modality, author, task, has_doi, recent, sort, search, owner, - * license, data_complete, has_hed, has_zarr, include_unknown, and the facet table + * license, data_complete, has_hed, has_zarr, has_zarr_verified, include_unknown, + * and the facet table * (epic #1144 phase 3, #1147) -- see shared/facets.ts for the full list of * facet query params (subjects, channels, sessions, size, files, citations, * duration, recording_length, recordings, unavailable, age, rate, @@ -1133,6 +1149,11 @@ export function registerCatalogRoutes(datasetRoutes: DatasetsRouter): void { ORDER BY created_at DESC LIMIT 1 ) AS latest_version, + -- Issue #1068 (epic #1181 phase 8): same derivation as the list + -- projection above (FACET_PROJECTION_COLUMNS) -- d.* alone does not + -- surface a json_extract expression. + json_extract(d.sweep_stamps, '$.zarr_verify_status') AS zarr_verify_status, + json_extract(d.sweep_stamps, '$.zarr_verified_at') AS zarr_verified_at, u.username as owner_username, u.github_username as owner_github FROM datasets d diff --git a/backend/src/services/dataset-filters.ts b/backend/src/services/dataset-filters.ts index 8e50d3d74..f07a2c849 100644 --- a/backend/src/services/dataset-filters.ts +++ b/backend/src/services/dataset-filters.ts @@ -70,6 +70,15 @@ export interface DatasetFilterOptions { * verified" rather than redefining this one out from under existing * callers (PR #1201 review, item 9). */ hasZarr?: boolean; + /** Only datasets whose Zarr copy has PASSED the standing fidelity + * verification sweep (issue #1068, epic #1181 phase 8): + * `has_zarr AND zarr_verify_status = 'verified'`. A strict narrowing of + * `hasZarr` above, not a replacement for it -- a fresh conversion is + * `has_zarr=true` with `zarr_verify_status` still null until the daily + * sweep (services/zarr-fidelity-sweep.ts) reaches it; the viewer keeps + * reading index.json regardless (ADR 0005). Same nullable-safe + * bespoke-filter shape as `hasZarr`, not a facet-table entry. */ + hasZarrVerified?: boolean; dataComplete?: boolean; recent?: number; licenseTiers?: LicenseTier[]; @@ -179,6 +188,16 @@ export function buildDatasetFilterClauses( // idx_datasets_zarr_status (migration 0035/0071) -- no new index needed. clauses += " AND d.zarr_status = 'ready' AND COALESCE(d.zarr_store_count, 0) > 0"; } + if (opts.hasZarrVerified) { + // #1068: has_zarr's predicate ANDed with the sweep's stamped verdict. + // json_extract returns SQL NULL for a never-swept row (sweep_stamps NULL + // or the key absent/JSON-null -- ADR 0035), which never equals the + // string literal, so an unswept or non-'verified' row is excluded, same + // as has_hed/has_zarr's own NULL-excludes-by-default convention. + clauses += + " AND d.zarr_status = 'ready' AND COALESCE(d.zarr_store_count, 0) > 0" + + " AND json_extract(d.sweep_stamps, '$.zarr_verify_status') = 'verified'"; + } if (opts.dataComplete) { // #970: same nullable-safe idiom as has_hed -- `= 1` excludes both 0 // (checked, incomplete) and NULL (not audited yet). Backed by diff --git a/backend/src/services/dataset-search.ts b/backend/src/services/dataset-search.ts index f8a41a518..981c50d87 100644 --- a/backend/src/services/dataset-search.ts +++ b/backend/src/services/dataset-search.ts @@ -411,6 +411,7 @@ export function hasActiveFilters(filters: DatasetFilterOptions): boolean { filters.hasDoi || filters.hasHed || filters.hasZarr || + filters.hasZarrVerified || filters.dataComplete || (filters.recent && filters.recent > 0) || (filters.licenseTiers && filters.licenseTiers.length > 0) || diff --git a/backend/test/catalog-has-zarr.test.ts b/backend/test/catalog-has-zarr.test.ts index 03707ae48..eafe701c7 100644 --- a/backend/test/catalog-has-zarr.test.ts +++ b/backend/test/catalog-has-zarr.test.ts @@ -141,6 +141,102 @@ describe("has_zarr filter: GET /datasets", () => { // (has_hed) and a facet-adjacent one (modality), through the real route -- // proves the two clauses AND together rather than one silently overriding // the other. +// Issue #1068, epic #1181 phase 8: has_zarr keeps its EXISTING meaning +// ("converted", ready + stores) under every caller that already relies on +// it; has_zarr_verified is a SEPARATE, stricter filter this sweep feeds. +describe("has_zarr_verified filter: GET /datasets (issue #1068, epic #1181 phase 8)", () => { + let db: Database; + let app: App; + + beforeEach(() => { + db = freshDb(); + app = newApp(); + insertDataset(db, "nm004301", { + name: "Ready with stores, verified", + zarr_status: "ready", + zarr_store_count: 3, + sweep_stamps: JSON.stringify({ zarr_verify_status: "verified" }), + }); + insertDataset(db, "nm004302", { + name: "Ready with stores, failed verification", + zarr_status: "ready", + zarr_store_count: 3, + sweep_stamps: JSON.stringify({ zarr_verify_status: "failed" }), + }); + insertDataset(db, "nm004303", { + name: "Ready with stores, unverifiable", + zarr_status: "ready", + zarr_store_count: 3, + sweep_stamps: JSON.stringify({ zarr_verify_status: "unverifiable" }), + }); + insertDataset(db, "nm004304", { + name: "Ready with stores, never swept", + zarr_status: "ready", + zarr_store_count: 3, + }); + insertDataset(db, "nm004305", { + name: "Not converted at all", + zarr_status: "pending", + }); + }); + + test("has_zarr=1 (UNCHANGED meaning) returns every converted row regardless of verify status", async () => { + const rows = await listDatasets(app, db, "has_zarr=1"); + expect(rows.map((r) => r.dataset_id).sort()).toEqual([ + "nm004301", + "nm004302", + "nm004303", + "nm004304", + ]); + }); + + test("has_zarr_verified=1 returns ONLY the verified row", async () => { + const rows = await listDatasets(app, db, "has_zarr_verified=1"); + expect(rows.map((r) => r.dataset_id)).toEqual(["nm004301"]); + }); + + test("has_zarr_verified=true (website convention) behaves identically to =1", async () => { + const rows = await listDatasets(app, db, "has_zarr_verified=true"); + expect(rows.map((r) => r.dataset_id)).toEqual(["nm004301"]); + }); + + test("no has_zarr_verified filter returns every row, including the unswept/failed/unverifiable ones", async () => { + const rows = await listDatasets(app, db, ""); + expect(rows.map((r) => r.dataset_id).sort()).toEqual([ + "nm004301", + "nm004302", + "nm004303", + "nm004304", + "nm004305", + ]); + }); + + test("the zarr_verify_status/zarr_verified_at fields are present on the list row", async () => { + const rows = await listDatasets(app, db, ""); + const verified = rows.find((r) => r.dataset_id === "nm004301") as unknown as { + zarr_verify_status?: string | null; + }; + const neverSwept = rows.find((r) => r.dataset_id === "nm004304") as unknown as { + zarr_verify_status?: string | null; + }; + expect(verified.zarr_verify_status).toBe("verified"); + expect(neverSwept.zarr_verify_status).toBeNull(); + }); + + test("GET /datasets/:id also derives zarr_verify_status/zarr_verified_at", async () => { + db.query( + "UPDATE datasets SET sweep_stamps = json_set(COALESCE(sweep_stamps, '{}'), '$.zarr_verified_at', '2026-09-01 00:00:00') WHERE dataset_id = 'nm004301'", + ).run(); + const res = await app.request("/nm004301", {}, env(db)); + expect(res.status).toBe(200); + const body = (await res.json()) as { + dataset: { zarr_verify_status?: string | null; zarr_verified_at?: string | null }; + }; + expect(body.dataset.zarr_verify_status).toBe("verified"); + expect(body.dataset.zarr_verified_at).toBe("2026-09-01 00:00:00"); + }); +}); + describe("has_zarr combined with another filter: GET /datasets", () => { let db: Database; let app: App; From 8584c3654925ec9ff6ccc04ae6b3504f6cdf3d40 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:30:52 -0700 Subject: [PATCH 05/15] feat: add zarr verify fields to contract --- shared/contract/dataset.ts | 47 +++++++++++++++++++++-- test/contract-schemas.test.ts | 72 +++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) diff --git a/shared/contract/dataset.ts b/shared/contract/dataset.ts index c46e20742..e09f28f74 100644 --- a/shared/contract/dataset.ts +++ b/shared/contract/dataset.ts @@ -115,6 +115,16 @@ const catalogItemObjectSchema = z zarr_deterministic: zeroOneNullable.optional(), zarr_failed_at: z.string().nullable().optional(), zarr_index_url: z.string().nullable().optional(), + // Issue #1068 (epic #1181 phase 8): the standing fidelity verification + // sweep's verdict, derived at read time from the JSON sweep_stamps + // column (ADR 0034/0035, no new column) -- see + // assertZarrVerifiedAtOnlyWhenStatusKnown below for the invariant. Both + // stay null/absent until the sweep (daily plus on-demand) reaches a + // freshly-converted dataset; a client that wants "converted AND + // verified" filters on `has_zarr_verified` rather than reading these + // two fields itself. + zarr_verify_status: z.enum(["verified", "failed", "unverifiable"]).nullable().optional(), + zarr_verified_at: z.string().nullable().optional(), total_recording_duration: z.number().nullable().optional(), recording_duration_min: z.number().nullable().optional(), recording_duration_max: z.number().nullable().optional(), @@ -154,9 +164,35 @@ function assertZarrIndexUrlOnlyWhenReady( } } -export const catalogItemSchema = catalogItemObjectSchema.superRefine( - assertZarrIndexUrlOnlyWhenReady, -); +/** + * Cross-field contract invariant (issue #1068, epic #1181 phase 8): + * `zarr_verified_at` is a timestamp of the LAST sweep run that produced + * `zarr_verify_status`, so it must be null/absent whenever the status itself + * is -- a dataset the sweep has never reached has neither. The reverse is + * not asserted: a non-null status with a null timestamp would be a stranger + * bug (the sweep only ever writes both together, see + * ZARR_FIDELITY_SWEEP_STAMP_SQL), but this direction is the one a producer + * could plausibly get wrong (stamping a status without its timestamp), so it + * is the one enforced. Shared by both schemas below, like + * assertZarrIndexUrlOnlyWhenReady. + */ +function assertZarrVerifiedAtRequiresStatus( + val: { zarr_verify_status?: string | null; zarr_verified_at?: string | null }, + ctx: z.RefinementCtx, +): void { + if (val.zarr_verified_at != null && val.zarr_verify_status == null) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "zarr_verified_at must be null/absent unless zarr_verify_status is non-null", + path: ["zarr_verified_at"], + }); + } +} + +export const catalogItemSchema = catalogItemObjectSchema.superRefine((val, ctx) => { + assertZarrIndexUrlOnlyWhenReady(val, ctx); + assertZarrVerifiedAtRequiresStatus(val, ctx); +}); export type CatalogItem = z.infer; /** @@ -207,7 +243,10 @@ export const datasetDetailSchema = catalogItemObjectSchema .optional(), }) .passthrough() - .superRefine(assertZarrIndexUrlOnlyWhenReady); + .superRefine((val, ctx) => { + assertZarrIndexUrlOnlyWhenReady(val, ctx); + assertZarrVerifiedAtRequiresStatus(val, ctx); + }); export type DatasetDetail = z.infer; /** diff --git a/test/contract-schemas.test.ts b/test/contract-schemas.test.ts index e121a7f7f..dde077731 100644 --- a/test/contract-schemas.test.ts +++ b/test/contract-schemas.test.ts @@ -181,6 +181,55 @@ describe("catalog item schema", () => { }), ).not.toThrow(); }); + + // Issue #1068, epic #1181 phase 8: the fidelity verification sweep's verdict. + test("accepts the #1068 zarr_verify_status/zarr_verified_at fields", () => { + const parsed = catalogItemSchema.parse({ + ...row, + zarr_status: "ready", + zarr_verify_status: "verified", + zarr_verified_at: "2026-09-02 00:00:00", + }); + expect(parsed.zarr_verify_status).toBe("verified"); + expect(parsed.zarr_verified_at).toBe("2026-09-02 00:00:00"); + }); + + test("zarr_verify_status/zarr_verified_at are optional and null (a fresh conversion the sweep hasn't reached yet)", () => { + expect(() => catalogItemSchema.parse(row)).not.toThrow(); + const parsed = catalogItemSchema.parse({ + ...row, + zarr_verify_status: null, + zarr_verified_at: null, + }); + expect(parsed.zarr_verify_status).toBeNull(); + expect(parsed.zarr_verified_at).toBeNull(); + }); + + test("rejects an out-of-enum zarr_verify_status", () => { + expect(() => catalogItemSchema.parse({ ...row, zarr_verify_status: "bogus" })).toThrow(); + }); + + // The cross-field invariant a regressed sweep write (or a future path that + // sets the two independently) would violate. + test("REJECTS zarr_verified_at set when zarr_verify_status is null", () => { + expect(() => + catalogItemSchema.parse({ + ...row, + zarr_verify_status: null, + zarr_verified_at: "2026-09-02 00:00:00", + }), + ).toThrow(); + }); + + test("ACCEPTS zarr_verified_at set when zarr_verify_status is non-null (the consistent pair)", () => { + expect(() => + catalogItemSchema.parse({ + ...row, + zarr_verify_status: "failed", + zarr_verified_at: "2026-09-02 00:00:00", + }), + ).not.toThrow(); + }); }); describe("detail schema: zarr_data_failures (#1191, fixes #1188)", () => { @@ -265,6 +314,29 @@ describe("detail schema: zarr_data_failures (#1191, fixes #1188)", () => { }), ).toThrow(); }); + + // Issue #1068, epic #1181 phase 8: same zarr_verified_at/zarr_verify_status + // invariant as catalogItemSchema, re-applied after `.extend()` -- proven + // separately for the same reason as the zarr_index_url pair above. + test("also REJECTS zarr_verified_at set when zarr_verify_status is null", () => { + expect(() => + datasetDetailSchema.parse({ + ...detailRow, + zarr_verify_status: null, + zarr_verified_at: "2026-09-02 00:00:00", + }), + ).toThrow(); + }); + + test("also ACCEPTS the consistent zarr_verify_status/zarr_verified_at pair", () => { + expect(() => + datasetDetailSchema.parse({ + ...detailRow, + zarr_verify_status: "unverifiable", + zarr_verified_at: "2026-09-02 00:00:00", + }), + ).not.toThrow(); + }); }); describe("search hit schema", () => { From 441addcd3dcd485a2d0180c677ba9d6f7cae3eac Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:31:01 -0700 Subject: [PATCH 06/15] feat: expose verify fields in zarr catalog --- backend/src/services/zarr-catalog.ts | 19 ++++++++++- backend/test/zarr-catalog.test.ts | 47 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/backend/src/services/zarr-catalog.ts b/backend/src/services/zarr-catalog.ts index 8fbae4f41..3515bb29e 100644 --- a/backend/src/services/zarr-catalog.ts +++ b/backend/src/services/zarr-catalog.ts @@ -75,6 +75,16 @@ export interface ZarrCatalogSourceRow { zarr_converted_at: string | null; zarr_source_commit: string | null; zarr_errors: number | null; + /** Issue #1068 (epic #1181 phase 8): the standing fidelity verification + * sweep's verdict/timestamp, projected from the JSON sweep_stamps column + * by the candidate SQL below (ADR 0034/0035 -- no new column). Null on a + * freshly-converted dataset the sweep has not reached yet; the catalog + * population itself is UNCHANGED (still every has_zarr row) -- a + * consumer that wants only verified entries filters on + * `zarr_verify_status` client-side or calls the API with + * `has_zarr_verified=1`. */ + zarr_verify_status: string | null; + zarr_verified_at: string | null; } export interface ZarrCatalogDataset { @@ -94,6 +104,9 @@ export interface ZarrCatalogDataset { zarr_converted_at: string | null; zarr_source_commit: string | null; zarr_errors: number | null; + /** Issue #1068 (epic #1181 phase 8). See {@link ZarrCatalogSourceRow}. */ + zarr_verify_status: string | null; + zarr_verified_at: string | null; index_url: string; } @@ -119,7 +132,9 @@ export const ZARR_CATALOG_CANDIDATE_SQL = `SELECT d.dataset_id, d.name, d.concept_doi, d.license, d.modalities, d.tasks, d.subject_count, d.has_hed, d.hed_version, d.zarr_status, d.zarr_store_count, d.recording_count, d.recordings_unavailable, d.total_recording_duration, - d.zarr_converted_at, d.zarr_source_commit, d.zarr_errors + d.zarr_converted_at, d.zarr_source_commit, d.zarr_errors, + json_extract(d.sweep_stamps, '$.zarr_verify_status') AS zarr_verify_status, + json_extract(d.sweep_stamps, '$.zarr_verified_at') AS zarr_verified_at FROM datasets d WHERE d.status = 'active' AND d.visibility = 'public' AND d.zarr_status = 'ready' AND COALESCE(d.zarr_store_count, 0) > 0 @@ -172,6 +187,8 @@ export function buildZarrCatalog( zarr_converted_at: row.zarr_converted_at, zarr_source_commit: row.zarr_source_commit, zarr_errors: row.zarr_errors, + zarr_verify_status: row.zarr_verify_status, + zarr_verified_at: row.zarr_verified_at, index_url: `${base}${row.dataset_id}/zarr/index.json`, })); return { diff --git a/backend/test/zarr-catalog.test.ts b/backend/test/zarr-catalog.test.ts index 4d3ed81e8..4c55e7c4b 100644 --- a/backend/test/zarr-catalog.test.ts +++ b/backend/test/zarr-catalog.test.ts @@ -143,6 +143,27 @@ describe("ZARR_CATALOG_CANDIDATE_SQL: the real exclusion predicate", () => { const rows = candidateRows(db); expect(rows.map((r) => r.dataset_id)).toEqual(["on500001", "on500002", "xx099901"]); }); + + // Issue #1068, epic #1181 phase 8: the two verify fields are derived from + // the JSON sweep_stamps column (ADR 0034/0035), not a stored column -- + // proved at the real SQL entry point, not just in buildZarrCatalog's pure + // unit tests below. + test("zarr_verify_status/zarr_verified_at are derived from sweep_stamps via json_extract", () => { + db.query("UPDATE datasets SET sweep_stamps = ? WHERE dataset_id = 'on500001'").run( + JSON.stringify({ zarr_verify_status: "verified", zarr_verified_at: "2026-09-01 00:00:00" }), + ); + const rows = candidateRows(db); + const row = rows.find((r) => r.dataset_id === "on500001"); + expect(row?.zarr_verify_status).toBe("verified"); + expect(row?.zarr_verified_at).toBe("2026-09-01 00:00:00"); + }); + + test("a never-swept row (sweep_stamps NULL) projects both verify fields as null", () => { + const rows = candidateRows(db); + const row = rows.find((r) => r.dataset_id === "on500001"); + expect(row?.zarr_verify_status).toBeNull(); + expect(row?.zarr_verified_at).toBeNull(); + }); }); describe("buildZarrCatalog: shape, ordering, and CSV-to-array parsing", () => { @@ -164,6 +185,9 @@ describe("buildZarrCatalog: shape, ordering, and CSV-to-array parsing", () => { zarr_converted_at: "2026-08-01T00:00:00.000Z", zarr_source_commit: "abc123", zarr_errors: 0, + // Issue #1068, epic #1181 phase 8. + zarr_verify_status: "verified", + zarr_verified_at: "2026-08-02T00:00:00.000Z", ...overrides, }); @@ -224,6 +248,29 @@ describe("buildZarrCatalog: shape, ordering, and CSV-to-array parsing", () => { expect(d.zarr_converted_at).toBe("2026-08-01T00:00:00.000Z"); expect(d.zarr_source_commit).toBe("abc123"); expect(d.zarr_errors).toBe(0); + expect(d.zarr_verify_status).toBe("verified"); + expect(d.zarr_verified_at).toBe("2026-08-02T00:00:00.000Z"); + }); + + // Issue #1068 (epic #1181 phase 8), decision 2: the catalog POPULATION is + // unchanged (still every has_zarr row), and the two verify fields ride + // along per entry -- null on a fresh conversion the sweep has not reached. + test("zarr_verify_status/zarr_verified_at are null on a fresh, unverified conversion", () => { + const catalog = buildZarrCatalog([row({ zarr_verify_status: null, zarr_verified_at: null })], { + contractBase: "https://zarr.nemar.org", + }); + expect(catalog.count).toBe(1); + expect(catalog.datasets[0].zarr_verify_status).toBeNull(); + expect(catalog.datasets[0].zarr_verified_at).toBeNull(); + }); + + test("a 'failed' verdict still rides along on its own entry -- population is not filtered by verdict", () => { + const catalog = buildZarrCatalog( + [row({ dataset_id: "on500009", zarr_verify_status: "failed" })], + { contractBase: "https://zarr.nemar.org" }, + ); + expect(catalog.count).toBe(1); + expect(catalog.datasets[0].zarr_verify_status).toBe("failed"); }); test("preserves the input row order (the SQL query owns ORDER BY dataset_id)", () => { From 0cdf3c50ab0ad6ccfdc6f56fd1ff88326b6b16a2 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:31:11 -0700 Subject: [PATCH 07/15] feat: add zarr-fidelity-sweep CLI command --- src/commands/admin.ts | 67 +++++++ src/lib/api/admin.ts | 47 +++++ test/api-export-surface.unit.test.ts | 2 + test/zarr-fidelity-sweep-cli.test.ts | 266 +++++++++++++++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 test/zarr-fidelity-sweep-cli.test.ts diff --git a/src/commands/admin.ts b/src/commands/admin.ts index c23436dd9..1cacdf055 100644 --- a/src/commands/admin.ts +++ b/src/commands/admin.ts @@ -44,6 +44,7 @@ import { type ReindexOptions, type ReindexResponse, type SummaryVersionCoverage, + type ZarrFidelitySweepBatchResponse, addCi, approveUser, approveUserById, @@ -90,6 +91,7 @@ import { validateCi, verifyImport, withdrawDataset, + zarrFidelitySweep, } from "../lib/api/admin.js"; import { applyS3Lock, getDatasetFiles } from "../lib/api/data.js"; import { @@ -5225,6 +5227,71 @@ dataIntegritySweepCommand adminCommand.addCommand(dataIntegritySweepCommand); +// ============================================================================ +// Zarr fidelity verification sweep (issue #1068, epic #1181 phase 8) +// ============================================================================ + +const zarrFidelitySweepCommand = new Command("zarr-fidelity-sweep").description( + "Re-verify published Zarr stores against their dataset's own BIDS metadata (#1068)", +); + +zarrFidelitySweepCommand + .option("--limit ", "Datasets per batch (server clamps to [1,100])", "25") + .option("--json", "Output raw JSON instead of the human summary") + .action(async (options: { limit?: string; json?: boolean }) => { + if (!requireAuth()) return; + + const limit = Number.parseInt(options.limit ?? "25", 10) || 25; + const spinner = ora("Verifying Zarr store fidelity...").start(); + + let res: ZarrFidelitySweepBatchResponse; + try { + res = await zarrFidelitySweep({ limit }); + spinner.stop(); + } catch (err) { + spinner.fail("Zarr fidelity sweep failed"); + console.error(chalk.red(errorDetail(err))); + process.exit(1); + return; + } + + if (options.json) { + console.log(JSON.stringify(res, null, 2)); + } else { + console.log(); + console.log( + chalk.cyan( + `processed=${res.processed} verified=${res.verified} failed=${res.failed} unverifiable=${res.unverifiable} errors=${res.errors.length} remaining=${res.remaining ?? "unknown"}`, + ), + ); + for (const r of res.results) { + const color = + r.verdict === "verified" + ? chalk.green + : r.verdict === "failed" + ? chalk.red + : chalk.yellow; + console.log( + ` ${color(r.verdict.padEnd(12))} ${r.dataset_id} (sampled=${r.sampled} checked=${r.checked})`, + ); + for (const ex of r.examples) { + console.log(` ${chalk.red(ex.code)}: ${ex.path}`); + } + } + for (const e of res.errors) { + console.log(` ${chalk.red("error")} ${e.dataset_id}: ${e.error}`); + } + } + // Non-zero exit on any per-dataset error OR an indeterminate (null) + // remaining, so a caller never reads a partial/uncertain sweep as + // success -- mirrors data-integrity-sweep/hed-sweep's convention. A + // 'failed' verdict is NOT itself a non-zero-exit condition: the sweep + // ran successfully and reported the truth, which is the whole point. + if (res.errors.length > 0 || res.remaining === null) process.exit(1); + }); + +adminCommand.addCommand(zarrFidelitySweepCommand); + // ============================================================================ // Doctor: diagnostic checks + remediation (#1130) // ============================================================================ diff --git a/src/lib/api/admin.ts b/src/lib/api/admin.ts index 2f07da2b5..be21687b6 100644 --- a/src/lib/api/admin.ts +++ b/src/lib/api/admin.ts @@ -1307,3 +1307,50 @@ export async function publishZarrCatalog(): Promise true, ); } + +// ============================================================================ +// Zarr fidelity verification sweep (issue #1068, epic #1181 phase 8) +// ============================================================================ + +/** One {path, code} mismatch example, bounded to 20 entries / 4 KB server-side. */ +export interface ZarrFidelityMismatchExample { + path: string; + code: "channel_count_mismatch" | "duration_mismatch" | "rate_mismatch"; +} + +/** Per-dataset outcome for every candidate the sweep reached a verdict for. */ +export interface ZarrFidelityDatasetResult { + dataset_id: string; + verdict: "verified" | "failed" | "unverifiable"; + sampled: number; + checked: number; + examples: ZarrFidelityMismatchExample[]; +} + +/** One batch of the zarr fidelity sweep + * (`POST /admin/datasets/zarr-fidelity-sweep`). */ +export interface ZarrFidelitySweepBatchResponse { + processed: number; + verified: number; + failed: number; + unverifiable: number; + results: ZarrFidelityDatasetResult[]; + errors: { dataset_id: string; error: string }[]; + /** Candidates still unverified after this run; null if the count query failed. */ + remaining: number | null; +} + +/** Run one bounded zarr fidelity sweep batch (server default 25, clamped to + * [1,100]). */ +export async function zarrFidelitySweep(options?: { + limit?: number; +}): Promise { + const params = new URLSearchParams(); + if (options?.limit != null) params.set("limit", String(options.limit)); + const query = params.toString() ? `?${params.toString()}` : ""; + return request( + `/admin/datasets/zarr-fidelity-sweep${query}`, + { method: "POST", headers: { "Content-Type": "application/json" } }, + true, + ); +} diff --git a/test/api-export-surface.unit.test.ts b/test/api-export-surface.unit.test.ts index a29965e73..6c4dbaa8a 100644 --- a/test/api-export-surface.unit.test.ts +++ b/test/api-export-surface.unit.test.ts @@ -137,6 +137,7 @@ const MODULE_EXPORTS: Record = { "validateCi", "verifyImport", "withdrawDataset", + "zarrFidelitySweep", ], notices: ["NOTICE_LEVELS", "createNotice", "deleteNotice", "getNotices", "listAdminNotices"], }; @@ -165,6 +166,7 @@ const POST_SPLIT_ADDITIONS = [ "doctorScan", // #1130, CLI wrapper for POST /admin/doctor/scan "getFacets", // #1149, epic #1144 phase 5b: GET /datasets/facets for shell completion "publishZarrCatalog", // #1062, epic #1181 phase 2: POST /admin/zarr-catalog/publish + "zarrFidelitySweep", // #1068, epic #1181 phase 8: POST /admin/datasets/zarr-fidelity-sweep ]; /** The api.ts monolith's runtime surface, captured at #908 commit 1. */ diff --git a/test/zarr-fidelity-sweep-cli.test.ts b/test/zarr-fidelity-sweep-cli.test.ts new file mode 100644 index 000000000..660f081df --- /dev/null +++ b/test/zarr-fidelity-sweep-cli.test.ts @@ -0,0 +1,266 @@ +/** + * CLI `nemar admin zarr-fidelity-sweep` (issue #1068, epic #1181 phase 8), + * driven through the real entry point (`bun run src/index.ts admin + * zarr-fidelity-sweep ...`). Mirrors test/has-zarr-cli-flag.test.ts's + * harness: a real subprocess CLI invocation pointed at a real local HTTP + * server via TEST_API_URL (src/lib/api/client.ts#getApiUrl) -- the + * recorded request is what the CLI's own `fetch()` actually sent, not a + * re-implementation of it. + * + * Unlike the has_zarr flag (an unauthenticated GET /datasets filter), this + * command is admin-only, so the harness also seeds a real config.json + * ({activeAccount, accounts: {: {apiKey}}}) into an isolated + * NEMAR_CONFIG_DIR -- the same shape src/lib/config.ts's Conf store reads + * (cwd: NEMAR_CONFIG_DIR) -- so isAuthenticated() is genuinely true rather + * than stubbed. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "bun"; + +const CLI_ENTRY = join(import.meta.dir, "..", "src", "index.ts"); +const REPO_ROOT = join(import.meta.dir, ".."); + +interface CaptureServer { + url: string; + requests: URL[]; + stop: () => void; +} + +function startCaptureServer(body: unknown, status = 200): CaptureServer { + const requests: URL[] = []; + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + // Real traffic every invocation fires that is not what these tests + // check (mirrors has-zarr-cli-flag.test.ts's exclusions). + if (url.pathname === "/notices") { + return new Response(JSON.stringify({ notices: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.pathname === "/datasets/facets") { + return new Response(JSON.stringify({}), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + requests.push(url); + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + }, + }); + return { + url: `http://localhost:${server.port}`, + requests, + stop: () => server.stop(true), + }; +} + +let configDir: string; + +function seedAuthenticatedConfig(): void { + writeFileSync( + join(configDir, "config.json"), + JSON.stringify({ + activeAccount: "zfscliuser", + accounts: { zfscliuser: { apiKey: "test-admin-key" } }, + }), + ); +} + +beforeEach(() => { + configDir = mkdtempSync(join(tmpdir(), "nemar-zarr-fidelity-sweep-cli-")); +}); + +afterEach(() => { + rmSync(configDir, { recursive: true, force: true }); +}); + +async function runCli( + args: string[], + testApiUrl: string, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const env = { + ...process.env, + NEMAR_CONFIG_DIR: configDir, + TEST_API_URL: testApiUrl, + NEMAR_NO_UPDATE_CHECK: "1", + NO_COLOR: "1", + }; + env.FORCE_COLOR = undefined; + env.CLICOLOR_FORCE = undefined; + const proc = spawn({ + cmd: ["bun", "run", CLI_ENTRY, ...args], + cwd: REPO_ROOT, + env, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const exitCode = await proc.exited; + return { stdout, stderr, exitCode }; +} + +const VERIFIED_RESPONSE = { + processed: 1, + verified: 1, + failed: 0, + unverifiable: 0, + results: [{ dataset_id: "on800001", verdict: "verified", sampled: 1, checked: 1, examples: [] }], + errors: [], + remaining: 0, +}; + +const FAILED_RESPONSE = { + processed: 1, + verified: 0, + failed: 1, + unverifiable: 0, + results: [ + { + dataset_id: "on800002", + verdict: "failed", + sampled: 1, + checked: 1, + examples: [{ path: "sub-01/eeg/sub-01_task-x_eeg.set", code: "channel_count_mismatch" }], + }, + ], + errors: [], + remaining: 0, +}; + +describe("nemar admin zarr-fidelity-sweep: auth", () => { + test("without a stored apiKey, requireAuth blocks before any request is sent", async () => { + const server = startCaptureServer(VERIFIED_RESPONSE); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep"], server.url); + expect(result.stdout).toContain("Not authenticated"); + expect(server.requests.length).toBe(0); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin zarr-fidelity-sweep: request shape", () => { + test("--limit is forwarded as ?limit=N on the real request", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer(VERIFIED_RESPONSE); + try { + const result = await runCli( + ["admin", "zarr-fidelity-sweep", "--limit", "10", "--json"], + server.url, + ); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].pathname).toBe("/admin/datasets/zarr-fidelity-sweep"); + expect(server.requests[0].searchParams.get("limit")).toBe("10"); + } finally { + server.stop(); + } + }); + + test("without --limit, the CLI's own default (25) is sent", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer(VERIFIED_RESPONSE); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.get("limit")).toBe("25"); + } finally { + server.stop(); + } + }); +}); + +describe("nemar admin zarr-fidelity-sweep: output and exit code", () => { + test("--json prints the raw response verbatim", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer(VERIFIED_RESPONSE); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout)).toEqual(VERIFIED_RESPONSE); + } finally { + server.stop(); + } + }); + + test("the human summary prints each dataset's verdict", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer(VERIFIED_RESPONSE); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("verified"); + expect(result.stdout).toContain("on800001"); + expect(result.stdout).toContain("processed=1"); + } finally { + server.stop(); + } + }); + + test("a 'failed' verdict prints its mismatch examples and still exits 0 (the sweep succeeded; it reported the truth)", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer(FAILED_RESPONSE); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep"], server.url); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("failed"); + expect(result.stdout).toContain("on800002"); + expect(result.stdout).toContain("channel_count_mismatch"); + } finally { + server.stop(); + } + }); + + test("a non-empty errors array exits non-zero", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer({ + processed: 1, + verified: 0, + failed: 0, + unverifiable: 0, + results: [], + errors: [{ dataset_id: "on800003", error: "s3: simulated timeout" }], + remaining: 0, + }); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("a null remaining count exits non-zero (an uncertain sweep is never reported as success)", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer({ + processed: 1, + verified: 1, + failed: 0, + unverifiable: 0, + results: [ + { dataset_id: "on800001", verdict: "verified", sampled: 1, checked: 1, examples: [] }, + ], + errors: [], + remaining: null, + }); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); +}); From a8781667a34a50844b93d64a424aac55015ba2de Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:31:20 -0700 Subject: [PATCH 08/15] feat: add --has-zarr-verified CLI flag --- src/commands/dataset.ts | 10 +++++ src/lib/api/datasets.ts | 8 ++++ test/has-zarr-cli-flag.test.ts | 73 ++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/src/commands/dataset.ts b/src/commands/dataset.ts index 8de3c3500..5412f3879 100644 --- a/src/commands/dataset.ts +++ b/src/commands/dataset.ts @@ -1880,6 +1880,10 @@ const listCommand = datasetCommand .option("--doi", "Show only datasets with DOIs") .option("--hed", "Show only datasets with HED annotations") .option("--has-zarr", "Show only datasets with a ready Zarr copy (#1062)") + .option( + "--has-zarr-verified", + "Show only datasets whose Zarr copy passed fidelity verification (#1068)", + ) .option("--complete", "Show only datasets verified data-complete (#970)") .option("--recent [days]", "Show recently published datasets") .addOption( @@ -1981,6 +1985,7 @@ Examples: hasDoi: !!options.doi, hasHed: !!options.hed, hasZarr: !!options.hasZarr, + hasZarrVerified: !!options.hasZarrVerified, dataComplete: !!options.complete, recent: options.recent ? Number.parseInt(options.recent, 10) || 30 : undefined, sort: options.sort, @@ -2094,6 +2099,10 @@ const searchCommand = datasetCommand .option("--doi", "Show only datasets with DOIs") .option("--hed", "Show only datasets with HED annotations") .option("--has-zarr", "Show only datasets with a ready Zarr copy (#1062)") + .option( + "--has-zarr-verified", + "Show only datasets whose Zarr copy passed fidelity verification (#1068)", + ) .option("--complete", "Show only datasets verified data-complete (#970)") .option("--recent [days]", "Show recently published datasets") .option( @@ -2141,6 +2150,7 @@ Examples: modality: options.modality, hasHed: !!options.hed, hasZarr: !!options.hasZarr, + hasZarrVerified: !!options.hasZarrVerified, limit: Number.parseInt(options.limit, 10), author: options.author, task: options.task, diff --git a/src/lib/api/datasets.ts b/src/lib/api/datasets.ts index 63c1a77d9..abead3d6b 100644 --- a/src/lib/api/datasets.ts +++ b/src/lib/api/datasets.ts @@ -180,6 +180,9 @@ export interface DatasetListFilters { hasHed?: boolean; /** Only datasets with a ready Zarr copy (#1062). Serialized as has_zarr=1. */ hasZarr?: boolean; + /** Only datasets whose Zarr copy has passed the fidelity verification + * sweep (#1068). Serialized as has_zarr_verified=1. */ + hasZarrVerified?: boolean; /** Only datasets verified data-complete (#970). Serialized as data_complete=1. */ dataComplete?: boolean; recent?: number; @@ -263,6 +266,9 @@ export interface DatasetSearchFilters { hasHed?: boolean; /** Only datasets with a ready Zarr copy (#1062). Serialized as has_zarr=1. */ hasZarr?: boolean; + /** Only datasets whose Zarr copy has passed the fidelity verification + * sweep (#1068). Serialized as has_zarr_verified=1. */ + hasZarrVerified?: boolean; limit?: number; author?: string; task?: string; @@ -319,6 +325,7 @@ export async function listDatasets( if (filters.hasDoi) params.set("has_doi", "true"); if (filters.hasHed) params.set("has_hed", "1"); if (filters.hasZarr) params.set("has_zarr", "1"); + if (filters.hasZarrVerified) params.set("has_zarr_verified", "1"); if (filters.dataComplete) params.set("data_complete", "1"); if (filters.recent) params.set("recent", String(filters.recent)); if (filters.sort) params.set("sort", filters.sort); @@ -397,6 +404,7 @@ export async function searchDatasets( if (filters.limit) params.set("limit", String(filters.limit)); if (filters.hasHed) params.set("has_hed", "1"); if (filters.hasZarr) params.set("has_zarr", "1"); + if (filters.hasZarrVerified) params.set("has_zarr_verified", "1"); if (filters.author) params.set("author", filters.author); if (filters.task) params.set("task", filters.task); if (filters.license) params.set("license", filters.license); diff --git a/test/has-zarr-cli-flag.test.ts b/test/has-zarr-cli-flag.test.ts index 1d28e2c51..6cee2d3b0 100644 --- a/test/has-zarr-cli-flag.test.ts +++ b/test/has-zarr-cli-flag.test.ts @@ -176,3 +176,76 @@ describe("nemar dataset search --has-zarr", () => { } }); }); + +// Issue #1068, epic #1181 phase 8: --has-zarr-verified is a SEPARATE, +// stricter filter alongside --has-zarr above (decision 2: has_zarr keeps +// its existing meaning; has_zarr_verified is additive, not a replacement). +describe("nemar dataset list --has-zarr-verified", () => { + test("maps to ?has_zarr_verified=1 on the real request", async () => { + const server = startCaptureServer(EMPTY_LIST_ENVELOPE); + try { + const result = await runCli(["dataset", "list", "--json", "--has-zarr-verified"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests.length).toBe(1); + expect(server.requests[0].searchParams.get("has_zarr_verified")).toBe("1"); + } finally { + server.stop(); + } + }); + + test("without the flag, has_zarr_verified is absent from the query string", async () => { + const server = startCaptureServer(EMPTY_LIST_ENVELOPE); + try { + const result = await runCli(["dataset", "list", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.has("has_zarr_verified")).toBe(false); + } finally { + server.stop(); + } + }); + + test("stacks with --has-zarr without either overriding the other", async () => { + const server = startCaptureServer(EMPTY_LIST_ENVELOPE); + try { + const result = await runCli( + ["dataset", "list", "--json", "--has-zarr", "--has-zarr-verified"], + server.url, + ); + expect(result.exitCode).toBe(0); + const params = server.requests[0].searchParams; + expect(params.get("has_zarr")).toBe("1"); + expect(params.get("has_zarr_verified")).toBe("1"); + } finally { + server.stop(); + } + }); +}); + +describe("nemar dataset search --has-zarr-verified", () => { + test("maps to ?has_zarr_verified=1 on the real request", async () => { + const server = startCaptureServer(EMPTY_SEARCH_ENVELOPE); + try { + const result = await runCli( + ["dataset", "search", "resting state", "--json", "--has-zarr-verified"], + server.url, + ); + expect(result.exitCode).toBe(0); + const params = server.requests[0].searchParams; + expect(params.get("has_zarr_verified")).toBe("1"); + expect(params.get("q")).toBe("resting state"); + } finally { + server.stop(); + } + }); + + test("without the flag, has_zarr_verified is absent from the query string", async () => { + const server = startCaptureServer(EMPTY_SEARCH_ENVELOPE); + try { + const result = await runCli(["dataset", "search", "resting state", "--json"], server.url); + expect(result.exitCode).toBe(0); + expect(server.requests[0].searchParams.has("has_zarr_verified")).toBe(false); + } finally { + server.stop(); + } + }); +}); From eda78be7e1c2dce283c0d59bcb1d358576e16e9a Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 11:32:42 -0700 Subject: [PATCH 09/15] docs: document fidelity sweep in AGENTS.md --- AGENTS.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 94aa77967..106318aa4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -379,6 +379,28 @@ Environments and pre-release checks: [`.context/release-safety-playbook.md`](.co normally throughout. The cohort stranded before the stamp existed (directory-format datasets converted before 2026-08-22, #1172) is recovered by `hallu-zarr.sh --backfill-dir-formats`, dry-run by default. + **Verification is a standing sweep, not a conversion-time gate alone** + (issue #1068, epic #1181 phase 8): the converter's own + `channel_count_mismatch` check withholds an unfaithful store when it is + built, but `has_zarr` KEEPS ITS EXISTING MEANING (converted: `zarr_status + = 'ready'` with at least one store) so no caller of `?has_zarr=1` sees a + silently narrower result set. `has_zarr_verified` is the new, separate, + stricter filter: converted AND the fidelity sweep's last verdict was + `verified`. `backend/src/services/zarr-fidelity-sweep.ts`'s + `runZarrFidelitySweep` re-derives ground truth per sampled recording + (`channels.tsv` row count, sidecar `SamplingFrequency`/`RecordingDuration`) + from the dataset's own git-tracked BIDS metadata via the public, + credential-free `raw.githubusercontent.com` content host (never the + GitHub API/App/PAT, which is what makes it safe on the non-prod cron too + -- `DEV_CRON_ALLOWLIST` in index.ts), and stamps `zarr_verify_status` + (`verified` / `failed` / `unverifiable`) plus `zarr_verified_at` into + `sweep_stamps` (ADR 0034/0035 -- no new column); a re-conversion (a + changed `zarr_source_commit`) re-arms verification. A fresh conversion + shows `zarr_verify_status: null` until the daily sweep (plus + `POST /admin/datasets/zarr-fidelity-sweep`, `nemar admin + zarr-fidelity-sweep`) reaches it; the viewer keeps reading index.json + regardless (ADR 0005 -- verification is reported, never a precondition + for serving). --- @@ -410,7 +432,7 @@ and what is historical. The entries worth knowing by name: | `nemar dataset publish` | request, status, resend | | `nemar dataset` (access) | request-access, access, invite, collaborators | | `nemar sandbox` | training run, status, reset — required before uploading | -| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, doctor, summary, notice, email-preferences, e2e-test | +| `nemar admin` | users, approve, revoke, role, notify, s3, repo, ci, doi, publish, revert, make-public, delete-dataset, bulk-delete, reindex, hed-sweep, data-integrity-sweep, zarr-fidelity-sweep, doctor, summary, notice, email-preferences, e2e-test | | `nemar admin import*` | OpenNeuro import, status, rollback, retry, verify, recover (issue #754, epic #967) | | `nemar admin fleet` | drift, enforce, revalidate — governance across dataset repos (epic #713) | | `nemar admin exemplar` | create, status, remint-dois — the staging exemplar fleet | From 2f2e1f1fc82d62be3903000eb02048a04578bdb9 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 12:12:49 -0700 Subject: [PATCH 10/15] fix: harden zarr fidelity sweep per PR 1203 review --- backend/src/services/zarr-fidelity-sweep.ts | 608 +++++++++---- backend/test/zarr-fidelity-sweep.test.ts | 939 +++++++++++++++++--- 2 files changed, 1273 insertions(+), 274 deletions(-) diff --git a/backend/src/services/zarr-fidelity-sweep.ts b/backend/src/services/zarr-fidelity-sweep.ts index e0d1a99c5..e64c46bb9 100644 --- a/backend/src/services/zarr-fidelity-sweep.ts +++ b/backend/src/services/zarr-fidelity-sweep.ts @@ -19,18 +19,27 @@ * here) -- they are always plain git blobs. Fetching them by RAW CONTENT URL * (`raw.githubusercontent.com////`) is therefore * sufficient, and is a plain unauthenticated public-CDN GET: no GitHub App - * installation token, no PAT, no shared API rate limit, and it 404s outright - * for a private repo. That is a materially different exposure than - * `signal-defaults-sweep.ts`'s `getBidsTreeStats`, which is PROD-ONLY - * specifically because it spends the shared GitHub App/PAT reading the - * shared `nemarDatasets` org (see that sweep's cron-wiring comment in - * index.ts). This sweep touches only S3 (the index) and this same public, - * credential-free content host, so -- like `recording-stats-sweep.ts` -- - * it is safe on the non-production cron: AGENTS.md's dev-cron fence exists - * to stop a job from emailing a real user, dispatching GitHub WORK (a - * mutation, or an authenticated read that spends the shared org's quota) - * against `nemarDatasets`, or mutating a real DOI/prod-bucket object; this - * sweep does none of those. + * installation token, no PAT, no shared API rate limit. It is also the + * reason candidates are restricted to PUBLIC, ACTIVE datasets (the + * candidate SQL): a private repo cannot be read anonymously, so verifying + * one would only ever produce `unverifiable` noise, never a real verdict. + * That is a materially different exposure than `signal-defaults-sweep.ts`'s + * `getBidsTreeStats`, which is PROD-ONLY specifically because it spends the + * shared GitHub App/PAT reading the shared `nemarDatasets` org (see that + * sweep's cron-wiring comment in index.ts). This sweep touches only S3 + * (the index) and this same public, credential-free content host, so -- + * like `recording-stats-sweep.ts` -- it is safe on the non-production cron. + * + * FAIL-OPEN ON THE CANDIDATE ROW, NEVER ON THE VERDICT. A transient infra + * failure (a thrown S3 fetch, a non-404 sidecar response, a network error, + * or the fetch budget running out mid-dataset) aborts THAT DATASET's + * verification for this run: nothing is stamped, the failure is recorded in + * the run's `errors`, and the row is untouched -- still a candidate next + * time (see `resolveSidecar`'s tri-state result and its callers). Only a + * clean, confirmed absence (404 at every nearest-first candidate path) is + * "no ground truth here", and only a value that actually parsed counts as + * checked (`sidecar_unparseable`/`store_metadata_invalid` below never + * silently pass as verified). * * NEAREST-FIRST, NOT A FULL BIDS-INHERITANCE WALK. generate_zarr.py's own * `expected_channel_count_for` / `power_line_frequency_for` resolve the @@ -40,12 +49,23 @@ * exists to avoid) or the version manifest (versioned by published tag, not * by the store's own `source_commit`). `bidsSidecarCandidates` below is the * brief's sanctioned fallback: a small, fixed, nearest-first candidate list - * (recording directory, subject directory, dataset root) that covers the - * placements real BIDS datasets actually use, tried in order and cached per - * dataset run so repeated subject-/root-level defaults cost one fetch each. - * `MAX_SIDECAR_FETCHES_PER_DATASET` is a hard safety net on top of that - * cache so one dataset with nothing but misses can't blow the Worker's - * subrequest budget. + * (recording directory, session directory, subject directory, dataset + * root) that covers the placements real BIDS datasets actually use, tried + * in order and cached per dataset run so a repeated session-/subject-/ + * root-level default costs one fetch each. + * + * TWO FETCH BUDGETS. `ZARR_FIDELITY_SWEEP_WIDE_BUDGET` bounds the WHOLE + * invocation (every index fetch plus every sidecar fetch, across every + * candidate dataset); once it is spent the batch stops and remaining + * candidates are left completely untouched (never fetched, never errored -- + * `budget_exhausted` in the result says why `processed` came in low). + * `ZARR_FIDELITY_MAX_SIDECAR_FETCHES_PER_DATASET` additionally bounds ONE + * dataset's sidecar fetches so a single pathological dataset (nothing but + * misses) cannot spend the whole sweep-wide budget by itself. Either budget + * running out mid-dataset surfaces as the SAME "error" outcome + * `resolveSidecar` already uses for a real infra failure (reason + * `budget_exhausted`), which is what aborts that one dataset per the + * fail-open rule above -- it is never turned into "absent". * * Modelled on recording-stats-sweep.ts's shape: candidate SQL, a bounded * per-invocation cap, three-way per-dataset error handling (throw/d1-error @@ -68,7 +88,8 @@ import type { PresignedUrlOptions } from "./s3.js"; * `MODALITY_RATES`, mirrored here -- kept in sync deliberately rather than * imported, since this is Python and this is the TypeScript verification * side reading the SAME published fact back out of the index). Keys are - * UPPERCASE to match a `group.modality` value uppercased at lookup time. */ + * UPPERCASE; lookup trims and uppercases the index's `modality` value + * before matching (issue #1068 PR #1203 review, item 8). */ export const ZARR_FIDELITY_MODALITY_RATE_CAPS: Record = { EEG: 250, MEG: 250, @@ -81,23 +102,32 @@ export const ZARR_FIDELITY_MODALITY_RATE_CAPS: Record = { export const ZARR_FIDELITY_MAX_SAMPLE_STORES = 40; /** Hard cap on `zarr_verify_examples`: at most this many {path, code} - * entries, and the cap below also bounds the serialized byte size. */ + * entries, and the cap below also bounds the serialized byte size. The + * total mismatch count (before truncation) is stamped separately as + * `zarr_verify_mismatch_count`, so a truncated list never hides how many + * mismatches were actually found (PR #1203 review, item 7). */ export const ZARR_FIDELITY_MAX_EXAMPLES = 20; /** Hard byte cap on the serialized `zarr_verify_examples` JSON array. */ export const ZARR_FIDELITY_MAX_EXAMPLES_BYTES = 4096; -/** Safety net on top of the per-dataset resolution cache: at most this many - * sidecar candidate fetches (across every sampled store) per dataset per - * run, so a dataset with nothing but 404s can't consume an unbounded - * subrequest budget. Sized generously above the common case (3 candidates - * x 2 suffix-kinds x a modest number of distinct directories among the - * sample, after subject-/root-level candidates are cached across stores). */ -export const ZARR_FIDELITY_MAX_SIDECAR_FETCHES = 150; +/** Per-dataset sidecar-fetch cap (PR #1203 review, item 3): 40 sampled + * stores x 2 files (channels.tsv + one modality sidecar) plus slack for + * multi-modality stores and nearest-first misses, so one pathological + * dataset cannot spend the whole sweep-wide budget below by itself. */ +export const ZARR_FIDELITY_MAX_SIDECAR_FETCHES_PER_DATASET = 90; + +/** Sweep-wide fetch budget (PR #1203 review, item 3): every index fetch + * plus every sidecar fetch, summed across the WHOLE invocation. Once + * spent, `runZarrFidelitySweep` stops the batch -- remaining candidates + * are left completely untouched, not errored, not stamped. */ +export const ZARR_FIDELITY_SWEEP_WIDE_BUDGET = 600; -/** Default / max datasets per invocation (decision 1). */ +/** Default / max datasets per invocation (decision 1; max lowered from 100 + * to 25 in PR #1203 review, item 3, so one admin call cannot alone spend + * the sweep-wide budget across pathological datasets). */ export const ZARR_FIDELITY_SWEEP_DEFAULT = 25; -export const ZARR_FIDELITY_SWEEP_MAX = 100; +export const ZARR_FIDELITY_SWEEP_MAX = 25; const GITHUB_RAW_ORIGIN = "https://raw.githubusercontent.com"; @@ -109,33 +139,49 @@ export interface ZarrFidelityMismatchExample { } /** - * Candidates (decision 1): converted-with-stores datasets never verified, OR - * verified against a commit that is no longer the dataset's current - * `zarr_source_commit` -- a re-conversion re-arms verification. `github_repo - * IS NOT NULL` is a defensive narrowing beyond the brief's literal predicate - * (not a change to it): a row with no repo has nothing this sweep could ever - * fetch a sidecar from, so it would only ever resolve `unverifiable` -- - * excluding it here just avoids wasted candidate slots, same reasoning - * channel-montage-sweep / signal-defaults-sweep already apply to their own - * candidate sets. + * Candidates (decision 1, amended by PR #1203 review items 5): converted, + * PUBLIC, active datasets never verified, OR verified against a commit that + * is no longer the dataset's current `zarr_source_commit`, OR stamped with + * a null/unusable commit. The last clause fixes a fossilisation bug the + * code reviewer reproduced: `zarr_verified_commit` written as JSON `null` + * (an unverifiable dataset with no fetchable index commit) makes + * `... != zarr_source_commit` evaluate to SQL NULL forever after, which is + * never true, so the row could never become a candidate again even once a + * real commit appeared. `runZarrFidelitySweep` also now stamps `''` + * instead of `null` for that case (belt and braces -- new writes never + * reproduce the fossil, and this predicate still catches any that already + * did or ever do again). + * + * `status = 'active' AND visibility = 'public'` (item 5): a private repo + * cannot be read anonymously via raw.githubusercontent.com, so including + * one here would only ever manufacture `unverifiable` noise. + * `github_repo IS NOT NULL` is a defensive narrowing beyond the brief's + * literal predicate (not a change to it): a row with no repo has nothing + * this sweep could ever fetch a sidecar from. */ export const ZARR_FIDELITY_SWEEP_CANDIDATE_SQL = `SELECT dataset_id, github_repo FROM datasets - WHERE zarr_status = 'ready' + WHERE status = 'active' + AND visibility = 'public' + AND zarr_status = 'ready' AND zarr_store_count > 0 AND github_repo IS NOT NULL AND ( json_extract(sweep_stamps, '$.zarr_verified_at') IS NULL + OR json_extract(sweep_stamps, '$.zarr_verified_commit') IS NULL OR json_extract(sweep_stamps, '$.zarr_verified_commit') != zarr_source_commit ) ORDER BY dataset_id LIMIT ?`; export const ZARR_FIDELITY_SWEEP_REMAINING_SQL = `SELECT COUNT(*) AS n FROM datasets - WHERE zarr_status = 'ready' + WHERE status = 'active' + AND visibility = 'public' + AND zarr_status = 'ready' AND zarr_store_count > 0 AND github_repo IS NOT NULL AND ( json_extract(sweep_stamps, '$.zarr_verified_at') IS NULL + OR json_extract(sweep_stamps, '$.zarr_verified_commit') IS NULL OR json_extract(sweep_stamps, '$.zarr_verified_commit') != zarr_source_commit )`; @@ -144,10 +190,11 @@ export const ZARR_FIDELITY_SWEEP_REMAINING_SQL = `SELECT COUNT(*) AS n FROM data * text (`.rules/testing.md`: never hand-copy). Writes ONLY `sweep_stamps` * (decision 1 / ADR 0034) -- no other `datasets` column changes, on any * verdict. `json(?)` wraps the examples parameter so it lands as a nested - * JSON array, not an escaped string (`json_set(x, '$.k', ?)` with a raw TEXT - * bind would store the literal characters `[...]`, unreadable by - * `json_extract('$.k[0]')`). Bind order: commit, status, examples-json, - * sampled, checked, dataset_id. + * JSON array, not an escaped string. Bind order: commit (never null -- + * `''` when the index has no usable commit, PR #1203 review item 5), + * status, examples-json, sampled, checked, checked_channels, + * checked_duration, checked_rate, unchecked, mismatch_count, + * examples_truncated (0/1), dataset_id. */ export const ZARR_FIDELITY_SWEEP_STAMP_SQL = `UPDATE datasets SET sweep_stamps = json_set( @@ -157,7 +204,13 @@ export const ZARR_FIDELITY_SWEEP_STAMP_SQL = `UPDATE datasets '$.zarr_verify_status', ?, '$.zarr_verify_examples', json(?), '$.zarr_verify_sampled', ?, - '$.zarr_verify_checked', ? + '$.zarr_verify_checked', ?, + '$.zarr_verify_checked_channels', ?, + '$.zarr_verify_checked_duration', ?, + '$.zarr_verify_checked_rate', ?, + '$.zarr_verify_unchecked', ?, + '$.zarr_verify_mismatch_count', ?, + '$.zarr_verify_examples_truncated', ? ) WHERE dataset_id = ?`; @@ -214,12 +267,18 @@ const FULL_COMMIT_RE = /^[0-9a-f]{40}$/i; * walk). `suffix` is the sidecar's own trailing name, e.g. `"channels.tsv"` * or `"eeg.json"`. * - * Three placements, nearest first, deduplicated: + * Four placements, nearest first, deduplicated (PR #1203 review, item 9): * 1. the recording's own directory, full BIDS entities (a per-recording * override); - * 2. the subject directory (session entity included when present), a - * dataset's most common "shared default" placement; - * 3. the dataset root, bare (the dataset-wide default). + * 2. the SESSION directory (`sub-XX/ses-YY/sub-XX_ses-YY_`), only + * when the recording carries both a `sub-` and a `ses-` entity -- a + * session-scoped default lives IN the session directory, never + * combined with the subject directory (the earlier, wrong shape this + * replaces: `sub-XX/sub-XX_ses-YY_` is not a path any BIDS + * writer produces); + * 3. the SUBJECT directory, WITHOUT the session entity + * (`sub-XX/sub-XX_`) -- a cross-session subject default; + * 4. the dataset root, bare (the dataset-wide default). * * Exported for direct unit testing. */ @@ -229,7 +288,6 @@ export function bidsSidecarCandidates(recordingPath: string, suffix: string): st const filename = parts[parts.length - 1]; const dir = parts.slice(0, -1).join("/"); - const subjectDir = parts[0].startsWith("sub-") ? parts[0] : null; const dot = filename.lastIndexOf("."); const stem = dot > 0 ? filename.slice(0, dot) : filename; @@ -242,6 +300,9 @@ export function bidsSidecarCandidates(recordingPath: string, suffix: string): st const subjectEntity = entityTokens.find((t) => t.startsWith("sub-")) ?? null; const sessionEntity = entityTokens.find((t) => t.startsWith("ses-")) ?? null; + const subjectDir = parts[0].startsWith("sub-") ? parts[0] : null; + const sessionDir = subjectDir && parts[1]?.startsWith("ses-") ? `${parts[0]}/${parts[1]}` : null; + const candidates: string[] = []; const add = (candidateDir: string, entities: string[]): void => { const name = entities.length > 0 ? `${entities.join("_")}_${suffix}` : suffix; @@ -249,20 +310,37 @@ export function bidsSidecarCandidates(recordingPath: string, suffix: string): st if (!candidates.includes(path)) candidates.push(path); }; - add(dir, entityTokens); - if (subjectDir) { - const subjectEntities = [subjectEntity, sessionEntity].filter((e): e is string => e !== null); - add(subjectDir, subjectEntities); + add(dir, entityTokens); // 1: recording-level override + if (sessionDir && subjectEntity && sessionEntity) { + add(sessionDir, [subjectEntity, sessionEntity]); // 2: session-level default + } + if (subjectDir && subjectEntity) { + add(subjectDir, [subjectEntity]); // 3: subject-level default, no session entity } - add("", []); + add("", []); // 4: dataset-wide default return candidates; } +/** + * Whether every group's `n_channels` is present and numeric (PR #1203 + * review, item 4). A store with zero groups is vacuously valid (its total + * is legitimately 0, a real fact to compare against channels.tsv); a store + * with a group whose `n_channels` is missing or non-numeric cannot be + * trusted for ANY check (not just the channel count), since the same + * malformed-metadata condition casts doubt on the rest of that group's + * fields too -- see `verifyStore`'s `store_metadata_invalid` branch. + */ +export function zarrFidelityStoreMetadataValid(groups: readonly ZarrFidelityGroupJson[]): boolean { + return groups.every((g) => toFiniteNumber(g.n_channels) !== null); +} + /** Total channels a store serves, summed across its groups -- mirrors * generate_zarr.py's `store_total_channels` exactly (the SAME rule the * converter's own conversion-time gate uses), so a post-hoc disagreement - * here reflects real drift, not a differently-derived total. */ + * here reflects real drift, not a differently-derived total. Callers must + * check {@link zarrFidelityStoreMetadataValid} first; this assumes every + * group's `n_channels` is already known-numeric. */ export function zarrFidelityStoreChannelTotal(groups: readonly ZarrFidelityGroupJson[]): number { return groups.reduce((sum, g) => sum + (toFiniteNumber(g.n_channels) ?? 0), 0); } @@ -321,14 +399,30 @@ async function fetchFidelityIndex( return (await response.json()) as ZarrFidelityIndexJson; } +/** Tri-state result of one candidate-path fetch attempt (PR #1203 review, + * item 1): a 404 is the expected "not at this candidate" signal; any other + * non-2xx (5xx, 429, ...) or a network throw is an infra ERROR, never + * folded into "absent" -- the caller must abort, not guess. */ +type SidecarFetchOutcome = + | { kind: "content"; body: string } + | { kind: "absent" } + | { kind: "error"; reason: string }; + +function rawContentUrl(base: string, repo: string, commit: string, path: string): string { + const encoded = path + .split("/") + .map((seg) => encodeURIComponent(seg)) + .join("/"); + return `${base}/${ORG_NAME}/${repo}/${commit}/${encoded}`; +} + /** * One candidate fetch against the public, credential-free content host (see - * the module doc for why this is dev-safe). A 404 is the expected "not at - * this candidate" signal; any other non-2xx or network error is logged and - * treated the same as a miss for resolution purposes -- fail-open, mirroring - * generate_zarr.py's `expected_channel_count_for`, which turns the gate off - * for a recording rather than treating an unreadable ground truth as a - * mismatch. + * the module doc for why this is dev-safe). Only a 404 is "absent"; a 5xx, + * a 429, any other non-2xx, or a network throw is `{kind:"error"}` (PR + * #1203 review, item 1) -- the caller (`resolveSidecar`) must abort that + * dataset's verification rather than silently treating an infra hiccup as + * "nothing here". */ async function fetchSidecarCandidate( base: string, @@ -336,66 +430,79 @@ async function fetchSidecarCandidate( commit: string, path: string, fetchImpl: typeof fetch, -): Promise { - const encoded = path - .split("/") - .map((seg) => encodeURIComponent(seg)) - .join("/"); - const url = `${base}/${ORG_NAME}/${repo}/${commit}/${encoded}`; +): Promise { + const url = rawContentUrl(base, repo, commit, path); let response: Response; try { response = await fetchImpl(url); } catch (err) { - console.warn( - `[zarr-fidelity-sweep] network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`, - ); - return null; + return { + kind: "error", + reason: `network error fetching ${url}: ${err instanceof Error ? err.message : String(err)}`, + }; } - if (response.status === 404) return null; + if (response.status === 404) return { kind: "absent" }; if (!response.ok) { - console.warn(`[zarr-fidelity-sweep] non-2xx ${response.status} fetching ${url}`); - return null; + return { kind: "error", reason: `HTTP ${response.status} fetching ${url}` }; } - return response.text(); + return { kind: "content", body: await response.text() }; } -/** Per-dataset-run resolution state: a content cache (path -> content, or - * null for a confirmed miss) shared across every sampled store so a - * subject-/root-level default is fetched at most once, and a hard fetch - * budget as the safety net described in the module doc. */ +/** Per-dataset-run resolution state: a content cache (path -> the tri-state + * outcome, minus "error" which is never cached -- see `resolveSidecar`) + * shared across every sampled store so a session-/subject-/root-level + * default is fetched at most once, plus the two fetch budgets (sweep-wide + * and per-dataset) and the "already logged this unknown modality" set + * (PR #1203 review, item 8). */ interface FidelityRunContext { - cache: Map; - budget: { remaining: number }; + datasetId: string; + cache: Map; + perDatasetBudget: { remaining: number }; + sweepBudget: { remaining: number }; githubRawBase: string; repo: string; commit: string; fetchImpl: typeof fetch; + loggedUnknownModalities: Set; } +/** Resolution result for one sidecar suffix against one recording: a hit + * (with the WINNING candidate path, for logging), a clean absence at every + * candidate, or an error that must abort the dataset (PR #1203 review, + * item 1). */ +type SidecarResolution = + | { kind: "content"; path: string; body: string } + | { kind: "absent" } + | { kind: "error"; reason: string }; + async function resolveSidecar( ctx: FidelityRunContext, recordingPath: string, suffix: string, -): Promise<{ path: string; content: string } | null> { +): Promise { for (const candidate of bidsSidecarCandidates(recordingPath, suffix)) { - if (ctx.cache.has(candidate)) { - const cached = ctx.cache.get(candidate) ?? null; - if (cached !== null) return { path: candidate, content: cached }; - continue; + const cached = ctx.cache.get(candidate); + if (cached) { + if (cached.kind === "content") return { kind: "content", path: candidate, body: cached.body }; + continue; // cached "absent" -- try the next candidate } - if (ctx.budget.remaining <= 0) return null; - ctx.budget.remaining--; - const content = await fetchSidecarCandidate( + if (ctx.sweepBudget.remaining <= 0 || ctx.perDatasetBudget.remaining <= 0) { + return { kind: "error", reason: "budget_exhausted" }; + } + ctx.sweepBudget.remaining--; + ctx.perDatasetBudget.remaining--; + const outcome = await fetchSidecarCandidate( ctx.githubRawBase, ctx.repo, ctx.commit, candidate, ctx.fetchImpl, ); - ctx.cache.set(candidate, content); - if (content !== null) return { path: candidate, content }; + if (outcome.kind === "error") return outcome; // never cached; the caller aborts + ctx.cache.set(candidate, outcome); + if (outcome.kind === "content") return { kind: "content", path: candidate, body: outcome.body }; } - return null; + return { kind: "absent" }; } /** @@ -404,8 +511,9 @@ async function resolveSidecar( * path order (`sampleEvenly`, reused from bids-tree.ts) plus every store * whose group has `n_channels === 1` -- a single-channel recording is the * shape most likely to reveal a truncation bug (biosigio#110's own - * signature), so it is never left to chance by the even spread. Exported - * for direct unit testing. + * signature, later confirmed genuine for nm000182/nm000183 -- see issue + * #1068), so it is never left to chance by the even spread. Exported for + * direct unit testing. */ export function zarrFidelitySelectSample( stores: readonly ZarrFidelityStoreJson[], @@ -431,60 +539,160 @@ export function zarrFidelitySelectSample( return sample; } +/** Bounded accumulator for mismatch examples (PR #1203 review, item 7): + * `count` is the TOTAL number of mismatches found, independent of + * truncation, so a truncated `examples` list never hides how many + * mismatches were actually found; `truncated` is set the moment either + * cap (20 entries, 4 KB serialized) would be exceeded. Exported for + * direct unit testing. */ +export interface ZarrFidelityMismatchAccumulator { + examples: ZarrFidelityMismatchExample[]; + count: number; + truncated: boolean; +} + +export function createZarrFidelityMismatchAccumulator(): ZarrFidelityMismatchAccumulator { + return { examples: [], count: 0, truncated: false }; +} + +export function recordZarrFidelityMismatch( + acc: ZarrFidelityMismatchAccumulator, + entry: ZarrFidelityMismatchExample, +): void { + acc.count++; + if (acc.truncated) return; + if (acc.examples.length >= ZARR_FIDELITY_MAX_EXAMPLES) { + acc.truncated = true; + return; + } + const candidate = [...acc.examples, entry]; + if ( + new TextEncoder().encode(JSON.stringify(candidate)).length > ZARR_FIDELITY_MAX_EXAMPLES_BYTES + ) { + acc.truncated = true; + return; + } + acc.examples.push(entry); +} + +/** Per-store verification result (PR #1203 review, items 1, 2, 4): `"ok"` + * carries per-CHECK-KIND checked flags (item 2) so a store that merely got + * a 200 it could not parse never silently counts as checked; `"invalid"` + * is `store_metadata_invalid` (item 4) -- no checks were even attempted; + * `"error"` must abort the whole dataset (item 1). */ +type StoreVerification = + | { + kind: "ok"; + checkedChannels: boolean; + checkedDuration: boolean; + checkedRate: boolean; + mismatches: ZarrFidelityMismatchExample[]; + } + | { kind: "invalid" } + | { kind: "error"; reason: string }; + +function logSidecarUnparseable(ctx: FidelityRunContext, path: string, detail: string): void { + console.warn( + `[zarr-fidelity-sweep] sidecar_unparseable dataset=${ctx.datasetId} url=${rawContentUrl(ctx.githubRawBase, ctx.repo, ctx.commit, path)} detail=${detail}`, + ); +} + /** - * Verify one sampled store against its own ground truth. Returns whether any - * ground truth was reachable at all (`checked`) and the mismatches found (a - * store can be `checked` with zero mismatches). + * Verify one sampled store against its own ground truth. + * + * `store_metadata_invalid` (item 4): a group with a missing/non-numeric + * `n_channels` makes the WHOLE store unverifiable -- not just the channel + * check -- because the same malformed field casts doubt on the group + * array's integrity in general, and the brief is explicit that this must + * never produce `channel_count_mismatch` or a `failed` verdict. + * + * `sidecar_unparseable` (item 2): a reachable (200) body that fails to + * parse -- `channels.tsv` with no usable data rows, or JSON that does not + * even parse -- is logged with the resolved URL and contributes nothing + * checked; it must never read as "verified". */ async function verifyStore( store: ZarrFidelityStoreJson & { path: string }, ctx: FidelityRunContext, -): Promise<{ checked: boolean; mismatches: ZarrFidelityMismatchExample[] }> { - const mismatches: ZarrFidelityMismatchExample[] = []; - let checked = false; +): Promise { const groups = (Array.isArray(store.groups) ? store.groups : []) as ZarrFidelityGroupJson[]; + if (!zarrFidelityStoreMetadataValid(groups)) { + console.warn( + `[zarr-fidelity-sweep] store_metadata_invalid dataset=${ctx.datasetId} store=${store.path}`, + ); + return { kind: "invalid" }; + } + + let checkedChannels = false; + let checkedDuration = false; + let checkedRate = false; + const mismatches: ZarrFidelityMismatchExample[] = []; + // Channel count: total store channels vs. channels.tsv's data-row count. - const channelsHit = await resolveSidecar(ctx, store.path, "channels.tsv"); - if (channelsHit) { - const parsed = parseChannelsTsv(channelsHit.content); - if (parsed) { - checked = true; + const channelsRes = await resolveSidecar(ctx, store.path, "channels.tsv"); + if (channelsRes.kind === "error") return { kind: "error", reason: channelsRes.reason }; + if (channelsRes.kind === "content") { + const parsed = parseChannelsTsv(channelsRes.body); + if (parsed && parsed.count >= 1) { + checkedChannels = true; const total = zarrFidelityStoreChannelTotal(groups); if (total < parsed.count) { mismatches.push({ path: store.path, code: "channel_count_mismatch" }); } + } else { + logSidecarUnparseable(ctx, channelsRes.path, "channels.tsv has no usable data rows"); } } - // Duration + rate, resolved once per distinct modality among this store's - // groups (a mixed-modality store is rare, but keeps this correct if one - // ever exists). + // Duration + rate, resolved once per distinct (trimmed, lowercased) + // modality among this store's groups (a mixed-modality store is rare, + // but keeps this correct if one ever exists). const byModality = new Map(); for (const g of groups) { - const modality = typeof g.modality === "string" ? g.modality.toLowerCase() : null; - if (!modality) continue; + const raw = typeof g.modality === "string" ? g.modality.trim() : ""; + if (!raw) continue; + const modality = raw.toLowerCase(); const list = byModality.get(modality) ?? []; list.push(g); byModality.set(modality, list); } for (const [modality, modalityGroups] of byModality) { - const sidecarHit = await resolveSidecar(ctx, store.path, `${modality}.json`); - if (!sidecarHit) continue; - checked = true; + const sidecarRes = await resolveSidecar(ctx, store.path, `${modality}.json`); + if (sidecarRes.kind === "error") return { kind: "error", reason: sidecarRes.reason }; + if (sidecarRes.kind !== "content") continue; + + try { + JSON.parse(sidecarRes.body); + } catch { + logSidecarUnparseable(ctx, sidecarRes.path, "invalid JSON"); + continue; + } - const recordingDuration = parseRecordingDuration(sidecarHit.content); + const recordingDuration = parseRecordingDuration(sidecarRes.body); if (recordingDuration !== null) { + checkedDuration = true; const storeDuration = zarrFidelityStoreDuration(modalityGroups); if (storeDuration !== null && Math.abs(storeDuration - recordingDuration) > 1) { mismatches.push({ path: store.path, code: "duration_mismatch" }); } } - const samplingFrequency = parseSamplingFrequency(sidecarHit.content); + const samplingFrequency = parseSamplingFrequency(sidecarRes.body); const cap = ZARR_FIDELITY_MODALITY_RATE_CAPS[modality.toUpperCase()]; - if (samplingFrequency !== null && cap !== undefined) { + if (cap === undefined) { + // Unknown modality (item 8): log once per DATASET (loggedUnknownModalities + // is per-dataset-run scoped, unlike the per-store cache), and skip only + // the rate check -- channel/duration checks above are unaffected. + if (!ctx.loggedUnknownModalities.has(modality)) { + ctx.loggedUnknownModalities.add(modality); + console.warn( + `[zarr-fidelity-sweep] unknown modality "${modality}" dataset=${ctx.datasetId}; skipping rate check`, + ); + } + } else if (samplingFrequency !== null) { + checkedRate = true; const expectedRate = Math.min(samplingFrequency, cap); for (const g of modalityGroups) { const rate = toFiniteNumber(g.rate); @@ -497,31 +705,26 @@ async function verifyStore( } } - return { checked, mismatches }; -} - -/** Append to a bounded examples array, respecting both the entry-count cap - * and the serialized-byte cap (decision 1: "hard cap 20 entries and 4 KB"). - * Exported for direct unit testing. */ -export function pushZarrFidelityExample( - examples: ZarrFidelityMismatchExample[], - entry: ZarrFidelityMismatchExample, -): void { - if (examples.length >= ZARR_FIDELITY_MAX_EXAMPLES) return; - const candidate = [...examples, entry]; - const bytes = new TextEncoder().encode(JSON.stringify(candidate)).length; - if (bytes > ZARR_FIDELITY_MAX_EXAMPLES_BYTES) return; - examples.push(entry); + return { kind: "ok", checkedChannels, checkedDuration, checkedRate, mismatches }; } interface DatasetVerificationOutcome { /** null means "could not even produce a verdict" -- an infra error the * caller should record and leave the row an untouched candidate. */ status: ZarrFidelityVerdict | null; - commit: string | null; + /** Never null when `status` is non-null: `''` when the index has no + * usable commit (PR #1203 review, item 5 -- never JSON null, which + * fossilises the candidate predicate). */ + commit: string; sampled: number; checked: number; + checkedChannels: number; + checkedDuration: number; + checkedRate: number; + unchecked: number; examples: ZarrFidelityMismatchExample[]; + mismatchCount: number; + examplesTruncated: boolean; error?: string; } @@ -532,69 +735,98 @@ async function verifyDataset( githubRawBase: string, fetchIndexImpl: typeof fetch, fetchSidecarImpl: typeof fetch, + sweepBudget: { remaining: number }, ): Promise { + const empty = { + commit: "", + sampled: 0, + checked: 0, + checkedChannels: 0, + checkedDuration: 0, + checkedRate: 0, + unchecked: 0, + examples: [], + mismatchCount: 0, + examplesTruncated: false, + }; + + if (sweepBudget.remaining <= 0) { + return { status: null, ...empty, error: "budget_exhausted" }; + } + sweepBudget.remaining--; + let index: ZarrFidelityIndexJson | null; try { index = await fetchFidelityIndex(s3Options, datasetId, fetchIndexImpl); } catch (err) { return { status: null, - commit: null, - sampled: 0, - checked: 0, - examples: [], + ...empty, error: `s3: ${err instanceof Error ? err.message : String(err)}`, }; } if (!index) { - return { - status: null, - commit: null, - sampled: 0, - checked: 0, - examples: [], - error: "zarr_status=ready but index.json is absent", - }; + return { status: null, ...empty, error: "zarr_status=ready but index.json is absent" }; } const commit = typeof index.source_commit === "string" && FULL_COMMIT_RE.test(index.source_commit) ? index.source_commit - : null; + : ""; const stores = (Array.isArray(index.stores) ? index.stores : []) as ZarrFidelityStoreJson[]; const sample = zarrFidelitySelectSample(stores); if (!commit) { // No fetchable ref -- ADR 0005: report the gap, don't fake a verdict. - return { - status: "unverifiable", - commit: null, - sampled: sample.length, - checked: 0, - examples: [], - }; + return { status: "unverifiable", ...empty, sampled: sample.length, unchecked: sample.length }; } const ctx: FidelityRunContext = { + datasetId, cache: new Map(), - budget: { remaining: ZARR_FIDELITY_MAX_SIDECAR_FETCHES }, + perDatasetBudget: { remaining: ZARR_FIDELITY_MAX_SIDECAR_FETCHES_PER_DATASET }, + sweepBudget, githubRawBase, repo, commit, fetchImpl: fetchSidecarImpl, + loggedUnknownModalities: new Set(), }; let checkedCount = 0; - const examples: ZarrFidelityMismatchExample[] = []; + let checkedChannelsCount = 0; + let checkedDurationCount = 0; + let checkedRateCount = 0; + const acc = createZarrFidelityMismatchAccumulator(); + for (const store of sample) { - const { checked, mismatches } = await verifyStore(store, ctx); - if (checked) checkedCount++; - for (const m of mismatches) pushZarrFidelityExample(examples, m); + const result = await verifyStore(store, ctx); + if (result.kind === "error") { + return { status: null, ...empty, commit, sampled: sample.length, error: result.reason }; + } + if (result.kind === "invalid") continue; // unverifiable for this store; not checked + if (result.checkedChannels) checkedChannelsCount++; + if (result.checkedDuration) checkedDurationCount++; + if (result.checkedRate) checkedRateCount++; + if (result.checkedChannels || result.checkedDuration || result.checkedRate) checkedCount++; + for (const m of result.mismatches) recordZarrFidelityMismatch(acc, m); } const status: ZarrFidelityVerdict = - checkedCount === 0 ? "unverifiable" : examples.length > 0 ? "failed" : "verified"; - return { status, commit, sampled: sample.length, checked: checkedCount, examples }; + checkedCount === 0 ? "unverifiable" : acc.count > 0 ? "failed" : "verified"; + return { + status, + commit, + sampled: sample.length, + checked: checkedCount, + checkedChannels: checkedChannelsCount, + checkedDuration: checkedDurationCount, + checkedRate: checkedRateCount, + unchecked: sample.length - checkedCount, + examples: acc.examples, + mismatchCount: acc.count, + examplesTruncated: acc.truncated, + }; } export interface ZarrFidelityDatasetResult { @@ -602,7 +834,13 @@ export interface ZarrFidelityDatasetResult { verdict: ZarrFidelityVerdict; sampled: number; checked: number; + checked_channels: number; + checked_duration: number; + checked_rate: number; + unchecked: number; examples: ZarrFidelityMismatchExample[]; + mismatch_count: number; + examples_truncated: boolean; } export interface ZarrFidelitySweepResult { @@ -619,6 +857,10 @@ export interface ZarrFidelitySweepResult { /** Candidates still unverified after this run; null if the count query * failed. */ remaining: number | null; + /** True when the sweep-wide fetch budget ran out before every requested + * candidate could be attempted (PR #1203 review, item 3): datasets past + * this point were never touched at all. */ + budget_exhausted: boolean; } /** @@ -630,16 +872,17 @@ export interface ZarrFidelitySweepResult { * callers can never drift. * * A dataset whose index can't even be fetched (S3 infra error, or - * `zarr_status='ready'` but index.json absent) is recorded in `errors` and - * left completely untouched -- no stamp write, stays a candidate for the - * next run -- exactly recording-stats-sweep's "throw -> stays a candidate" - * handling for the identical shape of failure. + * `zarr_status='ready'` but index.json absent), or whose sample hits an + * infra error or a spent fetch budget partway through verification, is + * recorded in `errors` and left completely untouched -- no stamp write, + * stays a candidate for the next run (PR #1203 review, item 1). * * Test-only DI seams (`s3Options.endpointUrl`, `githubRawBase`, - * `fetchIndexImpl`, `fetchSidecarImpl`) mirror `zarr-catalog.ts`'s - * `endpointUrl` / `runRecordingStatsSweep`'s `fetchIndex` idiom: every real - * caller omits them, so production always resolves the real S3 host and the - * real `raw.githubusercontent.com`. + * `fetchIndexImpl`, `fetchSidecarImpl`, `sweepWideBudget`) mirror + * `zarr-catalog.ts`'s `endpointUrl` / `runRecordingStatsSweep`'s + * `fetchIndex` idiom: every real caller omits them, so production always + * resolves the real S3 host, the real `raw.githubusercontent.com`, and the + * real {@link ZARR_FIDELITY_SWEEP_WIDE_BUDGET}. * * Throws only if the candidate query itself fails. Per-dataset failures are * collected into `errors`, never thrown. @@ -652,6 +895,7 @@ export async function runZarrFidelitySweep( githubRawBase?: string; fetchIndexImpl?: typeof fetch; fetchSidecarImpl?: typeof fetch; + sweepWideBudget?: number; }, ): Promise { const requested = opts?.limit ?? ZARR_FIDELITY_SWEEP_DEFAULT; @@ -672,14 +916,24 @@ export async function runZarrFidelitySweep( const githubRawBase = opts?.githubRawBase ?? GITHUB_RAW_ORIGIN; const fetchIndexImpl = opts?.fetchIndexImpl ?? fetch; const fetchSidecarImpl = opts?.fetchSidecarImpl ?? fetch; + const sweepBudget = { remaining: opts?.sweepWideBudget ?? ZARR_FIDELITY_SWEEP_WIDE_BUDGET }; let verified = 0; let failed = 0; let unverifiable = 0; + let budgetExhausted = false; const results: ZarrFidelityDatasetResult[] = []; const errors: { dataset_id: string; error: string }[] = []; for (const { dataset_id, github_repo } of candidates) { + if (sweepBudget.remaining <= 0) { + // Sweep-wide budget already spent by an earlier candidate: this and + // every remaining candidate are left completely untouched (PR #1203 + // review, item 3) -- no error entry, no stamp, no fetch attempted. + budgetExhausted = true; + break; + } + const repo = github_repo.split("/")[1] ?? github_repo; const outcome = await verifyDataset( dataset_id, @@ -688,10 +942,12 @@ export async function runZarrFidelitySweep( githubRawBase, fetchIndexImpl, fetchSidecarImpl, + sweepBudget, ); if (outcome.status === null) { errors.push({ dataset_id, error: outcome.error ?? "zarr-fidelity-sweep: unknown error" }); + if (outcome.error === "budget_exhausted") budgetExhausted = true; continue; } @@ -703,6 +959,12 @@ export async function runZarrFidelitySweep( JSON.stringify(outcome.examples), outcome.sampled, outcome.checked, + outcome.checkedChannels, + outcome.checkedDuration, + outcome.checkedRate, + outcome.unchecked, + outcome.mismatchCount, + outcome.examplesTruncated ? 1 : 0, dataset_id, ) .run(); @@ -722,7 +984,13 @@ export async function runZarrFidelitySweep( verdict: outcome.status, sampled: outcome.sampled, checked: outcome.checked, + checked_channels: outcome.checkedChannels, + checked_duration: outcome.checkedDuration, + checked_rate: outcome.checkedRate, + unchecked: outcome.unchecked, examples: outcome.examples, + mismatch_count: outcome.mismatchCount, + examples_truncated: outcome.examplesTruncated, }); if (outcome.status === "failed") { @@ -738,6 +1006,7 @@ export async function runZarrFidelitySweep( details: JSON.stringify({ sampled: outcome.sampled, checked: outcome.checked, + mismatch_count: outcome.mismatchCount, examples: outcome.examples, }), }).run(); @@ -752,12 +1021,13 @@ export async function runZarrFidelitySweep( .catch(() => null); return { - processed: candidates.length, + processed: results.length + errors.length, verified, failed, unverifiable, results, errors, remaining: remainingRow?.n ?? null, + budget_exhausted: budgetExhausted, }; } diff --git a/backend/test/zarr-fidelity-sweep.test.ts b/backend/test/zarr-fidelity-sweep.test.ts index a4907e112..97c868e2d 100644 --- a/backend/test/zarr-fidelity-sweep.test.ts +++ b/backend/test/zarr-fidelity-sweep.test.ts @@ -3,13 +3,20 @@ * phase 8) -- the entry point BOTH real callers * (`POST /admin/datasets/zarr-fidelity-sweep` and the daily cron) use. * + * Rewritten for PR #1203 review: tri-state sidecar fetch (item 1), checked + * only after a real parse (item 2), the two fetch budgets (item 3), + * store_metadata_invalid (item 4), the commit-fossilisation fix (item 5), + * mismatch_count/examples_truncated (item 7), modality normalization + + * once-per-dataset unknown-modality logging (item 8), and the corrected + * session-dir candidate order (item 9). + * * Drives the REAL function against a real D1 (bun:sqlite behind realD1), * substituting the two true network boundaries -- the S3 index GET and the * raw.githubusercontent.com sidecar GET -- with ONE real local `Bun.serve()` - * receiver, via the `s3Options.endpointUrl` / `githubRawBase` DI seams - * (`zarr-catalog.ts`'s `endpointUrl` idiom). Everything else -- the - * candidate query, sample selection, nearest-first resolution, the mismatch - * rules, the stamp write, the audit row -- runs for real. + * receiver, via the `s3Options.endpointUrl` / `githubRawBase` DI seams. + * Everything else -- the candidate query, sample selection, nearest-first + * resolution, the mismatch rules, the stamp write, the audit row -- runs + * for real. * * Per `.rules/testing.md`: never hand-copy a SQL statement -- the assertions * below read the stamped `sweep_stamps` JSON back with `json_extract` @@ -17,16 +24,19 @@ */ import type { Database } from "bun:sqlite"; -import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import type { Server } from "bun"; +import { parseRecordingDuration } from "../src/services/channel-montage"; import { type ZarrFidelityGroupJson, bidsSidecarCandidates, - pushZarrFidelityExample, + createZarrFidelityMismatchAccumulator, + recordZarrFidelityMismatch, runZarrFidelitySweep, zarrFidelitySelectSample, zarrFidelityStoreChannelTotal, zarrFidelityStoreDuration, + zarrFidelityStoreMetadataValid, } from "../src/services/zarr-fidelity-sweep"; import type { Bindings } from "../src/types/bindings"; import { freshDb, realD1 } from "./helpers/d1"; @@ -40,51 +50,62 @@ const COMMIT_B = "b".repeat(40); // --------------------------------------------------------------------------- describe("bidsSidecarCandidates: nearest-first, bounded, deduplicated", () => { - test("recording directory, subject directory, dataset root -- in that order", () => { + test("session-based recording: recording dir -> session dir -> subject dir (no session) -> root (item 9)", () => { const candidates = bidsSidecarCandidates( "sub-01/ses-01/eeg/sub-01_ses-01_task-rest_run-01_eeg.set", "channels.tsv", ); expect(candidates).toEqual([ "sub-01/ses-01/eeg/sub-01_ses-01_task-rest_run-01_channels.tsv", - "sub-01/sub-01_ses-01_channels.tsv", + "sub-01/ses-01/sub-01_ses-01_channels.tsv", + "sub-01/sub-01_channels.tsv", "channels.tsv", ]); }); - test("no subject entity and no dashed entity tokens: recording dir collapses to a bare name", () => { - // "file" has no "-", so it is not treated as a BIDS entity -- the - // recording-directory candidate is the bare suffix in that directory, - // and there is no subject-directory candidate at all (no "sub-*" path - // segment), leaving just the recording dir and the dataset root. + test("no session entity: recording dir -> subject dir -> root (no session-dir candidate)", () => { + const candidates = bidsSidecarCandidates("sub-02/eeg/sub-02_task-x_eeg.set", "eeg.json"); + expect(candidates).toEqual([ + "sub-02/eeg/sub-02_task-x_eeg.json", + "sub-02/sub-02_eeg.json", + "eeg.json", + ]); + }); + + test("no subject entity at all: recording dir -> root only", () => { const candidates = bidsSidecarCandidates("weird/path/file_eeg.set", "channels.tsv"); expect(candidates).toEqual(["weird/path/channels.tsv", "channels.tsv"]); }); test("a root-level recording (no directory) still yields the bare root candidate", () => { const candidates = bidsSidecarCandidates("sub-01_task-rest_eeg.set", "channels.tsv"); - expect(candidates).toContain("channels.tsv"); expect(candidates[candidates.length - 1]).toBe("channels.tsv"); }); +}); - test("works identically for a modality sidecar suffix", () => { - const candidates = bidsSidecarCandidates("sub-02/eeg/sub-02_task-x_eeg.set", "eeg.json"); - expect(candidates).toEqual([ - "sub-02/eeg/sub-02_task-x_eeg.json", - "sub-02/sub-02_eeg.json", - "eeg.json", - ]); +describe("zarrFidelityStoreMetadataValid: item 4", () => { + test("every group with a numeric n_channels is valid", () => { + expect(zarrFidelityStoreMetadataValid([{ n_channels: 10 }, { n_channels: 9 }])).toBe(true); + }); + + test("a group missing n_channels is invalid", () => { + expect(zarrFidelityStoreMetadataValid([{ n_channels: 10 }, { n_channels: undefined }])).toBe( + false, + ); + }); + + test("a group with a non-numeric n_channels is invalid", () => { + expect(zarrFidelityStoreMetadataValid([{ n_channels: "ten" }])).toBe(false); + }); + + test("zero groups is vacuously valid (a real, checkable total of 0)", () => { + expect(zarrFidelityStoreMetadataValid([])).toBe(true); }); }); describe("zarrFidelityStoreChannelTotal: sum across groups", () => { - test("sums n_channels across groups, treating missing/invalid as 0", () => { - const groups: ZarrFidelityGroupJson[] = [ - { n_channels: 10 }, - { n_channels: 9 }, - { n_channels: undefined }, - { n_channels: "not a number" }, - ]; + test("sums n_channels across groups", () => { + const groups: ZarrFidelityGroupJson[] = [{ n_channels: 10 }, { n_channels: 9 }]; expect(zarrFidelityStoreChannelTotal(groups)).toBe(19); }); @@ -119,9 +140,24 @@ describe("zarrFidelitySelectSample: decision 1's sampling rule", () => { expect(sample.length).toBe(40); }); - test("more than 40: spread evenly to 40, PLUS every n_channels===1 store outside the spread", () => { - // 100 stores; only store #50 has n_channels === 1, positioned to be - // unlikely to land in an even 40-of-100 spread by construction. + test("exactly 41 stores: 40 spread plus every n_channels===1 store, no duplicates (item 13)", () => { + // 41 stores; only store #40 (the last) has n_channels === 1. sampleEvenly + // over 41 items capped to 40 always includes the first AND last item (its + // own documented contract), so this fixture alone can't distinguish "the + // spread happened to include it" from "the forced-inclusion rule did" -- + // the second test below (#20, off-spread) is the one that actually + // exercises the forced-inclusion path. + const stores = Array.from({ length: 41 }, (_, i) => + store(`s${String(i).padStart(3, "0")}`, i === 40 ? 1 : 8), + ); + const sample = zarrFidelitySelectSample(stores); + expect(sample.length).toBe(40); + const ids = sample.map((s) => s.path); + expect(new Set(ids).size).toBe(ids.length); // no duplicates + expect(ids).toContain("s040"); + }); + + test("a n_channels===1 store OUTSIDE the even spread is still force-included", () => { const stores = Array.from({ length: 100 }, (_, i) => store(`s${String(i).padStart(3, "0")}`, i === 50 ? 1 : 8), ); @@ -137,28 +173,64 @@ describe("zarrFidelitySelectSample: decision 1's sampling rule", () => { }); }); -describe("pushZarrFidelityExample: bounded to 20 entries / 4 KB", () => { - test("stops appending past 20 entries", () => { - const examples: { path: string; code: "channel_count_mismatch" }[] = []; +describe("createZarrFidelityMismatchAccumulator / recordZarrFidelityMismatch (item 7)", () => { + test("stops appending past 20 entries but keeps counting", () => { + const acc = createZarrFidelityMismatchAccumulator(); for (let i = 0; i < 30; i++) { - pushZarrFidelityExample(examples, { path: `p${i}`, code: "channel_count_mismatch" }); + recordZarrFidelityMismatch(acc, { path: `p${i}`, code: "channel_count_mismatch" }); } - expect(examples.length).toBe(20); - expect(examples[0].path).toBe("p0"); - expect(examples[19].path).toBe("p19"); + expect(acc.examples.length).toBe(20); + expect(acc.examples[0].path).toBe("p0"); + expect(acc.examples[19].path).toBe("p19"); + expect(acc.count).toBe(30); + expect(acc.truncated).toBe(true); }); - test("stops appending once the serialized array would exceed 4 KB, even under 20 entries", () => { - const examples: { path: string; code: "channel_count_mismatch" }[] = []; + test("stops appending once the serialized array would exceed 4 KB, even under 20 entries -- count and truncated stay accurate (item 16)", () => { + const acc = createZarrFidelityMismatchAccumulator(); const longPath = "x".repeat(500); for (let i = 0; i < 20; i++) { - pushZarrFidelityExample(examples, { - path: `${longPath}${i}`, - code: "channel_count_mismatch", - }); + recordZarrFidelityMismatch(acc, { path: `${longPath}${i}`, code: "channel_count_mismatch" }); } - expect(examples.length).toBeLessThan(20); - expect(new TextEncoder().encode(JSON.stringify(examples)).length).toBeLessThanOrEqual(4096); + expect(acc.examples.length).toBeLessThan(20); + expect(new TextEncoder().encode(JSON.stringify(acc.examples)).length).toBeLessThanOrEqual(4096); + expect(acc.count).toBe(20); + expect(acc.truncated).toBe(true); + }); + + test("under both caps: nothing truncated, count matches examples length", () => { + const acc = createZarrFidelityMismatchAccumulator(); + recordZarrFidelityMismatch(acc, { path: "a", code: "duration_mismatch" }); + recordZarrFidelityMismatch(acc, { path: "b", code: "rate_mismatch" }); + expect(acc.examples.length).toBe(2); + expect(acc.count).toBe(2); + expect(acc.truncated).toBe(false); + }); +}); + +describe("parseRecordingDuration (item 15)", () => { + test("missing RecordingDuration -> null", () => { + expect(parseRecordingDuration(JSON.stringify({ SamplingFrequency: 250 }))).toBeNull(); + }); + + test("non-numeric RecordingDuration -> null", () => { + expect(parseRecordingDuration(JSON.stringify({ RecordingDuration: "120" }))).toBeNull(); + }); + + test("zero RecordingDuration -> null", () => { + expect(parseRecordingDuration(JSON.stringify({ RecordingDuration: 0 }))).toBeNull(); + }); + + test("negative RecordingDuration -> null", () => { + expect(parseRecordingDuration(JSON.stringify({ RecordingDuration: -5 }))).toBeNull(); + }); + + test("malformed JSON -> null, does not throw", () => { + expect(parseRecordingDuration("{not valid json")).toBeNull(); + }); + + test("a valid positive RecordingDuration parses", () => { + expect(parseRecordingDuration(JSON.stringify({ RecordingDuration: 120.5 }))).toBe(120.5); }); }); @@ -166,12 +238,16 @@ describe("pushZarrFidelityExample: bounded to 20 entries / 4 KB", () => { // Real-engine fixture server: one local Bun.serve() answering both the // signed S3 index GET (path `//zarr/index.json`) and the public // raw.githubusercontent.com sidecar GET (path -// `////`). +// `////`). Both support an explicit status +// override (for transient-infra-error tests, item 11) that wins over any +// registered fixture content. // --------------------------------------------------------------------------- let server: Server; const indexFixtures = new Map(); +const indexStatusOverrides = new Map(); const sidecarFixtures = new Map(); +const sidecarStatusOverrides = new Map(); beforeAll(() => { server = Bun.serve({ @@ -189,6 +265,8 @@ beforeAll(() => { parts[parts.length - 1] === "index.json" ) { const datasetId = parts[0]; + const statusOverride = indexStatusOverrides.get(datasetId); + if (statusOverride !== undefined) return new Response("error", { status: statusOverride }); const body = indexFixtures.get(datasetId); if (body === undefined) return new Response("not found", { status: 404 }); return new Response(JSON.stringify(body), { status: 200 }); @@ -197,6 +275,8 @@ beforeAll(() => { if (parts[0] === ORG_NAME) { const [, repo, commit, ...rest] = parts; const key = `${repo}/${commit}/${rest.join("/")}`; + const statusOverride = sidecarStatusOverrides.get(key); + if (statusOverride !== undefined) return new Response("error", { status: statusOverride }); const content = sidecarFixtures.get(key); if (content === undefined) return new Response("not found", { status: 404 }); return new Response(content, { status: 200 }); @@ -231,7 +311,8 @@ function runOpts() { }; } -/** Minimal, valid `datasets` row -- zarr-ready, converted, with a repo. */ +/** Minimal, valid `datasets` row -- zarr-ready, converted, public, active, + * with a repo. */ function seedDataset( id: string, repo: string, @@ -246,7 +327,7 @@ function seedDataset( zarr_status: "ready", zarr_store_count: 1, zarr_source_commit: COMMIT_A, - github_repo: `${ORG_NAME}/${repo}`, + github_repo: repo.includes("/") ? repo : `${ORG_NAME}/${repo}`, ...overrides, }; const keys = Object.keys(merged); @@ -264,7 +345,13 @@ function row(id: string): Record { json_extract(sweep_stamps, '$.zarr_verified_commit') AS zarr_verified_commit, json_extract(sweep_stamps, '$.zarr_verify_examples') AS zarr_verify_examples_raw, json_extract(sweep_stamps, '$.zarr_verify_sampled') AS zarr_verify_sampled, - json_extract(sweep_stamps, '$.zarr_verify_checked') AS zarr_verify_checked + json_extract(sweep_stamps, '$.zarr_verify_checked') AS zarr_verify_checked, + json_extract(sweep_stamps, '$.zarr_verify_checked_channels') AS zarr_verify_checked_channels, + json_extract(sweep_stamps, '$.zarr_verify_checked_duration') AS zarr_verify_checked_duration, + json_extract(sweep_stamps, '$.zarr_verify_checked_rate') AS zarr_verify_checked_rate, + json_extract(sweep_stamps, '$.zarr_verify_unchecked') AS zarr_verify_unchecked, + json_extract(sweep_stamps, '$.zarr_verify_mismatch_count') AS zarr_verify_mismatch_count, + json_extract(sweep_stamps, '$.zarr_verify_examples_truncated') AS zarr_verify_examples_truncated FROM datasets WHERE dataset_id = ?`, ) .get(id) as Record; @@ -283,14 +370,28 @@ function eegJson(samplingFrequency: number, recordingDuration: number): string { }); } +let warnSpy: typeof console.warn; +let warnCalls: unknown[][]; + beforeEach(() => { db = freshDb(); indexFixtures.clear(); + indexStatusOverrides.clear(); sidecarFixtures.clear(); + sidecarStatusOverrides.clear(); + warnCalls = []; + warnSpy = console.warn; + console.warn = (...args: unknown[]) => { + warnCalls.push(args); + }; +}); + +afterEach(() => { + console.warn = warnSpy; }); describe("runZarrFidelitySweep: per-dataset verdicts", () => { - test("VERIFIED: channel count, duration, and rate all agree with ground truth", () => { + test("VERIFIED: channel count, duration, and rate all agree with ground truth", async () => { const id = "on800001"; seedDataset(id, id); indexFixtures.set(id, { @@ -313,26 +414,42 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { eegJson(250, 120), ); - return runZarrFidelitySweep(env(), runOpts()).then((result) => { - expect(result.errors).toEqual([]); - expect(result.verified).toBe(1); - expect(result.failed).toBe(0); - expect(result.unverifiable).toBe(0); - expect(result.results).toEqual([ - { dataset_id: id, verdict: "verified", sampled: 1, checked: 1, examples: [] }, - ]); - - const r = row(id); - expect(r.zarr_verify_status).toBe("verified"); - expect(r.zarr_verified_commit).toBe(COMMIT_A); - expect(r.zarr_verify_sampled).toBe(1); - expect(r.zarr_verify_checked).toBe(1); - expect(JSON.parse(r.zarr_verify_examples_raw as string)).toEqual([]); - expect(r.zarr_verified_at).not.toBeNull(); - // Untouched columns: only sweep_stamps changed. - expect(r.name).toBe(id); - expect(r.zarr_store_count).toBe(1); - }); + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.errors).toEqual([]); + expect(result.verified).toBe(1); + expect(result.budget_exhausted).toBe(false); + expect(result.results).toEqual([ + { + dataset_id: id, + verdict: "verified", + sampled: 1, + checked: 1, + checked_channels: 1, + checked_duration: 1, + checked_rate: 1, + unchecked: 0, + examples: [], + mismatch_count: 0, + examples_truncated: false, + }, + ]); + + const r = row(id); + expect(r.zarr_verify_status).toBe("verified"); + expect(r.zarr_verified_commit).toBe(COMMIT_A); + expect(r.zarr_verify_sampled).toBe(1); + expect(r.zarr_verify_checked).toBe(1); + expect(r.zarr_verify_checked_channels).toBe(1); + expect(r.zarr_verify_checked_duration).toBe(1); + expect(r.zarr_verify_checked_rate).toBe(1); + expect(r.zarr_verify_unchecked).toBe(0); + expect(r.zarr_verify_mismatch_count).toBe(0); + expect(r.zarr_verify_examples_truncated).toBe(0); + expect(JSON.parse(r.zarr_verify_examples_raw as string)).toEqual([]); + expect(r.zarr_verified_at).not.toBeNull(); + // Untouched columns: only sweep_stamps changed. + expect(r.name).toBe(id); + expect(r.zarr_store_count).toBe(1); }); test("FAILED (channel_count_mismatch): store total channels fall short of channels.tsv", async () => { @@ -363,6 +480,8 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { expect(result.results[0].examples).toEqual([ { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "channel_count_mismatch" }, ]); + expect(result.results[0].mismatch_count).toBe(1); + expect(result.results[0].examples_truncated).toBe(false); const r = row(id); expect(r.zarr_verify_status).toBe("failed"); @@ -370,7 +489,6 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { { path: "sub-01/eeg/sub-01_task-rest_eeg.set", code: "channel_count_mismatch" }, ]); - // An audit_log row was written for the failure (decision 3). const audit = db .query( "SELECT action, resource_id, details FROM audit_log WHERE action = 'zarr_fidelity_failed'", @@ -401,7 +519,6 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, channelsTsv(19), ); - // RecordingDuration 120 vs the store's duration_s 100 -- a 20s gap. sidecarFixtures.set( `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, eegJson(250, 120), @@ -424,8 +541,6 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { { path: "sub-01/eeg/sub-01_task-rest_eeg.set", zarr: "z", - // SamplingFrequency 500 capped at EEG's 250 -> expected rate 250, - // but the store serves 500 (as if the cap were never applied). groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 500 }], }, ], @@ -446,7 +561,7 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { ]); }); - test("UNVERIFIABLE: no sidecar is reachable at any candidate path", async () => { + test("UNVERIFIABLE: no sidecar is reachable at any candidate path (clean 404s)", async () => { const id = "on800005"; seedDataset(id, id); indexFixtures.set(id, { @@ -460,20 +575,29 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { }, ], }); - // No sidecarFixtures registered at all -- every candidate 404s. + // No sidecarFixtures registered at all -- every candidate 404s (absent, + // not error). const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.errors).toEqual([]); expect(result.unverifiable).toBe(1); expect(result.results[0]).toEqual({ dataset_id: id, verdict: "unverifiable", sampled: 1, checked: 0, + checked_channels: 0, + checked_duration: 0, + checked_rate: 0, + unchecked: 1, examples: [], + mismatch_count: 0, + examples_truncated: false, }); const r = row(id); expect(r.zarr_verify_status).toBe("unverifiable"); expect(r.zarr_verify_checked).toBe(0); + expect(r.zarr_verify_unchecked).toBe(1); }); test("v1/v3 index tolerance: unrelated v3-only top-level and store fields do not break parsing", async () => { @@ -524,7 +648,7 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { expect(result.verified).toBe(1); }); - test("bounded examples: 25 mismatching stores cap zarr_verify_examples at 20 entries", async () => { + test("bounded examples: 25 mismatching stores cap examples at 20, but mismatch_count/examples_truncated are honest", async () => { const id = "on800007"; seedDataset(id, id, { zarr_store_count: 25 }); const stores = Array.from({ length: 25 }, (_, i) => { @@ -538,7 +662,6 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { indexFixtures.set(id, { source_commit: COMMIT_A, store_count: 25, stores }); for (let i = 0; i < 25; i++) { const p = `sub-${String(i).padStart(2, "0")}`; - // 5 channels declared, store only serves 1 -- every store mismatches. sidecarFixtures.set(`${id}/${COMMIT_A}/${p}/eeg/${p}_task-x_channels.tsv`, channelsTsv(5)); sidecarFixtures.set(`${id}/${COMMIT_A}/${p}/eeg/${p}_task-x_eeg.json`, eegJson(250, 10)); } @@ -548,12 +671,120 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { expect(result.results[0].sampled).toBe(25); expect(result.results[0].checked).toBe(25); expect(result.results[0].examples.length).toBe(20); + expect(result.results[0].mismatch_count).toBe(25); + expect(result.results[0].examples_truncated).toBe(true); const r = row(id); const examples = JSON.parse(r.zarr_verify_examples_raw as string) as unknown[]; expect(examples.length).toBe(20); + expect(r.zarr_verify_mismatch_count).toBe(25); + expect(r.zarr_verify_examples_truncated).toBe(1); + }); + + test("index.json absent (zarr_status='ready' but nothing at the S3 key) is an error, not a stamp", async () => { + const id = "on800009"; + seedDataset(id, id); + // No indexFixtures entry -- the fixture server 404s. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.errors).toEqual([ + { dataset_id: id, error: "zarr_status=ready but index.json is absent" }, + ]); + const r = row(id); + expect(r.zarr_verify_status).toBeNull(); + }); + + test("an index with no fetchable source_commit is unverifiable, not a crash", async () => { + const id = "on800010"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: "", // pre-phase-7 empty-string bug + store_count: 1, + stores: [{ path: "sub-01/eeg/sub-01_task-rest_eeg.set", zarr: "z", groups: [] }], + }); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.unverifiable).toBe(1); + expect(row(id).zarr_verify_status).toBe("unverifiable"); + // Item 5: the commit is stamped as '' (never JSON null), so the next + // sweep can re-arm this row once a real commit appears. + expect(row(id).zarr_verified_commit).toBe(""); + }); + + test("candidates are clamped to ZARR_FIDELITY_SWEEP_MAX (25) and remaining reflects real state", async () => { + for (let i = 0; i < 3; i++) { + const id = `on80101${i}`; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + } + const result = await runZarrFidelitySweep(env(), { ...runOpts(), limit: 1 }); + expect(result.processed).toBe(1); + expect(result.remaining).toBe(2); + }); + + test("a requested limit above ZARR_FIDELITY_SWEEP_MAX (25) is clamped down to it", async () => { + // 30 candidates, none with an index fixture -- every one is a real + // (absent-index) error, so `processed` reflects exactly how many were + // attempted, proving the SQL LIMIT bind itself was clamped to 25. None + // of them get stamped (an error never stamps), so all 30 are still + // candidates afterwards -- `remaining` here proves nothing about the + // clamp and is intentionally not asserted. + for (let i = 0; i < 30; i++) { + seedDataset(`on802${String(i).padStart(3, "0")}`, `on802${String(i).padStart(3, "0")}`); + } + const result = await runZarrFidelitySweep(env(), { ...runOpts(), limit: 999_999 }); + expect(result.processed).toBe(25); + expect(result.errors.length).toBe(25); + }); + + test("github_repo without an org prefix still resolves the repo correctly (item 17)", async () => { + const id = "on800011"; + // github_repo stored WITHOUT the "nemarDatasets/" prefix. + seedDataset(id, "on800011-bare-repo", { github_repo: "on800011-bare-repo" }); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `on800011-bare-repo/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `on800011-bare-repo/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); }); +}); +describe("runZarrFidelitySweep: re-conversion re-arming, including the fossilisation fix (item 5)", () => { test("a re-conversion (zarr_source_commit changes) re-arms verification", async () => { const id = "on800008"; seedDataset(id, id); @@ -582,12 +813,9 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { expect(first.processed).toBe(1); expect(row(id).zarr_verified_commit).toBe(COMMIT_A); - // Not re-selected while the store's commit is unchanged. const second = await runZarrFidelitySweep(env(), runOpts()); expect(second.processed).toBe(0); - // Simulate a re-conversion: the dataset's zarr_source_commit advances, - // and the published index now reflects the new commit. db.query("UPDATE datasets SET zarr_source_commit = ? WHERE dataset_id = ?").run(COMMIT_B, id); indexFixtures.set(id, buildIndex(COMMIT_B)); sidecarFixtures.set( @@ -605,59 +833,560 @@ describe("runZarrFidelitySweep: per-dataset verdicts", () => { expect(row(id).zarr_verified_commit).toBe(COMMIT_B); }); - test("index.json absent (zarr_status='ready' but nothing at the S3 key) is an error, not a stamp", async () => { - const id = "on800009"; + test("bad commit stamped, then re-converted with a real commit, then swept again (fossilisation regression)", async () => { + const id = "on800012"; + seedDataset(id, id, { zarr_source_commit: COMMIT_A }); + // Simulate a row ALREADY fossilised by the pre-fix code path: a stamp + // whose zarr_verified_commit is JSON null (what the old + // `outcome.commit` bind -- a raw `null` -- used to write), with + // zarr_verified_at already set. Without this PR's predicate fix + // (the added `IS NULL` branch), `json_extract(...) != zarr_source_commit` + // is SQL NULL when the stored value is JSON null -- never true -- so + // this row could never become a candidate again, even once a real + // commit landed. + db.query( + `UPDATE datasets + SET sweep_stamps = json_set( + COALESCE(sweep_stamps, '{}'), + '$.zarr_verified_at', datetime('now'), + '$.zarr_verified_commit', json('null'), + '$.zarr_verify_status', 'unverifiable' + ) + WHERE dataset_id = ?`, + ).run(id); + expect(row(id).zarr_verified_commit).toBeNull(); + + // A genuine re-conversion: the index now has a real, fetchable commit. + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.processed).toBe(1); // re-selected despite the null-commit fossil + expect(result.verified).toBe(1); + expect(row(id).zarr_verify_status).toBe("verified"); + expect(row(id).zarr_verified_commit).toBe(COMMIT_A); + + // And now it correctly stops being a candidate. + const second = await runZarrFidelitySweep(env(), runOpts()); + expect(second.processed).toBe(0); + }); +}); + +describe("runZarrFidelitySweep: tri-state fetch aborts the dataset, never fakes absence (item 1, 11)", () => { + test("a transient index 500 leaves the dataset in errors, unstamped, still a candidate", async () => { + const id = "on800020"; seedDataset(id, id); - // No indexFixtures entry -- the fixture server 404s. + indexStatusOverrides.set(id, 500); const result = await runZarrFidelitySweep(env(), runOpts()); - expect(result.errors).toEqual([ - { dataset_id: id, error: "zarr_status=ready but index.json is absent" }, - ]); - const r = row(id); - expect(r.zarr_verify_status).toBeNull(); + expect(result.results).toEqual([]); + expect(result.errors.length).toBe(1); + expect(result.errors[0].dataset_id).toBe(id); + expect(row(id).zarr_verify_status).toBeNull(); + + const second = await runZarrFidelitySweep(env(), runOpts()); + expect(second.processed).toBe(1); // still a candidate }); - test("an index with no fetchable source_commit is unverifiable, not a crash", async () => { - const id = "on800010"; + test("a transient sidecar 503 aborts that dataset's verification (not stamped, in errors)", async () => { + const id = "on800021"; seedDataset(id, id); indexFixtures.set(id, { - source_commit: "", // pre-phase-7 empty-string bug + source_commit: COMMIT_A, store_count: 1, - stores: [{ path: "sub-01/eeg/sub-01_task-rest_eeg.set", zarr: "z", groups: [] }], + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], }); + // Every channels.tsv candidate 503s. + for (const p of [ + "sub-01/eeg/sub-01_task-rest_channels.tsv", + "sub-01/sub-01_channels.tsv", + "channels.tsv", + ]) { + sidecarStatusOverrides.set(`${id}/${COMMIT_A}/${p}`, 503); + } const result = await runZarrFidelitySweep(env(), runOpts()); - expect(result.unverifiable).toBe(1); - expect(row(id).zarr_verify_status).toBe("unverifiable"); + expect(result.results).toEqual([]); + expect(result.errors.length).toBe(1); + expect(result.errors[0].dataset_id).toBe(id); + expect(row(id).zarr_verify_status).toBeNull(); }); - test("candidates are clamped to ZARR_FIDELITY_SWEEP_MAX and remaining reflects real state", async () => { - for (let i = 0; i < 3; i++) { - const id = `on80101${i}`; + test("a transient sidecar 429 aborts that dataset's verification the same way", async () => { + const id = "on800022"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + for (const p of [ + "sub-01/eeg/sub-01_task-rest_channels.tsv", + "sub-01/sub-01_channels.tsv", + "channels.tsv", + ]) { + sidecarStatusOverrides.set(`${id}/${COMMIT_A}/${p}`, 429); + } + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.results).toEqual([]); + expect(result.errors.length).toBe(1); + expect(row(id).zarr_verify_status).toBeNull(); + }); +}); + +describe("runZarrFidelitySweep: the two fetch budgets (item 3)", () => { + test("sweep-wide budget exhaustion stops the batch; untouched candidates are neither errored nor stamped", async () => { + const ids = ["on800090", "on800091", "on800092"]; + for (const id of ids) { seedDataset(id, id); indexFixtures.set(id, { source_commit: COMMIT_A, store_count: 1, stores: [ { - path: "sub-01/eeg/sub-01_task-rest_eeg.set", + path: "sub-01/eeg/sub-01_task-x_eeg.set", zarr: "z", groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], }, ], }); sidecarFixtures.set( - `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-x_channels.tsv`, channelsTsv(19), ); + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-x_eeg.json`, eegJson(250, 120)); + } + + // Budget of 1: the first dataset's index fetch spends it entirely, so + // its first sidecar resolution immediately sees an exhausted budget -- + // that dataset aborts (errors, unstamped); the other two are never + // even attempted. + const result = await runZarrFidelitySweep(env(), { ...runOpts(), sweepWideBudget: 1 }); + expect(result.budget_exhausted).toBe(true); + expect(result.errors.length).toBe(1); + expect(result.errors[0].error).toBe("budget_exhausted"); + expect(result.results).toEqual([]); + expect(result.processed).toBe(1); + // All three rows are still candidates: the errored one was never + // stamped, and the other two were never touched at all. + expect(result.remaining).toBe(3); + for (const id of ids) { + expect(row(id).zarr_verify_status).toBeNull(); + } + }); + + test("per-dataset sidecar budget exhaustion (90) aborts that one dataset without touching sweep-wide accounting for others", async () => { + const id = "on800095"; + const stores = Array.from({ length: 30 }, (_, i) => { + const p = `sub-${String(i).padStart(2, "0")}`; + return { + path: `${p}/eeg/${p}_task-x_eeg.set`, + zarr: `${p}.zarr`, + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }; + }); + seedDataset(id, id, { zarr_store_count: 30 }); + indexFixtures.set(id, { source_commit: COMMIT_A, store_count: 30, stores }); + // Nothing registered anywhere -- every candidate 404s (absent) until + // the per-dataset budget (90) is spent partway through the sample, + // at which point the NEXT resolution attempt reports budget_exhausted. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.results).toEqual([]); + expect(result.errors.length).toBe(1); + expect(result.errors[0].dataset_id).toBe(id); + expect(result.errors[0].error).toBe("budget_exhausted"); + expect(row(id).zarr_verify_status).toBeNull(); + }); +}); + +describe("runZarrFidelitySweep: checked only after a real parse (item 2)", () => { + test("a reachable but malformed JSON sidecar logs sidecar_unparseable and never counts as checked", async () => { + const id = "on800030"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + "{not valid json", + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + // Channels checked fine; duration/rate never checked (malformed JSON). + expect(result.results[0].verdict).toBe("verified"); // checked via channels alone, no mismatch + expect(result.results[0].checked_channels).toBe(1); + expect(result.results[0].checked_duration).toBe(0); + expect(result.results[0].checked_rate).toBe(0); + expect(warnCalls.some((c) => String(c[0]).includes("sidecar_unparseable"))).toBe(true); + expect(warnCalls.some((c) => String(c[0]).includes("eeg.json"))).toBe(true); + }); + + test("channels.tsv with zero usable data rows is unverifiable for the channel check, never verified off it", async () => { + const id = "on800031"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19 }], + }, + ], + }); + // Header only, no data rows. + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, "name\ttype"); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.unverifiable).toBe(1); + expect(result.results[0].checked_channels).toBe(0); + expect(warnCalls.some((c) => String(c[0]).includes("sidecar_unparseable"))).toBe(true); + }); + + test("per-check-kind checked counts: only channels checked when no modality sidecar exists at all", async () => { + const id = "on800032"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + // No eeg.json anywhere -- absent, not an error. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + expect(result.results[0].checked_channels).toBe(1); + expect(result.results[0].checked_duration).toBe(0); + expect(result.results[0].checked_rate).toBe(0); + expect(row(id).zarr_verify_checked_channels).toBe(1); + expect(row(id).zarr_verify_checked_duration).toBe(0); + expect(row(id).zarr_verify_checked_rate).toBe(0); + }); +}); + +describe("runZarrFidelitySweep: store_metadata_invalid never produces a false mismatch (item 4)", () => { + test("a group with a missing n_channels makes the store unverifiable, never channel_count_mismatch", async () => { + const id = "on800040"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + // n_channels missing entirely -- malformed, must not be treated as 0. + groups: [{ modality: "eeg", duration_s: 120, rate: 250 }], + }, + ], + }); + // channels.tsv genuinely has rows, which WOULD trivially "mismatch" + // against a naive total-of-0 -- the invalid-metadata guard must fire + // before that comparison is ever made. + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.unverifiable).toBe(1); + expect(result.failed).toBe(0); + expect(result.results[0].examples).toEqual([]); + expect(result.results[0].checked).toBe(0); + expect(warnCalls.some((c) => String(c[0]).includes("store_metadata_invalid"))).toBe(true); + }); +}); + +describe("runZarrFidelitySweep: modality normalization and once-per-dataset unknown-modality logging (item 8)", () => { + test("a modality with surrounding whitespace and mixed case still matches its rate cap", async () => { + const id = "on800050"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: " Eeg ", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + expect(result.results[0].checked_rate).toBe(1); + }); + + test("an unknown modality logs once per dataset and skips only the rate check", async () => { + const id = "on800051"; + seedDataset(id, id, { zarr_store_count: 2 }); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 2, + stores: [ + { + path: "sub-01/eyetrack/sub-01_task-x_eyetrack.set", + zarr: "z1", + groups: [{ modality: "eyetrack", n_channels: 4, duration_s: 60 }], + }, + { + path: "sub-02/eyetrack/sub-02_task-x_eyetrack.set", + zarr: "z2", + groups: [{ modality: "eyetrack", n_channels: 4, duration_s: 60 }], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eyetrack/sub-01_task-x_channels.tsv`, + channelsTsv(4), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/eyetrack/sub-01_task-x_eyetrack.json`, + eegJson(60, 60), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-02/eyetrack/sub-02_task-x_channels.tsv`, + channelsTsv(4), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-02/eyetrack/sub-02_task-x_eyetrack.json`, + eegJson(60, 60), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + // Duration checked fine for both stores; rate is never checked (unknown cap). + expect(result.results[0].checked_duration).toBe(2); + expect(result.results[0].checked_rate).toBe(0); + const unknownModalityWarnings = warnCalls.filter((c) => + String(c[0]).includes('unknown modality "eyetrack"'), + ); + expect(unknownModalityWarnings.length).toBe(1); // once per dataset, not once per store + }); +}); + +describe("runZarrFidelitySweep: BIDS inheritance end-to-end (item 10)", () => { + function baseIndex(path: string) { + return { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path, + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }; + } + + test("sidecar only at the dataset root -> verified", async () => { + const id = "on800060"; + seedDataset(id, id); + indexFixtures.set(id, baseIndex("sub-01/ses-01/eeg/sub-01_ses-01_task-rest_eeg.set")); + sidecarFixtures.set(`${id}/${COMMIT_A}/channels.tsv`, channelsTsv(19)); + sidecarFixtures.set(`${id}/${COMMIT_A}/eeg.json`, eegJson(250, 120)); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + expect(result.results[0].checked).toBe(1); + }); + + test("sidecar only at the session level -> verified", async () => { + const id = "on800061"; + seedDataset(id, id); + indexFixtures.set(id, baseIndex("sub-01/ses-01/eeg/sub-01_ses-01_task-rest_eeg.set")); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ses-01/sub-01_ses-01_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ses-01/sub-01_ses-01_eeg.json`, + eegJson(250, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + }); + + test("sidecar only at the subject level (no session entity) -> verified", async () => { + const id = "on800062"; + seedDataset(id, id); + indexFixtures.set(id, baseIndex("sub-01/eeg/sub-01_task-rest_eeg.set")); // no session + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/sub-01_channels.tsv`, channelsTsv(19)); + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/sub-01_eeg.json`, eegJson(250, 120)); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + }); +}); + +describe("runZarrFidelitySweep: partial reachability (item 12)", () => { + test("2 of 3 sampled stores checked, 1 absent everywhere -> verdict from the 2 checked, unchecked=1", async () => { + const id = "on800070"; + seedDataset(id, id, { zarr_store_count: 3 }); + const stores = ["a", "b", "c"].map((s) => ({ + path: `sub-${s}/eeg/sub-${s}_task-x_eeg.set`, + zarr: `${s}.zarr`, + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + })); + indexFixtures.set(id, { source_commit: COMMIT_A, store_count: 3, stores }); + for (const s of ["a", "b"]) { sidecarFixtures.set( - `${id}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + `${id}/${COMMIT_A}/sub-${s}/eeg/sub-${s}_task-x_channels.tsv`, + channelsTsv(19), + ); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-${s}/eeg/sub-${s}_task-x_eeg.json`, eegJson(250, 120), ); } - const result = await runZarrFidelitySweep(env(), { ...runOpts(), limit: 1 }); - expect(result.processed).toBe(1); - expect(result.remaining).toBe(2); + // sub-c: nothing registered anywhere -- clean absence, not an error. + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + expect(result.results[0].sampled).toBe(3); + expect(result.results[0].checked).toBe(2); + expect(result.results[0].unchecked).toBe(1); + expect(row(id).zarr_verify_unchecked).toBe(1); + }); +}); + +describe("runZarrFidelitySweep: a store with two groups of different modalities (item 13)", () => { + test("both modalities' sidecars are fetched separately; channel totals are summed", async () => { + const id = "on800080"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/ieeg/sub-01_task-x_ieeg.set", + zarr: "z", + groups: [ + { modality: "eeg", n_channels: 10, duration_s: 120, rate: 250 }, + { modality: "ieeg", n_channels: 8, duration_s: 120, rate: 1000 }, + ], + }, + ], + }); + // 18 total channels (10 + 8) declared in channels.tsv. + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_channels.tsv`, + channelsTsv(18), + ); + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_eeg.json`, eegJson(250, 120)); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_ieeg.json`, + eegJson(1000, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.verified).toBe(1); + expect(result.results[0].checked_channels).toBe(1); + expect(result.results[0].checked_duration).toBe(1); + expect(result.results[0].checked_rate).toBe(1); + expect(result.results[0].examples).toEqual([]); + }); + + test("a mismatch in EITHER modality's rate is still caught after two separate fetches", async () => { + const id = "on800081"; + seedDataset(id, id); + indexFixtures.set(id, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/ieeg/sub-01_task-x_ieeg.set", + zarr: "z", + groups: [ + { modality: "eeg", n_channels: 10, duration_s: 120, rate: 250 }, + { modality: "ieeg", n_channels: 8, duration_s: 120, rate: 5000 }, // wrong: cap is 1000 + ], + }, + ], + }); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_channels.tsv`, + channelsTsv(18), + ); + sidecarFixtures.set(`${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_eeg.json`, eegJson(250, 120)); + sidecarFixtures.set( + `${id}/${COMMIT_A}/sub-01/ieeg/sub-01_task-x_ieeg.json`, + eegJson(1000, 120), + ); + + const result = await runZarrFidelitySweep(env(), runOpts()); + expect(result.failed).toBe(1); + expect(result.results[0].examples).toEqual([ + { path: "sub-01/ieeg/sub-01_task-x_ieeg.set", code: "rate_mismatch" }, + ]); }); }); From ca0f04e6dd265d7f6bc25953ca25eaa32a838f94 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 12:12:59 -0700 Subject: [PATCH 11/15] fix: 502 when every sweep candidate errors --- .../src/routes/admin/zarr-fidelity-sweep.ts | 22 +++- .../test/zarr-fidelity-sweep-route.test.ts | 118 +++++++++++++++++- 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/backend/src/routes/admin/zarr-fidelity-sweep.ts b/backend/src/routes/admin/zarr-fidelity-sweep.ts index 110f8fb0f..3614e5474 100644 --- a/backend/src/routes/admin/zarr-fidelity-sweep.ts +++ b/backend/src/routes/admin/zarr-fidelity-sweep.ts @@ -40,6 +40,15 @@ export function registerZarrFidelitySweepRoutes( * 500 on a candidate-query error (e.g. migration 0073 not applied); * per-dataset failures are collected in the response body's `errors` * instead of throwing (see `runZarrFidelitySweep`'s doc comment). + * + * PR #1203 review, item 6: a candidate QUERY failure (thrown) is a 500 as + * before, but a call that reached D1 fine and processed at least one + * candidate, only to have EVERY one of them error out (zero real + * verdicts), answers 502 with `ok: false` -- a 200 there would read as + * "the sweep ran cleanly and verified nothing", which is not what + * happened. A partial result (some verdicts, some errors) is still a + * successful sweep and stays 200 `ok: true`; so does an empty candidate + * set (`processed: 0`) -- there was nothing to fail. */ admin.post("/datasets/zarr-fidelity-sweep", async (c) => { const limitRaw = Number.parseInt(c.req.query("limit") || "", 10); @@ -47,7 +56,18 @@ export function registerZarrFidelitySweepRoutes( try { const result = await sweep(c.env, { limit }); - return c.json(result); + const totalFailure = result.processed > 0 && result.errors.length === result.processed; + if (totalFailure) { + return c.json( + { + ...result, + ok: false, + error: `All ${result.processed} processed candidate(s) errored; see errors[]`, + }, + 502, + ); + } + return c.json({ ...result, ok: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error("[zarr-fidelity-sweep] candidate query failed:", msg); diff --git a/backend/test/zarr-fidelity-sweep-route.test.ts b/backend/test/zarr-fidelity-sweep-route.test.ts index 2e8084587..cea08af70 100644 --- a/backend/test/zarr-fidelity-sweep-route.test.ts +++ b/backend/test/zarr-fidelity-sweep-route.test.ts @@ -226,8 +226,21 @@ describe("POST /admin/datasets/zarr-fidelity-sweep: success against real S3/raw- }; expect(body.processed).toBe(1); expect(body.verified).toBe(1); + expect((body as unknown as { ok: boolean }).ok).toBe(true); expect(body.results).toEqual([ - { dataset_id: id, verdict: "verified", sampled: 1, checked: 1, examples: [] }, + { + dataset_id: id, + verdict: "verified", + sampled: 1, + checked: 1, + checked_channels: 1, + checked_duration: 1, + checked_rate: 1, + unchecked: 0, + examples: [], + mismatch_count: 0, + examples_truncated: false, + }, ]); const row = db @@ -238,7 +251,7 @@ describe("POST /admin/datasets/zarr-fidelity-sweep: success against real S3/raw- expect(row.s).toBe("verified"); }); - test("an empty candidate set still returns a well-formed 200", async () => { + test("an empty candidate set still returns a well-formed 200 with ok:true", async () => { const app = newApp({ sweep: (env, opts) => runZarrFidelitySweep(env, { @@ -253,9 +266,108 @@ describe("POST /admin/datasets/zarr-fidelity-sweep: success against real S3/raw- env(db), ); expect(res.status).toBe(200); - const body = (await res.json()) as { processed: number; results: unknown[]; remaining: number }; + const body = (await res.json()) as { + processed: number; + results: unknown[]; + remaining: number; + ok: boolean; + }; expect(body.processed).toBe(0); expect(body.results).toEqual([]); expect(body.remaining).toBe(0); + expect(body.ok).toBe(true); + }); + + // PR #1203 review, item 6: a call that reached D1 and processed at least + // one candidate, but every one of them errored (zero real verdicts), + // must not read as a clean 200 -- that would look like "ran fine, + // verified nothing", not "could not verify anything this run". + test("processed > 0 with every candidate erroring answers 502 with ok:false", async () => { + const id = "on810002"; + db.query( + `INSERT INTO datasets + (dataset_id, owner_user_id, name, visibility, status, is_sandbox, + zarr_status, zarr_store_count, zarr_source_commit, github_repo) + VALUES (?, -1, ?, 'public', 'active', 0, 'ready', 1, ?, ?)`, + ).run(id, id, COMMIT_A, `${ORG_NAME}/${id}`); + // No indexFixtures entry -- index.json 404s, which is a real per-dataset + // error (not stamped), so this is the "processed=1, errors=1" shape. + + const app = newApp({ + sweep: (env, opts) => + runZarrFidelitySweep(env, { + ...opts, + s3Options: { endpointUrl: `http://127.0.0.1:${server.port}` }, + githubRawBase: `http://127.0.0.1:${server.port}`, + }), + }); + const res = await app.request( + "/datasets/zarr-fidelity-sweep", + { method: "POST", headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, + env(db), + ); + expect(res.status).toBe(502); + const body = (await res.json()) as { + processed: number; + errors: { dataset_id: string; error: string }[]; + ok: boolean; + error: string; + }; + expect(body.processed).toBe(1); + expect(body.errors.length).toBe(1); + expect(body.ok).toBe(false); + expect(typeof body.error).toBe("string"); + }); + + test("a MIX of verdicts and errors still answers 200 with ok:true (a partial sweep is not a total failure)", async () => { + const verifiedId = "on810003"; + const erroredId = "on810004"; + for (const id of [verifiedId, erroredId]) { + db.query( + `INSERT INTO datasets + (dataset_id, owner_user_id, name, visibility, status, is_sandbox, + zarr_status, zarr_store_count, zarr_source_commit, github_repo) + VALUES (?, -1, ?, 'public', 'active', 0, 'ready', 1, ?, ?)`, + ).run(id, id, COMMIT_A, `${ORG_NAME}/${id}`); + } + indexFixtures.set(verifiedId, { + source_commit: COMMIT_A, + store_count: 1, + stores: [ + { + path: "sub-01/eeg/sub-01_task-rest_eeg.set", + zarr: "z", + groups: [{ modality: "eeg", n_channels: 19, duration_s: 120, rate: 250 }], + }, + ], + }); + sidecarFixtures.set( + `${verifiedId}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_channels.tsv`, + `name\ttype\n${Array.from({ length: 19 }, (_, i) => `Ch${i}\tEEG`).join("\n")}`, + ); + sidecarFixtures.set( + `${verifiedId}/${COMMIT_A}/sub-01/eeg/sub-01_task-rest_eeg.json`, + JSON.stringify({ SamplingFrequency: 250, RecordingDuration: 120 }), + ); + // erroredId has no index fixture -- it errors. + + const app = newApp({ + sweep: (env, opts) => + runZarrFidelitySweep(env, { + ...opts, + s3Options: { endpointUrl: `http://127.0.0.1:${server.port}` }, + githubRawBase: `http://127.0.0.1:${server.port}`, + }), + }); + const res = await app.request( + "/datasets/zarr-fidelity-sweep", + { method: "POST", headers: { Authorization: `Bearer ${ADMIN_KEY}` } }, + env(db), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { processed: number; verified: number; ok: boolean }; + expect(body.processed).toBe(2); + expect(body.verified).toBe(1); + expect(body.ok).toBe(true); }); }); From e22779e492959d705e914ba6a2dc8b78bf47ae85 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 12:13:07 -0700 Subject: [PATCH 12/15] feat: surface new sweep fields in the CLI --- src/commands/admin.ts | 9 +++++++-- src/lib/api/admin.ts | 19 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/commands/admin.ts b/src/commands/admin.ts index 1cacdf055..0fe4acc98 100644 --- a/src/commands/admin.ts +++ b/src/commands/admin.ts @@ -5236,7 +5236,7 @@ const zarrFidelitySweepCommand = new Command("zarr-fidelity-sweep").description( ); zarrFidelitySweepCommand - .option("--limit ", "Datasets per batch (server clamps to [1,100])", "25") + .option("--limit ", "Datasets per batch (server clamps to [1,25])", "25") .option("--json", "Output raw JSON instead of the human summary") .action(async (options: { limit?: string; json?: boolean }) => { if (!requireAuth()) return; @@ -5261,7 +5261,7 @@ zarrFidelitySweepCommand console.log(); console.log( chalk.cyan( - `processed=${res.processed} verified=${res.verified} failed=${res.failed} unverifiable=${res.unverifiable} errors=${res.errors.length} remaining=${res.remaining ?? "unknown"}`, + `processed=${res.processed} verified=${res.verified} failed=${res.failed} unverifiable=${res.unverifiable} errors=${res.errors.length} remaining=${res.remaining ?? "unknown"}${res.budget_exhausted ? " budget_exhausted=true" : ""}`, ), ); for (const r of res.results) { @@ -5277,6 +5277,11 @@ zarrFidelitySweepCommand for (const ex of r.examples) { console.log(` ${chalk.red(ex.code)}: ${ex.path}`); } + if (r.examples_truncated) { + console.log( + ` ${chalk.dim(`(${r.mismatch_count} mismatches total; showing first ${r.examples.length})`)}`, + ); + } } for (const e of res.errors) { console.log(` ${chalk.red("error")} ${e.dataset_id}: ${e.error}`); diff --git a/src/lib/api/admin.ts b/src/lib/api/admin.ts index be21687b6..0d7f06132 100644 --- a/src/lib/api/admin.ts +++ b/src/lib/api/admin.ts @@ -1324,11 +1324,20 @@ export interface ZarrFidelityDatasetResult { verdict: "verified" | "failed" | "unverifiable"; sampled: number; checked: number; + checked_channels: number; + checked_duration: number; + checked_rate: number; + unchecked: number; examples: ZarrFidelityMismatchExample[]; + mismatch_count: number; + examples_truncated: boolean; } /** One batch of the zarr fidelity sweep - * (`POST /admin/datasets/zarr-fidelity-sweep`). */ + * (`POST /admin/datasets/zarr-fidelity-sweep`). `ok` is false (with a 502 + * status) only when every processed candidate errored -- a partial mix of + * verdicts and errors, or an empty candidate set, is still `ok: true` + * (#1203 review, item 6). */ export interface ZarrFidelitySweepBatchResponse { processed: number; verified: number; @@ -1338,10 +1347,14 @@ export interface ZarrFidelitySweepBatchResponse { errors: { dataset_id: string; error: string }[]; /** Candidates still unverified after this run; null if the count query failed. */ remaining: number | null; + /** True when the sweep-wide fetch budget ran out before every requested + * candidate could be attempted; datasets past this point were never + * touched at all (#1203 review, item 3). */ + budget_exhausted: boolean; + ok: boolean; } -/** Run one bounded zarr fidelity sweep batch (server default 25, clamped to - * [1,100]). */ +/** Run one bounded zarr fidelity sweep batch (server default and max: 25). */ export async function zarrFidelitySweep(options?: { limit?: number; }): Promise { From 9c2373aec216b485285bad4adc0e7f178990d672 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 12:13:16 -0700 Subject: [PATCH 13/15] test: has_zarr_verified filters GET /datasets/search --- backend/test/catalog-has-zarr.test.ts | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/backend/test/catalog-has-zarr.test.ts b/backend/test/catalog-has-zarr.test.ts index eafe701c7..93d46da00 100644 --- a/backend/test/catalog-has-zarr.test.ts +++ b/backend/test/catalog-has-zarr.test.ts @@ -325,6 +325,47 @@ describe("has_zarr filter: GET /datasets/search", () => { }); }); +// Issue #1068, epic #1181 phase 8, PR #1203 review item 14: has_zarr_verified +// must filter GET /datasets/search server-side, not just the list endpoint. +describe("has_zarr_verified filter: GET /datasets/search", () => { + let db: Database; + let app: App; + + beforeEach(() => { + db = freshDb(); + app = newApp(); + insertDataset(db, "nm411001", { + name: "Search Verified Fixture Alpha", + zarr_status: "ready", + zarr_store_count: 5, + sweep_stamps: JSON.stringify({ zarr_verify_status: "verified" }), + }); + insertDataset(db, "nm411002", { + name: "Search Verified Fixture Beta", + zarr_status: "ready", + zarr_store_count: 5, + sweep_stamps: JSON.stringify({ zarr_verify_status: "failed" }), + }); + }); + + const Q = `q=${encodeURIComponent("Search Verified Fixture")}`; + + test("has_zarr_verified=1 excludes a has_zarr row that failed verification", async () => { + const ids = await searchIds(app, db, `${Q}&has_zarr_verified=1`); + expect(ids).toEqual(["nm411001"]); + }); + + test("no has_zarr_verified filter returns both datasets from search", async () => { + const ids = await searchIds(app, db, Q); + expect(ids.sort()).toEqual(["nm411001", "nm411002"]); + }); + + test("has_zarr_verified also applies on the exact-id lookup tier", async () => { + const idsWithFilter = await searchIds(app, db, "q=nm411002&has_zarr_verified=1"); + expect(idsWithFilter).not.toContain("nm411002"); + }); +}); + describe("the zarr fields are present on GET /datasets rows", () => { let db: Database; let app: App; From b7075440c832402b817d33033f6ff086e53efdef Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 12:13:25 -0700 Subject: [PATCH 14/15] docs: update fidelity sweep paragraph for review fixes --- AGENTS.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 106318aa4..f6344ce00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -392,11 +392,24 @@ Environments and pre-release checks: [`.context/release-safety-playbook.md`](.co from the dataset's own git-tracked BIDS metadata via the public, credential-free `raw.githubusercontent.com` content host (never the GitHub API/App/PAT, which is what makes it safe on the non-prod cron too - -- `DEV_CRON_ALLOWLIST` in index.ts), and stamps `zarr_verify_status` + -- `DEV_CRON_ALLOWLIST` in index.ts) -- candidates are restricted to + `status='active' AND visibility='public'` for exactly that reason: a + private repo can't be read anonymously, so it would only ever produce + `unverifiable` noise. It FAILS OPEN ON THE ROW, never on the verdict: a + transient infra error (a non-2xx that isn't 404, a network throw, or + either fetch budget running out mid-dataset -- a sweep-wide 600 and a + per-dataset 90, both index plus sidecar fetches) aborts that one + dataset's verification for the run -- nothing is stamped, it lands in the + run's `errors`, and the row stays a candidate; only a clean 404 at every + nearest-first candidate path is real absence, and only a value that + actually parsed counts as checked. Stamps `zarr_verify_status` (`verified` / `failed` / `unverifiable`) plus `zarr_verified_at` into `sweep_stamps` (ADR 0034/0035 -- no new column); a re-conversion (a - changed `zarr_source_commit`) re-arms verification. A fresh conversion - shows `zarr_verify_status: null` until the daily sweep (plus + changed `zarr_source_commit`) OR a stamped commit that is null (a fixed + fossilisation bug -- the write side now stamps `''`, never JSON `null`, + but the candidate predicate also re-arms on a null stamp so an + already-fossilised row un-sticks too) re-arms verification. A fresh + conversion shows `zarr_verify_status: null` until the daily sweep (plus `POST /admin/datasets/zarr-fidelity-sweep`, `nemar admin zarr-fidelity-sweep`) reaches it; the viewer keeps reading index.json regardless (ADR 0005 -- verification is reported, never a precondition From 54cc805c136204b4d3c5ff8d4a87ac0a2e81eca4 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Wed, 2 Sep 2026 13:03:39 -0700 Subject: [PATCH 15/15] fix: scope budget_exhausted to the sweep-wide budget --- backend/src/services/zarr-fidelity-sweep.ts | 43 +++++++++++++------ backend/test/zarr-fidelity-sweep.test.ts | 9 +++- src/commands/admin.ts | 16 ++++--- test/zarr-fidelity-sweep-cli.test.ts | 47 +++++++++++++++++++++ 4 files changed, 95 insertions(+), 20 deletions(-) diff --git a/backend/src/services/zarr-fidelity-sweep.ts b/backend/src/services/zarr-fidelity-sweep.ts index e64c46bb9..93693caf6 100644 --- a/backend/src/services/zarr-fidelity-sweep.ts +++ b/backend/src/services/zarr-fidelity-sweep.ts @@ -54,18 +54,23 @@ * in order and cached per dataset run so a repeated session-/subject-/ * root-level default costs one fetch each. * - * TWO FETCH BUDGETS. `ZARR_FIDELITY_SWEEP_WIDE_BUDGET` bounds the WHOLE - * invocation (every index fetch plus every sidecar fetch, across every - * candidate dataset); once it is spent the batch stops and remaining - * candidates are left completely untouched (never fetched, never errored -- + * TWO FETCH BUDGETS, TWO DISTINCT REASONS (PR #1203 review round 2). + * `ZARR_FIDELITY_SWEEP_WIDE_BUDGET` bounds the WHOLE invocation (every index + * fetch plus every sidecar fetch, across every candidate dataset); once it + * is spent the batch stops and remaining candidates are left completely + * untouched (never fetched, never errored -- the top-level * `budget_exhausted` in the result says why `processed` came in low). * `ZARR_FIDELITY_MAX_SIDECAR_FETCHES_PER_DATASET` additionally bounds ONE * dataset's sidecar fetches so a single pathological dataset (nothing but * misses) cannot spend the whole sweep-wide budget by itself. Either budget - * running out mid-dataset surfaces as the SAME "error" outcome - * `resolveSidecar` already uses for a real infra failure (reason - * `budget_exhausted`), which is what aborts that one dataset per the - * fail-open rule above -- it is never turned into "absent". + * running out mid-dataset surfaces as an "error" outcome from + * `resolveSidecar` (the same class of abort a real infra failure uses), + * but with a DIFFERENT reason string depending on which budget it was: + * `sweep_budget_exhausted` sets the top-level `budget_exhausted` flag + * (later candidates really are about to go untouched); a lone dataset's own + * `dataset_budget_exhausted` does not -- it is that ONE dataset's error, + * every other candidate in the batch is still processed normally. Neither + * is ever turned into "absent". * * Modelled on recording-stats-sweep.ts's shape: candidate SQL, a bounded * per-invocation cap, three-way per-dataset error handling (throw/d1-error @@ -486,8 +491,17 @@ async function resolveSidecar( if (cached.kind === "content") return { kind: "content", path: candidate, body: cached.body }; continue; // cached "absent" -- try the next candidate } - if (ctx.sweepBudget.remaining <= 0 || ctx.perDatasetBudget.remaining <= 0) { - return { kind: "error", reason: "budget_exhausted" }; + if (ctx.sweepBudget.remaining <= 0) { + // Checked first: if the sweep-wide budget is what's gone, every + // remaining candidate dataset is about to be left untouched too, so + // the top-level `budget_exhausted` flag (set from this exact reason + // string in `runZarrFidelitySweep`) must reflect that -- distinct + // from a single dataset merely hitting its OWN 90-fetch cap while the + // sweep-wide budget still has plenty left (PR #1203 review round 2). + return { kind: "error", reason: "sweep_budget_exhausted" }; + } + if (ctx.perDatasetBudget.remaining <= 0) { + return { kind: "error", reason: "dataset_budget_exhausted" }; } ctx.sweepBudget.remaining--; ctx.perDatasetBudget.remaining--; @@ -751,7 +765,7 @@ async function verifyDataset( }; if (sweepBudget.remaining <= 0) { - return { status: null, ...empty, error: "budget_exhausted" }; + return { status: null, ...empty, error: "sweep_budget_exhausted" }; } sweepBudget.remaining--; @@ -947,7 +961,12 @@ export async function runZarrFidelitySweep( if (outcome.status === null) { errors.push({ dataset_id, error: outcome.error ?? "zarr-fidelity-sweep: unknown error" }); - if (outcome.error === "budget_exhausted") budgetExhausted = true; + // Only the SWEEP-WIDE budget running out means later candidates are + // about to go untouched -- a lone dataset exhausting its own 90-fetch + // cap ("dataset_budget_exhausted") is just that one dataset's error; + // every other candidate is still processed normally (PR #1203 review + // round 2). + if (outcome.error === "sweep_budget_exhausted") budgetExhausted = true; continue; } diff --git a/backend/test/zarr-fidelity-sweep.test.ts b/backend/test/zarr-fidelity-sweep.test.ts index 97c868e2d..e3e9d94f0 100644 --- a/backend/test/zarr-fidelity-sweep.test.ts +++ b/backend/test/zarr-fidelity-sweep.test.ts @@ -994,7 +994,7 @@ describe("runZarrFidelitySweep: the two fetch budgets (item 3)", () => { const result = await runZarrFidelitySweep(env(), { ...runOpts(), sweepWideBudget: 1 }); expect(result.budget_exhausted).toBe(true); expect(result.errors.length).toBe(1); - expect(result.errors[0].error).toBe("budget_exhausted"); + expect(result.errors[0].error).toBe("sweep_budget_exhausted"); expect(result.results).toEqual([]); expect(result.processed).toBe(1); // All three rows are still candidates: the errored one was never @@ -1025,8 +1025,13 @@ describe("runZarrFidelitySweep: the two fetch budgets (item 3)", () => { expect(result.results).toEqual([]); expect(result.errors.length).toBe(1); expect(result.errors[0].dataset_id).toBe(id); - expect(result.errors[0].error).toBe("budget_exhausted"); + expect(result.errors[0].error).toBe("dataset_budget_exhausted"); expect(row(id).zarr_verify_status).toBeNull(); + // A single dataset hitting its OWN 90-fetch cap is that dataset's + // error alone -- the sweep-wide budget (600) still has plenty left, so + // the top-level flag must NOT read as "the batch was cut short" + // (PR #1203 review round 2). + expect(result.budget_exhausted).toBe(false); }); }); diff --git a/src/commands/admin.ts b/src/commands/admin.ts index 0fe4acc98..84671a9f4 100644 --- a/src/commands/admin.ts +++ b/src/commands/admin.ts @@ -5287,12 +5287,16 @@ zarrFidelitySweepCommand console.log(` ${chalk.red("error")} ${e.dataset_id}: ${e.error}`); } } - // Non-zero exit on any per-dataset error OR an indeterminate (null) - // remaining, so a caller never reads a partial/uncertain sweep as - // success -- mirrors data-integrity-sweep/hed-sweep's convention. A - // 'failed' verdict is NOT itself a non-zero-exit condition: the sweep - // ran successfully and reported the truth, which is the whole point. - if (res.errors.length > 0 || res.remaining === null) process.exit(1); + // Non-zero exit on any per-dataset error, an indeterminate (null) + // remaining, OR a sweep-wide budget cutoff (this batch is partial -- + // candidates past the cutoff were never even attempted), so a caller + // never reads a partial/uncertain sweep as success -- mirrors + // data-integrity-sweep/hed-sweep's convention. A 'failed' verdict is + // NOT itself a non-zero-exit condition: the sweep ran successfully and + // reported the truth, which is the whole point. + if (res.errors.length > 0 || res.remaining === null || res.budget_exhausted) { + process.exit(1); + } }); adminCommand.addCommand(zarrFidelitySweepCommand); diff --git a/test/zarr-fidelity-sweep-cli.test.ts b/test/zarr-fidelity-sweep-cli.test.ts index 660f081df..2e710bca6 100644 --- a/test/zarr-fidelity-sweep-cli.test.ts +++ b/test/zarr-fidelity-sweep-cli.test.ts @@ -263,4 +263,51 @@ describe("nemar admin zarr-fidelity-sweep: output and exit code", () => { server.stop(); } }); + + // PR #1203 review round 2: budget_exhausted=true with zero per-dataset + // errors used to exit 0 -- a sweep-wide budget cutoff means candidates + // past it were never even attempted, which is a partial run, not success. + test("budget_exhausted=true with no errors still exits non-zero (a partial sweep)", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer({ + processed: 1, + verified: 1, + failed: 0, + unverifiable: 0, + results: [ + { dataset_id: "on800001", verdict: "verified", sampled: 1, checked: 1, examples: [] }, + ], + errors: [], + remaining: 5, + budget_exhausted: true, + }); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(1); + } finally { + server.stop(); + } + }); + + test("budget_exhausted=false with no errors and a real remaining count exits 0", async () => { + seedAuthenticatedConfig(); + const server = startCaptureServer({ + processed: 1, + verified: 1, + failed: 0, + unverifiable: 0, + results: [ + { dataset_id: "on800001", verdict: "verified", sampled: 1, checked: 1, examples: [] }, + ], + errors: [], + remaining: 0, + budget_exhausted: false, + }); + try { + const result = await runCli(["admin", "zarr-fidelity-sweep", "--json"], server.url); + expect(result.exitCode).toBe(0); + } finally { + server.stop(); + } + }); });