From 2be27c1a96d63643c85a0a582678e8fbd7c78ee9 Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Sat, 22 Aug 2026 11:24:09 -0700 Subject: [PATCH 1/2] SSS-filter MaxShield MEG, or decline it specifically Implements ADR 0028, accepted 2026-08-20 and never built. on006012 and on006720 publish no Zarr index because MNE refuses raw Internal Active Shielding data, and all 396 recordings fail as file_read_error. Recordings carrying it are now corrected with mne.preprocessing. maxwell_filter using the dataset's own fine-calibration and cross-talk files, and served. Where that pair does not resolve the recording is declined with a new deterministic code, maxshield_uncalibrated, instead of an opaque read error: ADR 0028 rejects uncalibrated filtering because it is a weaker correction a consumer could not distinguish from a good one. The calibration pair needs its own resolver. Every existing sidecar resolver requires the candidate's entities to be a subset of the recording's, and sub-01_acq-calibration_meg.dat carries acq=calibration while the recording has no acq at all, so that rule rejects the very file it should find. acq names these sidecars rather than scoping them, so it is exempt here. Run over the real repository trees the resolver reproduces the ADR's projection exactly: 207 of 223 and 158 of 173, every decline under sub-emptyroom. Filtering is disclosed in the store root attributes and the index entry. ADR 0028 requires this: the result is a processed derivative unlike every other store, and a model training across datasets would otherwise silently mix filtered and unfiltered MEG. proc_history does not survive into the store, so it is written explicitly. Admission cannot detect MaxShield: it projects from annex pointers without downloading, and the BIDS sidecar carries no marker. It uses the one signal available from head_files, a resolvable calibration pair, and the worker does the real detection. MAXSHIELD_MEM_FACTOR is measured, not reasoned: 2.95 GiB peak on a 716 MiB recording, 4.2x, end to end through these functions on the conversion node. Load-bearing, since apply_worker_mem_limit sizes RLIMIT_DATA from the projection before the worker starts. An earlier draft of this work used a 12.2x factor taken from a pipeline production does not run. The filtered copy streams, so conversion adds nothing to the filter's peak. --- ...sss-filtered-before-serving-or-declined.md | 31 +++ scripts/zarr/generate_zarr.py | 255 +++++++++++++++++- scripts/zarr/test_generate_zarr.py | 125 +++++++++ 3 files changed, 405 insertions(+), 6 deletions(-) diff --git a/.context/decisions/0028-maxshield-meg-is-sss-filtered-before-serving-or-declined.md b/.context/decisions/0028-maxshield-meg-is-sss-filtered-before-serving-or-declined.md index 05841aa16..d372ae1c5 100644 --- a/.context/decisions/0028-maxshield-meg-is-sss-filtered-before-serving-or-declined.md +++ b/.context/decisions/0028-maxshield-meg-is-sss-filtered-before-serving-or-declined.md @@ -132,6 +132,37 @@ It came from the `zarr_errors` column and included the 43 calibration and cross-talk files themselves, which were being attempted as recordings before ADR 0027 excluded them. +## Implementation receipt (2026-08-22) + +This decision is **implemented** as of #1126; the status line stays `accepted` because +that field records the decision's standing, and the index test constrains it to +`proposed | accepted | superseded by ADR-NNNN`. The decision's own receipts verified that the correction is real; +what they did not carry is what it costs, which admission control needs. + +Measured on the conversion node against `on006720`'s `sub-155`, through the shipped +functions rather than a stand-in: + +- Detection (`is_maxshield_fif`, FIF header only, no preload): 1.5 s, 0.14 GiB peak. + Cheap enough to run on every FIF. +- `apply_sss` on a 716 MiB recording: 57.5 s, peak 2.95 GiB, i.e. **4.2x its on-disk size**. +- The filtered copy then takes the STREAMING conversion path, so `convert_recording` adds + nothing to that peak: end-to-end peak stayed 2.95 GiB. +- `info["maxshield"]` is clear on the output, as this decision's receipts predicted. + +Hence `MAXSHIELD_MEM_FACTOR = 6`. The pre-existing streaming floor of 4 GiB would have +cleared the largest recording in these datasets by under 5%, which is coincidence rather +than headroom: the floor was sized for conversion, and the filter phase is a separate cost +that peaks before conversion starts. Since `apply_worker_mem_limit` sizes `RLIMIT_DATA` +from the projection, an under-projection would not merely over-schedule the node, it would +kill the filter mid-run. + +Resolution coverage was re-derived from the current repository trees and reproduces this +decision's projection exactly: `on006720` 207 of 223, `on006012` 158 of 173, every declined +recording under `sub-emptyroom`. + +Conversion cost is real: roughly 5 minutes per recording (57 s filtering, ~260 s streaming +conversion), so the 365-recording backfill is on the order of 30 hours of node time. + Related: ADR 0027 (raw-only discovery, which excludes the calibration files as recordings). ## Note on eventual placement diff --git a/scripts/zarr/generate_zarr.py b/scripts/zarr/generate_zarr.py index 15c38feb8..8993598e7 100644 --- a/scripts/zarr/generate_zarr.py +++ b/scripts/zarr/generate_zarr.py @@ -457,6 +457,21 @@ def apply_worker_mem_limit( ".vhdr": float(os.environ.get("ZARR_INMEM_MEM_FACTOR_VHDR", "12")), } +# Signal-Space Separation (ADR 0028) runs BEFORE conversion and costs its own peak: +# it needs a fully preloaded float64 `Raw`, which is the anonymous memory RLIMIT_DATA +# counts. Measured end to end on the conversion node against on006720's `sub-155`, +# through these same functions: a 716 MiB recording peaked at 2.95 GiB, i.e. 4.2x its +# on-disk size, and the filtered copy then STREAMS, so conversion adds nothing to that +# peak. 6x is the measurement with headroom. +# +# This is load-bearing rather than an efficiency nicety. `apply_worker_mem_limit` +# sizes RLIMIT_DATA from the projection BEFORE the worker starts, so an +# under-projection does not merely over-schedule the node, it kills the filter +# mid-run. The streaming floor alone (4 GiB) happens to clear this dataset's largest +# recording by under 5%, which is coincidence rather than headroom -- and it is the +# filter phase, not the conversion, that the floor was never sized for. +MAXSHIELD_MEM_FACTOR = float(os.environ.get("ZARR_MAXSHIELD_MEM_FACTOR", "6")) + def note_measurement(result: dict, projections: dict, measured: dict) -> str | None: """Fold one `convert_one` result into ``measured``; return a warning to print @@ -564,8 +579,29 @@ class RecordingMemoryExceeded(Exception): code = "recording_memory_exceeded" +class MaxShieldUncalibrated(Exception): + """A MEG recording carrying raw Internal Active Shielding (MaxShield) data for + which the site-specific calibration pair does not resolve. + + MEGIN's position, which MNE enforces by refusing to read these files at all, is + that raw Internal Active Shielding data is not fit for analysis until the + shielding's effect has been modelled out. ADR 0028 decides we correct it with + Signal-Space Separation and serve the result -- but ONLY with the recording's own + fine-calibration and cross-talk files, because uncalibrated Signal-Space + Separation is a weaker correction whose quality varies by site and hardware, and + serving it under the same label would make the two indistinguishable. + + So this is the honest decline: a permanent property of what the dataset ships, + NOT in `RETRYABLE_CODES`. It exists to replace the opaque `file_read_error` that + gave a user no way to tell an unreadable file from a policy decision.""" + + code = "maxshield_uncalibrated" + + # Coded failures that are nevertheless NOT a permanent property of the data, so a # run consisting entirely of them must stay retryable. See `deterministic` in main(). +# `maxshield_uncalibrated` is deliberately absent: the calibration pair is either +# shipped with the dataset or it is not, and retrying cannot change that. RETRYABLE_CODES = frozenset({RecordingMemoryExceeded.code}) @@ -650,14 +686,28 @@ def streaming_peak_bytes(size_bytes: int, n_channels: int | None) -> int: def projected_peak_bytes( - primary_local: str, size_bytes: int, n_channels: int | None = None + primary_local: str, + size_bytes: int, + n_channels: int | None = None, + maxshield: bool = False, ) -> int: """Estimated peak RAM to convert this recording: the streaming path's channel-aware bound, or the float64 blow-up for the in-memory path. Drives the - skip guard (#909).""" - if should_stream(primary_local, size_bytes): - return streaming_peak_bytes(size_bytes, n_channels) - return int(size_bytes * inmem_factor_for(primary_local)) + skip guard (#909). + + `maxshield` adds the Signal-Space Separation phase (ADR 0028), which runs before + conversion and peaks independently of it. The two phases are sequential and the + `Raw` is released between them, so the recording's peak is the LARGER of the two + rather than their sum. + """ + conversion = ( + streaming_peak_bytes(size_bytes, n_channels) + if should_stream(primary_local, size_bytes) + else int(size_bytes * inmem_factor_for(primary_local)) + ) + if not maxshield: + return conversion + return max(conversion, int(size_bytes * MAXSHIELD_MEM_FACTOR)) def usable_ram_bytes(meminfo_path: str = "/proc/meminfo") -> int: @@ -786,6 +836,14 @@ def per_recording_ceiling_bytes() -> int: "The converted viewer copy carried fewer channels than this recording's " "channels.tsv declares, so it was withheld pending a converter fix." ), + # NEMAR-side (not a biosigIO code): ADR 0028. Surfaces MEGIN's own position, + # which is why the file cannot simply be shown, rather than a bare read error. + "maxshield_uncalibrated": ( + "This recording was acquired with internal active shielding, which distorts " + "the signal until it is corrected. Correcting it needs the site's " + "fine-calibration and cross-talk files, which this dataset does not provide " + "for this recording, so no viewer copy is offered." + ), } _GENERIC_REASON = _FALLBACK_REASONS["file_read_error"] @@ -848,6 +906,127 @@ def is_bids_calibration_file(path: str) -> bool: return path.endswith(_BIDS_CALIBRATION_SUFFIXES) +def maxshield_calibration_for( + primary_path: str, head_files: set[str] | frozenset[str] +) -> tuple[str, str] | None: + """The `(fine_calibration, cross_talk)` pair applying to `primary_path`, or None + when either is absent. + + Resolved by BIDS inheritance: the nearest match in the recording's own directory + or any ancestor, most specific winning. Both must resolve -- ADR 0028 declines + rather than run Signal-Space Separation uncalibrated, because an uncalibrated + correction is weaker in a way a consumer could not distinguish from a good one. + + These are exactly the files `is_bids_calibration_file` excludes from DISCOVERY. + That is not a contradiction and the two must not be conflated: they are never + recordings, and they are inputs to converting one. A future change that reads + "excluded from discovery" as "irrelevant to conversion" silently breaks MaxShield + support (ADR 0028's own warning). + + The entity-subset rule the other sidecar resolvers use does NOT work here, which + is why this is its own function. `sub-01_acq-calibration_meg.dat` carries + `acq=calibration`, while the recording `sub-01_task-rest_meg.fif` has no `acq` at + all, so the subset test rejects the very file it should find. `acq` is the entity + that NAMES these sidecars rather than scoping them, so it is exempt here. + """ + rec_dir = os.path.dirname(primary_path) + rec_ents = _bids_entities(filename_stem(primary_path)) + + def _nearest(suffix: str) -> str | None: + # Same two accepted spellings as the other sidecar resolvers: the + # entity-prefixed form beside the recording, or the bare form (no leading + # underscore) at a level that has no entities of its own. + bare = suffix.lstrip("_") + candidates: list[tuple[int, int, str]] = [] + for f in head_files: + if not (f.endswith(suffix) or os.path.basename(f) == bare): + continue + cdir = os.path.dirname(f) + if cdir and rec_dir != cdir and not rec_dir.startswith(cdir + "/"): + continue + cents = _bids_entities(filename_stem(f)) + cents.pop("acq", None) # names the sidecar, does not scope it + if any(rec_ents.get(k) != v for k, v in cents.items()): + continue + candidates.append((cdir.count("/") + (1 if cdir else 0), len(cents), f)) + if not candidates: + return None + candidates.sort() + return candidates[-1][2] + + cal = _nearest("_acq-calibration_meg.dat") + ctc = _nearest("_acq-crosstalk_meg.fif") + return (cal, ctc) if cal and ctc else None + + +def is_maxshield_fif(path: str) -> bool: + """True if `path` is a FIF carrying raw Internal Active Shielding data. + + Reads the FIF header only (`preload=False`), which is cheap even on a large + recording -- measured at 1.6 s and 0.13 GiB of peak RSS against a 716 MiB file on + the conversion node -- so this is affordable as a preflight on every FIF. + + `allow_maxshield="yes"` is the quiet form: it suppresses MNE's warning, which we + do not want in the log for a file we are about to correct properly. The flag we + read back, `info["maxshield"]`, is MNE's own; ADR 0028 established that it flips + True -> False once Signal-Space Separation has run, so the same field serves as + both the detector here and the verification afterwards. + + Anything that is not a readable FIF is not our business: return False and let the + normal conversion path produce its own typed failure. + """ + if lower_ext(path) != ".fif": + return False + try: + import mne # type: ignore[import-not-found] # lazy: runtime-only dep + + info = mne.io.read_raw_fif( + path, allow_maxshield="yes", preload=False, verbose="ERROR" + ).info + return bool(info.get("maxshield")) + except Exception: # noqa: BLE001 - unreadable here means "not the MaxShield path" + return False + + +def apply_sss(raw_path: str, calibration: str, cross_talk: str, out_path: str) -> dict: + """Signal-Space Separation filter a MaxShield recording, writing the corrected + recording to `out_path`. Returns the disclosure recorded in the store and index. + + ADR 0028: raw Internal Active Shielding data is never served. It is corrected + with the recording's own site-specific fine-calibration and cross-talk files, or + it is declined. `mne.preprocessing.maxwell_filter` is an open-source + implementation of the same Signal-Space Separation family as MEGIN's proprietary + MaxFilter, which the conversion host cannot run. + + The result is a PROCESSED DERIVATIVE, unlike every other store this converter + writes, which is why this returns a disclosure rather than filtering silently. + """ + import mne # type: ignore[import-not-found] # lazy: runtime-only dep + + raw = mne.io.read_raw_fif( + raw_path, allow_maxshield="yes", preload=True, verbose="ERROR" + ) + sss = mne.preprocessing.maxwell_filter( + raw, calibration=calibration, cross_talk=cross_talk, verbose="ERROR" + ) + # MNE's own guardrail, the check that refused to load the file, is satisfied by + # the output. If it is not, the correction did not happen and serving the result + # would be exactly the thing ADR 0028 forbids -- so fail rather than publish it. + if sss.info.get("maxshield"): + raise MaxShieldUncalibrated( + "Signal-Space Separation ran but the recording is still flagged as raw " + "Internal Active Shielding data; refusing to serve it" + ) + sss.save(out_path, overwrite=True, verbose="ERROR") + return { + "applied": True, + "method": "maxwell_filter", + "calibration": os.path.basename(calibration), + "cross_talk": os.path.basename(cross_talk), + "mne_version": mne.__version__, + } + + def is_excluded_from_discovery(path: str) -> bool: """True if `path` can never be a servable BIDS raw recording: it sits under an excluded tree, or it is a reserved BIDS calibration filename. @@ -2743,6 +2922,41 @@ def convert_one(primary: str, peak_bytes: int | None = None) -> dict: primary_local, events_local, primary_key = materialize_recording( c["repo"], c["bucket"], c["dataset_id"], primary, c["head_files"], c["head"], work ) + # ADR 0028. Substituted HERE rather than inside convert_recording, which + # derives modality, size and the streaming decision from the path it is given + # (a late rebind would leave all three describing the unfiltered file). The + # filtered copy lands in `work`, which the `finally` below already removes, + # and fix_source_file_attr still rewrites source_file to the BIDS path, so the + # scratch name never reaches the store. + sss_meta = None + if is_maxshield_fif(primary_local): + pair = maxshield_calibration_for(primary, c["head_files"]) + if pair is None: + raise MaxShieldUncalibrated( + "recording carries raw Internal Active Shielding data and this " + "dataset provides no fine-calibration / cross-talk pair for it" + ) + cal_rel, ctc_rel = pair + cal_local = os.path.join(work, os.path.basename(cal_rel)) + ctc_local = os.path.join(work, os.path.basename(ctc_rel)) + if c["local"]: + cal_local = os.path.join(c["repo"], cal_rel) + ctc_local = os.path.join(c["repo"], ctc_rel) + else: + for rel, dest in ((cal_rel, cal_local), (ctc_rel, ctc_local)): + found, _ = _fetch_blob( + c["repo"], c["bucket"], c["dataset_id"], rel, c["head"], dest + ) + if not found: + # Tracked at HEAD but unfetchable. Falling through to serve + # the recording unfiltered is exactly what ADR 0028 forbids. + raise MaxShieldUncalibrated( + f"calibration input {rel!r} is tracked at " + f"{c['head'][:8]} but could not be fetched" + ) + filtered = os.path.join(work, "sss_" + os.path.basename(primary_local)) + sss_meta = apply_sss(primary_local, cal_local, ctc_local, filtered) + primary_local = filtered plf = power_line_frequency_for(c["repo"], primary, c["head_files"], c["head"]) descs = event_descriptions_for(c["repo"], primary, c["head_files"], c["head"]) elec = electrode_positions_for(c["repo"], primary, c["head_files"], c["head"]) @@ -2757,6 +2971,11 @@ def convert_one(primary: str, peak_bytes: int | None = None) -> dict: # reproducible BIDS-repo-relative path before validating/uploading. # nemarOrg/nemar-cli#1102. fix_source_file_attr(store_local, primary) + if sss_meta: + # In the store as well as the index: a consumer reading the store + # directly (the ML streaming path) must not have to fetch index.json to + # learn that this signal is a processed derivative. + embed_root_attr(store_local, "sss", sss_meta) # Guard the --delete sync: an empty/partial store would otherwise wipe a # previously-valid one. zarr.json => v3 root. validate_store(store_local) @@ -2816,6 +3035,15 @@ def convert_one(primary: str, peak_bytes: int | None = None) -> dict: members = split_members_for(primary, c["head_files"]) if members: entry["split_members"] = members + # ADR 0028 requires this to be DISCLOSED, not merely auditable. Every other + # store is the source signal quantised and rate-capped and nothing more; this + # one has been processed. A model training across datasets would otherwise + # silently mix filtered and unfiltered MEG with no signal that it was doing + # so. MNE writes the parameters into the recording's own proc_history, but + # biosigIO's importer does not carry that into the store, so state it here + # (and in the store's root attributes) rather than assume it survives. + if sss_meta: + entry["sss"] = sss_meta return { "ok": True, "primary": primary, @@ -3197,8 +3425,23 @@ def record(r: dict, i: int) -> None: channel_counts[p] = expected_channel_count_for(repo, p, head_set, head) except Exception: # noqa: BLE001 - a projection input must not fail a run channel_counts[p] = None + # Whether a recording will need the Signal-Space Separation phase cannot be known + # here: that is `info["maxshield"]` inside the FIF, and admission deliberately + # projects from git-annex pointers WITHOUT downloading. The BIDS sidecar carries + # no MaxShield marker either. So use the one signal that IS available from + # `head_files` alone -- a resolvable fine-calibration / cross-talk pair -- and let + # the worker do the real detection. A dataset that ships the pair for a FIF that + # turns out not to be MaxShield is merely over-reserved for, which costs + # concurrency; under-reserving costs the node. + maxshield_hint = { + p: lower_ext(p) == ".fif" and maxshield_calibration_for(p, head_set) is not None + for p in convert + } projections = { - p: projected_peak_bytes(p, sizes[p], channel_counts.get(p)) for p in convert + p: projected_peak_bytes( + p, sizes[p], channel_counts.get(p), maxshield=maxshield_hint[p] + ) + for p in convert } peaks = { p: admission_reserve_bytes( diff --git a/scripts/zarr/test_generate_zarr.py b/scripts/zarr/test_generate_zarr.py index 1960f9139..5405fad0c 100644 --- a/scripts/zarr/test_generate_zarr.py +++ b/scripts/zarr/test_generate_zarr.py @@ -61,6 +61,10 @@ memory_failure_result, count_infra_failures, RecordingMemoryExceeded, + MaxShieldUncalibrated, + maxshield_calibration_for, + MAXSHIELD_MEM_FACTOR, + projected_peak_bytes, RETRYABLE_CODES, reason_for_code, should_stream, @@ -87,6 +91,7 @@ fix_source_file_attr, in_excluded_tree, is_bids_calibration_file, + _bids_entities, is_excluded_from_discovery, is_primary, is_split_fif, @@ -2103,6 +2108,126 @@ def test_no_size_field_returns_none(self): self.assertIsNone(annex_key_size("")) +class TestMaxShieldCalibrationResolution(unittest.TestCase): + """ADR 0028: Signal-Space Separation runs only with the recording's OWN + fine-calibration and cross-talk pair. Resolution is BIDS inheritance, and the + pair is all-or-nothing -- uncalibrated filtering is a weaker correction a + consumer could not distinguish from a good one, so it is declined instead.""" + + REC = "sub-01/meg/sub-01_task-rest_meg.fif" + CAL = "sub-01/meg/sub-01_acq-calibration_meg.dat" + CTC = "sub-01/meg/sub-01_acq-crosstalk_meg.fif" + + def test_pair_beside_the_recording(self): + self.assertEqual( + maxshield_calibration_for(self.REC, {self.REC, self.CAL, self.CTC}), + (self.CAL, self.CTC), + ) + + def test_the_entity_subset_rule_would_reject_these(self): + """The regression this resolver exists for. + + Every other sidecar resolver requires the candidate's entities to be a + SUBSET of the recording's. `sub-01_acq-calibration_meg.dat` carries + `acq=calibration` while the recording has no `acq` at all, so that rule + rejects the very file it should find. Pin the exemption. + """ + self.assertEqual(_bids_entities("sub-01_acq-calibration_meg")["acq"], "calibration") + self.assertNotIn("acq", _bids_entities("sub-01_task-rest_meg")) + self.assertIsNotNone( + maxshield_calibration_for(self.REC, {self.REC, self.CAL, self.CTC}) + ) + + def test_half_a_pair_is_no_pair(self): + for present in (self.CAL, self.CTC): + with self.subTest(present=present): + self.assertIsNone( + maxshield_calibration_for(self.REC, {self.REC, present}) + ) + + def test_neither_present(self): + self.assertIsNone(maxshield_calibration_for(self.REC, {self.REC})) + + def test_inherited_from_an_ancestor(self): + rec = "sub-06/ses-1/meg/sub-06_ses-1_task-rest_meg.fif" + head = {rec, "sub-06/sub-06_acq-calibration_meg.dat", + "sub-06/sub-06_acq-crosstalk_meg.fif"} + self.assertIsNotNone(maxshield_calibration_for(rec, head)) + + def test_bare_form_at_the_dataset_root(self): + head = {self.REC, "acq-calibration_meg.dat", "acq-crosstalk_meg.fif"} + self.assertIsNotNone(maxshield_calibration_for(self.REC, head)) + + def test_nearest_wins(self): + rec = "sub-06/ses-1/meg/sub-06_ses-1_task-rest_meg.fif" + near = "sub-06/ses-1/meg/sub-06_ses-1_acq-calibration_meg.dat" + head = {rec, near, "sub-06/ses-1/meg/sub-06_ses-1_acq-crosstalk_meg.fif", + "sub-06/sub-06_acq-calibration_meg.dat", + "sub-06/sub-06_acq-crosstalk_meg.fif"} + pair = maxshield_calibration_for(rec, head) + self.assertIsNotNone(pair) + self.assertEqual(pair[0], near) + + def test_another_subject_pair_does_not_apply(self): + head = {self.REC, "sub-02/meg/sub-02_acq-calibration_meg.dat", + "sub-02/meg/sub-02_acq-crosstalk_meg.fif"} + self.assertIsNone(maxshield_calibration_for(self.REC, head)) + + def test_a_sibling_session_does_not_apply(self): + # `sub-06/ses-2/` is not an ancestor of `sub-06/ses-1/`, so inheritance + # must not reach across it. + rec = "sub-06/ses-1/meg/sub-06_ses-1_task-rest_meg.fif" + head = {rec, "sub-06/ses-2/meg/sub-06_ses-2_acq-calibration_meg.dat", + "sub-06/ses-2/meg/sub-06_ses-2_acq-crosstalk_meg.fif"} + self.assertIsNone(maxshield_calibration_for(rec, head)) + + def test_calibration_files_stay_excluded_from_discovery(self): + """ADR 0028's own trap: these are inputs to conversion AND never + recordings. Both must remain true; conflating them breaks MaxShield.""" + for p in (self.CAL, self.CTC): + with self.subTest(p=p): + self.assertTrue(is_bids_calibration_file(p)) + self.assertFalse(is_primary(p)) + self.assertIsNotNone( + maxshield_calibration_for(self.REC, {self.REC, self.CAL, self.CTC}) + ) + + +class TestMaxShieldVerdictAndProjection(unittest.TestCase): + def test_decline_is_deterministic_not_retryable(self): + """The calibration pair is either shipped or it is not; retrying cannot + change that, so a dataset made entirely of these must be marked terminal + rather than burning five attempts.""" + self.assertNotIn(MaxShieldUncalibrated.code, RETRYABLE_CODES) + failures = [{"code": MaxShieldUncalibrated.code}] + self.assertEqual(count_infra_failures(failures, failures), 0) + + def test_decline_has_its_own_user_facing_reason(self): + reason = reason_for_code(MaxShieldUncalibrated.code) + self.assertNotEqual(reason, reason_for_code("file_read_error")) + self.assertIn("shielding", reason.lower()) + + def test_projection_covers_the_filter_phase(self): + """A streaming FIF is projected on the streaming bound, which for this + dataset's largest recording exceeds the measured filter peak by under 5%. + That is coincidence, not headroom, so the MaxShield term must raise it.""" + size = 927 * 1024**2 + p = "sub-01/meg/sub-01_task-rest_meg.fif" + plain = projected_peak_bytes(p, size, 328) + shielded = projected_peak_bytes(p, size, 328, maxshield=True) + self.assertGreater(shielded, plain) + self.assertGreaterEqual(shielded, int(size * MAXSHIELD_MEM_FACTOR)) + + def test_projection_takes_the_larger_phase_not_the_sum(self): + # The two phases are sequential and the Raw is released between them. + size = 8 * 1024**2 # small enough that conversion dominates + p = "sub-01/meg/sub-01_task-rest_meg.fif" + plain = projected_peak_bytes(p, size) + shielded = projected_peak_bytes(p, size, maxshield=True) + self.assertEqual(shielded, max(plain, int(size * MAXSHIELD_MEM_FACTOR))) + self.assertLess(shielded, plain + int(size * MAXSHIELD_MEM_FACTOR)) + + class TestFailureReasons(unittest.TestCase): """Typed data failures (recordings the viewer should explain) are carried into index.json `failures`; infra failures are not. Mirrors biosigIO's error codes.""" From 0453203337fe9f39466e63c86837dd80087fab7d Mon Sep 17 00:00:00 2001 From: Seyed Yahya Shirazi Date: Sat, 22 Aug 2026 11:43:33 -0700 Subject: [PATCH 2/2] Address review: silent regressions and a one-point factor Four findings from review of #1127. is_maxshield_fif swallowed every exception with no diagnostic. Every one it can swallow is a .fif whose header would not read, and returning False routes a genuinely shielded recording back down the pre-ADR-0028 path, where MNE refuses it and the operator sees an opaque file_read_error with no hint the correction was skipped. Now warns and says what to expect. maxwell_filter failures other than a missing pair were uncoded, so they retried the filter forever at about a minute a time. A calibration pair that is present but does not fit the recording is as permanent as a missing one, so it now gets the same terminal verdict. MemoryError still propagates, since that one really is transient. Local mode resolved the pair without checking it exists, while the remote branch declines cleanly when a tracked file cannot be materialised. A working tree can hold an annex pointer whose content was never fetched. The memory factor rested on one recording, which is exactly how the BrainVision factor was wrong by 2x. Measured across both datasets and the size range: 5.0x, 5.0x, 4.4x, 4.2x for 160, 164, 438 and 716 MiB. The ratio falls with size, so the worst case sits where the absolute number is trivial. 6 clears every point and clears the largest by 43%. Pinned by a test, along with the reserve at the end of the admission chain rather than the projection alone, since that is where the slack suppression lives. --- scripts/zarr/generate_zarr.py | 87 +++++++++++++++++++++++++----- scripts/zarr/test_generate_zarr.py | 20 +++++++ 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/scripts/zarr/generate_zarr.py b/scripts/zarr/generate_zarr.py index 8993598e7..d04303856 100644 --- a/scripts/zarr/generate_zarr.py +++ b/scripts/zarr/generate_zarr.py @@ -459,17 +459,34 @@ def apply_worker_mem_limit( # Signal-Space Separation (ADR 0028) runs BEFORE conversion and costs its own peak: # it needs a fully preloaded float64 `Raw`, which is the anonymous memory RLIMIT_DATA -# counts. Measured end to end on the conversion node against on006720's `sub-155`, -# through these same functions: a 716 MiB recording peaked at 2.95 GiB, i.e. 4.2x its -# on-disk size, and the filtered copy then STREAMS, so conversion adds nothing to that -# peak. 6x is the measurement with headroom. +# counts. The filtered copy then STREAMS, so conversion adds nothing to that peak. # -# This is load-bearing rather than an efficiency nicety. `apply_worker_mem_limit` -# sizes RLIMIT_DATA from the projection BEFORE the worker starts, so an -# under-projection does not merely over-schedule the node, it kills the filter -# mid-run. The streaming floor alone (4 GiB) happens to clear this dataset's largest -# recording by under 5%, which is coincidence rather than headroom -- and it is the -# filter phase, not the conversion, that the floor was never sized for. +# Measured through `apply_sss` itself on the conversion node, across both affected +# datasets rather than at one point -- the BrainVision entry above is the cautionary +# tale for a single-sample factor: +# +# on006720 sub-197 160 MiB -> 0.78 GiB 5.0x +# on006720 sub-155 164 MiB -> 0.79 GiB 5.0x +# on006012 sub-01 438 MiB -> 1.87 GiB 4.4x +# on006720 sub-155 716 MiB -> 2.95 GiB 4.2x +# +# The ratio FALLS with size as fixed overhead amortises, so the worst ratio sits where +# the absolute number is trivial and the largest recordings -- the ones that can +# actually exhaust the node -- are the cheapest per byte. 6x clears every observed +# point, and clears the largest by 43%. +# +# Note this deliberately does NOT get `MEM_LIMIT_SLACK`. Admission calls +# `admission_reserve_bytes(..., streamed=True)` for these recordings, since the +# CONVERSION streams, and that suppresses the 3x multiplier. That multiplier exists +# for factors that are "a guessed multiple of on-disk bytes"; this one is measured +# across the range it will be applied to, which is the condition the slack covers. +# Applying both would reserve ~18x on-disk and gut concurrency for no evidence. +# +# Load-bearing rather than an efficiency nicety: `apply_worker_mem_limit` sizes +# RLIMIT_DATA from this projection BEFORE the worker starts, so an under-projection +# does not merely over-schedule the node, it kills the filter mid-run. The streaming +# floor alone (4 GiB) happens to clear the largest recording by under 5%, which is +# coincidence -- that floor was sized for conversion, not for this phase. MAXSHIELD_MEM_FACTOR = float(os.environ.get("ZARR_MAXSHIELD_MEM_FACTOR", "6")) @@ -984,7 +1001,19 @@ def is_maxshield_fif(path: str) -> bool: path, allow_maxshield="yes", preload=False, verbose="ERROR" ).info return bool(info.get("maxshield")) - except Exception: # noqa: BLE001 - unreadable here means "not the MaxShield path" + except Exception as exc: # noqa: BLE001 - never fail a recording on the probe + # Returning False here routes the recording down the NORMAL path, which for + # a genuinely MaxShield file is the pre-ADR-0028 behaviour: MNE refuses it + # and the operator gets an opaque `file_read_error` with no hint that the + # correction was skipped because a header probe failed. That is a real + # regression risk (a truncated download, a missing split member -- MNE reads + # every split header even at preload=False), so it must not be silent. + print( + f"::warning::{path!r}: could not read the FIF header to test for " + f"Internal Active Shielding ({type(exc).__name__}: {exc}); converting it " + "as an ordinary recording. If it IS shielded, expect a file_read_error.", + flush=True, + ) return False @@ -1006,9 +1035,26 @@ def apply_sss(raw_path: str, calibration: str, cross_talk: str, out_path: str) - raw = mne.io.read_raw_fif( raw_path, allow_maxshield="yes", preload=True, verbose="ERROR" ) - sss = mne.preprocessing.maxwell_filter( - raw, calibration=calibration, cross_talk=cross_talk, verbose="ERROR" - ) + try: + sss = mne.preprocessing.maxwell_filter( + raw, calibration=calibration, cross_talk=cross_talk, verbose="ERROR" + ) + except MemoryError: + # Genuinely transient: the node was busy, not the data wrong. Let it reach + # the retryable `recording_memory_exceeded` verdict. + raise + except Exception as exc: # noqa: BLE001 - see below + # A calibration pair that is PRESENT but does not fit this recording (wrong + # sensor set, wrong site, malformed file) is just as permanent a property of + # what the dataset ships as a missing pair, so it gets the same terminal + # verdict. Left uncoded it would fall through to the generic handler as an + # infra failure and re-run maxwell_filter on every future pass forever -- + # the anti-pattern #1110 exists to prevent, at ~1 minute of compute a time. + raise MaxShieldUncalibrated( + f"Signal-Space Separation failed with this recording's calibration pair " + f"({os.path.basename(calibration)}, {os.path.basename(cross_talk)}): " + f"{type(exc).__name__}: {exc}" + ) from exc # MNE's own guardrail, the check that refused to load the file, is satisfied by # the output. If it is not, the correction did not happen and serving the result # would be exactly the thing ADR 0028 forbids -- so fail rather than publish it. @@ -2942,6 +2988,19 @@ def convert_one(primary: str, peak_bytes: int | None = None) -> dict: if c["local"]: cal_local = os.path.join(c["repo"], cal_rel) ctc_local = os.path.join(c["repo"], ctc_rel) + # The remote branch below decides cleanly when a tracked file cannot + # be materialised; local mode has to check for itself. A working tree + # can hold a git-annex POINTER whose content was never fetched, and + # `os.path.exists` is False for a dangling symlink -- so this catches + # the realistic case rather than letting apply_sss fail uncoded and + # retry the filter on every future run. + missing = [r for r, p in ((cal_rel, cal_local), (ctc_rel, ctc_local)) + if not os.path.exists(p)] + if missing: + raise MaxShieldUncalibrated( + "calibration input(s) present in the tree but without local " + f"content (run `git annex get`): {', '.join(missing)}" + ) else: for rel, dest in ((cal_rel, cal_local), (ctc_rel, ctc_local)): found, _ = _fetch_blob( diff --git a/scripts/zarr/test_generate_zarr.py b/scripts/zarr/test_generate_zarr.py index 5405fad0c..3f96cb3a6 100644 --- a/scripts/zarr/test_generate_zarr.py +++ b/scripts/zarr/test_generate_zarr.py @@ -2218,6 +2218,26 @@ def test_projection_covers_the_filter_phase(self): self.assertGreater(shielded, plain) self.assertGreaterEqual(shielded, int(size * MAXSHIELD_MEM_FACTOR)) + def test_factor_clears_every_measured_ratio(self): + """Pin the factor against the measurements it was derived from, so a future + edit cannot quietly drop it below an observed peak. Sizes in MiB, peaks in + GiB, measured through apply_sss on the conversion node.""" + for mib, gib in ((160, 0.78), (164, 0.79), (438, 1.87), (716, 2.95)): + with self.subTest(mib=mib): + self.assertGreater(mib * 1024**2 * MAXSHIELD_MEM_FACTOR, gib * 1024**3) + + def test_reserve_keeps_headroom_over_the_measured_peak(self): + """The projection is not the limit. What the worker actually gets is + admission_reserve_bytes, which for these recordings suppresses the usual 3x + slack because the CONVERSION streams -- so assert the end of that chain, not + just the projection. 716 MiB measured at 2.95 GiB.""" + size = 716 * 1024**2 + p = "sub-01/meg/sub-01_task-rest_meg.fif" + ceiling = 24 * 1024**3 + proj = projected_peak_bytes(p, size, 336, maxshield=True) + reserve = admission_reserve_bytes(proj, ceiling, streamed=should_stream(p, size)) + self.assertGreater(reserve, int(2.95 * 1024**3)) + def test_projection_takes_the_larger_phase_not_the_sum(self): # The two phases are sequential and the Raw is released between them. size = 8 * 1024**2 # small enough that conversion dominates