From abbb04cadf53b4ef5191cbcf0cbc0d93d82560c6 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 2 Oct 2026 13:28:38 +0000 Subject: [PATCH 1/3] test(xtask): make the WSL clock-step guard match lines and compare the twins Two gaps in `both_wsl_lanes_settle_the_clock_before_running_the_suite`. It found the step with `job_block.find(STEP)`, a substring search, while its doc comment claimed the step NAME was matched precisely so that the surrounding prose could not satisfy it. A comment that embeds the step's text keeps the old check green with the step deleted: measured by replacing the step line with `# disabled: - name: Settle the clock ...`, which the previous version passes. Both steps are now located by whole-line equality, and that mutant fails. It also pinned presence and order but not content, so the two hand-copied `run:` bodies could drift silently. That is the drift `apu_preflight_twins_do_not_drift` exists for, and once caught too late. The bodies are now extracted with the existing `run_block`/`dedent` helpers and compared byte for byte. They are identical today, 65 lines each, and a one-character edit to either now fails the test. Extracting a single step reuses a new `step_block`, which `gpu_preflight_steps` now calls too, rather than a second copy of its loop. The per-PR copy of the step also gains the pointer to its guarding test that only the nightly copy carried, so the guard is discoverable from both sides. Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 6 +++ .github/workflows/nightly.yml | 3 +- xtask/src/workflow_contract.rs | 69 +++++++++++++++++++--------- 3 files changed, 55 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index e813aace45..8f178e9033 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -1073,6 +1073,12 @@ jobs: # Every command is best-effort — this is a hazard to reduce, not a # precondition to enforce, and a lane that cannot settle its clock should # still run and fail with a real diagnosis. + # + # nightly.yml's `e2e-wsl-nightly` carries a copy of this step, kept in sync + # by hand until the lanes are deduplicated (#294); + # `both_wsl_lanes_settle_the_clock_before_running_the_suite` in + # xtask/src/workflow_contract.rs fails if either copy goes missing, moves + # after the suite, or drifts from the other. - name: Settle the clock before anything times a scenario shell: powershell run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a48b3dc687..0e7faf8cd6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1205,7 +1205,8 @@ jobs: # `SystemTime`. See that job's copy of this step for the full reasoning. # Kept in sync by hand until the two lanes are deduplicated (#294); # `both_wsl_lanes_settle_the_clock_before_running_the_suite` in - # xtask/src/workflow_contract.rs fails if either copy goes missing. + # xtask/src/workflow_contract.rs fails if either copy goes missing, moves + # after the suite, or drifts from the other. - name: Settle the clock before anything times a scenario shell: powershell run: | diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 89c0a77587..c3a3091a8d 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -946,23 +946,27 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" /// Every `GPU preflight` step block in `text`, in file order. fn gpu_preflight_steps(text: &str) -> Vec { let lines: Vec<&str> = text.lines().collect(); - let mut steps = Vec::new(); - for (i, line) in lines.iter().enumerate() { - if !line.trim_start().starts_with("- name: GPU preflight") { - continue; - } - let step_indent = indent_of(line); - let mut step = format!("{line}\n"); - for body in &lines[i + 1..] { - if !body.trim().is_empty() && indent_of(body) <= step_indent { - break; - } - step.push_str(body); - step.push('\n'); + lines + .iter() + .enumerate() + .filter(|(_, line)| line.trim_start().starts_with("- name: GPU preflight")) + .map(|(i, _)| step_block(&lines, i)) + .collect() + } + + /// The whole step whose `- name:` line is `lines[at]`: that line and every + /// line below it until one is indented no deeper than it. + fn step_block(lines: &[&str], at: usize) -> String { + let step_indent = indent_of(lines[at]); + let mut step = format!("{}\n", lines[at]); + for body in &lines[at + 1..] { + if !body.trim().is_empty() && indent_of(body) <= step_indent { + break; } - steps.push(step); + step.push_str(body); + step.push('\n'); } - steps + step } /// The lines of a step's `run: |` block, still indented. @@ -1087,13 +1091,19 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" /// nightly lane job-level `continue-on-error: true` means a regression would /// not even turn the run red. /// - /// Matching the step NAME rather than a substring of the job is deliberate — - /// the surrounding comments discuss the clock at length, so a bare substring - /// would stay satisfied by prose after the step itself was deleted, which is - /// exactly the failure this pins. + /// Steps are matched as WHOLE LINES, not found as substrings of the job: the + /// surrounding comments discuss the clock at length, and a comment line that + /// happened to embed the step's text would otherwise keep this satisfied after + /// the step itself was deleted — exactly the failure this pins. + /// + /// The two copies' `run:` bodies are also compared byte for byte, after + /// dedenting. They are kept in sync by hand until the lanes are deduplicated + /// (#294), and presence and order say nothing about what each copy runs — + /// the same drift `apu_preflight_twins_do_not_drift` once caught too late. #[test] fn both_wsl_lanes_settle_the_clock_before_running_the_suite() { const STEP: &str = " - name: Settle the clock before anything times a scenario"; + let mut bodies = Vec::new(); for (workflow, job, suite_step) in [ ( "e2e-selfhosted.yml", @@ -1107,22 +1117,37 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" ), ] { let job_block = nested_block(&read_workflow(workflow), job); - let settle = job_block.find(STEP).unwrap_or_else(|| { + let lines: Vec<&str> = job_block.lines().collect(); + let settle = lines.iter().position(|l| *l == STEP).unwrap_or_else(|| { panic!( "{workflow}'s `{job}` has no clock-settling step — a fresh WSL2 guest \ steps its clock backwards mid-run and cucumber subtracts SystemTime \ stamps, so the durations that lane reports become fiction" ) }); - let suite = job_block - .find(suite_step) + let suite = lines + .iter() + .position(|l| *l == suite_step) .unwrap_or_else(|| panic!("{workflow}'s `{job}` no longer runs `{suite_step}`")); assert!( settle < suite, "{workflow}'s `{job}` settles the clock AFTER starting the suite — the \ correction has to land while nothing is being timed" ); + let step = step_block(&lines, settle); + let body = run_block(&step) + .and_then(|block| dedent(&block)) + .unwrap_or_else(|| panic!("{workflow}'s clock-settling step has no `run: |` body")); + bodies.push((workflow, body)); } + let [(first, first_body), (second, second_body)] = bodies.as_slice() else { + unreachable!("exactly two lanes are checked above"); + }; + assert_eq!( + first_body, second_body, + "the clock-settling step has drifted between {first} and {second} — the \ + copies are kept in sync by hand until the lanes are deduplicated (#294)" + ); } #[test] From a8aa739bc435c7b41a936cbf1b7697689f705f50 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 5 Oct 2026 10:02:15 +0000 Subject: [PATCH 2/3] test(xtask): trim run-block separators and name the first drifted line The clock-settling comparison took in the blank line between that step and the next, because step_block and run_block both let blank lines through. Deleting that separator in one workflow failed the test with a "step has drifted" message for an edit outside the step. run_block now drops trailing blank lines, so every caller, including the APU preflight twin check, compares only the script. On a real drift the test now names the first differing body line and both versions of it, or the line counts when one body is a prefix of the other, instead of printing two 4 KB escaped strings. Also state that blank lines never end a step_block, and that the twin equality pins agreement between the lanes, not that either body still does the work. Signed-off-by: Roman Inflianskas --- xtask/src/workflow_contract.rs | 62 ++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index c3a3091a8d..ce94bd7a8b 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -955,7 +955,11 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } /// The whole step whose `- name:` line is `lines[at]`: that line and every - /// line below it until one is indented no deeper than it. + /// line below it until a non-blank one is indented no deeper than it. + /// + /// Blank lines never end a block, however they are indented — a `run: |` + /// body can contain them — so the blank separator before the next step is + /// carried along too. `run_block` drops it again. fn step_block(lines: &[&str], at: usize) -> String { let step_indent = indent_of(lines[at]); let mut step = format!("{}\n", lines[at]); @@ -970,16 +974,48 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } /// The lines of a step's `run: |` block, still indented. + /// + /// Trailing blank lines are dropped: they are the separator `step_block` + /// lets through, not part of the script, and keeping them would make two + /// identical scripts compare unequal over the spacing between steps. fn run_block(step: &str) -> Option> { let lines: Vec<&str> = step.lines().collect(); let run_at = lines.iter().position(|l| l.trim() == "run: |")?; let run_indent = indent_of(lines[run_at]); + let mut body: Vec<&str> = lines[run_at + 1..] + .iter() + .take_while(|l| l.trim().is_empty() || indent_of(l) > run_indent) + .copied() + .collect(); + while body.last().is_some_and(|l| l.trim().is_empty()) { + body.pop(); + } + Some(body) + } + + /// Where two line-oriented texts first differ, phrased for a failure + /// message; `None` when they are equal. + fn first_line_difference(a: &str, b: &str) -> Option { + if a == b { + return None; + } + let a_lines: Vec<&str> = a.lines().collect(); + let b_lines: Vec<&str> = b.lines().collect(); Some( - lines[run_at + 1..] - .iter() - .take_while(|l| l.trim().is_empty() || indent_of(l) > run_indent) - .copied() - .collect(), + match a_lines.iter().zip(&b_lines).position(|(x, y)| x != y) { + Some(i) => format!( + "first difference at body line {}:\n {:?}\n {:?}", + i + 1, + a_lines[i], + b_lines[i] + ), + None if a_lines.len() != b_lines.len() => format!( + "every shared line matches, but one body has {} lines and the other {}", + a_lines.len(), + b_lines.len() + ), + None => "the bodies differ only in line endings".to_string(), + }, ) } @@ -1100,6 +1136,8 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" /// dedenting. They are kept in sync by hand until the lanes are deduplicated /// (#294), and presence and order say nothing about what each copy runs — /// the same drift `apu_preflight_twins_do_not_drift` once caught too late. + /// Equality pins only that the two lanes agree, not that either body still + /// does the work: both gutted to an identical `echo hi` would pass. #[test] fn both_wsl_lanes_settle_the_clock_before_running_the_suite() { const STEP: &str = " - name: Settle the clock before anything times a scenario"; @@ -1143,11 +1181,13 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" let [(first, first_body), (second, second_body)] = bodies.as_slice() else { unreachable!("exactly two lanes are checked above"); }; - assert_eq!( - first_body, second_body, - "the clock-settling step has drifted between {first} and {second} — the \ - copies are kept in sync by hand until the lanes are deduplicated (#294)" - ); + if let Some(difference) = first_line_difference(first_body, second_body) { + panic!( + "the clock-settling step has drifted between {first} and {second} — the \ + copies are kept in sync by hand until the lanes are deduplicated (#294); \ + {difference}" + ); + } } #[test] From 8d6d9a428c7d15efa80805d74eff9f2a172ad273 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 5 Oct 2026 12:15:02 +0000 Subject: [PATCH 3/3] test(xtask): name the first drifted line in the APU preflight twin check apu_preflight_twins_do_not_drift still compared whole dedented scripts with assert_eq!, so a one-space drift printed two ~7 KB escaped strings that look identical in a terminal. Route that comparison through first_line_difference, as the clock-settling check already does. first_line_difference now takes the two workflow names, so the first differing line is labelled with its file and a length mismatch says which copy is shorter instead of "one body ... and the other". Signed-off-by: Roman Inflianskas --- xtask/src/workflow_contract.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index ce94bd7a8b..b8598e6793 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -994,8 +994,9 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" } /// Where two line-oriented texts first differ, phrased for a failure - /// message; `None` when they are equal. - fn first_line_difference(a: &str, b: &str) -> Option { + /// message and naming which file each side came from; `None` when they + /// are equal. + fn first_line_difference(a_name: &str, a: &str, b_name: &str, b: &str) -> Option { if a == b { return None; } @@ -1004,13 +1005,13 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" Some( match a_lines.iter().zip(&b_lines).position(|(x, y)| x != y) { Some(i) => format!( - "first difference at body line {}:\n {:?}\n {:?}", + "first difference at body line {}:\n {a_name}: {:?}\n {b_name}: {:?}", i + 1, a_lines[i], b_lines[i] ), None if a_lines.len() != b_lines.len() => format!( - "every shared line matches, but one body has {} lines and the other {}", + "every shared line matches, but {a_name} has {} lines and {b_name} has {}", a_lines.len(), b_lines.len() ), @@ -1181,7 +1182,7 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" let [(first, first_body), (second, second_body)] = bodies.as_slice() else { unreachable!("exactly two lanes are checked above"); }; - if let Some(difference) = first_line_difference(first_body, second_body) { + if let Some(difference) = first_line_difference(first, first_body, second, second_body) { panic!( "the clock-settling step has drifted between {first} and {second} — the \ copies are kept in sync by hand until the lanes are deduplicated (#294); \ @@ -1224,11 +1225,14 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" "{language} APU preflight count differs" ); for (i, (a, b)) in per_pr.iter().zip(nightly.iter()).enumerate() { - assert_eq!( - a, b, - "{language} APU preflight script #{i} has drifted between \ - e2e-selfhosted.yml and nightly.yml" - ); + if let Some(difference) = + first_line_difference("e2e-selfhosted.yml", a, "nightly.yml", b) + { + panic!( + "{language} APU preflight script #{i} has drifted between \ + e2e-selfhosted.yml and nightly.yml; {difference}" + ); + } } } }