Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/e2e-selfhosted.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1073,6 +1073,12 @@ jobs:
# Every command is best-effort — this is a hazard to reduce, not a
# precondition to enforce, and a lane that cannot settle its clock should
# still run and fail with a real diagnosis.
#
# nightly.yml's `e2e-wsl-nightly` carries a copy of this step, kept in sync
# by hand until the lanes are deduplicated (#294);
# `both_wsl_lanes_settle_the_clock_before_running_the_suite` in
# xtask/src/workflow_contract.rs fails if either copy goes missing, moves
# after the suite, or drifts from the other.
- name: Settle the clock before anything times a scenario
shell: powershell
run: |
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1205,7 +1205,8 @@ jobs:
# `SystemTime`. See that job's copy of this step for the full reasoning.
# Kept in sync by hand until the two lanes are deduplicated (#294);
# `both_wsl_lanes_settle_the_clock_before_running_the_suite` in
# xtask/src/workflow_contract.rs fails if either copy goes missing.
# xtask/src/workflow_contract.rs fails if either copy goes missing, moves
# after the suite, or drifts from the other.
- name: Settle the clock before anything times a scenario
shell: powershell
run: |
Expand Down
133 changes: 101 additions & 32 deletions xtask/src/workflow_contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -946,36 +946,77 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow"
/// Every `GPU preflight` step block in `text`, in file order.
fn gpu_preflight_steps(text: &str) -> Vec<String> {
let lines: Vec<&str> = text.lines().collect();
let mut steps = Vec::new();
for (i, line) in lines.iter().enumerate() {
if !line.trim_start().starts_with("- name: GPU preflight") {
continue;
}
let step_indent = indent_of(line);
let mut step = format!("{line}\n");
for body in &lines[i + 1..] {
if !body.trim().is_empty() && indent_of(body) <= step_indent {
break;
}
step.push_str(body);
step.push('\n');
lines
.iter()
.enumerate()
.filter(|(_, line)| line.trim_start().starts_with("- name: GPU preflight"))
.map(|(i, _)| step_block(&lines, i))
.collect()
}

/// The whole step whose `- name:` line is `lines[at]`: that line and every
/// line below it until a non-blank one is indented no deeper than it.
///
/// Blank lines never end a block, however they are indented — a `run: |`
/// body can contain them — so the blank separator before the next step is
/// carried along too. `run_block` drops it again.
fn step_block(lines: &[&str], at: usize) -> String {
let step_indent = indent_of(lines[at]);
let mut step = format!("{}\n", lines[at]);
for body in &lines[at + 1..] {
if !body.trim().is_empty() && indent_of(body) <= step_indent {
break;
}
steps.push(step);
step.push_str(body);
step.push('\n');
}
steps
step
}

/// The lines of a step's `run: |` block, still indented.
///
/// Trailing blank lines are dropped: they are the separator `step_block`
/// lets through, not part of the script, and keeping them would make two
/// identical scripts compare unequal over the spacing between steps.
fn run_block(step: &str) -> Option<Vec<&str>> {
let lines: Vec<&str> = step.lines().collect();
let run_at = lines.iter().position(|l| l.trim() == "run: |")?;
let run_indent = indent_of(lines[run_at]);
let mut body: Vec<&str> = lines[run_at + 1..]
.iter()
.take_while(|l| l.trim().is_empty() || indent_of(l) > run_indent)
.copied()
.collect();
while body.last().is_some_and(|l| l.trim().is_empty()) {
body.pop();
}
Some(body)
}

/// Where two line-oriented texts first differ, phrased for a failure
/// message and naming which file each side came from; `None` when they
/// are equal.
fn first_line_difference(a_name: &str, a: &str, b_name: &str, b: &str) -> Option<String> {
if a == b {
return None;
}
let a_lines: Vec<&str> = a.lines().collect();
let b_lines: Vec<&str> = b.lines().collect();
Some(
lines[run_at + 1..]
.iter()
.take_while(|l| l.trim().is_empty() || indent_of(l) > run_indent)
.copied()
.collect(),
match a_lines.iter().zip(&b_lines).position(|(x, y)| x != y) {
Some(i) => format!(
"first difference at body line {}:\n {a_name}: {:?}\n {b_name}: {:?}",
i + 1,
a_lines[i],
b_lines[i]
),
None if a_lines.len() != b_lines.len() => format!(
"every shared line matches, but {a_name} has {} lines and {b_name} has {}",
a_lines.len(),
b_lines.len()
),
None => "the bodies differ only in line endings".to_string(),
},
)
}

Expand Down Expand Up @@ -1087,13 +1128,21 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow"
/// nightly lane job-level `continue-on-error: true` means a regression would
/// not even turn the run red.
///
/// Matching the step NAME rather than a substring of the job is deliberate —
/// the surrounding comments discuss the clock at length, so a bare substring
/// would stay satisfied by prose after the step itself was deleted, which is
/// exactly the failure this pins.
/// Steps are matched as WHOLE LINES, not found as substrings of the job: the
/// surrounding comments discuss the clock at length, and a comment line that
/// happened to embed the step's text would otherwise keep this satisfied after
/// the step itself was deleted — exactly the failure this pins.
///
/// The two copies' `run:` bodies are also compared byte for byte, after

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Worth being explicit here about what byte-for-byte equality between the copies does not buy. I replaced both run: bodies with an identical echo hi and the test still passes: presence, ordering and mutual agreement are all satisfied by a step that does nothing.

That is not a defect -- it is the stated scope -- but because the copies are synced by hand, the normal way to edit this step is to change one and paste it over the other, which keeps them in agreement by construction. The one workflow that is routine is also the one this check is blind to. A sentence saying the comparison pins that the lanes agree, not that either body is still doing the work, would keep the next reader from over-trusting it.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added in a8aa739b: equality pins only that the two lanes agree, not that either body still does the work. Both gutted to an identical echo hi would still pass.

/// dedenting. They are kept in sync by hand until the lanes are deduplicated
/// (#294), and presence and order say nothing about what each copy runs —
/// the same drift `apu_preflight_twins_do_not_drift` once caught too late.
/// Equality pins only that the two lanes agree, not that either body still
/// does the work: both gutted to an identical `echo hi` would pass.
#[test]
fn both_wsl_lanes_settle_the_clock_before_running_the_suite() {
const STEP: &str = " - name: Settle the clock before anything times a scenario";
let mut bodies = Vec::new();
for (workflow, job, suite_step) in [
(
"e2e-selfhosted.yml",
Expand All @@ -1107,21 +1156,38 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow"
),
] {
let job_block = nested_block(&read_workflow(workflow), job);
let settle = job_block.find(STEP).unwrap_or_else(|| {
let lines: Vec<&str> = job_block.lines().collect();
let settle = lines.iter().position(|l| *l == STEP).unwrap_or_else(|| {
panic!(
"{workflow}'s `{job}` has no clock-settling step — a fresh WSL2 guest \
steps its clock backwards mid-run and cucumber subtracts SystemTime \
stamps, so the durations that lane reports become fiction"
)
});
let suite = job_block
.find(suite_step)
let suite = lines
.iter()
.position(|l| *l == suite_step)
.unwrap_or_else(|| panic!("{workflow}'s `{job}` no longer runs `{suite_step}`"));
assert!(
settle < suite,
"{workflow}'s `{job}` settles the clock AFTER starting the suite — the \
correction has to land while nothing is being timed"
);
let step = step_block(&lines, settle);
let body = run_block(&step)

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does not extract only the run: block. step_block skips blank lines when looking for the end of the step (!body.trim().is_empty() && ...), and run_block's take_while admits blank lines unconditionally, so the blank line that separates this step from the next one is pulled into the body along with the step's actual content.

Right now both workflows have exactly one blank line in that position, so the two bodies still match and the test passes. That is the only reason it passes. I deleted just that blank line from .github/workflows/nightly.yml (the one between exit 0 and - name: Ensure native build deps) and ran the test:

assertion `left == right` failed: the clock-settling step has drifted between
e2e-selfhosted.yml and nightly.yml -- the copies are kept in sync by hand

Nothing inside either step changed. A whitespace-only reflow of one workflow now fails a test that says the clock-settling step drifted, which would block unrelated work and send whoever hits it looking in the wrong place.

Trimming trailing blank lines off the extracted body before comparing closes it -- either .map(|b| b.trim_end().to_string()) here, or dropping trailing blanks from the vector run_block returns so every caller benefits.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in a8aa739b. run_block now drops trailing blank lines from what it returns, so the separator that step_block lets through never reaches a comparison. That also covers the APU preflight twin check, which had the same exposure. I reproduced your case before the fix: deleting the blank line after exit 0 in nightly.yml failed the test. It passes now, and so does adding extra blank lines there.

.and_then(|block| dedent(&block))
.unwrap_or_else(|| panic!("{workflow}'s clock-settling step has no `run: |` body"));
bodies.push((workflow, body));
}
let [(first, first_body), (second, second_body)] = bodies.as_slice() else {
unreachable!("exactly two lanes are checked above");
};
if let Some(difference) = first_line_difference(first, first_body, second, second_body) {
panic!(
"the clock-settling step has drifted between {first} and {second} — the \
copies are kept in sync by hand until the lanes are deduplicated (#294); \
{difference}"
);
}
}

Expand Down Expand Up @@ -1159,11 +1225,14 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow"
"{language} APU preflight count differs"
);
for (i, (a, b)) in per_pr.iter().zip(nightly.iter()).enumerate() {
assert_eq!(
a, b,
"{language} APU preflight script #{i} has drifted between \
e2e-selfhosted.yml and nightly.yml"
);
if let Some(difference) =
first_line_difference("e2e-selfhosted.yml", a, "nightly.yml", b)
{
panic!(
"{language} APU preflight script #{i} has drifted between \
e2e-selfhosted.yml and nightly.yml; {difference}"
);
}
}
}
}
Expand Down
Loading