From 11dd46a6f8c22fb8d44e7e82a18000d94564147e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 12:25:33 +0000 Subject: [PATCH 01/11] dev: narrow the pre-push cargo hooks to the crates a push affects The three cargo pre-push hooks (clippy, the e2e harness clippy and cargo test) ran over the whole workspace whatever the push contained, so a one-file change in a leaf crate cost a full workspace build and test run. They now run through `cargo xtask pre-push-check`, which takes the range prek passes to a pre-push hook and applies the selection CI already uses (`cargo xtask affected`): the changed crates plus every crate that depends on them, and the whole workspace for the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads or a path no crate owns. The directory-based selection cannot see a crate's tests reading a file from outside the crate, so the hook widens it: any Rust file also selects xtask (its contract tests scan every source file), README.md, docs/ and skills/ also select rocm and xtask, and the e2e PowerShell fixture also selects rocm-core, which includes it. A unit test scans the workspace for include_str!, include_bytes! and manifest-relative reads that leave their crate and fails when one is not covered, so a new cross-crate read cannot be missed silently. Whenever the affected crates cannot be determined (the base is not fetched, `cargo metadata` fails) a step runs over the whole workspace and prints why. A step is skipped only when the selection is known and has nothing for it to check, and it says so. The e2e harness clippy runs when e2e-cucumber is affected, and the test step never includes it, as before. `--dry-run` prints the scope and the cargo command without compiling. The step's range, `--base` override, `--dry-run` and cargo's exit status are tested through an injectable cargo runner. `affected` now lists both paths of a moved file (`--no-renames`), so a file moved out of one crate selects the crate that lost it as well as the one that gained it. Signed-off-by: Jussi Elo --- .pre-commit-config.yaml | 25 +- CONTRIBUTING.md | 4 +- docs/testing.md | 55 +- xtask/src/affected.rs | 72 ++- xtask/src/main.rs | 27 + xtask/src/prepush.rs | 1253 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 1424 insertions(+), 12 deletions(-) create mode 100644 xtask/src/prepush.rs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7dec5e6ff0..9877231133 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ # Setup: # uv tool install prek # or: cargo install --locked prek # prek install # install the pre-commit hook -# prek install -t pre-push # install the pre-push hook (clippy + tests) +# prek install -t pre-push # install the pre-push hook (clippy + tests of the affected crates) # # Run against everything on demand: # prek run --all-files @@ -130,12 +130,24 @@ repos: language: system types: [rust] pass_filenames: false + # The three pre-push cargo steps run through `cargo xtask pre-push-check`, + # which narrows each to the crates the push affects (the changed crates and + # everything that depends on them, selected as CI's `cargo xtask affected` + # does) and falls back to the whole workspace whenever it cannot tell. It + # prints the scope it chose, so these hooks are `verbose`. CI's Windows job, + # merge queue and `main` still check the whole workspace. Without the + # narrowing the steps are + # cargo clippy --workspace --all-targets -- -D warnings + # cargo clippy -p e2e-cucumber --test e2e -- -D warnings + # cargo test --workspace --all-targets --exclude e2e-cucumber + # and `cargo xtask pre-push-check --base ` runs one by hand. - id: cargo-clippy name: cargo clippy - entry: cargo clippy --workspace --all-targets -- -D warnings + entry: cargo xtask pre-push-check clippy language: system types: [rust] pass_filenames: false + verbose: true # shows which crates the push selected stages: [pre-push] # compiles the workspace; too slow for every commit groups: [local-tools] # runs in the dedicated `clippy` CI job - id: cargo-clippy-e2e @@ -143,19 +155,22 @@ repos: # e2e-cucumber's `e2e` test sets `harness = false, test = false` (see its # Cargo.toml), which also excludes it from `clippy --all-targets` above -- # leaving the ~1.6k lines of harness + step code unlinted. Clippy honours - # an explicit `--test`, so lint it directly, mirroring the CI job. - entry: cargo clippy -p e2e-cucumber --test e2e -- -D warnings + # an explicit `--test`, so the step lints it directly, mirroring the CI + # job, whenever e2e-cucumber is among the affected crates. + entry: cargo xtask pre-push-check clippy-e2e language: system types: [rust] pass_filenames: false + verbose: true stages: [pre-push] groups: [local-tools] # runs in the dedicated `clippy` CI job - id: cargo-test name: cargo test - entry: cargo test --workspace --all-targets --exclude e2e-cucumber + entry: cargo xtask pre-push-check test language: system types: [rust] pass_filenames: false + verbose: true stages: [pre-push] groups: [local-tools] # runs in the dedicated build-and-test CI job - id: powershell-script-analyzer diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc9b88922c..0fa54e2a85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,11 +43,13 @@ git clone https://github.com/ROCm/rocm-cli cd rocm-cli uv tool install prek # alternative: cargo install --locked prek prek install # fast checks on every commit -prek install -t pre-push # heavier checks on push (clippy + tests) +prek install -t pre-push # heavier checks on push (clippy + tests of the crates the push affects) ``` `prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), and the generated manifests (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`). +The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). + The manifest hooks only run when you change the dependency graph. They *rewrite* the generated file instead of only reporting that it is stale. When that happens, the commit stops so you can re-stage the refreshed file. The `THIRD_PARTY_NOTICES.txt` hook also needs the pinned generator. Without it, the hook skips and CI remains the gate: diff --git a/docs/testing.md b/docs/testing.md index 5497fac09f..dc203bd993 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -373,8 +373,59 @@ change that can't be confined to specific crates (the lockfile, the toolchain file, the workspace root manifest, CI config, or a doc an xtask test reads — `MANIFEST.md`, `CONTRIBUTING.md`, `docs/ci-hardware-testing.md`, `tests/e2e-cucumber/README.md`). On `main` -and in the merge queue the full workspace always runs. For local verification, keep using -`cargo test --workspace --all-targets` above — `affected` is a CI optimization. +and in the merge queue the full workspace always runs. For a full local verification, keep using +`cargo test --workspace --all-targets` above — `affected` is a CI optimization, and it also +narrows the local pre-push hooks (next section). + +### Pre-push hook scope + +The pre-push hooks `cargo-clippy`, `cargo-clippy-e2e` and `cargo-test` in +[.pre-commit-config.yaml](../.pre-commit-config.yaml) run through `cargo xtask pre-push-check` +([prepush.rs](../xtask/src/prepush.rs)), which applies the same selection to the range being +pushed (`PRE_COMMIT_FROM_REF..PRE_COMMIT_TO_REF`, the commits the remote does not have yet; +`origin/main...HEAD` when run by hand). Each step prints the scope it chose: + +- `clippy` runs `cargo clippy -p … --all-targets -- -D warnings` for the affected crates, + or `--workspace` when the selection is the whole workspace. +- `clippy-e2e` lints the e2e harness target only when `e2e-cucumber` is among the affected crates. +- `test` runs `cargo test -p … --all-targets` for the affected crates, never including + `e2e-cucumber` (its scenarios run through `cargo xtask e2e`), or + `--workspace --all-targets --exclude e2e-cucumber` for the whole workspace. + +The directory-based selection cannot see a crate's tests reading a file from outside the crate, +so the hook widens it: any Rust file also selects `xtask` (its contract tests scan every source +file), `README.md`, `docs/` and `skills/` also select `rocm` and `xtask` (the advised-commands +test reads them and `rocm` compiles in the assistant skill), and the e2e PowerShell fixture also +selects `rocm-core` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in +[prepush.rs](../xtask/src/prepush.rs), and a unit test scans the workspace for `include_str!`, +`include_bytes!` and manifest-relative reads that leave their crate and fails when one is not +covered, so a new cross-crate read cannot be added without a decision. + +A step that has nothing to check says so and exits successfully. A step whose affected crates +cannot be determined (the pushed range's base is not available locally, or `cargo metadata` +fails) runs over the whole workspace and prints why, so narrowing never replaces a check with +silence. Pushes that change no Rust file still skip the cargo hooks through their `types: [rust]` +filter, as before, including a lockfile-only push. + +The hook is an early warning, and CI is not narrowed everywhere. On pull requests the `clippy` +job lints the whole workspace and the `test` job (`Test (affected crates)`) runs only the crates +`cargo xtask affected` selects. The whole workspace's tests run in the `windows-debug` job on +every change, and in the merge queue and on `main`, where the Linux `test` job is forced to +`--workspace`. + +Known limits, unchanged by the narrowing: the steps check the working tree rather than the pushed +commit; under prek 0.4.5 a push of several refs, a tag or a deletion runs no hooks; and `-p` +builds can resolve Cargo features differently from `--workspace` ones. + +Preview what a push would check without compiling: + +```bash +cargo xtask pre-push-check test --dry-run --base origin/main +``` + +`--base` replaces the range's base; `clippy` and `clippy-e2e` take the same flags. The selection +logic is unit-tested in [prepush.rs](../xtask/src/prepush.rs); as dev tooling, with no change to +what the `rocm` binary does, it has no Gherkin scenario. The smoke path is the cross-platform local no-fallback acceptance surface. It uses an isolated config/data/cache root, does not install TheRock wheels, does diff --git a/xtask/src/affected.rs b/xtask/src/affected.rs index 7c09e1fadf..1d6cb1f041 100644 --- a/xtask/src/affected.rs +++ b/xtask/src/affected.rs @@ -28,7 +28,7 @@ use serde::Deserialize; /// Default base ref used when none is supplied on the command line. Matches /// [`crate::verify_commits`] so both commands diff against the same point. -const DEFAULT_BASE: &str = "origin/main"; +pub(crate) const DEFAULT_BASE: &str = "origin/main"; /// The workspace dependency graph, reduced to what selection needs: each /// member's source directory (workspace-relative, `/`-separated) and the @@ -311,9 +311,28 @@ fn build_graph(metadata: &Metadata) -> Graph { /// List files changed in `base...HEAD` (three-dot: relative to the merge base, /// the same "what this branch changed" diff layer one filters on). fn changed_files(base: &str) -> Result> { - let range = format!("{base}...HEAD"); - let output = Command::new("git") - .args(["diff", "--name-only", &range]) + changed_files_between(base, "HEAD") +} + +/// List files changed in `base...head`. +/// +/// `--no-renames` makes a moved file list both its old and its new path. +/// Without it git reports only the new path, so a file moved out of one crate +/// into another would select the destination crate and miss the one that lost +/// the file. +pub(crate) fn changed_files_between(base: &str, head: &str) -> Result> { + diff_name_only(None, base, head) +} + +/// [`changed_files_between`] in `dir` (the current directory when `None`). +fn diff_name_only(dir: Option<&std::path::Path>, base: &str, head: &str) -> Result> { + let range = format!("{base}...{head}"); + let mut git = Command::new("git"); + if let Some(dir) = dir { + git.arg("-C").arg(dir); + } + let output = git + .args(["diff", "--name-only", "--no-renames", &range]) .output() .context("failed to run `git diff`")?; if !output.status.success() { @@ -535,6 +554,51 @@ mod tests { assert_eq!(sel, Selection::Packages(want)); } + /// Run `git` in `dir` with a fixed identity and no signing, for the + /// throwaway repositories below. + fn git_in(dir: &std::path::Path, args: &[&str]) { + let status = Command::new("git") + .arg("-C") + .arg(dir) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .status() + .expect("run git"); + assert!(status.success(), "git {args:?} failed in {}", dir.display()); + } + + /// A file moved from one crate to another must select both crates. With git's + /// default rename detection only the new path is listed, the crate that lost + /// the file is not selected, and a push that breaks it is not checked. + #[test] + fn changed_files_between_lists_both_paths_of_a_rename() { + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path(); + git_in(root, &["init", "-q", "-b", "main"]); + std::fs::create_dir_all(root.join("crates/a/src")).expect("mkdir a"); + std::fs::create_dir_all(root.join("crates/b/src")).expect("mkdir b"); + std::fs::write(root.join("crates/a/src/moved.rs"), "pub fn f() {}\n").expect("write"); + git_in(root, &["add", "."]); + git_in(root, &["commit", "-q", "-m", "base"]); + git_in(root, &["checkout", "-q", "-b", "feature"]); + git_in( + root, + &["mv", "crates/a/src/moved.rs", "crates/b/src/moved.rs"], + ); + git_in(root, &["commit", "-q", "-m", "move"]); + + let mut files = diff_name_only(Some(root), "main", "feature").expect("git diff"); + files.sort(); + assert_eq!(files, ["crates/a/src/moved.rs", "crates/b/src/moved.rs"]); + } + #[test] fn cargo_args_rendering() { assert_eq!(Selection::Workspace.to_cargo_args(), "--workspace"); diff --git a/xtask/src/main.rs b/xtask/src/main.rs index fc2f1737ef..9ce4a86ebb 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -26,6 +26,7 @@ mod manifest; mod package; mod paths; mod powershell; +mod prepush; mod release_readiness; mod report_mailbox; mod report_rollup; @@ -106,6 +107,27 @@ pub(crate) enum Command { #[arg(long)] base: Option, }, + /// Run one cargo step of the pre-push hook over only the crates the push + /// affects: the changed crates plus their transitive dependents, selected + /// as `cargo xtask affected` does, from the range prek exports to the hook + /// (`PRE_COMMIT_FROM_REF`..`PRE_COMMIT_TO_REF`). + /// + /// The step runs over the whole workspace whenever the affected crates + /// cannot be determined, and prints why. It is skipped only when the + /// selection is known and has nothing for the step to check. CI always + /// checks the whole workspace. + PrePushCheck { + /// Which step to run. + #[arg(value_enum)] + check: prepush::Check, + /// Base ref for the `...HEAD` range, instead of the ref prek + /// provides. Without either it defaults to `origin/main`. + #[arg(long)] + base: Option, + /// Print the scope and the cargo command without running it. + #[arg(long)] + dry_run: bool, + }, /// Fail if any first-party crate has gained a normal/build dependency /// edge outside the allowlist pinned in `xtask/src/crate_edges.rs`. /// Dev-dependency edges are exempt (Cargo permits those to cycle). @@ -365,6 +387,11 @@ fn run() -> Result<()> { } => signing::verify(public_key.as_deref(), &input, &signature)?, Command::VerifyPinnedKeys => verify_pinned_keys::run()?, Command::Affected { base } => affected::run(base)?, + Command::PrePushCheck { + check, + base, + dry_run, + } => prepush::run(check, base, dry_run)?, Command::CheckCrateEdges => crate_edges::run()?, Command::Catalog { check } => catalog::run(check)?, Command::EvalFixtures { check } => eval_fixtures::run(check)?, diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs new file mode 100644 index 0000000000..dd6f4a7413 --- /dev/null +++ b/xtask/src/prepush.rs @@ -0,0 +1,1253 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! `cargo xtask pre-push-check`: the three cargo steps of the pre-push hook +//! (`clippy`, `clippy-e2e`, `test`), narrowed to the crates the push affects. +//! +//! Each step used to lint or test the whole workspace whatever the push +//! contained, so a one-file change in a leaf crate paid for every crate. The +//! selection is the one CI already uses ([`crate::affected`]): the crates whose +//! files changed plus every crate that transitively depends on them, with a +//! fall back to the whole workspace for anything that cannot be attributed to +//! one crate (the lockfile, the root manifest, the toolchain file, workflow +//! files, a doc an xtask test reads, an unrecognized path). On top of that the +//! hook widens for files that another crate's tests read from outside their own +//! crate directory ([`CROSS_CRATE_READS`]), which the directory-based selection +//! cannot see. CI stays the complete gate; this only makes the local early +//! warning cheaper. +//! +//! **Fail safe.** Whenever the affected set cannot be determined (`git diff` +//! fails because the base is not fetched, `cargo metadata` fails, ...) the step +//! runs over the whole workspace, as it did before, and says why. A step is +//! skipped only when the selection is known and contains nothing for that step +//! to check, and it prints the reason then too. +//! +//! The range comes from the refs prek exports to a pre-push hook: +//! `PRE_COMMIT_FROM_REF` is the commit the remote already has (the remote +//! branch tip for an update, the parent of the first unpushed commit for a new +//! branch) and `PRE_COMMIT_TO_REF` is what is being pushed. With neither, for a +//! manual run, it diffs `origin/main...HEAD` like `cargo xtask affected` does. +//! `--base` overrides the from-ref. +//! +//! As in [`crate::affected`], the decisions ([`push_from_env`], [`plan`], +//! [`hook_selection`], [`resolve_selection`]) are pure and unit-tested; running +//! git and cargo is confined to [`run`], and [`execute`] takes both as +//! parameters so its own behavior (the range it diffs, `--dry-run`, a failing +//! cargo) is tested too. + +use std::collections::BTreeSet; +use std::process::Command; + +use anyhow::{Context, Result, bail}; + +use crate::affected::{self, DEFAULT_BASE, Graph, Selection}; + +/// The e2e harness crate. `cargo test` excludes it (its scenarios run through +/// `cargo xtask e2e`), and its harness target is linted by its own step. +const E2E_CRATE: &str = "e2e-cucumber"; + +/// One cargo step of the pre-push hook. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub enum Check { + /// `cargo clippy --all-targets -- -D warnings` + Clippy, + /// `cargo clippy -p e2e-cucumber --test e2e -- -D warnings`: the harness + /// target that `--all-targets` leaves out of the step above. + ClippyE2e, + /// `cargo test --all-targets`, without the e2e harness. + Test, +} + +impl Check { + const fn label(self) -> &'static str { + match self { + Self::Clippy => "clippy", + Self::ClippyE2e => "clippy-e2e", + Self::Test => "test", + } + } +} + +/// The range being pushed: what the remote already has, and what is pushed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Push { + /// The remote's side of the range; `None` when prek gave none (a manual + /// run, or a brand-new remote), so the default base applies. + from: Option, + /// The commit being pushed. + to: String, +} + +impl Push { + fn base(&self) -> &str { + self.from.as_deref().unwrap_or(DEFAULT_BASE) + } +} + +/// Build the [`Push`] from the values of `PRE_COMMIT_FROM_REF` and +/// `PRE_COMMIT_TO_REF`. An unset, empty or all-zero from-ref (git's spelling of +/// "the remote has nothing") means no remote base; an empty to-ref means `HEAD`. +pub fn push_from_env(from: Option, to: Option) -> Push { + let from = from + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty() && !value.chars().all(|c| c == '0')); + let to = to + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "HEAD".to_string()); + Push { from, to } +} + +/// What a step does for a given selection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Plan { + /// Nothing for this step to check; the reason is printed. + Skip(String), + /// Run `cargo` with these arguments. + Cargo(Vec), +} + +/// Decide what `check` runs for `selection`. +/// +/// An empty package set is treated as the whole workspace: a bare +/// `cargo clippy --all-targets` in a virtual workspace lints every default +/// member anyway, and an empty set means the selection went wrong, not that +/// nothing changed. +pub fn plan(check: Check, selection: &Selection) -> Plan { + let packages = match selection { + Selection::Empty => { + return Plan::Skip("no Rust crate is affected by the pushed changes".to_string()); + } + Selection::Packages(packages) if !packages.is_empty() => Some(packages), + Selection::Workspace | Selection::Packages(_) => None, + }; + + match check { + Check::Clippy => { + let mut args = vec!["clippy".to_string()]; + args.extend(package_args(packages.map(|p| p.iter().map(String::as_str)))); + args.extend(["--all-targets", "--", "-D", "warnings"].map(String::from)); + Plan::Cargo(args) + } + Check::ClippyE2e => { + if packages.is_some_and(|p| !p.contains(E2E_CRATE)) { + return Plan::Skip(format!( + "{E2E_CRATE} is not among the affected crates, so its harness is unchanged" + )); + } + Plan::Cargo( + [ + "clippy", "-p", E2E_CRATE, "--test", "e2e", "--", "-D", "warnings", + ] + .map(String::from) + .to_vec(), + ) + } + Check::Test => { + let mut args = vec!["test".to_string()]; + match packages { + None => args.extend( + ["--workspace", "--all-targets", "--exclude", E2E_CRATE].map(String::from), + ), + Some(packages) => { + let testable = packages.iter().filter(|name| *name != E2E_CRATE); + let before = args.len(); + args.extend(package_args(Some(testable.map(String::as_str)))); + if args.len() == before { + return Plan::Skip(format!( + "the only affected crate is {E2E_CRATE}, which this step does not test" + )); + } + args.push("--all-targets".to_string()); + } + } + Plan::Cargo(args) + } + } +} + +/// `--workspace` for `None`, otherwise `-p ` for each package. +fn package_args<'a>(packages: Option>) -> Vec { + match packages { + None => vec!["--workspace".to_string()], + Some(names) => names + .flat_map(|name| ["-p".to_string(), name.to_string()]) + .collect(), + } +} + +/// Files a crate's tests (or `include_str!`) read from outside that crate's own +/// directory, with the crates that read them. [`affected::select`] attributes a +/// file to the crate whose directory holds it and treats Markdown and `docs/` +/// as inert, so without these entries a push that edits such a file would never +/// run the reader's tests: the old whole-workspace hook caught that, a narrowed +/// one would not. +/// +/// A pattern ending in `/` is a directory prefix, one starting with `*` is a +/// file-name suffix, anything else is an exact path. Files that no crate owns +/// and that are not Markdown (`install.sh`, `scripts/`, `.github/`, ...) need no +/// entry: `select` already sends them to the whole workspace. The guard +/// `every_cross_crate_read_is_covered_by_the_widening_rule` scans the workspace +/// for `include_str!`/`include_bytes!` and manifest-relative reads that leave +/// their crate and fails when one is not covered, so a new cross-crate read +/// cannot be missed silently. The e2e harness is not listed as a reader because +/// the `test` step never runs it. +/// +/// - `*.rs`: xtask's contract tests scan every Rust source file in the repo +/// (`ascii_output_contract`, `env_mutation_contract`) and the pinned-key test +/// reads `apps/rocm/src/therock/fetch.rs`. +/// - `README.md`, `docs/`, `skills/`: `rocm`'s `advised_commands` test checks +/// every `rocm ...` command shown there, `rocm` compiles in the assistant +/// skill, and xtask's tests read the pinned release key under `docs/keys` and +/// other docs. +/// - the e2e PowerShell fixture: `rocm-core`'s `fix` tests `include_str!` it +/// although it lives in the e2e crate. +pub(crate) const CROSS_CRATE_READS: &[(&str, &[&str])] = &[ + ("*.rs", &["xtask"]), + ("README.md", &["rocm", "xtask"]), + ("docs/", &["rocm", "xtask"]), + ("skills/", &["rocm", "xtask"]), + ("tests/e2e-cucumber/fixtures/", &["rocm-core"]), +]; + +fn read_pattern_matches(pattern: &str, file: &str) -> bool { + if let Some(suffix) = pattern.strip_prefix('*') { + file.ends_with(suffix) + } else if pattern.ends_with('/') { + file.starts_with(pattern) + } else { + file == pattern + } +} + +/// The hook's selection: [`affected::select`], widened with the reader crates of +/// every changed file in [`CROSS_CRATE_READS`] and their dependents. +pub fn hook_selection(changed_files: &[String], graph: &Graph) -> Selection { + hook_selection_with(changed_files, graph, CROSS_CRATE_READS) +} + +fn hook_selection_with( + changed_files: &[String], + graph: &Graph, + reads: &[(&str, &[&str])], +) -> Selection { + let base = affected::select(changed_files, graph); + if base == Selection::Workspace { + return base; + } + let readers: BTreeSet<&str> = reads + .iter() + .filter(|(pattern, _)| { + changed_files + .iter() + .any(|file| read_pattern_matches(pattern, file)) + }) + .flat_map(|(_, readers)| readers.iter().copied()) + .collect(); + if readers.is_empty() { + return base; + } + // A reader that is not a workspace member means the table is stale; run + // everything rather than guess. + let members: BTreeSet<&str> = graph + .crate_dirs + .iter() + .map(|(name, _)| name.as_str()) + .collect(); + if readers.iter().any(|reader| !members.contains(reader)) { + return Selection::Workspace; + } + let mut crates = match base { + Selection::Packages(packages) => packages, + Selection::Workspace | Selection::Empty => BTreeSet::new(), + }; + crates.extend(readers.into_iter().map(String::from)); + Selection::Packages(affected::reverse_closure(&crates, &graph.rev_deps)) +} + +/// Resolve the selection for the pushed changes, falling back to the whole +/// workspace (with the reason) when either input cannot be obtained. +pub fn resolve_selection( + load_graph: impl FnOnce() -> Result, + changed_files: impl FnOnce() -> Result>, +) -> (Selection, Option) { + let resolved = changed_files() + .context("could not list the changed files") + .and_then(|files| { + let graph = load_graph().context("could not read the workspace dependency graph")?; + Ok(hook_selection(&files, &graph)) + }); + match resolved { + Ok(selection) => (selection, None), + Err(error) => { + // git's usage text after the first line adds nothing to the reason. + let reason = format!("{error:#}"); + let first_line = reason.lines().next().unwrap_or_default().to_string(); + (Selection::Workspace, Some(first_line)) + } + } +} + +fn describe(selection: &Selection) -> String { + match selection { + Selection::Workspace => "the whole workspace".to_string(), + Selection::Empty => "no Rust crate".to_string(), + Selection::Packages(packages) => format!( + "{} affected crate(s): {}", + packages.len(), + packages + .iter() + .map(String::as_str) + .collect::>() + .join(", ") + ), + } +} + +/// The line printed before a step, naming its scope and why. +fn banner(check: Check, push: &Push, selection: &Selection, fallback: Option<&str>) -> String { + let range = format!("{}...{}", push.base(), push.to); + match fallback { + Some(reason) => format!( + "pre-push-check {}: could not determine the affected crates ({reason}); \ + checking the whole workspace", + check.label() + ), + None => format!( + "pre-push-check {}: {} for the changes in {range}; CI's Windows job, merge queue \ + and main still check the whole workspace", + check.label(), + describe(selection) + ), + } +} + +/// The refs prek exports to a pre-push hook. +const FROM_REF_VAR: &str = "PRE_COMMIT_FROM_REF"; +const TO_REF_VAR: &str = "PRE_COMMIT_TO_REF"; + +/// The pushed range as prek exported it; either side may be absent. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Env { + from_ref: Option, + to_ref: Option, +} + +impl Env { + /// Read the two refs through `lookup`, so tests need no process environment. + pub fn from_lookup(lookup: impl Fn(&str) -> Option) -> Self { + Self { + from_ref: lookup(FROM_REF_VAR), + to_ref: lookup(TO_REF_VAR), + } + } +} + +/// Run one step: resolve the range (an explicit `base` wins over prek's +/// from-ref), pick the affected crates, print the scope, and hand the planned +/// cargo arguments to `run_cargo` unless the step is skipped or `dry_run`. +/// +/// `load_graph`, `changed_files` (given the range's base and tip) and +/// `run_cargo` are parameters so the behavior around them is unit-tested. +pub fn execute( + check: Check, + base: Option, + dry_run: bool, + env: Env, + load_graph: impl FnOnce() -> Result, + changed_files: impl FnOnce(&str, &str) -> Result>, + mut run_cargo: impl FnMut(&[String]) -> Result<()>, +) -> Result<()> { + let push = push_from_env(base.or(env.from_ref), env.to_ref); + let (selection, fallback) = + resolve_selection(load_graph, || changed_files(push.base(), &push.to)); + println!("{}", banner(check, &push, &selection, fallback.as_deref())); + + let args = match plan(check, &selection) { + Plan::Skip(reason) => { + println!("pre-push-check {}: skipped, {reason}", check.label()); + return Ok(()); + } + Plan::Cargo(args) => args, + }; + + println!("pre-push-check {}: cargo {}", check.label(), args.join(" ")); + if dry_run { + return Ok(()); + } + run_cargo(&args) +} + +/// Run `cargo` with `args`, failing when it does. +fn spawn_cargo(args: &[String]) -> Result<()> { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let status = Command::new(cargo) + .args(args) + .status() + .context("failed to run cargo")?; + if !status.success() { + bail!("`cargo {}` failed ({status})", args.join(" ")); + } + Ok(()) +} + +/// Entry point for the `pre-push-check` subcommand. +pub fn run(check: Check, base: Option, dry_run: bool) -> Result<()> { + execute( + check, + base, + dry_run, + Env::from_lookup(|name| std::env::var(name).ok()), + affected::load_graph, + affected::changed_files_between, + spawn_cargo, + ) +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, BTreeSet}; + + use super::*; + + fn packages(names: &[&str]) -> Selection { + Selection::Packages(names.iter().map(|name| (*name).to_string()).collect()) + } + + fn args(plan: &Plan) -> String { + match plan { + Plan::Cargo(args) => args.join(" "), + Plan::Skip(reason) => panic!("expected a cargo run, got a skip: {reason}"), + } + } + + fn skip_reason(plan: &Plan) -> &str { + match plan { + Plan::Skip(reason) => reason, + Plan::Cargo(args) => panic!("expected a skip, got cargo {}", args.join(" ")), + } + } + + #[test] + fn a_real_from_ref_is_the_base() { + let push = push_from_env(Some("abc123".into()), Some("def456".into())); + assert_eq!(push.base(), "abc123"); + assert_eq!(push.to, "def456"); + } + + #[test] + fn no_remote_base_means_the_default_base() { + for from in [ + None, + Some(String::new()), + Some(" ".into()), + Some("0".repeat(40)), + ] { + let push = push_from_env(from.clone(), None); + assert_eq!(push.base(), DEFAULT_BASE, "from-ref {from:?}"); + assert_eq!(push.to, "HEAD"); + } + } + + #[test] + fn clippy_checks_the_whole_workspace_or_the_selected_crates() { + assert_eq!( + args(&plan(Check::Clippy, &Selection::Workspace)), + "clippy --workspace --all-targets -- -D warnings" + ); + assert_eq!( + args(&plan(Check::Clippy, &packages(&["rocm", "rocm-core"]))), + "clippy -p rocm -p rocm-core --all-targets -- -D warnings" + ); + } + + #[test] + fn test_excludes_the_e2e_harness_either_way() { + assert_eq!( + args(&plan(Check::Test, &Selection::Workspace)), + "test --workspace --all-targets --exclude e2e-cucumber" + ); + assert_eq!( + args(&plan( + Check::Test, + &packages(&["e2e-cucumber", "rocm-core"]) + )), + "test -p rocm-core --all-targets" + ); + } + + #[test] + fn test_is_skipped_with_a_reason_when_only_the_e2e_harness_is_affected() { + let plan = plan(Check::Test, &packages(&["e2e-cucumber"])); + assert!(skip_reason(&plan).contains("e2e-cucumber")); + } + + #[test] + fn the_e2e_harness_is_linted_only_when_it_is_affected() { + let want = "clippy -p e2e-cucumber --test e2e -- -D warnings"; + assert_eq!(args(&plan(Check::ClippyE2e, &Selection::Workspace)), want); + assert_eq!( + args(&plan( + Check::ClippyE2e, + &packages(&["e2e-cucumber", "xtask"]) + )), + want + ); + let plan = plan(Check::ClippyE2e, &packages(&["xtask"])); + assert!(skip_reason(&plan).contains("not among the affected crates")); + } + + #[test] + fn every_step_is_skipped_with_a_reason_when_no_crate_is_affected() { + for check in [Check::Clippy, Check::ClippyE2e, Check::Test] { + let plan = plan(check, &Selection::Empty); + assert!(skip_reason(&plan).contains("no Rust crate"), "{check:?}"); + } + } + + #[test] + fn an_empty_package_set_is_the_whole_workspace_not_a_bare_cargo_call() { + let empty = Selection::Packages(BTreeSet::new()); + assert_eq!( + args(&plan(Check::Clippy, &empty)), + "clippy --workspace --all-targets -- -D warnings" + ); + assert_eq!( + args(&plan(Check::Test, &empty)), + "test --workspace --all-targets --exclude e2e-cucumber" + ); + } + + fn graph() -> Graph { + let mut rev_deps = BTreeMap::new(); + rev_deps.insert("core".to_string(), vec!["app".to_string()]); + rev_deps.insert("app".to_string(), vec![]); + rev_deps.insert("xtask".to_string(), vec![]); + Graph { + crate_dirs: vec![ + ("core".into(), "crates/core".into()), + ("app".into(), "apps/app".into()), + ("xtask".into(), "xtask".into()), + ], + rev_deps, + } + } + + fn files(paths: &[&str]) -> Result> { + Ok(paths.iter().map(|path| (*path).to_string()).collect()) + } + + #[test] + fn a_known_selection_comes_from_the_changed_files_and_has_no_fallback_reason() { + let (selection, fallback) = + resolve_selection(|| Ok(graph()), || files(&["xtask/src/main.rs"])); + assert_eq!(selection, packages(&["xtask"])); + assert_eq!(fallback, None); + + // Any Rust file also selects xtask, whose contract tests scan them all. + let (selection, _) = + resolve_selection(|| Ok(graph()), || files(&["crates/core/src/lib.rs"])); + assert_eq!(selection, packages(&["core", "app", "xtask"])); + + let (selection, _) = resolve_selection(|| Ok(graph()), || files(&["CHANGELOG.md"])); + assert_eq!(selection, Selection::Empty); + } + + #[test] + fn a_failed_diff_falls_back_to_the_whole_workspace_and_says_why() { + let (selection, fallback) = resolve_selection( + || Ok(graph()), + || bail!("`git diff origin/main...HEAD` failed: unknown revision"), + ); + assert_eq!(selection, Selection::Workspace); + let reason = fallback.expect("a fallback reason"); + assert!( + reason.contains("could not list the changed files"), + "{reason}" + ); + assert!(reason.contains("unknown revision"), "{reason}"); + } + + #[test] + fn the_fallback_reason_is_one_line() { + let (_, fallback) = resolve_selection( + || Ok(graph()), + || bail!("`git diff x...HEAD` failed: unknown revision\nUse '--' to separate paths"), + ); + let reason = fallback.expect("a fallback reason"); + assert!(!reason.contains('\n'), "{reason}"); + assert!(reason.contains("unknown revision"), "{reason}"); + } + + #[test] + fn a_failed_graph_load_falls_back_to_the_whole_workspace_and_says_why() { + let (selection, fallback) = resolve_selection( + || bail!("`cargo metadata` failed"), + || files(&["xtask/src/main.rs"]), + ); + assert_eq!(selection, Selection::Workspace); + let reason = fallback.expect("a fallback reason"); + assert!(reason.contains("dependency graph"), "{reason}"); + } + + #[test] + fn a_shared_input_still_selects_the_whole_workspace() { + for path in [ + "Cargo.lock", + "Cargo.toml", + "rust-toolchain.toml", + ".github/workflows/ci.yml", + ] { + let (selection, fallback) = + resolve_selection(|| Ok(graph()), || files(&["xtask/src/main.rs", path])); + assert_eq!(selection, Selection::Workspace, "{path}"); + assert_eq!(fallback, None, "{path}"); + } + } + + #[test] + fn the_banner_names_the_scope_and_where_ci_still_checks_everything() { + let push = push_from_env(Some("abc123".into()), Some("def456".into())); + let line = banner(Check::Test, &push, &packages(&["rocm-core", "xtask"]), None); + assert!( + line.contains("2 affected crate(s): rocm-core, xtask"), + "{line}" + ); + assert!(line.contains("abc123...def456"), "{line}"); + assert!( + line.contains("Windows job, merge queue and main still check the whole workspace"), + "{line}" + ); + } + + #[test] + fn the_banner_of_a_fallback_names_the_reason_not_a_narrowed_scope() { + let push = push_from_env(None, None); + let line = banner( + Check::Clippy, + &push, + &Selection::Workspace, + Some("unknown revision"), + ); + assert!( + line.contains("could not determine the affected crates (unknown revision)"), + "{line}" + ); + assert!(line.contains("checking the whole workspace"), "{line}"); + } + + // ---- files read across crate boundaries ---- + + fn reader_graph() -> Graph { + let mut rev_deps = BTreeMap::new(); + rev_deps.insert( + "rocm-core".to_string(), + vec!["rocm".to_string(), "xtask".to_string()], + ); + for name in ["rocm", "xtask", "e2e-cucumber"] { + rev_deps.insert(name.to_string(), vec![]); + } + Graph { + crate_dirs: vec![ + ("rocm-core".into(), "crates/rocm-core".into()), + ("rocm".into(), "apps/rocm".into()), + ("xtask".into(), "xtask".into()), + ("e2e-cucumber".into(), "tests/e2e-cucumber".into()), + ], + rev_deps, + } + } + + fn selected(paths: &[&str]) -> Selection { + let changed: Vec = paths.iter().map(|path| (*path).to_string()).collect(); + hook_selection(&changed, &reader_graph()) + } + + fn selected_names(paths: &[&str]) -> BTreeSet { + match selected(paths) { + Selection::Packages(names) => names, + other => panic!("expected packages for {paths:?}, got {other:?}"), + } + } + + const FIXTURE: &str = "tests/e2e-cucumber/fixtures/powershell_text_rule_cases.txt"; + + /// The directory-based selection attributes the fixture and the step file to + /// the e2e crate and nothing else, so the `rocm-core` tests that + /// `include_str!` the fixture never ran and the `test` step skipped. + #[test] + fn a_fixture_edit_with_an_e2e_step_still_tests_the_crate_that_includes_it() { + let changed = [FIXTURE, "tests/e2e-cucumber/tests/e2e/diagnose_steps.rs"]; + assert!(selected_names(&changed).contains("rocm-core")); + let Plan::Cargo(args) = plan(Check::Test, &selected(&changed)) else { + panic!("the test step must run, not skip"); + }; + assert!(args.join(" ").contains("-p rocm-core"), "{args:?}"); + } + + #[test] + fn the_fixture_alone_selects_its_reader() { + assert!(selected_names(&[FIXTURE]).contains("rocm-core")); + } + + #[test] + fn docs_and_skills_select_the_crates_whose_tests_read_them() { + for path in [ + "README.md", + "docs/testing.md", + "skills/rocm-cli-assistant/SKILL.md", + ] { + let names = selected_names(&[path]); + assert!(names.contains("rocm"), "{path}: {names:?}"); + assert!(names.contains("xtask"), "{path}: {names:?}"); + assert!(!names.contains("rocm-core"), "{path}: {names:?}"); + } + } + + #[test] + fn any_rust_file_selects_xtask_for_its_repo_wide_source_scans() { + assert!(selected_names(&["apps/rocm/src/main.rs"]).contains("xtask")); + } + + #[test] + fn markdown_nothing_reads_still_selects_nothing() { + assert_eq!(selected(&["CHANGELOG.md"]), Selection::Empty); + } + + #[test] + fn the_whole_workspace_stays_the_whole_workspace() { + assert_eq!( + selected(&["Cargo.lock", "apps/rocm/src/main.rs"]), + Selection::Workspace + ); + } + + #[test] + fn a_reader_that_is_not_a_workspace_member_runs_everything() { + let changed = ["docs/testing.md".to_string()]; + let table: &[(&str, &[&str])] = &[("docs/", &["a-crate-that-was-renamed"])]; + assert_eq!( + hook_selection_with(&changed, &reader_graph(), table), + Selection::Workspace + ); + } + + #[test] + fn read_patterns_match_by_directory_suffix_or_exact_path() { + assert!(read_pattern_matches("docs/", "docs/keys/a.pem")); + assert!(!read_pattern_matches("docs/", "documents/a.md")); + assert!(read_pattern_matches("*.rs", "xtask/src/main.rs")); + assert!(!read_pattern_matches("*.rs", "xtask/Cargo.toml")); + assert!(read_pattern_matches("README.md", "README.md")); + assert!(!read_pattern_matches("README.md", "docs/README.md")); + } + + // ---- execute: the range, --dry-run and cargo's result ---- + + type Ranges = Vec<(String, String)>; + + fn env(from: Option<&str>, to: Option<&str>) -> Env { + Env { + from_ref: from.map(String::from), + to_ref: to.map(String::from), + } + } + + /// Run a step against canned inputs; `files` of `None` makes the diff fail. + /// Returns the result, every `(base, tip)` the diff was asked for, and the + /// argument lines handed to cargo. + fn run_step( + check: Check, + base: Option<&str>, + dry_run: bool, + env: Env, + files: Option<&[&str]>, + cargo_fails: bool, + ) -> (Result<()>, Ranges, Vec) { + let ranges = std::cell::RefCell::new(Vec::new()); + let cargo = std::cell::RefCell::new(Vec::new()); + let result = execute( + check, + base.map(String::from), + dry_run, + env, + || Ok(graph()), + |base, tip| { + ranges + .borrow_mut() + .push((base.to_string(), tip.to_string())); + match files { + Some(paths) => Ok(paths.iter().map(|path| (*path).to_string()).collect()), + None => bail!("`git diff` failed: unknown revision"), + } + }, + |args| { + cargo.borrow_mut().push(args.join(" ")); + if cargo_fails { + bail!("`cargo` failed (exit status: 101)") + } + Ok(()) + }, + ); + (result, ranges.into_inner(), cargo.into_inner()) + } + + const XTASK_ONLY: Option<&[&str]> = Some(&["xtask/src/main.rs"]); + + #[test] + fn a_passing_cargo_run_is_ok_and_ran_the_planned_arguments() { + let (result, _, cargo) = + run_step(Check::Test, None, false, env(None, None), XTASK_ONLY, false); + result.expect("a passing step"); + assert_eq!(cargo, ["test -p xtask --all-targets"]); + } + + #[test] + fn a_failing_cargo_run_fails_the_step() { + let (result, _, cargo) = + run_step(Check::Test, None, false, env(None, None), XTASK_ONLY, true); + let error = result.expect_err("a failing cargo must fail the hook"); + assert!(format!("{error:#}").contains("101"), "{error:#}"); + assert_eq!(cargo.len(), 1); + } + + #[test] + fn the_range_comes_from_the_exported_refs() { + let (_, ranges, _) = run_step( + Check::Test, + None, + true, + env(Some("envfrom"), Some("envto")), + XTASK_ONLY, + false, + ); + assert_eq!(ranges, [("envfrom".to_string(), "envto".to_string())]); + } + + #[test] + fn an_explicit_base_replaces_the_exported_from_ref_but_not_the_tip() { + let (_, ranges, _) = run_step( + Check::Test, + Some("override"), + true, + env(Some("envfrom"), Some("envto")), + XTASK_ONLY, + false, + ); + assert_eq!(ranges, [("override".to_string(), "envto".to_string())]); + } + + #[test] + fn without_refs_the_range_is_the_default_base_to_head() { + let (_, ranges, _) = run_step(Check::Test, None, true, env(None, None), XTASK_ONLY, false); + assert_eq!(ranges, [(DEFAULT_BASE.to_string(), "HEAD".to_string())]); + let (_, ranges, _) = run_step( + Check::Test, + None, + true, + env(Some("envfrom"), None), + XTASK_ONLY, + false, + ); + assert_eq!(ranges, [("envfrom".to_string(), "HEAD".to_string())]); + } + + #[test] + fn a_dry_run_selects_but_never_runs_cargo() { + let (result, ranges, cargo) = + run_step(Check::Test, None, true, env(None, None), XTASK_ONLY, false); + result.expect("a dry run"); + assert_eq!(ranges.len(), 1, "the selection is still computed"); + assert!(cargo.is_empty(), "{cargo:?}"); + } + + #[test] + fn a_skipped_step_never_runs_cargo() { + let (result, _, cargo) = run_step( + Check::Test, + None, + false, + env(None, None), + Some(&["CHANGELOG.md"]), + true, + ); + result.expect("a skipped step succeeds even if cargo would fail"); + assert!(cargo.is_empty(), "{cargo:?}"); + } + + #[test] + fn an_undeterminable_range_runs_the_whole_workspace() { + let (result, _, cargo) = run_step(Check::Test, None, false, env(None, None), None, false); + result.expect("the fallback run"); + assert_eq!( + cargo, + ["test --workspace --all-targets --exclude e2e-cucumber"] + ); + } + + #[test] + fn the_exported_refs_are_read_from_prek_s_variable_names() { + let lookup = |name: &str| match name { + "PRE_COMMIT_FROM_REF" => Some("remote-tip".to_string()), + "PRE_COMMIT_TO_REF" => Some("pushed-tip".to_string()), + _ => None, + }; + assert_eq!( + Env::from_lookup(lookup), + env(Some("remote-tip"), Some("pushed-tip")) + ); + assert_eq!(Env::from_lookup(|_| None), Env::default()); + } + + // ---- drift guard: every cross-crate read must be covered ---- + + use std::path::{Path, PathBuf}; + + /// A read that leaves the crate: a repo-relative path, or the repo root + /// itself when the code derives it and joins the rest elsewhere. + #[derive(Debug, PartialEq, Eq)] + enum Read { + Root, + Path(String), + } + + /// The text of the sole string literal in `args`, if that is all it holds. + fn sole_literal(args: &str) -> Option<&str> { + let inner = args.trim().strip_prefix('"')?; + let end = inner.find('"')?; + inner[end + 1..].trim().is_empty().then_some(&inner[..end]) + } + + /// Index of the `)` that closes the `(` at `open`, skipping string literals. + fn matching_paren(text: &str, open: usize) -> Option { + let bytes = text.as_bytes(); + let mut depth = 0usize; + let mut i = open; + while i < bytes.len() { + match bytes[i] { + b'"' => { + i += 1; + while i < bytes.len() && bytes[i] != b'"' { + i += if bytes[i] == b'\\' { 2 } else { 1 }; + } + } + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + _ => {} + } + i += 1; + } + None + } + + /// Lexically resolve `..` and `.`; `None` when the path leaves the root. + fn normalize(path: &str) -> Option { + let mut parts: Vec<&str> = Vec::new(); + for part in path.split('/') { + match part { + "" | "." => {} + ".." => { + parts.pop()?; + } + other => parts.push(other), + } + } + Some(parts.join("/")) + } + + /// Repo-relative targets of the `include_str!`/`include_bytes!` literals in + /// the file `file_rel`. + fn include_targets(file_rel: &str, text: &str) -> Vec { + let dir = file_rel.rsplit_once('/').map_or("", |(dir, _)| dir); + let mut targets = Vec::new(); + for macro_name in ["include_str", "include_bytes"] { + let needle = format!("{macro_name}!("); + for (at, _) in text.match_indices(&needle) { + let open = at + needle.len() - 1; + let Some(close) = matching_paren(text, open) else { + continue; + }; + if let Some(literal) = sole_literal(&text[open + 1..close]) { + targets.extend(normalize(&format!("{dir}/{literal}"))); + } + } + } + targets + } + + /// The reads that start at `env!("CARGO_MANIFEST_DIR")` and leave the crate + /// directory (through `.parent()`, `.ancestors()` or a `..` segment). Scratch + /// directories under `.rocm-work` and `target` are not reads of repo files. + fn manifest_reads(text: &str) -> Vec { + let needle = format!("env!(\"{}\")", "CARGO_MANIFEST_DIR"); + let bytes = text.as_bytes(); + let mut reads = Vec::new(); + for (at, _) in text.match_indices(&needle) { + let mut i = at + needle.len(); + let (mut escapes, mut segments) = (false, Vec::new()); + loop { + while i < bytes.len() && (bytes[i].is_ascii_whitespace() || bytes[i] == b')') { + i += 1; + } + if i >= bytes.len() || bytes[i] != b'.' { + break; + } + i += 1; + let name_start = i; + while i < bytes.len() && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') { + i += 1; + } + let name = &text[name_start..i]; + if i >= bytes.len() || bytes[i] != b'(' { + break; + } + let Some(close) = matching_paren(text, i) else { + break; + }; + let literal = sole_literal(&text[i + 1..close]); + match (name, literal) { + ("parent" | "ancestors", _) => escapes = true, + ("join", Some(path)) => { + for part in path.split('/') { + match part { + ".." => escapes = true, + "" | "." => {} + other => segments.push(other.to_string()), + } + } + } + _ => {} + } + i = close + 1; + } + if !escapes + || matches!( + segments.first().map(String::as_str), + Some(".rocm-work" | "target") + ) + { + continue; + } + reads.push(if segments.is_empty() { + Read::Root + } else { + Read::Path(segments.join("/")) + }); + } + reads + } + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + fn walk(dir: &Path, files: &mut Vec) { + const SKIPPED: &[&str] = &[ + "target", + ".git", + ".rocm-work", + "node_modules", + "third_party", + ]; + for entry in std::fs::read_dir(dir).expect("read dir").flatten() { + let path = entry.path(); + if path.is_dir() { + if !SKIPPED.contains(&entry.file_name().to_str().unwrap_or_default()) { + walk(&path, files); + } + } else { + files.push(path); + } + } + } + + fn relative(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .expect("under the repo root") + .to_string_lossy() + .replace('\\', "/") + } + + /// The workspace crates, found from their manifests on disk. + fn crates_on_disk(root: &Path, files: &[PathBuf]) -> Vec<(String, String)> { + files + .iter() + .filter(|path| path.file_name().is_some_and(|name| name == "Cargo.toml")) + .filter_map(|path| { + let manifest: toml::Value = + toml::from_str(&std::fs::read_to_string(path).ok()?).ok()?; + let name = manifest.get("package")?.get("name")?.as_str()?.to_string(); + let dir = relative(root, path.parent()?); + (!dir.is_empty()).then_some((name, dir)) + }) + .collect() + } + + /// Fails when a crate reads a file outside its own directory that the hook's + /// selection would not send to that crate's tests, or when a crate derives + /// the repo root without being declared as a reader. Add the path to + /// [`CROSS_CRATE_READS`] (or declare the crate) when this fails. + #[test] + fn every_cross_crate_read_is_covered_by_the_widening_rule() { + let root = repo_root(); + let mut files = Vec::new(); + walk(&root, &mut files); + let crate_dirs = crates_on_disk(&root, &files); + assert!( + crate_dirs.iter().any(|(name, _)| name == "xtask"), + "found no crates under {}", + root.display() + ); + let graph = Graph { + rev_deps: crate_dirs + .iter() + .map(|(name, _)| (name.clone(), Vec::new())) + .collect(), + crate_dirs, + }; + + let mut problems = Vec::new(); + let mut root_readers = BTreeSet::new(); + let mut checked = 0usize; + let mut check = |reader: &str, source: &str, target: &str| { + checked += 1; + // `test` never runs the e2e crate, so nothing selects it for its reads. + if reader == E2E_CRATE + || affected::owning_crate(target, &graph.crate_dirs) == Some(reader) + { + return; + } + let covered = match hook_selection(&[target.to_string()], &graph) { + Selection::Workspace => true, + Selection::Packages(names) => names.contains(reader), + Selection::Empty => false, + }; + if !covered { + problems.push(format!("{source} ({reader}) reads {target}")); + } + }; + + for path in files + .iter() + .filter(|path| path.extension().is_some_and(|ext| ext == "rs")) + { + let source = relative(&root, path); + let Some(reader) = affected::owning_crate(&source, &graph.crate_dirs) else { + continue; + }; + let Ok(text) = std::fs::read_to_string(path) else { + continue; + }; + for target in include_targets(&source, &text) { + check(reader, &source, &target); + } + for read in manifest_reads(&text) { + match read { + Read::Root => { + root_readers.insert(reader.to_string()); + } + Read::Path(target) => check(reader, &source, &target), + } + } + } + + // A scanner that finds nothing would pass vacuously. xtask's tests derive + // the repo root, and crates include files from outside themselves. + assert!( + root_readers.contains("xtask"), + "the manifest scanner found nothing" + ); + assert!(checked > 0, "the include scanner found nothing"); + + let declared: BTreeSet<&str> = CROSS_CRATE_READS + .iter() + .flat_map(|(_, readers)| readers.iter().copied()) + .collect(); + for reader in root_readers { + if reader != E2E_CRATE && !declared.contains(reader.as_str()) { + problems.push(format!( + "crate {reader} derives the repo root but is not a reader in CROSS_CRATE_READS" + )); + } + } + assert!( + problems.is_empty(), + "reads the narrowed hook would not run the reader's tests for:\n{}", + problems.join("\n") + ); + } + + #[test] + fn the_scanner_finds_includes_that_leave_the_crate() { + let include = |name: &str, path: &str| format!("let a = {name}!(\"{path}\");"); + let text = format!( + "{}\n{}\n{}", + include("include_str", "../../../tests/e2e-cucumber/fixtures/a.txt"), + include("include_bytes", "../data.bin"), + include("include_str", "../../../../../escapes.txt"), + ); + assert_eq!( + include_targets("crates/rocm-core/src/fix.rs", &text), + [ + "tests/e2e-cucumber/fixtures/a.txt", + "crates/rocm-core/data.bin" + ] + ); + } + + #[test] + fn the_scanner_follows_a_manifest_relative_chain_out_of_the_crate() { + let manifest = format!("env!(\"{}\")", "CARGO_MANIFEST_DIR"); + let chain = |tail: &str| format!("fn p() -> PathBuf {{ Path::new({manifest}){tail} }}"); + assert_eq!( + manifest_reads(&chain( + ".join(\"..\").join(\"..\").join(\"skills\").join(\"d\").join(\"f.md\")" + )), + [Read::Path("skills/d/f.md".to_string())] + ); + assert_eq!( + manifest_reads(&chain(".join(\"../..\").join(\"docs\")")), + [Read::Path("docs".to_string())] + ); + assert_eq!( + manifest_reads(&chain(".parent().expect(\"root\")")), + [Read::Root] + ); + assert_eq!( + manifest_reads(&chain(".ancestors().nth(2).expect(\"root\")")), + [Read::Root] + ); + } + + #[test] + fn the_scanner_ignores_scratch_dirs_and_reads_inside_the_crate() { + let manifest = format!("env!(\"{}\")", "CARGO_MANIFEST_DIR"); + let chain = |tail: &str| format!("fn p() -> PathBuf {{ PathBuf::from({manifest}){tail} }}"); + assert!( + manifest_reads(&chain( + ".join(\"..\").join(\"..\").join(\".rocm-work\").join(\"t\")" + )) + .is_empty() + ); + assert!(manifest_reads(&chain(".join(\"target\").join(\"x\")")).is_empty()); + assert!(manifest_reads(&chain(".join(\"features\")")).is_empty()); + } + + #[test] + fn the_scanner_ignores_a_join_argument_that_is_not_a_plain_literal() { + let manifest = format!("env!(\"{}\")", "CARGO_MANIFEST_DIR"); + let text = format!( + "let p = PathBuf::from({manifest}).parent().unwrap().join(format!(\"x-{{}}\", id));" + ); + assert_eq!(manifest_reads(&text), [Read::Root]); + } +} From 90a239c4e950a44087b576840c7b73ea3da27819 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 14:23:57 +0000 Subject: [PATCH 02/11] dev: recognise raw-string includes and skip nested checkouts in the cross-crate read guard The drift guard that keeps the pre-push selection in step with crate tests reading files outside their crate only recognised plain `"..."` literals, so `include_str!(r#"../../../README.md"#)` passed it even though a README edit would not select the reader crate. It now reads raw literals too. It also walked every directory, including nested checkouts such as `.claude/worktrees/...`, so a rocm-core fixture include in another branch's sources was reported as an uncovered read of this tree and could fail `cargo test -p xtask`. Like the other xtask source scans, it now skips every dot-directory. Each change has a test that fails when it is reverted. Signed-off-by: Jussi Elo --- xtask/src/prepush.rs | 74 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index dd6f4a7413..92f884b66e 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -913,10 +913,22 @@ mod tests { } /// The text of the sole string literal in `args`, if that is all it holds. + /// Plain (`"..."`) and raw (`r"..."`, `r#"..."#`) literals are recognised. fn sole_literal(args: &str) -> Option<&str> { - let inner = args.trim().strip_prefix('"')?; - let end = inner.find('"')?; - inner[end + 1..].trim().is_empty().then_some(&inner[..end]) + let args = args.trim(); + if let Some(inner) = args.strip_prefix('"') { + let end = inner.find('"')?; + return inner[end + 1..].trim().is_empty().then_some(&inner[..end]); + } + let raw = args.strip_prefix('r')?; + let hashes = raw.len() - raw.trim_start_matches('#').len(); + let body = raw[hashes..].strip_prefix('"')?; + let closing = format!("\"{}", "#".repeat(hashes)); + let end = body.find(&closing)?; + body[end + closing.len()..] + .trim() + .is_empty() + .then_some(&body[..end]) } /// Index of the `)` that closes the `(` at `open`, skipping string literals. @@ -1050,18 +1062,17 @@ mod tests { .to_path_buf() } + /// Collects the files under `dir`, skipping build output, vendored trees and + /// every dot-directory (`.git`, `.rocm-work`, and nested checkouts such as + /// `.claude/worktrees/...`, whose sources belong to another branch). fn walk(dir: &Path, files: &mut Vec) { - const SKIPPED: &[&str] = &[ - "target", - ".git", - ".rocm-work", - "node_modules", - "third_party", - ]; + const SKIPPED: &[&str] = &["target", "node_modules", "third_party"]; for entry in std::fs::read_dir(dir).expect("read dir").flatten() { let path = entry.path(); if path.is_dir() { - if !SKIPPED.contains(&entry.file_name().to_str().unwrap_or_default()) { + let name = entry.file_name(); + let name = name.to_str().unwrap_or_default(); + if !name.starts_with('.') && !SKIPPED.contains(&name) { walk(&path, files); } } else { @@ -1092,6 +1103,47 @@ mod tests { .collect() } + /// A raw-string include is a cross-crate read like any other; the scanner + /// must not skip it. + #[test] + fn include_targets_reads_plain_and_raw_string_literals() { + let file = "crates/reader/src/lib.rs"; + for text in [ + r#"include_str!("../../../README.md")"#, + r##"include_str!(r#"../../../README.md"#)"##, + r###"include_bytes!(r##"../../../README.md"##)"###, + r#"include_str!(r"../../../README.md")"#, + ] { + assert_eq!( + include_targets(file, text), + vec!["README.md".to_string()], + "{text}" + ); + } + assert!(include_targets(file, r#"include_str!(concat!("a", "b"))"#).is_empty()); + } + + /// A nested checkout (`.claude/worktrees/...`) holds another branch's + /// sources; scanning it reports reads that are not this tree's. + #[test] + fn walk_skips_dot_directories_and_build_output() { + let tmp = tempfile::tempdir().expect("temp dir"); + for file in [ + "crates/a/src/lib.rs", + ".claude/worktrees/w/crates/b/src/lib.rs", + ".git/hooks/x.rs", + "target/debug/y.rs", + ] { + let path = tmp.path().join(file); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(&path, "").expect("write"); + } + let mut files = Vec::new(); + walk(tmp.path(), &mut files); + let found: Vec = files.iter().map(|p| relative(tmp.path(), p)).collect(); + assert_eq!(found, vec!["crates/a/src/lib.rs".to_string()]); + } + /// Fails when a crate reads a file outside its own directory that the hook's /// selection would not send to that crate's tests, or when a crate derives /// the repo root without being declared as a reader. Add the path to From 73bfacc79ea07e0e5bf9448a6fe49bb4b9c28f27 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 14:38:28 +0000 Subject: [PATCH 03/11] dev: keep the raw-string include test from looking like an embedded data file The test added for raw-string includes spelled out `include_str!("../../../README.md")` in its source, which the ASCII output contract counts as a data file compiled into a binary, so `every_embedded_data_file_is_ascii` failed on the README's non-ASCII lines in the Linux, Windows and coverage jobs. The test now builds those arguments with `format!`, as the neighbouring scanner test does. Signed-off-by: Jussi Elo --- xtask/src/prepush.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 92f884b66e..862273bd64 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -1107,20 +1107,24 @@ mod tests { /// must not skip it. #[test] fn include_targets_reads_plain_and_raw_string_literals() { + // Built with `format!` so this file never contains a literal embed that + // `ascii_output_contract` would count as a data file compiled into a binary. + let call = |name: &str, arg: &str| format!("{name}!({arg})"); + let target = "../../../README.md"; let file = "crates/reader/src/lib.rs"; for text in [ - r#"include_str!("../../../README.md")"#, - r##"include_str!(r#"../../../README.md"#)"##, - r###"include_bytes!(r##"../../../README.md"##)"###, - r#"include_str!(r"../../../README.md")"#, + call("include_str", &format!("\"{target}\"")), + call("include_str", &format!("r\"{target}\"")), + call("include_str", &format!("r#\"{target}\"#")), + call("include_bytes", &format!("r##\"{target}\"##")), ] { assert_eq!( - include_targets(file, text), + include_targets(file, &text), vec!["README.md".to_string()], "{text}" ); } - assert!(include_targets(file, r#"include_str!(concat!("a", "b"))"#).is_empty()); + assert!(include_targets(file, &call("include_str", "concat!(\"a\", \"b\")")).is_empty()); } /// A nested checkout (`.claude/worktrees/...`) holds another branch's From 629fd30fa11e1b77a32ca970c98c4cf9ecc7df5a Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 17:15:17 +0000 Subject: [PATCH 04/11] dev: keep the rename test's git from inheriting a hook's GIT_DIR git exports an absolute `GIT_DIR` to a pre-push hook run from a linked worktree. The throwaway-repository helper and the directory-taking `diff_name_only` ran `git -C ` without clearing it, so in that setting they acted on the outer repository: the rename test could fail, and the helper's commit and staging landed on the developer's branch. Both now clear the variables git uses to select a repository, through one shared `scrub_git_env`, and a test checks that every one is cleared. Signed-off-by: Jussi Elo --- xtask/src/affected.rs | 53 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/xtask/src/affected.rs b/xtask/src/affected.rs index 1d6cb1f041..dde6e14630 100644 --- a/xtask/src/affected.rs +++ b/xtask/src/affected.rs @@ -324,12 +324,34 @@ pub(crate) fn changed_files_between(base: &str, head: &str) -> Result` must not inherit them, or it talks to the outer repo. +const GIT_REPO_ENV: &[&str] = &[ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR", + "GIT_PREFIX", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_NAMESPACE", +]; + +/// Clear [`GIT_REPO_ENV`] from `cmd`, for a git run against an explicit directory. +pub(crate) fn scrub_git_env(cmd: &mut Command) -> &mut Command { + for var in GIT_REPO_ENV { + cmd.env_remove(var); + } + cmd +} + /// [`changed_files_between`] in `dir` (the current directory when `None`). fn diff_name_only(dir: Option<&std::path::Path>, base: &str, head: &str) -> Result> { let range = format!("{base}...{head}"); let mut git = Command::new("git"); if let Some(dir) = dir { - git.arg("-C").arg(dir); + scrub_git_env(&mut git).arg("-C").arg(dir); } let output = git .args(["diff", "--name-only", "--no-renames", &range]) @@ -557,7 +579,8 @@ mod tests { /// Run `git` in `dir` with a fixed identity and no signing, for the /// throwaway repositories below. fn git_in(dir: &std::path::Path, args: &[&str]) { - let status = Command::new("git") + let mut git = Command::new("git"); + let status = scrub_git_env(&mut git) .arg("-C") .arg(dir) .args([ @@ -574,6 +597,32 @@ mod tests { assert!(status.success(), "git {args:?} failed in {}", dir.display()); } + /// A hook run from a linked worktree has `GIT_DIR` set; a git run against a + /// throwaway repository must not inherit it or it would act on the outer one. + #[test] + fn scrub_git_env_clears_every_repo_selecting_variable() { + let mut git = Command::new("git"); + git.env("GIT_DIR", "/outer/.git").env("GIT_INDEX_FILE", "x"); + scrub_git_env(&mut git); + let envs: std::collections::BTreeMap<_, _> = git + .get_envs() + .map(|(k, v)| (k.to_string_lossy().into_owned(), v.is_none())) + .collect(); + for var in GIT_REPO_ENV { + assert_eq!(envs.get(*var), Some(&true), "{var} is not cleared"); + } + assert!( + [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR" + ] + .iter() + .all(|var| GIT_REPO_ENV.contains(var)) + ); + } + /// A file moved from one crate to another must select both crates. With git's /// default rename detection only the new path is listed, the crate that lost /// the file is not selected, and a push that breaks it is not checked. From a12d0324ed93ed7402f6f6abed840b5670bd4e96 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 13:59:55 +0000 Subject: [PATCH 05/11] dev: skip the pre-push cargo steps that already passed on the same tree The pre-push hook had no memory, so a push retried after a dropped connection, a manual run before the push, or an amend that only rewords the message re-ran clippy and the tests on content it had just checked. After a step's cargo command exits 0, `cargo xtask pre-push-check` now records a stamp, and the next run of that step looks for it first. The stamp is keyed on the step and its exact cargo arguments (so a pass for a narrow selection never lets a wider one skip), the git tree hash of HEAD (so an amend with identical content still hits), the toolchain, and a digest of the CARGO_* and RUST* environment. It is only used and written when the working tree is clean, because cargo checks the working tree rather than the pushed commit, and nothing is recorded if the run changed the tree. Stamps live under the git common directory, so every worktree shares them, expire after seven days and are capped at 64; scratch files from a write that died are cleaned up. Any error reading, writing or verifying a stamp means the step runs, and git is run with GIT_DIR and the other repository-selecting variables removed when it is pointed at a directory. A skip prints what matched, what it does not cover (user-level cargo configuration, installed tools, files git ignores, flaky tests), and how to force a run (--rerun or ROCM_PRE_PUSH_RERUN=1). CONTRIBUTING.md, docs/testing.md and the hook comment describe the skip and say it shows this tree passed earlier, not that the checks ran during this push. Signed-off-by: Jussi Elo --- .pre-commit-config.yaml | 5 + CONTRIBUTING.md | 2 +- docs/testing.md | 42 +- xtask/src/main.rs | 33 +- xtask/src/prepush.rs | 789 ++++++++++++++++++- xtask/src/prepush_stamp.rs | 1459 ++++++++++++++++++++++++++++++++++++ 6 files changed, 2297 insertions(+), 33 deletions(-) create mode 100644 xtask/src/prepush_stamp.rs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9877231133..eaeef20d4d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -141,6 +141,11 @@ repos: # cargo clippy -p e2e-cucumber --test e2e -- -D warnings # cargo test --workspace --all-targets --exclude e2e-cucumber # and `cargo xtask pre-push-check --base ` runs one by hand. + # A step that already passed on the same tree (same cargo command, toolchain + # and environment, clean working tree) is skipped and says so; a green run + # records the pass under the git directory. `ROCM_PRE_PUSH_RERUN=1` (or + # `--rerun` by hand) ignores recorded passes, and `SKIP=` still + # skips a hook. - id: cargo-clippy name: cargo clippy entry: cargo xtask pre-push-check clippy diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fa54e2a85..f5577ef548 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ prek install -t pre-push # heavier checks on push (clippy + tests of the crat `prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), and the generated manifests (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`). -The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). +The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them and records a fresh pass; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). The manifest hooks only run when you change the dependency graph. They *rewrite* the generated file instead of only reporting that it is stale. When that happens, the commit stops so you can re-stage the refreshed file. diff --git a/docs/testing.md b/docs/testing.md index dc203bd993..749ffd4ec6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -417,14 +417,52 @@ Known limits, unchanged by the narrowing: the steps check the working tree rathe commit; under prek 0.4.5 a push of several refs, a tag or a deletion runs no hooks; and `-p` builds can resolve Cargo features differently from `--workspace` ones. -Preview what a push would check without compiling: +### Pass stamps + +A step that already passed on this exact content is skipped. After a step's cargo command exits 0, +[prepush_stamp.rs](../xtask/src/prepush_stamp.rs) records a stamp, and the next run of the same step +looks for it first. The stamp stands for: + +- the step and its exact cargo arguments, so a pass recorded for `-p xtask` never lets a later run + over `--workspace` skip (the selection is part of the key, not a separate check); +- the git *tree* hash of `HEAD`, not the commit hash, so an amend that only rewords the message, or + the same content pushed to a second remote, still hits; +- the compiler and cargo identity (`rustc -vV`, `cargo -vV`) and a digest of the `CARGO_*` and + `RUST*` environment (never the values), minus a short list of per-run variables such as + `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. + +The working tree has to be clean for a stamp to be used or written, because cargo checks the working +tree, not the pushed commit: with uncommitted changes the step runs and says why, and an untracked +file counts because it can hide a missing `git add`. The tree is probed again after the step, and +nothing is recorded if cargo changed it (a rewritten `Cargo.lock`, for example). A pass is only +written after cargo succeeded, per step, so a failed `test` does not erase the `clippy` pass that +the retry then skips. + +Stamps live in `rocm-cli-prepush-passes` under the git common directory, so every worktree of a +clone shares them, nothing lands in the work tree and `cargo clean` does not erase them. Each write +drops stamps older than seven days, keeps at most 64, and removes scratch files left by a write that +died (a write goes through a scratch file and a rename, and cleans it up when the rename fails). Any error locating, reading or verifying a +stamp means the step runs (git is run for an explicit directory with `GIT_DIR` and the other +repository-selecting variables removed, which git and prek export to a hook run from a linked +worktree); a stamp that does not match its key exactly, is dated in the future or is +unreadable is ignored (the step names it); a failed write is reported and never fails the hook. + +The skip message says what a stamp does not cover: user-level cargo configuration, installed tools, +files git ignores (the one thing a clean `git status` cannot see) and flaky tests. Force a run with `--rerun` or `ROCM_PRE_PUSH_RERUN=1` (the +hook has no flags, so use the variable there), or delete the directory the message names. `SKIP=` still skips a hook outright, and `verify-commits` is a separate hook that checks commit +signatures on every push. A skip proves that this tree passed these checks earlier, not that they ran +during this push; for a fresh run before relying on the result, use `--rerun`. + +Preview what a push would check without compiling (a dry run also says when a real run would skip): ```bash cargo xtask pre-push-check test --dry-run --base origin/main ``` `--base` replaces the range's base; `clippy` and `clippy-e2e` take the same flags. The selection -logic is unit-tested in [prepush.rs](../xtask/src/prepush.rs); as dev tooling, with no change to +logic and the stamps are unit-tested in [prepush.rs](../xtask/src/prepush.rs) and +[prepush_stamp.rs](../xtask/src/prepush_stamp.rs); as dev tooling, with no change to what the `rocm` binary does, it has no Gherkin scenario. The smoke path is the cross-platform local no-fallback acceptance surface. It diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9ce4a86ebb..11f4af0134 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,6 +27,7 @@ mod package; mod paths; mod powershell; mod prepush; +mod prepush_stamp; mod release_readiness; mod report_mailbox; mod report_rollup; @@ -116,6 +117,10 @@ pub(crate) enum Command { /// cannot be determined, and prints why. It is skipped only when the /// selection is known and has nothing for the step to check. CI always /// checks the whole workspace. + /// + /// A step that already passed on this exact tree (same cargo command, + /// toolchain and environment, clean working tree) is skipped, and a pass is + /// recorded after a green run; see `xtask/src/prepush_stamp.rs`. PrePushCheck { /// Which step to run. #[arg(value_enum)] @@ -127,6 +132,10 @@ pub(crate) enum Command { /// Print the scope and the cargo command without running it. #[arg(long)] dry_run: bool, + /// Run the step even when this tree already passed it, and record the + /// new result. `ROCM_PRE_PUSH_RERUN=1` does the same, for the hook. + #[arg(long)] + rerun: bool, }, /// Fail if any first-party crate has gained a normal/build dependency /// edge outside the allowlist pinned in `xtask/src/crate_edges.rs`. @@ -391,7 +400,8 @@ fn run() -> Result<()> { check, base, dry_run, - } => prepush::run(check, base, dry_run)?, + rerun, + } => prepush::run(check, base, dry_run, rerun)?, Command::CheckCrateEdges => crate_edges::run()?, Command::Catalog { check } => catalog::run(check)?, Command::EvalFixtures { check } => eval_fixtures::run(check)?, @@ -479,3 +489,24 @@ fn main() -> ExitCode { } } } + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::{Cli, Command}; + + fn rerun(args: &[&str]) -> bool { + let cli = Cli::try_parse_from(args).expect("the arguments parse"); + let Command::PrePushCheck { rerun, .. } = cli.command else { + panic!("expected pre-push-check"); + }; + rerun + } + + #[test] + fn pre_push_check_takes_rerun_and_defaults_to_off() { + assert!(rerun(&["xtask", "pre-push-check", "test", "--rerun"])); + assert!(!rerun(&["xtask", "pre-push-check", "test"])); + } +} diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 862273bd64..d6b4b28206 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -30,18 +30,27 @@ //! manual run, it diffs `origin/main...HEAD` like `cargo xtask affected` does. //! `--base` overrides the from-ref. //! +//! A step that already passed on this exact tree is skipped, and a green run is +//! recorded: see [`crate::prepush_stamp`] for what a pass is keyed on (the +//! step's cargo arguments, so a narrow pass never covers a wider run; the tree +//! hash of `HEAD`; the toolchain; the environment) and why a dirty working tree +//! neither uses nor writes one. +//! //! As in [`crate::affected`], the decisions ([`push_from_env`], [`plan`], //! [`hook_selection`], [`resolve_selection`]) are pure and unit-tested; running -//! git and cargo is confined to [`run`], and [`execute`] takes both as -//! parameters so its own behavior (the range it diffs, `--dry-run`, a failing -//! cargo) is tested too. +//! git and cargo is confined to [`run`], and [`execute`] takes both, the stamps +//! and the output as parameters so its own behavior (the range it diffs, +//! `--dry-run`, a failing cargo, a skipped or recorded pass, and what it prints +//! about each) is tested too. use std::collections::BTreeSet; +use std::io::Write; use std::process::Command; use anyhow::{Context, Result, bail}; use crate::affected::{self, DEFAULT_BASE, Graph, Selection}; +use crate::prepush_stamp::{self, Lookup, Stamps}; /// The e2e harness crate. `cargo test` excludes it (its scenarios run through /// `cargo xtask e2e`), and its harness target is linted by its own step. @@ -344,39 +353,86 @@ impl Env { } } +/// What one invocation asks for. +#[derive(Debug, Clone, Default)] +pub struct Request { + /// Replaces the range's base (`--base`). + pub base: Option, + /// Print the plan without running or recording anything (`--dry-run`). + pub dry_run: bool, + /// The refs prek exported. + pub env: Env, +} + /// Run one step: resolve the range (an explicit `base` wins over prek's /// from-ref), pick the affected crates, print the scope, and hand the planned -/// cargo arguments to `run_cargo` unless the step is skipped or `dry_run`. +/// cargo arguments to `run_cargo` unless the step is skipped, `dry_run`, or a +/// pass of exactly this command on this tree is on record (`stamps`). /// -/// `load_graph`, `changed_files` (given the range's base and tip) and -/// `run_cargo` are parameters so the behavior around them is unit-tested. +/// A pass is recorded only after `run_cargo` succeeded. `load_graph`, +/// `changed_files` (given the range's base and tip), `run_cargo` and `out` are +/// parameters so the behavior around them is unit-tested. pub fn execute( check: Check, - base: Option, - dry_run: bool, - env: Env, + request: Request, + stamps: &Stamps<'_>, load_graph: impl FnOnce() -> Result, changed_files: impl FnOnce(&str, &str) -> Result>, mut run_cargo: impl FnMut(&[String]) -> Result<()>, + out: &mut impl Write, ) -> Result<()> { + let label = check.label(); + let Request { base, dry_run, env } = request; let push = push_from_env(base.or(env.from_ref), env.to_ref); let (selection, fallback) = resolve_selection(load_graph, || changed_files(push.base(), &push.to)); - println!("{}", banner(check, &push, &selection, fallback.as_deref())); + writeln!( + out, + "{}", + banner(check, &push, &selection, fallback.as_deref()) + )?; let args = match plan(check, &selection) { Plan::Skip(reason) => { - println!("pre-push-check {}: skipped, {reason}", check.label()); + writeln!(out, "pre-push-check {label}: skipped, {reason}")?; return Ok(()); } Plan::Cargo(args) => args, }; - println!("pre-push-check {}: cargo {}", check.label(), args.join(" ")); + writeln!(out, "pre-push-check {label}: cargo {}", args.join(" "))?; + let key = match stamps.lookup(label, &args) { + Lookup::Hit { name, age_secs } => { + if dry_run { + writeln!( + out, + "pre-push-check {label}: a real run would skip this step, it passed {} ago", + prepush_stamp::age_text(age_secs) + )?; + } else { + let location = stamps.location(); + for line in prepush_stamp::skip_lines(label, &args, age_secs, &name, &location) { + writeln!(out, "{line}")?; + } + } + return Ok(()); + } + Lookup::Miss { key, note } => { + if let Some(note) = note { + writeln!(out, "pre-push-check {label}: {note}")?; + } + key + } + }; if dry_run { return Ok(()); } - run_cargo(&args) + + run_cargo(&args)?; + if let Some(note) = key.and_then(|key| stamps.record(&key)) { + writeln!(out, "pre-push-check {label}: {note}")?; + } + Ok(()) } /// Run `cargo` with `args`, failing when it does. @@ -393,15 +449,33 @@ fn spawn_cargo(args: &[String]) -> Result<()> { } /// Entry point for the `pre-push-check` subcommand. -pub fn run(check: Check, base: Option, dry_run: bool) -> Result<()> { - execute( - check, +pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Result<()> { + let store: Box = match prepush_stamp::FsStore::in_git_common_dir(None) + { + Ok(store) => Box::new(store), + Err(error) => Box::new(prepush_stamp::UnavailableStore(format!("{error:#}"))), + }; + let probe = prepush_stamp::SystemProbe::default(); + let stamps = Stamps::new( + &probe, + store.as_ref(), + prepush_stamp::now_secs(), + rerun, + |name| std::env::var(name).ok(), + ); + let request = Request { base, dry_run, - Env::from_lookup(|name| std::env::var(name).ok()), + env: Env::from_lookup(|name| std::env::var(name).ok()), + }; + execute( + check, + request, + &stamps, affected::load_graph, affected::changed_files_between, spawn_cargo, + &mut std::io::stdout().lock(), ) } @@ -410,6 +484,10 @@ mod tests { use std::collections::{BTreeMap, BTreeSet}; use super::*; + use crate::prepush_stamp::fakes::{FakeProbe, MemStore}; + + /// A fixed clock for the stamp tests. + const NOW: u64 = 1_800_000_000; fn packages(names: &[&str]) -> Selection { Selection::Packages(names.iter().map(|name| (*name).to_string()).collect()) @@ -754,24 +832,32 @@ mod tests { } } - /// Run a step against canned inputs; `files` of `None` makes the diff fail. - /// Returns the result, every `(base, tip)` the diff was asked for, and the - /// argument lines handed to cargo. - fn run_step( + /// Everything one `execute` call did. + struct StepRun { + result: Result<()>, + ranges: Ranges, + /// The argument lines handed to cargo. + cargo: Vec, + /// What the step printed. + out: String, + } + + /// Run a step against canned inputs, with the given stamps; `files` of + /// `None` makes the diff fail. + fn run_step_with( check: Check, - base: Option<&str>, - dry_run: bool, - env: Env, + request: Request, + stamps: &Stamps<'_>, files: Option<&[&str]>, cargo_fails: bool, - ) -> (Result<()>, Ranges, Vec) { + ) -> StepRun { let ranges = std::cell::RefCell::new(Vec::new()); let cargo = std::cell::RefCell::new(Vec::new()); + let mut out = Vec::new(); let result = execute( check, - base.map(String::from), - dry_run, - env, + request, + stamps, || Ok(graph()), |base, tip| { ranges @@ -789,8 +875,44 @@ mod tests { } Ok(()) }, + &mut out, ); - (result, ranges.into_inner(), cargo.into_inner()) + StepRun { + result, + ranges: ranges.into_inner(), + cargo: cargo.into_inner(), + out: String::from_utf8(out).expect("the step prints UTF-8"), + } + } + + /// Run a step with stamps that never apply (the working tree reads as dirty), + /// so these tests see only the range, `--dry-run` and cargo's result. + /// Returns the result, every `(base, tip)` the diff was asked for, and the + /// argument lines handed to cargo. + fn run_step( + check: Check, + base: Option<&str>, + dry_run: bool, + env: Env, + files: Option<&[&str]>, + cargo_fails: bool, + ) -> (Result<()>, Ranges, Vec) { + let probe = FakeProbe::clean("tree"); + *probe.tree.borrow_mut() = Err("the working tree is dirty".to_string()); + let store = MemStore::default(); + let stamps = Stamps { + probe: &probe, + store: &store, + now: NOW, + rerun: false, + }; + let request = Request { + base: base.map(String::from), + dry_run, + env, + }; + let run = run_step_with(check, request, &stamps, files, cargo_fails); + (run.result, run.ranges, run.cargo) } const XTASK_ONLY: Option<&[&str]> = Some(&["xtask/src/main.rs"]); @@ -900,6 +1022,615 @@ mod tests { assert_eq!(Env::from_lookup(|_| None), Env::default()); } + // ---- pass stamps: skip a step that already passed on this exact tree ---- + + const XTASK_TEST: &str = "test -p xtask --all-targets"; + const WIDE_TEST: &str = "test --workspace --all-targets --exclude e2e-cucumber"; + + fn stamps<'a>(probe: &'a FakeProbe, store: &'a MemStore, now: u64, rerun: bool) -> Stamps<'a> { + Stamps { + probe, + store, + now, + rerun, + } + } + + fn run_at( + stamps: &Stamps<'_>, + check: Check, + files: &[&str], + dry_run: bool, + cargo_fails: bool, + ) -> StepRun { + let request = Request { + dry_run, + ..Request::default() + }; + run_step_with(check, request, stamps, Some(files), cargo_fails) + } + + #[test] + fn a_green_run_is_recorded_and_the_next_identical_run_is_skipped_with_the_reason() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let first = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + first.result.expect("the first run"); + assert_eq!(first.cargo, [XTASK_TEST]); + assert_eq!(store.files.borrow().len(), 1, "a green run is recorded"); + + let second = run_at( + &stamps(&probe, &store, NOW + 90, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + second.result.expect("the skipped run"); + assert!( + second.cargo.is_empty(), + "the step must not run again: {:?}", + second.cargo + ); + assert!( + second + .out + .contains(&format!("pre-push-check test: skipped, this tree already passed `cargo {XTASK_TEST}` 1 min ago")), + "{}", + second.out + ); + // The skip names what it does not cover and how to force a run. + assert!( + second.out.contains("user-level cargo configuration"), + "{}", + second.out + ); + assert!(second.out.contains("--rerun"), "{}", second.out); + assert!(second.out.contains("files git ignores"), "{}", second.out); + assert!(second.out.contains("installed tools"), "{}", second.out); + assert!(second.out.contains("flaky tests"), "{}", second.out); + assert!( + second.out.contains(prepush_stamp::RERUN_VAR), + "{}", + second.out + ); + assert!(second.out.contains(""), "{}", second.out); + assert_eq!(store.files.borrow().len(), 1, "a skip records nothing"); + } + + #[test] + fn a_narrow_pass_never_lets_a_wider_selection_skip() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let narrow = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(narrow.cargo, [XTASK_TEST]); + + // The same tree, but the push now also touches the lockfile: the whole + // workspace. The earlier `-p xtask` pass says nothing about that. + let wide = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["Cargo.lock"], + false, + false, + ); + assert_eq!(wide.cargo, [WIDE_TEST], "{}", wide.out); + assert!(!wide.out.contains("skipped"), "{}", wide.out); + assert_eq!(store.files.borrow().len(), 2, "both passes are on record"); + + // And each is skipped on its own next run. + let again = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert!(again.cargo.is_empty(), "{}", again.out); + } + + #[test] + fn a_different_tree_runs_again() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("first"); + *probe.tree.borrow_mut() = Ok("tree-2".to_string()); + let second = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(second.cargo, [XTASK_TEST], "{}", second.out); + assert_eq!(store.files.borrow().len(), 2); + } + + #[test] + fn a_different_toolchain_or_environment_runs_again() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("first"); + + *probe.toolchain.borrow_mut() = Ok("rustc 1.97.0\ncargo 1.97.0\n".to_string()); + let toolchain = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!( + toolchain.cargo, + [XTASK_TEST], + "a new toolchain must not reuse the pass" + ); + + *probe.env.borrow_mut() = "env-b".to_string(); + let env = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!( + env.cargo, + [XTASK_TEST], + "a new RUSTFLAGS and the like must not reuse the pass" + ); + } + + #[test] + fn a_failed_step_records_nothing_and_a_passed_sibling_still_skips_on_retry() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Clippy, + &files, + false, + false, + ) + .result + .expect("clippy passes"); + let failed = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + true, + ); + failed.result.expect_err("test fails"); + assert_eq!( + store.files.borrow().len(), + 1, + "only the clippy pass is on record" + ); + assert!( + store + .files + .borrow() + .keys() + .all(|name| name.starts_with("clippy-")), + "{:?}", + store.files.borrow().keys() + ); + + // The retry skips the step that passed and re-runs the one that failed. + let clippy = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Clippy, + &files, + false, + false, + ); + assert!(clippy.cargo.is_empty(), "{}", clippy.out); + let test = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!(test.cargo, [XTASK_TEST]); + assert_eq!(store.files.borrow().len(), 2); + } + + #[test] + fn a_dirty_working_tree_neither_skips_nor_records_and_says_so() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("clean run"); + assert_eq!(store.files.borrow().len(), 1); + + *probe.tree.borrow_mut() = + Err("the working tree has 2 uncommitted or untracked path(s)".to_string()); + let dirty = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!( + dirty.cargo, + [XTASK_TEST], + "a recorded pass is not honored on a dirty tree" + ); + assert!( + dirty + .out + .contains("pass stamps are not used: the working tree has 2 uncommitted"), + "{}", + dirty.out + ); + assert_eq!(*store.writes.borrow(), 1, "and no pass is recorded from it"); + } + + #[test] + fn a_tree_the_run_changed_is_not_recorded() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let stamps = stamps(&probe, &store, NOW, false); + let mut out = Vec::new(); + execute( + Check::Test, + Request::default(), + &stamps, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + // cargo rewrote a tracked file (`Cargo.lock`, say) while it ran. + *probe.tree.borrow_mut() = + Err("the working tree has 1 uncommitted path".to_string()); + Ok(()) + }, + &mut out, + ) + .expect("the step itself passed"); + let out = String::from_utf8(out).expect("utf-8"); + assert!( + out.contains("pass not recorded: the working tree has 1 uncommitted"), + "{out}" + ); + assert!( + store.files.borrow().is_empty(), + "nothing is recorded for a tree cargo changed" + ); + } + + #[test] + fn rerun_ignores_a_recorded_pass_and_records_a_fresh_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("first"); + let forced = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + false, + ); + assert_eq!(forced.cargo, [XTASK_TEST], "--rerun must run the step"); + assert!( + forced.out.contains("recorded passes are ignored"), + "{}", + forced.out + ); + assert!( + forced.out.contains(prepush_stamp::RERUN_VAR), + "{}", + forced.out + ); + let recorded: Vec = store + .files + .borrow() + .values() + .map(|(text, _)| text.clone()) + .collect(); + assert_eq!(recorded.len(), 1, "the new pass replaces the old one"); + assert!( + recorded[0].contains(&format!("passed-at: {}", NOW + 100)), + "{recorded:?}" + ); + } + + #[test] + fn a_dry_run_reports_a_hit_and_never_runs_or_writes() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + let miss = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + true, + false, + ); + assert!(miss.cargo.is_empty()); + assert!(store.files.borrow().is_empty(), "a dry run records nothing"); + assert!(!miss.out.contains("would skip"), "{}", miss.out); + + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("real run"); + let writes = *store.writes.borrow(); + let hit = run_at( + &stamps(&probe, &store, NOW + 3600, false), + Check::Test, + &files, + true, + false, + ); + assert!(hit.cargo.is_empty()); + assert!( + hit.out + .contains("a real run would skip this step, it passed 1 h ago"), + "{}", + hit.out + ); + assert_eq!(*store.writes.borrow(), writes); + } + + #[test] + fn a_failed_write_is_reported_and_never_fails_the_step() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + *store.fail_writes.borrow_mut() = true; + let run = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + run.result + .expect("a stamp that cannot be written must not fail a green hook"); + assert_eq!(run.cargo, [XTASK_TEST]); + assert!( + run.out.contains( + "pass not recorded: could not write it under (read-only file system)" + ), + "{}", + run.out + ); + } + + #[test] + fn a_store_that_cannot_be_set_up_still_runs_the_step_and_says_why() { + let probe = FakeProbe::clean("tree-1"); + let store = prepush_stamp::UnavailableStore("not a git repository".to_string()); + let stamps = Stamps { + probe: &probe, + store: &store, + now: NOW, + rerun: false, + }; + let run = run_at(&stamps, Check::Test, &["xtask/src/main.rs"], false, false); + run.result.expect("the step runs"); + assert_eq!(run.cargo, [XTASK_TEST]); + assert!( + run.out + .contains("could not read the pass stamp (not a git repository)"), + "{}", + run.out + ); + assert!( + run.out + .contains("pass not recorded: could not write it under the stamp directory"), + "{}", + run.out + ); + } + + #[test] + fn a_corrupted_stamp_is_ignored_and_replaced_by_a_real_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("first"); + let name = store.files.borrow().keys().next().expect("a stamp").clone(); + store + .files + .borrow_mut() + .insert(name.clone(), ("not a stamp".to_string(), NOW)); + + let second = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &files, + false, + false, + ); + assert_eq!( + second.cargo, + [XTASK_TEST], + "garbage under the right name must not skip" + ); + assert!( + second + .out + .contains(&format!("ignoring the pass stamp {name}")), + "{}", + second.out + ); + let third = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &files, + false, + false, + ); + assert!( + third.cargo.is_empty(), + "the replacement is a real stamp: {}", + third.out + ); + } + + #[test] + fn a_skipped_selection_neither_looks_for_nor_writes_a_stamp() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let run = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["CHANGELOG.md"], + false, + false, + ); + assert!(run.cargo.is_empty()); + assert!( + run.out.contains("skipped, no Rust crate is affected"), + "{}", + run.out + ); + assert_eq!(*store.writes.borrow(), 0); + } + + #[test] + fn a_head_that_moved_during_the_step_is_not_recorded_and_the_note_names_head() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let stamps = stamps(&probe, &store, NOW, false); + let mut out = Vec::new(); + execute( + Check::Test, + Request::default(), + &stamps, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + // HEAD is a different commit by the time the step has finished. + *probe.tree.borrow_mut() = Ok("tree-2".to_string()); + Ok(()) + }, + &mut out, + ) + .expect("the step itself passed"); + let out = String::from_utf8(out).expect("utf-8"); + assert!( + out.contains("pass not recorded: HEAD moved while the step ran"), + "{out}" + ); + assert!( + store.files.borrow().is_empty(), + "nothing is written for a tree that was not checked" + ); + assert_eq!(*store.writes.borrow(), 0); + } + + /// A stamp for `xtask`'s test step on `tree-1`, written by a normal run. + fn with_a_recorded_pass() -> (FakeProbe, MemStore) { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ) + .result + .expect("the first run"); + assert_eq!(store.files.borrow().len(), 1); + (probe, store) + } + + #[test] + fn the_rerun_flag_forces_a_run_despite_a_matching_stamp() { + let (probe, store) = with_a_recorded_pass(); + let forced = Stamps::new(&probe, &store, NOW + 5, true, |_| None); + let run = run_at(&forced, Check::Test, &["xtask/src/main.rs"], false, false); + assert_eq!(run.cargo, [XTASK_TEST], "{}", run.out); + assert!( + run.out.contains("recorded passes are ignored"), + "{}", + run.out + ); + assert!(!run.out.contains("skipped"), "{}", run.out); + } + + #[test] + fn the_rerun_variable_forces_a_run_despite_a_matching_stamp() { + let (probe, store) = with_a_recorded_pass(); + let lookup = |name: &str| (name == prepush_stamp::RERUN_VAR).then(|| "1".to_string()); + let forced = Stamps::new(&probe, &store, NOW + 5, false, lookup); + let run = run_at(&forced, Check::Test, &["xtask/src/main.rs"], false, false); + assert_eq!(run.cargo, [XTASK_TEST], "{}", run.out); + assert!( + run.out.contains("recorded passes are ignored"), + "{}", + run.out + ); + } + + #[test] + fn without_the_flag_or_with_the_variable_off_the_same_stamp_still_skips() { + let (probe, store) = with_a_recorded_pass(); + let off = |name: &str| (name == prepush_stamp::RERUN_VAR).then(|| "0".to_string()); + for stamps in [ + Stamps::new(&probe, &store, NOW + 5, false, |_| None), + Stamps::new(&probe, &store, NOW + 5, false, off), + ] { + let run = run_at(&stamps, Check::Test, &["xtask/src/main.rs"], false, false); + assert!(run.cargo.is_empty(), "{}", run.out); + assert!( + run.out.contains("skipped, this tree already passed"), + "{}", + run.out + ); + } + } + // ---- drift guard: every cross-crate read must be covered ---- use std::path::{Path, PathBuf}; diff --git a/xtask/src/prepush_stamp.rs b/xtask/src/prepush_stamp.rs new file mode 100644 index 0000000000..e14eb8dc0e --- /dev/null +++ b/xtask/src/prepush_stamp.rs @@ -0,0 +1,1459 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Pass stamps for `cargo xtask pre-push-check`: remember that a cargo step +//! already passed on exactly this content, and skip it the next time. +//! +//! The pre-push hook has no memory, so a push retried after a dropped +//! connection, a manual run before the push, an amend that only rewords the +//! message, or the same tree pushed to a second remote re-ran every step on a +//! tree it had just checked. A stamp is written after a step's cargo command +//! exits 0 and is looked up before the next run of that step. +//! +//! **What a stamp stands for.** One step (`clippy`, `clippy-e2e` or `test`), the +//! exact cargo arguments it ran, the git *tree* hash of `HEAD` (not the commit +//! hash, so an amend or a re-commit with identical content still hits), the +//! compiler and cargo identity (`rustc -vV`, `cargo -vV`), and a digest of the +//! `CARGO_*`/`RUST*` environment. Because the arguments are part of the key, a +//! pass recorded for a narrow selection can never let a later, wider selection +//! skip: `-p xtask` and `--workspace` are different stamps. +//! +//! **The working tree must be clean.** Cargo checks the working tree, not the +//! pushed commit, so a stamp keyed on `HEAD`'s tree is only true when the +//! working tree *is* that tree. With uncommitted changes or untracked files +//! (an untracked `mod` target can hide a missing `git add`) a stamp is neither +//! honored nor written, and the step says why. After the run the tree is checked +//! again: if cargo changed it (a rewritten `Cargo.lock`, say), nothing is +//! recorded. +//! +//! **Fail safe.** Every error while locating the store, reading, parsing or +//! verifying a stamp means the step runs; a stamp that does not match its key +//! byte for byte, is unreadable, belongs to a different format, is dated in the +//! future or is older than [`MAX_AGE_SECS`] is ignored; a failed write is +//! reported and never fails the hook. A stamp is only written after cargo +//! succeeded. +//! +//! **Where.** Under the git *common* directory, so every linked worktree of a +//! clone shares the stamps, nothing lands in the work tree (no ignore rule, no +//! chance of committing one), and `cargo clean` does not erase them. Each write +//! prunes stamps older than [`MAX_AGE_SECS`] and keeps at most [`MAX_STAMPS`]. +//! +//! **Not covered**, and said in the skip message: user-level cargo +//! configuration (`~/.cargo/config.toml`), installed tools, files git ignores, +//! network-dependent or flaky tests. `--rerun` or [`RERUN_VAR`] ignores recorded +//! passes (and records a fresh one), and `SKIP=` still skips a hook +//! outright. The `verify-commits` hook is a separate hook and runs every time. +//! +//! As elsewhere in this crate the decisions are pure and the world is injected: +//! git and the toolchain sit behind [`Probe`], the filesystem behind [`Store`], +//! and the clock is a parameter, so the rules above are unit-tested without +//! cargo, git or a disk. + +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result, bail}; +use sha2::{Digest, Sha256}; + +use crate::affected; + +/// Setting this to anything but empty or `0` ignores recorded passes. +pub const RERUN_VAR: &str = "ROCM_PRE_PUSH_RERUN"; + +/// Directory name under the git common directory. +const STAMP_DIR: &str = "rocm-cli-prepush-passes"; +const STAMP_EXT: &str = ".stamp"; +/// Scratch files a write goes through before it is renamed into place. +const TMP_EXT: &str = ".tmp"; +const FORMAT_LINE: &str = "rocm-cli pre-push pass stamp v1"; + +/// The revision whose hash identifies the checked content. +const HEAD_TREE: &str = "HEAD^{tree}"; + +/// A pass older than this is ignored and pruned: toolchains, dependencies and +/// the registry move on, and a week is longer than any realistic retry loop. +pub const MAX_AGE_SECS: u64 = 7 * 24 * 60 * 60; + +/// At most this many stamps are kept; the oldest go first. +pub const MAX_STAMPS: usize = 64; + +/// A scratch file older than this belongs to a write that died; a younger one +/// may be a concurrent writer's, so it is left alone. +pub const TMP_MAX_AGE_SECS: u64 = 60 * 60; + +/// A stamp dated this far ahead of the clock is treated as corrupt. +const FUTURE_SLACK_SECS: u64 = 5 * 60; + +/// Environment variable prefixes that can change what a cargo step does. +/// Over-including only costs a re-run; leaving one out could skip a step that +/// would now fail. +const ENV_PREFIXES: &[&str] = &["CARGO_", "RUST"]; + +/// Variables under those prefixes that differ between runs without changing the +/// outcome: where artifacts go, terminal styling, and what `cargo run` sets for +/// the `xtask` binary itself. +const ENV_IGNORED: &[&str] = &[ + "CARGO_TARGET_DIR", + "CARGO_MAKEFLAGS", + "CARGO_MANIFEST_DIR", + "CARGO_MANIFEST_PATH", + "CARGO_BIN_NAME", + "CARGO_CRATE_NAME", + "CARGO_PRIMARY_PACKAGE", + "RUST_BACKTRACE", + "RUST_LIB_BACKTRACE", +]; +const ENV_IGNORED_PREFIXES: &[&str] = &["CARGO_TERM_", "CARGO_PKG_", "CARGO_BIN_EXE_"]; + +fn hex_digest(text: &str) -> String { + format!("{:x}", Sha256::digest(text.as_bytes())) +} + +/// Whether `--rerun` or [`RERUN_VAR`] asks for recorded passes to be ignored. +pub fn rerun_requested(flag: bool, lookup: impl Fn(&str) -> Option) -> bool { + flag || lookup(RERUN_VAR).is_some_and(|value| { + let value = value.trim(); + !value.is_empty() && value != "0" + }) +} + +/// Digest of the environment variables that can change a cargo step, in a +/// stable order. Only the digest is kept, never a value. +pub fn env_digest(vars: impl IntoIterator) -> String { + let mut relevant: Vec<(String, String)> = vars + .into_iter() + .filter(|(name, _)| { + ENV_PREFIXES.iter().any(|prefix| name.starts_with(prefix)) + && !ENV_IGNORED.contains(&name.as_str()) + && !ENV_IGNORED_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .collect(); + relevant.sort(); + let mut text = String::new(); + for (name, value) in relevant { + text.push_str(&name); + text.push('='); + text.push_str(&value); + text.push('\n'); + } + hex_digest(&text) +} + +/// Everything one pass depends on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Key { + step: String, + args: String, + tree: String, + toolchain: String, + env: String, +} + +impl Key { + fn canonical(&self) -> String { + format!( + "step: {}\ncargo: {}\ntree: {}\ntoolchain: {}\nenv: {}\n", + self.step, self.args, self.tree, self.toolchain, self.env + ) + } + + fn file_name(&self) -> String { + let digest = hex_digest(&self.canonical()); + format!("{}-{}{STAMP_EXT}", self.step, &digest[..32]) + } + + fn stamp_text(&self, passed_at: u64) -> String { + format!( + "{FORMAT_LINE}\n{}passed-at: {passed_at}\n", + self.canonical() + ) + } +} + +/// What git and the toolchain say about the state being checked. +pub trait Probe { + /// The tree hash of `HEAD`, only when the working tree is exactly that tree. + /// `Err` carries the reason stamps cannot be trusted right now. + fn clean_tree(&self) -> std::result::Result; + /// A description of the compiler and cargo that will run the step. + fn toolchain(&self) -> std::result::Result; + /// [`env_digest`] of the current environment. + fn env_digest(&self) -> String; +} + +/// Where stamps are kept. Names are file names inside the store. +pub trait Store { + fn read(&self, name: &str) -> Result>; + fn write(&self, name: &str, contents: &str) -> Result<()>; + /// Every stamp as `(name, modified time in seconds since the epoch)`. + fn list(&self) -> Result>; + fn remove(&self, name: &str) -> Result<()>; + /// Where the stamps are, for messages. + fn location(&self) -> String; +} + +/// The outcome of looking for a recorded pass. +#[derive(Debug, PartialEq, Eq)] +pub enum Lookup { + /// A matching pass is on record: skip the step. + Hit { name: String, age_secs: u64 }, + /// Run the step. `key` is `Some` when a pass can be recorded afterwards, and + /// `note` explains anything the user should know (a dirty tree, a rejected + /// stamp, `--rerun`). + Miss { + key: Option, + note: Option, + }, +} + +/// Stamp handling for one invocation, with the world injected. +pub struct Stamps<'a> { + pub probe: &'a dyn Probe, + pub store: &'a dyn Store, + /// Seconds since the epoch. + pub now: u64, + /// Ignore recorded passes (but still record a new one). + pub rerun: bool, +} + +impl<'a> Stamps<'a> { + /// The stamps for one invocation. `rerun_flag` is `--rerun`; `lookup` reads + /// the environment for [`RERUN_VAR`], so tests need no process environment. + pub fn new( + probe: &'a dyn Probe, + store: &'a dyn Store, + now: u64, + rerun_flag: bool, + lookup: impl Fn(&str) -> Option, + ) -> Self { + Self { + probe, + store, + now, + rerun: rerun_requested(rerun_flag, lookup), + } + } +} + +impl Stamps<'_> { + fn key(&self, step: &str, args: &[String]) -> std::result::Result { + let tree = self.probe.clean_tree()?; + let toolchain = hex_digest(&self.probe.toolchain()?); + Ok(Key { + step: step.to_string(), + args: args.join(" "), + tree, + toolchain, + env: self.probe.env_digest(), + }) + } + + /// Look for a pass of `step` run with `args` on the current tree. + pub fn lookup(&self, step: &str, args: &[String]) -> Lookup { + let key = match self.key(step, args) { + Ok(key) => key, + Err(why) => { + return Lookup::Miss { + key: None, + note: Some(format!("pass stamps are not used: {why}")), + }; + } + }; + if self.rerun { + return Lookup::Miss { + key: Some(key), + note: Some(format!( + "recorded passes are ignored because --rerun or {RERUN_VAR} is set" + )), + }; + } + let name = key.file_name(); + let text = match self.store.read(&name) { + Ok(Some(text)) => text, + Ok(None) => { + return Lookup::Miss { + key: Some(key), + note: None, + }; + } + Err(error) => { + return Lookup::Miss { + key: Some(key), + note: Some(format!( + "could not read the pass stamp ({error:#}); running the step" + )), + }; + } + }; + match self.verify(&key, &text) { + Ok(Some(age_secs)) => Lookup::Hit { name, age_secs }, + Ok(None) => Lookup::Miss { + key: Some(key), + note: None, + }, + Err(why) => Lookup::Miss { + key: Some(key), + note: Some(format!("ignoring the pass stamp {name}: {why}")), + }, + } + } + + /// `Ok(Some(age))` for a valid, fresh stamp, `Ok(None)` for an expired one, + /// `Err(reason)` for one that is not what its name says. + fn verify(&self, key: &Key, text: &str) -> std::result::Result, String> { + let expected = format!("{FORMAT_LINE}\n{}", key.canonical()); + let rest = text + .strip_prefix(&expected) + .ok_or("it does not match this step, tree, toolchain and environment")?; + let passed_at: u64 = rest + .strip_prefix("passed-at: ") + .and_then(|value| value.trim().parse().ok()) + .ok_or("its time stamp is unreadable")?; + if passed_at > self.now.saturating_add(FUTURE_SLACK_SECS) { + return Err("it is dated in the future".to_string()); + } + let age = self.now.saturating_sub(passed_at); + Ok((age <= MAX_AGE_SECS).then_some(age)) + } + + /// Record that the step passed. Returns a note when nothing was recorded. + /// + /// The tree is probed again first: a stamp is only true when the working + /// tree is still the one the key was computed for, and a step that rewrote + /// it (`Cargo.lock`, a generated file) did not check `HEAD`'s tree. + pub fn record(&self, key: &Key) -> Option { + match self.probe.clean_tree() { + Ok(tree) if tree == key.tree => {} + Ok(_) => return Some("pass not recorded: HEAD moved while the step ran".to_string()), + Err(why) => return Some(format!("pass not recorded: {why}")), + } + if let Err(error) = self + .store + .write(&key.file_name(), &key.stamp_text(self.now)) + { + return Some(format!( + "pass not recorded: could not write it under {} ({error:#})", + self.store.location() + )); + } + self.prune(); + None + } + + /// Drop stamps past [`MAX_AGE_SECS`], then the oldest beyond [`MAX_STAMPS`], + /// and scratch files left by a write that died ([`TMP_MAX_AGE_SECS`]). + /// Best effort: a file that cannot be removed is only wasted space. + fn prune(&self) { + let Ok(mut entries) = self.store.list() else { + return; + }; + entries.retain(|(name, modified)| { + let age = self.now.saturating_sub(*modified); + let expired = if name.ends_with(TMP_EXT) { + age > TMP_MAX_AGE_SECS + } else { + age > MAX_AGE_SECS + }; + if expired { + let _ = self.store.remove(name); + } + !expired + }); + // A scratch file is not a pass: it does not count toward the cap. + entries.retain(|(name, _)| !name.ends_with(TMP_EXT)); + entries.sort_by_key(|(_, modified)| std::cmp::Reverse(*modified)); + for (name, _) in entries.iter().skip(MAX_STAMPS) { + let _ = self.store.remove(name); + } + } + + /// Where the stamps live, for the skip message. + pub fn location(&self) -> String { + self.store.location() + } +} + +/// `3 min`, `2 h`, `1 d`: how long ago a pass was recorded. +pub fn age_text(secs: u64) -> String { + match secs { + 0..=59 => "under a minute".to_string(), + 60..=3599 => format!("{} min", secs / 60), + 3600..=86_399 => format!("{} h", secs / 3600), + _ => format!("{} d", secs / 86_400), + } +} + +/// The lines printed when a step is skipped because its pass is on record. +pub fn skip_lines( + step: &str, + args: &[String], + age_secs: u64, + name: &str, + location: &str, +) -> Vec { + vec![ + format!( + "pre-push-check {step}: skipped, this tree already passed `cargo {}` {} ago \ + (stamp {name})", + args.join(" "), + age_text(age_secs) + ), + format!( + "pre-push-check {step}: that does not cover your user-level cargo configuration, \ + installed tools, files git ignores or flaky tests; run it again with --rerun \ + or {RERUN_VAR}=1, or delete {location}" + ), + ] +} + +// ---- the real world: git, the toolchain, the filesystem ---- + +/// A `git` command for the repository in `dir`. With an explicit directory the +/// repository-selecting variables are removed ([`affected::scrub_git_env`]): git +/// and prek export an absolute `GIT_DIR` to a pre-push hook run from a linked +/// worktree, and `git -C ` would otherwise still act on the repository +/// being pushed. With `None` the environment is kept, because in the hook that +/// inherited `GIT_DIR` correctly names the repository being pushed. +fn git_command(dir: Option<&Path>) -> Command { + let mut git = Command::new("git"); + if let Some(dir) = dir { + affected::scrub_git_env(&mut git).arg("-C").arg(dir); + } + git +} + +/// Run `git` with `args` (in `dir`, or the current directory) and return its +/// trimmed stdout. +fn git_stdout(dir: Option<&Path>, args: &[&str]) -> std::result::Result { + let output = git_command(dir) + .args(args) + .output() + .map_err(|error| format!("could not run git: {error}"))?; + if !output.status.success() { + return Err(format!( + "`git {}` failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// ` -vV`, or the reason it could not run. +fn version_text(tool: &std::ffi::OsStr) -> std::result::Result { + let output = Command::new(tool) + .arg("-vV") + .output() + .map_err(|error| format!("could not run {}: {error}", tool.to_string_lossy()))?; + if !output.status.success() { + return Err(format!("`{} -vV` failed", tool.to_string_lossy())); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +/// [`Probe`] backed by the `git` and toolchain on `PATH`, for the repository in +/// `dir` (the current directory when `None`). +#[derive(Default)] +pub struct SystemProbe { + pub dir: Option, +} + +impl Probe for SystemProbe { + fn clean_tree(&self) -> std::result::Result { + let dir = self.dir.as_deref(); + let status = git_stdout(dir, &["status", "--porcelain", "--untracked-files=normal"])?; + let dirty = status + .lines() + .filter(|line| !line.trim().is_empty()) + .count(); + if dirty > 0 { + return Err(format!( + "the working tree has {dirty} uncommitted or untracked path(s), and cargo checks \ + the working tree rather than HEAD; commit or remove them (`git status` lists them)" + )); + } + git_stdout(dir, &["rev-parse", HEAD_TREE]) + } + + fn toolchain(&self) -> std::result::Result { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + Ok(format!( + "{}\n{}", + version_text(std::ffi::OsStr::new("rustc"))?, + version_text(&cargo)? + )) + } + + fn env_digest(&self) -> String { + env_digest(std::env::vars()) + } +} + +/// Stamps in a directory on disk. +pub struct FsStore { + dir: PathBuf, +} + +impl FsStore { + /// The stamp directory under the git common directory of the repository in + /// `dir` (the current directory when `None`), shared by every worktree of + /// that clone. + pub fn in_git_common_dir(dir: Option<&Path>) -> Result { + let common = git_stdout(dir, &["rev-parse", "--git-common-dir"]) + .map_err(|why| anyhow::anyhow!(why))?; + let common = PathBuf::from(common); + // git prints the path relative to where it ran when it is inside there. + let common = if common.is_absolute() { + common + } else { + let base = match dir { + Some(dir) => dir.to_path_buf(), + None => std::env::current_dir().context("could not read the current directory")?, + }; + base.join(common) + }; + Ok(Self { + dir: common.join(STAMP_DIR), + }) + } + + fn path(&self, name: &str) -> Result { + if name.is_empty() || name == "." || name == ".." || name.contains(['/', '\\']) { + bail!("refusing the stamp name {name:?}"); + } + Ok(self.dir.join(name)) + } +} + +impl Store for FsStore { + fn read(&self, name: &str) -> Result> { + match std::fs::read_to_string(self.path(name)?) { + Ok(text) => Ok(Some(text)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error).context("failed to read the stamp"), + } + } + + fn write(&self, name: &str, contents: &str) -> Result<()> { + std::fs::create_dir_all(&self.dir) + .with_context(|| format!("failed to create {}", self.dir.display()))?; + let target = self.path(name)?; + // Write beside the target and rename, so a reader never sees half a stamp. + let scratch = self + .dir + .join(format!(".{name}.{}{TMP_EXT}", std::process::id())); + let result = std::fs::write(&scratch, contents) + .context("failed to write the stamp") + .and_then(|()| { + std::fs::rename(&scratch, &target).context("failed to move the stamp into place") + }); + if result.is_err() { + // Do not leave the scratch file behind; `prune` only sees what a + // later successful write lists. + let _ = std::fs::remove_file(&scratch); + } + result + } + + fn list(&self) -> Result> { + let mut entries = Vec::new(); + let read_dir = match std::fs::read_dir(&self.dir) { + Ok(read_dir) => read_dir, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(entries), + Err(error) => return Err(error).context("failed to list the stamps"), + }; + for entry in read_dir.flatten() { + let name = entry.file_name().to_string_lossy().into_owned(); + if !name.ends_with(STAMP_EXT) && !name.ends_with(TMP_EXT) { + continue; + } + let modified = entry + .metadata() + .and_then(|meta| meta.modified()) + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |since| since.as_secs()); + entries.push((name, modified)); + } + Ok(entries) + } + + fn remove(&self, name: &str) -> Result<()> { + match std::fs::remove_file(self.path(name)?) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).context("failed to remove the stamp"), + } + } + + fn location(&self) -> String { + self.dir.display().to_string() + } +} + +/// A store that could not be set up: every read and write reports why, so the +/// step runs and says so. +pub struct UnavailableStore(pub String); + +impl Store for UnavailableStore { + fn read(&self, _name: &str) -> Result> { + bail!("{}", self.0) + } + fn write(&self, _name: &str, _contents: &str) -> Result<()> { + bail!("{}", self.0) + } + fn list(&self) -> Result> { + bail!("{}", self.0) + } + fn remove(&self, _name: &str) -> Result<()> { + bail!("{}", self.0) + } + fn location(&self) -> String { + "the stamp directory".to_string() + } +} + +/// Seconds since the epoch, or 0 when the clock is before it (which makes every +/// stamp look dated in the future, so none is honored). +pub fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_secs()) +} + +#[cfg(test)] +pub(crate) mod fakes { + use std::cell::RefCell; + use std::collections::BTreeMap; + + use anyhow::{Result, bail}; + + use super::{Probe, Store}; + + /// A probe whose answers a test sets, and which counts its calls. + pub struct FakeProbe { + pub tree: RefCell>, + pub toolchain: RefCell>, + pub env: RefCell, + } + + impl FakeProbe { + pub fn clean(tree: &str) -> Self { + Self { + tree: RefCell::new(Ok(tree.to_string())), + toolchain: RefCell::new(Ok("rustc 1.96.0\ncargo 1.96.0\n".to_string())), + env: RefCell::new("env-a".to_string()), + } + } + } + + impl Probe for FakeProbe { + fn clean_tree(&self) -> std::result::Result { + self.tree.borrow().clone() + } + fn toolchain(&self) -> std::result::Result { + self.toolchain.borrow().clone() + } + fn env_digest(&self) -> String { + self.env.borrow().clone() + } + } + + /// An in-memory store; `fail_writes` makes every write error. + #[derive(Default)] + pub struct MemStore { + pub files: RefCell>, + pub fail_writes: RefCell, + pub writes: RefCell, + } + + impl Store for MemStore { + fn read(&self, name: &str) -> Result> { + Ok(self.files.borrow().get(name).map(|(text, _)| text.clone())) + } + fn write(&self, name: &str, contents: &str) -> Result<()> { + if *self.fail_writes.borrow() { + bail!("read-only file system"); + } + *self.writes.borrow_mut() += 1; + // The mtime is the stamp's own `passed-at`, as on a real disk. + let modified = contents + .lines() + .find_map(|line| line.strip_prefix("passed-at: ")) + .and_then(|value| value.parse().ok()) + .unwrap_or(0); + self.files + .borrow_mut() + .insert(name.to_string(), (contents.to_string(), modified)); + Ok(()) + } + fn list(&self) -> Result> { + Ok(self + .files + .borrow() + .iter() + .map(|(name, (_, modified))| (name.clone(), *modified)) + .collect()) + } + fn remove(&self, name: &str) -> Result<()> { + self.files.borrow_mut().remove(name); + Ok(()) + } + fn location(&self) -> String { + "".to_string() + } + } +} + +#[cfg(test)] +mod tests { + use super::fakes::{FakeProbe, MemStore}; + use super::*; + + const NOW: u64 = 1_800_000_000; + + fn args(text: &str) -> Vec { + text.split_whitespace().map(String::from).collect() + } + + fn stamps<'a>(probe: &'a FakeProbe, store: &'a MemStore, now: u64) -> Stamps<'a> { + Stamps { + probe, + store, + now, + rerun: false, + } + } + + /// Record a pass of `test -p xtask` and return the stamp's file name. + fn record_pass(probe: &FakeProbe, store: &MemStore, now: u64) -> String { + let stamps = stamps(probe, store, now); + let Lookup::Miss { key: Some(key), .. } = stamps.lookup("test", &args("test -p xtask")) + else { + panic!("expected a miss with a key"); + }; + assert_eq!(stamps.record(&key), None); + key.file_name() + } + + // ---- the key ---- + + fn base_key() -> Key { + Key { + step: "test".to_string(), + args: "test -p xtask".to_string(), + tree: "tree".to_string(), + toolchain: "toolchain".to_string(), + env: "env".to_string(), + } + } + + #[test] + fn every_part_of_the_key_changes_the_stamp_name() { + let base = base_key(); + let variants = [ + Key { + step: "clippy".into(), + ..base.clone() + }, + Key { + args: "test --workspace".into(), + ..base.clone() + }, + Key { + tree: "other".into(), + ..base.clone() + }, + Key { + toolchain: "other".into(), + ..base.clone() + }, + Key { + env: "other".into(), + ..base.clone() + }, + ]; + for variant in &variants { + assert_ne!(variant.file_name(), base.file_name(), "{variant:?}"); + } + assert_eq!( + base.file_name(), + base_key().file_name(), + "the name is stable" + ); + let name = base.file_name(); + assert!( + name.starts_with("test-") && name.strip_suffix(".stamp").is_some(), + "{name}" + ); + } + + // ---- age: expiry and a clock that disagrees ---- + + #[test] + fn a_pass_is_honored_up_to_the_maximum_age_and_not_a_second_longer() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + record_pass(&probe, &store, NOW); + let at = |now| stamps(&probe, &store, now).lookup("test", &args("test -p xtask")); + assert!(matches!(at(NOW + MAX_AGE_SECS), Lookup::Hit { .. })); + // Expired: a quiet miss that is allowed to record a fresh pass. + match at(NOW + MAX_AGE_SECS + 1) { + Lookup::Miss { key, note } => { + assert!(key.is_some()); + assert_eq!(note, None); + } + Lookup::Hit { .. } => panic!("an expired pass must not skip the step"), + } + } + + #[test] + fn a_stamp_dated_in_the_future_is_ignored_and_says_so() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let name = record_pass(&probe, &store, NOW + 10 * 60); + match stamps(&probe, &store, NOW).lookup("test", &args("test -p xtask")) { + Lookup::Miss { note, .. } => { + let note = note.expect("a note"); + assert!( + note.contains(&name) && note.contains("dated in the future"), + "{note}" + ); + } + Lookup::Hit { .. } => panic!("a future-dated stamp must not skip the step"), + } + // A little clock skew is tolerated. + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + record_pass(&probe, &store, NOW + 60); + assert!(matches!( + stamps(&probe, &store, NOW).lookup("test", &args("test -p xtask")), + Lookup::Hit { .. } + )); + } + + #[test] + fn a_stamp_from_another_format_or_with_a_bad_time_is_ignored() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let name = record_pass(&probe, &store, NOW); + let good = store.files.borrow()[&name].0.clone(); + for (broken, why) in [ + ( + good.replace("pass stamp v1", "pass stamp v2"), + "does not match", + ), + (good.replace("tree: t", "tree: u"), "does not match"), + ( + good.replace(&format!("passed-at: {NOW}"), "passed-at: soon"), + "time stamp is unreadable", + ), + ( + good.replace(&format!("passed-at: {NOW}\n"), ""), + "time stamp is unreadable", + ), + ] { + store.files.borrow_mut().insert(name.clone(), (broken, NOW)); + match stamps(&probe, &store, NOW + 1).lookup("test", &args("test -p xtask")) { + Lookup::Miss { note, .. } => { + let note = note.expect("a note"); + assert!(note.contains(why), "{note}"); + } + Lookup::Hit { .. } => panic!("a broken stamp must not skip the step: {why}"), + } + } + } + + // ---- expiry and the size cap ---- + + #[test] + fn recording_prunes_expired_stamps_and_keeps_the_newest_up_to_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert( + "old-a.stamp".into(), + (String::new(), NOW - MAX_AGE_SECS - 10), + ); + files.insert( + "old-b.stamp".into(), + (String::new(), NOW - MAX_AGE_SECS - 20), + ); + for index in 0..(MAX_STAMPS + 5) { + files.insert( + format!("fresh-{index:03}.stamp"), + (String::new(), NOW - 1000 + index as u64), + ); + } + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + assert_eq!( + files.len(), + MAX_STAMPS, + "{:?}", + files.keys().collect::>() + ); + assert!(!files.contains_key("old-a.stamp") && !files.contains_key("old-b.stamp")); + assert!( + !files.contains_key("fresh-000.stamp"), + "the oldest beyond the cap go first" + ); + assert!( + files.contains_key(&format!("fresh-{:03}.stamp", MAX_STAMPS + 4)), + "the newest stay" + ); + assert!( + files.keys().any(|name| name.starts_with("test-")), + "and so does the new pass" + ); + } + + #[test] + fn expired_stamps_are_pruned_even_when_the_store_is_far_below_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert("old.stamp".into(), (String::new(), NOW - MAX_AGE_SECS - 1)); + files.insert("edge.stamp".into(), (String::new(), NOW - MAX_AGE_SECS)); + files.insert("fresh.stamp".into(), (String::new(), NOW - 60)); + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + assert!( + !files.contains_key("old.stamp"), + "{:?}", + files.keys().collect::>() + ); + assert!( + files.contains_key("edge.stamp"), + "a stamp exactly at the limit is still honored" + ); + assert!(files.contains_key("fresh.stamp")); + } + + // ---- the environment ---- + + fn vars(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect() + } + + #[test] + fn the_environment_digest_follows_cargo_and_rust_settings_only() { + let base = env_digest(vars(&[("PATH", "/bin"), ("RUSTFLAGS", "-D warnings")])); + // Settings that can change a step's outcome change the digest ... + assert_ne!( + base, + env_digest(vars(&[("RUSTFLAGS", "-C target-cpu=native")])) + ); + assert_ne!( + base, + env_digest(vars(&[ + ("RUSTFLAGS", "-D warnings"), + ("CARGO_BUILD_TARGET", "x") + ])) + ); + // ... unrelated or per-run variables do not, whatever their order. + assert_eq!( + base, + env_digest(vars(&[ + ("HOME", "/home/a"), + ("CARGO_TARGET_DIR", "/elsewhere"), + ("CARGO_TERM_COLOR", "always"), + ("CARGO_PKG_NAME", "xtask"), + ("CARGO_MANIFEST_DIR", "/repo/xtask"), + ("RUST_BACKTRACE", "1"), + ("RUSTFLAGS", "-D warnings"), + ])) + ); + assert_eq!( + env_digest(vars(&[("RUSTFLAGS", "a"), ("CARGO_INCREMENTAL", "0")])), + env_digest(vars(&[("CARGO_INCREMENTAL", "0"), ("RUSTFLAGS", "a")])), + ); + } + + #[test] + fn no_environment_value_ends_up_in_a_stamp() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + *probe.env.borrow_mut() = env_digest(vars(&[("CARGO_REGISTRY_TOKEN", "hunter2-secret")])); + let name = record_pass(&probe, &store, NOW); + let text = store.files.borrow()[&name].0.clone(); + assert!( + !text.contains("hunter2") && !text.contains("CARGO_REGISTRY_TOKEN"), + "{text}" + ); + } + + #[test] + fn rerun_comes_from_the_flag_or_a_non_empty_non_zero_variable() { + let var = |value: Option<&'static str>| { + move |name: &str| { + (name == RERUN_VAR) + .then(|| value.map(String::from)) + .flatten() + } + }; + assert!(rerun_requested(true, var(None))); + assert!(rerun_requested(false, var(Some("1")))); + assert!(rerun_requested(false, var(Some("yes")))); + assert!(!rerun_requested(false, var(Some("0")))); + assert!(!rerun_requested(false, var(Some("")))); + assert!(!rerun_requested(false, var(Some(" ")))); + assert!(!rerun_requested(false, var(None))); + } + + #[test] + fn ages_read_as_minutes_hours_or_days() { + assert_eq!(age_text(5), "under a minute"); + assert_eq!(age_text(90), "1 min"); + assert_eq!(age_text(2 * 3600 + 5), "2 h"); + assert_eq!(age_text(3 * 86_400), "3 d"); + } + + // ---- the real store on a real disk ---- + + #[test] + fn the_file_store_round_trips_lists_only_stamps_and_removes_idempotently() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + assert_eq!( + store.read("a.stamp").expect("read"), + None, + "a missing stamp is not an error" + ); + assert!( + store.list().expect("list").is_empty(), + "nor is a missing directory" + ); + + store.write("a.stamp", "one").expect("write"); + store + .write("a.stamp", "two") + .expect("an overwrite replaces it"); + assert_eq!(store.read("a.stamp").expect("read").as_deref(), Some("two")); + std::fs::write(dir.path().join("passes/notes.txt"), "x").expect("a foreign file"); + + let listed = store.list().expect("list"); + assert_eq!(listed.len(), 1, "{listed:?}"); + assert_eq!(listed[0].0, "a.stamp"); + assert!( + listed[0].1 > 1_000_000_000, + "the modified time is read: {listed:?}" + ); + let leftovers = std::fs::read_dir(dir.path().join("passes")) + .expect("dir") + .count(); + assert_eq!( + leftovers, 2, + "no scratch file is left behind (the stamp and the foreign file)" + ); + + store.remove("a.stamp").expect("remove"); + store.remove("a.stamp").expect("removing it again is fine"); + assert_eq!(store.read("a.stamp").expect("read"), None); + } + + #[test] + fn the_file_store_refuses_names_that_leave_its_directory() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + for name in ["../escape.stamp", "a/b.stamp", "a\\b.stamp", "..", ".", ""] { + // The guard itself refuses, not whatever the filesystem then does. + let refusals = [ + store.write(name, "x").unwrap_err(), + store.read(name).unwrap_err(), + store.remove(name).unwrap_err(), + ]; + for error in refusals { + assert!( + format!("{error:#}").contains("refusing the stamp name"), + "{name}: {error:#}" + ); + } + } + assert!(!dir.path().join("escape.stamp").exists()); + } + + // ---- git: the tree hash, the dirty check and the shared location ---- + + fn git_in(dir: &Path, args: &[&str]) -> String { + let output = git_command(Some(dir)) + .args([ + "-c", + "user.name=t", + "-c", + "user.email=t@example.com", + "-c", + "commit.gpgsign=false", + ]) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {args:?} failed in {}", + dir.display() + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + /// A repository with one commit, in a fresh directory. + fn repo() -> tempfile::TempDir { + let dir = tempfile::tempdir().expect("tempdir"); + git_in(dir.path(), &["init", "-q"]); + std::fs::write(dir.path().join("a.txt"), "one\n").expect("write"); + git_in(dir.path(), &["add", "a.txt"]); + git_in(dir.path(), &["commit", "-q", "-m", "first"]); + dir + } + + fn probe_for(dir: &Path) -> SystemProbe { + SystemProbe { + dir: Some(dir.to_path_buf()), + } + } + + #[test] + fn a_clean_tree_is_keyed_by_its_tree_hash_which_an_amend_does_not_change() { + let repo = repo(); + let probe = probe_for(repo.path()); + let tree = probe.clean_tree().expect("a clean tree"); + assert_eq!(tree, git_in(repo.path(), &["rev-parse", HEAD_TREE])); + + let commit = git_in(repo.path(), &["rev-parse", "HEAD"]); + git_in(repo.path(), &["commit", "-q", "--amend", "-m", "reworded"]); + assert_ne!( + git_in(repo.path(), &["rev-parse", "HEAD"]), + commit, + "the commit hash changed" + ); + assert_eq!( + probe.clean_tree().expect("still clean"), + tree, + "the tree hash did not" + ); + + std::fs::write(repo.path().join("a.txt"), "two\n").expect("write"); + git_in(repo.path(), &["commit", "-q", "-am", "second"]); + assert_ne!( + probe.clean_tree().expect("clean"), + tree, + "new content is a new tree" + ); + } + + #[test] + fn uncommitted_or_untracked_files_make_the_tree_unusable_for_stamps() { + let repo = repo(); + let probe = probe_for(repo.path()); + probe.clean_tree().expect("clean to start with"); + + std::fs::write(repo.path().join("a.txt"), "changed\n").expect("write"); + let why = probe.clean_tree().expect_err("a modified tracked file"); + assert!(why.contains("1 uncommitted or untracked path"), "{why}"); + assert!( + why.contains("git status"), + "the reason says how to fix it: {why}" + ); + git_in(repo.path(), &["checkout", "--", "a.txt"]); + probe.clean_tree().expect("clean again"); + + // An untracked file can hide a missing `git add`, so it counts too. + std::fs::write(repo.path().join("new.rs"), "mod x;\n").expect("write"); + assert!(probe.clean_tree().is_err()); + } + + #[test] + fn linked_worktrees_share_one_stamp_directory_outside_every_work_tree() { + let repo = repo(); + let linked = tempfile::tempdir().expect("tempdir"); + let linked_path = linked.path().join("wt"); + git_in( + repo.path(), + &[ + "worktree", + "add", + "-q", + linked_path.to_str().expect("utf-8"), + "-b", + "side", + ], + ); + + let main = FsStore::in_git_common_dir(Some(repo.path())).expect("store"); + let side = FsStore::in_git_common_dir(Some(&linked_path)).expect("store"); + let canonical = |store: &FsStore| { + std::fs::canonicalize(store.dir.parent().expect("parent")).expect("exists") + }; + assert_eq!( + canonical(&main), + canonical(&side), + "one directory for the whole clone" + ); + assert!(main.dir.ends_with(STAMP_DIR)); + assert!(canonical(&main).ends_with(".git"), "{:?}", canonical(&main)); + + // A stamp written from one worktree is read from the other, and writing + // it does not dirty either work tree. + main.write("t.stamp", "shared").expect("write"); + assert_eq!( + side.read("t.stamp").expect("read").as_deref(), + Some("shared") + ); + probe_for(repo.path()) + .clean_tree() + .expect("the main work tree is still clean"); + probe_for(&linked_path) + .clean_tree() + .expect("so is the linked one"); + } + + #[test] + fn a_directory_git_does_not_know_has_no_store_and_no_usable_tree() { + let dir = tempfile::tempdir().expect("tempdir"); + // The temp directory is not normally inside a repository; if it were, + // git would answer and there would be nothing to assert. + if git_stdout(Some(dir.path()), &["rev-parse", "--git-dir"]).is_err() { + assert!(probe_for(dir.path()).clean_tree().is_err()); + assert!(FsStore::in_git_common_dir(Some(dir.path())).is_err()); + } + } + + #[test] + fn the_toolchain_is_identified_from_rustc_and_cargo() { + let text = SystemProbe::default() + .toolchain() + .expect("rustc and cargo run in the test environment"); + assert!(text.contains("rustc") && text.contains("cargo"), "{text}"); + } + + // ---- hermetic git: a hostile ambient GIT_DIR must not redirect any of it ---- + + const HOSTILE_CHILD_VAR: &str = "ROCM_STAMP_TEST_HOSTILE_CHILD"; + + #[test] + fn an_explicit_directory_drops_the_repository_selecting_environment() { + let with_dir = git_command(Some(Path::new("scratch"))); + let envs: Vec<_> = with_dir.get_envs().collect(); + for var in [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_COMMON_DIR", + ] { + assert!( + envs.iter() + .any(|(name, value)| name.to_str() == Some(var) && value.is_none()), + "{var} must be removed for a command pointed at a directory" + ); + } + // Without a directory the hook's own environment is kept: there GIT_DIR + // correctly names the repository being pushed. + assert_eq!(git_command(None).get_envs().count(), 0); + } + + /// Does nothing on its own. The test below re-runs this binary with a hostile + /// git environment and this variable set, and this body then drives the real + /// probe, store and test helpers against a scratch repository. + #[test] + fn hostile_environment_child() { + let Some(dir) = std::env::var_os(HOSTILE_CHILD_VAR) else { + return; + }; + let dir = PathBuf::from(dir); + assert!( + std::env::var_os("GIT_DIR").is_some(), + "the child is supposed to run under a hostile GIT_DIR" + ); + let probe = probe_for(&dir); + let tree = git_in(&dir, &["rev-parse", HEAD_TREE]); + assert_eq!( + probe.clean_tree().expect("the scratch repository's tree"), + tree + ); + + // Committing through the helper lands in the scratch repository. + std::fs::write(dir.join("b.txt"), "two\n").expect("write"); + git_in(&dir, &["add", "b.txt"]); + git_in(&dir, &["commit", "-q", "-m", "second"]); + assert_ne!(probe.clean_tree().expect("clean"), tree); + + let store = FsStore::in_git_common_dir(Some(&dir)).expect("store"); + let scratch = std::fs::canonicalize(&dir).expect("canonical"); + let stamp_parent = + std::fs::canonicalize(store.dir.parent().expect("parent")).expect("exists"); + assert!( + stamp_parent.starts_with(&scratch), + "the store must be in the scratch repository, not the outer one: {stamp_parent:?}" + ); + println!("hostile-child-ran"); + } + + #[test] + fn git_driven_behaviour_ignores_a_hostile_git_environment() { + let outer = repo(); + let inner = repo(); + let outer_git = outer.path().join(".git"); + let outer_head = git_in(outer.path(), &["rev-parse", "HEAD"]); + + let output = Command::new(std::env::current_exe().expect("this test binary")) + .args([ + "--exact", + "prepush_stamp::tests::hostile_environment_child", + "--nocapture", + "--test-threads=1", + ]) + .env(HOSTILE_CHILD_VAR, inner.path()) + .env("GIT_DIR", &outer_git) + .env("GIT_WORK_TREE", outer.path()) + .env("GIT_INDEX_FILE", outer_git.join("index")) + .env("GIT_COMMON_DIR", &outer_git) + .output() + .expect("re-run the test binary"); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + assert!( + stdout.contains("hostile-child-ran"), + "the child did not run its body: {stdout}" + ); + + // The outer repository is exactly as it was: no stray commit, nothing staged. + assert_eq!(git_in(outer.path(), &["rev-parse", "HEAD"]), outer_head); + assert_eq!(git_in(outer.path(), &["rev-list", "--count", "HEAD"]), "1"); + assert_eq!(git_in(outer.path(), &["status", "--porcelain"]), ""); + } + + // ---- scratch files left by a write that died ---- + + #[test] + fn a_write_that_cannot_be_moved_into_place_leaves_no_scratch_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + // A non-empty directory where the stamp should go makes the rename fail. + std::fs::create_dir_all(store.dir.join("a.stamp")).expect("mkdir"); + std::fs::write(store.dir.join("a.stamp").join("inner"), "x").expect("write"); + + let error = store + .write("a.stamp", "x") + .expect_err("a directory is in the way"); + assert!( + format!("{error:#}").contains("failed to move the stamp into place"), + "{error:#}" + ); + let names: Vec = std::fs::read_dir(&store.dir) + .expect("dir") + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + names, + ["a.stamp"], + "no scratch file is left behind: {names:?}" + ); + } + + #[test] + fn stale_scratch_files_are_pruned_and_fresh_ones_kept_and_none_count_toward_the_cap() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + { + let mut files = store.files.borrow_mut(); + files.insert( + ".a.stamp.1.tmp".into(), + (String::new(), NOW - TMP_MAX_AGE_SECS - 1), + ); + files.insert( + ".b.stamp.2.tmp".into(), + (String::new(), NOW - TMP_MAX_AGE_SECS), + ); + files.insert(".c.stamp.3.tmp".into(), (String::new(), NOW - 60)); + for index in 0..MAX_STAMPS { + files.insert( + format!("fresh-{index:03}.stamp"), + (String::new(), NOW - 1000 + index as u64), + ); + } + } + record_pass(&probe, &store, NOW); + let files = store.files.borrow(); + let names: Vec<&String> = files.keys().collect(); + assert!( + !files.contains_key(".a.stamp.1.tmp"), + "a dead writer's scratch file goes: {names:?}" + ); + assert!( + files.contains_key(".b.stamp.2.tmp"), + "one exactly at the bound stays" + ); + assert!( + files.contains_key(".c.stamp.3.tmp"), + "a young one may be a live writer's" + ); + let stamps = files + .keys() + .filter(|name| name.ends_with(STAMP_EXT)) + .count(); + assert_eq!( + stamps, MAX_STAMPS, + "scratch files do not push stamps out: {names:?}" + ); + } + + #[test] + fn a_real_store_prunes_a_dead_writers_scratch_file() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = FsStore { + dir: dir.path().join("passes"), + }; + std::fs::create_dir_all(&store.dir).expect("mkdir"); + let now = now_secs(); + let age_of = |name: &str, secs: u64| { + let path = store.dir.join(name); + std::fs::write(&path, "x").expect("write"); + std::fs::File::options() + .write(true) + .open(&path) + .expect("open") + .set_modified(SystemTime::now() - std::time::Duration::from_secs(secs)) + .expect("set the modified time"); + path + }; + let dead = age_of(".a.stamp.1.tmp", 2 * 60 * 60); + let live = age_of(".b.stamp.2.tmp", 60); + + let probe = FakeProbe::clean("t"); + let stamps = Stamps::new(&probe, &store, now, false, |_| None); + let Lookup::Miss { key: Some(key), .. } = stamps.lookup("test", &args("test -p xtask")) + else { + panic!("expected a miss with a key"); + }; + assert_eq!(stamps.record(&key), None); + assert!( + !dead.exists(), + "the dead writer's scratch file is removed from disk" + ); + assert!(live.exists(), "the young one is left alone"); + } + + // ---- Stamps::new: the --rerun flag and the variable reach the lookup ---- + + #[test] + fn stamps_new_reads_the_flag_and_the_variable() { + let (probe, store) = (FakeProbe::clean("t"), MemStore::default()); + let var = + |value: &'static str| move |name: &str| (name == RERUN_VAR).then(|| value.to_string()); + assert!(Stamps::new(&probe, &store, NOW, true, |_| None).rerun); + assert!(Stamps::new(&probe, &store, NOW, false, var("1")).rerun); + assert!(!Stamps::new(&probe, &store, NOW, false, var("0")).rerun); + assert!(!Stamps::new(&probe, &store, NOW, false, |_| None).rerun); + } +} From 2bde69670e2063d0c5326ee2b6af75aacf5bfa6f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 14:52:35 +0000 Subject: [PATCH 06/11] test(xtask): pin that an empty changed-file list runs the whole workspace on the stamped path A pushed range that lists no changed files means the range is wrong, not that nothing changed. resolve_selection already turns that into the whole workspace, and the stamped path goes through it, but nothing exercised the two together, so a change that let an empty list through to the "no Rust crate" skip would also bypass the pass stamps. The new test records a narrow pass, then runs with an empty list and expects the whole-workspace test command to run, to be recorded as its own pass and to be skipped only on its own next run, and expects the lint step to fall back the same way. It fails when the empty-range check in resolve_selection is removed. Signed-off-by: Jussi Elo --- xtask/src/prepush.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 705c760aa0..c4e8044f8a 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -1185,6 +1185,73 @@ mod tests { assert_eq!(store.files.borrow().len(), 1, "a skip records nothing"); } + /// A pushed range that lists no changed files means the range is wrong, not + /// that nothing changed, so on the stamped path too it runs the whole + /// workspace: it is never skipped as "no Rust crate", an earlier narrow pass + /// does not cover it, and it is skipped only by its own earlier pass. + #[test] + fn an_empty_changed_file_list_runs_the_whole_workspace_on_the_stamped_path() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let narrow = run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &["xtask/src/main.rs"], + false, + false, + ); + assert_eq!(narrow.cargo, [XTASK_TEST]); + + let empty = run_at( + &stamps(&probe, &store, NOW + 5, false), + Check::Test, + &[], + false, + false, + ); + empty.result.expect("the empty-range run"); + assert_eq!(empty.cargo, [WIDE_TEST], "{}", empty.out); + assert!( + empty + .out + .contains("the pushed range lists no changed files"), + "{}", + empty.out + ); + assert!(!empty.out.contains("skipped"), "{}", empty.out); + assert_eq!(store.files.borrow().len(), 2, "the wide pass is recorded"); + + let again = run_at( + &stamps(&probe, &store, NOW + 10, false), + Check::Test, + &[], + false, + false, + ); + assert!(again.cargo.is_empty(), "{:?}", again.cargo); + assert!( + again + .out + .contains(&format!("this tree already passed `cargo {WIDE_TEST}`")), + "{}", + again.out + ); + + // The lint step falls back the same way and is not skipped as "no crate". + let clippy = run_at( + &stamps(&probe, &store, NOW + 15, false), + Check::Clippy, + &[], + false, + false, + ); + assert_eq!( + clippy.cargo, + ["clippy --workspace --all-targets -- -D warnings"], + "{}", + clippy.out + ); + } + #[test] fn a_narrow_pass_never_lets_a_wider_selection_skip() { let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); From 7886647f752a2ffc3bbb8334679c9743fd47361c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 14:56:59 +0000 Subject: [PATCH 07/11] dev: drop this step's earlier pass before a --rerun so a failed rerun cannot leave it behind --rerun exists for the cases the pass key does not cover, such as a change to user-level cargo configuration or an installed tool. Before, a rerun ignored the recorded pass but left it in place, so when the rerun failed the next ordinary run still skipped on the pass the rerun was meant to double-check. A rerun that is going to run now removes exactly this step's matching stamp first, through a new Stamps::invalidate, and a successful rerun records a fresh pass as before. Stamps of other steps and other selections are different files and stay, and --dry-run still touches nothing. New tests cover pass, failed rerun and ordinary retry (which must run the step), a failed rerun leaving sibling stamps alone, a dry-run rerun leaving every stamp in place, and a passing rerun replacing the pass; each fails when its part of the change is removed. Signed-off-by: Jussi Elo --- CONTRIBUTING.md | 2 +- docs/testing.md | 5 +- xtask/src/prepush.rs | 190 +++++++++++++++++++++++++++++++++++++ xtask/src/prepush_stamp.rs | 19 +++- 4 files changed, 213 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 859ed0f3e1..8704da4e68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ prek install -t pre-push # heavier checks on push (clippy + tests of the crat `prek` runs the same checks locally that CI enforces: `cargo fmt`, `clippy`, `cargo test`, `ruff` (Python), `shellcheck` (shell), PowerShell syntax, license headers (`hawkeye`), markdown links (`lychee`), and the generated manifests (`MANIFEST.md`, `THIRD_PARTY_NOTICES.txt`). -The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally or the pushed range lists no changed files. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them and records a fresh pass; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). +The pre-push hooks (`cargo clippy`, the e2e harness clippy and `cargo test`) check only the crates your push affects: the crates whose files changed plus every crate that depends on them, selected the way CI's `cargo xtask affected` does. They check the whole workspace when a change cannot be confined to specific crates (the lockfile, the root manifest, the toolchain file, workflow files, a doc an xtask test reads, or a path no crate owns), and when the affected crates cannot be determined at all, for example because the remote branch tip is not fetched locally or the pushed range lists no changed files. They also add the crates whose tests read files from outside their own crate: any Rust file adds `xtask`, whose contract tests scan every source file, and `README.md`, `docs/` and `skills/` add `rocm` and `xtask`. Each step prints the scope it chose and why. A step that already passed on this exact content is skipped, and says so: after a green run it records a stamp keyed on the git tree hash of `HEAD`, the exact cargo command (so a pass for a narrow selection never covers a wider one), the toolchain and the `CARGO_*` and `RUST*` environment, under the clone's `.git` directory where every worktree shares it, so a retried push, a manual run before the push or an amend that only rewords the message does not pay for the same checks twice. Stamps are only used and written when the working tree is clean (cargo checks the working tree, not the pushed commit, and an untracked file can hide a missing `git add`), expire after seven days, and do not cover your user-level cargo configuration, installed tools, files git ignores or flaky tests. A skip shows that this tree passed those checks earlier, not that they ran during this push; the `cargo test --workspace` and `cargo clippy --workspace` commands you run by hand are unaffected, and `--rerun` gives a fresh run when you want proof the checks ran for this push. `cargo xtask pre-push-check --rerun`, or `ROCM_PRE_PUSH_RERUN=1` for the hook, ignores them, drops this step's earlier pass so a failed rerun cannot leave it to skip the next run, and records a fresh pass when the step succeeds; `SKIP=` still skips a hook outright, and the commit-signature hook runs on every push. A push that changes no Rust file runs none of the cargo steps, whatever else it changes (a lockfile-only push included). A green hook is an early warning and not the gate: on pull requests CI lints the whole workspace, but its Linux test job runs only the affected crates, and the whole workspace's tests run in the Windows debug job, in the merge queue and on `main`. To see what a push would check without compiling anything, run `cargo xtask pre-push-check test --dry-run --base origin/main` (the steps are `clippy`, `clippy-e2e` and `test`). The manifest hooks only run when you change the dependency graph. They *rewrite* the generated file instead of only reporting that it is stale. When that happens, the commit stops so you can re-stage the refreshed file. diff --git a/docs/testing.md b/docs/testing.md index 71c81108d2..05cfeaacfe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -483,7 +483,10 @@ files git ignores (the one thing a clean `git status` cannot see) and flaky test hook has no flags, so use the variable there), or delete the directory the message names. `SKIP=` still skips a hook outright, and `verify-commits` is a separate hook that checks commit signatures on every push. A skip proves that this tree passed these checks earlier, not that they ran -during this push; for a fresh run before relying on the result, use `--rerun`. +during this push; for a fresh run before relying on the result, use `--rerun`. A rerun replaces the recorded pass: it drops +this step's earlier pass before running, so if the rerun fails the next ordinary run runs the step +instead of skipping on the pass it was meant to double-check, and a successful rerun records a fresh +one. Other steps' and other selections' passes are left alone, and `--dry-run` never touches a stamp. Preview what a push would check without compiling (a dry run also says when a real run would skip): diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index c4e8044f8a..95fbc30c2d 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -441,6 +441,14 @@ pub fn execute( return Ok(()); } + // `--rerun` replaces the recorded pass: drop it first, so a run that fails + // leaves nothing for the next ordinary run to skip on, and write a fresh one + // only once cargo succeeded. + if stamps.rerun { + if let Some(note) = key.as_ref().and_then(|key| stamps.invalidate(key)) { + writeln!(out, "pre-push-check {label}: {note}")?; + } + } run_cargo(&args)?; if let Some(note) = key.and_then(|key| stamps.record(&key)) { writeln!(out, "pre-push-check {label}: {note}")?; @@ -1480,6 +1488,188 @@ mod tests { ); } + /// `--rerun` is used when something the key omits may have changed (user-level + /// cargo configuration, an installed tool). If that rerun fails, the earlier + /// pass must not survive to skip the next ordinary run. + #[test] + fn a_failed_rerun_does_not_leave_the_earlier_pass_to_skip_the_next_ordinary_run() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first, passing run"); + assert_eq!(store.files.borrow().len(), 1); + + let failed = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + true, + ); + assert!(failed.result.is_err(), "the rerun fails"); + assert_eq!(failed.cargo, [XTASK_TEST], "--rerun must run the step"); + assert!( + store.files.borrow().is_empty(), + "the earlier pass is dropped before the rerun and nothing replaces it" + ); + + let retry = run_at( + &stamps(&probe, &store, NOW + 200, false), + Check::Test, + &files, + false, + false, + ); + retry.result.expect("the ordinary retry"); + assert_eq!( + retry.cargo, + [XTASK_TEST], + "the ordinary retry must run the step: {}", + retry.out + ); + assert!(!retry.out.contains("skipped"), "{}", retry.out); + assert_eq!( + store.files.borrow().len(), + 1, + "the passing retry is recorded" + ); + } + + #[test] + fn a_failed_rerun_keeps_the_passes_of_other_steps_and_other_selections() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + for (check, files) in [ + (Check::Test, &["xtask/src/main.rs"][..]), + (Check::Test, &["Cargo.lock"][..]), + (Check::Clippy, &["xtask/src/main.rs"][..]), + ] { + run_at( + &stamps(&probe, &store, NOW, false), + check, + files, + false, + false, + ) + .result + .expect("a recorded pass"); + } + assert_eq!(store.files.borrow().len(), 3); + + let failed = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &["xtask/src/main.rs"], + false, + true, + ); + assert!(failed.result.is_err()); + assert_eq!( + store.files.borrow().len(), + 2, + "only the stamp the rerun replaces is dropped" + ); + // The sibling passes still skip. + for (check, files) in [ + (Check::Test, &["Cargo.lock"][..]), + (Check::Clippy, &["xtask/src/main.rs"][..]), + ] { + let again = run_at( + &stamps(&probe, &store, NOW + 200, false), + check, + files, + false, + false, + ); + assert!(again.cargo.is_empty(), "{:?}: {}", again.cargo, again.out); + } + } + + #[test] + fn a_dry_run_rerun_leaves_every_stamp_in_place() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first run"); + let before = store.files.borrow().clone(); + + let dry = run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + true, + false, + ); + dry.result.expect("the dry run"); + assert!( + dry.cargo.is_empty(), + "a dry run runs nothing: {:?}", + dry.cargo + ); + assert_eq!( + *store.files.borrow(), + before, + "a dry run neither removes nor rewrites a stamp" + ); + } + + #[test] + fn a_passing_rerun_replaces_the_pass_with_a_fresh_one() { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let files = ["xtask/src/main.rs"]; + run_at( + &stamps(&probe, &store, NOW, false), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the first run"); + run_at( + &stamps(&probe, &store, NOW + 100, true), + Check::Test, + &files, + false, + false, + ) + .result + .expect("the passing rerun"); + assert_eq!( + store.files.borrow().len(), + 1, + "one pass, replaced not added" + ); + let after = run_at( + &stamps(&probe, &store, NOW + 130, false), + Check::Test, + &files, + false, + false, + ); + assert!(after.cargo.is_empty(), "{}", after.out); + assert!( + after.out.contains( + "this tree already passed `cargo test -p xtask --all-targets` under a minute ago" + ), + "the fresh pass is the one that skips: {}", + after.out + ); + } + #[test] fn rerun_ignores_a_recorded_pass_and_records_a_fresh_one() { let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); diff --git a/xtask/src/prepush_stamp.rs b/xtask/src/prepush_stamp.rs index e14eb8dc0e..07bbca8ccc 100644 --- a/xtask/src/prepush_stamp.rs +++ b/xtask/src/prepush_stamp.rs @@ -42,7 +42,9 @@ //! **Not covered**, and said in the skip message: user-level cargo //! configuration (`~/.cargo/config.toml`), installed tools, files git ignores, //! network-dependent or flaky tests. `--rerun` or [`RERUN_VAR`] ignores recorded -//! passes (and records a fresh one), and `SKIP=` still skips a hook +//! passes: it drops this step's earlier pass before running, so a rerun that +//! fails leaves nothing for the next ordinary run to skip on, and it records a +//! fresh pass when the step succeeds. `SKIP=` still skips a hook //! outright. The `verify-commits` hook is a separate hook and runs every time. //! //! As elsewhere in this crate the decisions are pure and the world is injected: @@ -320,6 +322,21 @@ impl Stamps<'_> { Ok((age <= MAX_AGE_SECS).then_some(age)) } + /// Drop the recorded pass for exactly this key, before a rerun that will + /// replace it. A rerun exists because something the key does not cover may + /// have changed (user-level cargo configuration, an installed tool), so if it + /// fails the earlier pass must not survive to skip the next ordinary run. + /// Other steps' and other trees' stamps are different files and stay. + /// Returns a note when the stamp could not be removed. + pub fn invalidate(&self, key: &Key) -> Option { + self.store.remove(&key.file_name()).err().map(|error| { + format!( + "could not remove the earlier pass stamp ({error:#}); if this run fails, the \ + next ordinary run may still skip the step" + ) + }) + } + /// Record that the step passed. Returns a note when nothing was recorded. /// /// The tree is probed again first: a stamp is only true when the working From 7934447a6be2c1641a9ced2a6c3b84e74525f3e0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:02:13 +0000 Subject: [PATCH 08/11] dev: read the process environment as raw bytes so a non-Unicode variable cannot abort the pre-push hook The environment part of the pass key came from std::env::vars(), which panics as soon as any variable holds bytes that are not valid Unicode, before the digest could filter out the unrelated ones. On Linux a single stray variable was enough to make `pre-push-check` abort even with --rerun, for a feature that is optional. The probe now reads std::env::vars_os(), and env_digest takes OsStr-like pairs: names are matched as text, which is enough because every prefix and exempt name is ASCII, and values are hashed as the bytes they are, so a relevant non-Unicode value still changes the digest and two different ones do not collapse into one. For Unicode values the hashed bytes are the same as before, so recorded stamps stay valid, and a test pins that. The two other environment lookups in the hook (the rerun variable and the git refs) went through std::env::var(..).ok(), which does not panic but reads a non-Unicode value as unset, so a non-Unicode ROCM_PRE_PUSH_RERUN was ignored; they now go through var_os. New tests cover unrelated and relevant non-Unicode bytes in names and values, and a child-process test that runs the real probe under non-Unicode variables; each fails when its part is reverted. Signed-off-by: Jussi Elo --- docs/testing.md | 3 +- xtask/src/prepush.rs | 11 ++- xtask/src/prepush_stamp.rs | 163 ++++++++++++++++++++++++++++++++++--- 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 05cfeaacfe..3f08f89dd5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -460,7 +460,8 @@ looks for it first. The stamp stands for: the same content pushed to a second remote, still hits; - the compiler and cargo identity (`rustc -vV`, `cargo -vV`) and a digest of the `CARGO_*` and `RUST*` environment (never the values), minus a short list of per-run variables such as - `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. + `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. The environment is read as raw bytes, so a variable whose value is not valid + Unicode never stops the hook: an unrelated one is ignored and a relevant one is hashed as the bytes it is. The working tree has to be clean for a stamp to be used or written, because cargo checks the working tree, not the pushed commit: with uncommitted changes the step runs and says why, and an untracked diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 95fbc30c2d..d44c4e459e 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -469,6 +469,13 @@ fn spawn_cargo(args: &[String]) -> Result<()> { Ok(()) } +/// A process environment variable as text. A value that is not valid Unicode is +/// still a set value (`std::env::var` reports it as an error, which `.ok()` would +/// read as unset, so a non-Unicode `ROCM_PRE_PUSH_RERUN` would be ignored). +fn process_env(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + /// Entry point for the `pre-push-check` subcommand. pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Result<()> { let store: Box = match prepush_stamp::FsStore::in_git_common_dir(None) @@ -482,12 +489,12 @@ pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Re store.as_ref(), prepush_stamp::now_secs(), rerun, - |name| std::env::var(name).ok(), + process_env, ); let request = Request { base, dry_run, - env: Env::from_lookup(|name| std::env::var(name).ok()), + env: Env::from_lookup(process_env), }; execute( check, diff --git a/xtask/src/prepush_stamp.rs b/xtask/src/prepush_stamp.rs index 07bbca8ccc..7ce41bb30c 100644 --- a/xtask/src/prepush_stamp.rs +++ b/xtask/src/prepush_stamp.rs @@ -110,7 +110,11 @@ const ENV_IGNORED: &[&str] = &[ const ENV_IGNORED_PREFIXES: &[&str] = &["CARGO_TERM_", "CARGO_PKG_", "CARGO_BIN_EXE_"]; fn hex_digest(text: &str) -> String { - format!("{:x}", Sha256::digest(text.as_bytes())) + hex_digest_bytes(text.as_bytes()) +} + +fn hex_digest_bytes(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) } /// Whether `--rerun` or [`RERUN_VAR`] asks for recorded passes to be ignored. @@ -123,26 +127,42 @@ pub fn rerun_requested(flag: bool, lookup: impl Fn(&str) -> Option) -> b /// Digest of the environment variables that can change a cargo step, in a /// stable order. Only the digest is kept, never a value. -pub fn env_digest(vars: impl IntoIterator) -> String { - let mut relevant: Vec<(String, String)> = vars +pub fn env_digest(vars: impl IntoIterator) -> String +where + K: AsRef, + V: AsRef, +{ + // The process environment is read as `OsString`s: a variable holding bytes + // that are not valid Unicode (on Linux, any variable at all) must not abort + // an optional check, so nothing here needs a `String`. Names are matched as + // text, which is enough because every prefix and exempt name is ASCII, and + // values are hashed as the bytes they are. + let mut relevant: Vec<(Vec, Vec)> = vars .into_iter() .filter(|(name, _)| { + let name = name.as_ref().to_string_lossy(); ENV_PREFIXES.iter().any(|prefix| name.starts_with(prefix)) - && !ENV_IGNORED.contains(&name.as_str()) + && !ENV_IGNORED.contains(&name.as_ref()) && !ENV_IGNORED_PREFIXES .iter() .any(|prefix| name.starts_with(prefix)) }) + .map(|(name, value)| { + ( + name.as_ref().as_encoded_bytes().to_vec(), + value.as_ref().as_encoded_bytes().to_vec(), + ) + }) .collect(); relevant.sort(); - let mut text = String::new(); + let mut bytes = Vec::new(); for (name, value) in relevant { - text.push_str(&name); - text.push('='); - text.push_str(&value); - text.push('\n'); + bytes.extend_from_slice(&name); + bytes.push(b'='); + bytes.extend_from_slice(&value); + bytes.push(b'\n'); } - hex_digest(&text) + hex_digest_bytes(&bytes) } /// Everything one pass depends on. @@ -506,7 +526,7 @@ impl Probe for SystemProbe { } fn env_digest(&self) -> String { - env_digest(std::env::vars()) + env_digest(std::env::vars_os()) } } @@ -1348,6 +1368,127 @@ mod tests { assert_eq!(git_in(outer.path(), &["status", "--porcelain"]), ""); } + // ---- non-Unicode environment variables must never abort the optional stamp ---- + + #[cfg(unix)] + fn non_unicode(prefix: &str, tail: &[u8]) -> std::ffi::OsString { + use std::os::unix::ffi::OsStringExt; + let mut bytes = prefix.as_bytes().to_vec(); + bytes.extend_from_slice(tail); + std::ffi::OsString::from_vec(bytes) + } + + /// Unicode values hash to the same digest they always did, so stamps recorded + /// before the digest read the environment as bytes stay valid. + #[test] + fn the_environment_digest_of_unicode_values_is_the_digest_of_the_text() { + assert_eq!( + env_digest(vars(&[("RUSTFLAGS", "-D warnings"), ("PATH", "/bin")])), + hex_digest("RUSTFLAGS=-D warnings\n") + ); + } + + #[cfg(unix)] + #[test] + fn an_unrelated_variable_with_non_unicode_bytes_is_ignored_and_does_not_panic() { + let base = env_digest(vars(&[("RUSTFLAGS", "-D warnings")])); + let with_noise = env_digest([ + ( + std::ffi::OsString::from("RUSTFLAGS"), + std::ffi::OsString::from("-D warnings"), + ), + ( + std::ffi::OsString::from("PATH"), + non_unicode("/bin", &[0xff]), + ), + (non_unicode("HOME_", &[0xfe]), std::ffi::OsString::from("x")), + ( + std::ffi::OsString::from("LANG"), + non_unicode("", &[0x80, 0x81]), + ), + ]); + assert_eq!( + with_noise, base, + "unrelated variables never reach the digest" + ); + } + + #[cfg(unix)] + #[test] + fn a_relevant_variable_with_non_unicode_bytes_is_hashed_as_the_bytes_it_is() { + let digest = |value: &[u8]| { + env_digest([( + std::ffi::OsString::from("RUSTFLAGS"), + non_unicode("", value), + )]) + }; + let absent = env_digest(Vec::<(String, String)>::new()); + assert_ne!(digest(&[0xff]), absent, "the variable counts"); + // A lossy conversion would turn both bytes into U+FFFD and collapse them. + assert_ne!( + digest(&[0xff]), + digest(&[0xfe]), + "the bytes are hashed, not a lossy text" + ); + assert_eq!(digest(&[0xff]), digest(&[0xff]), "and the digest is stable"); + + // The same holds when the invalid bytes are in the name, after a prefix. + let named = |tail: &[u8]| { + env_digest([(non_unicode("CARGO_X", tail), std::ffi::OsString::from("1"))]) + }; + assert_ne!(named(&[0xff]), absent); + assert_ne!(named(&[0xff]), named(&[0xfe])); + } + + const NON_UNICODE_CHILD_VAR: &str = "ROCM_STAMP_TEST_NON_UNICODE_CHILD"; + + /// Does nothing on its own. The test below re-runs this binary with + /// non-Unicode bytes in unrelated and in relevant variables, and this body + /// then drives the real probe: `std::env::vars()` would panic here. + #[test] + fn non_unicode_environment_child() { + if std::env::var_os(NON_UNICODE_CHILD_VAR).is_none() { + return; + } + let probe = SystemProbe::default(); + let first = Probe::env_digest(&probe); + assert_eq!(first, Probe::env_digest(&probe), "the digest is stable"); + assert_eq!(first.len(), 64, "a SHA-256 digest: {first}"); + println!("non-unicode-child-ran {first}"); + } + + #[cfg(unix)] + #[test] + fn the_real_probe_computes_its_digest_under_non_unicode_environment_variables() { + let run = |rustflags: &[u8]| { + let output = Command::new(std::env::current_exe().expect("this test binary")) + .args([ + "--exact", + "prepush_stamp::tests::non_unicode_environment_child", + "--nocapture", + "--test-threads=1", + ]) + .env(NON_UNICODE_CHILD_VAR, "1") + .env("ROCM_STAMP_TEST_UNRELATED", non_unicode("v", &[0xff, 0xfe])) + .env(non_unicode("UNRELATED_", &[0xff]), "x") + .env("RUSTFLAGS", non_unicode("", rustflags)) + .output() + .expect("re-run the test binary"); + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "{stdout}\n{stderr}"); + stdout + .split_once("non-unicode-child-ran ") + .and_then(|(_, rest)| rest.split_whitespace().next()) + .unwrap_or_else(|| panic!("the child did not run its body: {stdout}")) + .to_string() + }; + let one = run(&[0xff]); + // A relevant non-Unicode value changes the digest, losslessly. + assert_ne!(one, run(&[0xfe])); + assert_eq!(one, run(&[0xff])); + } + // ---- scratch files left by a write that died ---- #[test] From 84d72c3b745e41f4f2a331f3e064df630500d1bb Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:09:40 +0000 Subject: [PATCH 09/11] dev: collapse the nested check that drops the earlier pass before a rerun Clippy's collapsible_if lint, with warnings denied as in CI, flagged the nested `if` in `execute` that drops this step's earlier pass before a `--rerun`; it is now one condition with a let chain, and behaves exactly as before. Signed-off-by: Jussi Elo --- xtask/src/prepush.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index d44c4e459e..39254b9360 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -444,10 +444,10 @@ pub fn execute( // `--rerun` replaces the recorded pass: drop it first, so a run that fails // leaves nothing for the next ordinary run to skip on, and write a fresh one // only once cargo succeeded. - if stamps.rerun { - if let Some(note) = key.as_ref().and_then(|key| stamps.invalidate(key)) { - writeln!(out, "pre-push-check {label}: {note}")?; - } + if stamps.rerun + && let Some(note) = key.as_ref().and_then(|key| stamps.invalidate(key)) + { + writeln!(out, "pre-push-check {label}: {note}")?; } run_cargo(&args)?; if let Some(note) = key.and_then(|key| stamps.record(&key)) { From d3184226f76ba5f4073da868c0cb923fb5caa3e5 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:29:44 +0000 Subject: [PATCH 10/11] dev: test that --rerun and ROCM_PRE_PUSH_RERUN reach the stamp logic through the real wiring The tests of the stamp logic built Stamps by hand, so nothing proved that the flag parsed in main.rs, and the variable read by run(), actually arrive at it: a regression that always passed rerun=false would not have been caught. run() is now a thin wrapper over run_in(), which takes the stamp store, the probe, the clock and the environment lookup as a World and holds the wiring from the flag and the variable into Stamps; the pre-push-check arm of main.rs is a function that takes run as a parameter, so a test sees what the dispatch hands over. New tests parse real command lines and check that every flag arrives unchanged, and run run_in over a fake store with a recorded pass: without the flag or the variable the step is skipped, with --rerun or with the variable set it runs, and 0 or empty means off. Each fails when its part is broken: hard-coding rerun=false in the dispatch fails the dispatch test, hard-coding it in run_in fails the flag test, and dropping the variable lookup fails the variable test. The thin real-world run() wrapper, which builds the real git store, is not covered by a unit test. Signed-off-by: Jussi Elo --- xtask/src/main.rs | 71 ++++++++++++++++++-- xtask/src/prepush.rs | 151 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 204 insertions(+), 18 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index eb32cff137..de223062de 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -43,7 +43,7 @@ mod workflow_contract; use std::path::PathBuf; use std::process::ExitCode; -use anyhow::Result; +use anyhow::{Result, bail}; use clap::{Parser, Subcommand}; #[derive(Parser)] @@ -411,12 +411,7 @@ fn run() -> Result<()> { } => signing::verify(public_key.as_deref(), &input, &signature)?, Command::VerifyPinnedKeys => verify_pinned_keys::run()?, Command::Affected { base } => affected::run(base)?, - Command::PrePushCheck { - check, - base, - dry_run, - rerun, - } => prepush::run(check, base, dry_run, rerun)?, + command @ Command::PrePushCheck { .. } => run_pre_push_check(command, prepush::run)?, Command::CheckCrateEdges => crate_edges::run()?, Command::CheckModuleSize => module_size::run()?, Command::CheckModuleSizeAdvisory => module_size::run_advisory()?, @@ -497,6 +492,25 @@ fn run() -> Result<()> { Ok(()) } +/// The `pre-push-check` arm of [`run`]: hand each argument to `run` as parsed. +/// `run` is a parameter so a test sees exactly what arrives, in particular that +/// `--rerun` is not dropped on the way to the stamp logic. +fn run_pre_push_check( + command: Command, + run: impl FnOnce(prepush::Check, Option, bool, bool) -> Result<()>, +) -> Result<()> { + let Command::PrePushCheck { + check, + base, + dry_run, + rerun, + } = command + else { + bail!("not a pre-push-check command"); + }; + run(check, base, dry_run, rerun) +} + fn main() -> ExitCode { match run() { Ok(()) => ExitCode::SUCCESS, @@ -521,6 +535,49 @@ mod tests { rerun } + /// What the dispatch hands to `prepush::run` for these arguments: the real + /// clap parse, then the `pre-push-check` arm, with `run` recording its input. + fn handed_to_run(args: &[&str]) -> (String, Option, bool, bool) { + let cli = Cli::try_parse_from(args).expect("the arguments parse"); + let mut seen = None; + super::run_pre_push_check(cli.command, |check, base, dry_run, rerun| { + seen = Some((format!("{check:?}"), base, dry_run, rerun)); + Ok(()) + }) + .expect("the dispatch succeeds"); + seen.expect("the arm calls run") + } + + #[test] + fn the_dispatch_hands_every_flag_to_run_unchanged() { + assert_eq!( + handed_to_run(&["xtask", "pre-push-check", "test"]), + ("Test".to_string(), None, false, false) + ); + assert_eq!( + handed_to_run(&[ + "xtask", + "pre-push-check", + "clippy", + "--base", + "origin/main", + "--dry-run", + "--rerun", + ]), + ( + "Clippy".to_string(), + Some("origin/main".to_string()), + true, + true + ) + ); + // `--rerun` alone, and `--dry-run` alone, are not mixed up with each other. + assert!(handed_to_run(&["xtask", "pre-push-check", "test", "--rerun"]).3); + assert!(!handed_to_run(&["xtask", "pre-push-check", "test", "--rerun"]).2); + assert!(handed_to_run(&["xtask", "pre-push-check", "test", "--dry-run"]).2); + assert!(!handed_to_run(&["xtask", "pre-push-check", "test", "--dry-run"]).3); + } + #[test] fn pre_push_check_takes_rerun_and_defaults_to_off() { assert!(rerun(&["xtask", "pre-push-check", "test", "--rerun"])); diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 39254b9360..9ac9fec066 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -476,6 +476,23 @@ fn process_env(name: &str) -> Option { std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) } +/// What the `pre-push-check` command line asked for. +pub struct Flags { + pub base: Option, + pub dry_run: bool, + pub rerun: bool, +} + +/// The world one invocation runs against: where passes are kept, how the tree +/// and toolchain are probed, the clock, and how a process variable is read. +pub struct World<'a> { + pub probe: &'a dyn prepush_stamp::Probe, + pub store: &'a dyn prepush_stamp::Store, + /// Seconds since the epoch. + pub now: u64, + pub env: &'a dyn Fn(&str) -> Option, +} + /// Entry point for the `pre-push-check` subcommand. pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Result<()> { let store: Box = match prepush_stamp::FsStore::in_git_common_dir(None) @@ -484,26 +501,59 @@ pub fn run(check: Check, base: Option, dry_run: bool, rerun: bool) -> Re Err(error) => Box::new(prepush_stamp::UnavailableStore(format!("{error:#}"))), }; let probe = prepush_stamp::SystemProbe::default(); - let stamps = Stamps::new( - &probe, - store.as_ref(), - prepush_stamp::now_secs(), + run_in( + check, + Flags { + base, + dry_run, + rerun, + }, + &World { + probe: &probe, + store: store.as_ref(), + now: prepush_stamp::now_secs(), + env: &process_env, + }, + affected::load_graph, + affected::changed_files_between, + spawn_cargo, + &mut std::io::stdout().lock(), + ) +} + +/// [`run`] with the world injected: this is where `--rerun` and +/// `ROCM_PRE_PUSH_RERUN` reach the stamp logic, so a test drives it with a fake +/// store and sees whether a recorded pass is honored. +pub fn run_in( + check: Check, + flags: Flags, + world: &World<'_>, + load_graph: impl FnOnce() -> Result, + changed_files: impl FnOnce(&str, &str) -> Result>, + run_cargo: impl FnMut(&[String]) -> Result<()>, + out: &mut impl Write, +) -> Result<()> { + let Flags { + base, + dry_run, rerun, - process_env, - ); + } = flags; + let stamps = Stamps::new(world.probe, world.store, world.now, rerun, |name| { + (world.env)(name) + }); let request = Request { base, dry_run, - env: Env::from_lookup(process_env), + env: Env::from_lookup(|name| (world.env)(name)), }; execute( check, request, &stamps, - affected::load_graph, - affected::changed_files_between, - spawn_cargo, - &mut std::io::stdout().lock(), + load_graph, + changed_files, + run_cargo, + out, ) } @@ -1931,6 +1981,85 @@ mod tests { (probe, store) } + // ---- run_in: the flag and the variable reach the stamp logic ---- + + /// Drive the real wiring below `main`: `run_in` builds the stamps from the + /// flag and the environment, here over a fake store. With `recorded_pass` + /// a first plain run records a pass for the tree; the run that follows is + /// the one measured, and the result says whether it ran cargo. + fn measured_run_ran_cargo(rerun: bool, rerun_var: Option<&str>, recorded_pass: bool) -> bool { + let (probe, store) = (FakeProbe::clean("tree-1"), MemStore::default()); + let env = |name: &str| { + if name == prepush_stamp::RERUN_VAR { + rerun_var.map(String::from) + } else { + None + } + }; + let ran = std::cell::Cell::new(false); + let run_once = |now: u64, rerun: bool| { + ran.set(false); + run_in( + Check::Test, + Flags { + base: None, + dry_run: false, + rerun, + }, + &World { + probe: &probe, + store: &store, + now, + env: &env, + }, + || Ok(graph()), + |_, _| Ok(vec!["xtask/src/main.rs".to_string()]), + |_| { + ran.set(true); + Ok(()) + }, + &mut Vec::new(), + ) + .expect("the step"); + ran.get() + }; + if recorded_pass { + // Nothing is on record yet, so this run runs cargo and records a pass. + assert!(run_once(NOW, false), "the first run runs the step"); + assert_eq!(store.files.borrow().len(), 1, "and records a pass"); + } + run_once(NOW + 60, rerun) + } + + #[test] + fn without_the_flag_or_the_variable_a_recorded_pass_skips_the_step() { + assert!(!measured_run_ran_cargo(false, None, true)); + assert!( + !measured_run_ran_cargo(false, Some("0"), true), + "0 means off" + ); + assert!( + !measured_run_ran_cargo(false, Some(""), true), + "empty means off" + ); + } + + #[test] + fn the_flag_reaches_the_stamp_logic_and_reruns_despite_a_recorded_pass() { + assert!(measured_run_ran_cargo(true, None, true)); + } + + #[test] + fn the_variable_reaches_the_stamp_logic_and_reruns_despite_a_recorded_pass() { + assert!(measured_run_ran_cargo(false, Some("1"), true)); + } + + #[test] + fn with_no_recorded_pass_the_step_runs_either_way() { + assert!(measured_run_ran_cargo(false, None, false)); + assert!(measured_run_ran_cargo(true, None, false)); + } + #[test] fn the_rerun_flag_forces_a_run_despite_a_matching_stamp() { let (probe, store) = with_a_recorded_pass(); From ac19326c9dc63be6ddb9919a6540f24438aedae2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:30:59 +0000 Subject: [PATCH 11/11] dev: say exactly what a killed write leaves behind and wrap the pass-stamp docs to the file's width A write goes through a scratch file and a rename and removes the scratch file itself when the rename fails, but a process killed mid-write leaves its scratch file behind until a later write prunes it, once it is more than an hour old; a younger one may belong to a concurrent writer. docs/testing.md and the comment on the age limit now say that, and nothing in the docs or comments claims otherwise. The paragraphs this pull request added to docs/testing.md that ran well past the rest of the file are wrapped to its width; no other text is touched. Signed-off-by: Jussi Elo --- docs/testing.md | 31 ++++++++++++++++++------------- xtask/src/prepush_stamp.rs | 5 +++-- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3f08f89dd5..0f3cd0b810 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -460,8 +460,9 @@ looks for it first. The stamp stands for: the same content pushed to a second remote, still hits; - the compiler and cargo identity (`rustc -vV`, `cargo -vV`) and a digest of the `CARGO_*` and `RUST*` environment (never the values), minus a short list of per-run variables such as - `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. The environment is read as raw bytes, so a variable whose value is not valid - Unicode never stops the hook: an unrelated one is ignored and a relevant one is hashed as the bytes it is. + `CARGO_TARGET_DIR` and `CARGO_TERM_COLOR`. The environment is read as raw bytes, so a variable + whose value is not valid Unicode never stops the hook: an unrelated one is ignored and a + relevant one is hashed as the bytes it is. The working tree has to be clean for a stamp to be used or written, because cargo checks the working tree, not the pushed commit: with uncommitted changes the step runs and says why, and an untracked @@ -472,22 +473,26 @@ the retry then skips. Stamps live in `rocm-cli-prepush-passes` under the git common directory, so every worktree of a clone shares them, nothing lands in the work tree and `cargo clean` does not erase them. Each write -drops stamps older than seven days, keeps at most 64, and removes scratch files left by a write that -died (a write goes through a scratch file and a rename, and cleans it up when the rename fails). Any error locating, reading or verifying a +drops stamps older than seven days and keeps at most 64. A write goes through a scratch file and a +rename, and removes the scratch file itself when the rename fails; a process killed mid-write +leaves its scratch file behind until a later write prunes it, once it is more than an hour old (a +younger one may belong to a concurrent writer). Any error locating, reading or verifying a stamp means the step runs (git is run for an explicit directory with `GIT_DIR` and the other repository-selecting variables removed, which git and prek export to a hook run from a linked worktree); a stamp that does not match its key exactly, is dated in the future or is unreadable is ignored (the step names it); a failed write is reported and never fails the hook. -The skip message says what a stamp does not cover: user-level cargo configuration, installed tools, -files git ignores (the one thing a clean `git status` cannot see) and flaky tests. Force a run with `--rerun` or `ROCM_PRE_PUSH_RERUN=1` (the -hook has no flags, so use the variable there), or delete the directory the message names. `SKIP=` still skips a hook outright, and `verify-commits` is a separate hook that checks commit -signatures on every push. A skip proves that this tree passed these checks earlier, not that they ran -during this push; for a fresh run before relying on the result, use `--rerun`. A rerun replaces the recorded pass: it drops -this step's earlier pass before running, so if the rerun fails the next ordinary run runs the step -instead of skipping on the pass it was meant to double-check, and a successful rerun records a fresh -one. Other steps' and other selections' passes are left alone, and `--dry-run` never touches a stamp. +The skip message says what a stamp does not cover: user-level cargo configuration, installed +tools, files git ignores (the one thing a clean `git status` cannot see) and flaky tests. Force +a run with `--rerun` or `ROCM_PRE_PUSH_RERUN=1` (the hook has no flags, so use the variable +there), or delete the directory the message names. `SKIP=` still skips a hook outright, +and `verify-commits` is a separate hook that checks commit signatures on every push. A skip +proves that this tree passed these checks earlier, not that they ran during this push; for a +fresh run before relying on the result, use `--rerun`. A rerun replaces the recorded pass: it +drops this step's earlier pass before running, so if the rerun fails the next ordinary run runs +the step instead of skipping on the pass it was meant to double-check, and a successful rerun +records a fresh one. Other steps' and other selections' passes are left alone, and `--dry-run` +never touches a stamp. Preview what a push would check without compiling (a dry run also says when a real run would skip): diff --git a/xtask/src/prepush_stamp.rs b/xtask/src/prepush_stamp.rs index 7ce41bb30c..46d5d0290a 100644 --- a/xtask/src/prepush_stamp.rs +++ b/xtask/src/prepush_stamp.rs @@ -81,8 +81,9 @@ pub const MAX_AGE_SECS: u64 = 7 * 24 * 60 * 60; /// At most this many stamps are kept; the oldest go first. pub const MAX_STAMPS: usize = 64; -/// A scratch file older than this belongs to a write that died; a younger one -/// may be a concurrent writer's, so it is left alone. +/// A write killed mid-way leaves its scratch file behind until a later write +/// prunes it, once it is older than this: an older scratch file belongs to a +/// write that died, a younger one may be a concurrent writer's and is left alone. pub const TMP_MAX_AGE_SECS: u64 = 60 * 60; /// A stamp dated this far ahead of the clock is treated as corrupt.