fork: outside PRs run nothing, one checkout without actions/checkout, contract tests

.github/fork-ci-tests.txt

@@ -6,6 +6,7 @@
 # A present test that is not listed fails. A listed file that is not present yet
 # is a note; one that was present and then deleted fails.
 .github/scripts/fork-ci-tests.test.cjs
+.github/scripts/fork-contracts.test.py
 .github/scripts/fork-follow.test.sh
 .github/scripts/fork-release-workflow.test.py
 .github/scripts/fork-release.test.sh

.github/scripts/fork-follow.test.sh

@@ -5,24 +5,9 @@
 set -uo pipefail
 
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-dir="$(realpath "${FORK_FOLLOW_DIR:-$here}")"
-workflow="$(realpath "${FORK_FOLLOW_WORKFLOW:-$here/../workflows/fork-follow.yml}")"
 root="$(mktemp -d)"
 trap 'rm -rf "$root"' EXIT
 
-# Upstream "up/t3code" and this fork "me/t3code" live under $root/gh; git reaches them through the
-# github.com URLs the scripts use.
-export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig"
-git config --global user.name test
-git config --global user.email test@example.invalid
-git config --global init.defaultBranch main
-git config --global advice.detachedHead false
-git config --global url."$root/gh/".insteadOf https://github.com/
-export REPOSITORY=me/t3code GITHUB_REPOSITORY=me/t3code UPSTREAM=up/t3code
-unset GITHUB_OUTPUT GITHUB_STEP_SUMMARY GITHUB_ACTIONS UPSTREAM_URL ORIGIN RUN_SHA PUSH_URL \
-  SERIES_BRANCH FORK_PUSH_TOKEN OWN_WORKFLOWS HEAD_SHA TREE_REF SINCE REPO_SECRET_PRESENT \
-  FORK_PUSH_ENVIRONMENT
-
 # Fake gh, served from $root/api:
 #   gh api [-X M] PATH [--paginate] [--jq EXPR]     gh issue list|comment|create|close ...
 # A file $root/api/fail_<word> makes every call whose path contains <word> fail. Run listings are
@@ -86,6 +71,23 @@ case "$method $path" in
 esac
 if [[ -n "$jq" ]]; then jq -r "$jq" <<< "$body"; else echo "$body"; fi
 GH
+
+dir="$(realpath "${FORK_FOLLOW_DIR:-$here}")"
+workflow="$(realpath "${FORK_FOLLOW_WORKFLOW:-$here/../workflows/fork-follow.yml}")"
+
+# Upstream "up/t3code" and this fork "me/t3code" live under $root/gh; git reaches them through the
+# github.com URLs the scripts use.
+export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig"
+git config --global user.name test
+git config --global user.email test@example.invalid
+git config --global init.defaultBranch main
+git config --global advice.detachedHead false
+git config --global url."$root/gh/".insteadOf https://github.com/
+export REPOSITORY=me/t3code GITHUB_REPOSITORY=me/t3code UPSTREAM=up/t3code
+unset GITHUB_OUTPUT GITHUB_STEP_SUMMARY GITHUB_ACTIONS UPSTREAM_URL ORIGIN RUN_SHA PUSH_URL \
+  SERIES_BRANCH FORK_PUSH_TOKEN OWN_WORKFLOWS HEAD_SHA TREE_REF SINCE REPO_SECRET_PRESENT \
+  FORK_PUSH_ENVIRONMENT
+
 export FAKE_API="$root/api"
 
 failures=0
@@ -655,7 +657,10 @@ echo '{"workflows":[{"id":9,"state":"active","path":".github/workflows/fork-test
 D
 chmod +x "$root/decoy/gh"
 grep -vxF "$FORK_GH_STUB_MARKER" "$root/bin/gh" > "$root/nomarker/gh" && chmod +x "$root/nomarker/gh"
-gh_scripts=(fork-workflows.sh fork-follow-guard.sh fork-follow-report.sh)
+# Every script here that sources the guard (fork-contracts.test.py requires it of each that calls gh).
+mapfile -t gh_scripts < <(cd "$dir" && grep -lxF '. "$(dirname "${BASH_SOURCE[0]}")/fork-gh-guard.sh"' fork-*.sh | grep -v '\.test\.sh$' | sort)
+check "gh: the refusal tests cover the scripts that source the guard" \
+  eval '(for s in fork-follow-guard.sh fork-follow-report.sh fork-release.sh fork-workflows.sh; do [[ " ${gh_scripts[*]} " == *" $s "* ]] || exit 1; done)'
 refuses() { # refuses ENV...: every gh script, run with ENV, fails, says why, and calls no gh
   local s
   for s in "${gh_scripts[@]}"; do
@@ -724,11 +729,8 @@ top_level="$advance_if"
 while [[ "$top_level" =~ \([^()]*\) ]]; do top_level="${top_level//"${BASH_REMATCH[0]}"/_}"; done
 check "workflow: advance's if: is a conjunction (no top-level ||) of the plan's success and this attempt's guard" \
   eval '[[ -n "$advance_if" && "$top_level" != *"||"* && "$top_level" == *"$attempt_check &&"* && "$top_level" == *"needs.plan.result == '"'"'success'"'"' &&"* ]]'
-# A refusal that only warns is no refusal: continue-on-error is allowed in the Report job alone (the
-# guard, the Environment re-check and the sweep before the move must stop their job).
-report_start="$(grep -n "^  report:$" "$workflow" | cut -d: -f1)"
-check "workflow: continue-on-error appears only in the Report job (never on the guard, the re-check or the sweep before the move)" \
-  eval '[[ -n "$report_start" ]] && ! awk -v r="$report_start" "/continue-on-error/ && NR < r { bad = 1 } END { exit !bad }" "$workflow"'
+# A refusal that only warns is no refusal. fork-contracts.test.py ("refusals") checks every way to
+# swallow one (continue-on-error, || true, || :, ; true, set +e, a trap, an if:, another shell, BASH_ENV).
 check "workflow: the move step runs only when the checks before it passed (no if:)" \
   eval 'awk "/- name: Move the series branch/ { s = 1; next } s && /^      - / { exit } s && /^        if:/ { bad = 1 } END { exit bad }" <<< "$advance_job"'
 check "workflow: action none deletes a leftover candidate branch" \
@@ -740,8 +742,10 @@ token_jobs="$(grep -vF "$presence" "$workflow" | awk '/^  [a-z_-]+:$/ { job = $1
 env_jobs="$(awk '/^  [a-z_-]+:$/ { job = $1 } /^    environment: fork-push$/ { print job }' "$workflow" | sort -u)"
 check "workflow: the jobs that read FORK_PUSH_TOKEN, and only they, run in the Environment fork-push" \
   eval '[[ -n "$token_jobs" && "$token_jobs" == "$env_jobs" ]] && grep -qx "  FORK_PUSH_ENVIRONMENT: fork-push" "$workflow"'
-check "workflow: every checkout drops its credentials" \
-  eval '[[ $(grep -c "actions/checkout@" "$workflow") == $(grep -c "persist-credentials: false" "$workflow") ]]'
+# actions/checkout fails on upstream's gitlinks without .gitmodules once it cleans up credentials;
+# every job uses the shared plain-git step (fork-contracts.test.py runs it end to end).
+check "workflow: no job uses actions/checkout; four plain-git checkouts, none with a token" \
+  eval '! grep -q "actions/checkout@" "$workflow" && [[ $(grep -c "^          git init -q$" "$workflow") == 4 ]]'
 check "workflow: every action is pinned by commit" \
   eval '! grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vqE "@[0-9a-f]{40}( |$)"'
 check "workflow: the report runs on every run (also action none)" \

.github/scripts/fork-gh-guard.sh

@@ -9,7 +9,7 @@
 # Anything else refuses: the script exits 1 when it sources this file, before any gh call, and
 # every later call goes through the gh function below, which checks again (a script may change
 # PATH). The marker must match FORK_GH_STUB_MARKER in fork-test-gh.sh. Tests: fork-follow.test.sh
-# and fork-release.test.sh.
+# and fork-release.test.sh; fork-contracts.test.py requires this file of every script calling gh.
 
 FORK_GH_STUB_MARKER="# fork-gh-stub: a test stand-in for gh that never reaches GitHub"
 

.github/scripts/fork-release-workflow.test.py

@@ -62,9 +62,15 @@ def all_steps():
 check("release-desktop.yml gets exactly the secrets it declares", not wrong, ", ".join(wrong))
 check("FORK_PUSH_TOKEN appears nowhere", "FORK_PUSH_TOKEN" not in text)
 
-bad = [j for j, s in all_steps() if str(s.get("uses", "")).startswith("actions/checkout@")
-       and (s.get("with") or {}).get("persist-credentials") is not False]
-check("every checkout sets persist-credentials: false", not bad, ", ".join(bad))
+# actions/checkout fails on upstream's gitlinks without .gitmodules once it cleans up credentials;
+# every job checks out with the shared plain-git step instead (fork-contracts.test.py runs it).
+bad = [j for j, s in all_steps() if str(s.get("uses", "")).startswith("actions/checkout@")]
+check("no job uses actions/checkout", not bad, ", ".join(bad))
+plain = {j for j, s in all_steps() if s.get("name") == "Checkout" and "git init" in str(s.get("run", ""))}
+check("every job that checks out uses the plain-git step, with no token",
+      plain == {"preflight", "build_bundle", "assemble", "publish"}
+      and all("github.token" not in yaml.safe_dump(s) and "secrets." not in yaml.safe_dump(s)
+              for _, s in all_steps() if s.get("name") == "Checkout"), ", ".join(sorted(plain)))
 
 release_steps = [(j, s) for j, s in all_steps() if str(s.get("uses", "")).startswith("softprops/action-gh-release@")]
 check("one release-action step", len(release_steps) == 1)
@@ -132,9 +138,10 @@ def index(pred):
 check("the former name T3CODE_RELEASE_REPOSITORY appears nowhere", "T3CODE_RELEASE_REPOSITORY" not in text)
 
 # The write job's guard script comes from the workflow's own commit, not the commit being built.
-guard = [s for s in steps(jobs["publish"]) if str(s.get("uses", "")).startswith("actions/checkout@")]
-check("publish checks out its guard script at github.workflow_sha",
-      len(guard) == 1 and (guard[0].get("with") or {}).get("ref") == "${{ github.workflow_sha }}")
+guard = [s for s in steps(jobs["publish"]) if "git init" in str(s.get("run", ""))]
+check("publish checks out only its guard scripts, at github.workflow_sha",
+      len(guard) == 1 and guard[0].get("env") == {"REF": "${{ github.workflow_sha }}", "HISTORY": "tip",
+                                                  "SPARSE": "/.github/scripts/"})
 
 if failures:
     print(f"{len(failures)} failed")

.github/scripts/fork-release.test.sh

@@ -4,21 +4,9 @@
 set -euo pipefail
 
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-script="$(realpath "${FORK_RELEASE_SH:-$here/fork-release.sh}")"
 root="$(mktemp -d)"
 trap 'rm -rf "$root"' EXIT
 
-# Upstream "up/t3code" and this fork "me/t3code" live under $root/gh; git reaches them through the
-# github.com URLs the workflow uses.
-export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig"
-git config --global user.name test
-git config --global user.email test@example.invalid
-git config --global init.defaultBranch main
-git config --global advice.detachedHead false
-git config --global url."$root/gh/".insteadOf https://github.com/
-export REPOSITORY=me/t3code GITHUB_REPOSITORY=me/t3code
-unset GITHUB_OUTPUT GITHUB_STEP_SUMMARY UPSTREAM UPSTREAM_URL ORIGIN
-
 # Fake gh: `gh api [-X M] PATH [--paginate] [--jq EXPR] [-f/-F k=v]...` served from $root/api.
 # fork_test_gh (fork-test-gh.sh) installs it first on PATH, with no token, a .invalid GH_HOST and
 # FORK_TEST_MODE, so the script refuses to run if any other gh would answer.
@@ -57,6 +45,20 @@ case "$method $path" in
 esac
 if [[ -n "$jq" ]]; then jq -r "$jq" <<< "$body"; else echo "$body"; fi
 GH
+
+script="$(realpath "${FORK_RELEASE_SH:-$here/fork-release.sh}")"
+
+# Upstream "up/t3code" and this fork "me/t3code" live under $root/gh; git reaches them through the
+# github.com URLs the workflow uses.
+export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig"
+git config --global user.name test
+git config --global user.email test@example.invalid
+git config --global init.defaultBranch main
+git config --global advice.detachedHead false
+git config --global url."$root/gh/".insteadOf https://github.com/
+export REPOSITORY=me/t3code GITHUB_REPOSITORY=me/t3code
+unset GITHUB_OUTPUT GITHUB_STEP_SUMMARY UPSTREAM UPSTREAM_URL ORIGIN
+
 export FAKE_API="$root/api"
 
 failures=0

.github/scripts/fork-schema-check.test.cjs

@@ -448,22 +448,134 @@ test("any action the workflow uses is pinned by commit SHA", () => {
   for (const line of uses) assert.match(line, /@[0-9a-f]{40}$/, line);
 });
 
-test("the workflow runs the default branch's copy and never runs pull request code", () => {
-  const workflow = fs.readFileSync(WORKFLOW, "utf8");
+// The shell scripts of a workflow: each `run:` value with every line after it that is indented past
+// the `run:` key, blank lines included. That covers a `run: |` or `>` block, a value that starts on
+// the next line (`run:` alone) and a plain or quoted scalar continued on later lines; YAML reads all
+// of them as one script.
+function runScripts(workflow) {
+  const lines = workflow.split("\n");
+  const scripts = [];
+  for (let i = 0; i < lines.length; i++) {
+    const m = /^(\s*(?:-\s+)?)run:(?:\s+(.*))?$/.exec(lines[i]);
+    if (!m) continue;
+    const head = (m[2] ?? "").trim();
+    const body = head === "" || /^[|>][-+0-9]*\s*(#.*)?$/.test(head) ? [] : [head];
+    while (
+      i + 1 < lines.length &&
+      (lines[i + 1].trim() === "" || lines[i + 1].search(/\S/) > m[1].length)
+    ) {
+      body.push(lines[++i]);
+    }
+    scripts.push(body.join("\n"));
+  }
+  return scripts;
+}
+
+// Why a workflow may run pull request code or hand it a token; empty when it cannot.
+function prCodeProblems(workflow) {
   const code = workflow.replace(/^\s*#.*$/gm, "");
-  assert.match(code, /^on:\n {2}pull_request_target:/m);
-  assert.doesNotMatch(code, /^\s+pull_request:/m);
+  const problems = [];
+  const expect = (ok, why) => ok || problems.push(why);
+  expect(/^on:\n {2}pull_request_target:/m.test(code), "not triggered by pull_request_target");
+  expect(!/^\s+pull_request:/m.test(code), "triggered by pull_request");
   // The only checkout is of the default branch (`github.sha` under pull_request_target).
-  assert.doesNotMatch(code, /actions\/checkout|\bref:|secrets\.|GITHUB_TOKEN/);
-  assert.deepEqual(code.match(/git (checkout|switch|worktree|reset|read-tree)\b.*/g), [
-    'git checkout -q --detach "$DEFAULT_SHA"',
+  const banned = /actions\/checkout|\bref:|secrets\.|GITHUB_TOKEN|github\.token/.exec(code);
+  expect(!banned, `uses ${banned?.[0]}`);
+  const checkouts = code.match(/git (checkout|switch|worktree|reset|read-tree)\b.*/g) ?? [];
+  expect(
+    checkouts.length === 1 && checkouts[0] === 'git checkout -q --detach "$DEFAULT_SHA"',
+    `checks out ${checkouts.join(", ")}`,
+  );
+  expect(
+    JSON.stringify(code.match(/DEFAULT_SHA:.*/g)) ===
+      JSON.stringify(["DEFAULT_SHA: ${{ github.sha }}"]),
+    "DEFAULT_SHA is not github.sha",
+  );
+  // Event values reach the shell only through env, never pasted into a script.
+  for (const script of runScripts(code)) {
+    expect(
+      !/\$\{\{/.test(script),
+      `an expression inside a run script: ${script.trim().split("\n")[0]}`,
+    );
+  }
+  expect(
+    /node \.github\/scripts\/fork-schema-check\.cjs "\$BASE_SHA" "\$HEAD_SHA"/.test(code),
+    "does not run the check on $BASE_SHA $HEAD_SHA",
+  );
+  return problems;
+}
+
+test("the workflow runs the default branch's copy and never runs pull request code", () => {
+  assert.deepEqual(prCodeProblems(fs.readFileSync(WORKFLOW, "utf8")), []);
+});
+
+test("runScripts reads one-line runs and whole run blocks", () => {
+  const scripts = runScripts(
+    [
+      "      - run: echo one",
+      "      - name: x",
+      "        run: |",
+      "          a",
+      "",
+      "          b",
+      "      - run: >-",
+      "          c",
+      "        env: {}",
+      "      - run:",
+      "          d",
+      "      - run: e",
+      "          f",
+      "        name: y",
+    ].join("\n"),
+  );
+  assert.deepEqual(scripts, [
+    "echo one",
+    "          a\n\n          b",
+    "          c",
+    "          d",
+    "e\n          f",
   ]);
-  assert.deepEqual(code.match(/DEFAULT_SHA:.*/g), ["DEFAULT_SHA: ${{ github.sha }}"]);
-  // Event values reach the shell only through env, never pasted into the script.
-  for (const line of code.split("\n").filter((l) => /run:/.test(l))) {
-    assert.doesNotMatch(line, /\$\{\{/, line);
+});
+
+test("a workflow that pastes PR values into a script or reaches a token fails the check", () => {
+  const workflow = fs.readFileSync(WORKFLOW, "utf8");
+  // The checkout step's `run: |` block: a line after its first, where a one-line check misses it.
+  const block = /^ +run: \|\n( +)(\S.*)\n/m;
+  assert.match(workflow, block);
+  const mutants = {
+    "a PR title inside a run block": workflow.replace(
+      block,
+      (all, indent) => `${all}${indent}echo "\${{ github.event.pull_request.title }}"\n`,
+    ),
+    "a PR head ref on a later line of a run block": workflow.replace(
+      block,
+      (all, indent) => `${all}${indent}true\n\n${indent}git log -1 \${{ github.head_ref }}\n`,
+    ),
+    "github.token in an env": workflow.replace(
+      /^( +)env:\n/m,
+      (line, indent) => `${line}${indent}  GH_TOKEN: \${{ github.token }}\n`,
+    ),
+    "GITHUB_TOKEN in an env": workflow.replace(
+      /^( +)env:\n/m,
+      (line, indent) => `${line}${indent}  T: \${{ secrets.GITHUB_TOKEN }}\n`,
+    ),
+    "a pull_request trigger": workflow.replace(/^on:\n/m, "on:\n  pull_request:\n"),
+    // `run:` alone, the script on the next line.
+    "an expression in a run value that starts on the next line": workflow.replace(
+      /^( +)run: (node --test .*)$/m,
+      (all, indent, cmd) =>
+        `${indent}run:\n${indent}  echo \${{ github.event.pull_request.title }} && ${cmd}`,
+    ),
+    // A plain one-line run continued on the next line: YAML folds it into the same script.
+    "an expression on the continuation of a plain run": workflow.replace(
+      /^( +)run: (node --test .*)$/m,
+      (all, indent, cmd) => `${all}\n${indent}  \${{ github.head_ref }}`,
+    ),
+  };
+  for (const [label, text] of Object.entries(mutants)) {
+    assert.notEqual(text, workflow, label);
+    assert.notDeepEqual(prCodeProblems(text), [], label);
   }
-  assert.match(code, /node \.github\/scripts\/fork-schema-check\.cjs "\$BASE_SHA" "\$HEAD_SHA"/);
 });
 
 // End to end: the real git in a scratch repository, the way CI calls the check.

.github/scripts/fork-secret-scan.test.sh

@@ -6,10 +6,20 @@
 set -euo pipefail
 
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-script="$(realpath "${FORK_SECRET_SCAN_SH:-$here/fork-secret-scan.sh}")"
 root="$(mktemp -d)"
 trap 'rm -rf "$root"' EXIT
 
+# No script tested here calls gh. The stub (fork-test-gh.sh: no token, a .invalid GH_HOST) refuses
+# every call, so a scan that starts calling gh fails here instead of reaching GitHub.
+. "$here/fork-test-gh.sh"
+mkdir -p "$root/bin"
+fork_test_gh "$root/bin" <<'GH'
+echo "fake gh: fork-secret-scan.test.sh expects no gh call (gh $*)" >&2
+exit 97
+GH
+
+script="$(realpath "${FORK_SECRET_SCAN_SH:-$here/fork-secret-scan.sh}")"
+
 export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL="$root/gitconfig"
 git config --global user.name test
 git config --global user.email test@example.invalid

.github/scripts/fork-test-gh.sh

@@ -1,6 +1,6 @@
 # shellcheck shell=bash
-# Sourced first by every fork test that runs a script calling gh (fork-follow.test.sh,
-# fork-release.test.sh). `fork_test_gh DIR < STUB` cuts the test off from GitHub before any script
+# Sourced first by every fork shell test (fork-contracts.test.py requires it of each *.test.sh).
+# `fork_test_gh DIR < STUB` cuts the test off from GitHub before any script
 # runs: no GitHub token, GH_HOST a .invalid host and an empty gh config (so even a real gh has
 # nothing to reach and no login), and the stub read from stdin as DIR/gh, first on PATH, with the
 # marker line fork-gh-guard.sh looks for. FORK_TEST_MODE then makes the scripts refuse whenever

.github/workflows/fork-checks.yml

@@ -14,9 +14,18 @@ on:
 permissions:
   contents: read
 
+# Every job but `outside_pr` starts with the same condition: on a pull request, only one whose branch
+# is in this repository. A pull request from another fork runs none of them, not even after approval
+# (the repository asks approval only for pull_request runs, and pull_request_target skips that, but
+# this file always comes from the default branch). It gets only `outside_pr`, which checks nothing
+# out, runs nothing from the pull request, gets no token and fails. A job skipped by its if: counts
+# as passed, so a ruleset that requires "No database schema changes" must also require "Outside pull
+# request". A head repository GitHub no longer knows reads as "another fork". Test:
+# fork-contracts.test.py.
 jobs:
   no_schema_changes:
     name: No database schema changes
+    if: ${{ (!github.event.pull_request || github.event.pull_request.head.repo.full_name == github.repository) }}
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -46,3 +55,15 @@ jobs:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: node .github/scripts/fork-schema-check.cjs "$BASE_SHA" "$HEAD_SHA"
+
+  outside_pr:
+    name: Outside pull request
+    if: ${{ github.event.pull_request && github.event.pull_request.head.repo.full_name != github.repository }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions: {}
+    steps:
+      - name: Refuse an outside pull request
+        run: |
+          echo "::error::outside PR: not tested here; a maintainer must review"
+          exit 1