Add language-agnostic emitter-diff tool

.chronus/changes/emitter-diff-tool-2026-7-2-0-0-0.md

@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/http-client-python"
+---
+
+Add a `diff-spector-tests` script wired to the new language-agnostic emitter-diff tool, plus its PR CI workflow

.github/actions/setup/action.yml

@@ -11,10 +11,10 @@ runs:
 
   steps:
     - name: Install pnpm
-      uses: pnpm/action-setup@v3
+      uses: pnpm/action-setup@v6
 
     - name: Set node version to ${{ inputs.node-version }}
-      uses: actions/setup-node@v4
+      uses: actions/setup-node@v5
       with:
         node-version: ${{ inputs.node-version }}
         cache: pnpm

.github/workflows/ci-emitter-diff-python.yml

@@ -0,0 +1,147 @@
+name: "python / emitter diff"
+
+# Diffs generated code between this PR's emitter and the merge-base baseline.
+# Informational only: reports a sticky PR comment + HTML artifact, and fails the
+# job only on a tool/build error (never on a diff). See eng/emitter-diff.
+#
+# The tool runs `npm run regenerate` verbatim in each tree. It auto-prepares the
+# baseline tree it fetches from GitHub (via the preset's --setup: install +
+# build + venv), so this workflow only builds the head (PR) emitter itself.
+
+on:
+  pull_request:
+    branches: [main, release/*]
+    paths:
+      - "packages/http-client-python/**"
+      - "eng/emitter-diff/**"
+      - ".github/workflows/ci-emitter-diff-python.yml"
+  workflow_dispatch:
+    inputs:
+      baseline:
+        description: "Baseline emitter ref (local path or github ref). Defaults to the PR merge-base. A gh:/github: ref is fetched and auto-prepared (install + build) by the tool; a local: path is used as-is."
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  emitter-diff:
+    name: "Generate & Diff"
+    runs-on: ubuntu-latest
+    # Fork PRs are intentionally skipped
+    if: github.event.pull_request.head.repo.fork != true
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: ./.github/actions/setup
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install repo dependencies
+        run: pnpm install
+
+      - name: Determine baseline
+        id: baseline
+        # Dispatch input is untrusted: pass via env, reference only as "$VAR".
+        env:
+          BASELINE_INPUT: ${{ github.event.inputs.baseline || '' }}
+          BASE_REF: ${{ github.event.pull_request.base.ref || github.event.repository.default_branch }}
+        run: |
+          input="$BASELINE_INPUT"
+          if [ -n "$input" ]; then
+            echo "ref=$input" >> "$GITHUB_OUTPUT"
+            echo "Baseline (explicit): $input"
+          else
+            # merge-base = a real commit on the base branch; survives squash/rebase.
+            git fetch --no-tags origin "$BASE_REF"
+            base_sha="$(git merge-base FETCH_HEAD HEAD)"
+            [ -n "$base_sha" ] || { echo "::error::No merge-base with $BASE_REF."; exit 1; }
+            # gh:<sha> = this repo at the merge-base. The tool fetches that tree
+            # and auto-preps it (install + build + venv) via the preset's --setup.
+            echo "ref=gh:$base_sha" >> "$GITHUB_OUTPUT"
+            echo "Baseline (merge-base): gh:$base_sha"
+          fi
+
+      - name: Prepare head emitter (build + venv)
+        working-directory: packages/http-client-python
+        # http-client-python is excluded from the pnpm workspace (standalone,
+        # registry-versioned deps), so the repo-level `pnpm install` above does
+        # NOT provision its node_modules. Install them first (scripts skipped),
+        # then `npm run setup` (build + venv). Mirrors the baseline tree prep the
+        # tool does via the python preset's --setup.
+        run: |
+          npm install --ignore-scripts
+          npm run setup
+
+      - name: Run emitter diff
+        id: diff
+        working-directory: packages/http-client-python
+        env:
+          BASELINE_REF: ${{ steps.baseline.outputs.ref }}
+          RUNNER_TEMP: ${{ runner.temp }}
+        run: |
+          set +e
+          # No --fail-on-diff (informational); tool/build errors still exit non-zero
+          # (checked in "Fail on tool error").
+          npm run diff-spector-tests -- \
+            --ci \
+            --baseline "$BASELINE_REF" \
+            --html "$RUNNER_TEMP/emitter-diff.html" \
+            --md "$RUNNER_TEMP/emitter-diff.md" \
+            | tee "$RUNNER_TEMP/emitter-diff.log"
+          echo "status=${PIPESTATUS[0]}" >> "$GITHUB_OUTPUT"
+          # Reuse the tool's own summary line (strip ANSI) instead of re-parsing.
+          summary="$(sed -r 's/\x1b\[[0-9;]*m//g' "$RUNNER_TEMP/emitter-diff.log" \
+            | grep -oE 'Diff summary: [0-9]+ file\(s\), \+[0-9]+ / -[0-9]+' | head -1)"
+          echo "summary=${summary:-No changes to generated output.}" >> "$GITHUB_OUTPUT"
+          # Render the diff inline on the run page.
+          if [ -f "$RUNNER_TEMP/emitter-diff.md" ]; then
+            cat "$RUNNER_TEMP/emitter-diff.md" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload HTML diff
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: emitter-diff-html
+          path: ${{ runner.temp }}/emitter-diff.html
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Comment on PR
+        if: always() && github.event_name == 'pull_request'
+        continue-on-error: true
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          BASELINE: ${{ steps.baseline.outputs.sha || steps.baseline.outputs.ref }}
+          STATUS: ${{ steps.diff.outputs.status }}
+          SUMMARY: ${{ steps.diff.outputs.summary }}
+        with:
+          script: |
+            const marker = "<!-- emitter-diff-python -->";
+            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const { STATUS, SUMMARY, BASELINE } = process.env;
+            let body = `${marker}\n### Python emitter diff\nBaseline \`${BASELINE}\` vs this PR.\n\n`;
+            body += STATUS !== "0"
+              ? `⚠️ **emitter-diff failed** (exit \`${STATUS}\`) — tool/build error, not a diff. See the [run](${runUrl}).\n`
+              : `${SUMMARY}\n\nRendered diff: inline on the [run summary](${runUrl}), or the **emitter-diff-html** artifact.\n`;
+            body += `\n_Informational check (eng/emitter-diff); does not block the PR._`;
+            const { owner, repo } = context.repo, issue_number = context.issue.number;
+            const { data } = await github.rest.issues.listComments({ owner, repo, issue_number });
+            const hit = data.find((c) => c.body && c.body.includes(marker));
+            if (hit) await github.rest.issues.updateComment({ owner, repo, comment_id: hit.id, body });
+            else await github.rest.issues.createComment({ owner, repo, issue_number, body });
+
+      - name: Fail on tool error
+        if: always()
+        env:
+          STATUS: ${{ steps.diff.outputs.status }}
+        run: '[ "$STATUS" = "0" ] || { echo "::error::emitter-diff failed (exit $STATUS)."; exit 1; }'

cspell.yaml

@@ -155,6 +155,7 @@ words:
   - MACVMIMAGEM
   - marshal
   - mday
+  - Menlo
   - methodsubscriptionid
   - mgmt
   - mgmtplane

eng/emitter-diff/README.md

@@ -0,0 +1,199 @@
+# emitter-diff
+
+A language-agnostic tool for **diffing the generated code produced by two versions of a
+TypeSpec emitter**.
+
+It runs the emitter's own regenerate command against a **baseline** source tree and a **head**
+source tree, then shows the diff between the two generated outputs. Use it locally during
+development and in CI on PRs.
+
+The tool contains **zero language logic**. An emitter integrates by naming three things — its
+regenerate command, the package directory to run it in, and the generated-code directory to diff —
+either as flags or via a built-in `--emitter` preset.
+
+## How it works
+
+```text
+   baseline tree ─► run <command> in <emitter-path> ─► snapshot <generated-code-path> ─┐
+                                                                                        ├─► git diff ─► terminal / HTML
+   head tree ─────► run <command> in <emitter-path> ─► snapshot <generated-code-path> ─┘
+```
+
+- A **source tree** is resolved for each side (`--baseline` / `--head`).
+- The **`--command`** is run verbatim (tokenized to argv — no shell) inside
+  `<tree>/<emitter-path>`. This is the emitter's _unmodified_ regenerate command; the tool does
+  not reach into it.
+- The `<emitter-path>/<generated-code-path>` subtree is snapshotted for each side and the
+  two snapshots are diffed. `--generated-code-path` accepts **multiple roots** — an emitter that
+  writes generated code to several directories (e.g. Go) passes a comma-separated list or repeats
+  the flag; each root is snapshotted under its own relative path so outputs never collide.
+
+Because the tool just runs a command, **any emitter with a regenerate script works** — no per-language
+plugin code.
+
+## Usage
+
+```bash
+# Using a built-in preset (fills in command + paths):
+node eng/emitter-diff/src/cli.ts --emitter python --baseline gh:<sha>
+
+# Fully explicit (no preset needed):
+node eng/emitter-diff/src/cli.ts \
+  --command "npm run regenerate" \
+  --emitter-path packages/http-client-python \
+  --generated-code-path tests/generated \
+  --baseline gh:<sha>
+
+# Multiple generated roots (e.g. Go writes several) — comma-separated (or repeat the flag):
+node eng/emitter-diff/src/cli.ts \
+  --command "npm run tspcompile" \
+  --emitter-path packages/typespec-go \
+  --generated-code-path test/http-specs,test/azure-http-specs \
+  --baseline github:Azure/autorest.go@<sha>
+```
+
+> This tool is a set of plain `.ts` scripts — not an installed package. It runs through `node`
+> (which executes TypeScript directly on the versions this repo supports), so there is nothing to
+> build. Typecheck with `npx tsc -p eng/emitter-diff`.
+
+> **Build your checkout first.** `--head` defaults to the current working tree, and the tool runs
+> the regenerate command against it **as-is** — it never installs or builds your checkout (see
+> [Command prep](#command-prep---setup)). Build the emitter for the head side before diffing; for
+> **python** that's `npm run setup` in `packages/http-client-python` (builds the emitter and creates
+> the venv `regenerate` requires). Only trees the tool fetches from GitHub are auto-prepared.
+
+### Emitter config
+
+| Flag                           | Meaning                                                                                                                                                                                                                                            |
+| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `--emitter <name>`             | Built-in preset that fills in the three fields below.                                                                                                                                                                                              |
+| `--command <cmd>`              | Regenerate command, run verbatim in `--emitter-path`.                                                                                                                                                                                              |
+| `--emitter-path <path>`        | Package dir (relative to a tree root) to run the command in.                                                                                                                                                                                       |
+| `--generated-code-path <path>` | Generated-code dir (relative to `--emitter-path`) to diff. Accepts **multiple roots** — comma-separated (`a,b`) or by repeating the flag — for an emitter with several generated roots (e.g. Go); each is snapshotted under its own relative path. |
+
+A preset supplies all three; each flag still overrides the preset value. To onboard a new language,
+add a row to `EMITTER_DEFAULTS` in `src/registry.ts` — or just pass the three flags directly and
+skip `--emitter`.
+
+### Refs
+
+| Syntax                            | Meaning                              |
+| --------------------------------- | ------------------------------------ |
+| `local:/path` or `./path`         | a local source folder (run in place) |
+| `github:owner/repo@<sha\|branch>` | a GitHub source at a ref             |
+| `gh:<sha\|branch>`                | this repo (origin remote) at a ref   |
+
+`--head` defaults to the **current working tree**. `--baseline` defaults to the `upstream` remote's
+repo at its default branch, falling back to `origin` (e.g. `github:microsoft/typespec@main`).
+
+### Command prep (`--setup`)
+
+The `--command` is run **as-is** — it does not install deps or build. Instead, prep is handled by
+**`--setup`**, which runs **only in a tree the tool freshly fetched from GitHub** (a `gh:`/`github:`
+ref). The current working tree and user-provided `local:` paths are assumed already built and are
+**never touched** (so setup never mutates your checkout or a prepared CI worktree).
+
+- A **preset** supplies sensible setup defaults, so `--emitter python --baseline gh:main` installs +
+  builds the fetched baseline automatically (python: `npm install --ignore-scripts` → `npm run setup`;
+  typescript: `pnpm install` → `npm run build`).
+- Override with one or more `--setup <cmd>` (each runs in order, in `<tree>/<emitter-path>`), or
+  disable entirely with `--no-setup`.
+
+Each `--setup` command — like `--command` — is tokenized to argv and run **without a shell**, so
+multi-step pipelines (`&&`, `|`) are not supported; pass repeated `--setup` flags instead.
+
+### Common options
+
+By default the tool writes a **clickable HTML report** (`emitter-diff.html`) into the work dir and
+prints a `file://` link to it.
+
+- `--baseline <ref>` / `--head <ref>`: the two source trees to compare.
+- `--setup <cmd>` (repeatable) / `--no-setup`: prep commands for freshly fetched GitHub trees.
+- `--work-dir <dir>`: scratch dir for snapshots (default: a fresh temp dir).
+- `--sequential`: regenerate baseline then head one after another instead of in parallel. Useful on
+  a single machine where running both at once oversubscribes the CPU (each regenerate already fans
+  out across cores) or trips generator races.
+- `--ci`: disable the local baseline-output cache (intended for CI).
+- `--html <file>`: write the rendered HTML report to this path.
+- `--md <file>`: write a Markdown report (collapsible per-file `diff` blocks) to this path. Handy for
+  a CI job summary (`$GITHUB_STEP_SUMMARY`) or a PR comment body.
+- `--no-group`: render every file separately in the HTML/Markdown reports. By default, files that
+  share the same change — the same added/removed lines, even when the surrounding generated code
+  differs — are merged into a single collapsible group (with the shared diff shown once and every
+  once and every affected file listed), so a repeated change across many generated files is reviewed
+  once instead of N times.
+- `--fail-on-diff`: exit non-zero when output differs (exit `2` = diff present, `1` = hard error).
+- `-- <args>`: everything after `--` is appended to `--command` verbatim on **both** sides, so the
+  diff stays apples-to-apples. Use it to regenerate only a subset of tests by forwarding the
+  regenerate script's own filter flags.
+
+### Regenerating a subset of tests
+
+The tool doesn't define its own test-filter flags — it forwards `-- <args>` to each emitter's
+regenerate command, which owns the filtering. For the **Python** emitter (`regenerate.ts`):
+
+- `-n, --name <pattern>`: case-insensitive substring match on package name.
+- `-f, --flavor <azure|unbranded>`: limit to one flavor.
+- `-j, --jobs <n>`: parallel job count.
+
+```sh
+# Only the authentication packages, azure flavor
+node ../../eng/emitter-diff/src/cli.ts --emitter python -- --name authentication --flavor azure
+
+# Via the package script (first `--` is npm's, second is emitter-diff's passthrough separator)
+npm run diff-spector-tests -- -- --name type/array
+```
+
+`--name` filters the spec set already bundled in the package's `node_modules`
+(`@azure-tools/azure-http-specs` + `@typespec/http-specs`); it does not point the test set at an
+arbitrary folder or ref. Other emitters expose their own filter flags — pass whatever their
+regenerate command accepts.
+
+## CI integration
+
+`.github/workflows/ci-emitter-diff-<lang>.yml` runs on PRs that touch the language emitter or this
+tool. The **baseline** is the base-branch commit the PR is based on (the `git merge-base` with the
+target branch). Because the tool runs the regenerate command as-is, the workflow **prepares both
+trees** (installs deps, builds the emitter, creates any venv) before invoking the tool, then:
+
+- posts a **sticky PR comment** (updated in place on each push) linking the diff artifact, and
+- uploads the rendered **HTML report** as an artifact.
+
+**Informational:** the check **always passes unless the tool hits a real tool/build error** — a
+generated-output diff does not fail the PR. CI runs the tool without `--fail-on-diff`, so a diff
+still exits `0`; only a non-zero exit (a build/venv/generate failure) fails the job.
+
+**Fork PRs are not run.** The job checks out and executes the PR's code (builds the emitter, runs
+`regenerate`), so a job-level `if` guard restricts it to same-repo PRs — it skips any PR whose head
+is a fork.
+
+## Adding a new language
+
+Either add a preset row to `EMITTER_DEFAULTS` (`src/registry.ts`):
+
+```ts
+rust: {
+  command: "npm run tspcompile",
+  emitterPath: "packages/typespec-rust",
+  generatedCodePath: "test/generated",
+},
+```
+
+…or skip the preset entirely and pass `--command` / `--emitter-path` /
+`--generated-code-path` directly. Either way, ensure each side's tree can actually run the
+command (see **Command prep** above). The orchestrator, ref resolver, and diff engine need no
+changes.
+
+## Notes & limitations
+
+- `--html` renders a self-contained, GitHub-style HTML report (inline CSS, no external requests).
+  `--md` renders a Markdown report (collapsible per-file `diff` blocks) suitable for a CI job
+  summary or PR comment. Both reports group files that share the same change into one block by
+  default (`--no-group` to disable), so reviewing a repeated diff is a one-time effort. The diff
+  itself is produced by `git diff --no-index`; the tool leans on only a couple of small repo dev
+  dependencies (`execa`, `picocolors`) for process spawning and terminal coloring.
+- For github refs (including fork repos), the resolver uses a detached, commit-keyed cached git
+  worktree under the temp directory, created from an isolated cache repo (not from your active
+  checkout). Repeated runs on the same commit reuse it.
+- Spec inputs come from each side's own dependencies (e.g. `node_modules/@typespec/http-specs`).
+  Spec versions rarely change, so any drift between baseline/head is treated as acceptable noise.