diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9e5462135..1e01c2867 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,7 +23,28 @@ # # Mistral AI — onboarded: 2026-06-22 — partner: @peymanmohajerian — dri: @truptiparkar7 # /samples/python/mistral/ @FILL_IN +#### docs-referenced-file guard (Docs team) ################################################## +# These three paths ARE the guard that stops a PR from breaking a published Learn article. +# They must not be editable without AI Platform Docs review, or the gate can be disabled in the +# same PR that breaks a reference. Also mark `docs-referenced-files / check` as a required status +# check on `main` -- CODEOWNERS alone only routes review, it does not block a merge. +/.github/docs-referenced-files.json @microsoft-foundry/AI-Platform-Docs +/.github/scripts/check-docs-referenced-files.py @microsoft-foundry/AI-Platform-Docs +/.github/workflows/docs-referenced-files.yml @microsoft-foundry/AI-Platform-Docs + #### files referenced in docs (DO NOT EDIT, except for Docs team!!!) ########################################## +# +# MIGRATION IN PROGRESS -- this per-file block is being replaced by +# `.github/docs-referenced-files.json`, which the `docs-referenced-files / check` workflow +# enforces automatically. That check fails the PR when a referenced file is deleted or renamed, +# or when a snippet delimiter comment or a named notebook cell is removed. +# +# Once that check is required on `main`, delete the entries below: the docs team no longer needs +# to own or approve routine edits to these samples, only to be consulted when a reference breaks. +# Until then, keep both in sync. +# +# Questions: General - AI Platform Docs <7f0acdb3.microsoft.com@amer.teams.ms> +# /infrastructure/infrastructure-setup-bicep/01-connections/connection-key-vault.bicep @microsoft-foundry/AI-Platform-Docs /infrastructure/infrastructure-setup-bicep/05-custom-policy-definitions/deny-disallowed-connections.json @microsoft-foundry/AI-Platform-Docs /infrastructure/infrastructure-setup-terraform/00-basic-azurerm/code/main.tf @microsoft-foundry/AI-Platform-Docs diff --git a/.github/docs-referenced-files.json b/.github/docs-referenced-files.json new file mode 100644 index 000000000..edcd388a8 --- /dev/null +++ b/.github/docs-referenced-files.json @@ -0,0 +1,96 @@ +{ + "schemaVersion": 1, + "$comment": "Files that published Microsoft Learn articles include by reference. Owned by the AI Platform Docs team. To protect a file, add its path to 'files' -- one string per line is all you need. Renaming or deleting a listed file, or removing one of its snippet delimiter comments or named notebook cells, breaks the docs build; .github/workflows/docs-referenced-files.yml enforces this on every PR. Do not edit without the docs team.", + "contact": { + "team": "@microsoft-foundry/AI-Platform-Docs", + "teamsChannelEmail": "General - AI Platform Docs <7f0acdb3.microsoft.com@amer.teams.ms>" + }, + "files": [ + "infrastructure/infrastructure-setup-bicep/01-connections/connection-key-vault.bicep", + "infrastructure/infrastructure-setup-bicep/05-custom-policy-definitions/deny-disallowed-connections.json", + "infrastructure/infrastructure-setup-terraform/00-basic-azurerm/code/main.tf", + "infrastructure/infrastructure-setup-terraform/00-basic-azurerm/code/providers.tf", + "infrastructure/infrastructure-setup-terraform/00-basic-azurerm/code/variables.tf", + "infrastructure/infrastructure-setup-terraform/00-basic/code/main.tf", + "infrastructure/infrastructure-setup-terraform/00-basic/code/providers.tf", + "infrastructure/infrastructure-setup-terraform/00-basic/code/variables.tf", + "samples-classic/REST/quickstart/quickstart.sh", + "samples-classic/csharp/quickstart/Samples/.env.example", + "samples-classic/csharp/quickstart/Samples/AgentFileSearch.cs", + "samples-classic/csharp/quickstart/Samples/AgentService.cs", + "samples-classic/csharp/quickstart/Samples/SimpleInference.cs", + "samples-classic/java/quickstart/src/main/java/com/azure/ai/foundry/samples/AgentSample.java", + "samples-classic/java/quickstart/src/main/java/com/azure/ai/foundry/samples/ChatCompletionSample.java", + "samples-classic/java/quickstart/src/main/java/com/azure/ai/foundry/samples/FileSearchAgentSample.java", + "samples-classic/python/quickstart/create_project.py", + "samples-classic/python/quickstart/quickstart.py", + "samples-classic/typescript/quickstart/.env.template", + "samples-classic/typescript/quickstart/src/quickstart.ts", + "samples/REST/quickstart/quickstart-chat-with-agent.sh", + "samples/REST/quickstart/quickstart-create-agent.sh", + "samples/REST/quickstart/quickstart-responses.sh", + "samples/csharp/enterprise-agent-tutorial/1-idea-to-prototype/Evaluate/Program.cs", + "samples/csharp/enterprise-agent-tutorial/1-idea-to-prototype/ModernWorkplaceAssistant/Program.cs", + "samples/csharp/foundry-local/audio-transcription-example/Program.cs", + "samples/csharp/foundry-local/embeddings/Program.cs", + "samples/csharp/foundry-local/foundry-local-web-server/Program.cs", + "samples/csharp/foundry-local/live-audio-transcription/Program.cs", + "samples/csharp/foundry-local/native-chat-completions/NativeChatCompletions.csproj", + "samples/csharp/foundry-local/native-chat-completions/Program.cs", + "samples/csharp/foundry-local/tool-calling-foundry-local-sdk/Program.cs", + "samples/csharp/foundry-local/tool-calling-foundry-local-web-server/Program.cs", + "samples/csharp/foundry-local/tutorial-chat-assistant/Program.cs", + "samples/csharp/foundry-local/tutorial-document-summarizer/Program.cs", + "samples/csharp/foundry-local/tutorial-tool-calling/Program.cs", + "samples/csharp/foundry-local/tutorial-voice-to-text/Program.cs", + "samples/csharp/quickstart/chat-with-agent/quickstart-chat-with-agent.cs", + "samples/csharp/quickstart/create-agent/quickstart-create-agent.cs", + "samples/csharp/quickstart/responses/quickstart-responses.cs", + "samples/java/quickstart/chat-with-agent/src/main/java/com/azure/ai/agents/ChatWithAgent.java", + "samples/java/quickstart/create-agent/src/main/java/com/azure/ai/agents/CreateAgent.java", + "samples/java/quickstart/responses/src/main/java/com/azure/ai/agents/CreateResponse.java", + "samples/javascript/foundry-local/audio-transcription-example/app.js", + "samples/javascript/foundry-local/embeddings/app.js", + "samples/javascript/foundry-local/langchain-integration-example/app.js", + "samples/javascript/foundry-local/live-audio-transcription/app.js", + "samples/javascript/foundry-local/native-chat-completions/app.js", + "samples/javascript/foundry-local/tool-calling-foundry-local/src/app.js", + "samples/javascript/foundry-local/tutorial-chat-assistant/app.js", + "samples/javascript/foundry-local/tutorial-document-summarizer/app.js", + "samples/javascript/foundry-local/tutorial-tool-calling/app.js", + "samples/javascript/foundry-local/tutorial-voice-to-text/app.js", + "samples/javascript/foundry-local/web-server-example/app.js", + "samples/python/enterprise-agent-tutorial/1-idea-to-prototype/evaluate.py", + "samples/python/enterprise-agent-tutorial/1-idea-to-prototype/main.py", + "samples/python/foundry-local/audio-transcription/src/app.py", + "samples/python/foundry-local/embeddings/src/app.py", + "samples/python/foundry-local/langchain-integration/src/app.py", + "samples/python/foundry-local/live-audio-transcription/src/app.py", + "samples/python/foundry-local/native-chat-completions/src/app.py", + "samples/python/foundry-local/tool-calling/src/app.py", + "samples/python/foundry-local/tutorial-chat-assistant/src/app.py", + "samples/python/foundry-local/tutorial-document-summarizer/src/app.py", + "samples/python/foundry-local/tutorial-tool-calling/src/app.py", + "samples/python/foundry-local/tutorial-voice-to-text/src/app.py", + "samples/python/foundry-local/web-server/src/app.py", + "samples/python/foundry-models/model-router/model-router-chat-completions-observability.py", + "samples/python/foundry-models/model-router/model-router-chat-completions.py", + "samples/python/foundry-models/model-router/model-router-foundry-responses.py", + "samples/python/quickstart/chat-with-agent/quickstart-chat-with-agent.py", + "samples/python/quickstart/create-agent/quickstart-create-agent.py", + "samples/python/quickstart/responses/quickstart-responses.py", + "samples/rust/foundry-local/audio-transcription-example/src/main.rs", + "samples/rust/foundry-local/embeddings/src/main.rs", + "samples/rust/foundry-local/foundry-local-webserver/src/main.rs", + "samples/rust/foundry-local/live-audio-transcription/src/main.rs", + "samples/rust/foundry-local/native-chat-completions/src/main.rs", + "samples/rust/foundry-local/tool-calling-foundry-local/src/main.rs", + "samples/rust/foundry-local/tutorial-chat-assistant/src/main.rs", + "samples/rust/foundry-local/tutorial-document-summarizer/src/main.rs", + "samples/rust/foundry-local/tutorial-tool-calling/src/main.rs", + "samples/rust/foundry-local/tutorial-voice-to-text/src/main.rs", + "samples/typescript/quickstart/chat-with-agent/src/quickstart-chat-with-agent.ts", + "samples/typescript/quickstart/create-agent/src/quickstart-create-agent.ts", + "samples/typescript/quickstart/responses/src/quickstart-responses.ts" + ] +} diff --git a/.github/scripts/check-docs-referenced-files.README.md b/.github/scripts/check-docs-referenced-files.README.md new file mode 100644 index 000000000..d383b5c19 --- /dev/null +++ b/.github/scripts/check-docs-referenced-files.README.md @@ -0,0 +1,151 @@ +# check-docs-referenced-files.py + +Published Microsoft Learn articles include code from this repo **by reference**: + +``` +:::code language="python" source="~/foundry-samples/samples/python/quickstart/create-agent/quickstart-create-agent.py" id="create_agent"::: +``` + +Three changes on this side break the docs build: + +| # | Change | Why it breaks | +|---|--------|---------------| +| 1 | The file is **renamed or moved** | `source=` no longer resolves | +| 2 | The file is **deleted** | `source=` no longer resolves | +| 3 | A snippet delimiter comment (`# ` … `# `) is **removed** | `id=` no longer resolves | +| 4 | A referenced **notebook cell** is deleted, or its `"metadata": {"name": …}` is removed | the article's cell reference no longer resolves | + +This check replaces the old process (every referenced file listed in `CODEOWNERS`, docs team +manually approving every PR that touched one) with an automated PR gate. + +**Contact for the docs team:** `General - AI Platform Docs <7f0acdb3.microsoft.com@amer.teams.ms>` +(emailing that address posts to the team's Teams channel), or @-mention +`@microsoft-foundry/AI-Platform-Docs` on the PR. + +## The manifest + +`.github/docs-referenced-files.json` is the source of truth. It is owned by the AI Platform Docs +team, and it replaces the per-file block in `CODEOWNERS`. Normally an entry is just the path: + +```json +"files": [ + "samples/python/quickstart/create-agent/quickstart-create-agent.py", + "samples/python/fine-tuning/distillation.ipynb" +] +``` + +That is the whole maintenance story: **one line per file, added when an article starts referencing +a sample and removed when it stops.** You never list snippet ids by hand. + +A bare path means `mode: "auto"`. The checker reads the file at the **pull request's base +revision**, records which snippet ids and named notebook cells it had, and requires the head +revision to still have all of them. So: + +* Removing a `# ` delimiter, or a notebook cell whose `metadata.name` an article + references, fails the build. +* Adding new snippets, renaming nothing, and freely editing the code *inside* a region all pass. +* Only per-cell `metadata.name` counts — notebook-level `kernelspec`/`language_info` `name` keys + are ignored, so a raw `"name":` search can't produce false positives. + +The tradeoff of the bare form: expectations come from the base revision, so a break that was +already merged in an earlier PR is not re-reported on later PRs. That is why CI always passes +`--base-ref` and fails loudly if it cannot resolve one. + +### Pinning an entry (optional) + +To make a file's expectations independent of the base revision — or to protect only *some* of a +file's regions — write the long form instead. Generate it with `--seed --pin`. + +```json +{ + "path": "samples/python/quickstart/create-agent/quickstart-create-agent.py", + "mode": "delimited", + "snippets": ["create_agent"] +} +``` + +* `mode: "delimited"` — every listed snippet id must exist exactly once as a matched + `` / `` comment pair with non-empty content between them. +* `mode: "notebook-cell"` — `.ipynb` only. Every listed name must match exactly one cell's + `metadata.name`, and that cell must be non-empty. +* `mode: "whole-file"` — the article includes the entire file; only its exact path is checked. +* `note` — optional free text (e.g. which article references it). + +Pinned entries are checked against the current tree directly, so a break that slipped through +earlier keeps being reported. `--seed` never downgrades a pinned entry back to a bare path. + +## Running it + +```bash +# validate the committed tree +python .github/scripts/check-docs-referenced-files.py + +# validate your local edits before you push +python .github/scripts/check-docs-referenced-files.py --worktree + +# what CI runs: compare against the branch you'll merge into +python .github/scripts/check-docs-referenced-files.py --base-ref origin/main +``` + +Exit codes: `0` intact · `1` a docs reference is broken · `2` checker/manifest/git error. + +## Maintenance (docs team) + +```bash +# regenerate every entry from the working tree (re-reads snippet ids from the files) +python .github/scripts/check-docs-referenced-files.py --seed + +# start from the CODEOWNERS block owned by @microsoft-foundry/AI-Platform-Docs +python .github/scripts/check-docs-referenced-files.py --seed --from-codeowners + +# add or refresh one file +python .github/scripts/check-docs-referenced-files.py --seed --path samples/python/x/app.py +``` + +`--seed` writes bare paths. Add `--pin` to write the long form instead, inferring `mode` and +`snippets` from the file's current contents (comment delimiters for code files, +`cells[*].metadata.name` for notebooks). Review a pinned diff — if an article only includes *some* +of a file's regions, trim `snippets` to the ones the docs actually use so contributors aren't +blocked on delimiters nobody publishes. + +To add or drop a file day to day, just edit the `files` list by hand; the path is all it needs. + +## Renaming or deleting a referenced file + +The rename cannot be atomic across two repos, so the safe order is: + +1. Add the sample at the new path, keeping the old path and its delimiters in place. +2. Add the new path to the manifest. +3. Docs team updates and publishes the Learn articles. +4. After publication, docs team removes the old manifest entry. +5. Delete the old path. + +## Supported file types + +Delimiter scanning is extension-aware: `#` for Python/Bash/Terraform/YAML/`.env`, `//` for +C#/Java/JS/TS/Rust/Bicep/Go, `` for XML/`.csproj`/Markdown, `--` for SQL. Files are decoded +as `utf-8-sig` (BOM-tolerant — several `Program.cs` files start with one) and CRLF-normalized. + +Only comment-only lines whose entire body is `` or `` count, so `List` and +JSON strings are never mistaken for delimiters. `.json` has no comment syntax and must be listed as +`whole-file`; `.ipynb` uses `notebook-cell` (or `whole-file`) and is rejected in `delimited` mode. +An unknown extension is a hard error (exit 2) rather than a silent zero-tag pass; add its comment +syntax to the marker tables in the script. + +## Enforcement + +`.github/workflows/docs-referenced-files.yml` runs this on every PR (`pull_request`, never +`pull_request_target`, `contents: read`, no secrets, no `paths:` filter so the required check can +never conclude `skipped`). Mark **`docs-referenced-files / check`** as a required status check on +`main`. + +The gate is only as strong as the files that implement it. Protect all three from unreviewed edits +(branch-ruleset path restriction, or enforced code-owner review with stale approvals dismissed): + +* `.github/docs-referenced-files.json` +* `.github/scripts/check-docs-referenced-files.py` +* `.github/workflows/docs-referenced-files.yml` + +Once the required check and that protection are both live, the per-file +`@microsoft-foundry/AI-Platform-Docs` block in `.github/CODEOWNERS` can be deleted — the docs team +stops owning sample code and owns only the manifest. diff --git a/.github/scripts/check-docs-referenced-files.py b/.github/scripts/check-docs-referenced-files.py new file mode 100644 index 000000000..d4a704b28 --- /dev/null +++ b/.github/scripts/check-docs-referenced-files.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python3 +"""Guard the sample files that Microsoft Learn articles include by reference. + +Learn articles pull code out of this repo with `:::code source="" id="":::`. +Three changes here break the published docs build: + + 1. the referenced file is DELETED + 2. the referenced file is RENAMED or MOVED (the `source=` path stops resolving) + 3. a snippet delimiter comment (`# ` ... `# `) is REMOVED, + so the `id=` region stops resolving + 4. for a notebook, a referenced CELL is deleted, or its `"metadata": {"name": "..."}` is + removed, so the article's cell reference stops resolving + +`.github/docs-referenced-files.json` is the manifest of what the docs reference. Day to day it is +just a list of paths -- add a line to protect a file, delete a line to stop protecting it: + + "files": [ + "samples/python/quickstart/create-agent/quickstart-create-agent.py", + "samples/REST/quickstart/quickstart-responses.sh" + ] + +A path's snippet ids are discovered from the file itself, so nobody has to maintain them by hand. +Expectations are read from the PR BASE revision, which is what makes a REMOVED tag detectable: the +base still has the delimiter the head just lost. An entry can instead be written as an object to +pin an explicit mode/snippet list, which makes that entry independent of the base revision. + +Exit codes: + 0 every referenced file and snippet is intact + 1 a docs reference is broken -- the PR must not merge until the docs team is contacted + 2 checker/manifest/git error (fail loud, never fail open) + +Usage: + python .github/scripts/check-docs-referenced-files.py --base-ref origin/main # full check + python .github/scripts/check-docs-referenced-files.py --worktree --base-ref origin/main + python .github/scripts/check-docs-referenced-files.py --ref --base-ref + +Docs-team maintenance (regenerates the manifest from the working tree): + python .github/scripts/check-docs-referenced-files.py --seed --from-codeowners + python .github/scripts/check-docs-referenced-files.py --seed --path samples/python/x/app.py + python .github/scripts/check-docs-referenced-files.py --seed --pin # explicit snippet lists +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path, PurePosixPath + +EXIT_OK = 0 +EXIT_VIOLATION = 1 +EXIT_ERROR = 2 + +SCHEMA_VERSION = 1 +DEFAULT_MANIFEST = ".github/docs-referenced-files.json" +CODEOWNERS_PATH = ".github/CODEOWNERS" +DOCS_TEAM = "@microsoft-foundry/AI-Platform-Docs" +DOCS_TEAM_EMAIL = "General - AI Platform Docs <7f0acdb3.microsoft.com@amer.teams.ms>" + +MODE_AUTO = "auto" +MODE_DELIMITED = "delimited" +MODE_WHOLE_FILE = "whole-file" +MODE_NOTEBOOK_CELL = "notebook-cell" +VALID_MODES = (MODE_AUTO, MODE_DELIMITED, MODE_WHOLE_FILE, MODE_NOTEBOOK_CELL) + +NOTEBOOK_EXT = ".ipynb" + +# A snippet delimiter is a COMMENT-ONLY line: `` or `` and nothing else. Requiring the +# whole comment body to be the tag is what keeps C# generics (`List`) and JSON +# strings out of the tag set. +TAG_RE = re.compile(r"^$") + +# Comment syntax per extension. Unknown extensions are rejected for `delimited` entries rather +# than silently scanned with a guessed marker (a guess that finds zero tags would fail open). +LINE_COMMENT_MARKERS: dict[str, tuple[str, ...]] = { + ".bicep": ("//",), + ".cs": ("//",), + ".env": ("#",), + ".example": ("#",), # .env.example + ".go": ("//",), + ".java": ("//",), + ".js": ("//",), + ".jsx": ("//",), + ".mjs": ("//",), + ".ps1": ("#",), + ".py": ("#",), + ".rs": ("//",), + ".sh": ("#",), + ".sql": ("--",), + ".template": ("#",), # .env.template + ".tf": ("#", "//"), + ".tfvars": ("#", "//"), + ".ts": ("//",), + ".tsx": ("//",), + ".yaml": ("#",), + ".yml": ("#",), +} + +BLOCK_COMMENT_MARKERS: dict[str, tuple[str, str]] = { + ".csproj": (""), + ".html": (""), + ".md": (""), + ".props": (""), + ".targets": (""), + ".xml": (""), +} + +# Formats with no comment syntax at all: they can only ever be whole-file references. +NO_COMMENT_EXTS = {".json", ".ipynb"} + +REGULAR_FILE_MODES = {"100644", "100755"} + +CONTACT = ( + "Contact the Microsoft Foundry docs team (AI Platform Docs) BEFORE merging this PR:\n" + f" Teams channel: {DOCS_TEAM_EMAIL}\n" + f" Or on this PR: @-mention {DOCS_TEAM}\n" + "They must retarget the affected Learn articles first; only then can the manifest entry " + f"in {DEFAULT_MANIFEST} be updated." +) + + +class CheckerError(Exception): + """Configuration/manifest/git problem -- exit 2, never a silent pass.""" + + +# -------------------------------------------------------------------------------------- +# git plumbing +# -------------------------------------------------------------------------------------- +def git(repo_root: Path, *args: str, binary: bool = False) -> bytes | str: + proc = subprocess.run( + ["git", "-C", str(repo_root), *args], + capture_output=True, + ) + if proc.returncode != 0: + raise CheckerError( + f"git {' '.join(args)} failed ({proc.returncode}): " + f"{proc.stderr.decode('utf-8', 'replace').strip()}" + ) + return proc.stdout if binary else proc.stdout.decode("utf-8", "replace") + + +def repo_root_from(start: Path) -> Path: + proc = subprocess.run( + ["git", "-C", str(start), "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise CheckerError(f"not inside a git repository: {start}") + return Path(proc.stdout.strip()) + + +class Tree: + """The set of tracked paths to validate, plus how to read their content. + + `ref` mode reads a committed revision. `worktree` mode reads the index for path existence + (so exact case comes from git, not from a case-insensitive macOS/Windows filesystem) and + on-disk content for uncommitted edits. + """ + + def __init__(self, repo_root: Path, ref: str | None, worktree: bool) -> None: + self.repo_root = repo_root + self.ref = ref + self.worktree = worktree + self.modes: dict[str, str] = {} + if worktree: + self._load_index() + else: + self._load_ref() + + @property + def label(self) -> str: + return "working tree" if self.worktree else f"revision {self.ref}" + + def _load_ref(self) -> None: + assert self.ref is not None + try: + git(self.repo_root, "rev-parse", "--verify", f"{self.ref}^{{commit}}") + except CheckerError as exc: + raise CheckerError( + f"cannot resolve ref '{self.ref}'. In CI, check out with fetch-depth: 0; " + f"locally, try `git fetch --unshallow`.\n {exc}" + ) from exc + raw = git(self.repo_root, "ls-tree", "-r", "-z", self.ref, binary=True) + for record in raw.split(b"\0"): + if not record: + continue + meta, _, path = record.partition(b"\t") + fields = meta.split(b" ") + if len(fields) < 3: + continue + self.modes[path.decode("utf-8", "surrogateescape")] = fields[0].decode() + + def _load_index(self) -> None: + raw = git(self.repo_root, "ls-files", "-s", "-z", binary=True) + for record in raw.split(b"\0"): + if not record: + continue + meta, _, path = record.partition(b"\t") + fields = meta.split(b" ") + if len(fields) < 3: + continue + self.modes[path.decode("utf-8", "surrogateescape")] = fields[0].decode() + + def file_mode(self, path: str) -> str | None: + return self.modes.get(path) + + def read_text(self, path: str) -> str: + if self.worktree: + disk = self.repo_root / path + if disk.is_symlink(): + raise CheckerError(f"{path}: worktree path is a symbolic link, not a regular file") + if disk.is_file(): + raw = disk.read_bytes() + else: # staged deletion or a staged-but-not-materialized path + raw = git(self.repo_root, "cat-file", "blob", f":{path}", binary=True) + else: + raw = git(self.repo_root, "cat-file", "blob", f"{self.ref}:{path}", binary=True) + try: + text = raw.decode("utf-8-sig") # utf-8-sig strips a leading BOM + except UnicodeDecodeError as exc: + raise CheckerError(f"{path}: not valid UTF-8, cannot scan for snippet tags ({exc})") + return text.replace("\r\n", "\n").replace("\r", "\n") + + def exists_on_disk(self, path: str) -> bool: + return (self.repo_root / path).is_file() + + +# -------------------------------------------------------------------------------------- +# snippet tag parsing +# -------------------------------------------------------------------------------------- +class Tag: + __slots__ = ("name", "closing", "line") + + def __init__(self, name: str, closing: bool, line: int) -> None: + self.name = name + self.closing = closing + self.line = line + + +def comment_markers(path: str) -> tuple[tuple[str, ...], tuple[str, str] | None]: + """Return (line markers, block marker pair) for a path, or raise for unsupported types.""" + suffix = PurePosixPath(path).suffix.lower() + if suffix == NOTEBOOK_EXT: + raise CheckerError( + f"{path}: notebooks identify snippets by cell metadata, not comment delimiters. " + f'Use "mode": "{MODE_NOTEBOOK_CELL}" with the cell names, or ' + f'"mode": "{MODE_WHOLE_FILE}".' + ) + if suffix in NO_COMMENT_EXTS: + raise CheckerError( + f"{path}: '{suffix}' has no comment syntax, so it cannot carry snippet delimiters. " + f"Use \"mode\": \"{MODE_WHOLE_FILE}\" for this entry." + ) + line = LINE_COMMENT_MARKERS.get(suffix, ()) + block = BLOCK_COMMENT_MARKERS.get(suffix) + if not line and not block: + raise CheckerError( + f"{path}: unknown file type '{suffix or '(no extension)'}'. Add its comment syntax to " + f"LINE_COMMENT_MARKERS/BLOCK_COMMENT_MARKERS in {Path(__file__).name}, or use " + f'"mode": "{MODE_WHOLE_FILE}".' + ) + return line, block + + +def parse_tags(text: str, path: str) -> list[Tag]: + """Extract snippet delimiter tags: comment-only lines whose entire body is ``/``.""" + line_markers, block = comment_markers(path) + tags: list[Tag] = [] + for number, raw_line in enumerate(text.split("\n"), start=1): + stripped = raw_line.strip().lstrip("\ufeff").strip() + body: str | None = None + if block and stripped.startswith(block[0]): + inner = stripped[len(block[0]) :] + if inner.endswith(block[1]): + body = inner[: -len(block[1])].strip() + if body is None: + for marker in line_markers: + if stripped.startswith(marker): + body = stripped[len(marker) :].strip() + break + if body is None or not TAG_RE.match(body): + continue + closing = body.startswith(" bool: + lines = text.split("\n") + return any(line.strip() for line in lines[open_line : close_line - 1]) + + +def notebook_cell_names(text: str, path: str) -> list[str]: + """Return `cells[*].metadata.name` values, in order. + + Only per-cell metadata counts. Notebook-level metadata (kernelspec/language_info `name`) + is deliberately ignored -- a raw text search for `"name":` would match those and produce + false positives. + """ + try: + document = json.loads(text) + except json.JSONDecodeError as exc: + raise CheckerError(f"{path}: not valid notebook JSON ({exc})") from exc + if not isinstance(document, dict): + raise CheckerError(f"{path}: notebook JSON must be an object") + cells = document.get("cells") + if not isinstance(cells, list): + raise CheckerError(f"{path}: notebook has no 'cells' array") + names: list[str] = [] + for cell in cells: + if not isinstance(cell, dict): + continue + metadata = cell.get("metadata") + if not isinstance(metadata, dict): + continue + name = metadata.get("name") + if isinstance(name, str) and name: + names.append(name) + return names + + +def notebook_cell_source(text: str, name: str) -> str: + """Concatenated source of the first cell whose metadata.name is `name` ('' if absent).""" + document = json.loads(text) + for cell in document.get("cells", []): + if not isinstance(cell, dict): + continue + metadata = cell.get("metadata") + if not isinstance(metadata, dict) or metadata.get("name") != name: + continue + source = cell.get("source", "") + if isinstance(source, list): + return "".join(str(part) for part in source) + return str(source) + return "" + + +def check_notebook_cells(text: str, path: str, expected: list[str]) -> list[str]: + """Verify each expected named cell still exists, keeps its name, and has content.""" + names = notebook_cell_names(text, path) + problems: list[str] = [] + for name in expected: + count = names.count(name) + if count == 0: + problems.append( + f'cell `{name}` is gone -- no cell has "metadata": {{"name": "{name}"}}. ' + "Either the cell was deleted or its name metadata was removed; a Learn article " + "includes that cell by this name." + ) + continue + if count > 1: + problems.append( + f"cell `{name}`: {count} cells claim this name -- it must be unique so the " + "article includes an unambiguous cell" + ) + continue + if not notebook_cell_source(text, name).strip(): + problems.append( + f"cell `{name}`: the cell is empty -- the docs would publish a blank code block" + ) + return problems + + +def check_snippets(text: str, path: str, expected: list[str]) -> list[str]: + """Return human-readable problems with the expected snippet regions in `text`.""" + tags = parse_tags(text, path) + problems: list[str] = [] + + # Structural nesting: nested regions are fine, crossing regions are not. + stack: list[Tag] = [] + for tag in tags: + if not tag.closing: + stack.append(tag) + continue + if not stack: + problems.append(f"line {tag.line}: `` has no matching opening tag") + elif stack[-1].name != tag.name: + problems.append( + f"line {tag.line}: `` closes out of order " + f"(expected `` opened on line {stack[-1].line}) -- " + "snippet regions may nest but must not overlap" + ) + stack.pop() + else: + stack.pop() + for tag in stack: + problems.append(f"line {tag.line}: `<{tag.name}>` is never closed") + + for name in expected: + opens = [t for t in tags if t.name == name and not t.closing] + closes = [t for t in tags if t.name == name and t.closing] + if not opens and not closes: + problems.append( + f"snippet `{name}` is gone -- both `<{name}>` and `` are missing" + ) + continue + if not opens: + problems.append(f"snippet `{name}`: opening tag `<{name}>` was removed") + continue + if not closes: + problems.append(f"snippet `{name}`: closing tag `` was removed") + continue + if len(opens) > 1 or len(closes) > 1: + problems.append( + f"snippet `{name}`: expected exactly one `<{name}>`/`` pair, found " + f"{len(opens)} opening and {len(closes)} closing tags" + ) + continue + if opens[0].line > closes[0].line: + problems.append( + f"snippet `{name}`: `` (line {closes[0].line}) comes before " + f"`<{name}>` (line {opens[0].line})" + ) + continue + if not region_has_content(text, opens[0].line, closes[0].line): + problems.append( + f"snippet `{name}`: the region between lines {opens[0].line} and " + f"{closes[0].line} is empty -- the docs would publish a blank code block" + ) + return problems + + +# -------------------------------------------------------------------------------------- +# manifest +# -------------------------------------------------------------------------------------- +class Entry: + __slots__ = ("path", "mode", "snippets", "note") + + def __init__(self, path: str, mode: str, snippets: list[str], note: str) -> None: + self.path = path + self.mode = mode + self.snippets = snippets + self.note = note + + +def load_manifest(manifest_path: Path) -> list[Entry]: + if not manifest_path.is_file(): + raise CheckerError(f"manifest not found: {manifest_path}") + try: + data = json.loads(manifest_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise CheckerError(f"{manifest_path}: invalid JSON ({exc})") from exc + if not isinstance(data, dict): + raise CheckerError(f"{manifest_path}: top level must be an object") + version = data.get("schemaVersion") + if version != SCHEMA_VERSION: + raise CheckerError( + f"{manifest_path}: schemaVersion {version!r} is not supported " + f"(this checker understands {SCHEMA_VERSION})" + ) + files = data.get("files") + if not isinstance(files, list) or not files: + raise CheckerError(f"{manifest_path}: 'files' must be a non-empty array") + + entries: list[Entry] = [] + seen: set[str] = set() + for index, item in enumerate(files): + where = f"{manifest_path}: files[{index}]" + # A bare string is the everyday form: just the path. Everything about it is discovered, + # and snippet removal is caught by comparing against the PR base. + if isinstance(item, str): + item = {"path": item, "mode": MODE_AUTO} + if not isinstance(item, dict): + raise CheckerError(f"{where}: must be a path string or an object") + path = item.get("path") + if not isinstance(path, str) or not path: + raise CheckerError(f"{where}: 'path' must be a non-empty string") + if path.startswith("/") or "\\" in path or ".." in PurePosixPath(path).parts: + raise CheckerError( + f"{where}: 'path' must be a repo-relative POSIX path without '..' (got {path!r})" + ) + if path in seen: + raise CheckerError(f"{where}: duplicate path {path!r}") + seen.add(path) + mode = item.get("mode", MODE_AUTO) + if mode not in VALID_MODES: + raise CheckerError(f"{where}: 'mode' must be one of {VALID_MODES} (got {mode!r})") + snippets = item.get("snippets", []) + if not isinstance(snippets, list) or not all( + isinstance(s, str) and s for s in snippets + ): + raise CheckerError(f"{where}: 'snippets' must be an array of non-empty strings") + if len(set(snippets)) != len(snippets): + raise CheckerError(f"{where}: 'snippets' contains duplicates") + if mode == MODE_AUTO and snippets: + raise CheckerError( + f"{where}: mode '{MODE_AUTO}' discovers snippets itself and must not list them " + f"(use '{MODE_DELIMITED}'/'{MODE_NOTEBOOK_CELL}' to pin an explicit list)" + ) + if mode == MODE_DELIMITED and not snippets: + raise CheckerError( + f"{where}: mode '{MODE_DELIMITED}' requires at least one snippet id " + f"(use '{MODE_WHOLE_FILE}' when the article includes the entire file)" + ) + if mode == MODE_NOTEBOOK_CELL: + if PurePosixPath(path).suffix.lower() != NOTEBOOK_EXT: + raise CheckerError( + f"{where}: mode '{MODE_NOTEBOOK_CELL}' only applies to {NOTEBOOK_EXT} files" + ) + if not snippets: + raise CheckerError( + f"{where}: mode '{MODE_NOTEBOOK_CELL}' requires at least one cell name " + f"(use '{MODE_WHOLE_FILE}' when the article includes the whole notebook)" + ) + if mode == MODE_DELIMITED and PurePosixPath(path).suffix.lower() == NOTEBOOK_EXT: + raise CheckerError( + f"{where}: notebooks use cell metadata names -- " + f"use mode '{MODE_NOTEBOOK_CELL}' instead of '{MODE_DELIMITED}'" + ) + if mode == MODE_WHOLE_FILE and snippets: + raise CheckerError(f"{where}: mode '{MODE_WHOLE_FILE}' must not list snippets") + note = item.get("note", "") + if not isinstance(note, str): + raise CheckerError(f"{where}: 'note' must be a string") + entries.append(Entry(path, mode, snippets, note)) + return entries + + +# -------------------------------------------------------------------------------------- +# validation +# -------------------------------------------------------------------------------------- +def rename_hints(repo_root: Path, base_ref: str, tree: Tree) -> dict[str, str]: + """old path -> new path, from git's rename detection. Diagnostics only, never a verdict.""" + # In worktree mode, diff base against the working tree (no second revision) so an + # uncommitted `git mv` still produces a hint. + args = ["diff", "-M", "--name-status", "-z", base_ref] + if not tree.worktree: + args.append(tree.ref or "HEAD") + raw = git(repo_root, *args, binary=True) + fields = [f.decode("utf-8", "surrogateescape") for f in raw.split(b"\0") if f] + hints: dict[str, str] = {} + index = 0 + while index < len(fields): + status = fields[index] + if status[:1] in ("R", "C") and index + 2 < len(fields): + hints[fields[index + 1]] = fields[index + 2] + index += 3 + else: + index += 2 + return hints + + +def discover_expected(text: str, path: str) -> tuple[str, list[str]]: + """Work out what a path's protected snippets are, from its own content. + + Returns (effective mode, snippet names). A file with no recognizable snippet markers -- + or a type that cannot carry them, like .json -- is treated as a whole-file reference. + """ + suffix = PurePosixPath(path).suffix.lower() + if suffix == NOTEBOOK_EXT: + deduped = list(dict.fromkeys(notebook_cell_names(text, path))) + return (MODE_NOTEBOOK_CELL, deduped) if deduped else (MODE_WHOLE_FILE, []) + if suffix in NO_COMMENT_EXTS: + return MODE_WHOLE_FILE, [] + tags = parse_tags(text, path) + deduped = list(dict.fromkeys(tag.name for tag in tags if not tag.closing)) + return (MODE_DELIMITED, deduped) if deduped else (MODE_WHOLE_FILE, []) + + +def resolve_entry( + entry: Entry, head_text: str, base_text: str | None +) -> tuple[str, list[str], str | None]: + """Return (mode, expected snippet names, warning) for an entry. + + Explicit modes are used as written. For `auto`, expectations come from the file as it exists + at the PR BASE -- that is what makes removal detectable: the base still has the tag or named + cell the head just lost. With no base available we can only validate the head's own structure. + """ + if entry.mode != MODE_AUTO: + return entry.mode, entry.snippets, None + if base_text is not None: + mode, names = discover_expected(base_text, entry.path) + return mode, names, None + mode, names = discover_expected(head_text, entry.path) + return ( + mode, + names, + f"{entry.path}: no base revision available, so snippet REMOVAL cannot be detected for " + "this auto entry; only the current file's own structure was validated.", + ) + + +def validate( + entries: list[Entry], + tree: Tree, + hints: dict[str, str], + base: Tree | None = None, + warnings: list[str] | None = None, +) -> list[str]: + failures: list[str] = [] + for entry in entries: + mode = tree.file_mode(entry.path) + if mode is None: + hint = hints.get(entry.path) + moved = f"\n git suggests it moved to: {hint}" if hint else "" + failures.append( + f"[missing] {entry.path}\n" + f" This file is referenced by a published Learn article and is no longer at " + f"this exact path in the {tree.label} (deleted, renamed, or moved).{moved}" + ) + continue + if mode not in REGULAR_FILE_MODES: + failures.append( + f"[not a file] {entry.path}\n" + f" Expected a regular file; git reports mode {mode} " + "(symlink, submodule, or directory)." + ) + continue + if tree.worktree and not tree.exists_on_disk(entry.path): + failures.append( + f"[missing] {entry.path}\n" + " Tracked by git but not present on disk (unstaged deletion)." + ) + continue + if entry.mode == MODE_WHOLE_FILE: + continue + try: + text = tree.read_text(entry.path) + base_text = None + if entry.mode == MODE_AUTO and base is not None: + if base.file_mode(entry.path) in REGULAR_FILE_MODES: + base_text = base.read_text(entry.path) + else: + # New in this PR (or absent from the base): nothing to compare against, so + # validate the head's own structure instead of silently expecting nothing. + base_text = text + effective_mode, expected, warning = resolve_entry(entry, text, base_text) + # `warning` only fires when no base exists at all, which main() already reports once; + # repeating it per entry would bury the real output under one line per manifest path. + del warning + if effective_mode == MODE_WHOLE_FILE: + continue + if effective_mode == MODE_NOTEBOOK_CELL: + problems = check_notebook_cells(text, entry.path, expected) + else: + problems = check_snippets(text, entry.path, expected) + except CheckerError as exc: + failures.append(f"[unreadable] {entry.path}\n {exc}") + continue + if problems: + detail = "\n".join(f" {p}" for p in problems) + label = ( + "notebook cell broken" + if effective_mode == MODE_NOTEBOOK_CELL + else "snippet broken" + ) + failures.append(f"[{label}] {entry.path}\n{detail}") + return failures + + +# -------------------------------------------------------------------------------------- +# seeding (docs-team maintenance) +# -------------------------------------------------------------------------------------- +def codeowners_docs_paths(repo_root: Path) -> list[str]: + """Docs-team-owned FILE patterns in CODEOWNERS, minus this check's own enforcement paths.""" + enforcement = { + DEFAULT_MANIFEST, + ".github/scripts/check-docs-referenced-files.py", + ".github/workflows/docs-referenced-files.yml", + } + paths: list[str] = [] + codeowners = repo_root / CODEOWNERS_PATH + if not codeowners.is_file(): + raise CheckerError(f"{CODEOWNERS_PATH} not found") + for line in codeowners.read_text(encoding="utf-8").split("\n"): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + fields = stripped.split() + if DOCS_TEAM.lower() not in (f.lower() for f in fields[1:]): + continue + pattern = fields[0] + if pattern.endswith("/"): + continue # directory ownership is not a file-level docs reference + relative = pattern.lstrip("/") + if relative in enforcement: + continue # the gate's own files are owned, but they are not docs-referenced samples + paths.append(relative) + return sorted(set(paths)) + + +def seed(repo_root: Path, manifest_path: Path, paths: list[str], tree: Tree, pin: bool) -> int: + """Write the manifest. + + Default output is a bare path string per file -- the everyday form the docs team maintains by + adding or deleting one line. `--pin` instead records the discovered mode and snippet ids + explicitly, which makes the check independent of the PR base for those entries. + """ + existing: dict[str, Entry] = {} + if manifest_path.is_file(): + try: + for entry in load_manifest(manifest_path): + existing[entry.path] = entry + except CheckerError: + existing = {} + + files: list[object] = [] + for path in paths: + if tree.file_mode(path) is None: + print(f"skip (not tracked): {path}", file=sys.stderr) + continue + previous = existing.get(path) + if not pin and previous is None: + files.append(path) + continue + if not pin and previous is not None: + item: dict[str, object] = {"path": path, "mode": previous.mode} + if previous.snippets: + item["snippets"] = previous.snippets + if previous.note: + item["note"] = previous.note + files.append(item if previous.mode != MODE_AUTO or previous.note else path) + continue + try: + entry_mode, snippets = discover_expected(tree.read_text(path), path) + except CheckerError as exc: + print(f"skip (cannot scan): {path}: {exc}", file=sys.stderr) + entry_mode, snippets = MODE_WHOLE_FILE, [] + item = {"path": path, "mode": entry_mode} + if snippets: + item["snippets"] = snippets + if previous and previous.note: + item["note"] = previous.note + files.append(item) + + document = { + "schemaVersion": SCHEMA_VERSION, + "$comment": ( + "Files that published Microsoft Learn articles include by reference. Owned by the " + "AI Platform Docs team. To protect a file, add its path to 'files' -- one string per " + "line is all you need. Renaming or deleting a listed file, or removing one of its " + "snippet delimiter comments or named notebook cells, breaks the docs build; " + ".github/workflows/docs-referenced-files.yml enforces this on every PR. " + "Do not edit without the docs team." + ), + "contact": { + "team": DOCS_TEAM, + "teamsChannelEmail": DOCS_TEAM_EMAIL, + }, + "files": sorted( + files, key=lambda item: item if isinstance(item, str) else str(item["path"]) + ), + } + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + print(f"wrote {manifest_path} with {len(files)} entries") + return EXIT_OK + + +# -------------------------------------------------------------------------------------- +# cli +# -------------------------------------------------------------------------------------- +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Verify that docs-referenced sample files and snippet delimiters are intact.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--manifest", default=DEFAULT_MANIFEST, help="manifest path") + parser.add_argument("--ref", default="HEAD", help="revision to validate (default: HEAD)") + parser.add_argument( + "--worktree", + action="store_true", + help="validate the working tree (staged + unstaged edits) instead of a revision", + ) + parser.add_argument( + "--base-ref", + default=None, + help=( + "the PR base revision. Auto entries take their expected snippets from here, so this " + "is what makes a REMOVED tag/cell detectable. Also supplies rename hints." + ), + ) + parser.add_argument("--seed", action="store_true", help="docs team: regenerate the manifest") + parser.add_argument( + "--pin", + action="store_true", + help="with --seed: write explicit mode/snippets instead of bare paths", + ) + parser.add_argument( + "--from-codeowners", + action="store_true", + help=f"with --seed: take paths owned by {DOCS_TEAM} in {CODEOWNERS_PATH}", + ) + parser.add_argument( + "--path", + action="append", + default=[], + help="with --seed: add/refresh a specific path (repeatable)", + ) + return parser + + +def main(argv: list[str]) -> int: + args = build_parser().parse_args(argv) + try: + repo_root = repo_root_from(Path.cwd()) + manifest_path = repo_root / args.manifest + + if args.seed: + tree = Tree(repo_root, None, worktree=True) + paths = list(args.path) + if args.from_codeowners: + paths.extend(codeowners_docs_paths(repo_root)) + if not paths and manifest_path.is_file(): + paths = [entry.path for entry in load_manifest(manifest_path)] + elif args.path and manifest_path.is_file(): + paths.extend(entry.path for entry in load_manifest(manifest_path)) + if not paths: + raise CheckerError("--seed needs --from-codeowners, --path, or an existing manifest") + return seed(repo_root, manifest_path, sorted(set(paths)), tree, args.pin) + + entries = load_manifest(manifest_path) + tree = Tree(repo_root, None if args.worktree else args.ref, worktree=args.worktree) + hints: dict[str, str] = {} + base: Tree | None = None + warnings: list[str] = [] + if args.base_ref: + try: + base = Tree(repo_root, args.base_ref, worktree=False) + hints = rename_hints(repo_root, args.base_ref, tree) + except CheckerError as exc: + print(f"note: base revision unavailable ({exc})", file=sys.stderr) + base = None + needs_base = any(entry.mode == MODE_AUTO for entry in entries) + if needs_base and base is None: + warnings.append( + "no --base-ref given: auto entries fall back to structure-only validation, so a " + "REMOVED snippet tag or named notebook cell will NOT be reported. CI always " + "passes the PR base." + ) + failures = validate(entries, tree, hints, base, warnings) + except CheckerError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return EXIT_ERROR + + for warning in dict.fromkeys(warnings): + print(f"warning: {warning}", file=sys.stderr) + + if not failures: + print( + f"OK: all {len(entries)} docs-referenced files and their snippet regions are intact " + f"({tree.label})." + ) + return EXIT_OK + + print("") + print("=" * 88) + print(" DOCS BUILD WOULD BREAK -- this PR changes files that Microsoft Learn publishes") + print("=" * 88) + print("") + for failure in failures: + print(failure) + print("") + print("-" * 88) + print("How to fix:") + print(" * Deleted/renamed a file? Restore the original path. Learn articles point at the") + print(" exact path; a rename must be published on the docs side FIRST.") + print(" * Removed a `# ` / `# ` comment? Put it back -- those delimiters mark") + print(" the exact region an article includes; they are not dead code.") + print(" * Deleted a notebook cell, or its `\"metadata\": {\"name\": ...}`? Restore both --") + print(" articles include notebook cells by that name.") + print(" * The change is genuinely required? It still cannot merge as-is.") + print("") + print(CONTACT) + print("-" * 88) + return EXIT_VIOLATION + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/scripts/test/test_docs_referenced_files.py b/.github/scripts/test/test_docs_referenced_files.py new file mode 100644 index 000000000..f4e58e54e --- /dev/null +++ b/.github/scripts/test/test_docs_referenced_files.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 +"""Contract tests for check-docs-referenced-files.py. + +Hermetic: every case builds a throwaway git repository in a temp dir, so nothing here depends on +the state of this repo. Run directly: python .github/scripts/test/test_docs_referenced_files.py +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts" / "check-docs-referenced-files.py" +WORKFLOW = ROOT / "workflows" / "docs-referenced-files.yml" +MANIFEST = ROOT.parent / ".github" / "docs-referenced-files.json" + +SPEC = importlib.util.spec_from_file_location("docs_referenced_files", CHECKER) +assert SPEC and SPEC.loader +checker = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(checker) + +PY_SAMPLE = """# +agent = client.create_agent() +# + +# +agent.run() +# +""" + +CS_SAMPLE = """\ufeff// +var items = new List(); +// +""" + + +def run_git(cwd: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(cwd), *args], check=True, capture_output=True) + + +class RepoFixture: + """A temp git repo with a manifest and sample files.""" + + def __init__(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) / "repo" + self.root.mkdir() + run_git(self.root, "init", "-q", "-b", "main") + run_git(self.root, "config", "user.email", "t@example.com") + run_git(self.root, "config", "user.name", "T") + run_git(self.root, "config", "commit.gpgsign", "false") + + def write(self, rel: str, text: str, newline: str = "\n") -> None: + target = self.root / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text.replace("\n", newline), encoding="utf-8", newline="") + + def write_manifest(self, files: list) -> None: + (self.root / ".github").mkdir(parents=True, exist_ok=True) + (self.root / ".github" / "docs-referenced-files.json").write_text( + json.dumps({"schemaVersion": 1, "files": files}, indent=2) + "\n", + encoding="utf-8", + ) + + def commit(self, message: str = "c") -> None: + run_git(self.root, "add", "-A") + run_git(self.root, "commit", "-q", "-m", message) + + def check(self, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(CHECKER), *args], + cwd=self.root, + capture_output=True, + text=True, + ) + + def cleanup(self) -> None: + self.tmp.cleanup() + + +class CheckerContractTests(unittest.TestCase): + def setUp(self) -> None: + self.repo = RepoFixture() + self.addCleanup(self.repo.cleanup) + self.repo.write("samples/app.py", PY_SAMPLE) + self.repo.write_manifest( + [ + { + "path": "samples/app.py", + "mode": "delimited", + "snippets": ["create_agent", "run_agent"], + } + ] + ) + self.repo.commit("initial") + + # --- happy path ------------------------------------------------------------------- + def test_intact_tree_passes(self) -> None: + result = self.repo.check() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("intact", result.stdout) + + # --- condition 2: deletion -------------------------------------------------------- + def test_deleted_file_fails(self) -> None: + (self.repo.root / "samples/app.py").unlink() + self.repo.commit("delete") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + self.assertIn("docs team", result.stdout.lower()) + + # --- condition 1: rename/move ----------------------------------------------------- + def test_rename_fails_with_hint(self) -> None: + base = subprocess.run( + ["git", "-C", str(self.repo.root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + run_git(self.repo.root, "mv", "samples/app.py", "samples/renamed.py") + self.repo.commit("rename") + result = self.repo.check("--base-ref", base) + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + self.assertIn("samples/renamed.py", result.stdout) + + def test_rename_with_edits_still_fails_without_hint(self) -> None: + run_git(self.repo.root, "mv", "samples/app.py", "samples/renamed.py") + (self.repo.root / "samples/renamed.py").write_text( + "# totally rewritten\n", encoding="utf-8" + ) + self.repo.commit("rename+rewrite") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + + def test_case_only_rename_fails(self) -> None: + run_git(self.repo.root, "mv", "samples/app.py", "samples/App.py") + self.repo.commit("case rename") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + + # --- condition 3: snippet delimiters ---------------------------------------------- + def test_removed_opening_tag_fails(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE.replace("# \n", "")) + self.repo.commit("drop open tag") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("create_agent", result.stdout) + + def test_removed_closing_tag_fails(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE.replace("# \n", "")) + self.repo.commit("drop close tag") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("run_agent", result.stdout) + + def test_whole_snippet_removed_fails(self) -> None: + self.repo.write("samples/app.py", "# \nagent.run()\n# \n") + self.repo.commit("drop region") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("`create_agent` is gone", result.stdout) + + def test_emptied_region_fails(self) -> None: + self.repo.write( + "samples/app.py", + "# \n# \n# \nagent.run()\n# \n", + ) + self.repo.commit("empty region") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("empty", result.stdout) + + def test_moved_region_still_passes(self) -> None: + self.repo.write( + "samples/app.py", + "import os\n\n" + "# \nagent.run()\n# \n\n" + "# \nagent = client.create_agent()\n# \n", + ) + self.repo.commit("reorder") + self.assertEqual(self.repo.check().returncode, 0) + + def test_crossed_regions_fail(self) -> None: + self.repo.write( + "samples/app.py", + "# \na\n# \nb\n# \nc\n# \n", + ) + self.repo.commit("crossed") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("out of order", result.stdout) + + def test_duplicate_tag_fails(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE + PY_SAMPLE) + self.repo.commit("duplicated") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("exactly one", result.stdout) + + # --- parser robustness ------------------------------------------------------------ + def test_bom_and_crlf_csharp_tags_are_found(self) -> None: + self.repo.write("samples/Program.cs", CS_SAMPLE, newline="\r\n") + self.repo.write_manifest( + [ + { + "path": "samples/app.py", + "mode": "delimited", + "snippets": ["create_agent", "run_agent"], + }, + { + "path": "samples/Program.cs", + "mode": "delimited", + "snippets": ["complete_code"], + }, + ] + ) + self.repo.commit("csharp") + self.assertEqual(self.repo.check().returncode, 0) + + def test_generic_type_is_not_a_tag(self) -> None: + tags = checker.parse_tags( + "var items = new List();\n", "samples/Program.cs" + ) + self.assertEqual(tags, []) + + def test_xml_comment_tags_are_found(self) -> None: + tags = checker.parse_tags( + "\n \n \n \n\n", + "samples/x.csproj", + ) + self.assertEqual([(t.name, t.closing) for t in tags], [("deps", False), ("deps", True)]) + + def test_json_cannot_be_delimited(self) -> None: + with self.assertRaises(checker.CheckerError): + checker.parse_tags('{"a": ""}\n', "samples/policy.json") + + def test_unknown_extension_is_rejected(self) -> None: + with self.assertRaises(checker.CheckerError): + checker.parse_tags("# \n", "samples/thing.wat") + + def test_whole_file_entry_only_checks_existence(self) -> None: + self.repo.write("samples/policy.json", '{"a": 1}\n') + self.repo.write_manifest([{"path": "samples/policy.json", "mode": "whole-file"}]) + self.repo.commit("json") + self.assertEqual(self.repo.check().returncode, 0) + (self.repo.root / "samples/policy.json").unlink() + self.repo.commit("rm json") + self.assertEqual(self.repo.check().returncode, 1) + + def test_symlink_replacement_is_rejected(self) -> None: + target = self.repo.root / "samples/app.py" + target.unlink() + os.symlink("elsewhere.py", target) + self.repo.commit("symlink") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("not a file", result.stdout) + + # --- worktree / local pre-push mode ------------------------------------------------ + def test_worktree_mode_sees_uncommitted_edit(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE.replace("# \n", "")) + result = self.repo.check("--worktree") + self.assertEqual(result.returncode, 1) + self.assertIn("run_agent", result.stdout) + # ...and the committed revision is still clean + self.assertEqual(self.repo.check("--ref", "HEAD").returncode, 0) + + def test_worktree_mode_sees_unstaged_deletion(self) -> None: + (self.repo.root / "samples/app.py").unlink() + result = self.repo.check("--worktree") + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + + def test_worktree_mode_rejects_unstaged_symlink_replacement(self) -> None: + self.repo.write("samples/elsewhere.py", PY_SAMPLE) + target = self.repo.root / "samples/app.py" + target.unlink() + os.symlink("elsewhere.py", target) + result = self.repo.check("--worktree") + self.assertEqual(result.returncode, 1) + self.assertIn("symbolic link", result.stdout) + + # --- fail-loud behavior ------------------------------------------------------------- + def test_missing_manifest_is_error_not_pass(self) -> None: + (self.repo.root / ".github" / "docs-referenced-files.json").unlink() + self.repo.commit("rm manifest") + result = self.repo.check() + self.assertEqual(result.returncode, 2) + self.assertIn("manifest not found", result.stderr) + + def test_malformed_manifest_is_error(self) -> None: + (self.repo.root / ".github" / "docs-referenced-files.json").write_text( + "{not json", encoding="utf-8" + ) + self.repo.commit("bad manifest") + self.assertEqual(self.repo.check().returncode, 2) + + def test_schema_version_mismatch_is_error(self) -> None: + (self.repo.root / ".github" / "docs-referenced-files.json").write_text( + json.dumps({"schemaVersion": 99, "files": []}), encoding="utf-8" + ) + self.repo.commit("bad schema") + self.assertEqual(self.repo.check().returncode, 2) + + def test_absolute_and_traversal_paths_rejected(self) -> None: + for bad in ("/samples/app.py", "../outside.py"): + with self.subTest(path=bad): + self.repo.write_manifest([{"path": bad, "mode": "whole-file"}]) + self.assertEqual(self.repo.check().returncode, 2) + + def test_duplicate_manifest_entry_rejected(self) -> None: + self.repo.write_manifest( + [ + {"path": "samples/app.py", "mode": "whole-file"}, + {"path": "samples/app.py", "mode": "whole-file"}, + ] + ) + self.assertEqual(self.repo.check().returncode, 2) + + def test_delimited_entry_without_snippets_rejected(self) -> None: + self.repo.write_manifest([{"path": "samples/app.py", "mode": "delimited"}]) + self.assertEqual(self.repo.check().returncode, 2) + + def test_unresolvable_ref_is_error(self) -> None: + result = self.repo.check("--ref", "does-not-exist") + self.assertEqual(result.returncode, 2) + self.assertIn("cannot resolve ref", result.stderr) + + # --- notebooks: named cells must survive ------------------------------------------- + def _notebook(self, cells: list[dict]) -> str: + return json.dumps( + { + "cells": cells, + "metadata": { + # Notebook-level `name` keys that a naive `"name":` search would wrongly match. + "kernelspec": {"name": "python3", "display_name": "Python 3"}, + "language_info": {"name": "python"}, + }, + "nbformat": 4, + "nbformat_minor": 5, + }, + indent=1, + ) + + def _seed_notebook(self, cells: list[dict], names: list[str]) -> None: + self.repo.write("samples/demo.ipynb", self._notebook(cells)) + self.repo.write_manifest( + [{"path": "samples/demo.ipynb", "mode": "notebook-cell", "snippets": names}] + ) + self.repo.commit("notebook") + + NAMED_CELL = { + "cell_type": "code", + "metadata": {"name": "gen-training-data-locally"}, + "source": ["print('train')\n"], + "outputs": [], + "execution_count": None, + } + PLAIN_CELL = { + "cell_type": "code", + "metadata": {}, + "source": ["print('other')\n"], + "outputs": [], + "execution_count": None, + } + + def test_notebook_named_cell_intact_passes(self) -> None: + self._seed_notebook( + [self.PLAIN_CELL, self.NAMED_CELL], ["gen-training-data-locally"] + ) + result = self.repo.check() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_notebook_deleted_cell_fails(self) -> None: + self._seed_notebook( + [self.PLAIN_CELL, self.NAMED_CELL], ["gen-training-data-locally"] + ) + self.repo.write("samples/demo.ipynb", self._notebook([self.PLAIN_CELL])) + self.repo.commit("delete named cell") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[notebook cell broken] samples/demo.ipynb", result.stdout) + self.assertIn("gen-training-data-locally", result.stdout) + + def test_notebook_removed_name_metadata_fails(self) -> None: + self._seed_notebook([self.NAMED_CELL], ["gen-training-data-locally"]) + stripped = dict(self.NAMED_CELL, metadata={}) + self.repo.write("samples/demo.ipynb", self._notebook([stripped])) + self.repo.commit("strip name metadata") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("name metadata was removed", result.stdout) + + def test_notebook_renamed_cell_fails(self) -> None: + self._seed_notebook([self.NAMED_CELL], ["gen-training-data-locally"]) + renamed = dict(self.NAMED_CELL, metadata={"name": "something-else"}) + self.repo.write("samples/demo.ipynb", self._notebook([renamed])) + self.repo.commit("rename cell") + self.assertEqual(self.repo.check().returncode, 1) + + def test_notebook_emptied_cell_fails(self) -> None: + self._seed_notebook([self.NAMED_CELL], ["gen-training-data-locally"]) + emptied = dict(self.NAMED_CELL, source=[]) + self.repo.write("samples/demo.ipynb", self._notebook([emptied])) + self.repo.commit("empty cell") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("empty", result.stdout) + + def test_notebook_duplicate_cell_name_fails(self) -> None: + self._seed_notebook( + [self.NAMED_CELL, dict(self.NAMED_CELL)], ["gen-training-data-locally"] + ) + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("claim this name", result.stdout) + + def test_notebook_edits_to_cell_body_still_pass(self) -> None: + self._seed_notebook([self.NAMED_CELL], ["gen-training-data-locally"]) + edited = dict(self.NAMED_CELL, source=["print('completely rewritten')\n"]) + self.repo.write("samples/demo.ipynb", self._notebook([edited])) + self.repo.commit("edit body") + self.assertEqual(self.repo.check().returncode, 0) + + def test_notebook_level_metadata_name_is_not_a_cell(self) -> None: + text = self._notebook([self.PLAIN_CELL]) + self.assertEqual(checker.notebook_cell_names(text, "samples/demo.ipynb"), []) + + def test_notebook_rejects_delimited_mode(self) -> None: + self.repo.write("samples/demo.ipynb", self._notebook([self.NAMED_CELL])) + self.repo.write_manifest( + [{"path": "samples/demo.ipynb", "mode": "delimited", "snippets": ["x"]}] + ) + self.repo.commit("bad mode") + result = self.repo.check() + self.assertEqual(result.returncode, 2) + self.assertIn("notebook-cell", result.stderr) + + def test_notebook_cell_mode_rejected_for_non_notebook(self) -> None: + self.repo.write_manifest( + [{"path": "samples/app.py", "mode": "notebook-cell", "snippets": ["x"]}] + ) + self.assertEqual(self.repo.check().returncode, 2) + + def test_malformed_notebook_is_reported(self) -> None: + self.repo.write("samples/demo.ipynb", "{not json") + self.repo.write_manifest( + [{"path": "samples/demo.ipynb", "mode": "notebook-cell", "snippets": ["x"]}] + ) + self.repo.commit("broken notebook") + result = self.repo.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[unreadable]", result.stdout) + + def test_notebook_whole_file_mode_ignores_cells(self) -> None: + self.repo.write("samples/demo.ipynb", self._notebook([self.PLAIN_CELL])) + self.repo.write_manifest([{"path": "samples/demo.ipynb", "mode": "whole-file"}]) + self.repo.commit("whole notebook") + self.assertEqual(self.repo.check().returncode, 0) + + # --- seeding ------------------------------------------------------------------------- + def test_seed_writes_bare_paths_by_default(self) -> None: + self.repo.write("samples/policy.json", '{"a": 1}\n') + self.repo.commit("add json") + result = self.repo.check( + "--seed", "--path", "samples/app.py", "--path", "samples/policy.json" + ) + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads( + (self.repo.root / ".github" / "docs-referenced-files.json").read_text() + ) + # samples/app.py was already pinned by the fixture, and a reseed must not downgrade it. + self.assertIsInstance(data["files"][0], dict) + self.assertEqual(data["files"][0]["path"], "samples/app.py") + # A freshly seeded path is written in the everyday bare form. + self.assertEqual(data["files"][1], "samples/policy.json") + + def test_seed_pin_infers_notebook_cell_mode(self) -> None: + self.repo.write("samples/demo.ipynb", self._notebook([self.PLAIN_CELL, self.NAMED_CELL])) + self.repo.commit("add notebook") + result = self.repo.check("--seed", "--pin", "--path", "samples/demo.ipynb") + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads( + (self.repo.root / ".github" / "docs-referenced-files.json").read_text() + ) + entry = { + item["path"]: item for item in data["files"] if isinstance(item, dict) + }["samples/demo.ipynb"] + self.assertEqual(entry["mode"], "notebook-cell") + self.assertEqual(entry["snippets"], ["gen-training-data-locally"]) + + def test_seed_pin_infers_modes_and_snippets(self) -> None: + self.repo.write("samples/policy.json", '{"a": 1}\n') + self.repo.commit("add json") + result = self.repo.check( + "--seed", "--pin", "--path", "samples/app.py", "--path", "samples/policy.json" + ) + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads( + (self.repo.root / ".github" / "docs-referenced-files.json").read_text() + ) + by_path = {item["path"]: item for item in data["files"]} + self.assertEqual(by_path["samples/app.py"]["snippets"], ["create_agent", "run_agent"]) + self.assertEqual(by_path["samples/policy.json"]["mode"], "whole-file") + + def test_seed_preserves_existing_pinned_entry(self) -> None: + result = self.repo.check("--seed", "--path", "samples/app.py") + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads( + (self.repo.root / ".github" / "docs-referenced-files.json").read_text() + ) + self.assertEqual( + data["files"], + [ + { + "path": "samples/app.py", + "mode": "delimited", + "snippets": ["create_agent", "run_agent"], + } + ], + ) + + def test_seed_preserves_annotated_auto_entry(self) -> None: + self.repo.write_manifest( + [{"path": "samples/app.py", "mode": "auto", "note": "Docs uses this file."}] + ) + result = self.repo.check("--seed", "--path", "samples/app.py") + self.assertEqual(result.returncode, 0, result.stderr) + data = json.loads( + (self.repo.root / ".github" / "docs-referenced-files.json").read_text() + ) + self.assertEqual( + data["files"], + [{"path": "samples/app.py", "mode": "auto", "note": "Docs uses this file."}], + ) + + +class AutoModeTests(unittest.TestCase): + """Bare-path manifest entries: expectations discovered from the file, removal caught vs base.""" + + def setUp(self) -> None: + self.repo = RepoFixture() + self.addCleanup(self.repo.cleanup) + self.repo.write("samples/app.py", PY_SAMPLE) + # A bare string, not an object -- the everyday form the docs team maintains. + self.repo.write_manifest(["samples/app.py"]) + self.repo.commit("initial") + self.base = subprocess.run( + ["git", "-C", str(self.repo.root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + def check(self, *args: str) -> subprocess.CompletedProcess: + return self.repo.check("--base-ref", self.base, *args) + + def test_bare_path_entry_passes_when_intact(self) -> None: + result = self.check() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_bare_path_detects_removed_tag_against_base(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE.replace("# \n", "")) + self.repo.commit("drop tag") + result = self.check() + self.assertEqual(result.returncode, 1) + self.assertIn("create_agent", result.stdout) + + def test_bare_path_detects_whole_region_removal(self) -> None: + self.repo.write("samples/app.py", "# \nagent.run()\n# \n") + self.repo.commit("drop region") + result = self.check() + self.assertEqual(result.returncode, 1) + self.assertIn("`create_agent` is gone", result.stdout) + + def test_bare_path_detects_deletion(self) -> None: + (self.repo.root / "samples/app.py").unlink() + self.repo.commit("delete") + result = self.check() + self.assertEqual(result.returncode, 1) + self.assertIn("[missing] samples/app.py", result.stdout) + + def test_bare_path_allows_adding_a_new_snippet(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE + "\n# \nx = 1\n# \n") + self.repo.commit("add snippet") + self.assertEqual(self.check().returncode, 0) + + def test_bare_path_allows_editing_snippet_body(self) -> None: + self.repo.write( + "samples/app.py", + PY_SAMPLE.replace("agent = client.create_agent()", "agent = build() # rewritten"), + ) + self.repo.commit("edit body") + self.assertEqual(self.check().returncode, 0) + + def test_bare_unknown_extension_requires_explicit_whole_file_mode(self) -> None: + self.repo.write("samples/unknown.wat", "# \nvalue\n# \n") + self.repo.write_manifest(["samples/app.py", "samples/unknown.wat"]) + self.repo.commit("add unknown file") + base = subprocess.run( + ["git", "-C", str(self.repo.root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + self.repo.write("samples/unknown.wat", "value\n") + self.repo.commit("drop unknown tag") + result = self.repo.check("--base-ref", base) + self.assertEqual(result.returncode, 1) + self.assertIn("unknown file type", result.stdout) + + def test_bare_path_notebook_cell_removal_detected(self) -> None: + notebook = json.dumps( + { + "cells": [ + { + "cell_type": "code", + "metadata": {"name": "gen-training-data-locally"}, + "source": ["print('x')\n"], + "outputs": [], + "execution_count": None, + } + ], + "metadata": {"language_info": {"name": "python"}}, + "nbformat": 4, + "nbformat_minor": 5, + }, + indent=1, + ) + self.repo.write("samples/demo.ipynb", notebook) + self.repo.write_manifest(["samples/app.py", "samples/demo.ipynb"]) + self.repo.commit("add notebook") + base = subprocess.run( + ["git", "-C", str(self.repo.root), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + stripped = json.loads(notebook) + stripped["cells"][0]["metadata"] = {} + self.repo.write("samples/demo.ipynb", json.dumps(stripped, indent=1)) + self.repo.commit("strip cell name") + result = self.repo.check("--base-ref", base) + self.assertEqual(result.returncode, 1) + self.assertIn("gen-training-data-locally", result.stdout) + + def test_file_added_in_this_pr_is_structurally_validated(self) -> None: + # Not present at the base: there is nothing to compare to, so the head's own structure + # is checked rather than the entry silently expecting nothing. + self.repo.write("samples/new.py", "# \nx = 1\n") + self.repo.write_manifest(["samples/app.py", "samples/new.py"]) + self.repo.commit("add new file") + result = self.check() + self.assertEqual(result.returncode, 1) + self.assertIn("never closed", result.stdout) + + def test_missing_base_warns_and_degrades_loudly(self) -> None: + self.repo.write("samples/app.py", PY_SAMPLE.replace("# \n", "")) + self.repo.commit("drop tag") + result = self.repo.check() # no --base-ref + self.assertIn("no --base-ref given", result.stderr) + # Structure is still checked, so the now-orphaned closing tag is caught. + self.assertEqual(result.returncode, 1) + + def test_auto_entry_may_not_list_snippets(self) -> None: + self.repo.write_manifest( + [{"path": "samples/app.py", "mode": "auto", "snippets": ["create_agent"]}] + ) + result = self.check() + self.assertEqual(result.returncode, 2) + self.assertIn("discovers snippets itself", result.stderr) + + def test_pinned_entry_needs_no_base(self) -> None: + self.repo.write_manifest( + [{"path": "samples/app.py", "mode": "delimited", "snippets": ["create_agent"]}] + ) + self.repo.write("samples/app.py", "# \nagent.run()\n# \n") + self.repo.commit("drop pinned snippet") + result = self.repo.check() # no base at all + self.assertEqual(result.returncode, 1) + self.assertIn("`create_agent` is gone", result.stdout) + self.assertNotIn("no --base-ref given", result.stderr) + + +class RepoManifestTests(unittest.TestCase): + """The real manifest and workflow in THIS repo must stay coherent.""" + + def test_repo_manifest_is_valid(self) -> None: + entries = checker.load_manifest(MANIFEST) + self.assertGreater(len(entries), 0) + + def test_workflow_is_fork_safe_and_unskippable(self) -> None: + text = WORKFLOW.read_text(encoding="utf-8") + directives = "\n".join( + line for line in text.split("\n") if not line.lstrip().startswith("#") + ) + self.assertIn("name: docs-referenced-files", directives) + self.assertIn("on:\n pull_request:", directives) + self.assertNotIn("pull_request_target", directives) + self.assertIn("permissions:\n contents: read", directives) + self.assertNotIn("secrets.", directives) + # A `paths:` filter under pull_request would let a required check conclude `skipped`. + pr_block = directives.split("on:\n pull_request:", 1)[1].split("\npermissions:", 1)[0] + self.assertNotIn("paths:", pr_block) + self.assertIn("fetch-depth: 0", directives) + self.assertIn("check-docs-referenced-files.py", directives) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/docs-referenced-files.yml b/.github/workflows/docs-referenced-files.yml new file mode 100644 index 000000000..f3189e12d --- /dev/null +++ b/.github/workflows/docs-referenced-files.yml @@ -0,0 +1,67 @@ +name: docs-referenced-files + +# Protects the sample files that PUBLISHED Microsoft Learn articles include by reference +# (`:::code source="" id="":::`). Three changes break the docs build: +# 1. the referenced file is deleted +# 2. the referenced file is renamed/moved +# 3. a `# ` / `# ` delimiter comment is removed +# `.github/docs-referenced-files.json` is the manifest of what the docs reference; the checker +# derives expected regions from the PR base, validates the PR head, and fails with instructions +# to contact the docs team. +# +# This replaces per-file CODEOWNERS entries + manual docs-team approval on every PR. +# +# FORK-SAFE BY CONSTRUCTION, following the conventions documented in validate.yml: +# - on: pull_request (NEVER pull_request_target) -> fork PRs run with a read-only token. +# - permissions: contents: read only. No secrets, no OIDC, no privileged runner. +# - No `paths:` filter: a required check must never conclude `skipped` (GitHub treats a skipped +# required check as satisfied, which would silently open the gate). +# - Single job with a stable name, so the required-check context is exactly +# `docs-referenced-files / check`. +# The checker only reads the repo and runs git plumbing; it executes no sample code. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + +permissions: + contents: read + +concurrency: + group: docs-referenced-files-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + name: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # Full history so --base-ref can resolve the PR base commit. The base contains the + # expected regions that the head is required to preserve. + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Verify docs-referenced files and snippet delimiters + env: + # Exact SHA from the event payload, not a moving branch ref. The manifest lists bare + # paths, so the checker reads each file's expected snippet ids from the BASE revision -- + # that is what makes a REMOVED delimiter detectable. Load-bearing, not a hint: if the + # base is unavailable the job fails rather than silently degrading to a weaker check. + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + run: | + set -euo pipefail + if [ -z "${BASE_SHA:-}" ]; then + echo "::error::No base SHA in the event payload; cannot compare against the PR base." + exit 1 + fi + if ! git rev-parse --verify --quiet "${BASE_SHA}^{commit}" >/dev/null; then + echo "::error::Base commit ${BASE_SHA} is missing. Check out with fetch-depth: 0." + exit 1 + fi + python .github/scripts/check-docs-referenced-files.py --ref HEAD --base-ref "$BASE_SHA" diff --git a/.github/workflows/scripts-selftest.yml b/.github/workflows/scripts-selftest.yml index 02a328752..a429e9315 100644 --- a/.github/workflows/scripts-selftest.yml +++ b/.github/workflows/scripts-selftest.yml @@ -25,6 +25,8 @@ on: - '.github/workflows/validation-report.yml' - '.github/workflows/validation-pilot.yml' - '.github/scripts/discover-validation-samples.py' + - '.github/docs-referenced-files.json' + - '.github/workflows/docs-referenced-files.yml' permissions: contents: read @@ -38,6 +40,23 @@ jobs: - name: validate.yml trusted-gate structure exit gate run: bash .github/scripts/test/run-workflow-structure-tests.sh + # --- Docs-referenced-file guard: hermetic Python unittest, temp git repos only ----------------- + # Contract tests for the gate that stops a PR from breaking a published Learn article by + # deleting/renaming a referenced sample or removing a snippet delimiter or named notebook cell. + # The gate itself is `docs-referenced-files.yml`; this job only proves the checker still behaves. + docs-referenced-files-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: check-docs-referenced-files contract tests + run: python .github/scripts/test/test_docs_referenced_files.py + # --- Detector harness: hermetic (git + coreutils only), runs fully GREEN anywhere -------------- detect-harness: runs-on: ubuntu-latest