diff --git a/shape-doctor.py b/shape-doctor.py index 1d1b3331..1f93561c 100755 --- a/shape-doctor.py +++ b/shape-doctor.py @@ -2021,10 +2021,27 @@ def check_manifest_kinds(ctx: Context) -> Row: next command -- register a kind, or remove the file from the root -- that is the wrong remedy for a merge conflict in a project's own manifest. So it is skipped, and the row says where it went. + + A FILE THAT DOES NOT LOAD IS NOT AN UNREGISTERED KIND (#291). It used to + share that case's next command -- register the kind, or remove the file + from the root -- which no file without a readable `kind:` can follow, and + which told a reader with a conflicted `contracts/shape-pin.yaml` to + delete the pin. The row now quotes what the loader said about each one, + and `unreadable_next` picks the command. That sentence stays the + unregistered kind's alone, and where a row has more than one fault -- an + unregistered kind, a kind registered to a gate this root never runs, a + `family.yaml` beside `project.yaml` -- it names the restore alone, first, + because the row cannot say what kind a file declares until it loads, and + because a command and a sentence joined by `;` would be a second command + in a pasted line (#277). The reason still names every fault, and the + next run names the rest. `restore` and `refusals` join `detail` only + when a file does not load, so every other root reports the row it + always did. """ known: list[str] = [] unknown: list[str] = [] unreadable: list[str] = [] + refusals: dict[str, dict] = {} #: `(path, kind, entry)` for each manifest whose registered gate never #: runs on this kind of root (#327). ungated: list[tuple] = [] @@ -2034,14 +2051,26 @@ def check_manifest_kinds(ctx: Context) -> Row: for path in manifest_candidates(ctx.root): if path == elsewhere: continue - data = Context._read(path) + rel = path.relative_to(ctx.root).as_posix() + # `load_yaml` and not `Context._read`, which says None for both a + # refusal and an empty file: the row quotes the refusal, less the + # path it opens with, which the row has just named. + try: + data = load_yaml(path) + except Refusal as exc: + unreadable.append(rel) + refusals[rel] = {"code": exc.code, + "detail": exc.detail.removeprefix(f"{path}: ")} + continue if not isinstance(data, dict): - unreadable.append(path.relative_to(ctx.root).as_posix()) + unreadable.append(rel) + refusals[rel] = {"code": None, "detail": ( + "it holds no YAML document" if data is None else + f"it loads as a {type(data).__name__}, not a mapping")} continue kind = data.get("kind") if not isinstance(kind, str) or not kind: continue - rel = path.relative_to(ctx.root).as_posix() if (ctx.kind == PROJECT and rel == FAMILY_MANIFEST and kind == "family-manifest"): beside = family_beside_project(ctx, data) @@ -2079,20 +2108,26 @@ def check_manifest_kinds(ctx: Context) -> Row: if unknown: parts.append("no validator registered for kind " + ", ".join(unknown)) - if unreadable: - parts.append("unreadable as YAML: " + ", ".join(unreadable)) - if unknown or unreadable: fixes.append("register the kind in shape-doctor.py's " "MANIFEST_VALIDATORS, beside the check that " "validates it, or remove the file from the root") + if unreadable: + said = {rel: ": ".join(filter(None, (why["code"], why["detail"]))) + for rel, why in refusals.items()} + parts.append("unreadable as YAML: " + ", ".join( + f"{rel} ({said[rel]})" for rel in unreadable)) if ungated_said is not None: parts.append(ungated_said[0]) fixes.append(ungated_said[1]) if beside is not None: parts.append(beside[0]) fixes.append(FAMILY_BESIDE_FIX) + command = "; ".join(fixes) + if unreadable: + restore, command = unreadable_next(ctx, unreadable) + detail.update({"restore": restore, "refusals": refusals}) return Row("manifest-kinds", LABEL_MANIFEST_KINDS, FINDING, - "; ".join(parts) + aside, "; ".join(fixes), detail) + "; ".join(parts) + aside, command, detail) return Row("manifest-kinds", LABEL_MANIFEST_KINDS, OK, f"{len(known)} manifest(s), every declared kind has a " f"registered validator{aside}", None, detail) @@ -2204,9 +2239,15 @@ def git_answer(args: list, cwd: Path, binary: bool = False): PIN_HISTORY_DEPTH = 50 -def pin_reads_at(ctx: Context, rev: str) -> bool: - """Whether the pin `rev` holds reads as one, by `read_pin`'s own checks - (#237). +def pin_reads_at(ctx: Context, rev: str, pin: str = SHAPE_PIN) -> bool: + """Whether the pin `rev` holds reads as one: `pin_at` names something.""" + return pin_at(ctx, rev, pin) is not None + + +def pin_at(ctx: Context, rev: str, pin: str = SHAPE_PIN): + """What the pin `rev` holds reads as, by `read_pin`'s own checks (#237), + or None where it does not read: for the shape pin the paths its + `files:` rows name, never empty, and for any other file the mapping. READ BEFORE IT IS NAMED: a NEXT that puts back a pin out of history must name one that reads. Rolling back to a pin that is broken too @@ -2226,22 +2267,35 @@ def pin_reads_at(ctx: Context, rev: str) -> bool: pin's bytes raises one now, so one means a caller out of step with `check_pin`, and the doctor fails loudly rather than answer that no commit holds a pin to put back. + + ANY OTHER FILE `tool_written` ADMITS READS WHERE IT LOADS AS A MAPPING + (#291). That is the question the `manifest kinds` row asks of it, and + the one reader this file holds for a leg pin or a copy, so what the row + puts back is what the row then reads. + + THE ROWS, NOT A YES, for the shape pin (review of PR #358): `rev` may + be an index stage, `:2` or `:3`, and the rows of the side of a merge + `pin_restore` takes whole are the copies that side's tool wrote, so + they are what it names with the pin. """ - blob = git_answer(["cat-file", "blob", f"{rev}:./{SHAPE_PIN}"], + blob = git_answer(["cat-file", "blob", f"{rev}:./{pin}"], ctx.root, binary=True) if blob is None: - return False + return None us = ctx.update_shape() try: text = blob.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") - us.check_pin(us.parse_yaml(text), f"{rev}:{SHAPE_PIN}", ctx.root) + data = us.parse_yaml(text) + if pin != SHAPE_PIN: + return data if isinstance(data, dict) else None + _pin, rows = us.check_pin(data, f"{rev}:{SHAPE_PIN}", ctx.root) except (Refusal, us.YamlError, ValueError): - return False - return True + return None + return [row.path for row in rows] -def pin_restore(ctx: Context, - detail: dict | None = None) -> tuple[str, str | None]: +def pin_restore(ctx: Context, detail: dict | None = None, + pin: str = SHAPE_PIN) -> tuple[str, str | None]: """`(restore, command)`: the git command that puts back a pin a tool wrote, chosen from the pin's own git state, read-only (#237). @@ -2252,10 +2306,14 @@ def pin_restore(ctx: Context, unmerged a merge left the pin conflicted. `checkout --ours`, or `--theirs` where only that side holds a pin, of every - conflicted path the shape writes WHOLE -- the pin, the - copies and their collision copies -- because a merge of - two `update-shape.py apply` runs conflicts in them - together, and one side taken whole is one tool's output. + conflicted path the shape writes WHOLE -- the pin, and + the copies that side's pin has `files:` rows for, their + collision copies included -- because a merge of two + `update-shape.py apply` runs conflicts in them together, + and one side taken whole is one tool's output. Those rows + and not this standard's copy lists: a file the pin has no + row for is not a shape copy, though the lists name its + path (review of PR #358). The manifest and README.md are never named, though `apply` writes lines of them: every other line is the project's own, and one side taken whole drops the other side's @@ -2293,18 +2351,57 @@ def pin_restore(ctx: Context, path is read relative to the root (`ls-files` and pathspecs are, and `:./` is), so each one names the root's own file even where git answers from a checkout the root sits inside. + + ANY FILE UNDER `contracts/` A TOOL WROTE, NOT THE SHAPE PIN ALONE (#291). + `pin` is the root's own `contracts/shape-pin.yaml` unless the caller + names another file `tool_written` admits -- a pin `project.yaml` + declares, for a leg or a neutral product, or a copy the shape pin has a + row for -- which is how the `manifest kinds` row answers each one that + does not load. The note says `copy` where the file is not a pin, and + `pin_reads_at` says what reads. Only the shape pin takes more paths with + it in a merge -- the copies `apply` writes in the same commit -- and + only its note names a conflicted manifest or README.md for a person. + Any other file is named alone. A leg pin moves with its gitlink, but + `checkout --ours` on a conflicted gitlink exits 0 and leaves all three + stages where they were, so naming it would be a command that changes + nothing. Finishing the merge takes the gitlink, and the `pins` row then + says whether the two agree. + + A FILE A PERSON MAY HAVE EDITED IS SHOWN, NEVER PUT BACK, in every + state (review of PR #358). That is a copy, which can hold an edit of + the project's -- the merge by hand `update-shape.py` asks for where a + copy changed on both sides above all -- and a neutral product pin, + which no tool writes after the scaffold, so every change to one is a + person's. `edited_by_hand` says which. Each of the three commands above + drops that edit: the other side of their merge and the hunks git merged + cleanly, their uncommitted work, their commit. So the state picks a + read-only command instead: `diff` for `unmerged`, which shows the hunks + git could not merge, `diff HEAD` for `uncommitted`, and `diff + ~1` for `committed`, every change since the newest one + that reads. Mending the file, or merging it, is the person's, as the + `both` exit of `update-shape.py` says of a copy changed on both sides: + a merge by hand, then `--accept-local` on it, which is what the `shape + currency` row names for the same file. """ root = ctx.root git_c = f"git -C {quote_arg(root)}" + what = "pin" if pin.endswith("-pin.yaml") else "copy" + shown = edited_by_hand(ctx, pin) + why = BY_HAND_WHY[what] raw = git_answer(["--no-optional-locks", "status", "--porcelain=v1", - "--", SHAPE_PIN], root, binary=True) + "--", pin], root, binary=True) xy = raw.decode("utf-8", "replace")[:2] if raw else "" + if xy in UNMERGED_STATUS and shown: + return "unmerged", ( + f"{git_c} diff -- {quote_arg(pin)} # a merge left the {what} " + "conflicted, and this shows the hunks git could not merge. " + f"{why}, so neither side is taken whole: merge it by hand, " + "keeping the hunks git merged, then finish the merge as git says " + "and run this doctor again") if xy in UNMERGED_STATUS: us = ctx.update_shape() kind = us.root_kind(root) - whole = {SHAPE_PIN} - for rel in kind.copy_source: - whole.update((rel, f"{us.COLLISION_DIR}/{rel}")) + whole = {pin} # ` SP SP TAB `, one record per stage. stages: dict[str, set] = {} listing = git_answer(["ls-files", "-u", "-z"], root, binary=True) @@ -2313,19 +2410,21 @@ def pin_restore(ctx: Context, if path: stages.setdefault(path.decode("utf-8", "replace"), set()).add(meta.rsplit(b" ", 1)[-1]) - pin_stages = stages.get(SHAPE_PIN, set()) + pin_stages = stages.get(pin, set()) side, stage = (("--ours", b"2") if b"2" in pin_stages else ("--theirs", b"3") if b"3" in pin_stages else (None, None)) if side: + if pin == SHAPE_PIN: + whole.update(pin_at(ctx, f":{stage.decode()}") or ()) note = ("--theirs takes the other side. Take one side whole" if b"3" in pin_stages and side == "--ours" - else "only that side of the merge holds a pin. Take it " - "whole") + else f"only that side of the merge holds a {what}. Take " + "it whole") named = sorted(name for name, held in stages.items() if name in whole and stage in held) - left = sorted(name for name in (kind.manifest, us.README) - if name in stages) + left = (sorted(name for name in (kind.manifest, us.README) + if name in stages) if pin == SHAPE_PIN else []) then = ", then finish" if left: if detail is not None: @@ -2341,40 +2440,220 @@ def pin_restore(ctx: Context, f"{git_c} checkout {side} -- " + " ".join(quote_arg(name) for name in named) + f" # {note} for every path named and never a " - f"hand-merged pin{then} the merge as git says and run " + f"hand-merged {what}{then} the merge as git says and run " "this doctor again") - elif pin_reads_at(ctx, "HEAD"): + elif pin_reads_at(ctx, "HEAD", pin): + if xy and shown: + return "uncommitted", ( + f"{git_c} diff HEAD -- {quote_arg(pin)} # HEAD holds a " + f"{what} that reads, and this shows the uncommitted change " + f"that breaks the {what} here. {why}. Git keeps no other copy " + "of the change until it is committed, so nothing is put back. " + f"Mend the change, or take the {what} as last committed once " + "nothing in it is worth keeping, then run this doctor again") if xy: return "uncommitted", ( - f"{git_c} checkout HEAD -- {quote_arg(SHAPE_PIN)} # puts " - "back the pin as last committed and discards the uncommitted " + f"{git_c} checkout HEAD -- {quote_arg(pin)} # puts back " + f"the {what} as last committed and discards the uncommitted " "change that broke it, which no tool of this standard writes") return "hidden", ( - f"{git_c} ls-files -v -- {quote_arg(SHAPE_PIN)} # HEAD holds a " - "pin that reads and git status shows no change to the copy here, " - "so no commit broke it and nothing in history is put back. S in " - "the first column is skip-worktree, as a sparse checkout sets " - "it, and a lowercase h is assume-unchanged: either hides the " + f"{git_c} ls-files -v -- {quote_arg(pin)} # HEAD holds a " + f"{what} that reads and git status shows no change to the copy " + "here, so no commit broke it and nothing in history is put back. " + "S in the first column is skip-worktree, as a sparse checkout " + "sets it, and a lowercase h is assume-unchanged: either hides the " "copy here from git. Any other letter means the file itself " "cannot be read, as the refusal says. Clearing that is for a " "person, then run this doctor again") history = git_answer(["rev-list", "-n", str(PIN_HISTORY_DEPTH), "HEAD", - "--", SHAPE_PIN], root) + "--", pin], root) for breaking in (history or "").split(): before = f"{breaking}~1" - if git_answer(["cat-file", "-e", f"{before}:./{SHAPE_PIN}"], + if git_answer(["cat-file", "-e", f"{before}:./{pin}"], root) is None: break - if pin_reads_at(ctx, before): + if not pin_reads_at(ctx, before, pin): + continue + if shown: return "committed", ( - f"{git_c} checkout {quote_arg(before)} -- " - f"{quote_arg(SHAPE_PIN)} # the pin as it was before " - f"{breaking[:12]}, the commit that broke it, and the newest " - "pin in its history that reads. Commit that rather than " - "editing the pin") + f"{git_c} diff {quote_arg(before)} -- {quote_arg(pin)} # " + f"the newest {what} in its history that reads is the one " + f"before {breaking[:12]}, the commit that broke it, and this " + f"shows every change since. {why}, so nothing is rolled " + f"back: mend the {what}, commit that and run this doctor " + "again") + return "committed", ( + f"{git_c} checkout {quote_arg(before)} -- " + f"{quote_arg(pin)} # the {what} as it was before " + f"{breaking[:12]}, the commit that broke it, and the newest " + f"{what} in its history that reads. Commit that rather than " + f"editing the {what}") return "none", None +def declared_pins(ctx: Context) -> set: + """Every pin this root DECLARES, at the path the tool that writes it + writes it to (#291). + + The shape pin, which every root carries. `contracts/-pin.yaml` + for each leg `project.yaml` declares, the spelling `check_legs` reads, + `validate-pins.py` checks and `bump-leg.py` writes. And + `contracts/-pin.yaml` for each name its + `neutral_product_pins:` lists, which is `shape_materialize`'s + `neutral_pin_path` and the spelling `validate-pins.py` checks, spelled + here as `check_legs` spells a leg's. A role or a product that is not + text names no pin. + """ + pins = {SHAPE_PIN} + pins.update(f"contracts/{leg['role']}-pin.yaml" for leg in ctx.legs() + if isinstance(leg.get("role"), str) and leg["role"]) + return pins | neutral_pins(ctx) + + +def neutral_pins(ctx: Context) -> set: + """`contracts/-pin.yaml` for each name the root's + `neutral_product_pins:` lists, as `declared_pins` spells them (#291).""" + products = (ctx.manifest or {}).get("neutral_product_pins") + if not isinstance(products, list): + return set() + return {f"contracts/{product.lower()}-pin.yaml" for product in products + if isinstance(product, str) and product} + + +def pinned_copies(ctx: Context) -> set: + """The root paths the root's own shape pin has `files:` rows for: its + shape copies, read as `update-shape.py` reads them (review of PR #358). + + THE ROWS, NOT THIS STANDARD'S COPY LISTS. A file with no row is not a + shape copy (`update-shape.py`'s own rule, and AGENTS.md's): an adopted + project that merged a copy away drops its row, and the file left at its + path is the project's own, as is the source's own file at a path an + adoption collided on, whose copy went under `shape/` with a row naming + that path. The lists name both paths all the same. Where the pin does + not read, no row says anything, so nothing is a copy for that run: the + safe side, as for a `project.yaml` that does not load. + """ + us = ctx.update_shape() + try: + _pin, rows = us.read_pin(ctx.root) + except (Refusal, us.YamlError, OSError, ValueError): + return set() + return {row.path for row in rows} + + +def edited_by_hand(ctx: Context, rel: str) -> bool: + """Whether `rel` is a file a tool wrote that a person may have edited + since, so that `pin_restore` shows it and puts nothing back (review of + PR #358): a copy the shape pin has a row for, or a neutral product pin. + + A copy can hold the project's own edit: `update-shape.py` calls one + `locally-modified` and leaves it, and where a copy changed on both + sides it asks for a merge by hand. A neutral product pin is written by + `scaffold-project.py --pin` and by no tool after it, and the manifest + template says to add one by hand, so every change to one is a + person's. The shape pin and a leg pin are a tool's alone: a person + re-pins a leg with `bump-leg.py`, and the shape with `update-shape.py`. + """ + return rel != SHAPE_PIN and (rel in pinned_copies(ctx) + or rel in neutral_pins(ctx)) + + +#: Why `pin_restore` puts nothing back for a file `edited_by_hand`, by what +#: its note calls the file. No `;`, apostrophe or backtick (#277). +BY_HAND_WHY = { + "copy": ("A copy can hold an edit of the project, such as the merge by " + "hand update-shape.py asks for where a copy changed on both " + "sides"), + "pin": ("No tool of this standard writes a neutral product pin after the " + "scaffold, and every change to one since was made by a person"), +} + + +def tool_written(ctx: Context, rel: str) -> bool: + """Whether `rel`, a root path under `contracts/`, is a file a tool of + this standard writes, so that git holds the last good bytes (#291). + + WHAT THE ROOT DECLARES DECIDES, NOT THE NAME. A file is a tool's where + it is one of `declared_pins` or one of `pinned_copies`, the rows of the + root's own shape pin, which under `contracts/` is the naming policy + where the root keeps it. The shape pin is written by + `scaffold-project.py`, `adopt-project.py` and `update-shape.py apply`, a + leg pin by those and by `bump-leg.py`, and a neutral product pin by + `scaffold-project.py --pin`. A copy or a neutral product pin can hold a + person's edit, which is why `pin_restore` shows one and puts nothing + back (`edited_by_hand`). Anything else under `contracts/`, and + `project.yaml` or `family.yaml`, is the project's own. A + `contracts/release-pin.yaml` that no leg and no product names is a file + the project wrote, whatever it is called, and putting it back from git + would discard the edit, the merged hunk or the commit somebody made + (codex, review of PR #358). So is a `contracts/repository-naming.yaml` + the shape pin has no row for, though this standard copies a file there. + + WHERE `project.yaml` DOES NOT LOAD, IT DECLARES NOTHING. A leg pin or a + neutral product pin that does not load either is then the project's for + that run, and is left alone. That is the safe side. Reading the roles + from HEAD's `project.yaml` instead would decide what git puts back from + a declaration the tree may disagree with, as the other side of a merge + that adds or renames a leg does. A file wrongly called a tool's loses a + person's edit, and one wrongly called the project's costs a run: once + `project.yaml` loads, the next run names the pin's restore. The shape + pin and its copies hang on no manifest: the shape pin stays a tool's, + and the copies are what its rows say, whatever `project.yaml` holds. + """ + return rel in declared_pins(ctx) or rel in pinned_copies(ctx) + + +#: The note on the `manifest kinds` row's next command where no file that +#: does not load has a copy git can put back (#291). No `;`, apostrophe or +#: backtick, so a pasted line is one command in any shell (#277). +UNREADABLE_BY_HAND = ( + "the reason names each file that does not load and why, and none is a " + "file this root declares a tool writes with a copy that reads in its " + "history, so what each should say -- a merge conflict resolved, a hand " + "edit put right -- is for a person to decide. This row reads them again " + "once they load") + + +def unreadable_next(ctx: Context, unreadable: list) -> tuple[dict, str]: + """`(restore, command)` for the `manifest kinds` row's files that do not + load (#291): each file's case, and the one command the row prints. + + A FILE A TOOL WROTE IS PUT BACK FROM GIT by `pin_restore`. That is the + helper the `shape currency` row uses for the shape pin, so both rows + print the same command for the same file. The shape pin goes first + where it has a command, then each other file in the row's order. A row + prints one command, and the next run names the next file. A copy or a + neutral product pin, which a person may have edited, gets the + read-only command `pin_restore` picks for it instead, a `git diff` + that shows what breaks it, and nothing is put back. + + A FILE OF THE PROJECT'S OWN IS NOT, whatever it is called, and its case + is `project`, with git not asked. `tool_written` reads what the root + declares, not the name. The git-state choice does not fit it: each of + the three commands discards somebody's edit -- the other side of their + merge, their uncommitted work, their last commit -- and nothing in this + standard says which bytes are right. Where no file has a command to + name, the command is this doctor again, to run once a person has mended + the file, and its note says that what the file should say is theirs to + decide. #205's `manifest` row answers a `project.yaml` that does not + load the same way: the gate that refuses it, and that note. + """ + restore: dict[str, str] = {} + commands: dict[str, str] = {} + for rel in unreadable: + if tool_written(ctx, rel): + restore[rel], found = pin_restore(ctx, pin=rel) + if found: + commands[rel] = found + else: + restore[rel] = "project" + command = commands.get(SHAPE_PIN) or next(iter(commands.values()), None) + if command is None: + command = (f"{PYTHON} {quote_arg(ctx.shape / 'shape-doctor.py')} " + f"--root {quote_arg(ctx.root)} # {UNREADABLE_BY_HAND}") + return restore, command + + def unreadable_pin_row(ctx: Context, exc: Refusal) -> Row: """The `shape currency` row for a root whose OWN pin does not read as one (#237): a FINDING, the refusal quoted whole, and the pin put back. diff --git a/tests/test_shape_doctor.py b/tests/test_shape_doctor.py index 7c38f621..b50fda87 100644 --- a/tests/test_shape_doctor.py +++ b/tests/test_shape_doctor.py @@ -27,6 +27,7 @@ from __future__ import annotations import ast +import hashlib import importlib.util import json import os @@ -4469,6 +4470,1013 @@ def test_the_doctor_writes_no_bytecode_into_the_standard(standard, project, assert left == [], left +# --- the `manifest kinds` row for a file that does not load (#291) ---------- +# +# The row lists each file under `contracts/` that does not load as a mapping, +# and its NEXT was the unregistered kind's sentence: register the kind in +# shape-doctor.py's MANIFEST_VALIDATORS, or remove the file from the root. +# For a conflicted `contracts/shape-pin.yaml` that told the reader to delete +# the pin, and it was not a command a shell could even split. A file a tool +# wrote is now put back from the root's own history by `pin_restore`, the +# helper the `shape currency` row uses (#237). A file of the project's own +# is left to a person. A copy the shape pin has a row for, and a neutral +# product pin, can hold a person's edit, so the NEXT shows it and puts +# nothing back (review of PR #358). Each test runs the printed NEXT, then +# asks again. + +SPEC_PIN = "contracts/spec-pin.yaml" +CODE_PIN = "contracts/code-pin.yaml" +#: The one shape copy a root keeps under `contracts/`. +NAMING_COPY = "contracts/repository-naming.yaml" +OWN_CONTRACT = "contracts/notes.yaml" +#: A file of the project's own that is NAMED like a pin. No leg has the role +#: `release`, `neutral_product_pins:` does not name it, and it is not a shape +#: copy, so it is the project's whatever it is called (codex, PR #358). +OWN_PIN_NAMED = "contracts/release-pin.yaml" +RELEASE = (b"schedule: weekly\nowner: release-team\nchannels:\n" + b" - stable\nnotes: first\n") +#: The pin `neutral_product_pins:` declares for `openGlass`, at the path +#: `scaffold-project.py --pin` writes it to. +NEUTRAL_PIN = "contracts/openglass-pin.yaml" +#: A slip that stops a file loading, appended to what it held. +SLIP = b"\t: : [\n" + +#: How each restore's note names the file it puts back, `{what}` being +#: `pin` or `copy`. +NOTE_SAYS = {"unmerged": "never a hand-merged {what}", + "uncommitted": "puts back the {what} as last committed", + "committed": "the {what} as it was before"} + + +def conflict_in(root: Path, rel: str, key: str, *also: str) -> None: + """Leave `rel` conflicted, `UU`, the way two branches leave it. + + Two branches from HEAD each give the `key:` line of `rel` their own + 40-digit value and append a line to each of `also`, and the second + merges the first. Bytes throughout, as `break_the_pin` writes them. + """ + base = git("rev-parse", "HEAD", cwd=root).stdout.strip() + for branch, digit in (("edit-a", "1"), ("edit-b", "2")): + git("checkout", "-q", "-b", branch, base, cwd=root) + path = root / rel + path.write_bytes(pin_with(rf"^{key}: .*$", f'{key}: "{digit * 40}"')( + path.read_text(encoding="utf-8"))) + for name in also: + other = root / name + other.write_bytes(other.read_bytes() + f"# {branch}\n".encode()) + git("add", "--", rel, *also, cwd=root) + commit_in(root, branch) + merged = subprocess.run(["git", "merge", "-q", "edit-a"], cwd=str(root), + capture_output=True, text=True, check=False, + env={**os.environ, **GIT_IDENTITY}) + assert merged.returncode != 0, merged.stdout + merged.stderr + status = git("status", "--porcelain=v1", cwd=root).stdout + for name in (rel, *also): + assert f"UU {name}" in status, status + + +def own_contract(root: Path) -> Path: + """A committed `contracts/notes.yaml` with no `kind:`, the project's own + data, which loads and which the row passes over in silence.""" + own = root / OWN_CONTRACT + own.write_bytes(b"colour: blue\n") + git("add", "--", OWN_CONTRACT, cwd=root) + commit_in(root, "a contract of the project's own") + return own + + +def kinds_and_currency(standard: Path, root: Path) -> tuple[dict, dict]: + rows = rows_of(doctor(standard, root, "--json")) + return rows["manifest-kinds"], rows["shape-currency"] + + +def assert_no_registration(row: dict) -> None: + """The issue's (a) and (b): the NEXT is a command and never the + unregistered kind's sentence, whose second half deletes the file.""" + assert row["next"], row + for words in ("register the kind", "remove the file", + "MANIFEST_VALIDATORS"): + assert words not in row["next"], (words, row["next"]) + assert_a_plain_note(row["next"]) + + +def shape_pin_conflicted(root: Path) -> list: + conflict_in(root, SHAPE_PIN, "commit") + return ["checkout", "--ours", "--", SHAPE_PIN] + + +def shape_pin_not_utf8_committed(root: Path) -> list: + pin = root / SHAPE_PIN + pin.write_bytes(pin.read_bytes() + b"\xff\xfe\n") + git("add", "--", SHAPE_PIN, cwd=root) + breaking = commit_in(root, "a byte that is not UTF-8") + commit_in(root, "an unrelated commit after it", "--allow-empty") + return ["checkout", f"{breaking}~1", "--", SHAPE_PIN] + + +def shape_pin_a_list(root: Path) -> list: + (root / SHAPE_PIN).write_bytes(b"- one\n- two\n") + return ["checkout", "HEAD", "--", SHAPE_PIN] + + +def leg_pin_conflicted(root: Path) -> list: + # `Makefile`, a copy the shape writes, and README.md, which holds the + # line `apply` writes, conflict beside it. Only the shape pin takes the + # copies with it and names README.md for a person: a leg pin is named + # alone, and its note says nothing of either. + conflict_in(root, SPEC_PIN, "commit", "Makefile", "README.md") + return ["checkout", "--ours", "--", SPEC_PIN] + + +def leg_pin_not_utf8(root: Path) -> list: + pin = root / SPEC_PIN + pin.write_bytes(pin.read_bytes() + b"\xff\n") + return ["checkout", "HEAD", "--", SPEC_PIN] + + +def leg_pin_not_utf8_committed(root: Path) -> list: + # The leg pin's own history, not the shape pin's: the walk lists the + # commits that touched THIS file, and asks each parent for THIS file. + pin = root / SPEC_PIN + pin.write_bytes(pin.read_bytes() + b"\xff\n") + git("add", "--", SPEC_PIN, cwd=root) + breaking = commit_in(root, "a byte that is not UTF-8 in the spec pin") + commit_in(root, "an unrelated commit after it", "--allow-empty") + return ["checkout", f"{breaking}~1", "--", SPEC_PIN] + + +def leg_pin_past_a_list_in_its_history(root: Path) -> list: + # Two commits in a row broke it, the first into a list, which loads and + # is not a mapping. The walk steps past that parent to the pin before + # both, because putting back the list would leave the row where it is. + pin = root / SPEC_PIN + good = pin.read_bytes() + pin.write_bytes(b"- a\n") + git("add", "--", SPEC_PIN, cwd=root) + first = commit_in(root, "the spec pin made a list") + pin.write_bytes(good + b"\xff\n") + git("add", "--", SPEC_PIN, cwd=root) + commit_in(root, "then a byte that is not UTF-8 in it") + commit_in(root, "an unrelated commit after it", "--allow-empty") + return ["checkout", f"{first}~1", "--", SPEC_PIN] + + +@pytest.mark.parametrize("breaks, rel, code, restore, what, compliant", [ + pytest.param(shape_pin_conflicted, SHAPE_PIN, "yaml-unparsable", + "unmerged", "pin", False, id="the-shape-pin-conflicted"), + pytest.param(shape_pin_not_utf8_committed, SHAPE_PIN, "yaml-unreadable", + "committed", "pin", True, + id="the-shape-pin-not-utf-8-committed"), + pytest.param(shape_pin_a_list, SHAPE_PIN, None, "uncommitted", "pin", + True, id="the-shape-pin-a-list"), + pytest.param(leg_pin_conflicted, SPEC_PIN, "yaml-unparsable", + "unmerged", "pin", False, id="a-leg-pin-conflicted"), + pytest.param(leg_pin_not_utf8, SPEC_PIN, "yaml-unreadable", + "uncommitted", "pin", True, id="a-leg-pin-not-utf-8"), + pytest.param(leg_pin_not_utf8_committed, SPEC_PIN, "yaml-unreadable", + "committed", "pin", True, + id="a-leg-pin-not-utf-8-committed"), + pytest.param(leg_pin_past_a_list_in_its_history, SPEC_PIN, + "yaml-unreadable", "committed", "pin", True, + id="a-leg-pin-past-a-list-in-its-history"), +]) +def test_a_file_a_tool_wrote_that_does_not_load_is_put_back_from_git( + standard, project, breaks, rel, code, restore, what, compliant): + """The row names the git command for the file's own state, the loader's + words for it, and never the unregistered kind's sentence. + + Run as printed, the command puts back the bytes a tool wrote, and the + row reads every file again. Where the break was the only fault the + doctor is COMPLIANT. A merge is left for its owner to finish. + """ + expected = breaks(project) + result = doctor(standard, project, "--json") + assert result.returncode == 1, result.stdout + result.stderr + rows = rows_of(result) + row = rows["manifest-kinds"] + assert row["status"] == "FINDING", row + assert row["detail"]["unreadable"] == [rel], row["detail"] + assert_no_registration(row) + assert row["detail"]["restore"] == {rel: restore}, row["detail"] + said = row["detail"]["refusals"][rel] + assert said["code"] == code, said + if code is None: + # What a file that loads as something else is said to hold. + assert said["detail"] == "it loads as a list, not a mapping", said + # The loader's own words, less the path it opens with. + words = f"{code}: {said['detail']}" if code else said["detail"] + assert f"unreadable as YAML: {rel} ({words})" in row["reason"], \ + row["reason"] + assert Path(rel).name not in said["detail"], said + argv = pasted(row["next"]) + assert argv[:2] == ["git", "-C"], row["next"] + assert argv[3:] == expected, argv + note = row["next"].split(" #", 1)[1] + assert NOTE_SAYS[restore].format(what=what) in note, note + assert "README.md" not in note and "a person merges" not in note, note + if rel == SHAPE_PIN: + # One helper, so the two rows about the one file name one command. + assert row["next"] == rows["shape-currency"]["next"], rows + run_pasted(row["next"]) + again = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert again["status"] == "ok", again + assert again["detail"]["unreadable"] == [], again + if compliant: + compliant_again(standard, project) + + +def project_contract_conflicted(root: Path) -> tuple[str, str]: + own_contract(root) + conflict_in(root, OWN_CONTRACT, "colour") + return OWN_CONTRACT, "project" + + +def leg_pin_in_a_squashed_root_commit(root: Path) -> tuple[str, str]: + leg_pin_not_utf8(root) + git("checkout", "-q", "--orphan", "squashed", cwd=root) + git("add", "--", SPEC_PIN, cwd=root) + commit_in(root, "the whole history, squashed into one commit") + return SPEC_PIN, "none" + + +def leg_pin_added_back_after_its_deletion(root: Path) -> tuple[str, str]: + # The walk stops at the commit that added the file back: its parent + # holds none, so the pin from before the deletion is not this file's + # history to put back, though the parent holds a shape pin. + pin = root / SPEC_PIN + good = pin.read_bytes() + git("rm", "-q", "--", SPEC_PIN, cwd=root) + commit_in(root, "the spec pin deleted") + pin.write_bytes(good + b"\xff\n") + git("add", "--", SPEC_PIN, cwd=root) + commit_in(root, "the spec pin added back, with a byte that is not UTF-8") + return SPEC_PIN, "none" + + +def own_pin_named(root: Path) -> Path: + """`contracts/release-pin.yaml`, committed: the project's own file, + named like a pin, which nothing in the root declares.""" + path = root / OWN_PIN_NAMED + path.write_bytes(RELEASE) + git("add", "--", OWN_PIN_NAMED, cwd=root) + commit_in(root, "the project's own release train") + return path + + +def own_pin_named_edited_with_a_slip(root: Path) -> tuple[str, str]: + # A person's edit, not yet committed, with a slip in it. `checkout + # HEAD` would drop `schedule: daily` along with the slip. + own_pin_named(root).write_bytes(RELEASE.replace(b"weekly", b"daily") + + SLIP) + return OWN_PIN_NAMED, "project" + + +def own_pin_named_conflicted(root: Path) -> tuple[str, str]: + # Both sides change `schedule:`, and the other side also adds a channel + # that git merges cleanly. `checkout --ours` would take this side whole + # and drop `- nightly`. + own_pin_named(root) + base = git("rev-parse", "HEAD", cwd=root).stdout.strip() + for branch, body in ( + ("edit-a", RELEASE.replace(b"weekly", b"daily").replace( + b" - stable\n", b" - stable\n - nightly\n")), + ("edit-b", RELEASE.replace(b"weekly", b"hourly"))): + git("checkout", "-q", "-b", branch, base, cwd=root) + (root / OWN_PIN_NAMED).write_bytes(body) + git("add", "--", OWN_PIN_NAMED, cwd=root) + commit_in(root, branch) + merged = subprocess.run(["git", "merge", "-q", "edit-a"], cwd=str(root), + capture_output=True, text=True, check=False, + env={**os.environ, **GIT_IDENTITY}) + assert merged.returncode != 0, merged.stdout + merged.stderr + status = git("status", "--porcelain=v1", cwd=root).stdout + assert f"UU {OWN_PIN_NAMED}" in status, status + assert b" - nightly\n" in (root / OWN_PIN_NAMED).read_bytes() + return OWN_PIN_NAMED, "project" + + +def own_pin_named_broken_by_a_commit(root: Path) -> tuple[str, str]: + # A commit that hands the train to another team, with a slip in it. + # `checkout ~1` would drop `owner: platform-team` with the slip. + path = own_pin_named(root) + path.write_bytes(RELEASE.replace(b"release-team", b"platform-team") + + SLIP) + git("add", "--", OWN_PIN_NAMED, cwd=root) + commit_in(root, "the release train moves to the platform team") + commit_in(root, "an unrelated commit after it", "--allow-empty") + return OWN_PIN_NAMED, "project" + + +@pytest.mark.parametrize("breaks", [ + pytest.param(project_contract_conflicted, + id="a-contract-of-the-projects-own-conflicted"), + pytest.param(own_pin_named_edited_with_a_slip, + id="a-file-of-the-projects-own-named-like-a-pin-edited"), + pytest.param(own_pin_named_conflicted, + id="a-file-of-the-projects-own-named-like-a-pin-conflicted"), + pytest.param(own_pin_named_broken_by_a_commit, + id="a-file-of-the-projects-own-named-like-a-pin-committed"), + pytest.param(leg_pin_in_a_squashed_root_commit, + id="a-leg-pin-with-no-good-copy-in-its-history"), + pytest.param(leg_pin_added_back_after_its_deletion, + id="a-leg-pin-added-back-after-its-deletion"), +]) +def test_a_file_git_cannot_put_back_is_left_to_a_person(standard, project, + breaks): + """No git command to name: the NEXT is this doctor again, and its note + says what the file should say is for a person to decide. + + A file of the project's own is never put back from git, because each + of the three commands throws away an edit somebody made, so git is not + asked (`project`). That holds for one named like a pin, in each of the + three states: what the root declares decides, not the name. A pin with + no good copy in its history has nothing to put back (`none`). Run as + printed, the doctor finds the same file and touches none of it. Once a + person mends the file, the row reads it. + """ + rel, restore = breaks(project) + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["status"] == "FINDING", row + assert row["detail"]["unreadable"] == [rel], row["detail"] + assert_no_registration(row) + assert row["detail"]["restore"] == {rel: restore}, row["detail"] + assert "checkout" not in row["next"], row["next"] + argv = pasted(row["next"]) + assert argv[0] == repo_shape.PYTHON, argv + assert Path(argv[1]).name == DOCTOR and argv[2] == "--root", argv + assert Path(argv[3]).resolve() == project.resolve(), argv + assert "for a person to decide" in row["next"].split(" #", 1)[1] + before = (project / rel).read_bytes() + proc = subprocess.run(argv, capture_output=True, text=True, check=False, + stdin=subprocess.DEVNULL, + env={**os.environ, **GIT_IDENTITY}) + assert proc.returncode == 1, proc.stdout + proc.stderr + assert f"unreadable as YAML: {rel} (" in proc.stdout, proc.stdout + assert (project / rel).read_bytes() == before + if restore == "project": + (project / rel).write_bytes(b"colour: green\n") + again = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert again["status"] == "ok", again + + +def test_an_unregistered_kind_beside_a_file_that_does_not_load(standard, + project): + """Both in the reason, and the NEXT is the restore: the row cannot say + what kind a file declares until it loads. Run, the row is the one the + unregistered kind alone got, sentence, keys and all.""" + (project / "contracts" / "inventory.yaml").write_text( + "schema_version: 1\nkind: service-inventory\nservices: []\n", + encoding="utf-8") + alone = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert alone["next"].startswith( + "register the kind in shape-doctor.py's MANIFEST_VALIDATORS"), alone + assert set(alone["detail"]) == {"validated", "unregistered", "unreadable", + "registered_kinds"}, alone["detail"] + break_the_pin(project, conflicted_pin) + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["unreadable"] == [SHAPE_PIN], row["detail"] + assert_no_registration(row) + assert row["reason"].startswith( + "no validator registered for kind contracts/inventory.yaml " + "(service-inventory); unreadable as YAML: contracts/shape-pin.yaml " + "(yaml-unparsable: line "), row["reason"] + assert row["detail"]["unregistered"] == alone["detail"]["unregistered"] + assert row["detail"]["restore"] == {SHAPE_PIN: "uncommitted"} + assert pasted(row["next"])[3:] == ["checkout", "HEAD", "--", SHAPE_PIN] + run_pasted(row["next"]) + after = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert after == alone, (after, alone) + + +def test_the_shape_pin_goes_first_and_each_run_names_the_next_file( + standard, project): + """Three files that do not load: two pins and one of the project's own. + + The row prints one command. The shape pin's comes first, the command + the `shape currency` row prints too, then the other pin's, then this + doctor for the file a person mends. The project's file is never touched. + """ + own = own_contract(project) + for rel in (CODE_PIN, OWN_CONTRACT, SHAPE_PIN): + path = project / rel + path.write_bytes(path.read_bytes() + b"\xff\n") + kinds, currency = kinds_and_currency(standard, project) + assert kinds["detail"]["unreadable"] == [CODE_PIN, OWN_CONTRACT, + SHAPE_PIN], kinds["detail"] + assert_no_registration(kinds) + assert kinds["detail"]["restore"] == { + CODE_PIN: "uncommitted", OWN_CONTRACT: "project", + SHAPE_PIN: "uncommitted"}, kinds["detail"] + assert kinds["next"] == currency["next"], (kinds, currency) + assert pasted(kinds["next"])[3:] == ["checkout", "HEAD", "--", SHAPE_PIN] + run_pasted(kinds["next"]) + kinds, _ = kinds_and_currency(standard, project) + assert kinds["detail"]["unreadable"] == [CODE_PIN, OWN_CONTRACT], kinds + assert pasted(kinds["next"])[3:] == ["checkout", "HEAD", "--", CODE_PIN] + assert_no_registration(kinds) + run_pasted(kinds["next"]) + kinds, _ = kinds_and_currency(standard, project) + assert kinds["detail"]["unreadable"] == [OWN_CONTRACT], kinds + assert kinds["detail"]["restore"] == {OWN_CONTRACT: "project"}, kinds + assert Path(pasted(kinds["next"])[1]).name == DOCTOR, kinds["next"] + assert_no_registration(kinds) + assert own.read_bytes() == b"colour: blue\n\xff\n" + + +def declare_a_neutral_pin(root: Path) -> None: + """`neutral_product_pins:` naming `openGlass` in `project.yaml`, and the + pin it declares committed beside it, as `scaffold-project.py --pin` + leaves the two.""" + manifest = root / "project.yaml" + text = manifest.read_bytes() + assert text.count(b"neutral_product_pins: []\n") == 1, text + manifest.write_bytes(text.replace(b"neutral_product_pins: []\n", + b"neutral_product_pins:\n" + b" - openGlass\n")) + (root / NEUTRAL_PIN).write_bytes( + b"pin_role: neutral-product\nproduct: openGlass\n" + b'commit: "' + b"1" * 40 + b'"\n') + git("add", "--", "project.yaml", NEUTRAL_PIN, cwd=root) + commit_in(root, "a pin on the neutral product openGlass") + + +def test_a_pin_is_what_the_root_declares_not_what_it_is_called(standard, + project): + """A neutral product pin `project.yaml` declares is asked of git, and a + file of the project's own named like a pin is not. + + The declared pin's case is its git state, and its NEXT the `diff HEAD` + that shows the change: no tool writes a neutral product pin after the + scaffold, so the change is a person's and nothing is put back (review + of PR #358). The file named like a pin is `project`, git not asked. + Run as printed, neither file's bytes move. + + IN PROCESS, through the row alone: a declared neutral pin makes the + `pins` row recheck its digest, and with no checkout of the product + beside the root that asks `gh`, which no test here may reach. + """ + declare_a_neutral_pin(project) + own = own_pin_named(project) + for rel in (NEUTRAL_PIN, OWN_PIN_NAMED): + path = project / rel + path.write_bytes(path.read_bytes() + b"\xff\n") + neutral = (project / NEUTRAL_PIN).read_bytes() + module = doctor_module(standard) + row = module.check_manifest_kinds(module.Context(project)).as_dict() + assert row["detail"]["unreadable"] == [NEUTRAL_PIN, OWN_PIN_NAMED], row + assert row["detail"]["restore"] == {NEUTRAL_PIN: "uncommitted", + OWN_PIN_NAMED: "project"}, row + assert_no_registration(row) + assert pasted(row["next"])[3:] == ["diff", "HEAD", "--", + NEUTRAL_PIN], row["next"] + # As bytes: the diff shows the byte that is not UTF-8. + run_read_only(standard, project, row) + again = module.check_manifest_kinds(module.Context(project)).as_dict() + assert again == row, (again, row) + assert (project / NEUTRAL_PIN).read_bytes() == neutral + assert own.read_bytes() == RELEASE + b"\xff\n" + + +def test_a_leg_pin_is_left_to_a_person_while_project_yaml_does_not_load( + standard, project): + """`project.yaml` declares the leg pins, so where it does not load it + declares none, and a leg pin that does not load either is left alone + for that run. The shape pin and the copies are declared by no manifest, + and stay a tool's. Once `project.yaml` loads, the row puts the leg pin + back. + + In process, because at this branch a root whose `project.yaml` does not + load is not a shape root, and the doctor asks no row of it. + """ + for rel in ("project.yaml", SPEC_PIN): + path = project / rel + path.write_bytes(path.read_bytes() + b"\xff\n") + module = doctor_module(standard) + ctx = module.Context(project) + assert ctx.manifest is None + row = module.check_manifest_kinds(ctx).as_dict() + assert row["detail"]["restore"][SPEC_PIN] == "project", row["detail"] + assert Path(pasted(row["next"])[1]).name == DOCTOR, row["next"] + assert module.tool_written(ctx, SHAPE_PIN) + assert module.tool_written(ctx, NAMING_COPY) + git("checkout", "-q", "HEAD", "--", "project.yaml", cwd=project) + row = module.check_manifest_kinds(module.Context(project)).as_dict() + assert row["detail"]["restore"] == {SPEC_PIN: "uncommitted"}, row + assert pasted(row["next"])[3:] == ["checkout", "HEAD", "--", SPEC_PIN] + + +def test_an_uncommitted_change_to_a_copy_is_shown_and_kept(standard, + project): + """A copy can hold the project's own edit -- the merge by hand that + `update-shape.py` asks for where a copy changed on both sides -- and + git keeps no other copy of it until it is committed. So where HEAD's + copy reads and the change here breaks it, the NEXT shows the change and + puts nothing back. Run as printed, the bytes are still the person's. + """ + copy = project / NAMING_COPY + copy.write_bytes(copy.read_bytes() + SLIP) + before = copy.read_bytes() + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["restore"] == {NAMING_COPY: "uncommitted"}, row + assert_no_registration(row) + assert pasted(row["next"])[3:] == ["diff", "HEAD", "--", NAMING_COPY], \ + row["next"] + assert "checkout" not in row["next"], row["next"] + assert "nothing is put back" in row["next"].split(" #", 1)[1] + run_pasted(row["next"]) + assert copy.read_bytes() == before + again = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert again["detail"]["restore"] == {NAMING_COPY: "uncommitted"}, again + + +def test_a_family_yaml_beside_a_file_that_does_not_load_names_the_restore_alone( + standard, project, holder): + """#211 and #291 in one row: the reason says both, and the NEXT is the + restore ALONE, because joined to #211's sentence by `;` it would be a + second command in a pasted line (#277). Run, the row is #211's.""" + family_beside(project, holder) + break_the_pin(project, conflicted_pin) + kinds, currency = kinds_and_currency(standard, project) + assert FAMILY_BESIDE_OPENING in kinds["reason"], kinds["reason"] + assert "unreadable as YAML: contracts/shape-pin.yaml (" in \ + kinds["reason"], kinds["reason"] + assert_no_registration(kinds) + assert kinds["next"] == currency["next"], (kinds, currency) + assert "family.py" not in kinds["next"], kinds["next"] + run_pasted(kinds["next"]) + beside_row(doctor(standard, project, "--json")) + + +def test_a_contracts_family_yaml_beside_a_file_that_does_not_load_names_the_restore_alone( + standard, project, holder): + """#327 and #291 in one row: a `contracts/family.yaml` no gate in an + assembly root reads, beside a conflicted pin. The reason and `detail` + say both, and the NEXT is the restore ALONE, because #327's `take ... + out of this root` joined to it by `;` would be a second command in a + pasted line (#277). Run, the row is #327's, and its own NEXT, run in + turn, leaves the root COMPLIANT.""" + elsewhere = ("no validator that runs in an assembly root is registered " + "for kind contracts/family.yaml (family-manifest: " + "scripts/validate-family.py runs only in a family holder)") + shutil.copy2(holder["root"] / "family.yaml", + project / "contracts" / "family.yaml") + commit_all(project, "A family manifest under contracts/") + break_the_pin(project, conflicted_pin) + kinds, currency = kinds_and_currency(standard, project) + assert elsewhere in kinds["reason"], kinds["reason"] + assert "unreadable as YAML: contracts/shape-pin.yaml (" in \ + kinds["reason"], kinds["reason"] + assert set(kinds["detail"]) == MANIFEST_KINDS_DETAIL_KEYS | { + "registered_elsewhere", "restore", "refusals"}, kinds["detail"] + assert_no_registration(kinds) + assert kinds["next"] == currency["next"], (kinds, currency) + assert " rm -- " not in kinds["next"], kinds["next"] + run_pasted(kinds["next"]) + after = doctor(standard, project, "--json") + row = only_finding_is_manifest_kinds(after) + assert row["reason"] == elsewhere, row["reason"] + assert set(row["detail"]) == MANIFEST_KINDS_DETAIL_KEYS | { + "registered_elsewhere"}, row["detail"] + run_the_removal(removal_named(standard, row, json.loads(after.stdout), + ["contracts/family.yaml"]), project) + final = doctor(standard, project, "--json") + assert final.returncode == 0, final.stdout + final.stderr + assert json.loads(final.stdout)["verdict"] == "COMPLIANT" + + +# --- a copy or a neutral pin is a person's to mend (review of PR #358) ------ +# +# Codex on 2745890, and rv358's probes P10 to P13. A copy can hold the +# project's own edit, above all the merge by hand `update-shape.py` asks for +# where a copy changed on both sides, so its `unmerged` and `committed` +# states get what its `uncommitted` state got: a read-only `git diff` and a +# person's merge, never one side taken whole and never a rollback (codex +# r4250578605). A copy is what the shape pin's `files:` rows name, not what +# this standard's copy lists name (codex r4250578619). And no tool writes a +# neutral product pin after the scaffold, so a change to one is a person's. + +#: Lines a person writes into the naming copy. Comments, so that any copy +#: of the policy stays YAML, and the end of the file is where both sides of +#: a merge write, so git cannot merge the two tails. +PROJECT_LINE = b"# Atlas_docs: a line the project keeps in its copy\n" +PROJECT_TAIL = b"# a closing line the project wrote\n" +SHAPE_TAIL = b"# a closing line the shape wrote\n" +#: A conflict hunk, as git leaves one in the working file. +CONFLICT_HUNK = re.compile( + rb"(?ms)^<<<<<<< [^\n]*\n(.*?)^=======\n(.*?)^>>>>>>> [^\n]*\n") + + +def insert_after_first_line(path: Path, line: bytes) -> None: + first, rest = path.read_bytes().split(b"\n", 1) + path.write_bytes(first + b"\n" + line + rest) + + +def append_to(path: Path, line: bytes) -> None: + path.write_bytes(path.read_bytes() + line) + + +def repin(root: Path, rel: str) -> None: + """The shape pin's `sha256:` for `rel` set to the file's bytes, as + `update-shape.py apply` sets it once it has re-copied the file.""" + pin = root / SHAPE_PIN + digest = hashlib.sha256((root / rel).read_bytes()).hexdigest() + row = (rb'(?m)^( - path: ' + re.escape(rel.encode()) + + rb'\n sha256: )"[0-9a-f]{64}"$') + text, count = re.subn(row, rb'\1"' + digest.encode() + rb'"', + pin.read_bytes()) + assert count == 1, rel + pin.write_bytes(text) + + +def merge_two_edits(root: Path, edit_a, edit_b, *paths: str) -> str: + """`edit-a` and `edit-b` from HEAD, each making its own edit of `paths`, + and `edit-b` merging `edit-a`, so `--ours` is edit-b's side and git + merges edit-a's hunks where it can. The status the merge leaves.""" + base = git("rev-parse", "HEAD", cwd=root).stdout.strip() + for branch, edit in (("edit-a", edit_a), ("edit-b", edit_b)): + git("checkout", "-q", "-b", branch, base, cwd=root) + edit(root) + git("add", "--", *paths, cwd=root) + commit_in(root, branch) + merged = subprocess.run(["git", "merge", "-q", "edit-a"], cwd=str(root), + capture_output=True, text=True, check=False, + env={**os.environ, **GIT_IDENTITY}) + assert merged.returncode != 0, merged.stdout + merged.stderr + return git("status", "--porcelain=v1", cwd=root).stdout + + +def a_copy_conflicted_on_its_own(root: Path) -> None: + """P10: edit-a is the project's own edit of the naming copy, a line + near the top and a line at the end; edit-b is an `apply`-like re-copy + that writes the end its own way and re-pins the copy. Git merges the + project's line and conflicts on the end. Only the copy is `UU`: the + shape pin changed on one side alone.""" + def project_edit(r: Path) -> None: + insert_after_first_line(r / NAMING_COPY, PROJECT_LINE) + append_to(r / NAMING_COPY, PROJECT_TAIL) + + def apply_like(r: Path) -> None: + append_to(r / NAMING_COPY, SHAPE_TAIL) + repin(r, NAMING_COPY) + + status = merge_two_edits(root, project_edit, apply_like, NAMING_COPY, + SHAPE_PIN) + assert f"UU {NAMING_COPY}" in status and SHAPE_PIN not in status, status + held = (root / NAMING_COPY).read_bytes() + for line in (PROJECT_LINE, PROJECT_TAIL, SHAPE_TAIL): + assert line in held, line + + +def merge_by_hand(path: Path) -> None: + """A person's merge: both sides of every conflict hunk kept, and the + hunks git merged left as they are.""" + held = path.read_bytes() + merged, count = CONFLICT_HUNK.subn(rb"\1\2", held) + assert count == 1 and b"<<<<<<<" not in merged, held[-400:] + path.write_bytes(merged) + + +def run_read_only(standard: Path, root: Path, row: dict) -> bytes: + """Run the printed NEXT, which must be `git -C diff ...`, and + require that it changed nothing: the file's bytes, the status, HEAD. + Its output.""" + argv = pasted(row["next"]) + assert argv[:2] == ["git", "-C"] and argv[3] == "diff", row["next"] + assert "checkout" not in row["next"], row["next"] + rel = argv[-1] + before = ((root / rel).read_bytes(), + git("status", "--porcelain=v1", cwd=root).stdout, + git("rev-parse", "HEAD", cwd=root).stdout) + proc = subprocess.run(argv, capture_output=True, check=False, + stdin=subprocess.DEVNULL, + env={**os.environ, **GIT_IDENTITY}) + assert proc.returncode == 0, proc.stdout + proc.stderr + assert ((root / rel).read_bytes(), + git("status", "--porcelain=v1", cwd=root).stdout, + git("rev-parse", "HEAD", cwd=root).stdout) == before + return proc.stdout + + +def test_a_copy_conflicted_on_its_own_is_shown_and_merged_by_a_person( + standard, project): + """P10 (codex r4250578605). The NEXT was `checkout --ours`, the + apply-like side whole: run, and the merge finished as its note said, + the root was COMPLIANT and HEAD's copy had lost the project's line and + its tail, which git had merged or held for a person. + + Now the NEXT is the `diff` that shows the hunk git could not merge, + its note says a person merges it, and it changes nothing. Merged by + hand, both sides kept, the row reads the copy again. + """ + a_copy_conflicted_on_its_own(project) + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["unreadable"] == [NAMING_COPY], row["detail"] + assert row["detail"]["restore"] == {NAMING_COPY: "unmerged"}, row + assert_no_registration(row) + assert pasted(row["next"])[3:] == ["diff", "--", NAMING_COPY], \ + row["next"] + note = row["next"].split(" #", 1)[1] + assert "neither side is taken whole" in note, note + assert "merge it by hand, keeping the hunks git merged" in note, note + shown = run_read_only(standard, project, row) + assert SHAPE_TAIL.strip() in shown and PROJECT_TAIL.strip() in shown, \ + shown + merge_by_hand(project / NAMING_COPY) + git("add", "--", NAMING_COPY, cwd=project) + commit_in(project, "the naming copy merged by hand") + again = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert again["status"] == "ok", again + kept = git("show", f"HEAD:{NAMING_COPY}", cwd=project).stdout + for line in (PROJECT_LINE, PROJECT_TAIL, SHAPE_TAIL): + assert line.decode() in kept, line + + +def test_a_conflicted_copys_next_agrees_with_shape_currency_and_agents_md( + standard, project): + """P10b. In the same report the `shape currency` row names `apply + --accept-local` on the copy, which keeps the local bytes, while the + `manifest kinds` row took one side whole. AGENTS.md says `both` is a + merge: by hand, commit, then `--accept-local` on the file merged. + + Now the `manifest kinds` row asks for that merge, and the two rows + are one path: the merge by hand, then the `shape currency` row's own + NEXT, run as its note spells it, ends COMPLIANT with every line of + both sides in the copy and its pin. + """ + a_copy_conflicted_on_its_own(project) + kinds, currency = kinds_and_currency(standard, project) + assert "merge it by hand" in kinds["next"].split(" #", 1)[1], kinds + assert "checkout" not in kinds["next"], kinds["next"] + assert f"--accept-local {NAMING_COPY}" in currency["next"], currency + merge_by_hand(project / NAMING_COPY) + git("add", "--", NAMING_COPY, cwd=project) + commit_in(project, "the naming copy merged by hand") + kinds, currency = kinds_and_currency(standard, project) + assert kinds["status"] == "ok", kinds + argv = pasted(currency["next"]) + assert Path(argv[1]).name == "update-shape.py" and argv[2] == "check", \ + currency["next"] + then = currency["next"].split(" #", 1)[1].split() + assert then[:2] == ["then", "apply"], currency["next"] + assert then[then.index("--accept-local") + 1] == NAMING_COPY, then + applied = run_script(standard / "update-shape.py", "apply", "--root", + str(project), "--upstream", str(standard), *then[2:]) + assert applied.returncode == 0, applied.stdout + applied.stderr + compliant_again(standard, project) + kept = git("show", f"HEAD:{NAMING_COPY}", cwd=project).stdout + for line in (PROJECT_LINE, PROJECT_TAIL, SHAPE_TAIL): + assert line.decode() in kept, line + + +def drop_the_copys_row(root: Path) -> None: + """The naming copy's `files:` row taken out of the shape pin and + committed, as an adopted project that made the file its own does.""" + pin = root / SHAPE_PIN + row = (rb'(?m)^ - path: ' + re.escape(NAMING_COPY.encode()) + + rb'\n sha256: "[0-9a-f]{64}"\n') + text, count = re.subn(row, b"", pin.read_bytes()) + assert count == 1 + pin.write_bytes(text) + git("add", "--", SHAPE_PIN, cwd=root) + commit_in(root, "the naming policy is the project's own now") + + +def test_a_file_the_shape_pin_has_no_row_for_is_the_projects(standard, + project): + """P11a (codex r4250578619). The pin has no row for + `contracts/repository-naming.yaml`, so the file there is the + project's, and the root is COMPLIANT. A commit of the project's own + breaks it: the NEXT was `checkout ~1`, the copy this + standard's lists name, and it dropped the project's line. + + Now the copy is what the rows say, so the file is `project`, git is not + asked, and the NEXT is this doctor again, which touches nothing. + """ + drop_the_copys_row(project) + compliant_again(standard, project) + own = project / NAMING_COPY + insert_after_first_line(own, PROJECT_LINE) + append_to(own, SLIP) + git("add", "--", NAMING_COPY, cwd=project) + commit_in(project, "the project's own naming examples, with a slip") + commit_in(project, "an unrelated commit after it", "--allow-empty") + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["restore"] == {NAMING_COPY: "project"}, row + assert "checkout" not in row["next"], row["next"] + argv = pasted(row["next"]) + assert Path(argv[1]).name == DOCTOR, row["next"] + before = own.read_bytes() + proc = subprocess.run(argv, capture_output=True, text=True, check=False, + stdin=subprocess.DEVNULL, + env={**os.environ, **GIT_IDENTITY}) + assert proc.returncode == 1, proc.stdout + proc.stderr + assert own.read_bytes() == before and PROJECT_LINE in before + module = doctor_module(standard) + ctx = module.Context(project) + assert NAMING_COPY not in module.pinned_copies(ctx) + assert "Makefile" in module.pinned_copies(ctx) + assert not module.tool_written(ctx, NAMING_COPY) + + +def collision_layout(root: Path) -> None: + """An in-place adoption that collided on the naming policy: the shape's + copy under `shape/`, the pin's row naming it there, as + `shape_materialize` writes the two, and the source's own file at + `contracts/repository-naming.yaml`, which no row names.""" + copy = (root / NAMING_COPY).read_bytes() + (root / "shape" / "contracts").mkdir(parents=True) + (root / "shape" / NAMING_COPY).write_bytes(copy) + pin = root / SHAPE_PIN + text = pin.read_bytes() + row = f" - path: {NAMING_COPY}\n".encode() + assert text.count(row) == 1, text + pin.write_bytes(text.replace(row, f" - path: shape/{NAMING_COPY}\n" + .encode())) + git("add", "--", SHAPE_PIN, f"shape/{NAMING_COPY}", cwd=root) + commit_in(root, "the shape's copy under shape/, the source's own file " + "at contracts/") + + +def test_the_sources_own_file_at_a_collided_copy_path_is_the_projects( + standard, project): + """P11b (codex r4250578619). In the collision layout the source's own + `contracts/repository-naming.yaml` is conflicted, the other side's line + merged cleanly. The NEXT was `checkout --ours`, which dropped that + line. Now the file is `project`, and the NEXT is this doctor again, + which leaves the merge as git left it.""" + collision_layout(project) + compliant_again(standard, project) + + def ours(r: Path) -> None: + insert_after_first_line(r / NAMING_COPY, PROJECT_LINE) + append_to(r / NAMING_COPY, PROJECT_TAIL) + + status = merge_two_edits( + project, ours, lambda r: append_to(r / NAMING_COPY, SHAPE_TAIL), + NAMING_COPY) + assert f"UU {NAMING_COPY}" in status, status + own = project / NAMING_COPY + before = own.read_bytes() + assert PROJECT_LINE in before + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["restore"] == {NAMING_COPY: "project"}, row + assert "checkout" not in row["next"], row["next"] + assert Path(pasted(row["next"])[1]).name == DOCTOR, row["next"] + run = subprocess.run(pasted(row["next"]), capture_output=True, + text=True, check=False, stdin=subprocess.DEVNULL, + env={**os.environ, **GIT_IDENTITY}) + assert run.returncode == 1, run.stdout + run.stderr + assert own.read_bytes() == before + assert f"UU {NAMING_COPY}" in git("status", "--porcelain=v1", + cwd=project).stdout + + +def test_a_conflicted_shape_pin_takes_only_the_copies_its_side_has_rows_for( + standard, project): + """Codex r4250578619 in the shape pin's own merge (#267): the pin and + the source's own `contracts/repository-naming.yaml` conflicted + together, in the collision layout. The NEXT took that file whole with + the pin, because this standard's copy lists name its path. + + Now the side taken is read for its rows, which name the copy under + `shape/` and not the source's file, so the pin is named alone, by both + rows, and the source's file keeps both sides for its owner. + """ + collision_layout(project) + + def side(digit: str, tail: bytes): + def edit(r: Path) -> None: + break_the_pin(r, pin_with(r'^commit: .*$', + f'commit: "{digit * 40}"')) + append_to(r / NAMING_COPY, tail) + return edit + + status = merge_two_edits(project, side("1", PROJECT_TAIL), + side("2", SHAPE_TAIL), SHAPE_PIN, NAMING_COPY) + assert f"UU {SHAPE_PIN}" in status and f"UU {NAMING_COPY}" in status, \ + status + own = project / NAMING_COPY + before = own.read_bytes() + kinds, currency = kinds_and_currency(standard, project) + assert currency["detail"]["restore"] == "unmerged", currency["detail"] + assert pasted(currency["next"])[3:] == [ + "checkout", "--ours", "--", SHAPE_PIN], currency["next"] + assert kinds["next"] == currency["next"], (kinds, currency) + assert kinds["detail"]["restore"] == {NAMING_COPY: "project", + SHAPE_PIN: "unmerged"}, kinds + run_pasted(currency["next"]) + assert own.read_bytes() == before + again = rows_of(doctor(standard, project, "--json"))["shape-currency"] + assert again["detail"].get("error") not in READ_PIN_CODES, again + + +def test_a_committed_hand_merge_of_a_copy_is_shown_not_rolled_back( + standard, project): + """P12 (codex r4250578605, the committed state). A hand merge of the + copy, as `update-shape.py`'s `both` exit asks for, committed with a + slip in it. The NEXT was `checkout ~1`, which dropped the + merge from the tree. + + Now the NEXT is the `diff` from the newest copy that reads, which shows + the merge and the slip and changes nothing. Mended and committed, the + row reads the copy again, with the merge in it. + """ + copy = project / NAMING_COPY + insert_after_first_line(copy, PROJECT_LINE) + append_to(copy, SLIP) + git("add", "--", NAMING_COPY, cwd=project) + breaking = commit_in(project, "the hand merge, with a slip") + commit_in(project, "an unrelated commit after it", "--allow-empty") + row = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert row["detail"]["restore"] == {NAMING_COPY: "committed"}, row + assert_no_registration(row) + assert pasted(row["next"])[3:] == ["diff", f"{breaking}~1", "--", + NAMING_COPY], row["next"] + note = row["next"].split(" #", 1)[1] + assert breaking[:12] in note and "nothing is rolled back" in note, note + shown = run_read_only(standard, project, row) + assert PROJECT_LINE.strip() in shown and SLIP.strip() in shown, shown + copy.write_bytes(copy.read_bytes()[:-len(SLIP)]) + git("add", "--", NAMING_COPY, cwd=project) + commit_in(project, "the slip mended") + again = rows_of(doctor(standard, project, "--json"))["manifest-kinds"] + assert again["status"] == "ok", again + assert PROJECT_LINE.decode() in git("show", f"HEAD:{NAMING_COPY}", + cwd=project).stdout + + +def neutral_pin_body(digit: str) -> bytes: + return (b"pin_role: neutral-product\nproduct: openGlass\n" + b'commit: "' + digit.encode() * 40 + b'"\n') + + +def neutral_pin_re_pinned_by_hand(root: Path) -> tuple[str, list]: + # P13: re-pinned by hand to another commit, with a slip, uncommitted. + # `checkout HEAD` discarded the re-pin. + (root / NEUTRAL_PIN).write_bytes(neutral_pin_body("2") + SLIP) + return "uncommitted", ["diff", "HEAD", "--", NEUTRAL_PIN] + + +def neutral_pin_re_pin_committed(root: Path) -> tuple[str, list]: + # The same re-pin committed. `checkout ~1` rolled it back. + (root / NEUTRAL_PIN).write_bytes(neutral_pin_body("2") + SLIP) + git("add", "--", NEUTRAL_PIN, cwd=root) + breaking = commit_in(root, "openGlass re-pinned by hand, with a slip") + commit_in(root, "an unrelated commit after it", "--allow-empty") + return "committed", ["diff", f"{breaking}~1", "--", NEUTRAL_PIN] + + +def neutral_pin_re_pins_conflicted(root: Path) -> tuple[str, list]: + # Two people re-pin it, each to their own commit. `checkout --ours` + # took one of them whole. + status = merge_two_edits( + root, + lambda r: (r / NEUTRAL_PIN).write_bytes(neutral_pin_body("2")), + lambda r: (r / NEUTRAL_PIN).write_bytes(neutral_pin_body("3")), + NEUTRAL_PIN) + assert f"UU {NEUTRAL_PIN}" in status, status + return "unmerged", ["diff", "--", NEUTRAL_PIN] + + +@pytest.mark.parametrize("breaks", [ + pytest.param(neutral_pin_re_pinned_by_hand, + id="re-pinned-by-hand-uncommitted"), + pytest.param(neutral_pin_re_pin_committed, id="a-hand-re-pin-committed"), + pytest.param(neutral_pin_re_pins_conflicted, + id="two-hand-re-pins-conflicted"), +]) +def test_a_neutral_product_pin_is_shown_never_put_back(standard, project, + breaks): + """P13. No tool writes a neutral product pin after the scaffold, and the + manifest template says to add one by hand, so every change to one is a + person's. In each git state the NEXT is a read-only `diff`, run as + printed it changes nothing, and the person's re-pin is still there. + + In process, for the reason `test_a_pin_is_what_the_root_declares_not_ + what_it_is_called` gives: a declared neutral pin sends the `pins` row + to `gh`. + """ + declare_a_neutral_pin(project) + restore, expected = breaks(project) + before = (project / NEUTRAL_PIN).read_bytes() + module = doctor_module(standard) + row = module.check_manifest_kinds(module.Context(project)).as_dict() + assert row["detail"]["unreadable"] == [NEUTRAL_PIN], row["detail"] + assert row["detail"]["restore"] == {NEUTRAL_PIN: restore}, row + assert_no_registration(row) + assert pasted(row["next"])[3:] == expected, row["next"] + assert "No tool of this standard writes a neutral product pin" in \ + row["next"].split(" #", 1)[1], row["next"] + # One `so` to a sentence (review of PR #358): the reason ended in its own + # ", so every change to one was made by a person", and the note's ", so + # neither side is taken whole" or ", so nothing is rolled back" followed. + assert not any(sentence.count(", so ") > 1 for sentence in + row["next"].split(" #", 1)[1].split(". ")), row["next"] + run_read_only(standard, project, row) + assert (project / NEUTRAL_PIN).read_bytes() == before + assert b"2" * 40 in before + + # --- placement: code in spec, spec in code (#99) ---------------------------- # # Brett Heap, 2026-09-10: "we have to look for code in spec and spec in code".