diff --git a/templates/family-root/scripts/bootstrap.py b/templates/family-root/scripts/bootstrap.py index aec82b91..418a898c 100755 --- a/templates/family-root/scripts/bootstrap.py +++ b/templates/family-root/scripts/bootstrap.py @@ -54,9 +54,11 @@ out; where every member was checked out the run fails all the same, since git updated none of them. A work tree file git cannot parse at all, a conflict marker or an open quote left in it, is said and - failed on the same way, and its cure is the file mended by hand. Git - exits 0 over the second where a recursive clone ran first, and 128 - (`No url found for submodule path`) in a plain clone; a missing + failed on the same way, and its cure is the file mended by hand; in + the index's or HEAD's file git reads the sections before such a line + and goes on, and so does this (#856). Git exits 0 over the second + where a recursive clone ran first, and 128 (`No url found for + submodule path`) in a plain clone; a missing credential exits non-zero too, so what the holder records is the test, never git's words. The line says what cures each (`cures`): `validate-family.py` names the commit for the first two; for the @@ -66,7 +68,13 @@ sections give, or a gitlink in members/ that no row names; `git rm --cached` takes out one outside members/, which the validator does not read; and where the gitlink is a row's own, with one section, - the line says to write its section back. + the line says to write its section back. Where git read the index's + or HEAD's file only up to a line it cannot parse, or up to a key it + takes no value for (a bare `url` line, which `git config --list` + takes), a section or url it did not read may be past that line, so + for those three the line says to put that file back, mend the line + or give the key a value or take it out, and commit it, and writes + no section back (#856). (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project and knows how to bootstrap itself — it puts its legs on their @@ -132,12 +140,21 @@ NO_URL = "git has no url for that path" EXITED_0 = "git exited 0 and checked nothing out there" STOPPED = "git stopped at {}, a gitlink it has no url for" +#: The whys a `.gitmodules` git read only up to a line it cannot parse, or +#: up to a key with no value it stops at, may be the cause of, since what +#: it did not read is past that line (`Gitmodules.cut`, #856). +CUT_WHYS = frozenset({NO_SECTION, NO_URL, STOPPED}) +#: HEAD's `.gitmodules`, as `git config --blob` is given it. +HEAD_GITMODULES = f"HEAD:{GITMODULES}" #: What git 2.43 dies on in a `.gitmodules` it reads (#768), as #: `parse_config` in its submodule-config.c reads a submodule's key: these #: keys with no value (a `branch` with none crashes it, `BRANCH_READ_SINCE`), #: and an `update` other than these four. A `shallow` must be a boolean and -#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). +#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). The +#: gate's `_values_refused`, `_git_takes_bool` and `GIT_TAKES` say the same, +#: and `tests/test_family_gitmodules_values.py` holds the two scripts to one +#: answer: a change made here is made there too (#856). NO_VALUE_REFUSED = frozenset({"path", "url", "update", "ignore", "branch"}) #: The first git that reads a `branch` with no value as it reads a `url` #: with none, `missing value` and `bad config line`, which dies in the work @@ -328,9 +345,21 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None, url no clone can check it out. - A row git exited 0 over: the update that checks it out whatever the clone chose. `--init` with the path registers a member a subset - clone left out, and `--checkout` overrides an `update = none`.""" + clone left out, and `--checkout` overrides an `update = none`. + - The index's or HEAD's `.gitmodules`, a blob, which git read only up + to a line it cannot parse, or up to a key with no value it stops at + (`Gitmodules.cut`, #856), for a row with no section or no url there, + or the gitlink git stopped at (`CUT_WHYS`): that file put back, + mended and committed, which `gitmodules_cures` says. A section or + url git did not read may be past the line, so no other cure is said + for those: a section written back where the work tree holds no file + would be the only one in it, and `validate-family.py` reads HEAD's + file alone, not the index's.""" whys = {why for _, _, why in left} said = gitmodules_cures(whys, modules) + if modules.cut: + left = [entry for entry in left if entry[2] not in CUT_WHYS] + whys -= CUT_WHYS if whys & {NO_GITLINK, NO_SECTION}: said.append(f"`{PYTHON} scripts/validate-family.py` names the cause " "and the commit that cures it.") @@ -374,7 +403,21 @@ def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]: """`cures`' sentences for a `.gitmodules` git reads nothing from, each in the order `cures` says them (#648, #768): a merge left unmerged, for a row `whys` names so; each value git dies on; a work tree file - git cannot parse.""" + git cannot parse. And for a blob git read only up to such a line, or + up to a key with no value it stops at, where a row `whys` names is + one of `CUT_WHYS` (#856): the command that puts that file back in the + work tree, `git checkout` from the index where git read the index's, + which keeps what was staged, and from HEAD where it read HEAD's, then + the file mended, or the key given a value git takes (`TAKES`) or its + line taken out, and committed. The index's is put back with + `--ignore-skip-worktree-bits`: in a sparse checkout that leaves + `.gitmodules` out, its entry is skip-worktree, and `git checkout -- + .gitmodules` matches no file there (RV877's NIT-3, measured, git + 2.43.0); with the flag it puts the file back there and in any other + work tree. HEAD's is put back only where the index holds no entry, + so no skip-worktree bit, and `git checkout HEAD` puts it back in a + sparse checkout too. `git config -f .gitmodules --list` + takes a key with no value, so it is pointed at for the line alone.""" said = [] if UNMERGED in whys: said.append(f"Resolve the merge in {GITMODULES} and `git add " @@ -392,6 +435,21 @@ def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]: said.append(f"Mend {GITMODULES} by hand (`git config -f " f"{GITMODULES} --list` says where it breaks), and " "commit it.") + if modules.cut and whys & CUT_WHYS: + head = modules.cut == HEAD_GITMODULES + put_back = shlex.join([ + "git", "checkout", + *(["HEAD"] if head else ["--ignore-skip-worktree-bits"]), + "--", GITMODULES]) + whose = f"{'HEAD' if head else 'the index'}'s {GITMODULES}" + key = modules.cut_key + mend = (f"`{spelled(key)}`, a key with no value: give it a value " + f"git takes, {TAKES[key.rpartition('.')[2]]}, or take that " + "line out" if key else + "a line it cannot parse: mend it by hand (`git config -f " + f"{GITMODULES} --list` says where it breaks)") + said.append(f"`{put_back}` puts back {whose}, which git read only " + f"up to {mend}, and commit it.") return said @@ -422,7 +480,16 @@ class Gitmodules(NamedTuple): it reads the file, None for a key with no value (#768): git reads nothing from it then, and the three are empty. `unreadable` says the work tree's file is one `git config --list` cannot parse, which git - dies reading: the three are empty then too.""" + dies reading: the three are empty then too. `cut` is the blob git read + only up to such a line, as `gitmodules_source` names it, + `HEAD:.gitmodules` or the index's object id, where it read an entry + before it, or the blob is the index's (`gitmodules_listed`, #856): + the three hold what it read, + and `cures` writes no section back. Empty where git read the whole + file, or nothing. `cut_key` is the `submodule..` with no + value that cut it, where that is what git stopped at + (`ends_blob_reading`), a line `git config --list` takes, and empty + where a line it cannot parse did.""" given: Counter read: dict[str, str] @@ -430,15 +497,22 @@ class Gitmodules(NamedTuple): unmerged: bool = False refused: tuple[tuple[str, str | None], ...] = () unreadable: bool = False + cut: str = "" + cut_key: str = "" def gitmodules_sections(root: Path) -> Gitmodules: """`.gitmodules` as `git submodule update` reads it: the work tree's file, which a fresh clone takes from HEAD, or, where the work tree has none, as a deletion not yet committed or a sparse checkout leaves it, - the index's, then HEAD's (`gitmodules_source`, #768). Nothing where - none holds one, or git cannot read it: git has no url for any - submodule then either. Nothing, and `unreadable`, where that is the + the index's, then HEAD's (`gitmodules_source`, #768), each of the + last two, a blob, up to a line git cannot parse, as git reads it, + and `cut` where it read an entry before that line, or the blob is + the index's (`gitmodules_listed`, #856), or up to a key with no value + git stops at, `cut` and `cut_key` (`gitmodules_entries`). Nothing + where none holds one, or git cannot read it: git has no url for any + submodule then either. + Nothing, and `unreadable`, where that is the work tree's file and `git config --list` cannot parse it: git dies reading it (`bad config line`), before it checks out any member. Nothing, and `unmerged`, where the index holds the file only at a @@ -455,19 +529,19 @@ def gitmodules_sections(root: Path) -> Gitmodules: source = gitmodules_source(root) if source is None: return nothing._replace(unmerged=True) - try: - listed = git_out(["config", "-z", *source, "--list"], cwd=root, - binary=True) - except Refusal: + listed = gitmodules_listed(root, source) + if listed is None: return nothing._replace(unreadable=os.path.isfile(root / GITMODULES)) - entries = gitmodules_entries(root, listed, source[0] == "--blob") + printed, cut = listed + entries, ended = gitmodules_entries(root, printed, source[0] == "--blob") refused = tuple(dict.fromkeys( (f"submodule.{name}.{key}", value) for name, key, value, ignored in entries if not ignored and refused_value(root, key, value))) if refused: return nothing._replace(refused=refused) - return read_sections(entries) + return read_sections(entries)._replace( + cut=source[1] if cut or ended else "", cut_key=ended) def gitmodules_source(root: Path) -> list[str] | None: @@ -480,7 +554,7 @@ def gitmodules_source(root: Path) -> list[str] | None: cwd=root, binary=True) except Refusal: stages = b"" - source = ["--blob", f"HEAD:{GITMODULES}"] + source = ["--blob", HEAD_GITMODULES] for record in stages.split(b"\0"): meta, _, name = record.partition(b"\t") fields = meta.split() @@ -494,8 +568,56 @@ def gitmodules_source(root: Path) -> list[str] | None: return source +def gitmodules_listed(root: Path, source: list[str] + ) -> tuple[bytes, bool] | None: + """What `git config -z --no-includes --list` prints of the + `.gitmodules` that `source` names (`gitmodules_source`), and whether + git stopped short of the file's end at a line it cannot parse, or + None where git cannot read it (#856). + + A BLOB IS READ UP TO A LINE GIT CANNOT PARSE. `git config --blob` + prints the entries before that line, then `bad config line`, and + exits 128. Git's submodule reader, which reads the index's or HEAD's + file where the work tree has none, takes those same entries, says + the same line as an error, not a die, and goes on (measured, git + 2.43.0): so what was printed is what git read, and a bad line before + the first section leaves nothing, as git reads nothing. A work tree + file with such a line is None: git dies reading it. Git is run as + `git_out` runs it. + + TOLD BY THE EXIT CODE, NEVER GIT'S WORDS, which a git built with its + translations prints in the person's language: a blob's listing that + exits 128 is read for what it printed, whatever it says on stderr. + The other 128 met is a blob git cannot find (`HEAD:.gitmodules` + where HEAD holds none, #611), which prints nothing, and empty + stdout reads as the same `Gitmodules` that None gives for a blob: + no entry, and nothing `unreadable`, since a blob is read only where + the work tree holds no file. So HEAD's, printing nothing, is no + `cut`: it may be no file at all. The index's is, printing nothing or + not: `git ls-files -s` gave its object id, so the blob is there, and + a 128 can only be a line git cannot parse, the first included + (RV877's SHOULD-2). + + NO `[include]` IS FOLLOWED (`--no-includes`), as git's submodule + reader follows none. `git config --blob` follows one by default: a + relative one stopped its listing there (`relative config includes + must come from files`), where git reads on, and an absolute one's file + was listed with the blob's own entries, where git reads the blob's + alone. A work tree file's includes are not followed by default + either.""" + proc = subprocess.run(["git", "config", "-z", "--no-includes", *source, + "--list"], + cwd=str(root), capture_output=True, check=False) + if proc.returncode == 0: + return proc.stdout, False + if source[0] == "--blob" and proc.returncode == 128: + return proc.stdout, bool(proc.stdout) or source[1] != HEAD_GITMODULES + return None + + def gitmodules_entries(root: Path, listed: bytes, blob: bool - ) -> list[tuple[str, str, str | None, bool]]: + ) -> tuple[list[tuple[str, str, str | None, bool]], + str]: """`(name, key, value, ignored)` for each `submodule..` that `git config -z --list` printed in `listed`, in file order, as git's submodule reader meets them: None for a key with no value, and @@ -503,7 +625,8 @@ def gitmodules_entries(root: Path, listed: bytes, blob: bool a `..` component, split on `/` and on a backslash (git warns `ignoring suspicious submodule name`), a value git would die on included. In the index's or HEAD's file, a `blob`, git reads nothing after a key it - refuses with no value (`ends_blob_reading`).""" + refuses with no value (`ends_blob_reading`), and that key, as + `submodule..`, is returned beside them; else nothing.""" entries: list[tuple[str, str, str | None, bool]] = [] for record in listed.split(b"\0"): key, valued, value = record.decode("utf-8", @@ -515,9 +638,9 @@ def gitmodules_entries(root: Path, listed: bytes, blob: bool ignored = not name or ".." in name.replace("\\", "/").split("/") if blob and not ignored and not valued and ends_blob_reading( root, variable): - break + return entries, f"submodule.{name}.{variable}" entries.append((name, variable, value if valued else None, ignored)) - return entries + return entries, "" def ends_blob_reading(root: Path, key: str) -> bool: @@ -525,7 +648,12 @@ def ends_blob_reading(root: Path, key: str) -> bool: HEAD's `.gitmodules`, a blob, where `bad config line` is an error and no die (#768): each key git refuses with no value, but a `branch` only where git does not crash on it (`crashes_on_branch`); before 2.43.1 - git crashes on it there too, and `refused_value` names it.""" + git crashes on it there too, and `refused_value` names it. Measured on + git 2.43.0 (#856): a bare `url`, `path`, `update` or `ignore` in a + blob is `missing value` and `bad config line`, and git reads no + further; a bare `branch` crashes it (exit 139); a bare `shallow`, + `fetchRecurseSubmodules` or `active` is read as true, and git reads + on.""" return key in NO_VALUE_REFUSED and (key != "branch" or not crashes_on_branch(root)) diff --git a/templates/family-root/scripts/validate-family.py b/templates/family-root/scripts/validate-family.py index 9e77c3ce..ace9c3ec 100755 --- a/templates/family-root/scripts/validate-family.py +++ b/templates/family-root/scripts/validate-family.py @@ -1483,6 +1483,10 @@ def _head_listed(root: Path) -> tuple[bool, bytes] | None: #: What git takes for a `submodule..` whose value it dies on as #: it reads `.gitmodules` (#768), as `gitmodules-value-refused` says it. +#: This, `_values_refused` and `_git_takes_bool` say what bootstrap's +#: `TAKES`, `refused_value` and `git_bool` say, and +#: `tests/test_family_gitmodules_values.py` holds the two scripts to one +#: answer: a change made here is made there too (#856). GIT_TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", "shallow": "a boolean, `true` or `false`", "fetchrecursesubmodules": "a boolean or `on-demand`", diff --git a/tests/test_family_gitmodules_values.py b/tests/test_family_gitmodules_values.py index 3a6610ac..c2d1777a 100644 --- a/tests/test_family_gitmodules_values.py +++ b/tests/test_family_gitmodules_values.py @@ -21,6 +21,50 @@ `submodule..active` decides before `submodule.active`'s patterns; and a plain path pattern names only itself and what is under it. +AND #856'S TWO LINES, the first reproduced on #839's head 77f8a6c8: + +- Where git reads the index's or HEAD's `.gitmodules`, a blob, a line it + cannot parse there is an error, not a die: git reads the entries before + it and goes on. Bootstrap read none, so with no credential each row was + "no section of .gitmodules gives that path" and the run failed where + the members were the credential's. It reads what git reads now. +- The rules of which values git dies on are written in both scripts. One + table of cases is run through both, and their answers held to each + other, and what both say git takes is read from bootstrap's `TAKES`. + +AND #877'S FIRST FIX ROUND, after lane 4's RV877, each reproduced on its +head 40ba4208 first: + +- Where the line hid a section or a url, the cure was the section written + back, which in a work tree with no `.gitmodules` writes a file of that + section alone, and git then dies at the member it had read. The cure is + the file git read put back, mended and committed now, typed as printed. +- An `[include]` in the index's or HEAD's file is followed by neither git + nor bootstrap: a relative one cut bootstrap's reading short there, and + an absolute one's file was read as the blob's own. + +AND ITS SECOND, each reproduced on its first's head 1b87c57d first: + +- A key git takes no value for, a bare `url` line, inside a section of + the index's or HEAD's file: `git config --blob` takes it, git's reading + stops there, and the cure was the section written back, which cannot + cure, the key staying in the file. The cure is the file put back with + that key named, to be given a value or taken out, typed as printed. + +AND ITS THIRD, after RV877's re-check, each reproduced on the restacked +head 0afe0afe first: + +- A line git cannot parse first in the index's file, HEAD's whole: git + reads nothing, and bootstrap said the validator names the cause, which + reads HEAD's and cannot. The index's blob is there, so the read is cut, + and the cure puts the index's back. HEAD's cut at its first line stays + the validator's: HEAD may hold no file at all (#611). +- In a sparse checkout that leaves `.gitmodules` out, `git checkout -- + .gitmodules` matched no file. The index's put-back passes + `--ignore-skip-worktree-bits`, typed as printed there. +- A git that says `bad config line` in other words, and a cut blob beside + a row of another why, are each held by a test. + THE SAME FAMILY, OFFLINE RULES AND STATES as `test_family_bootstrap_gate_order.py` (#648), whose helpers are imported: a plain clone of the holder, `git submodule update` given the fixture's @@ -30,7 +74,9 @@ from __future__ import annotations +import os import re +import shutil import sys import pytest @@ -40,10 +86,10 @@ from test_family import ORG, family, validate # noqa: F401 from test_family_add_refusals import holder, offline # noqa: F401 from test_family_bootstrap_gate_order import ( - CHECKOUT_CURE, DUPLICATE, EXITED_0, FETCHED, NO_URL, NOT_ALL_FETCHED, - SECTION_CURE, STOPPED, after_first_line, bootstrap, cloned, edited_clone, - failed_on, fetching, not_fetched, type_line, url_edited, with_config, - without_url, + CHECKOUT_CURE, DUPLICATE, EXITED_0, FETCHED, NO_SECTION, NO_URL, + NOT_ALL_FETCHED, SECTION_CURE, STOPPED, after_first_line, bootstrap, + cloned, edited_clone, failed_on, fetching, not_fetched, type_line, + url_edited, with_config, without_url, ) from test_family_dry_run import isolated from test_family_head_lost_gitmodules import ( @@ -74,12 +120,26 @@ "branch-no-value": ("branch", "branch", None), } -#: What both tools say git takes for each key. -TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`", - "shallow": "a boolean, `true` or `false`", - "fetchrecursesubmodules": "a boolean or `on-demand`", - "ignore": "one of `untracked`, `dirty`, `all` or `none`", - "path": "a path", "url": "a url", "branch": "a branch name"} + +def by_path(script: str, name: str): + """The family root's `scripts/