bootstrap.py: a .gitmodules blob with a line git cannot parse is read up to that line, as git reads it, and a parity test holds the value rules both family scripts write to one answer (#856)

templates/family-root/scripts/bootstrap.py

@@ -54,9 +54,11 @@
       out; where every member was checked out the run fails all the same,
       since git updated none of them. A work tree file git cannot parse at
       all, a conflict marker or an open quote left in it, is said and
-      failed on the same way, and its cure is the file mended by hand. Git
-      exits 0 over the second where a recursive clone ran first, and 128
-      (`No url found for submodule path`) in a plain clone; a missing
+      failed on the same way, and its cure is the file mended by hand; in
+      the index's or HEAD's file git reads the sections before such a line
+      and goes on, and so does this (#856). Git exits 0 over the second
+      where a recursive clone ran first, and 128 (`No url found for
+      submodule path`) in a plain clone; a missing
       credential exits non-zero too, so what the holder records is the
       test, never git's words. The line says what cures each (`cures`):
       `validate-family.py` names the commit for the first two; for the
@@ -66,7 +68,13 @@
       sections give, or a gitlink in members/ that no row names; `git rm
       --cached` takes out one outside members/, which the validator does
       not read; and where the gitlink is a row's own, with one section,
-      the line says to write its section back.
+      the line says to write its section back. Where git read the index's
+      or HEAD's file only up to a line it cannot parse, or up to a key it
+      takes no value for (a bare `url` line, which `git config --list`
+      takes), a section or url it did not read may be past that line, so
+      for those three the line says to put that file back, mend the line
+      or give the key a value or take it out, and commit it, and writes
+      no section back (#856).
 
   (b) RUNS EACH MEMBER'S OWN `make bootstrap`. Each member is a whole project
       and knows how to bootstrap itself — it puts its legs on their
@@ -132,12 +140,21 @@
 NO_URL = "git has no url for that path"
 EXITED_0 = "git exited 0 and checked nothing out there"
 STOPPED = "git stopped at {}, a gitlink it has no url for"
+#: The whys a `.gitmodules` git read only up to a line it cannot parse, or
+#: up to a key with no value it stops at, may be the cause of, since what
+#: it did not read is past that line (`Gitmodules.cut`, #856).
+CUT_WHYS = frozenset({NO_SECTION, NO_URL, STOPPED})
+#: HEAD's `.gitmodules`, as `git config --blob` is given it.
+HEAD_GITMODULES = f"HEAD:{GITMODULES}"
 
 #: What git 2.43 dies on in a `.gitmodules` it reads (#768), as
 #: `parse_config` in its submodule-config.c reads a submodule's key: these
 #: keys with no value (a `branch` with none crashes it, `BRANCH_READ_SINCE`),
 #: and an `update` other than these four. A `shallow` must be a boolean and
-#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`).
+#: a `fetchrecursesubmodules` a boolean or `on-demand` (`git_bool`). The
+#: gate's `_values_refused`, `_git_takes_bool` and `GIT_TAKES` say the same,
+#: and `tests/test_family_gitmodules_values.py` holds the two scripts to one
+#: answer: a change made here is made there too (#856).
 NO_VALUE_REFUSED = frozenset({"path", "url", "update", "ignore", "branch"})
 #: The first git that reads a `branch` with no value as it reads a `url`
 #: with none, `missing value` and `bad config line`, which dies in the work
@@ -328,9 +345,21 @@ def cures(left: list[tuple[dict, str, str]], stopper: str | None,
       url no clone can check it out.
     - A row git exited 0 over: the update that checks it out whatever the
       clone chose. `--init` with the path registers a member a subset
-      clone left out, and `--checkout` overrides an `update = none`."""
+      clone left out, and `--checkout` overrides an `update = none`.
+    - The index's or HEAD's `.gitmodules`, a blob, which git read only up
+      to a line it cannot parse, or up to a key with no value it stops at
+      (`Gitmodules.cut`, #856), for a row with no section or no url there,
+      or the gitlink git stopped at (`CUT_WHYS`): that file put back,
+      mended and committed, which `gitmodules_cures` says. A section or
+      url git did not read may be past the line, so no other cure is said
+      for those: a section written back where the work tree holds no file
+      would be the only one in it, and `validate-family.py` reads HEAD's
+      file alone, not the index's."""
     whys = {why for _, _, why in left}
     said = gitmodules_cures(whys, modules)
+    if modules.cut:
+        left = [entry for entry in left if entry[2] not in CUT_WHYS]
+        whys -= CUT_WHYS
     if whys & {NO_GITLINK, NO_SECTION}:
         said.append(f"`{PYTHON} scripts/validate-family.py` names the cause "
                     "and the commit that cures it.")
@@ -374,7 +403,21 @@ def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]:
     """`cures`' sentences for a `.gitmodules` git reads nothing from, each
     in the order `cures` says them (#648, #768): a merge left unmerged,
     for a row `whys` names so; each value git dies on; a work tree file
-    git cannot parse."""
+    git cannot parse. And for a blob git read only up to such a line, or
+    up to a key with no value it stops at, where a row `whys` names is
+    one of `CUT_WHYS` (#856): the command that puts that file back in the
+    work tree, `git checkout` from the index where git read the index's,
+    which keeps what was staged, and from HEAD where it read HEAD's, then
+    the file mended, or the key given a value git takes (`TAKES`) or its
+    line taken out, and committed. The index's is put back with
+    `--ignore-skip-worktree-bits`: in a sparse checkout that leaves
+    `.gitmodules` out, its entry is skip-worktree, and `git checkout --
+    .gitmodules` matches no file there (RV877's NIT-3, measured, git
+    2.43.0); with the flag it puts the file back there and in any other
+    work tree. HEAD's is put back only where the index holds no entry,
+    so no skip-worktree bit, and `git checkout HEAD` puts it back in a
+    sparse checkout too. `git config -f .gitmodules --list`
+    takes a key with no value, so it is pointed at for the line alone."""
     said = []
     if UNMERGED in whys:
         said.append(f"Resolve the merge in {GITMODULES} and `git add "
@@ -392,6 +435,21 @@ def gitmodules_cures(whys: set[str], modules: Gitmodules) -> list[str]:
         said.append(f"Mend {GITMODULES} by hand (`git config -f "
                     f"{GITMODULES} --list` says where it breaks), and "
                     "commit it.")
+    if modules.cut and whys & CUT_WHYS:
+        head = modules.cut == HEAD_GITMODULES
+        put_back = shlex.join([
+            "git", "checkout",
+            *(["HEAD"] if head else ["--ignore-skip-worktree-bits"]),
+            "--", GITMODULES])
+        whose = f"{'HEAD' if head else 'the index'}'s {GITMODULES}"
+        key = modules.cut_key
+        mend = (f"`{spelled(key)}`, a key with no value: give it a value "
+                f"git takes, {TAKES[key.rpartition('.')[2]]}, or take that "
+                "line out" if key else
+                "a line it cannot parse: mend it by hand (`git config -f "
+                f"{GITMODULES} --list` says where it breaks)")
+        said.append(f"`{put_back}` puts back {whose}, which git read only "
+                    f"up to {mend}, and commit it.")
     return said
 
 
@@ -422,23 +480,39 @@ class Gitmodules(NamedTuple):
     it reads the file, None for a key with no value (#768): git reads
     nothing from it then, and the three are empty. `unreadable` says the
     work tree's file is one `git config --list` cannot parse, which git
-    dies reading: the three are empty then too."""
+    dies reading: the three are empty then too. `cut` is the blob git read
+    only up to such a line, as `gitmodules_source` names it,
+    `HEAD:.gitmodules` or the index's object id, where it read an entry
+    before it, or the blob is the index's (`gitmodules_listed`, #856):
+    the three hold what it read,
+    and `cures` writes no section back. Empty where git read the whole
+    file, or nothing. `cut_key` is the `submodule.<name>.<key>` with no
+    value that cut it, where that is what git stopped at
+    (`ends_blob_reading`), a line `git config --list` takes, and empty
+    where a line it cannot parse did."""
 
     given: Counter
     read: dict[str, str]
     urls: dict[str, str]
     unmerged: bool = False
     refused: tuple[tuple[str, str | None], ...] = ()
     unreadable: bool = False
+    cut: str = ""
+    cut_key: str = ""
 
 
 def gitmodules_sections(root: Path) -> Gitmodules:
     """`.gitmodules` as `git submodule update` reads it: the work tree's
     file, which a fresh clone takes from HEAD, or, where the work tree has
     none, as a deletion not yet committed or a sparse checkout leaves it,
-    the index's, then HEAD's (`gitmodules_source`, #768). Nothing where
-    none holds one, or git cannot read it: git has no url for any
-    submodule then either. Nothing, and `unreadable`, where that is the
+    the index's, then HEAD's (`gitmodules_source`, #768), each of the
+    last two, a blob, up to a line git cannot parse, as git reads it,
+    and `cut` where it read an entry before that line, or the blob is
+    the index's (`gitmodules_listed`, #856), or up to a key with no value
+    git stops at, `cut` and `cut_key` (`gitmodules_entries`). Nothing
+    where none holds one, or git cannot read it: git has no url for any
+    submodule then either.
+    Nothing, and `unreadable`, where that is the
     work tree's file and `git config --list` cannot parse it: git dies
     reading it (`bad config line`), before it checks out any member.
     Nothing, and `unmerged`, where the index holds the file only at a
@@ -455,19 +529,19 @@ def gitmodules_sections(root: Path) -> Gitmodules:
     source = gitmodules_source(root)
     if source is None:
         return nothing._replace(unmerged=True)
-    try:
-        listed = git_out(["config", "-z", *source, "--list"], cwd=root,
-                         binary=True)
-    except Refusal:
+    listed = gitmodules_listed(root, source)
+    if listed is None:
         return nothing._replace(unreadable=os.path.isfile(root / GITMODULES))
-    entries = gitmodules_entries(root, listed, source[0] == "--blob")
+    printed, cut = listed
+    entries, ended = gitmodules_entries(root, printed, source[0] == "--blob")
     refused = tuple(dict.fromkeys(
         (f"submodule.{name}.{key}", value)
         for name, key, value, ignored in entries
         if not ignored and refused_value(root, key, value)))
     if refused:
         return nothing._replace(refused=refused)
-    return read_sections(entries)
+    return read_sections(entries)._replace(
+        cut=source[1] if cut or ended else "", cut_key=ended)
 
 
 def gitmodules_source(root: Path) -> list[str] | None:
@@ -480,7 +554,7 @@ def gitmodules_source(root: Path) -> list[str] | None:
                          cwd=root, binary=True)
     except Refusal:
         stages = b""
-    source = ["--blob", f"HEAD:{GITMODULES}"]
+    source = ["--blob", HEAD_GITMODULES]
     for record in stages.split(b"\0"):
         meta, _, name = record.partition(b"\t")
         fields = meta.split()
@@ -494,16 +568,65 @@ def gitmodules_source(root: Path) -> list[str] | None:
     return source
 
 
+def gitmodules_listed(root: Path, source: list[str]
+                      ) -> tuple[bytes, bool] | None:
+    """What `git config -z --no-includes --list` prints of the
+    `.gitmodules` that `source` names (`gitmodules_source`), and whether
+    git stopped short of the file's end at a line it cannot parse, or
+    None where git cannot read it (#856).
+
+    A BLOB IS READ UP TO A LINE GIT CANNOT PARSE. `git config --blob`
+    prints the entries before that line, then `bad config line`, and
+    exits 128. Git's submodule reader, which reads the index's or HEAD's
+    file where the work tree has none, takes those same entries, says
+    the same line as an error, not a die, and goes on (measured, git
+    2.43.0): so what was printed is what git read, and a bad line before
+    the first section leaves nothing, as git reads nothing. A work tree
+    file with such a line is None: git dies reading it. Git is run as
+    `git_out` runs it.
+
+    TOLD BY THE EXIT CODE, NEVER GIT'S WORDS, which a git built with its
+    translations prints in the person's language: a blob's listing that
+    exits 128 is read for what it printed, whatever it says on stderr.
+    The other 128 met is a blob git cannot find (`HEAD:.gitmodules`
+    where HEAD holds none, #611), which prints nothing, and empty
+    stdout reads as the same `Gitmodules` that None gives for a blob:
+    no entry, and nothing `unreadable`, since a blob is read only where
+    the work tree holds no file. So HEAD's, printing nothing, is no
+    `cut`: it may be no file at all. The index's is, printing nothing or
+    not: `git ls-files -s` gave its object id, so the blob is there, and
+    a 128 can only be a line git cannot parse, the first included
+    (RV877's SHOULD-2).
+
+    NO `[include]` IS FOLLOWED (`--no-includes`), as git's submodule
+    reader follows none. `git config --blob` follows one by default: a
+    relative one stopped its listing there (`relative config includes
+    must come from files`), where git reads on, and an absolute one's file
+    was listed with the blob's own entries, where git reads the blob's
+    alone. A work tree file's includes are not followed by default
+    either."""
+    proc = subprocess.run(["git", "config", "-z", "--no-includes", *source,
+                           "--list"],
+                          cwd=str(root), capture_output=True, check=False)
+    if proc.returncode == 0:
+        return proc.stdout, False
+    if source[0] == "--blob" and proc.returncode == 128:
+        return proc.stdout, bool(proc.stdout) or source[1] != HEAD_GITMODULES
+    return None
+
+
 def gitmodules_entries(root: Path, listed: bytes, blob: bool
-                       ) -> list[tuple[str, str, str | None, bool]]:
+                       ) -> tuple[list[tuple[str, str, str | None, bool]],
+                                  str]:
     """`(name, key, value, ignored)` for each `submodule.<name>.<key>` that
     `git config -z --list` printed in `listed`, in file order, as git's
     submodule reader meets them: None for a key with no value, and
     `ignored` for a section git ignores whole, whose name is empty or has
     a `..` component, split on `/` and on a backslash (git warns `ignoring
     suspicious submodule name`), a value git would die on included. In the
     index's or HEAD's file, a `blob`, git reads nothing after a key it
-    refuses with no value (`ends_blob_reading`)."""
+    refuses with no value (`ends_blob_reading`), and that key, as
+    `submodule.<name>.<key>`, is returned beside them; else nothing."""
     entries: list[tuple[str, str, str | None, bool]] = []
     for record in listed.split(b"\0"):
         key, valued, value = record.decode("utf-8",
@@ -515,17 +638,22 @@ def gitmodules_entries(root: Path, listed: bytes, blob: bool
         ignored = not name or ".." in name.replace("\\", "/").split("/")
         if blob and not ignored and not valued and ends_blob_reading(
                 root, variable):
-            break
+            return entries, f"submodule.{name}.{variable}"
         entries.append((name, variable, value if valued else None, ignored))
-    return entries
+    return entries, ""
 
 
 def ends_blob_reading(root: Path, key: str) -> bool:
     """Whether a `key` with no value ends git's reading of the index's or
     HEAD's `.gitmodules`, a blob, where `bad config line` is an error and
     no die (#768): each key git refuses with no value, but a `branch` only
     where git does not crash on it (`crashes_on_branch`); before 2.43.1
-    git crashes on it there too, and `refused_value` names it."""
+    git crashes on it there too, and `refused_value` names it. Measured on
+    git 2.43.0 (#856): a bare `url`, `path`, `update` or `ignore` in a
+    blob is `missing value` and `bad config line`, and git reads no
+    further; a bare `branch` crashes it (exit 139); a bare `shallow`,
+    `fetchRecurseSubmodules` or `active` is read as true, and git reads
+    on."""
     return key in NO_VALUE_REFUSED and (key != "branch"
                                         or not crashes_on_branch(root))
 

templates/family-root/scripts/validate-family.py

@@ -1483,6 +1483,10 @@ def _head_listed(root: Path) -> tuple[bool, bytes] | None:
 
 #: What git takes for a `submodule.<name>.<key>` whose value it dies on as
 #: it reads `.gitmodules` (#768), as `gitmodules-value-refused` says it.
+#: This, `_values_refused` and `_git_takes_bool` say what bootstrap's
+#: `TAKES`, `refused_value` and `git_bool` say, and
+#: `tests/test_family_gitmodules_values.py` holds the two scripts to one
+#: answer: a change made here is made there too (#856).
 GIT_TAKES = {"update": "one of `none`, `checkout`, `rebase` or `merge`",
              "shallow": "a boolean, `true` or `false`",
              "fetchrecursesubmodules": "a boolean or `on-demand`",