tests/conftest.py: one helper builds the url.<file URI>.insteadOf rewrite that clone_just_pushed, read_through_git and the adopt tests' pasted each built, and the empty-mount submodule update --init sites use it (#662)

tests/conftest.py

@@ -68,6 +68,60 @@
 FILE_PROTOCOL = ["-c", "protocol.file.allow=always"]
 
 
+# --- one rewrite, for every read of a just-pushed local remote (#440, #662) --
+#
+# A clone of a plain path is git's local clone, which walks the source's
+# `objects/` one file at a time; a clone of a `file://` URL asks upload-pack,
+# which reads the object store (`clone_just_pushed` says why that matters).
+# What turns the first into the second for the legs a submodule clones from
+# the plain paths `.gitmodules` records is a `url.<file URI>.insteadOf`
+# rewrite, and it is written HERE, once, in three forms: the prefixes, the
+# `(key, prefix)` entries, and the `-c` arguments of one git command.
+# `read_through_git` below puts the entries in the environment instead.
+
+def insteadof_prefixes(remotes: Path | str, *,
+                       typed: bool = False) -> list[str]:
+    """The prefixes a `url.<file URI>.insteadOf` rewrite names so that git
+    reads the bare remotes under `remotes` through the transport and not
+    through their files: the directory with a separator after it, so a
+    rewrite of `<remotes>/` does not reach a neighbour called
+    `<remotes>-other`, and a `remotes` given with or without a trailing
+    separator gives the same list.
+
+    RESOLVED, always: `scaffold-project.py` writes `.gitmodules`'s leg URLs
+    from the directory resolved, so that is where a submodule's clone starts,
+    spelled with the system's separator and with a slash, which are two
+    prefixes on Windows and one elsewhere. TYPED, on request: the directory
+    as the caller spelled it, made absolute and a symlink in it kept, which is
+    how a line a tool prints names a remote; it comes first. None is listed
+    twice."""
+    resolved = Path(remotes).resolve()
+    directories = [Path(os.path.abspath(remotes)), resolved] if typed \
+        else [resolved]
+    return list(dict.fromkeys(
+        spelling for directory in directories
+        for spelling in (os.path.join(str(directory), ""),
+                         directory.as_posix().rstrip("/") + "/")))
+
+
+def insteadof_entries(remotes: Path | str, *,
+                      typed: bool = False) -> list[tuple[str, str]]:
+    """`(key, prefix)` for each of `insteadof_prefixes`: the key is the one
+    `url.<file URI of the directory resolved>.insteadOf` that sends them all
+    to the same `file://` URL."""
+    uri = Path(remotes).resolve().as_uri().rstrip("/") + "/"
+    return [(f"url.{uri}.insteadOf", prefix)
+            for prefix in insteadof_prefixes(remotes, typed=typed)]
+
+
+def insteadof_args(remotes: Path | str, *, typed: bool = False) -> list[str]:
+    """`insteadof_entries` as the `-c <key>=<prefix>` pairs of one git
+    command. A `-c` is not stored: a clone made with it keeps the plain path
+    it was given as its `origin`, and a submodule its `.gitmodules` URL."""
+    return [arg for key, prefix in insteadof_entries(remotes, typed=typed)
+            for arg in ("-c", f"{key}={prefix}")]
+
+
 def clone_just_pushed(remote: Path, dest: Path,
                       branch: str | None = None, *, recurse: bool = True,
                       bare: bool = False, config: Sequence[str] = (),
@@ -93,7 +147,9 @@ def clone_just_pushed(remote: Path, dest: Path,
     `url.<base>.insteadOf` for the one command: a `-c` is not stored, so the
     clone's own `origin` and `submodule.<name>.url` keep the plain paths. The
     prefix is the directory RESOLVED, as `scaffold-project.py` writes it into
-    `.gitmodules`, in both of a Windows path's spellings.
+    `.gitmodules`, in both of a Windows path's spellings
+    (`insteadof_prefixes`, shared with `read_through_git` and
+    `submodule_update_through_git`).
 
     `branch` is `-b <branch>`: `adopt-project.py execute` pushes its split
     branch into the SOURCE, a plain repository, which is then `remote`.
@@ -110,12 +166,7 @@ def clone_just_pushed(remote: Path, dest: Path,
     names a path, and a project's name can be any text.
     """
     remote = Path(remote)
-    remotes = remote.parent.resolve()
-    uri = remotes.as_uri().rstrip("/") + "/"
-    prefixes = dict.fromkeys([os.path.join(str(remotes), ""),
-                              remotes.as_posix().rstrip("/") + "/"])
-    rewrites = [arg for prefix in prefixes
-                for arg in ("-c", f"url.{uri}.insteadOf={prefix}")]
+    rewrites = insteadof_args(remote.parent)
     if bare:
         mode = ["--bare"]
     else:
@@ -138,6 +189,30 @@ def cloned_just_pushed(remote: Path, dest: Path, branch: str | None = None,
     return Path(dest)
 
 
+def submodule_update_through_git(root: Path, remotes: Path,
+                                 *options: str
+                                 ) -> subprocess.CompletedProcess:
+    """`git submodule update <options>` in `root`, failing the test where it
+    fails (`git` below), with each mount it CLONES read through git and not
+    through the remote's files (#662, #440).
+
+    An `--init` of a mount git keeps no repository for is git's local clone of
+    the plain path `.gitmodules` records for it: the walk of the remote's
+    `objects/` that `clone_just_pushed` keeps off, taken by a submodule.
+    So the plain paths under `remotes` are rewritten to `file://` for the one
+    command, as `clone_just_pushed` rewrites them for its legs, and the file
+    transport is allowed, as every test that clones a local path allows it.
+    `options` follow `update`: `("-q", "--init")`, `("--init", "--recursive",
+    "--", "code")`.
+
+    An `update` over a mount whose repository git already keeps FETCHES, and
+    a fetch asks upload-pack whatever the URL is: a call that only brings
+    such a mount to its gitlink needs neither the rewrite nor this wrapper.
+    """
+    return git(*FILE_PROTOCOL, *insteadof_args(remotes),
+               "submodule", "update", *options, cwd=root)
+
+
 # --- git's automatic maintenance, OFF for every git the session starts (#645)
 #
 # A COPY IS A WALK OF THE FILES TOO. `shutil.copytree` reads `objects/` one
@@ -1525,21 +1600,18 @@ def read_through_git(remotes: Path) -> dict:
     """The environment that makes every git a pasted line starts read the
     bare repositories in `remotes` through git, not through their files
     (#440): `clone_just_pushed`'s `url.<file URI>.insteadOf` rewrites, as
-    `GIT_CONFIG_COUNT` entries, because the line under test is run exactly
-    as it was printed and a submodule's own clone inherits the environment.
-    The directory as typed and as resolved, so the root's URL and the legs'
-    `.gitmodules` URLs are both rewritten."""
-    uri = remotes.resolve().as_uri().rstrip("/") + "/"
-    prefixes = list(dict.fromkeys(
-        spelling for directory in (Path(os.path.abspath(remotes)),
-                                   remotes.resolve())
-        for spelling in (os.path.join(str(directory), ""),
-                         directory.as_posix().rstrip("/") + "/")))
-    env = {"GIT_CONFIG_COUNT": str(len(prefixes))}
-    for index, prefix in enumerate(prefixes):
-        env[f"GIT_CONFIG_KEY_{index}"] = f"url.{uri}.insteadOf"
-        env[f"GIT_CONFIG_VALUE_{index}"] = prefix
-    return env
+    `GIT_CONFIG_*` entries, because the line under test is run exactly as it
+    was printed and a submodule's own clone inherits the environment. The
+    directory as typed and as resolved (`insteadof_entries`), so the root's
+    URL and the legs' `.gitmodules` URLs are both rewritten.
+
+    The entries are APPENDED to the ones `os.environ` holds
+    (`appended_to_command_scope`), which every caller lays this over at the
+    same moment, so the session's own (`no_auto_maintenance`'s, and the two
+    the Windows job sets) still reach the pasted line, as they reach every
+    other git a test starts."""
+    return appended_to_command_scope(
+        os.environ, insteadof_entries(remotes, typed=True))
 
 
 def pasted(lines, cwd: Path,

tests/test_adopt_e2e.py

@@ -11,7 +11,6 @@
 from __future__ import annotations
 
 import importlib.util
-import os
 import re
 import shlex
 import shutil
@@ -21,8 +20,8 @@
 
 import pytest
 
-from conftest import (ADOPT, REPO, cloned_just_pushed, git, make_source_repo,
-                      resolve, run_script, write_plan)
+from conftest import (ADOPT, REPO, cloned_just_pushed, git, insteadof_args,
+                      make_source_repo, resolve, run_script, write_plan)
 
 sys.path.insert(0, str(REPO / "scripts"))
 from repo_shape import load_yaml  # noqa: E402
@@ -243,21 +242,16 @@ def pasted(line: str, cwd: Path, remotes: Path) -> subprocess.CompletedProcess:
     object files one by one, and maintenance a push left running can pack
     them away mid-copy (`failed to copy file`, on the macOS runner). So
     `--no-local` reads the root through git, and each leg's plain path under
-    `remotes`, as `.gitmodules` records it (the directory resolved, in both
-    of a Windows path's spellings), is read as its `file://` URI. Every
-    printed argument is kept as printed, `-c protocol.file.allow=always`
-    among them; a URI is the `file` transport too, so a line without it is
-    refused here as it is refused pasted."""
+    `remotes`, as `.gitmodules` records it (`insteadof_args`: the directory
+    resolved, in both of a Windows path's spellings), is read as its
+    `file://` URI. Every printed argument is kept as printed, `-c
+    protocol.file.allow=always` among them; a URI is the `file` transport
+    too, so a line without it is refused here as it is refused pasted."""
     argv = shlex.split(line)
-    resolved = remotes.resolve()
-    uri = resolved.as_uri().rstrip("/") + "/"
-    prefixes = dict.fromkeys([os.path.join(str(resolved), ""),
-                              resolved.as_posix().rstrip("/") + "/"])
     at = argv.index("clone")
     return subprocess.run(
-        [*argv[:at], *(arg for prefix in prefixes
-                       for arg in ("-c", f"url.{uri}.insteadOf={prefix}")),
-         "clone", "--no-local", *argv[at + 1:]],
+        [*argv[:at], *insteadof_args(remotes), "clone", "--no-local",
+         *argv[at + 1:]],
         cwd=str(cwd), capture_output=True, encoding="utf-8", errors="replace",
         check=False)
 

tests/test_clone_just_pushed.py

@@ -10,17 +10,29 @@
 no loose object, which a clone that walked the remote's own files would not
 (`assert_store_read_through_git`). The option itself is asked of git beside
 it, and the last case is the clone that fails.
+
+#662 wrote the rewrite those clones carry once (`insteadof_prefixes`, its
+`(key, prefix)` entries and its `-c` arguments) and gave its other two users
+the same list: `read_through_git`, which puts it in the environment of a
+pasted line, and `submodule_update_through_git`, for the `--init` of a mount a
+test brings up after the clone. The cases at the end ask git what each form
+does to a plain path, with `git ls-remote --get-url`, which expands a URL as a
+fetch would and talks to nothing.
 """
 
 from __future__ import annotations
 
 import os
+import subprocess
+from pathlib import Path
 
 import pytest
 
-from conftest import (PROJECT, assert_read_through_git,
+from conftest import (FILE_PROTOCOL, PROJECT, assert_read_through_git,
                       assert_store_read_through_git, clone_just_pushed,
-                      cloned_just_pushed, git)
+                      cloned_just_pushed, git, insteadof_args,
+                      insteadof_entries, insteadof_prefixes, read_through_git,
+                      submodule_update_through_git)
 
 
 @pytest.fixture
@@ -100,3 +112,161 @@ def test_a_clone_that_fails_fails_the_test_and_says_why(tmp_path):
     assert clone_just_pushed(missing, tmp_path / "clone").returncode != 0
     with pytest.raises(AssertionError, match="git clone of .* failed"):
         cloned_just_pushed(missing, tmp_path / "clone")
+
+
+# --- the rewrite, written once (#662) ----------------------------------------
+
+def spellings(directory: Path) -> set[str]:
+    """The prefixes of an existing `directory` as a person would write them:
+    the system's separator after it, and a slash. One on POSIX, two on
+    Windows. Written out here and not read from the helper."""
+    return {str(directory) + os.sep, directory.as_posix() + "/"}
+
+
+def get_url(url: str, *config: str, env: dict | None = None,
+            cwd: Path) -> str:
+    """What git makes of `url` as a fetch would, `insteadOf` rewrites
+    included, and talks to nothing."""
+    return subprocess.run(["git", *config, "ls-remote", "--get-url", url],
+                          cwd=str(cwd), capture_output=True, text=True,
+                          check=True, env=env).stdout.strip()
+
+
+@pytest.fixture
+def remotes(tmp_path) -> Path:
+    directory = tmp_path / "remotes"
+    directory.mkdir()
+    return directory
+
+
+@pytest.mark.parametrize("typed", [False, True], ids=["resolved", "typed"])
+@pytest.mark.parametrize("trailing", ["", "/", os.sep],
+                         ids=["none", "slash", "separator"])
+def test_a_directory_is_named_with_a_separator_after_it_given_or_not(
+        remotes, typed, trailing):
+    """The prefix ends with a separator whether the directory was given with
+    one or without, so a rewrite of `<remotes>/` never reaches a neighbour
+    that begins with the same text, and each spelling is listed once. A
+    directory with no link in its path is typed and resolved at once."""
+    prefixes = insteadof_prefixes(str(remotes) + trailing, typed=typed)
+    assert set(prefixes) == spellings(remotes.resolve()), prefixes
+    assert len(prefixes) == len(set(prefixes)), prefixes
+    assert prefixes == insteadof_prefixes(remotes, typed=typed), prefixes
+
+
+def a_link_to(directory: Path, tmp_path: Path) -> Path:
+    """`tmp_path/link`, a symlink to `directory`, or a skip where this
+    account cannot make one."""
+    link = tmp_path / "link"
+    try:
+        link.symlink_to(directory, target_is_directory=True)
+    except (OSError, NotImplementedError):
+        pytest.skip("this account cannot make a symlink")
+    return link
+
+
+def test_a_typed_directory_adds_the_spelling_it_was_given_first(tmp_path):
+    """TYPED: a symlink kept, as a line a tool prints names a remote. The
+    resolved directory is the one `.gitmodules` records, and stays."""
+    real = tmp_path / "real"
+    real.mkdir()
+    link = a_link_to(real, tmp_path)
+    resolved = spellings(real.resolve())
+    typed = spellings(Path(os.path.abspath(link)))
+    assert typed.isdisjoint(resolved), "premise: a link that is not its target"
+    assert set(insteadof_prefixes(link)) == resolved
+    prefixes = insteadof_prefixes(link, typed=True)
+    assert set(prefixes[:len(typed)]) == typed, prefixes
+    assert set(prefixes) == typed | resolved, prefixes
+    assert len(prefixes) == len(set(prefixes)), prefixes
+    # Both go to the one URL: the resolved directory's, where the legs are.
+    uri = real.resolve().as_uri().rstrip("/") + "/"
+    assert {key for key, _ in insteadof_entries(link, typed=True)} == {
+        f"url.{uri}.insteadOf"}
+
+
+def test_every_entry_sends_its_prefix_to_the_resolved_directorys_uri(
+        remotes):
+    uri = remotes.resolve().as_uri().rstrip("/") + "/"
+    entries = insteadof_entries(remotes, typed=True)
+    assert {key for key, _ in entries} == {f"url.{uri}.insteadOf"}, entries
+    assert [prefix for _, prefix in entries] == insteadof_prefixes(
+        remotes, typed=True)
+    assert insteadof_args(remotes, typed=True) == [
+        arg for key, prefix in entries for arg in ("-c", f"{key}={prefix}")]
+
+
+def test_the_arguments_and_the_environment_send_a_plain_path_to_its_uri(
+        remotes, tmp_path):
+    """Both forms of the one rewrite do to a path under `remotes` what
+    `clone_just_pushed` needs, and leave a neighbour's path as it is."""
+    plain = str(remotes / f"{PROJECT}.git")
+    neighbour = str(remotes) + f"-other/{PROJECT}.git"
+    uri = (remotes.resolve() / f"{PROJECT}.git").as_uri()
+    environ = {**os.environ, **read_through_git(remotes)}
+    assert get_url(plain, *insteadof_args(remotes), cwd=tmp_path) == uri
+    assert get_url(plain, env=environ, cwd=tmp_path) == uri
+    assert get_url(neighbour, *insteadof_args(remotes),
+                   cwd=tmp_path) == neighbour
+    assert get_url(neighbour, env=environ, cwd=tmp_path) == neighbour
+
+
+def test_the_environment_form_reads_a_path_typed_through_a_symlink(
+        remotes, tmp_path):
+    """`read_through_git` is the one user of `typed=True`: the line a tool
+    prints names the remotes as the person typed them, a symlink and all, and
+    only the TYPED spelling matches that path. Both go to the resolved
+    directory's URI."""
+    link = a_link_to(remotes, tmp_path)
+    environ = {**os.environ, **read_through_git(link)}
+    assert get_url(str(link / "A.git"), env=environ, cwd=tmp_path) == (
+        remotes.resolve() / "A.git").as_uri()
+    assert get_url(str(remotes / "A.git"), env=environ, cwd=tmp_path) == (
+        remotes.resolve() / "A.git").as_uri()
+
+
+def test_the_environment_form_keeps_what_the_environment_holds(
+        remotes, tmp_path, monkeypatch):
+    """APPENDED, as `appended_to_command_scope` appends the session's own
+    settings: an entry the process already has is neither overwritten nor
+    counted away, so the pasted line sees it as every other git does."""
+    monkeypatch.setenv("GIT_CONFIG_COUNT", "1")
+    monkeypatch.setenv("GIT_CONFIG_KEY_0", "url.held://base/.insteadOf")
+    monkeypatch.setenv("GIT_CONFIG_VALUE_0", "held://")
+    entries = read_through_git(remotes)
+    assert "GIT_CONFIG_KEY_0" not in entries, entries
+    assert entries["GIT_CONFIG_COUNT"] == str(
+        1 + len(insteadof_prefixes(remotes, typed=True))), entries
+    environ = {**os.environ, **entries}
+    assert get_url("held://x", env=environ, cwd=tmp_path) == "held://base/x"
+    assert get_url(str(remotes / "A.git"), env=environ, cwd=tmp_path) == (
+        remotes.resolve() / "A.git").as_uri()
+
+
+def test_an_init_of_an_empty_mount_is_read_through_git_by_the_wrapper(
+        scaffolded, remote, tmp_path):
+    """#662. The mount is empty and git keeps no repository for it, so its
+    `--init` is a clone of the plain path `.gitmodules` records. Through
+    `submodule_update_through_git` the leg arrives as one pack with no loose
+    object, and the `submodule.<name>.url` git registers is still the plain
+    path, as `-c` stores nothing. The same `--init` of the other leg, run
+    plain, is git's local clone and leaves the remote's loose objects loose:
+    the control that makes the first half mean something."""
+    clone = cloned_just_pushed(remote, tmp_path / "clone", recurse=False)
+    assert not any((clone / "spec").iterdir()), "premise: an empty mount"
+    assert not any((clone / "code").iterdir()), "premise: an empty mount"
+
+    submodule_update_through_git(clone, scaffolded["remotes"], "-q", "--init",
+                                 "--", "spec")
+    assert any((clone / "spec").iterdir()), "the leg was not checked out"
+    assert_store_read_through_git(clone / "spec", "the leg, via the wrapper")
+    registered = git("config", "--local", "--get", "submodule.spec.url",
+                     cwd=clone).stdout.strip()
+    assert registered.endswith(f"{PROJECT}-spec.git"), registered
+    assert not registered.startswith("file:"), registered
+
+    git(*FILE_PROTOCOL, "submodule", "update", "-q", "--init", "--", "code",
+        cwd=clone)
+    assert any((clone / "code").iterdir()), "the control was not checked out"
+    with pytest.raises(AssertionError, match="loose objects"):
+        assert_store_read_through_git(clone / "code", "the leg, plain")