diff --git a/adopt-project.py b/adopt-project.py index a8b1ef34..83da3349 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -1762,6 +1762,52 @@ def _os_reason(exc: OSError) -> str: return exc.strerror or type(exc).__name__ +def _refused_at(exc: OSError, directory: Path) -> bool: + """Whether `exc`, raised by a command run with `cwd=directory`, is the + operating system refusing to enter `directory` (#918): `subprocess` + names the directory as the exception's `filename` then. A program that + could not be started is named by the program instead, and is not this: + whether that is a question the caller can answer stays the caller's, + as `git_out` leaves it.""" + return exc.filename == str(directory) + + +def _recorded_spelling(recorded: str) -> tuple[str, str]: + """`(spelling, said)`: `recorded`, a path a plan records as text, as the + disk is asked for it in this locale, and what a refusal that it names + nothing adds, which is "" where it is asked as written (#918, #457's + L1-86). + + A PLAN IS UTF-8 TEXT, and `plan` records a path as `os.fsdecode` read + it: under a UTF-8 locale or Python's UTF-8 mode each character as + itself and a byte that is not UTF-8 as its surrogate, and under the C + locale every byte outside ASCII as its surrogate. Either way the path's + bytes are the text's UTF-8 with `surrogateescape`, as `_path_argument` + spells a path for git. Where this locale's file-name encoding can spell + the text, it is asked as written, as before. Where it cannot, as the + ASCII of `LC_ALL=C PYTHONUTF8=0` cannot spell `Thing\\u00e9` from a plan + written in UTF-8 mode, `Path.exists()` met the `UnicodeEncodeError` + inside and answered False, and `check` refused `plan-source-gone`, + "names nothing that exists", of a source that is there. It is asked as + those UTF-8 bytes instead, which `os.fsdecode` spells in this locale; + and where they name nothing, `said` names the spelling asked and this + locale's encoding, which cannot spell the text: a plan written under + another locale of one byte to a character, Latin-1 say, recorded + other bytes. A surrogate no byte is read as cannot be those bytes + either, and no name on the disk holds it: it is asked as written.""" + try: + os.fsencode(recorded) + return recorded, "" + except UnicodeEncodeError: + pass + try: + spelling = os.fsdecode(recorded.encode("utf-8", "surrogateescape")) + except UnicodeEncodeError: + return recorded, "" + return spelling, (f" as UTF-8 spells it (this locale's file names, " + f"{sys.getfilesystemencoding()}, cannot spell it)") + + # --------------------------------------------------------------------------- # What a `--source` names, and where its clone goes (#652) # --------------------------------------------------------------------------- @@ -2012,7 +2058,11 @@ def _source_spec(spec: str) -> tuple[Path | None, str | None]: if _source_there(spec, candidate, name): if name is not None: raise _ambiguous(spec, name) - return _top_level(candidate, f"--source {spec!r}"), None + return _top_level( + candidate, f"--source {spec!r}", + lambda exc: _source_unreadable(f"--source {spec!r}", exc, + "whether a repository is there") + ), None if name is None: raise Refusal( SOURCE_UNRESOLVABLE, @@ -2022,7 +2072,8 @@ def _source_spec(spec: str) -> tuple[Path | None, str | None]: return None, name -def _top_level(path: Path, named: str) -> Path: +def _top_level(path: Path, named: str, + refused: Callable[[OSError], Refusal]) -> Path: """`path`, a local source that exists, resolved, where it is the top-level directory of a repository; refused as `source-not-top-level` where it is not a directory, or is a directory inside a repository @@ -2047,13 +2098,26 @@ def _top_level(path: Path, named: str) -> Path: a later read refused it on main. On main a directory inside a repository was read as the source: `git ls-tree` there lists what is under it alone, so `plan` planned that, exit 0, and the plan recorded - it.""" + it. + + A DIRECTORY THIS USER CANNOT ENTER IS `refused()`, the + caller's `source-unreadable`, before git reads anything in it (#918, + #457's L1-105): git runs IN the directory, and at mode 000 `subprocess` + raised the `PermissionError` of `cwd=` out of `plan`, `check` and + `execute`, a traceback. No other `OSError` is: a git that cannot be + started is raised, as `git_out` leaves it (`_refused_at`).""" resolved = Path(os.path.realpath(path)) if not resolved.is_dir(): raise Refusal(SOURCE_NOT_TOP_LEVEL, f"{named} is not a directory, and {TOP_LEVEL_ONLY}", TOP_LEVEL_REMEDY) - if git_out(["rev-parse", "--is-inside-git-dir"], cwd=resolved) == "true": + try: + inside = git_out(["rev-parse", "--is-inside-git-dir"], cwd=resolved) + except OSError as exc: + if not _refused_at(exc, resolved): + raise + raise refused(exc) from exc + if inside == "true": git_dir = os.fsdecode(git_out(["rev-parse", "--absolute-git-dir"], cwd=resolved, binary=True).rstrip(b"\n")) if Path(os.path.realpath(git_dir)) != resolved: @@ -2421,13 +2485,22 @@ def _not_a_whole_clone(path: Path, commit: str) -> str | None: another repository, which git reads as that one; the third a removed pack, which a clone holds all its objects in. Only what a cleaner leaves is asked: whether every object is whole is `git fsck`'s question, - which a large source would pay for on every run.""" + which a large source would pay for on every run. + + A DIRECTORY THIS USER CANNOT ENTER IS NEITHER (#918, #457's L1-105): the + `OSError` of `cwd=` (`_refused_at`) is raised, for the caller to name. + It was "git finds no repository there", and the clone, still there, was + cloned again or refused as gone.""" if not path.is_dir(): return "it is not a directory" try: top = os.fsdecode(git_out(["rev-parse", "--show-toplevel"], cwd=path, binary=True).strip()) - except (Refusal, OSError): + except Refusal: + return "git finds no repository there" + except OSError as exc: + if _refused_at(exc, path): + raise return "git finds no repository there" if os.path.realpath(top) != os.path.realpath(path): return f"git reads it as part of the repository at {_spelled(top)}" @@ -6360,12 +6433,14 @@ def _keep_the_clone_at_the_record(self, made: list | None) -> None: is at the plan's own `source.local_path` (#803, #457's L1-97): it is the plan's source again, as the clone `_recorded_source` makes there is. On main `check` removed it, and the next run cloned it once - more.""" + more. The record is spelled as `_recorded_source` asks the disk for + it (`_recorded_spelling`, #918).""" record = self.source.get("local_path") if not (made and record): return try: - at = os.path.realpath(os.path.expanduser(str(record))) + at = os.path.realpath(os.path.expanduser( + _recorded_spelling(str(record))[0])) except (OSError, ValueError): return made[:] = [claim for claim in made @@ -6434,34 +6509,52 @@ def _recorded_source(self, local_path: str, work_root: Path, raised it, a `PermissionError` traceback; on 3.14 it answered False, so the path was `plan-source-gone`, "names nothing that exists", or, a clone `plan` made, "no longer exists" and was cloned again from - GitHub beside the one still there.""" - path = Path(os.path.expanduser(local_path)) - try: - there = _is_there(path) - except OSError as exc: - raise _source_unreadable( - f"the plan's `source.local_path`, {_spelled(local_path)}", exc, + GitHub beside the one still there. So is a directory this user + cannot enter (#918): git runs in it, and at mode 000 that was a + `PermissionError` traceback where `plan` did not make it, and "no + longer whole (git finds no repository there)" where it did. + + A PATH THIS LOCALE CANNOT SPELL IS ASKED AS ITS UTF-8 BYTES + (`_recorded_spelling`, #918): a plan written in UTF-8 mode for + `Thing\\u00e9` and checked under `LC_ALL=C PYTHONUTF8=0` was + `plan-source-gone`, "names nothing that exists", of a source that is + there. It is named as the plan records it.""" + spelling, said = _recorded_spelling(local_path) + path = Path(os.path.expanduser(spelling)) + named = f"the plan's `source.local_path`, {_spelled(local_path)}" + + def unreadable(exc: OSError) -> Refusal: + return _source_unreadable( + named, exc, ("whether `plan`'s clone is still there, to be read, or gone, " "to be cloned again," if self.source.get("cloned") is True else "whether the source is still there"), "pass the source again with --source, which is held to the " - "plan's `commit:`") from exc + "plan's `commit:`") + + try: + there = _is_there(path) + except OSError as exc: + raise unreadable(exc) from exc if self.source.get("cloned") is not True: if there: - return Source.in_place(_top_level( - path, f"the plan's `source.local_path`, " - f"{_spelled(local_path)},"), work_root, self.read_at) - raise self._gone(local_path, "names nothing that exists", + return Source.in_place(_top_level(path, f"{named},", + unreadable), + work_root, self.read_at) + raise self._gone(local_path, "names nothing that exists" + said, "the plan does not record it as a clone `plan` " "made (`source.cloned` is not `true`), so " "nothing clones it again") - broken = _not_a_whole_clone(path, self.source_commit) if there \ - else None + try: + broken = _not_a_whole_clone(path, self.source_commit) if there \ + else None + except OSError as exc: + raise unreadable(exc) from exc if there and broken is None: return Source.in_place(Path(os.path.realpath(path)), work_root, self.read_at) state = (f"is a clone `plan` made that is no longer whole ({broken})" - if there else "no longer exists") + if there else "no longer exists" + said) repository = self._recorded_repository() if repository is None: raise self._gone(local_path, state, "the plan records no " @@ -16470,8 +16563,9 @@ def main(argv: list[str] | None = None) -> int: "and one that is neither a path that exists nor " "an `org/repo`; `source-unreadable`, a path this " "user cannot reach (under a directory it may not " - "search), which is never read as one that is not " - "there, nor cloned as an `org/repo`.") + "search, or one it may not enter), which is never " + "read as one that is not there, nor cloned as an " + "`org/repo`.") plan.add_argument("--project", required=True, help="the assembly-root name; for an in-place adoption " "it is the source repository's own name. " diff --git a/docs/cli.md b/docs/cli.md index 3ca5f67e..87a225e3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -461,9 +461,9 @@ options: `source-unresolvable` refuses an empty value, and one that is neither a path that exists nor an `org/repo`; `source-unreadable`, a path this user cannot reach - (under a directory it may not search), which is never - read as one that is not there, nor cloned as an - `org/repo`. + (under a directory it may not search, or one it may + not enter), which is never read as one that is not + there, nor cloned as an `org/repo`. --project PROJECT the assembly-root name; for an in-place adoption it is the source repository's own name. `execute` refuses one over 95 characters without --local-remote-dir: diff --git a/scripts/family.py b/scripts/family.py index 82270793..c43ed105 100755 --- a/scripts/family.py +++ b/scripts/family.py @@ -7671,6 +7671,18 @@ def _member_head(member: Path) -> tuple[str | None, str | None]: #365): the rollback would have taken the holder's commit for the member's. A `.git` whose `lstat` is refused is raised (`_is_there`, #430), not read as no checkout. + + THE BRANCH IS READ AS THE BYTES GIT WROTE, as `_holder_branch` reads the + holder's (#918, #457's L1-70): exactly one newline and exactly + `refs/heads/` come off the bytes, and the rest is decoded with + `surrogateescape`, as #425 decodes a source's default branch. Read + strictly, a member checked out on a branch that is not UTF-8 + (`caf<0xE9>`) raised `UnicodeDecodeError` out of `PinRollback`, so + `bump` and `remove` ended in a traceback before they wrote anything. + Kept as a lone surrogate, the name is handed back to git as the byte it + is (`_head_target`, `_head_back`), so a rollback puts the member back + ON that branch. And `strip()` no longer takes a trailing U+00A0 off a + branch's name, which named another branch (#765). """ if not _is_there(member / ".git"): return None, None @@ -7680,11 +7692,13 @@ def _member_head(member: Path) -> tuple[str | None, str | None]: except Refusal: at = None try: - ref = git_out(["symbolic-ref", "-q", "HEAD"], cwd=member) + ref = git_out(["symbolic-ref", "-q", "HEAD"], cwd=member, binary=True) except Refusal: - ref = "" - return at, (ref[len(HEADS_PREFIX):] if ref.startswith(HEADS_PREFIX) - else None) + ref = b"" + ref = ref[:-1] if ref.endswith(b"\n") else ref + heads = HEADS_PREFIX.encode("ascii") + return at, (ref[len(heads):].decode("utf-8", "surrogateescape") + if ref.startswith(heads) else None) def _head_target(member: Path, at: str, branch: str | None) -> list[str]: @@ -8118,7 +8132,7 @@ def _undo_work_tree(self) -> str | None: def _head_said(self) -> str: at = (self.at or "")[:12] if self.branch is not None: - return f"{self.path} back on {self.branch} at {at}" + return f"{self.path} back on {_shown(self.branch)} at {at}" return f"{self.path} checked out at {at} again" def _work_tree_said(self) -> str: @@ -8127,7 +8141,7 @@ def _work_tree_said(self) -> str: if self.at is None: return f"{self.path} checked out again" said = (f"{self.path} checked out again at {self.at[:12]}" - + (f" on {self.branch}" if self.branch else "") + + (f" on {_shown(self.branch)}" if self.branch else "") + " from the object store `git rm` keeps") if self.legs: said += (", and " + " and ".join(leg for leg, _, _, _ in self.legs) diff --git a/tests/test_adopt_unenterable_source_and_locale.py b/tests/test_adopt_unenterable_source_and_locale.py new file mode 100644 index 00000000..08e650ad --- /dev/null +++ b/tests/test_adopt_unenterable_source_and_locale.py @@ -0,0 +1,375 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`adopt-project.py`: a `--source`, or a plan's `source.local_path`, that +this user cannot enter is refused by name, and a recorded path this locale +cannot spell is asked as the UTF-8 bytes the plan holds (#918, #457's L1-105 +and L1-86). + +ON 00f6d7f7 (2026-10-20), Python 3.12.3, a non-root user, a stub `gh` that +refuses every call and local remotes: + +- L1-105. `plan --source `, `` itself at mode 000 (one under a + parent at mode 000 is #430's slice H, `source-unreadable` already): a + `PermissionError` traceback from `git rev-parse` in `_top_level`, exit 1. + `check` of a plan whose `local_path` is at mode 000: the same traceback; + recorded as a clone `plan` made, `plan-source-gone`, "a clone `plan` made + that is no longer whole (git finds no repository there)", or, with a + `source.repository`, a NOTE and `gh repo clone` of a clone still there. +- L1-86. A plan written in UTF-8 mode for a source directory `Thing\\u00e9`, + checked under `LC_ALL=C PYTHONUTF8=0`: `plan-source-gone`, "names nothing + that exists", exit 2, of a source that is there, and `plan ok` in UTF-8 + mode. + +Now the first is `source-unreadable`, naming the path and the operating +system's reason, with `chmod u+rx` as the cure, before anything is cloned or +written; and the second reads the source, `plan ok`, as UTF-8 mode does. +Where the UTF-8 bytes name nothing either, `plan-source-gone` says it asked +them and names the locale's encoding, which cannot spell the path. + +A real mode runs in a child, skipped on Windows and as root +(`needs_a_mode`); the same questions are asked in-process with git's +`cwd=` refused, which runs everywhere. A C locale is a POSIX idea, so the +locale's tests skip where `c_locale_or_skip` finds none. +""" + +from __future__ import annotations + +import errno +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import ADOPT, git, make_source_repo, run_script, spelled +from test_adopt_c_locale_bytes import in_the_locale +from test_adopt_git_rm_bytes import C_LOCALE, c_locale_or_skip +from test_adopt_temp_work_roots import (PLANNED, PROJECT, SMALL_TREE, + planned_from_github, recorded) +from test_adopt_unsearchable_source import ( # noqa: F401 + CURE, DENIED, UNREADABLE, URL, adopter, assert_no_false_answer, + needs_a_mode, planned_locally, recording_gh, run, said, unsearchable, +) + +#: What a refused `cwd=` was read as for a clone `plan` made, on 00f6d7f7. +NO_REPOSITORY = "git finds no repository there" +#: The way past a recorded source `source-unreadable` names. +INSTEAD = ("or pass the source again with --source, which is held to the " + "plan's `commit:`.") +#: What `source-unreadable` cannot tell of a recorded source, by +#: `source.cloned`. +UNTOLD = {False: "whether the source is still there", + True: "whether `plan`'s clone is still there, to be read, or gone, " + "to be cloned again,"} +#: A source directory outside ASCII: `Thing` and U+00E9, as #918 measured. +ACCENTED = "Thing\u00e9" +#: UTF-8 mode under the C locale: how the plan is written. +UTF8_MODE = {"LC_ALL": "C", "PYTHONUTF8": "1"} + +#: The child of `test_the_clone_at_the_record_is_kept_under_the_c_locale`: +#: `Plan._keep_the_clone_at_the_record` of a plan recording argv[2], a JSON +#: string, for the clones whose paths are the hex of their bytes in argv[3:] +#: (every argument is ASCII, as the locale under test spells nothing else). +#: It prints one line of ASCII JSON: whether this locale spells the record, +#: and the hex of each clone left. +KEEP_THE_CLONE = r""" +import importlib.util, json, os, sys +from pathlib import Path +from types import SimpleNamespace +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location( + "adopt_keep_the_clone", os.fsdecode(bytes.fromhex(sys.argv[1]))) +adopter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(adopter) +record = json.loads(sys.argv[2]) +made = [] +for spelled in sys.argv[3:]: + at = os.fsdecode(bytes.fromhex(spelled)) + made.append(adopter._Claim(Path(at), at, (0, 0), (0, 0), False)) +try: + os.fsencode(record) + spells = True +except UnicodeEncodeError: + spells = False +adopter.Plan._keep_the_clone_at_the_record( + SimpleNamespace(source={"local_path": record}), made) +print(json.dumps({"spells": spells, + "left": [os.fsencode(c.made_at).hex() for c in made]})) +""" + + +def refused_cwd(adopter, monkeypatch, directory: str, + error: OSError | None = None) -> None: + """`git_out` run with `cwd=` raises what `subprocess` raises + for a directory this user cannot enter, `PermissionError` naming it; + or `error`, for every call. Every other call is git's own.""" + real = adopter.git_out + + def refusing(args, cwd, *rest, **kwargs): + if error is not None: + raise error + if os.path.realpath(cwd) == os.path.realpath(directory): + raise PermissionError(errno.EACCES, "Permission denied", str(cwd)) + return real(args, cwd, *rest, **kwargs) + + monkeypatch.setattr(adopter, "git_out", refusing) + + +# --- L1-105: a `--source` this user cannot enter ---------------------------- + +@needs_a_mode +def test_a_source_this_user_cannot_enter_is_refused_by_name(tmp_path): + """`--source` names a repository at mode 000 itself: a traceback on + 00f6d7f7. Now `source-unreadable`, with the cure; no plan is written, + nothing is cloned and `gh` is never asked.""" + source = make_source_repo(tmp_path / PROJECT, tree=SMALL_TREE, edits=()) + plan = tmp_path / "adoption-plan.yaml" + env, log = recording_gh(tmp_path / "gh-bin") + + with unsearchable(source): + proc = run_script(ADOPT, "plan", "--source", str(source), *PLANNED, + "--out", str(plan), "--work-dir", + str(tmp_path / "work"), env=env) + + assert proc.returncode == 2, said(proc) + assert (f"{UNREADABLE}--source {str(source)!r}: {DENIED}, so whether a " + "repository is there cannot be told") in proc.stderr + assert CURE in proc.stderr + assert_no_false_answer(said(proc)) + assert not plan.exists() + assert not (tmp_path / "work" / "source-readonly").exists() + assert log.read_bytes() == b"" + + +def test_a_source_git_cannot_enter_is_refused_on_every_platform( + adopter, monkeypatch, tmp_path): + """The same, in-process, with the `cwd=` of git refused, so it runs as + root and on Windows too.""" + source = make_source_repo(tmp_path / PROJECT, tree=SMALL_TREE, edits=()) + refused_cwd(adopter, monkeypatch, str(source)) + + with pytest.raises(adopter.Refusal) as raised: + adopter._source_spec(str(source)) + + assert raised.value.code == "source-unreadable" + assert raised.value.detail == (f"--source {str(source)!r}: {DENIED}, so " + "whether a repository is there cannot be " + "told") + assert CURE in raised.value.remediation + + +def test_a_git_that_cannot_be_started_is_not_read_as_the_source( + adopter, monkeypatch, tmp_path): + """Only the directory's own refusal is `source-unreadable`: a git that + cannot be started names the program, not the directory, and is raised + as `git_out` leaves it, never said of the source.""" + source = make_source_repo(tmp_path / PROJECT, tree=SMALL_TREE, edits=()) + refused_cwd(adopter, monkeypatch, str(source), FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), "git")) + + with pytest.raises(FileNotFoundError): + adopter._source_spec(str(source)) + + +# --- L1-105: a recorded source this user cannot enter ----------------------- + +@needs_a_mode +@pytest.mark.parametrize(("verb", "cloned"), [ + ("check", False), ("check", True), ("execute", True), +]) +def test_a_recorded_source_this_user_cannot_enter_is_refused_by_name( + tmp_path, verb, cloned): + """The plan's `source.local_path` at mode 000: on 00f6d7f7 a traceback + where `plan` was given the path, and where it is recorded as a clone + `plan` made, "no longer whole (git finds no repository there)", so the + plan's `source.repository` was cloned again, `gh` asked. Now + `source-unreadable` in `check` and `execute` alike, nothing cloned and + `gh` never asked.""" + source = make_source_repo(tmp_path / PROJECT, tree=SMALL_TREE, edits=()) + git("remote", "add", "origin", f"{URL}.git", cwd=source) + plan = planned_locally(tmp_path, source, cloned) + local_path = str(recorded(plan)["local_path"]) + env, log = recording_gh(tmp_path / "gh-bin") + work = tmp_path / f"{verb}-work" + + with unsearchable(source): + proc = run_script(ADOPT, verb, "--plan", str(plan), "--work-dir", + str(work), "--local-remote-dir", + str(tmp_path / "remotes"), + *(["--yes"] if verb == "execute" else []), env=env) + + assert proc.returncode == 2, said(proc) + assert (f"{UNREADABLE}the plan's `source.local_path`, " + f"{spelled(local_path)}: {DENIED}, so {UNTOLD[cloned]} cannot be " + "told") in proc.stderr + assert CURE in proc.stderr + assert INSTEAD in proc.stderr + assert "plan-source-gone" not in said(proc) + assert NO_REPOSITORY not in said(proc) + assert_no_false_answer(said(proc)) + assert log.read_bytes() == b"" + assert not (work / "source-readonly").exists() + assert not (tmp_path / "remotes").exists() + + +def test_a_recorded_clone_git_cannot_enter_is_not_cloned_again( + adopter, monkeypatch, capsys, tmp_path): + """`plan --source testorg/Thing` records its clone, and git cannot enter + it: on 00f6d7f7 it was "no longer whole (git finds no repository + there)", and cloned again from GitHub. Now `source-unreadable`, and no + second clone is asked for. In-process, so it runs everywhere.""" + planned = planned_from_github(adopter, monkeypatch, tmp_path) + assert len(planned.calls) == 1, planned.calls + refused_cwd(adopter, monkeypatch, planned.clone) + + code, out, err = run(adopter, capsys, "check", "--plan", + str(planned.plan), "--work-dir", + str(tmp_path / "check-work"), "--local-remote-dir", + str(tmp_path / "remotes")) + + assert code == 2, out + err + assert (f"{UNREADABLE}the plan's `source.local_path`, " + f"{spelled(planned.clone)}: {DENIED}, so {UNTOLD[True]} cannot be " + "told") in err + assert NO_REPOSITORY not in out + err + assert_no_false_answer(out + err) + assert len(planned.calls) == 1, planned.calls + + +def test_a_clone_whose_git_cannot_be_started_is_still_no_repository( + adopter, monkeypatch, tmp_path): + """`_not_a_whole_clone` raises only the directory's own refusal: a git + that cannot be started is what it always was, no repository there.""" + clone = make_source_repo(tmp_path / PROJECT, tree=SMALL_TREE, edits=()) + refused_cwd(adopter, monkeypatch, str(clone), FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), "git")) + + assert adopter._not_a_whole_clone(clone, "") == NO_REPOSITORY + + +# --- L1-86: a plan written in UTF-8 mode, checked under the C locale --------- + +def written_in_utf8_mode(monkeypatch, tmp_path: Path, cloned: bool + ) -> tuple[Path, Path, Path, dict]: + """A source at `/Thing\\u00e9`, its `origin` on GitHub, and the plan + `plan` writes for it in UTF-8 mode, recorded as a clone `plan` made + where `cloned` says so: `(source, plan, gh log, gh environment)`.""" + in_the_locale(monkeypatch, UTF8_MODE) + source = make_source_repo(tmp_path / ACCENTED, tree=SMALL_TREE, edits=()) + git("remote", "add", "origin", f"{URL}.git", cwd=source) + env, log = recording_gh(tmp_path / "gh-bin") + plan = tmp_path / "adoption-plan.yaml" + written = run_script(ADOPT, "plan", "--source", str(source), *PLANNED, + "--out", str(plan), "--work-dir", + str(tmp_path / "plan-work"), env=env) + assert written.returncode == 0, said(written) + assert recorded(plan)["local_path"].endswith(f"/{ACCENTED}"), ( + "premise: the plan holds the character itself") + if cloned: + text = plan.read_bytes() + assert text.count(b"\n cloned: false\n") == 1, text + plan.write_bytes(text.replace(b"\n cloned: false\n", + b"\n cloned: true\n")) + return source, plan, log, env + + +def checked(plan: Path, tmp_path: Path, name: str, env: dict): + """`check --plan ` in a child, in the locale this test set.""" + return run_script(ADOPT, "check", "--plan", str(plan), "--work-dir", + str(tmp_path / name), "--local-remote-dir", + str(tmp_path / "remotes"), env=env) + + +@pytest.mark.parametrize("cloned", [False, True], ids=["given", "cloned"]) +def test_a_plan_written_in_utf8_mode_is_read_under_the_c_locale( + monkeypatch, tmp_path, cloned): + """#918's measurement: on 00f6d7f7 `plan-source-gone`, "names nothing + that exists", and, recorded as a clone `plan` made, "no longer exists" + and `gh repo clone` asked. Now `check` reads the source where it is, + `plan ok`, exit 0, as it does in UTF-8 mode, and `gh` is never asked.""" + _, plan, log, env = written_in_utf8_mode(monkeypatch, tmp_path, cloned) + utf8 = checked(plan, tmp_path, "utf8-work", env) + in_the_locale(monkeypatch, C_LOCALE) + + proc = checked(plan, tmp_path, "c-work", env) + + assert utf8.returncode == 0, said(utf8) + assert proc.returncode == 0, said(proc) + assert "plan ok" in proc.stdout + assert "Traceback" not in proc.stderr + assert "plan-source-gone" not in said(proc) + assert "no longer exists" not in said(proc) + assert log.read_bytes() == b"" + + +def test_a_plan_path_this_locale_cannot_spell_and_utf8_does_not_find( + monkeypatch, tmp_path): + """The source moved away after `plan`: under the C locale its UTF-8 + bytes name nothing either, and `plan-source-gone` says it asked them, + naming this locale's encoding; in UTF-8 mode, which spells the path as + written, the refusal is as it always was.""" + source, plan, log, env = written_in_utf8_mode(monkeypatch, tmp_path, + False) + source.rename(tmp_path / "moved") + utf8 = checked(plan, tmp_path, "utf8-work", env) + in_the_locale(monkeypatch, C_LOCALE) + encoding = subprocess.run( + [sys.executable, "-c", + "import sys; print(sys.getfilesystemencoding())"], + capture_output=True, text=True, check=True, + stdin=subprocess.DEVNULL).stdout.strip() + + proc = checked(plan, tmp_path, "c-work", env) + + gone = "REFUSED plan-source-gone: the plan's `source.local_path`, " + rest = (", and the plan does not record it as a clone `plan` made " + "(`source.cloned` is not `true`), so nothing clones it again") + assert utf8.returncode == 2, said(utf8) + assert f"`, names nothing that exists{rest}" in utf8.stderr + assert gone in utf8.stderr + assert proc.returncode == 2, said(proc) + assert gone in proc.stderr + assert (f"`, names nothing that exists as UTF-8 spells it (this locale's " + f"file names, {encoding}, cannot spell it){rest}") in proc.stderr + assert "Traceback" not in proc.stderr + assert log.read_bytes() == b"" + + +def test_a_surrogate_no_byte_is_read_as_is_asked_as_written(adopter): + """A lone surrogate outside U+DC80 to U+DCFF, which a hand edit can put + in a plan, is no byte `surrogateescape` reads: no name on the disk holds + it, and it is asked as written, as `Path.exists()` asked it.""" + assert adopter._recorded_spelling("Thing\ud800") == ("Thing\ud800", "") + + +def test_the_clone_at_the_record_is_kept_under_the_c_locale(tmp_path): + """`_keep_the_clone_at_the_record` (#803, #918): `check --source` takes + the clone this run made off its list of clones to remove where it is at + the plan's own `source.local_path`. A plan written in UTF-8 mode records + `Thing\\u00e9` as itself, which `LC_ALL=C PYTHONUTF8=0` cannot spell: if + the record is asked as written, `os.path.realpath` meets the + `UnicodeEncodeError` and the method returns, the clone at the record + stays on the list and `check` removes it, #803's bug again under the C + locale. Asked as the UTF-8 bytes the plan holds, that one clone is taken + off, and a clone made elsewhere stays. The method runs in a child that + is the C locale itself, with a real directory at the record.""" + env = c_locale_or_skip() + base = Path(os.path.realpath(tmp_path)) + clone = base / ACCENTED / "source-readonly" / "Thing" + elsewhere = base / "work" / "source-readonly" / "Thing" + for directory in (clone, elsewhere): + directory.mkdir(parents=True) + + child = subprocess.run( + [sys.executable, "-c", KEEP_THE_CLONE, os.fsencode(ADOPT).hex(), + json.dumps(str(clone)), os.fsencode(clone).hex(), + os.fsencode(elsewhere).hex()], + capture_output=True, env=env, stdin=subprocess.DEVNULL, check=False) + + assert child.returncode == 0, child.stderr.decode("utf-8", "replace") + answer = json.loads(child.stdout.decode("ascii")) + assert answer["spells"] is False, "premise: this locale cannot spell it" + left = [os.fsdecode(bytes.fromhex(held)) for held in answer["left"]] + assert left == [str(elsewhere)], ( + f"the clone at the record is not taken off: {left}") diff --git a/tests/test_family_member_branch_not_utf8.py b/tests/test_family_member_branch_not_utf8.py new file mode 100644 index 00000000..8cbb8c8e --- /dev/null +++ b/tests/test_family_member_branch_not_utf8.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`scripts/family.py`: a member checked out on a branch that is not UTF-8 is +read as the bytes git wrote, so `bump` and `remove` run, and a rollback puts +the member back ON that branch (#918, #457's L1-70). + +ON 00f6d7f7 (2026-10-20): `_member_head` read `git symbolic-ref -q HEAD` +through `git_out`, which decodes UTF-8 strictly, and caught only `Refusal`. +`PinRollback` records the member's HEAD before the first write, so for a +member on `caf<0xE9>` it raised `UnicodeDecodeError`: `bump` and `remove` +ended in a traceback, exit 1, before they wrote anything. Now the name is +read as `_holder_branch` reads the holder's (#765): one newline and +`refs/heads/` off the bytes, the rest decoded with `surrogateescape`, as +#425 decodes a source's default branch; and git is handed it back as the +byte it is. + +The family, its offline environment and the `pre-commit` hook that refuses +are `tests/test_family_bump_remove_rollback.py`'s: the hook refuses the one +commit after the first write, so each verb's rollback runs, and the member +must come back on its branch at its commit. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from conftest import (NOT_UTF_8_BRANCH, git, git_answer_bytes, + on_a_branch_that_is_not_utf_8) +from test_family import family, family_module # noqa: F401 +from test_family_add_refusals import holder, offline # noqa: F401 +from test_family_bump_remove_rollback import ( + BUMP_PUT_BACK, REMOVE_PUT_BACK, assert_same, bump, refusing_hook, remove, + snapshot, +) +from test_family_dry_run import isolated +from test_family_member_path import MEMBER, older + +PATH = f"members/{MEMBER}" +#: The ref a checkout on `NOT_UTF_8_BRANCH` is attached to, as git prints it. +ATTACHED = (b"refs/heads/" + NOT_UTF_8_BRANCH.encode("utf-8", + "surrogateescape") + + b"\n") +#: How each verb's sentence begins once its rollback has put everything +#: back, and what it puts back. +UNDONE = {"bump": ("Nothing was bumped: the bump has been undone", + BUMP_PUT_BACK), + "remove": ("Nothing was removed: the unmount has been undone", + REMOVE_PUT_BACK)} + + +@pytest.mark.parametrize("verb", ["bump", "remove"]) +def test_a_member_on_a_branch_that_is_not_utf8_is_put_back_on_it( + family, holder, offline, tmp_path, verb): + """The member's checkout is on `caf<0xE9>` at the commit the holder + pins, and the holder's `pre-commit` hook refuses: on 00f6d7f7 a + `UnicodeDecodeError` traceback, exit 1. Now each verb runs to the + commit the hook refuses, exit 2, and its rollback puts the holder back + as it was and the member back on `caf<0xE9>`, attached, at its + commit; and the sentence names the branch as `_shown` does, U+FFFD for + the byte, not as the lone surrogate stderr writes with + `backslashreplace`.""" + env = isolated(offline, tmp_path) + mount = holder / PATH + on_a_branch_that_is_not_utf_8(mount) + at = git_answer_bytes(mount, "rev-parse", "HEAD").decode("ascii").strip() + refusing_hook(holder, tmp_path) + before = snapshot(holder, env) + begins, put_back = UNDONE[verb] + + result = (bump(holder, env, older(family)) if verb == "bump" + else remove(holder, env)) + + assert "Traceback" not in result.stderr, result.stderr + assert result.returncode == 2, result.stdout + result.stderr + sentence = result.stderr.splitlines()[-1] + assert sentence.startswith(begins), sentence + assert " on caf" in sentence and at[:12] in sentence, sentence + # The branch as a refusal shows a name read with `surrogateescape` + # (`_shown`, U+FFFD), never as the lone surrogate `backslashreplace` + # writes to stderr, `caf` and a backslash and `udce9`. + assert " on caf\ufffd" in sentence, sentence + assert "\\udce9" not in sentence, sentence + assert git_answer_bytes(mount, "symbolic-ref", "-q", "HEAD") == ATTACHED + assert git_answer_bytes(mount, "rev-parse", "HEAD").decode( + "ascii").strip() == at + assert_same(before, snapshot(holder, env), put_back) + + +def checkout_on(tmp_path: Path, branch: str) -> tuple[Path, str]: + """A repository of one commit, switched to a new branch `branch`, made + as `on_a_branch_that_is_not_utf_8` makes one where it is that branch: + `(it, its commit)`.""" + repository = tmp_path / "member" + repository.mkdir() + git("init", "-q", "-b", "main", cwd=repository) + git("-c", "user.name=T", "-c", "user.email=t@example.invalid", "commit", + "-q", "--allow-empty", "-m", "one", cwd=repository) + if branch == NOT_UTF_8_BRANCH: + on_a_branch_that_is_not_utf_8(repository) + else: + subprocess.run(["git", "switch", "-q", "-c", branch], + cwd=str(repository), capture_output=True, check=True) + return repository, git("rev-parse", "HEAD", cwd=repository + ).stdout.strip() + + +@pytest.mark.parametrize("branch", [ + pytest.param(NOT_UTF_8_BRANCH, id="not-utf8"), + pytest.param("feat\u00a0", id="trailing-no-break-space"), +]) +def test_the_member_head_is_the_branch_git_wrote(tmp_path, branch): + """`_member_head` reads the branch git wrote: a byte that is not UTF-8 + as its surrogate, where it raised `UnicodeDecodeError`; and a trailing + U+00A0, which `strip()` took off, so the rollback named `feat`.""" + repository, at = checkout_on(tmp_path, branch) + + assert family_module()._member_head(repository) == (at, branch)