diff --git a/adopt-project.py b/adopt-project.py index 169ad39f..0c0f31b7 100755 --- a/adopt-project.py +++ b/adopt-project.py @@ -1818,20 +1818,64 @@ def _recorded_spelling(recorded: str) -> tuple[str, str]: locale's encoding, which cannot spell the text: a plan written under another locale of one byte to a character, Latin-1 say, recorded other bytes. A surrogate no byte is read as cannot be those bytes - either, and no name on the disk holds it: it is asked as written.""" + either, and no name on the disk holds it: it is asked as written. + Where this locale spells the text otherwise than its UTF-8, as Latin-1 + spells `\\u00e9` as one byte, `_recorded_path` asks those bytes too + (#1078).""" try: os.fsencode(recorded) return recorded, "" except UnicodeEncodeError: pass - try: - spelling = os.fsdecode(recorded.encode("utf-8", "surrogateescape")) - except UnicodeEncodeError: + spelling = _utf8_spelling(recorded) + if spelling is None: return recorded, "" return spelling, (f" as UTF-8 spells it (this locale's file names, " f"{sys.getfilesystemencoding()}, cannot spell it)") +def _utf8_spelling(recorded: str) -> str | None: + """`recorded`'s UTF-8 bytes, a byte that is not UTF-8 as its surrogate + (`surrogateescape`), as `os.fsdecode` spells them in this locale: the + name a plan written in UTF-8 records (#918, #1078). None for a surrogate + no byte is read as, which no name on the disk holds, and on Windows for + a byte that is not UTF-8, which `os.fsdecode` refuses there.""" + try: + return os.fsdecode(recorded.encode("utf-8", "surrogateescape")) + except UnicodeError: + return None + + +def _recorded_path(recorded: str) -> tuple[Path, bool, str]: + """`(path, there, said)`: `recorded`, a path a plan records as text, as + `_recorded_spelling` spells it, `~` expanded; whether anything is there, + asked as `_is_there` asks it, so a `stat` the operating system refused + is raised (#430); and `_recorded_spelling`'s `said`. + + WHERE THAT NAMES NOTHING, THE PLAN'S UTF-8 BYTES ARE ASKED TOO (#1078). + Under a locale of one byte to a character, Latin-1 say, `os.fsencode` + spells `Thing\\u00e9` as written, its last character as the one byte + 0xE9, so `_recorded_spelling` asks it as written; but a plan written in + UTF-8 holds the name `plan` read there, whose bytes are 0xC3 0xA9, and + `check` refused `plan-source-gone`, "names nothing that exists", of a + source that is there. The spelling as written is asked FIRST, so a plan + written under that locale, which records the name as that locale read + it, reads as it did. Where neither names anything, the path is the + spelling as written, and every word of a refusal is as before. Under a + UTF-8 locale, and for text that is all ASCII, the two are one name, + asked once.""" + spelling, said = _recorded_spelling(recorded) + path = Path(os.path.expanduser(spelling)) + if _is_there(path): + return path, True, said + utf8 = _utf8_spelling(recorded) + if utf8 is not None and utf8 != spelling: + other = Path(os.path.expanduser(utf8)) + if _is_there(other): + return other, True, said + return path, False, said + + # --------------------------------------------------------------------------- # What a `--source` names, and where its clone goes (#652) # --------------------------------------------------------------------------- @@ -6482,13 +6526,12 @@ def _keep_the_clone_at_the_record(self, made: list | None) -> None: the plan's source again, as the clone `_recorded_source` makes there is. On main `check` removed it, and the next run cloned it once more. The record is spelled as `_recorded_source` asks the disk for - it (`_recorded_spelling`, #918).""" + it (`_recorded_path`, #918, #1078).""" record = self.source.get("local_path") if not (made and record): return try: - at = os.path.realpath(os.path.expanduser( - _recorded_spelling(str(record))[0])) + at = os.path.realpath(_recorded_path(str(record))[0]) except (OSError, ValueError): return made[:] = [claim for claim in made @@ -6566,9 +6609,10 @@ def _recorded_source(self, local_path: str, work_root: Path, (`_recorded_spelling`, #918): a plan written in UTF-8 mode for `Thing\\u00e9` and checked under `LC_ALL=C PYTHONUTF8=0` was `plan-source-gone`, "names nothing that exists", of a source that is - there. It is named as the plan records it.""" - spelling, said = _recorded_spelling(local_path) - path = Path(os.path.expanduser(spelling)) + there. It is named as the plan records it. A path this locale + spells otherwise than UTF-8, as Latin-1 does, is asked as written + and, where that names nothing, as the plan's UTF-8 bytes too + (`_recorded_path`, #1078): it was `plan-source-gone` too.""" named = f"the plan's `source.local_path`, {_spelled(local_path)}" def unreadable(exc: OSError) -> Refusal: @@ -6581,7 +6625,7 @@ def unreadable(exc: OSError) -> Refusal: "plan's `commit:`") try: - there = _is_there(path) + path, there, said = _recorded_path(local_path) except OSError as exc: raise unreadable(exc) from exc if self.source.get("cloned") is not True: diff --git a/tests/test_adopt_one_byte_locale.py b/tests/test_adopt_one_byte_locale.py new file mode 100644 index 00000000..7825e785 --- /dev/null +++ b/tests/test_adopt_one_byte_locale.py @@ -0,0 +1,242 @@ +# SPDX-License-Identifier: Apache-2.0 +"""`adopt-project.py check`: a plan written in UTF-8 is read under a locale +of one byte to a character, Latin-1 say, where its `source.local_path` is +asked as written and, where that names nothing, as the plan's UTF-8 bytes +too (#1078, found in the review of #1072). + +ON 2a119f28 (2026-10-21), Python 3.12.3, a Latin-1 locale built with +`localedef` (`sys.getfilesystemencoding()` `iso8859-1`), `PYTHONUTF8=0`: a +plan written in UTF-8 mode for a source directory `Thing\\u00e9` was +`plan-source-gone`, "names nothing that exists", exit 2, of a source that +is there; recorded as a clone `plan` made, it "no longer exists" and `gh +repo clone` was asked. `os.fsencode` spells the recorded text in Latin-1, +the last character as the one byte 0xE9, but the name on the disk is the +two bytes 0xC3 0xA9 `plan` read. Under C and POSIX the same plan was `plan +ok` (#1072, which asks the UTF-8 bytes where the locale cannot spell the +text). + +Now `check` reads the source, `plan ok`, under Latin-1 as under C and +POSIX. The spelling as written is still asked first, and where neither +names anything the refusal is word for word as before. + +THE LOCALE IS BUILT HERE, in the module's temporary directory, from a +charmap and a locale source this file writes, so it needs only `localedef` +and a C library that reads `LOCPATH` (glibc): a host without the `locales` +package has no `en_US` source and no charmaps to build one from. ISO-8859-1 +maps each byte to the code point of the same number. The tests skip, with +`localedef`'s own words, where no such locale can be built, and on Windows, +which hands a command line to `CreateProcessW` as UTF-16. +""" + +from __future__ import annotations + +import codecs +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import ADOPT, blank_unnamed_pin_sources, spelled +from test_adopt_git_rm_bytes import C_LOCALE, LOCALE_KEYS, c_locale_or_skip +from test_adopt_temp_work_roots import recorded +from test_adopt_unenterable_source_and_locale import ( + ACCENTED, kept_clones, written_in_utf8_mode, +) + +#: The locale #1078 names, as `LC_ALL` names the one built here. +LATIN1_NAME = "en_US.ISO-8859-1" +#: `codecs.lookup(...).name` of Latin-1, which a child in it must report. +LATIN1 = codecs.lookup("iso8859-1").name +#: The POSIX locale, Python's UTF-8 mode off, as #1072 kept it. +POSIX_LOCALE = {"LC_ALL": "POSIX", "PYTHONUTF8": "0"} +#: A child's file name encoding and its preferred encoding, one per line. +ASK_ENCODINGS = ("import codecs, locale, sys; " + "print(codecs.lookup(sys.getfilesystemencoding()).name); " + "print(codecs.lookup(locale.getpreferredencoding(False))" + ".name)") +#: The child of `test_the_record_is_asked_as_written_then_as_utf8`: +#: `_recorded_path` of the record in argv[2], a JSON string. It prints one +#: line of ASCII JSON: the hex of the path's bytes, whether it is there, +#: `said`, and this locale's file name encoding. +RECORDED_PATH = r""" +import importlib.util, json, os, sys +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location( + "adopt_recorded_path", os.fsdecode(bytes.fromhex(sys.argv[1]))) +adopter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(adopter) +path, there, said = adopter._recorded_path(json.loads(sys.argv[2])) +print(json.dumps({"path": os.fsencode(path).hex(), "there": there, + "said": said, "encoding": sys.getfilesystemencoding()})) +""" + + +def charmap() -> str: + """ISO-8859-1 as `localedef -f` reads a charmap: each byte, `/x00` to + `/xff`, the code point of the same number.""" + return (" ISO-8859-1\n %\n /\n" + " 1\n 1\nCHARMAP\n" + + "".join(f" /x{byte:02x}\n" for byte in range(256)) + + "END CHARMAP\n") + + +@pytest.fixture(scope="module") +def latin1(tmp_path_factory) -> dict: + """What a child's environment adds to run in a Latin-1 locale built + here: `LOCPATH`, `LC_ALL` naming it, and `PYTHONUTF8=0`. Built by + `localedef -c`, which writes a locale whose categories other than + `LC_CTYPE` are empty and says so (exit 1, warnings); a child started in + it must report Latin-1 as its file name encoding and its preferred + encoding, or every test here skips with `localedef`'s words.""" + if os.name == "nt": + pytest.skip("a locale of one byte to a character is a POSIX idea: " + "Windows hands a command line to CreateProcessW as UTF-16") + localedef = shutil.which("localedef") + if localedef is None: + pytest.skip("no `localedef` here to build a Latin-1 locale with") + kit = tmp_path_factory.mktemp("latin1") + (kit / "ISO-8859-1").write_text(charmap(), encoding="ascii") + (kit / "latin1").write_text("LC_CTYPE\nEND LC_CTYPE\n", encoding="ascii") + (kit / "locales").mkdir() + built = subprocess.run( + [localedef, "-c", "-i", str(kit / "latin1"), "-f", + str(kit / "ISO-8859-1"), str(kit / "locales" / LATIN1_NAME)], + capture_output=True, text=True, errors="replace", + stdin=subprocess.DEVNULL, check=False) + locale = {"LOCPATH": str(kit / "locales"), "LC_ALL": LATIN1_NAME, + "PYTHONUTF8": "0"} + env = {key: value for key, value in os.environ.items() + if not key.startswith(LOCALE_KEYS)} + asked = subprocess.run([sys.executable, "-c", ASK_ENCODINGS], + capture_output=True, text=True, errors="replace", + env={**env, **locale}, stdin=subprocess.DEVNULL, + check=False) + if asked.stdout.split() != [LATIN1, LATIN1]: + words = " ".join(line for line in built.stderr.splitlines() + if not line.startswith("[warning]")) + pytest.skip(f"localedef could not build a Latin-1 locale here (exit " + f"{built.returncode}: {words[-300:]!r}); a child in it " + f"reports {asked.stdout.split()}") + return locale + + +def checked(plan: Path, tmp_path: Path, name: str, env: dict): + """`check --plan ` in a child in the locale `env` names, its + output read as Latin-1, which reads every byte: a child in that locale + writes `\\u00e9` as the one byte 0xE9, and a child under C or POSIX + writes ASCII.""" + asked = {**env, "LANES_LANE": ""} + return subprocess.run( + [sys.executable, str(ADOPT), "check", "--plan", str(plan), + "--work-dir", str(tmp_path / name), "--local-remote-dir", + str(tmp_path / "remotes")], + capture_output=True, encoding=LATIN1, + env=blank_unnamed_pin_sources({**os.environ, **asked}, asked), + stdin=subprocess.DEVNULL, check=False) + + +def said(proc) -> str: + """What a child printed, stdout then stderr.""" + return proc.stdout + proc.stderr + + +@pytest.mark.parametrize("cloned", [False, True], ids=["given", "cloned"]) +def test_a_plan_written_in_utf8_is_read_under_a_one_byte_locale( + latin1, monkeypatch, tmp_path, cloned): + """#1078's measurement: on 2a119f28 `plan-source-gone`, "names nothing + that exists", under Latin-1, and, recorded as a clone `plan` made, "no + longer exists" and `gh repo clone` asked. Now `plan ok`, exit 0, under + Latin-1, and under C and POSIX as #1072 left them; `gh` is never + asked.""" + _, plan, log, env = written_in_utf8_mode(monkeypatch, tmp_path, cloned) + + for name, locale in (("latin1", latin1), ("c", C_LOCALE), + ("posix", POSIX_LOCALE)): + proc = checked(plan, tmp_path, f"{name}-work", {**env, **locale}) + + assert proc.returncode == 0, (name, said(proc)) + assert "plan ok" in proc.stdout, name + assert "Traceback" not in proc.stderr, name + assert "plan-source-gone" not in said(proc), name + assert "no longer exists" not in said(proc), name + assert log.read_bytes() == b"" + + +def test_where_neither_spelling_names_anything_the_words_are_as_before( + latin1, monkeypatch, tmp_path): + """The source moved away after `plan`: under Latin-1 neither the + spelling as written nor the UTF-8 bytes name anything, and + `plan-source-gone` is word for word what it was, the path named as the + plan records it; nothing says a second spelling was asked.""" + source, plan, log, env = written_in_utf8_mode(monkeypatch, tmp_path, + False) + local_path = str(recorded(plan)["local_path"]) + source.rename(tmp_path / "moved") + + proc = checked(plan, tmp_path, "latin1-work", {**env, **latin1}) + + assert proc.returncode == 2, said(proc) + assert ("REFUSED plan-source-gone: the plan's `source.local_path`, " + f"{spelled(local_path)}, names nothing that exists, and the plan " + "does not record it as a clone `plan` made (`source.cloned` is " + "not `true`), so nothing clones it again") in proc.stderr + assert "UTF-8" not in said(proc) + assert "Traceback" not in proc.stderr + assert log.read_bytes() == b"" + + +@pytest.mark.parametrize("locale", ["latin1", "c"]) +@pytest.mark.parametrize("present", ["utf8", "both", "neither"]) +def test_the_record_is_asked_as_written_then_as_utf8( + latin1, tmp_path, locale, present): + """`_recorded_path` of `/Thing\\u00e9`, as a plan written in UTF-8 + records it, with a directory at its UTF-8 bytes, at its Latin-1 byte as + well, or at neither. Under Latin-1 the spelling as written (0xE9) is + asked first, and the UTF-8 bytes (0xC3 0xA9) only where it names + nothing; where neither is there the path is the spelling as written. + Under C, which cannot spell the text, only the UTF-8 bytes are asked + and `said` names them, as #1072 left it.""" + env = c_locale_or_skip() + if locale == "latin1": + env.update(latin1) + base = os.fsencode(os.path.realpath(tmp_path)) + utf8 = base + b"/" + ACCENTED.encode("utf-8") + one_byte = base + b"/" + ACCENTED.encode(LATIN1) + for made in {"utf8": [utf8], "both": [utf8, one_byte], + "neither": []}[present]: + os.mkdir(made) + + child = subprocess.run( + [sys.executable, "-c", RECORDED_PATH, os.fsencode(ADOPT).hex(), + json.dumps(os.fsdecode(base) + "/" + ACCENTED)], + capture_output=True, env=env, stdin=subprocess.DEVNULL, check=False) + + assert child.returncode == 0, child.stderr.decode("utf-8", "replace") + answer = json.loads(child.stdout.decode("ascii")) + first = one_byte if locale == "latin1" else utf8 + assert bytes.fromhex(answer["path"]) == ( + utf8 if present == "utf8" else first), answer + assert answer["there"] is (present != "neither"), answer + assert answer["said"] == ("" if locale == "latin1" else + f" as UTF-8 spells it (this locale's file " + f"names, {answer['encoding']}, cannot spell " + "it)"), answer + + +def test_the_clone_at_the_record_is_kept_under_a_one_byte_locale( + latin1, tmp_path): + """`_keep_the_clone_at_the_record` (#803, #918) under Latin-1, which + spells `Thing\\u00e9` as written: the record asked only so names a path + that is not there, so the clone this run made at the plan's UTF-8 bytes + stayed on the list of clones to remove, #803's bug again. Asked as + `_recorded_source` asks it, that one clone is taken off, and a clone + made elsewhere stays.""" + spells, left, elsewhere = kept_clones( + tmp_path, {**c_locale_or_skip(), **latin1}) + assert spells is True, "premise: Latin-1 spells the record" + assert left == [elsewhere], ( + f"the clone at the record is not taken off: {left}") diff --git a/tests/test_adopt_unenterable_source_and_locale.py b/tests/test_adopt_unenterable_source_and_locale.py index 08e650ad..723f82a3 100644 --- a/tests/test_adopt_unenterable_source_and_locale.py +++ b/tests/test_adopt_unenterable_source_and_locale.py @@ -67,10 +67,10 @@ #: UTF-8 mode under the C locale: how the plan is written. UTF8_MODE = {"LC_ALL": "C", "PYTHONUTF8": "1"} -#: The child of `test_the_clone_at_the_record_is_kept_under_the_c_locale`: +#: The child `kept_clones` runs, for the C locale (#918) and Latin-1 (#1078): #: `Plan._keep_the_clone_at_the_record` of a plan recording argv[2], a JSON #: string, for the clones whose paths are the hex of their bytes in argv[3:] -#: (every argument is ASCII, as the locale under test spells nothing else). +#: (every argument is ASCII, which the C locale spells and Latin-1 alike). #: It prints one line of ASCII JSON: whether this locale spells the record, #: and the hex of each clone left. KEEP_THE_CLONE = r""" @@ -343,18 +343,12 @@ def test_a_surrogate_no_byte_is_read_as_is_asked_as_written(adopter): assert adopter._recorded_spelling("Thing\ud800") == ("Thing\ud800", "") -def test_the_clone_at_the_record_is_kept_under_the_c_locale(tmp_path): - """`_keep_the_clone_at_the_record` (#803, #918): `check --source` takes - the clone this run made off its list of clones to remove where it is at - the plan's own `source.local_path`. A plan written in UTF-8 mode records - `Thing\\u00e9` as itself, which `LC_ALL=C PYTHONUTF8=0` cannot spell: if - the record is asked as written, `os.path.realpath` meets the - `UnicodeEncodeError` and the method returns, the clone at the record - stays on the list and `check` removes it, #803's bug again under the C - locale. Asked as the UTF-8 bytes the plan holds, that one clone is taken - off, and a clone made elsewhere stays. The method runs in a child that - is the C locale itself, with a real directory at the record.""" - env = c_locale_or_skip() +def kept_clones(tmp_path: Path, env: dict) -> tuple[bool, list[str], str]: + """`KEEP_THE_CLONE` in a child started with `env`, for a record at + `/Thing\\u00e9/source-readonly/Thing` and two real clones, one at the + record and one elsewhere: `(spells, left, elsewhere)`, whether that + locale spells the record, the clones left on the list to remove, and + the one made elsewhere (#803, #918, #1078).""" base = Path(os.path.realpath(tmp_path)) clone = base / ACCENTED / "source-readonly" / "Thing" elsewhere = base / "work" / "source-readonly" / "Thing" @@ -369,7 +363,22 @@ def test_the_clone_at_the_record_is_kept_under_the_c_locale(tmp_path): assert child.returncode == 0, child.stderr.decode("utf-8", "replace") answer = json.loads(child.stdout.decode("ascii")) - assert answer["spells"] is False, "premise: this locale cannot spell it" left = [os.fsdecode(bytes.fromhex(held)) for held in answer["left"]] - assert left == [str(elsewhere)], ( + return answer["spells"], left, str(elsewhere) + + +def test_the_clone_at_the_record_is_kept_under_the_c_locale(tmp_path): + """`_keep_the_clone_at_the_record` (#803, #918): `check --source` takes + the clone this run made off its list of clones to remove where it is at + the plan's own `source.local_path`. A plan written in UTF-8 mode records + `Thing\\u00e9` as itself, which `LC_ALL=C PYTHONUTF8=0` cannot spell: if + the record is asked as written, `os.path.realpath` meets the + `UnicodeEncodeError` and the method returns, the clone at the record + stays on the list and `check` removes it, #803's bug again under the C + locale. Asked as the UTF-8 bytes the plan holds, that one clone is taken + off, and a clone made elsewhere stays. The method runs in a child that + is the C locale itself, with a real directory at the record.""" + spells, left, elsewhere = kept_clones(tmp_path, c_locale_or_skip()) + assert spells is False, "premise: this locale cannot spell it" + assert left == [elsewhere], ( f"the clone at the record is not taken off: {left}")