From d6d1df2340652efe87c0cefbcad4eac8f804ce32 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:05:35 +0000 Subject: [PATCH 01/13] conftest.py and four test files: one helper asks whether a system can carry a name or branch that is not UTF-8, `os.fsencode` first and the disk's EILSEQ second, and two pins hold `bump-leg.py`'s UTF-8 decode and the leg finding's role, so a test no longer skips on the disk alone or passes under a decode as ASCII or a finding named by path (#663) #461's five test lines, lane 2's L2-52 and lane 4's R423 N1 and N2 and R455 F1 and F3. Test only: no tool and no template changes, and `validate-manifest.py`, a pinned copy, is read and not edited. - `tests/conftest.py` holds `skip_where_a_name_cannot_be_not_utf_8`, which `tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py` each had a copy of. It asks `skip_where_fsencode_gives_a_surrogate_no_byte` first (Windows' `os.fsencode` makes three bytes of U+DCFF, so no name that stands for a byte is carried there, though NTFS would take the file), then the disk (macOS answers EILSEQ); any other error is the test's own. - `make_file` and the `--branch a\udcffb` test call it, so they skip on Windows too, where the disk takes the name and the child then refuses the branch. `test_a_branch_the_system_carries_has_no_fault` asks the same function instead of its own copy of the question. - `test_a_name_that_is_utf8_under_quote_path_false_is_still_refused` asserts that the line naming `caf.yml` holds no literal `\x`, in any locale, so `bump-leg.py` decoding git's bytes as ASCII fails it. - `_leg_branch_findings` is asked of a leg whose role is not its path, so naming the leg by `path` fails it. - `tests/test_non_utf8_name_helpers.py` holds the helpers to this, with Windows' `os.fsencode` and macOS's EILSEQ emulated: CI is Linux only, and no Windows or macOS run was made. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 83 +++++++++++++++++ tests/test_adopt_pins.py | 20 +--- tests/test_bootstrap_branch_bytes.py | 74 ++++++++++++--- tests/test_non_utf8_git_reads.py | 77 +++++++++++++--- tests/test_non_utf8_name_helpers.py | 133 +++++++++++++++++++++++++++ tests/test_pin_source_unreadable.py | 21 +---- 6 files changed, 346 insertions(+), 62 deletions(-) create mode 100644 tests/test_non_utf8_name_helpers.py diff --git a/tests/conftest.py b/tests/conftest.py index 33cbf476..0e9da01d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,7 @@ from __future__ import annotations +import errno import functools import importlib.util import os @@ -864,6 +865,88 @@ def a_hook_that_unstages(hooks: Path, path: str = PINNED_COPY) -> Path: return hooks +# --- a file name or a branch that is not UTF-8 (#663) ------------------------ +# +# A test that needs a file name or a branch holding a byte that is not UTF-8 +# (Python keeps the byte as a lone surrogate, U+DC80 to U+DCFF, which +# `os.fsencode` gives back as the byte on POSIX) asks these helpers whether +# this system can carry it, so each system's answer is written in one place +# and a change to it reaches every test that asks. +# `tests/test_non_utf8_name_helpers.py` holds them to it, Windows included, by +# emulating its `os.fsencode`: CI runs the suite on Linux only (#597). + +#: The surrogate the helpers ask about: the byte 0xFF, which no UTF-8 text has. +NOT_UTF_8_PROBE = "\udcff" + + +def skip_where_fsencode_gives_a_surrogate_no_byte(text: str) -> None: + """Skip where `text` holds a byte-carrying surrogate (U+DC80 to U+DCFF) + and this system's `os.fsencode` makes no single byte of one. + + That is Windows: its `os.fsencode` is UTF-8 with `surrogatepass`, so it + writes the three bytes of a sequence that is not UTF-8 for EVERY + surrogate, and none of them is a byte of a name. A disk probe cannot be + trusted to ask it: CPython hands a str path to `CreateFileW` as UTF-16 + and NTFS most likely stores a lone surrogate unchecked (read, not run), + so the file is made, and git for Windows then sees the name through its + own UTF-16 conversion, which is not the byte the test meant. A command + line is the same (`CreateProcessW`), which is why `bootstrap.py`'s + `_argument_fault` refuses U+DC80 to U+DCFF there. Asked through the `os` + module at call time, so a test emulates Windows with + `monkeypatch.setattr(os, "fsencode", )`. + """ + if (any("\udc80" <= char <= "\udcff" for char in text) + and len(os.fsencode(NOT_UTF_8_PROBE)) != 1): + pytest.skip("this system's `os.fsencode` makes no byte of a surrogate " + "that stands for a byte that is not UTF-8") + + +def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: + """Skip, as `POSIX_NAMES` skips on Windows, where a file name that is not + UTF-8 cannot be made under `directory`. Two systems say so: + + - `skip_where_fsencode_gives_a_surrogate_no_byte` (Windows), asked first, + before the disk is touched; + - a filesystem that refuses the name: macOS answers EILSEQ ("Illegal byte + sequence"), so no clone can sit at such a path there. + + The second is asked by making a directory `probe` under + `directory` and removing it; any other error is the test's own and is + raised. + """ + probe = directory / f"probe{NOT_UTF_8_PROBE}" + skip_where_fsencode_gives_a_surrogate_no_byte(probe.name) + try: + probe.mkdir() + except OSError as exc: + if exc.errno != errno.EILSEQ: + raise + pytest.skip("needs a file name that is not UTF-8, which this " + "filesystem refuses (EILSEQ)") + probe.rmdir() + + +def windows_fsencode(text: str) -> bytes: + """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every + lone surrogate is the three bytes of a sequence that is not UTF-8. For + `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the + guard above in front of a test that Linux would let through.""" + return text.encode("utf-8", "surrogatepass") + + +def skip_message(call, *args) -> str | None: + """The reason `call(*args)` skips the running test with, or None where it + returns; any other exception goes on. For a test OF a skip helper: a + skip that escapes a test ends it SKIPPED, which no run fails on, so a + helper that skipped where it should not would pass `pytest.raises` and + every other assertion that expects no skip.""" + try: + call(*args) + except pytest.skip.Exception as skipped: + return skipped.msg + return None + + # --- the `gh` guard (#296) --------------------------------------------------- # # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still diff --git a/tests/test_adopt_pins.py b/tests/test_adopt_pins.py index 719af0f7..7b3bb313 100644 --- a/tests/test_adopt_pins.py +++ b/tests/test_adopt_pins.py @@ -34,7 +34,6 @@ from __future__ import annotations -import errno import importlib.util import json import os @@ -50,7 +49,8 @@ from conftest import (ADOPT, REPO, SCAFFOLD, SYNTHETIC_TREE, WINDOWS_SKIP, as_human, cloned_just_pushed, findings, git, make_source_repo, planned, refused_before_any_leg, - resolve, rmtree, run_script) + resolve, rmtree, run_script, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -855,22 +855,6 @@ def test_a_pin_source_check_would_refuse_is_refused_by_plan( UNREAD = "unreadclone" -def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: - """Skip, as `POSIX_NAMES` skips on Windows, on a filesystem that refuses - a file name that is not UTF-8: macOS answers EILSEQ ("Illegal byte - sequence"), so no clone can sit at such a path there. Asked of a - directory made under `directory`; any other error is the test's own.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name, reason", [ pytest.param(f"{UNREAD}‍x", "it holds an invisible formatting character", id="zwj"), diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index bab82102..3f63761e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -45,7 +45,10 @@ import pytest -from conftest import REPO, git, load_without_bytecode +from conftest import (REPO, git, load_without_bytecode, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bootstrap_spelling import (ASSEMBLY_TEMPLATE, DASH, bootstrap_of, edit, pin12, run) from test_stdout_encoding import child_environ, run_tool_bytes @@ -349,12 +352,12 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( `os.fsencode` carries it, and git is handed that byte, as on origin/main. `--branch` is the only way it reaches step (a) with the validators out of it (the manifest validator names it, #280's rule for a surrogate). Made on - a disk that takes the byte, skipped on one that does not.""" - probe = tmp_path / "probe-\udcff" - try: - probe.write_bytes(b"") - except (OSError, ValueError): - pytest.skip("this disk takes no file name with the byte 0xff") + a system and a disk that take the byte, skipped on one that does not: the + shared helper asks `os.fsencode` first, which is what Windows answers + (NTFS takes the name, and `_argument_fault` refuses the branch there, + which `test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused` + holds), and then the disk, which macOS refuses (#663).""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) code, out, err = run(bootstrap_of(project), "--skip-validators", "--branch", "a\udcffb", encoding=ENCODING, tmp_path=tmp_path, cwd=project) @@ -370,12 +373,40 @@ def test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made( assert head == b"refs/heads/a\xffb\n", head +def test_the_branch_test_skips_where_fsencode_makes_no_byte_of_the_surrogate( + project, tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R455 F1): on NTFS the disk takes + `probe\\udcff`, the child is given `--branch a\\udcffb`, and + `_argument_fault` refuses U+DCFF where `os.fsencode` writes three bytes, + so the `branch a\\udcffb created` line the test expects is absent. With + Windows' `os.fsencode` the test is therefore SKIPPED, before it runs + bootstrap: it is called here as it is, under the emulation, and the legs + stand as they were.""" + before = legs_state(project) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message( + test_a_name_that_is_not_utf8_is_still_carried_and_the_branch_made, + project, tmp_path) + assert reason is not None, "the test ran where `os.fsencode` gives no byte" + assert "os.fsencode" in reason, reason + assert legs_state(project) == before + + @pytest.fixture(scope="module") def bootstrap_module(): """The template run as a module, no bytecode written beside it.""" return load_without_bytecode(ASSEMBLY_TEMPLATE, "bootstrap_branch_bytes") +@pytest.fixture(scope="module") +def validator_module(): + """`validate-manifest.py`, the same template, as a module: the file is a + pinned copy, so its functions are read and never changed here.""" + return load_without_bytecode( + REPO / "templates" / "assembly-root" / "scripts" + / "validate-manifest.py", "validate_manifest_branch_bytes") + + @pytest.mark.parametrize("branch", [ "main", "release/1.2", "feature/日本", "a‍b", "a\xe9b", "a b", "a\x1bb", "a..b", "", "\udc80", "a\udcffb"]) @@ -385,9 +416,7 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): the byte-carrying surrogates U+DC80 to U+DCFF are the commands'. On a system whose `os.fsencode` does not carry U+DC80 and U+DCFF (Windows) the next test is the one that holds.""" - if any("\udc80" <= char <= "\udcff" for char in branch) and len( - os.fsencode("\udcff")) != 1: - pytest.skip("this system's `os.fsencode` makes no byte of U+DCFF") + skip_where_fsencode_gives_a_surrogate_no_byte(branch) assert bootstrap_module._argument_fault(branch) is None @@ -424,13 +453,28 @@ def test_where_a_surrogate_is_three_bytes_and_no_byte_it_is_refused( assert fault.startswith("U+DC"), fault -def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module): +def test_the_two_templates_agree_on_what_a_fault_is(bootstrap_module, + validator_module): """`validate-manifest.py` finds ANY surrogate (#280's rule, which a UTF-8 file settles), where `bootstrap.py` asks only whether a command can be given the value: so every value bootstrap refuses, the validator names.""" - validator = load_without_bytecode( - REPO / "templates" / "assembly-root" / "scripts" - / "validate-manifest.py", "validate_manifest_branch_bytes") for branch in ("a\x00b", "a\ud83db", "a\ude00", "\udc00"): assert bootstrap_module._argument_fault(branch) is not None - assert validator._branch_finding("c", "w", branch) != [] + assert validator_module._branch_finding("c", "w", branch) != [] + + +@pytest.mark.parametrize("role", ["spec", "code"]) +def test_the_leg_finding_names_the_leg_by_its_role_and_not_its_path( + validator_module, role): + """THE NAME IS THE ROLE (#663, R455 F3, mutant R2: naming `path` passed + all 54 tests). The templates' roles equal their paths (`spec` is mounted + at `spec`), so nothing the scaffold writes can tell the two apart, and + the finding is asked of a leg whose path is not its role, as a spec leg + mounted at `contracts` is. A unit test on the validator, which this file + loads for its last test too: it is a pinned copy and is not edited.""" + [finding] = validator_module._leg_branch_findings( + [{"role": role, "path": "contracts", "branch": "a\x00b"}]) + assert finding.startswith( + f"FINDING manifest-leg-branch: leg '{role}': branch is 'a\\x00b', " + "which holds U+0000, a NUL"), finding + assert "contracts" not in finding, finding diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index c79bada2..18fbb7e9 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -32,11 +32,13 @@ NO NETWORK AND NO GITHUB. The scaffold and `family.py init` here are run as child processes into LOCAL bare remotes (`--local-remote-dir`), which never calls `gh`. A name that is not UTF-8 is made by `os.fsdecode` of its bytes, so -a POSIX filesystem takes it as it is; where the disk refuses (macOS, Windows) -the test skips, and where the ENVIRONMENT cannot carry a byte (Windows') the -`user.name` rows skip. The checkout under test for `_shape_revision` and -`_shape_snapshot` is a COPY of the standard in a repository of its own -(`standard`), made dirty one test at a time, never this checkout. +a POSIX filesystem takes it as it is; where the system cannot carry the byte +the test skips (`make_file`: Windows' `os.fsencode` makes no byte of it, and +macOS's disk refuses it), and where the ENVIRONMENT cannot carry a byte +(Windows') the `user.name` rows skip. The checkout under test for +`_shape_revision` and `_shape_snapshot` is a COPY of the standard in a +repository of its own (`standard`), made dirty one test at a time, never this +checkout. """ from __future__ import annotations @@ -49,7 +51,10 @@ import pytest -from conftest import ORG, PROJECT, REPO, cloned_just_pushed, git, run_script +from conftest import (ORG, PROJECT, REPO, cloned_just_pushed, git, run_script, + skip_message, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) from test_bump_leg import branched, bump sys.path.insert(0, str(REPO / "scripts")) @@ -85,7 +90,11 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: """Write `data` at `path`, or skip: a name that is not UTF-8 needs a - filesystem that stores bytes (Linux does; APFS and NTFS refuse it).""" + system that carries the byte and a filesystem that stores it (Linux does; + APFS refuses it, and on Windows `os.fsencode` makes no byte of it, which + is asked FIRST: NTFS most likely takes the surrogate, and the file would + then not be the one the test means). A name that is UTF-8 is written.""" + skip_where_fsencode_gives_a_surrogate_no_byte(path.name) try: path.write_bytes(data) except (OSError, UnicodeError) as exc: @@ -93,6 +102,47 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: f"UTF-8: {exc}") +def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( + tmp_path, monkeypatch): + """THE WINDOWS GUARD, EMULATED (#663, R423 N1): where `os.fsencode` gives + a surrogate three bytes, NTFS most likely takes `caf\\udce9.yml` anyway + and git for Windows would see another name, so `make_file` skips BEFORE it + writes, and nothing is on the disk. Linux takes the name, so a file + standing there would be the failure.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target) + assert reason is not None, "make_file wrote the name instead of skipping" + assert "os.fsencode" in reason, reason + assert not target.exists(), "the file was written before the skip" + + +def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( + tmp_path, monkeypatch): + """The guard is for the byte and no wider: Windows holds `caf\\u00e9.yml` + as text, and the tests that use it must run there.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + target = tmp_path / UTF8 + assert skip_message(make_file, target, b"on: [pull_request]\n") is None + assert target.read_bytes() == b"on: [pull_request]\n" + + +def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( + tmp_path): + """The control: where `os.fsencode` carries the byte and the disk takes it + (Linux), the file is written under the name that is not UTF-8, so the + guard skipped nothing.""" + if len(os.fsencode("\udce9")) != 1: + pytest.skip("this system's `os.fsencode` makes no byte of U+DCE9") + target = tmp_path / NOT_UTF8 + reason = skip_message(make_file, target, b"on: [push]\n") + if reason is not None and "filesystem" in reason: + pytest.skip(reason) + assert reason is None, reason + assert os.fsencode(target.name) == b"caf\xe9.yml" + assert target.read_bytes() == b"on: [push]\n" + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -205,16 +255,21 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( Its bytes decode, and it was refused on main; it still is, and the refusal still prints it as git did. (The name is matched on its ASCII ends: a child's stderr is decoded by the locale here, which is not UTF-8 - on every platform.)""" + on every platform. What the middle of the name is NOT is held instead, in + any locale: the tool decodes git's bytes as UTF-8, so `\\u00e9` is one + character, where a decode as ASCII would show its two bytes as `\\xc3\\xa9`, + and a locale's decode of what the child printed is mojibake at worst and + never a `\\x`: #423's mutant R1, #663.)""" dirty_root(project, UTF8, quote_path=False) before = found(project) result = bump(project, scaffolded, advanced, *extra) output = result.stderr + result.stdout assert result.returncode == 2, output assert "REFUSED bump-leg-root-dirty: " in result.stderr, output - assert any(line.startswith(" M .github/workflows/caf") - and line.endswith(".yml") - for line in result.stderr.splitlines()), output + [shown] = [line for line in result.stderr.splitlines() + if line.startswith(" M .github/workflows/caf")] + assert shown.endswith(".yml"), output + assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py new file mode 100644 index 00000000..8c387eee --- /dev/null +++ b/tests/test_non_utf8_name_helpers.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The two helpers a test asks whether a name that is not UTF-8 can be +carried here (#663, #461's L2-52). + +`skip_where_a_name_cannot_be_not_utf_8` was two helpers with one body +(`tests/test_adopt_pins.py` and `tests/test_pin_source_unreadable.py`), and +`make_file` and the `--branch a\\udcffb` test each asked a smaller question of +their own, or none: on Windows the disk takes the name, so no probe of the +disk skips there. Now `tests/conftest.py` holds both questions once, the +system's first (`os.fsencode`) and the disk's second (EILSEQ), and this file +holds them to it. + +CI runs the suite on Linux only (#597), so what Windows answers is EMULATED: +`os.fsencode` is replaced for one test by `windows_fsencode`, which gives a +surrogate three bytes, as Windows' does. No Windows run was made. macOS's +EILSEQ is emulated the same way, by a `mkdir` that answers it. +""" + +from __future__ import annotations + +import errno +import os +import sys +from pathlib import Path + +import pytest + +from conftest import (NOT_UTF_8_PROBE, skip_message, + skip_where_a_name_cannot_be_not_utf_8, + skip_where_fsencode_gives_a_surrogate_no_byte, + windows_fsencode) + +LINUX_ONLY = pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Linux stores any byte in a file name; the other systems are " + "asked by the tests below that emulate them") + + +def mkdir_answers(code: int, said: str): + """A `Path.mkdir` that fails as the disk would with `code`.""" + def refuse(self, *args, **kwargs): + raise OSError(code, said, str(self)) + return refuse + + +# --- the system's question: what `os.fsencode` makes of the surrogate -------- + +@LINUX_ONLY +def test_a_system_that_carries_the_byte_is_not_skipped(): + """The control: Linux's `os.fsencode` gives U+DCFF back as the one byte + 0xFF, so no text is skipped.""" + assert os.fsencode(NOT_UTF_8_PROBE) == b"\xff" + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"caf{NOT_UTF_8_PROBE}.yml") is None + + +def test_where_fsencode_makes_three_bytes_of_the_surrogate_the_name_is_skipped( + monkeypatch): + """Windows' `os.fsencode` (emulated): U+DCFF is the three bytes of a + sequence that is not UTF-8, none of them 0xFF, so a name that holds a + byte-carrying surrogate is skipped, and each one from U+DC80 to U+DCFF.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert len(os.fsencode(NOT_UTF_8_PROBE)) == 3 + for surrogate in ("\udc80", "\udce9", "\udcff"): + reason = skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + f"a{surrogate}b") + assert reason is not None, f"U+{ord(surrogate):04X} was not skipped" + assert "os.fsencode" in reason, reason + + +@pytest.mark.parametrize("text", [ + "plain", "café.yml", "feature/日本", "", "a\ud83db"], + ids=["ascii", "text", "cjk", "empty", "surrogate-without-a-byte"]) +def test_where_fsencode_makes_three_bytes_a_name_that_is_text_is_not_skipped( + monkeypatch, text): + """The guard is for the surrogates that stand for a byte (U+DC80 to + U+DCFF) and no wider: text is the same on every system, and a surrogate + that stands for no byte is not this helper's to judge (`_argument_fault` + refuses it, as `bootstrap.py`'s tests hold).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + assert skip_message(skip_where_fsencode_gives_a_surrogate_no_byte, + text) is None + + +# --- the shared helper: the system, then the disk ---------------------------- + +@LINUX_ONLY +def test_a_disk_that_takes_the_name_is_not_skipped_and_the_probe_is_removed( + tmp_path): + """The control: no skip, and nothing is left in the directory.""" + assert skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path) is None + assert list(tmp_path.iterdir()) == [] + + +def test_where_fsencode_makes_three_bytes_the_disk_is_not_asked( + tmp_path, monkeypatch): + """THE SYSTEM IS ASKED FIRST (#663): on Windows the disk would take the + name, so a skip that waited for the disk would never come. The directory + does not exist, so an `mkdir` under it would be FileNotFoundError and not + a skip: the skip is the answer, and the disk was not touched.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, + tmp_path / "absent") + assert reason is not None, "the helper went on to ask the disk" + assert "os.fsencode" in reason, reason + + +def test_a_disk_that_answers_eilseq_is_skipped(tmp_path, monkeypatch): + """macOS (emulated): APFS answers EILSEQ, `Illegal byte sequence`, to a + file name that is not UTF-8, and no clone can sit at such a path.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", + mkdir_answers(errno.EILSEQ, "Illegal byte sequence")) + reason = skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert reason is not None, "EILSEQ was not a skip" + assert "EILSEQ" in reason, reason + + +@pytest.mark.parametrize("code", [errno.EACCES, errno.ENOSPC, errno.EEXIST], + ids=["EACCES", "ENOSPC", "EEXIST"]) +def test_any_other_error_from_the_disk_is_the_tests_own( + tmp_path, monkeypatch, code): + """Only EILSEQ is the filesystem's refusal of the name: a full disk, a + folder nobody can write in or a probe left by an earlier run is a + failure of the test, raised as it was, and never a skip that hides it.""" + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + monkeypatch.setattr(Path, "mkdir", mkdir_answers(code, os.strerror(code))) + with pytest.raises(OSError) as raised: + skip_message(skip_where_a_name_cannot_be_not_utf_8, tmp_path) + assert raised.value.errno == code diff --git a/tests/test_pin_source_unreadable.py b/tests/test_pin_source_unreadable.py index 177e8c94..c43a41ad 100644 --- a/tests/test_pin_source_unreadable.py +++ b/tests/test_pin_source_unreadable.py @@ -41,7 +41,8 @@ import pytest -from conftest import ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, run_script +from conftest import (ADOPT, ORG, REPO, SCAFFOLD, git, make_source_repo, + run_script, skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) from repo_shape import load_yaml, parse_yaml, tree_digest # noqa: E402 @@ -592,22 +593,6 @@ def test_a_typed_dot_is_still_the_directory_the_tool_runs_in( ] -def skip_where_no_file_name_is_not_utf_8(directory: Path) -> None: - """Skip, as a Windows command line skips, where the filesystem refuses - a file name that is not UTF-8, so that no clone can sit at one: macOS - answers EILSEQ ("Illegal byte sequence"). Asked by making one under - `directory`; any other error is the test's to report.""" - probe = directory / "probe\udcff" - try: - probe.mkdir() - except OSError as exc: - if exc.errno != errno.EILSEQ: - raise - pytest.skip("needs a file name that is not UTF-8, which this " - "filesystem refuses (EILSEQ)") - probe.rmdir() - - @pytest.mark.parametrize("name", CLONE_NAMES) def test_the_pin_file_says_how_the_digest_was_read_and_not_where( name, tmp_path, adoptable, glass): @@ -618,7 +603,7 @@ def test_the_pin_file_says_how_the_digest_was_read_and_not_where( --all`, read as bytes).""" product, commit = glass if "\udcff" in name: - skip_where_no_file_name_is_not_utf_8(tmp_path) + skip_where_a_name_cannot_be_not_utf_8(tmp_path) clones = tmp_path / "clones" shutil.copytree(product, clones / name, symlinks=True) result = run("scaffold-run", tmp_path, adoptable, str(clones / name), From 0b1d1ddbeeca662500534f58c39e5b5a0c67a70d Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 14:15:43 +0000 Subject: [PATCH 02/13] test_bootstrap_branch_bytes.py: the no-fault test's Windows guard is held by an emulation, so dropping the shared helper from it fails a test on Linux too (#663) `test_a_branch_the_system_carries_has_no_fault` now asks `skip_where_fsencode_gives_a_surrogate_no_byte` where it had its own copy of the question. The new test calls it under Windows' `os.fsencode` (emulated): a branch holding a byte-carrying surrogate is skipped, plain text and text with an accent run. Removing the call fails the two skipped rows. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bootstrap_branch_bytes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_bootstrap_branch_bytes.py b/tests/test_bootstrap_branch_bytes.py index 3f63761e..5a79ff8e 100644 --- a/tests/test_bootstrap_branch_bytes.py +++ b/tests/test_bootstrap_branch_bytes.py @@ -420,6 +420,24 @@ def test_a_branch_the_system_carries_has_no_fault(bootstrap_module, branch): assert bootstrap_module._argument_fault(branch) is None +@pytest.mark.parametrize("branch, skipped", [ + pytest.param("a\udcffb", True, id="byte-carrying-surrogate"), + pytest.param("\udc80", True, id="the-first-one"), + pytest.param("release/1.2", False, id="plain"), + pytest.param("a\xe9b", False, id="text"), +]) +def test_the_no_fault_test_skips_where_fsencode_makes_no_byte_of_the_branch( + bootstrap_module, monkeypatch, branch, skipped): + """THE WINDOWS GUARD, EMULATED (#663): under Windows' `os.fsencode` the + test above skips a branch holding a byte-carrying surrogate, which + `_argument_fault` refuses there, and runs every other (the next test + holds the refusal).""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(test_a_branch_the_system_carries_has_no_fault, + bootstrap_module, branch) + assert (reason is not None) is skipped, reason + + @pytest.mark.parametrize("branch, fault", [ ("a\x00b", "U+0000, a NUL"), ("\x00", "U+0000, a NUL"), From 6ba8c5d2d6fec3a779963713593e5377c76bbadf Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Fri, 16 Oct 2026 18:40:04 +0000 Subject: [PATCH 03/13] conftest.py and two test files: `skip_message` is held by a test that fails where a skip escapes it, `windows_fsencode` passes bytes through as `os.fsencode` does, and the no-`\x` check skips where the file name encoding is not UTF-8, so a leaked skip, an emulation that breaks on bytes and a Latin-1 host no longer pass silently or fail on a tool that is right (#663) The review of #723 (0b1d1ddb) found no MUST and three things to take. SHOULD-1: `skip_message` had no test of its own. A skip that escapes it ends the test SKIPPED, and no run fails on a skip, so the mutant where one escapes gave `10 passed, 3 skipped` on the helpers file and seven emulation tests silently skipped. `tests/test_non_utf8_name_helpers.py` now holds it with three tests: it returns the reason a call skips with, it returns None where the call does not skip, and it lets any other exception go on. The first catches the escape and fails it with `pytest.fail`, because a bare assert on an escaped skip would end that test SKIPPED too. NIT-2: `windows_fsencode` took `str` only. `os.fsencode` passes bytes through on every system, so the emulation now does, and two tests hold it (the surrogate and text, and the bytes, compared with `os.fsencode` itself). NIT-1: on a Latin-1 file name encoding `make_file` writes `caf\xe9.yml` as the single byte 0xE9, the tool prints `caf\xe9.yml`, and the new `"\\x" not in shown` assertion would fail although the tool is right. It is now `assert_shown_without_hex_escapes`, which skips where `os.fsencode(UTF8) != UTF8.encode("utf-8")` and is called after the test's other assertions, so a skip loses none of them. Two tests hold it under an emulated Latin-1 `os.fsencode` (it skips) and Windows' (it asserts, and fails a line that holds a `\x`). Tests only; no tool and no template changes. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 7 +++-- tests/test_non_utf8_git_reads.py | 44 ++++++++++++++++++++++++++- tests/test_non_utf8_name_helpers.py | 47 +++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0e9da01d..a5313d72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -926,11 +926,14 @@ def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: probe.rmdir() -def windows_fsencode(text: str) -> bytes: +def windows_fsencode(text: str | bytes) -> bytes: """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every - lone surrogate is the three bytes of a sequence that is not UTF-8. For + lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes + go through as they are, as `os.fsencode` passes them on every system. For `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the guard above in front of a test that Linux would let through.""" + if isinstance(text, bytes): + return text return text.encode("utf-8", "surrogatepass") diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index 18fbb7e9..4d19a998 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -143,6 +143,48 @@ def test_make_file_writes_the_name_that_is_not_utf8_where_the_system_has_it( assert target.read_bytes() == b"on: [push]\n" +def assert_shown_without_hex_escapes(shown: str, output: str) -> None: + """Assert that the refusal's line for `caf\u00e9.yml` holds no literal + `\\x`, or skip where this system's file name encoding is not UTF-8. + + On a Latin-1 host `make_file` writes the one byte 0xE9 for it, and the + tool, which decodes git's bytes as UTF-8, rightly prints `caf\\xe9.yml`: + the assertion would fail though the tool is right (#663, NIT-1). So it + is asked after the system's encoding is, and a system whose encoding is + not UTF-8 skips. (Where `make_file` could not write the name at all, an + ASCII host, it skipped already.)""" + if os.fsencode(UTF8) != UTF8.encode("utf-8"): + pytest.skip("this system's file name encoding is not UTF-8, so a " + "UTF-8 name is not written as the bytes the tool decodes") + assert "\\x" not in shown, output + + +def test_the_hex_escape_check_skips_where_the_file_name_encoding_is_not_utf_8( + monkeypatch): + """THE LATIN-1 HOST, EMULATED (#663, NIT-1): `os.fsencode` gives the name + `caf\u00e9.yml` as the one byte 0xE9, the line holds a `\\x`, and the + check skips, where it would otherwise fail on a tool that is right.""" + monkeypatch.setattr(os, "fsencode", lambda name: name.encode("latin-1")) + reason = skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xe9.yml", "output") + assert reason is not None, "the check asserted instead of skipping" + assert "UTF-8" in reason, reason + + +def test_the_hex_escape_check_asserts_where_the_file_name_encoding_is_utf_8( + monkeypatch): + """The control: Windows' `os.fsencode` (emulated), like Linux's and + macOS's, is UTF-8 for text, so the check is not skipped, and it still + fails a line that holds a `\\x` and passes one that does not.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(assert_shown_without_hex_escapes, + " M .github/workflows/caf\u00e9.yml", "output") + assert reason is None, reason + with pytest.raises(AssertionError): + skip_message(assert_shown_without_hex_escapes, + r" M .github/workflows/caf\xc3\xa9.yml", "output") + + def git_bytes(*args: str, cwd: Path) -> bytes: """`git ` in `cwd`, stdout as the bytes git wrote. `conftest.git` decodes as text, which is the failure under test.""" @@ -269,9 +311,9 @@ def test_a_name_that_is_utf8_under_quote_path_false_is_still_refused( [shown] = [line for line in result.stderr.splitlines() if line.startswith(" M .github/workflows/caf")] assert shown.endswith(".yml"), output - assert "\\x" not in shown, output assert "Traceback" not in output, output assert found(project) == before + assert_shown_without_hex_escapes(shown, output) def test_the_refusal_names_every_dirty_file_whatever_its_bytes( diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 8c387eee..47ae0d13 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -14,6 +14,10 @@ `os.fsencode` is replaced for one test by `windows_fsencode`, which gives a surrogate three bytes, as Windows' does. No Windows run was made. macOS's EILSEQ is emulated the same way, by a `mkdir` that answers it. + +The two tools the tests below are written with are held here too: +`skip_message`, which every one of them reads a skip with, and +`windows_fsencode`, the emulation itself. """ from __future__ import annotations @@ -43,6 +47,49 @@ def refuse(self, *args, **kwargs): return refuse +# --- `skip_message`: how every test below reads a skip ----------------------- +# +# A skip that ESCAPES a helper ends its test SKIPPED, and no run fails on a +# skip: a `skip_message` that let one out would turn every assertion below +# that expects a skip, or expects none, into a silent skip. So the first test +# catches the escape and fails it. + +def test_skip_message_returns_the_reason_a_call_skips_with(): + try: + reason = skip_message(lambda: pytest.skip("x")) + except pytest.skip.Exception: + pytest.fail("the skip escaped skip_message, which would end the test " + "SKIPPED, and no run fails on that") + assert reason == "x" + + +def test_skip_message_returns_none_where_the_call_does_not_skip(): + assert skip_message(lambda: None) is None + + +def test_skip_message_lets_any_other_exception_go_on(): + def not_a_skip(): + raise ZeroDivisionError("not a skip") + + with pytest.raises(ZeroDivisionError): + skip_message(not_a_skip) + + +# --- `windows_fsencode`: the emulation is Windows' `os.fsencode` -------------- + +def test_windows_fsencode_makes_three_bytes_of_a_surrogate_and_utf_8_of_text(): + assert windows_fsencode("\udcff") == b"\xed\xb3\xbf" + assert windows_fsencode("caf\u00e9.yml") == "caf\u00e9.yml".encode("utf-8") + + +def test_windows_fsencode_passes_bytes_through_as_os_fsencode_does(): + """`os.fsencode` takes bytes on every system and gives them back; the + emulation that stands in for it must not fail on them.""" + data = b"caf\xe9.yml" + assert os.fsencode(data) == data + assert windows_fsencode(data) == data + + # --- the system's question: what `os.fsencode` makes of the surrogate -------- @LINUX_ONLY From 41849725ce9e309eb364233e6adcfa61e3f34882 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 10:40:26 +0000 Subject: [PATCH 04/13] test_stdout_encoding.py, test_adopt_git_reads.py, test_adopt_template_values.py, test_adopt_plan_escapes.py, test_clone_url_bytes.py and test_non_utf8_git_reads.py: the four tests and `directory_named` that made a directory named with a byte that is not UTF-8 ask `skip_where_a_name_cannot_be_not_utf_8` and skip on its two answers only, `directory_named` keeps its own Windows skip for a control character and no longer skips a bidi override there, and three tests of `test_non_utf8_git_reads.py` skip where an ASCII file name encoding cannot carry their text, so a full disk or a `tmp_path` nobody can write in fails the test instead of skipping it and a C locale shows no failure that is not a defect (#757) The four tests (in `test_stdout_encoding.py`, `test_adopt_git_reads.py`, `test_adopt_template_values.py` and `test_adopt_plan_escapes.py`) and `directory_named` each made the directory themselves and turned ANY `OSError` into `pytest.skip`, beside the helper #663 added to `tests/conftest.py`. That helper skips on the two answers that mean "this system cannot carry the name" (Windows' `os.fsencode` first, macOS's EILSEQ second) and raises the rest. They now call it, and each keeps its own second skip (git will not work under the name; a listing that shows a rewritten name); the directory they make after the probe is no longer guarded. The file twin in `test_adopt_plan_escapes.py` asks the same question of a file name and calls the helper too. `directory_named` asks the helper for a name that is not UTF-8 and keeps its own Windows skip for a control character, which Windows refuses in any name. The `bidi-override` row, which NTFS holds, now runs on Windows (reasoned; CI is Linux only, no Windows run was made). Under `PYTHONUTF8=0 LC_ALL=C PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, and `tests/test_non_utf8_git_reads.py` failed six tests that hold no defect: `test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte` (the issue's line) and the five rows that put `Jos\u00E9`, a U+200C name or a U+202E name into a child's environment, where `subprocess` raises `UnicodeEncodeError` before the tool runs. The first now ends as its neighbour does, on a skip that names the filesystem; the others skip where `os.fsencode` cannot carry the value. `tests/test_non_utf8_name_callers.py` calls each caller as a function with the disk answering as macOS (EILSEQ on the probe: a skip), as a folder nobody can write in (EACCES on the probe, and on the test's own directory: a failure), and `directory_named` as Windows (a control character skips, a bidi override does not), and holds the two C-locale skips with an ASCII `os.fsencode` and a `write_bytes` that raises as it would. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_git_reads.py | 13 +- tests/test_adopt_plan_escapes.py | 21 +- tests/test_adopt_template_values.py | 13 +- tests/test_clone_url_bytes.py | 39 ++-- tests/test_non_utf8_git_reads.py | 23 ++- tests/test_non_utf8_name_callers.py | 285 ++++++++++++++++++++++++++++ tests/test_stdout_encoding.py | 9 +- 7 files changed, 358 insertions(+), 45 deletions(-) create mode 100644 tests/test_non_utf8_name_callers.py diff --git a/tests/test_adopt_git_reads.py b/tests/test_adopt_git_reads.py index a327e56f..d97353a4 100644 --- a/tests/test_adopt_git_reads.py +++ b/tests/test_adopt_git_reads.py @@ -31,7 +31,8 @@ import pytest -from conftest import REPO, git, make_source_repo +from conftest import (REPO, git, make_source_repo, + skip_where_a_name_cannot_be_not_utf_8) from test_adopt_template_values import (ADOPT_REFUSED, PLAN_FILE, PLAN_OK, PROJECT, REMOTES, VERIFIED, WORK, adopted, answer, @@ -427,12 +428,10 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): --source ` ended in this issue's traceback; it now adopts. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() try: source = make_source_repo(parent / "Thing") except AssertionError as exc: diff --git a/tests/test_adopt_plan_escapes.py b/tests/test_adopt_plan_escapes.py index cc1ef280..c6041a72 100644 --- a/tests/test_adopt_plan_escapes.py +++ b/tests/test_adopt_plan_escapes.py @@ -37,7 +37,8 @@ import pytest from conftest import (ADOPT, REPO, blank_unnamed_pin_sources, git, - make_source_repo, resolve) + make_source_repo, resolve, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import repo_shape # noqa: E402 @@ -398,12 +399,10 @@ def test_a_source_directory_that_is_not_utf8_is_shown_spelled(adopter, `UnicodeEncodeError` on a strict UTF-8 stdout after the plan was written. Both show it spelled now. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() try: source = make_source_repo(parent / "Thing") except AssertionError as exc: @@ -458,13 +457,7 @@ def test_a_leg_extracts_a_path_that_is_not_utf8(adopter, tmp_path): lifting that refusal reaches this, so the extraction is tested here, directly. Skipped where no file can be named with that byte (Windows, macOS's APFS): filter-repo works in a clone, which checks the name out.""" - try: - probe = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9.txt"))) - probe.write_bytes(b"") - probe.unlink() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no file here is named b'caf\\xe9.txt': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) source = odd_source(tmp_path, b"caf\xe9.txt") commit = git("rev-parse", "HEAD", cwd=source).stdout.strip() remote = tmp_path / "Northwind-code.git" diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index faf3a008..1351c43a 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -50,7 +50,8 @@ import pytest from conftest import (ADOPT, REPO, WINDOWS_SKIP, blank_unnamed_pin_sources, - make_source_repo, resolve) + make_source_repo, resolve, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -292,12 +293,10 @@ def test_execute_refuses_a_local_source_directory_that_is_not_utf8(tmp_path): path, which `_remote_repository` reads as UTF-8. Skipped where no directory can be named with the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" - try: - parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), - b"caf\xe9"))) - parent.mkdir() - except (OSError, UnicodeError) as exc: - pytest.skip(f"no directory here is named b'caf\\xe9': {exc!r}") + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), + b"caf\xe9"))) + parent.mkdir() refused_then_moved(tmp_path, parent, b"a byte that is not UTF-8") diff --git a/tests/test_clone_url_bytes.py b/tests/test_clone_url_bytes.py index 373b8de7..13c0bca5 100644 --- a/tests/test_clone_url_bytes.py +++ b/tests/test_clone_url_bytes.py @@ -44,7 +44,8 @@ import pytest -from conftest import ORG, SCAFFOLD, blank_unnamed_pin_sources +from conftest import (ORG, SCAFFOLD, blank_unnamed_pin_sources, + skip_where_a_name_cannot_be_not_utf_8) # Fixtures of the #282 file, on the harness this one shares with it: the # scaffold loaded as a module, and the guard that no process is started. from test_scaffold_free_text_values import ( # noqa: F401 - fixtures @@ -133,18 +134,34 @@ def family_refusal(url: str, reason: str) -> str: f"{FAMILY_REMEDIATION}\n") +def skip_where_this_system_cannot_name(tmp_path: Path, raw: bytes) -> None: + """Skip where `tmp_path/` cannot be made on this system. Two + questions, each asked only of the names it is about: + + - a name that is not UTF-8 goes to `skip_where_a_name_cannot_be_not_utf_8`, + which skips on the two answers that mean "this system cannot carry + it" (Windows names its files in UTF-16, and macOS answers EILSEQ) and + raises any other error of the disk; + - a control character, 0x00 to 0x1F, is refused by Windows in any name, + and a name that holds one is skipped there. + + Nothing else is skipped: U+202E, which NTFS holds, is a name Windows + makes as Linux does (#757; reasoned, no Windows run: CI is Linux only). + `tests/test_non_utf8_name_callers.py` holds the answers by emulation.""" + try: + raw.decode("utf-8") + except UnicodeDecodeError: + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + if os.name == "nt" and any(byte < 0x20 for byte in raw): + pytest.skip("Windows refuses a control character in a file name") + + def directory_named(tmp_path: Path, raw: bytes) -> Path: - """`tmp_path/` made from BYTES, empty; the test is skipped where the - disk cannot hold such a name (macOS refuses a name that is not UTF-8 and - Windows names its files in UTF-16, and refuses a control character).""" - if os.name == "nt": - pytest.skip("Windows names its files in UTF-16, which has no byte " - "that is not UTF-8, and refuses a control character") + """`tmp_path/` made from BYTES, empty; the test is skipped where this + system cannot hold such a name, and fails on any other error of the disk.""" + skip_where_this_system_cannot_name(tmp_path, raw) made = os.fsencode(tmp_path) + os.fsencode(os.sep) + raw - try: - os.mkdir(made) - except (OSError, ValueError, UnicodeError) as error: - pytest.skip(f"this disk cannot hold the name {raw!r}: {error}") + os.mkdir(made) path = Path(os.fsdecode(made)) assert path.is_dir() assert os.listdir(path) == [] diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index 4d19a998..ebe28bcd 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -123,7 +123,10 @@ def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( as text, and the tests that use it must run there.""" monkeypatch.setattr(os, "fsencode", windows_fsencode) target = tmp_path / UTF8 - assert skip_message(make_file, target, b"on: [pull_request]\n") is None + reason = skip_message(make_file, target, b"on: [pull_request]\n") + if reason is not None and "filesystem" in reason: + pytest.skip(reason) + assert reason is None, reason assert target.read_bytes() == b"on: [pull_request]\n" @@ -490,6 +493,22 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, "Windows converts it") LEGACY_NAME = "Jos\udce9" + +def skip_where_the_environment_cannot_carry(value: str) -> None: + """Skip where this system's file name encoding cannot encode `value`, as + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII): `subprocess` + puts each environment value through `os.fsencode` before it starts the + child, so a `user.name` of `Jos\\u00e9` raises `UnicodeEncodeError` in the + test and not in the tool (#757). A name that is ASCII, or a lone + surrogate that `os.fsencode` gives back as its byte, is carried.""" + try: + os.fsencode(value) + except UnicodeEncodeError: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) cannot carry {value!r} " + "in an environment") + + REFUSED = "REFUSED family-unwritable-value: " #: How the refusal names the creator when no flag supplied it: `_manifest_flags` #: (#307), so that the two refusals agree on who is meant. @@ -623,6 +642,7 @@ def test_a_user_name_that_is_text_still_initialises(tmp_path, value, dry_run): exit 0, `created_by: "Jose"`. Text a file holds is not refused: an accent, a U+200C ZERO WIDTH NON-JOINER, a backslash and a quote, which the scaffold lets through as well.""" + skip_where_the_environment_cannot_carry(value) result = init(tmp_path, user_name(value), dry_run=dry_run) assert result.returncode == 0, result.stderr + result.stdout assert "REFUSED" not in result.stderr, result.stderr @@ -643,6 +663,7 @@ def test_every_other_name_no_file_can_hold_is_refused_the_same_way( asks it of the same setting: a line break, a bidi control and a control character in `git config user.name` are refused in its words, where the byte that is not UTF-8 is the case the issue lists.""" + skip_where_the_environment_cannot_carry(value) result = init(tmp_path, user_name(value), dry_run=True) assert result.returncode == 2, result.stderr + result.stdout assert result.stderr.startswith( diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py new file mode 100644 index 00000000..57f4dbc0 --- /dev/null +++ b/tests/test_non_utf8_name_callers.py @@ -0,0 +1,285 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A test that needs a directory named with a byte that is not UTF-8 asks the +one helper whether this system can carry it, and every other error of the disk +is the test's own (#757, #461). + +Five tests and `directory_named` each made the directory themselves and turned +ANY `OSError` into `pytest.skip`, beside `skip_where_a_name_cannot_be_not_utf_8` +(#663), which skips on the two answers that mean "this system cannot carry the +name" -- Windows' `os.fsencode` and macOS's EILSEQ -- and raises the rest. So a +disk that filled up, or a `tmp_path` this user cannot write in, read as SKIPPED +there and as a failure in the helper's own tests, and no run fails on a skip. + +Each caller is called here as a plain function, with the disk answering as it +would on the systems this suite does not run on (CI is Linux only, #597), and +asked what it did: + +- the PROBE is refused with EILSEQ (macOS): the test skips, on the helper's + reason; +- the PROBE is refused with EACCES: the test fails with `PermissionError`; +- the test's OWN directory is refused with EACCES, the probe not: the test + fails with `PermissionError`, where it skipped. + +Windows is emulated too, for the one caller whose own question is wider than +the helper's: `directory_named` skips there for a control character, which +Windows refuses in any file name, and for no other name that is text. No +Windows run was made. + +The last two groups hold the C-locale half of the same issue. Under +`LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, +so `make_file` cannot write `caf\\u00e9.yml` and `subprocess` cannot put a +`user.name` of `Jos\\u00e9` into a child's environment: the tests that do it +skip where this system cannot, and are held to it here with an ASCII +`os.fsencode` and a `write_bytes` that raises as it would. +""" + +from __future__ import annotations + +import errno +import os +import sys +from pathlib import Path + +import pytest + +import test_adopt_git_reads as git_reads +import test_adopt_plan_escapes as plan_escapes +import test_adopt_template_values as template_values +import test_clone_url_bytes as clone_url_bytes +import test_non_utf8_git_reads as non_utf8_git_reads +import test_stdout_encoding as stdout_encoding +from conftest import NOT_UTF_8_PROBE, skip_message, windows_fsencode +# The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a +# module, which two of its tests ask for. +from test_adopt_plan_escapes import adopter # noqa: F401 - fixture + +#: Where the probe of the helper is: a directory named `probe`. +PROBE = "probe" + +#: Every caller of the helper, as `(id, the call, whether it makes a directory +#: of its own after the probe)`; the call is given a `tmp_path` and the +#: `adopter` fixture. `directory_named` is called with the name the issue's +#: own row uses, `caf` and byte 0xE9. +CALLERS = [ + pytest.param( + lambda tmp_path, adopter: stdout_encoding + .test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes( + tmp_path, "utf-8:surrogateescape"), + True, id="stdout-encoding"), + pytest.param( + lambda tmp_path, adopter: git_reads + .test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path), + True, id="adopt-git-reads"), + pytest.param( + lambda tmp_path, adopter: template_values + .test_execute_refuses_a_local_source_directory_that_is_not_utf8( + tmp_path), + True, id="adopt-template-values"), + pytest.param( + lambda tmp_path, adopter: plan_escapes + .test_a_source_directory_that_is_not_utf8_is_shown_spelled( + adopter, tmp_path), + True, id="adopt-plan-escapes-directory"), + pytest.param( + lambda tmp_path, adopter: clone_url_bytes.directory_named( + tmp_path, b"caf\xe9"), + True, id="clone-url-bytes"), + pytest.param( + lambda tmp_path, adopter: plan_escapes + .test_a_leg_extracts_a_path_that_is_not_utf8(adopter, tmp_path), + False, id="adopt-plan-escapes-file"), +] +EVERY_CALLER = [pytest.param(row.values[0], id=row.id) for row in CALLERS] +THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN = [ + pytest.param(row.values[0], id=row.id) for row in CALLERS if row.values[1]] + + +def refusing(code: int, *, probe: bool): + """An `os.mkdir` that fails with `code` for the probe (`probe=True`) or + for every directory but the probe, and makes the rest. `Path.mkdir` calls + it too.""" + real = os.mkdir + + def mkdir(path, *args, **kwargs): + name = os.path.basename(os.fsdecode(path)) + if name.startswith(PROBE) is probe: + raise OSError(code, os.strerror(code), os.fsdecode(path)) + return real(path, *args, **kwargs) + return mkdir + + +@pytest.fixture +def a_system_that_carries_the_byte(): + if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: + pytest.skip("on this system the first question is the one that skips") + + +# --- the disk's refusals, on the systems CI does not run -------------------- + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", EVERY_CALLER) +def test_a_disk_that_refuses_the_name_with_eilseq_skips_the_caller( + tmp_path, monkeypatch, adopter, call): + """macOS (emulated): the probe is refused as `Illegal byte sequence`, so + the test is SKIPPED, on the helper's reason.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(call, tmp_path, adopter) + assert reason is not None, "the caller went on past the probe's EILSEQ" + assert "EILSEQ" in reason, reason + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", EVERY_CALLER) +def test_a_disk_that_refuses_the_probe_with_eacces_fails_the_caller( + tmp_path, monkeypatch, adopter, call): + """A folder nobody can write in: the helper raises it, as its own tests + hold, so the test FAILS, and no run reads it as a skip.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=True)) + with pytest.raises(PermissionError): + reason = skip_message(call, tmp_path, adopter) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN) +def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( + tmp_path, monkeypatch, adopter, call): + """The probe is made and removed, and then the directory the test means + to use is refused with EACCES: that is the test's own, raised, where it + was a skip that said "no directory here is named ...".""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=False)) + with pytest.raises(PermissionError): + reason = skip_message(call, tmp_path, adopter) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + +# --- `directory_named`'s own question: Windows, emulated -------------------- + +def windows_names_it(monkeypatch) -> None: + """Windows, as far as `skip_where_this_system_cannot_name` can ask it: + `os.name` is `nt` and `os.fsencode` gives a surrogate three bytes.""" + monkeypatch.setattr(os, "name", "nt") + monkeypatch.setattr(os, "fsencode", windows_fsencode) + + +def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_nothing( + tmp_path, monkeypatch): + windows_names_it(monkeypatch) + reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, b"caf\xe9") + assert reason is not None, "a name that is not UTF-8 was not skipped" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("raw", [b"line\nbreak", b"a\x1bb"], + ids=["line-feed", "escape"]) +def test_where_windows_refuses_a_control_character_directory_named_skips( + tmp_path, monkeypatch, raw): + windows_names_it(monkeypatch) + reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) + assert reason is not None, f"{raw!r} was not skipped on Windows" + assert "control character" in reason, reason + + +def test_where_windows_holds_a_bidi_override_directory_named_does_not_skip( + tmp_path, monkeypatch): + """The row #459 names (R433 O5): NTFS holds U+202E, so the `bidi-override` + row of the clone-URL tests runs on Windows as it does here. Reasoned, and + asked here only of the skip: no Windows run was made.""" + windows_names_it(monkeypatch) + raw = "\u202eevil".encode("utf-8") + assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) is None + + +@pytest.mark.skipif(os.name == "nt", reason="Windows is the case above") +@pytest.mark.parametrize("raw", [b"line\nbreak", b"a\x1bb"], + ids=["line-feed", "escape"]) +def test_a_system_that_is_not_windows_makes_a_control_character_directory( + tmp_path, raw): + """The control: a POSIX disk takes the name, so no skip.""" + assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, + tmp_path, raw) is None + assert os.path.basename(clone_url_bytes.directory_named( + tmp_path, raw)) == os.fsdecode(raw) + + +# --- under an ASCII file name encoding (LC_ALL=C PYTHONUTF8=0) -------------- + +def ascii_fsencode(value): + """`os.fsencode` where the file name encoding is ASCII, with + `surrogateescape`: text outside ASCII raises, and a lone surrogate that + stands for a byte is that byte.""" + value = os.fspath(value) + if isinstance(value, bytes): + return value + return value.encode("ascii", "surrogateescape") + + +def test_make_files_utf8_name_test_skips_where_the_encoding_is_ascii( + tmp_path, monkeypatch): + """`make_file` cannot write `caf\\u00e9.yml` where the file name encoding is + ASCII, and says so as a skip on `filesystem`. The test about the guard + reads that skip as it reads it in its neighbour, and is SKIPPED: it failed + on `assert skip_message(make_file, ...) is None`, on a system that holds + no defect. Emulated: `write_bytes` raises what the ASCII encoding does.""" + real = Path.write_bytes + + def write_bytes(self, data): + for index, char in enumerate(self.name): + if not char.isascii(): + raise UnicodeEncodeError("ascii", self.name, index, index + 1, + "ordinal not in range(128)") + return real(self, data) + monkeypatch.setattr(Path, "write_bytes", write_bytes) + reason = skip_message( + non_utf8_git_reads + .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte, + tmp_path, monkeypatch) + assert reason is not None, "the test went on past an ASCII encoding" + assert "filesystem" in reason, reason + + +@pytest.mark.parametrize("value", ["Jos\u00e9", "Mehr\u200cdad", "\u202eevil"], + ids=["accent", "zwnj", "u202e"]) +def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( + monkeypatch, value): + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message( + non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) + assert reason is not None, f"{value!r} was carried by an ASCII encoding" + assert "environment" in reason, reason + + +@pytest.mark.parametrize("value", ["Jose", "Jos\udce9", "Ada\nLovelace", + "a\x1bb"], + ids=["ascii", "byte", "line-feed", "escape"]) +def test_a_user_name_the_encoding_carries_is_not_skipped_even_where_it_is_ascii( + monkeypatch, value): + """ASCII text, a lone surrogate that is a byte, and the control characters + are the environment's own: the tests of them still run.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + assert skip_message( + non_utf8_git_reads.skip_where_the_environment_cannot_carry, + value) is None + + +@pytest.mark.parametrize("test, args", [ + pytest.param("test_a_user_name_that_is_text_still_initialises", + ("Jos\u00e9", True), id="text-still-initialises"), + pytest.param("test_every_other_name_no_file_can_hold_is_refused_the_same_way", + ("\u202eevil", "it holds a bidirectional control"), + id="every-other-name"), +]) +def test_the_two_tests_that_pass_the_name_in_an_environment_skip_where_it_is_ascii( + tmp_path, monkeypatch, test, args): + """Both of them fail on the `UnicodeEncodeError` that `subprocess` raises + in the test (the tool is not reached), under an ASCII encoding.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(getattr(non_utf8_git_reads, test), tmp_path, *args) + assert reason is not None, "the test ran, and raised or passed on ASCII" + assert "environment" in reason, reason diff --git a/tests/test_stdout_encoding.py b/tests/test_stdout_encoding.py index 621b43c7..d3e0f7e9 100644 --- a/tests/test_stdout_encoding.py +++ b/tests/test_stdout_encoding.py @@ -51,7 +51,8 @@ import pytest from conftest import (ADOPT, ORG, PROJECT, REPO, SCAFFOLD, - blank_unnamed_pin_sources, git, make_source_repo) + blank_unnamed_pin_sources, git, make_source_repo, + skip_where_a_name_cannot_be_not_utf_8) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -206,11 +207,9 @@ def test_a_name_that_is_not_utf8_still_comes_out_as_its_own_bytes(tmp_path, `日本.md` -- escaped on cp1252 -- where plain `backslashreplace` would print `\\udcff` instead. POSIX only: neither Windows nor macOS holds a file name that is not UTF-8.""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = os.path.join(os.fsencode(tmp_path), b"b\xffd") - try: - os.mkdir(parent) - except (OSError, UnicodeError) as exc: - pytest.skip(f"this filesystem holds no name that is not UTF-8: {exc}") + os.mkdir(parent) if b"b\xffd" not in os.listdir(os.fsencode(tmp_path)): pytest.skip("this filesystem rewrote a name that is not UTF-8") source = make_source_repo(tmp_path / "Thing", tree={ From 6b5e9074ab01fcc89381b98d97444d370c27634a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 11:24:28 +0000 Subject: [PATCH 05/13] test_non_utf8_git_reads.py and test_non_utf8_name_callers.py: the `make_file` test asks the file name encoding first and keeps its strict assert, the rows that let the probe through ask the real disk first, and each `pytest.raises` block holds one call, so a write the disk refuses fails the test and no skip is made of it, a disk that refuses the probe skips the rows and does not fail them, and the new file has no S5778 smell (#757) A fix round for lane 1's review of #806 at 41849725 (three SHOULD FIX, one nit). 1. The `make_file` test skipped on `make_file`'s own "filesystem" reason, which `make_file` gives for ANY `OSError` of its write, so a write refused with EACCES or ENOSPC for the plain UTF-8 name `caf` and U+00E9 plus `.yml` ended the test SKIPPED, and the reason it gave was false. The test now calls `skip_where_the_environment_cannot_carry(UTF8)` first, before it swaps in the Windows `os.fsencode`, and keeps the strict `assert skip_message(make_file, ...) is None`: it skips for the file name encoding and for nothing else. The helper's reason loses " in an environment", which was wrong for a file name; the callers file reads "file name encoding". The ASCII row now patches `os.fsencode`, and two rows make `Path.write_bytes` raise EACCES and ENOSPC and expect the test to FAIL. 2. The own-directory rows let the probe through to the real disk, and their fixture asked only `os.fsencode`, so on macOS's APFS or a ZFS with `utf8only` the real probe answers EILSEQ, the helper skips, and the row failed. `a_disk_that_takes_the_probe` asks the real disk first and those rows alone use it; a row makes the disk answer EILSEQ and asks the fixture. 3. `fail_where_the_caller_ends` holds the `pytest.fail`, so each `pytest.raises` block holds one call (python:S5778, two places, and the new EACCES/ENOSPC rows). 4. The Windows rows for a byte and a control character now call `directory_named` itself, so its order (ask, then make) is held: a version that skipped after `os.mkdir` leaves a directory or fails on `os.fsencode` of a path. The bidi row asks `skip_where_this_system_cannot_name` and is named for it, because `directory_named` goes on to make a `Path`, and under an emulated `os.name` of `nt` that is a `WindowsPath`. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_non_utf8_git_reads.py | 26 +++--- tests/test_non_utf8_name_callers.py | 124 +++++++++++++++++++--------- 2 files changed, 100 insertions(+), 50 deletions(-) diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index ebe28bcd..ff81a3b3 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -120,13 +120,14 @@ def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( def test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte( tmp_path, monkeypatch): """The guard is for the byte and no wider: Windows holds `caf\\u00e9.yml` - as text, and the tests that use it must run there.""" + as text, and the tests that use it must run there. Skipped, before the + Windows `os.fsencode` is put in, where this system's own file name + encoding cannot write `caf\\u00e9.yml` (ASCII); and for nothing else, so a + write the disk refuses fails the test (#757).""" + skip_where_the_environment_cannot_carry(UTF8) monkeypatch.setattr(os, "fsencode", windows_fsencode) target = tmp_path / UTF8 - reason = skip_message(make_file, target, b"on: [pull_request]\n") - if reason is not None and "filesystem" in reason: - pytest.skip(reason) - assert reason is None, reason + assert skip_message(make_file, target, b"on: [pull_request]\n") is None assert target.read_bytes() == b"on: [pull_request]\n" @@ -496,17 +497,18 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, def skip_where_the_environment_cannot_carry(value: str) -> None: """Skip where this system's file name encoding cannot encode `value`, as - under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII): `subprocess` - puts each environment value through `os.fsencode` before it starts the - child, so a `user.name` of `Jos\\u00e9` raises `UnicodeEncodeError` in the - test and not in the tool (#757). A name that is ASCII, or a lone - surrogate that `os.fsencode` gives back as its byte, is carried.""" + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of + a file name (`make_file` cannot write `caf\\u00e9.yml`) and of an + environment value: `subprocess` puts each one through `os.fsencode` + before it starts the child, so a `user.name` of `Jos\\u00e9` raises + `UnicodeEncodeError` in the test and not in the tool (#757). Text that is + ASCII, or a lone surrogate that `os.fsencode` gives back as its byte, is + carried.""" try: os.fsencode(value) except UnicodeEncodeError: pytest.skip(f"this system's file name encoding " - f"({sys.getfilesystemencoding()}) cannot carry {value!r} " - "in an environment") + f"({sys.getfilesystemencoding()}) cannot carry {value!r}") REFUSED = "REFUSED family-unwritable-value: " diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 57f4dbc0..3b45ffe0 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -18,26 +18,30 @@ reason; - the PROBE is refused with EACCES: the test fails with `PermissionError`; - the test's OWN directory is refused with EACCES, the probe not: the test - fails with `PermissionError`, where it skipped. + fails with `PermissionError`, where it skipped. These rows let the probe + through to the real disk, so they ask it first and skip where it answers + EILSEQ (macOS's APFS, a ZFS with `utf8only`): they would FAIL there on a + system that holds no defect. Windows is emulated too, for the one caller whose own question is wider than the helper's: `directory_named` skips there for a control character, which Windows refuses in any file name, and for no other name that is text. No Windows run was made. -The last two groups hold the C-locale half of the same issue. Under +The last groups hold the C-locale half of the same issue. Under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` the file name encoding is ASCII, so `make_file` cannot write `caf\\u00e9.yml` and `subprocess` cannot put a -`user.name` of `Jos\\u00e9` into a child's environment: the tests that do it -skip where this system cannot, and are held to it here with an ASCII -`os.fsencode` and a `write_bytes` that raises as it would. +`user.name` of `Jos\\u00e9` into a child's environment: the three tests that +do it skip where this system's encoding cannot, held here with an ASCII +`os.fsencode`, and for that reason ALONE: a write the disk refuses (EACCES, +ENOSPC) still fails the `make_file` test, which a skip on `make_file`'s own +"filesystem" reason would have turned into a skip. """ from __future__ import annotations import errno import os -import sys from pathlib import Path import pytest @@ -48,7 +52,8 @@ import test_clone_url_bytes as clone_url_bytes import test_non_utf8_git_reads as non_utf8_git_reads import test_stdout_encoding as stdout_encoding -from conftest import NOT_UTF_8_PROBE, skip_message, windows_fsencode +from conftest import (NOT_UTF_8_PROBE, skip_message, + skip_where_a_name_cannot_be_not_utf_8, windows_fsencode) # The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a # module, which two of its tests ask for. from test_adopt_plan_escapes import adopter # noqa: F401 - fixture @@ -108,12 +113,29 @@ def mkdir(path, *args, **kwargs): return mkdir +def fail_where_the_caller_ends(call, *args) -> None: + """Fail where `call(*args)` ends without an error, SKIPPED or not: the + caller was to raise. A `pytest.raises` block may hold one call and no + more (python:S5778), so the failure is made here and not beside it.""" + reason = skip_message(call, *args) + pytest.fail("the caller ended " + + (f"SKIPPED: {reason}" if reason else "without an error")) + + @pytest.fixture def a_system_that_carries_the_byte(): if len(os.fsencode(NOT_UTF_8_PROBE)) != 1: pytest.skip("on this system the first question is the one that skips") +@pytest.fixture +def a_disk_that_takes_the_probe(tmp_path, a_system_that_carries_the_byte): + """The rows that let the probe through to the real disk ask it first: + macOS's APFS, or a ZFS with `utf8only`, answers EILSEQ to it, and those + rows would FAIL there on a system that holds no defect.""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) + + # --- the disk's refusals, on the systems CI does not run -------------------- @pytest.mark.usefixtures("a_system_that_carries_the_byte") @@ -136,12 +158,10 @@ def test_a_disk_that_refuses_the_probe_with_eacces_fails_the_caller( hold, so the test FAILS, and no run reads it as a skip.""" monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=True)) with pytest.raises(PermissionError): - reason = skip_message(call, tmp_path, adopter) - pytest.fail("the caller ended " - + (f"SKIPPED: {reason}" if reason else "without an error")) + fail_where_the_caller_ends(call, tmp_path, adopter) -@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") @pytest.mark.parametrize("call", THOSE_WITH_A_DIRECTORY_OF_THEIR_OWN) def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( tmp_path, monkeypatch, adopter, call): @@ -150,9 +170,20 @@ def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( was a skip that said "no directory here is named ...".""" monkeypatch.setattr(os, "mkdir", refusing(errno.EACCES, probe=False)) with pytest.raises(PermissionError): - reason = skip_message(call, tmp_path, adopter) - pytest.fail("the caller ended " - + (f"SKIPPED: {reason}" if reason else "without an error")) + fail_where_the_caller_ends(call, tmp_path, adopter) + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +def test_the_fixture_that_guards_those_rows_asks_the_real_disk( + tmp_path, monkeypatch, request): + """`a_disk_that_takes_the_probe` skips where the disk answers EILSEQ to the + probe, as APFS does; asked here after the disk is made to, because a + fixture runs before the test body and Linux's disk would never say it.""" + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(request.getfixturevalue, + "a_disk_that_takes_the_probe") + assert reason is not None, "the fixture let a refused probe through" + assert "EILSEQ" in reason, reason # --- `directory_named`'s own question: Windows, emulated -------------------- @@ -166,9 +197,12 @@ def windows_names_it(monkeypatch) -> None: def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_nothing( tmp_path, monkeypatch): + """`directory_named` itself is called, so that it asks before it makes + anything: a version that skipped after its `os.mkdir` would leave a + directory here, or fail on `os.fsencode` of a path under the emulation.""" windows_names_it(monkeypatch) - reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, - tmp_path, b"caf\xe9") + reason = skip_message(clone_url_bytes.directory_named, tmp_path, + b"caf\xe9") assert reason is not None, "a name that is not UTF-8 was not skipped" assert "os.fsencode" in reason, reason assert list(tmp_path.iterdir()) == [] @@ -179,17 +213,20 @@ def test_where_windows_cannot_carry_the_byte_directory_named_skips_and_makes_not def test_where_windows_refuses_a_control_character_directory_named_skips( tmp_path, monkeypatch, raw): windows_names_it(monkeypatch) - reason = skip_message(clone_url_bytes.skip_where_this_system_cannot_name, - tmp_path, raw) + reason = skip_message(clone_url_bytes.directory_named, tmp_path, raw) assert reason is not None, f"{raw!r} was not skipped on Windows" assert "control character" in reason, reason + assert list(tmp_path.iterdir()) == [] -def test_where_windows_holds_a_bidi_override_directory_named_does_not_skip( +def test_where_windows_holds_a_bidi_override_the_name_check_does_not_skip( tmp_path, monkeypatch): """The row #459 names (R433 O5): NTFS holds U+202E, so the `bidi-override` row of the clone-URL tests runs on Windows as it does here. Reasoned, and - asked here only of the skip: no Windows run was made.""" + asked here only of the skip: no Windows run was made. It asks + `skip_where_this_system_cannot_name` and not `directory_named`, which + goes on to make a `Path`, and under an emulated `os.name` of `nt` that is + a `WindowsPath`, which Linux cannot make.""" windows_names_it(monkeypatch) raw = "\u202eevil".encode("utf-8") assert skip_message(clone_url_bytes.skip_where_this_system_cannot_name, @@ -220,28 +257,39 @@ def ascii_fsencode(value): return value.encode("ascii", "surrogateescape") +MAKE_FILE_TEST = ( + non_utf8_git_reads + .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte) + + def test_make_files_utf8_name_test_skips_where_the_encoding_is_ascii( tmp_path, monkeypatch): """`make_file` cannot write `caf\\u00e9.yml` where the file name encoding is - ASCII, and says so as a skip on `filesystem`. The test about the guard - reads that skip as it reads it in its neighbour, and is SKIPPED: it failed - on `assert skip_message(make_file, ...) is None`, on a system that holds - no defect. Emulated: `write_bytes` raises what the ASCII encoding does.""" - real = Path.write_bytes + ASCII, so the test about the guard failed on `assert skip_message(...) + is None`, on a system that holds no defect. It asks the encoding first, + and is SKIPPED on that reason.""" + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(MAKE_FILE_TEST, tmp_path, monkeypatch) + assert reason is not None, "the test went on past an ASCII encoding" + assert "file name encoding" in reason, reason + + +@pytest.mark.parametrize("code", [errno.EACCES, errno.ENOSPC], + ids=["EACCES", "ENOSPC"]) +def test_a_write_the_disk_refuses_still_fails_the_make_file_test( + tmp_path, monkeypatch, code): + """The name is plain UTF-8, so a refused write is the disk's and not the + encoding's: the test FAILS on its assert, as it does without this change. + A skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and + no run fails on that.""" + non_utf8_git_reads.skip_where_the_environment_cannot_carry( + non_utf8_git_reads.UTF8) def write_bytes(self, data): - for index, char in enumerate(self.name): - if not char.isascii(): - raise UnicodeEncodeError("ascii", self.name, index, index + 1, - "ordinal not in range(128)") - return real(self, data) + raise OSError(code, os.strerror(code), str(self)) monkeypatch.setattr(Path, "write_bytes", write_bytes) - reason = skip_message( - non_utf8_git_reads - .test_make_file_writes_a_name_that_is_utf8_where_fsencode_makes_no_byte, - tmp_path, monkeypatch) - assert reason is not None, "the test went on past an ASCII encoding" - assert "filesystem" in reason, reason + with pytest.raises(AssertionError): + fail_where_the_caller_ends(MAKE_FILE_TEST, tmp_path, monkeypatch) @pytest.mark.parametrize("value", ["Jos\u00e9", "Mehr\u200cdad", "\u202eevil"], @@ -252,7 +300,7 @@ def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( reason = skip_message( non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) assert reason is not None, f"{value!r} was carried by an ASCII encoding" - assert "environment" in reason, reason + assert "file name encoding" in reason, reason @pytest.mark.parametrize("value", ["Jose", "Jos\udce9", "Ada\nLovelace", @@ -282,4 +330,4 @@ def test_the_two_tests_that_pass_the_name_in_an_environment_skip_where_it_is_asc monkeypatch.setattr(os, "fsencode", ascii_fsencode) reason = skip_message(getattr(non_utf8_git_reads, test), tmp_path, *args) assert reason is not None, "the test ran, and raised or passed on ASCII" - assert "environment" in reason, reason + assert "file name encoding" in reason, reason From b18b243e1ec7fe39060e5401d95e7edc8c259692 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 19:19:32 +0000 Subject: [PATCH 06/13] tests: three tests ask the helper whether a name can be made, `make_file` skips on EILSEQ or a `UnicodeError` only and the U+202E directory has no guard, so a full disk or a read-only `tmp_path` fails the test, not SKIPPED (#834) Five tests that make a directory or a file named with a byte that is not UTF-8, or with U+202E, turned ANY `OSError` into `pytest.skip`, beside `skip_where_a_name_cannot_be_not_utf_8` (#663). A disk that filled up, a read-only `tmp_path` or a folder this user cannot write in read as SKIPPED, and no run fails on a skip. - `test_a_checkout_whose_path_is_not_utf8_is_a_checkout` (test_cli_reference.py) and `test_a_name_that_is_not_utf8_is_never_an_exception_quoted_as_a_finding` (test_doctor_validator_verdicts.py) ask the helper on `tmp_path` and make their directory with no `try`. - `test_a_branch_whose_name_is_not_utf8_is_read_back_on_every_run` (test_bootstrap_leg_moves.py) asks the helper in place of its `write_bytes` probe, which skipped on `OSError` and `ValueError`. - `make_file` (test_non_utf8_git_reads.py) keeps the `UnicodeError` skip (an ASCII file name encoding, #757) and skips an `OSError` only where its errno is EILSEQ (macOS); ENOSPC, EROFS and EACCES are raised, so each caller inherits a failure and no skip. #757's test of a refused write (test_non_utf8_name_callers.py) now expects that `OSError`, where it expected the AssertionError of the test's own "not a skip" assert. - The U+202E directory (test_adopt_template_values.py) is valid text, which no system known here refuses, so its `mkdir` is unguarded. Read, not run, on macOS and Windows; Linux takes it. tests/test_non_utf8_oserror_sites.py calls each as a function, with the disk answering as it would on a system CI does not run (Linux only, #597): EILSEQ on the probe (macOS) skips, EACCES, ENOSPC and EROFS on the probe or on the test's own name raise, and an `os.fsencode` that gives a surrogate three bytes (Windows) skips the three helper callers before the disk is touched. On the base 30 of its 32 tests fail, and so do the 2 changed ones of test_non_utf8_name_callers.py; the other 2 hold what both keep, the EILSEQ skip and the `UnicodeError` skip of `make_file`. No Windows or macOS run was made. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_template_values.py | 10 +- tests/test_bootstrap_leg_moves.py | 13 +- tests/test_cli_reference.py | 10 +- tests/test_doctor_validator_verdicts.py | 16 +- tests/test_non_utf8_git_reads.py | 11 +- tests/test_non_utf8_name_callers.py | 10 +- tests/test_non_utf8_oserror_sites.py | 239 ++++++++++++++++++++++++ 7 files changed, 279 insertions(+), 30 deletions(-) create mode 100644 tests/test_non_utf8_oserror_sites.py diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index fd60cf3b..0cb86dcf 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -306,12 +306,12 @@ def test_execute_refuses_a_local_source_directory_holding_a_bidi_control( """A source under a directory whose name begins with U+202E RIGHT-TO-LEFT OVERRIDE: on main `execute` adopted it, and the shape's README showed the rest of its `git clone` line reversed. Now it is refused as the - byte is, and moving the source adopts.""" + byte is, and moving the source adopts. The directory is made with no + guard: U+202E is valid text, which a POSIX disk, NTFS and APFS hold + (read, not run on the last two), so an error of the disk is the + test's own and is raised, not skipped (#834).""" parent = tmp_path / "\u202ecaf" - try: - parent.mkdir() - except OSError as exc: - pytest.skip(f"no directory here is named '\\u202ecaf': {exc!r}") + parent.mkdir() refused_then_moved(tmp_path, parent, b"a bidirectional control") diff --git a/tests/test_bootstrap_leg_moves.py b/tests/test_bootstrap_leg_moves.py index b0a950a9..41f1d43b 100644 --- a/tests/test_bootstrap_leg_moves.py +++ b/tests/test_bootstrap_leg_moves.py @@ -55,7 +55,8 @@ import pytest -from conftest import IDENTITY_CONFIG, git, run_script +from conftest import (IDENTITY_CONFIG, git, run_script, + skip_where_a_name_cannot_be_not_utf_8) from test_bootstrap_after_bump import ( # noqa: F401 ADDED, EDITED, MAIN, NOTHING_ELSE, bootstrapped, bumped, failing_post_checkout, head_ref, left_branch, moved_and_refused, rev, @@ -546,11 +547,11 @@ def test_a_branch_whose_name_is_not_utf8_is_read_back_on_every_run( the check of where `checkout -b` left HEAD (S3) reads HEAD back; the second places the leg on it. Each read `symbolic-ref` decoded as UTF-8, and on base the second run ended in a `UnicodeDecodeError` traceback. - Made on a disk that takes the byte, skipped on one that does not.""" - try: - (tmp_path / "probe-\udcff").write_bytes(b"") - except (OSError, ValueError): - pytest.skip("this disk takes no file name with the byte 0xff") + Made on a system and a disk that take the byte, skipped on one that + does not: `skip_where_a_name_cannot_be_not_utf_8` asks. Any other + error of the disk, a full one or a read-only `tmp_path`, fails the + test (#834).""" + skip_where_a_name_cannot_be_not_utf_8(tmp_path) leg = root / "spec" pin = rev(leg) git("checkout", "-q", "--detach", pin, cwd=leg) diff --git a/tests/test_cli_reference.py b/tests/test_cli_reference.py index a770e1bf..6863056b 100644 --- a/tests/test_cli_reference.py +++ b/tests/test_cli_reference.py @@ -49,7 +49,8 @@ import pytest -from conftest import REPO, WINDOWS_SKIP, git +from conftest import (REPO, WINDOWS_SKIP, git, + skip_where_a_name_cannot_be_not_utf_8) REFERENCE = REPO / "docs" / "cli.md" GENERATOR = REPO / "scripts" / "render-cli-reference.py" @@ -642,12 +643,9 @@ def test_a_checkout_whose_path_ends_in_a_space_is_a_checkout(tmp_path): @WINDOWS_SKIP def test_a_checkout_whose_path_is_not_utf8_is_a_checkout(tmp_path): + skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = tmp_path / os.fsdecode(b"bad\xffname") - try: - parent.mkdir() - except OSError as error: - pytest.skip(f"this file system cannot hold a name that is not UTF-8: " - f"{error}") + parent.mkdir() tree = a_checkout_at(parent / "openRepoShape") with pytest.raises(UnicodeDecodeError): os.fsencode(tree).decode("utf-8") diff --git a/tests/test_doctor_validator_verdicts.py b/tests/test_doctor_validator_verdicts.py index 7c41223c..0f4f5ebc 100644 --- a/tests/test_doctor_validator_verdicts.py +++ b/tests/test_doctor_validator_verdicts.py @@ -48,7 +48,8 @@ import pytest -from conftest import WINDOWS_SKIP, clone_just_pushed, run_script +from conftest import (WINDOWS_SKIP, clone_just_pushed, run_script, + skip_where_a_name_cannot_be_not_utf_8) #: The layer stand-in a chain descendant pins, from the module that scaffolds #: one: a second way of building it would test a different one. from test_scaffold_pin_and_reuse import layer_repo @@ -542,16 +543,15 @@ def test_a_name_that_is_not_utf8_is_never_an_exception_quoted_as_a_finding( today the row is the crash row; a script that prints such a name will make it the `naming-unclassified` FINDING, and either is a true row -- what this test holds is that the exception is never the finding. - `WINDOWS_SKIP`: Windows has no name that is not UTF-8, and a disk that - refuses one (APFS) skips. + `WINDOWS_SKIP`: Windows has no name that is not UTF-8. A system or a + disk that cannot carry one (APFS's EILSEQ) skips, asked of + `skip_where_a_name_cannot_be_not_utf_8`; any other error of the disk, + a full one or a read-only `tmp_path`, fails the test (#834). """ + skip_where_a_name_cannot_be_not_utf_8(tmp_path) name = os.fsdecode(b"Atl\xffas") here = tmp_path / "elsewhere" / name - try: - here.mkdir(parents=True) - except OSError as exc: - pytest.skip(f"this filesystem refuses a name that is not UTF-8: " - f"{exc}") + here.mkdir(parents=True) result = doctor(standard, here, "--json") payload = payload_of(result) row = rows_of(result)["naming"] diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index ff81a3b3..e35b64c2 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -43,6 +43,7 @@ from __future__ import annotations +import errno import os import shutil import subprocess @@ -93,11 +94,19 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: system that carries the byte and a filesystem that stores it (Linux does; APFS refuses it, and on Windows `os.fsencode` makes no byte of it, which is asked FIRST: NTFS most likely takes the surrogate, and the file would - then not be the one the test means). A name that is UTF-8 is written.""" + then not be the one the test means). A name that is UTF-8 is written. + + Two answers are the system's: a file name encoding that cannot spell + the name (`UnicodeError`: ASCII under `LC_ALL=C`) and a disk that + refuses the name (EILSEQ: macOS). Every other `OSError`, a full disk + (ENOSPC), a read-only one (EROFS) or a folder nobody can write in + (EACCES), is the test's own and is raised (#834).""" skip_where_fsencode_gives_a_surrogate_no_byte(path.name) try: path.write_bytes(data) except (OSError, UnicodeError) as exc: + if isinstance(exc, OSError) and exc.errno != errno.EILSEQ: + raise pytest.skip("this filesystem cannot hold a file name that is not " f"UTF-8: {exc}") diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 3b45ffe0..13aa5518 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -279,17 +279,19 @@ def test_make_files_utf8_name_test_skips_where_the_encoding_is_ascii( def test_a_write_the_disk_refuses_still_fails_the_make_file_test( tmp_path, monkeypatch, code): """The name is plain UTF-8, so a refused write is the disk's and not the - encoding's: the test FAILS on its assert, as it does without this change. - A skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and - no run fails on that.""" + encoding's: `make_file` raises it (#834), and the test FAILS on it. A + skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and + no run fails on that; before #834 it did skip, and the test failed on + its assert that the call is not a skip.""" non_utf8_git_reads.skip_where_the_environment_cannot_carry( non_utf8_git_reads.UTF8) def write_bytes(self, data): raise OSError(code, os.strerror(code), str(self)) monkeypatch.setattr(Path, "write_bytes", write_bytes) - with pytest.raises(AssertionError): + with pytest.raises(OSError) as raised: fail_where_the_caller_ends(MAKE_FILE_TEST, tmp_path, monkeypatch) + assert raised.value.errno == code @pytest.mark.parametrize("value", ["Jos\u00e9", "Mehr\u200cdad", "\u202eevil"], diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py new file mode 100644 index 00000000..9b4109fc --- /dev/null +++ b/tests/test_non_utf8_oserror_sites.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A test that makes a directory or a file named with a byte that is not UTF-8 +(or with U+202E) skips on the system's answer and on no other error of the +disk (#834, #461). + +Five more places beside the ones #757 moved onto `skip_where_a_name_cannot_be_ +not_utf_8` (#663) turned ANY `OSError` into `pytest.skip`, so a disk that +filled up, a read-only `tmp_path` or a folder this user cannot write in read as +SKIPPED, and no run fails on a skip: + +- `test_a_checkout_whose_path_is_not_utf8_is_a_checkout` + (`tests/test_cli_reference.py`) and + `test_a_name_that_is_not_utf8_is_never_an_exception_quoted_as_a_finding` + (`tests/test_doctor_validator_verdicts.py`) made their own directory; +- `test_a_branch_whose_name_is_not_utf8_is_read_back_on_every_run` + (`tests/test_bootstrap_leg_moves.py`) probed the disk with `write_bytes`; +- `make_file` (`tests/test_non_utf8_git_reads.py`) skipped on any `OSError` of + its `write_bytes`, so each of its callers inherited the skip; +- the U+202E directory of `tests/test_adopt_template_values.py` is valid text, + which no system known here refuses, so it skipped for nothing but a fault. + +The first three now ask the helper, which skips on the two answers that mean +"this system cannot carry the name" (Windows' `os.fsencode` and macOS's EILSEQ) +and raises the rest. `make_file` skips on EILSEQ and on a `UnicodeError` (an +ASCII file name encoding, #757) and raises the rest, and the U+202E test makes +its directory with no guard. + +Each is called here as a plain function, with the disk answering as it would on +a system this suite does not run on (CI is Linux only, #597), and asked what +it did. No Windows or macOS run was made: + +- a disk that refuses the PROBE with EILSEQ (macOS): the test skips, on the + helper's reason; +- a disk that refuses the probe, or the test's own name, with EACCES, ENOSPC + or EROFS: the test raises it, where it skipped; +- Windows, by an `os.fsencode` that gives a surrogate three bytes: the three + tests that ask the helper skip before the disk is touched. +""" + +from __future__ import annotations + +import errno +import os +from pathlib import Path + +import pytest + +import test_adopt_template_values as template_values +import test_bootstrap_leg_moves as leg_moves +import test_cli_reference as cli_reference +import test_doctor_validator_verdicts as doctor_verdicts +import test_non_utf8_git_reads as non_utf8_git_reads +from conftest import skip_message, windows_fsencode +# The fixtures and the two functions of #757's callers file, which ask the same +# questions of the same helper. +from test_non_utf8_name_callers import ( # noqa: F401 - fixtures + PROBE, a_disk_that_takes_the_probe, a_system_that_carries_the_byte, + fail_where_the_caller_ends) + +#: The three tests that ask `skip_where_a_name_cannot_be_not_utf_8` for the +#: name of a directory under their `tmp_path`, each called with that and, for +#: the two that take a fixture of the project, `None`: they raise or skip +#: before they use it. +ASK_THE_HELPER = [ + pytest.param( + lambda tmp_path: cli_reference + .test_a_checkout_whose_path_is_not_utf8_is_a_checkout(tmp_path), + id="cli-reference"), + pytest.param( + lambda tmp_path: doctor_verdicts + .test_a_name_that_is_not_utf8_is_never_an_exception_quoted_as_a_finding( + None, tmp_path), + id="doctor-validator-verdicts"), + pytest.param( + lambda tmp_path: leg_moves + .test_a_branch_whose_name_is_not_utf8_is_read_back_on_every_run( + None, tmp_path), + id="bootstrap-leg-moves"), +] +#: Those of them that go on to make a directory of their own after the probe. +THEN_MAKE_THEIR_OWN = [row for row in ASK_THE_HELPER + if row.id != "bootstrap-leg-moves"] + +#: What the disk refuses with, and the exception Python raises for it. +REFUSALS = [pytest.param(errno.EACCES, PermissionError, id="EACCES"), + pytest.param(errno.ENOSPC, OSError, id="ENOSPC"), + pytest.param(errno.EROFS, OSError, id="EROFS")] + + +def the_probe(name: str) -> bool: + return name.startswith(PROBE) + + +def anything_but_the_probe(name: str) -> bool: + return not the_probe(name) + + +def the_error_of(kind, call, *args) -> OSError: + """The `kind` of error `call(*args)` raises. A call that skips or ends + without one FAILS the test, as `fail_where_the_caller_ends` has it.""" + with pytest.raises(kind) as raised: + fail_where_the_caller_ends(call, *args) + return raised.value + + +def a_disk_that_refuses(monkeypatch, code: int, names=None) -> None: + """From here on a disk that answers `code` to every directory (`os.mkdir`, + which `Path.mkdir` calls) and every file (`Path.write_bytes`) whose last + part `names` accepts, to all of them where it is None, and makes the + rest.""" + real_mkdir = os.mkdir + real_write_bytes = Path.write_bytes + + def refused(name: str) -> bool: + return names is None or names(name) + + def mkdir(path, *args, **kwargs): + if refused(os.path.basename(os.fsdecode(path))): + raise OSError(code, os.strerror(code), os.fsdecode(path)) + return real_mkdir(path, *args, **kwargs) + + def write_bytes(self, data): + if refused(self.name): + raise OSError(code, os.strerror(code), str(self)) + return real_write_bytes(self, data) + + monkeypatch.setattr(os, "mkdir", mkdir) + monkeypatch.setattr(Path, "write_bytes", write_bytes) + + +# --- the three tests that ask the helper ------------------------------------ + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", ASK_THE_HELPER) +def test_a_disk_that_refuses_the_probe_with_eilseq_skips_the_test( + tmp_path, monkeypatch, call): + """macOS (emulated): the probe is `Illegal byte sequence`, so the test is + SKIPPED, on the helper's reason, and nothing else is asked of the disk.""" + a_disk_that_refuses(monkeypatch, errno.EILSEQ, the_probe) + reason = skip_message(call, tmp_path) + assert reason is not None, "the test went on past the probe's EILSEQ" + assert "EILSEQ" in reason, reason + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("code, kind", REFUSALS) +@pytest.mark.parametrize("call", ASK_THE_HELPER) +def test_a_disk_that_refuses_the_probe_for_any_other_reason_fails_the_test( + tmp_path, monkeypatch, call, code, kind): + """A full disk, a read-only one or a folder nobody can write in: the + helper raises it, so the test FAILS, and no run reads it as a skip.""" + a_disk_that_refuses(monkeypatch, code, the_probe) + assert the_error_of(kind, call, tmp_path).errno == code + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +@pytest.mark.parametrize("code, kind", REFUSALS) +@pytest.mark.parametrize("call", THEN_MAKE_THEIR_OWN) +def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( + tmp_path, monkeypatch, call, code, kind): + """The probe is made and removed, and then the directory the test means + to use is refused: that is the test's own, raised, where it was a skip. The + rows ask the real disk for the probe first, and skip where it answers + EILSEQ: they would FAIL there on a system that holds no defect.""" + a_disk_that_refuses(monkeypatch, code, anything_but_the_probe) + assert the_error_of(kind, call, tmp_path).errno == code + + +@pytest.mark.parametrize("call", ASK_THE_HELPER) +def test_where_fsencode_makes_no_byte_of_the_surrogate_the_test_skips_first( + tmp_path, monkeypatch, call): + """Windows (emulated): the helper is asked BEFORE the disk, which would + take the name, so the test skips on the system's reason and nothing is + made under `tmp_path`: a test that made its directory first, or probed + the disk and went on, ends without a skip here.""" + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, "the test went on where fsencode gives no byte" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +# --- `make_file` ------------------------------------------------------------ + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +def test_make_file_skips_where_the_disk_answers_eilseq(tmp_path, monkeypatch): + """macOS (emulated): APFS refuses a file name that is not UTF-8 with + `Illegal byte sequence`, which is the disk's answer, so the file is not + written and the test that asked is skipped.""" + a_disk_that_refuses(monkeypatch, errno.EILSEQ) + target = tmp_path / non_utf8_git_reads.NOT_UTF8 + reason = skip_message(non_utf8_git_reads.make_file, target) + assert reason is not None, "make_file went on past the disk's EILSEQ" + assert "filesystem" in reason, reason + + +def test_make_file_skips_where_the_file_name_encoding_cannot_spell_the_name( + tmp_path, monkeypatch): + """ASCII under `LC_ALL=C` (#757): the write raises `UnicodeEncodeError` + before the disk is asked, which is the system's answer.""" + def write_bytes(self, data): + raise UnicodeEncodeError("ascii", self.name, 0, 1, + "ordinal not in range(128)") + monkeypatch.setattr(Path, "write_bytes", write_bytes) + target = tmp_path / non_utf8_git_reads.UTF8 + reason = skip_message(non_utf8_git_reads.make_file, target) + assert reason is not None, "make_file raised where the encoding says no" + assert "filesystem" in reason, reason + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("name", [non_utf8_git_reads.NOT_UTF8, + non_utf8_git_reads.UTF8], + ids=["not-utf-8", "utf-8"]) +@pytest.mark.parametrize("code, kind", REFUSALS) +def test_make_file_raises_any_other_error_of_the_disk( + tmp_path, monkeypatch, code, kind, name): + """A full disk (ENOSPC), a read-only one (EROFS) or a folder nobody can + write in (EACCES) is the test's own, for a name that is not UTF-8 and for + one that is: raised as it was, and never a skip that hides it.""" + a_disk_that_refuses(monkeypatch, code) + error = the_error_of(kind, non_utf8_git_reads.make_file, tmp_path / name) + assert error.errno == code + + +# --- the U+202E directory --------------------------------------------------- + +@pytest.mark.parametrize("code, kind", REFUSALS) +def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( + tmp_path, monkeypatch, code, kind): + """U+202E is valid text: no probe is asked, and the directory is made with + no guard, so what the disk answers to it is raised. It was a skip that said + "no directory here is named ..." for a full disk or a read-only one.""" + a_disk_that_refuses(monkeypatch, code) + error = the_error_of( + kind, template_values + .test_execute_refuses_a_local_source_directory_holding_a_bidi_control, + tmp_path) + assert error.errno == code From efb17273667ef85759a50541d211415ecf712a25 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 20:13:13 +0000 Subject: [PATCH 07/13] tests: the emulated disk answers every errno but EILSEQ, `make_file` gives its two skips their own reasons, and the U+202E test fails where git fails, so a guard that skips the rest or a full disk after the `mkdir` no longer passes (#834) Fix round on #845, after lane 4's review of b18b243e (READY, no MUST). SHOULD 1. `REFUSALS` held ENOSPC, EROFS and EACCES, so a guard that re-raised those three and skipped every other error passed all of the tests. The emulated disk now answers every errno `errno.errorcode` holds but EILSEQ (129 of them), and each caller must raise it, as Python makes it (`PermissionError` for EACCES). The sweep stops at the first errno that fails and names it, so a caller that goes on past a refused name runs its test once and not 129 times. The explicit three-row families are gone, as the sweeps cover them; the new file has 18 tests where it had 32. SHOULD 2. The U+202E test's docstring claimed more than `refused_then_moved` did: its `make_source_repo` call still skipped on an `AssertionError`, so a disk that filled after the `mkdir` read as SKIPPED. Made true and not narrowed: `refused_then_moved` takes `skip_where_git_fails` (default True, as #757 left it for the name that is not UTF-8, which git may not work under), and the U+202E test passes False, as the name is valid text and git failing there is the disk. Two new tests hold it, and a third holds the kept skip for the byte name, which stays a known limit. NIT 3. `make_file` gives the two skips their own reasons: "this system's file name encoding cannot spell ..." for a `UnicodeError`, and "this filesystem refuses a file name that is not UTF-8 (EILSEQ)" for EILSEQ. The controls pin both. NIT 5. The new own-directory test is renamed for its sites, so a `-k` selection no longer picks up #757's test of the same name. NIT 6. The module docstring keeps `skip_where_a_name_cannot_be_not_utf_8` on one line. NIT 4 stays: moving the emulated disk and the two fixtures into conftest would add hunks to a file #814 and #818 also edit. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_adopt_template_values.py | 25 +++- tests/test_non_utf8_git_reads.py | 22 +-- tests/test_non_utf8_oserror_sites.py | 216 +++++++++++++++++++-------- 3 files changed, 183 insertions(+), 80 deletions(-) diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index 0cb86dcf..991687f5 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -246,16 +246,25 @@ def test_plan_names_the_elector_it_took_from_git_config(tmp_path): b"again") -def refused_then_moved(base: Path, parent: Path, reason: bytes) -> None: +def refused_then_moved(base: Path, parent: Path, reason: bytes, + skip_where_git_fails: bool = True) -> None: """A source in the directory `parent`: `plan` and `check` pass it, as #247 made them, and `execute` refuses its path as the assembly root's clone URL, giving `reason`, before any leg exists. Then the remediation the refusal gives, followed as it is worded -- `parent` renamed, then `plan --source `, `check` and `execute` run again -- ends - `adoption verified`.""" + `adoption verified`. + + A source repository that git cannot make under `parent` skips the test + where `skip_where_git_fails` is true, for a name that is not UTF-8, which + git may not work under. A name that is valid text asks for it false, and + the failure is raised: the cause is the disk or git, never the name + (#834).""" try: source = make_source_repo(parent / "Thing") except AssertionError as exc: + if not skip_where_git_fails: + raise pytest.skip(f"git will not work under {parent.name!r} here: {exc}") runs = plan_check_execute(base, source) assert_no_traceback_and_no_leg(base, runs) @@ -306,13 +315,15 @@ def test_execute_refuses_a_local_source_directory_holding_a_bidi_control( """A source under a directory whose name begins with U+202E RIGHT-TO-LEFT OVERRIDE: on main `execute` adopted it, and the shape's README showed the rest of its `git clone` line reversed. Now it is refused as the - byte is, and moving the source adopts. The directory is made with no - guard: U+202E is valid text, which a POSIX disk, NTFS and APFS hold - (read, not run on the last two), so an error of the disk is the - test's own and is raised, not skipped (#834).""" + byte is, and moving the source adopts. Nothing here is guarded: U+202E + is valid text, which a POSIX disk, NTFS and APFS hold and git works + under (reasoned, not run, on the last two), so an error of the disk when + the directory is made, and a failure of git when the source is, are the + test's own and are raised, not skipped (#834).""" parent = tmp_path / "\u202ecaf" parent.mkdir() - refused_then_moved(tmp_path, parent, b"a bidirectional control") + refused_then_moved(tmp_path, parent, b"a bidirectional control", + skip_where_git_fails=False) # --- `check` and `execute`: a plan value edited by hand ---------------------- diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index e35b64c2..dc31a1c2 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -96,19 +96,23 @@ def make_file(path: Path, data: bytes = b"on: [push]\n") -> None: is asked FIRST: NTFS most likely takes the surrogate, and the file would then not be the one the test means). A name that is UTF-8 is written. - Two answers are the system's: a file name encoding that cannot spell - the name (`UnicodeError`: ASCII under `LC_ALL=C`) and a disk that - refuses the name (EILSEQ: macOS). Every other `OSError`, a full disk - (ENOSPC), a read-only one (EROFS) or a folder nobody can write in - (EACCES), is the test's own and is raised (#834).""" + Two answers are the system's, each skipped on a reason of its own: a + file name encoding that cannot spell the name (`UnicodeError`: ASCII + under `LC_ALL=C`) and a disk that refuses the name (EILSEQ: macOS). + Every other `OSError`, a full disk (ENOSPC), a read-only one (EROFS) + or a folder nobody can write in (EACCES), is the test's own and is + raised (#834).""" skip_where_fsencode_gives_a_surrogate_no_byte(path.name) try: path.write_bytes(data) - except (OSError, UnicodeError) as exc: - if isinstance(exc, OSError) and exc.errno != errno.EILSEQ: + except UnicodeError as exc: + pytest.skip("this system's file name encoding cannot spell " + f"{path.name!r}: {exc}") + except OSError as exc: + if exc.errno != errno.EILSEQ: raise - pytest.skip("this filesystem cannot hold a file name that is not " - f"UTF-8: {exc}") + pytest.skip("this filesystem refuses a file name that is not " + f"UTF-8 (EILSEQ): {exc}") def test_make_file_skips_where_fsencode_makes_no_byte_and_writes_nothing( diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index 9b4109fc..8a88254c 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -3,10 +3,10 @@ (or with U+202E) skips on the system's answer and on no other error of the disk (#834, #461). -Five more places beside the ones #757 moved onto `skip_where_a_name_cannot_be_ -not_utf_8` (#663) turned ANY `OSError` into `pytest.skip`, so a disk that -filled up, a read-only `tmp_path` or a folder this user cannot write in read as -SKIPPED, and no run fails on a skip: +Five more places beside the ones #757 moved onto +`skip_where_a_name_cannot_be_not_utf_8` (#663) turned ANY `OSError` into +`pytest.skip`, so a disk that filled up, a read-only `tmp_path` or a folder +this user cannot write in read as SKIPPED, and no run fails on a skip: - `test_a_checkout_whose_path_is_not_utf8_is_a_checkout` (`tests/test_cli_reference.py`) and @@ -17,13 +17,16 @@ - `make_file` (`tests/test_non_utf8_git_reads.py`) skipped on any `OSError` of its `write_bytes`, so each of its callers inherited the skip; - the U+202E directory of `tests/test_adopt_template_values.py` is valid text, - which no system known here refuses, so it skipped for nothing but a fault. + which no system known here refuses, so it skipped for nothing but a fault, + and so did the source repository `refused_then_moved` makes in it, when git + failed. The first three now ask the helper, which skips on the two answers that mean "this system cannot carry the name" (Windows' `os.fsencode` and macOS's EILSEQ) and raises the rest. `make_file` skips on EILSEQ and on a `UnicodeError` (an -ASCII file name encoding, #757) and raises the rest, and the U+202E test makes -its directory with no guard. +ASCII file name encoding, #757), each on a reason of its own, and raises the +rest. The U+202E test makes its directory, and its source repository, with no +guard. Each is called here as a plain function, with the disk answering as it would on a system this suite does not run on (CI is Linux only, #597), and asked what @@ -31,10 +34,16 @@ - a disk that refuses the PROBE with EILSEQ (macOS): the test skips, on the helper's reason; -- a disk that refuses the probe, or the test's own name, with EACCES, ENOSPC - or EROFS: the test raises it, where it skipped; +- a disk that refuses the probe, or the test's own name, with ANY other errno, + every one `errno.errorcode` holds: the test raises it, where it skipped. The + errnos are drawn from there and not listed, so a guard that re-raises the + three somebody thought of (ENOSPC, EROFS, EACCES) and skips the rest, EIO or + EDQUOT, fails here; - Windows, by an `os.fsencode` that gives a surrogate three bytes: the three - tests that ask the helper skip before the disk is touched. + tests that ask the helper skip before the disk is touched; +- git failing under the U+202E directory fails the test, and under the + directory named with a byte that is not UTF-8 still skips it (#757 kept that + second skip, and it is a known limit: a full disk skips there). """ from __future__ import annotations @@ -81,10 +90,20 @@ THEN_MAKE_THEIR_OWN = [row for row in ASK_THE_HELPER if row.id != "bootstrap-leg-moves"] -#: What the disk refuses with, and the exception Python raises for it. -REFUSALS = [pytest.param(errno.EACCES, PermissionError, id="EACCES"), - pytest.param(errno.ENOSPC, OSError, id="ENOSPC"), - pytest.param(errno.EROFS, OSError, id="EROFS")] +#: Every answer a disk can give but the one that means "this system cannot +#: carry the name": drawn from `errno.errorcode` and not listed, so an errno +#: nobody thought of is asked too. +EVERY_OTHER_ERRNO = sorted(code for code in errno.errorcode + if code != errno.EILSEQ) + +#: A `git` failure as the disk gives it: what `make_source_repo` raises when +#: `git commit` cannot write an object. +GIT_FAILED = ("git commit failed: error: unable to write file " + ".git/objects/xx: No space left on device") + + +def a_source_git_cannot_make(*args, **kwargs): + raise AssertionError(GIT_FAILED) def the_probe(name: str) -> bool: @@ -95,37 +114,69 @@ def anything_but_the_probe(name: str) -> bool: return not the_probe(name) -def the_error_of(kind, call, *args) -> OSError: - """The `kind` of error `call(*args)` raises. A call that skips or ends - without one FAILS the test, as `fail_where_the_caller_ends` has it.""" - with pytest.raises(kind) as raised: - fail_where_the_caller_ends(call, *args) - return raised.value +class Disk: + """What `a_disk_that_refuses` puts in: it answers `code`, which a test may + change, to every name `names` accepts, and to all of them where that is + None.""" + + def __init__(self, code: int, names=None): + self.code = code + self.names = names + def refuses(self, name: str) -> bool: + return self.names is None or self.names(name) -def a_disk_that_refuses(monkeypatch, code: int, names=None) -> None: + def error(self, path) -> OSError: + return OSError(self.code, os.strerror(self.code), os.fsdecode(path)) + + +def a_disk_that_refuses(monkeypatch, code: int, names=None) -> Disk: """From here on a disk that answers `code` to every directory (`os.mkdir`, which `Path.mkdir` calls) and every file (`Path.write_bytes`) whose last part `names` accepts, to all of them where it is None, and makes the rest.""" + disk = Disk(code, names) real_mkdir = os.mkdir real_write_bytes = Path.write_bytes - def refused(name: str) -> bool: - return names is None or names(name) - def mkdir(path, *args, **kwargs): - if refused(os.path.basename(os.fsdecode(path))): - raise OSError(code, os.strerror(code), os.fsdecode(path)) + if disk.refuses(os.path.basename(os.fsdecode(path))): + raise disk.error(path) return real_mkdir(path, *args, **kwargs) def write_bytes(self, data): - if refused(self.name): - raise OSError(code, os.strerror(code), str(self)) + if disk.refuses(self.name): + raise disk.error(self) return real_write_bytes(self, data) monkeypatch.setattr(os, "mkdir", mkdir) monkeypatch.setattr(Path, "write_bytes", write_bytes) + return disk + + +def the_error_of(kind, call, *args) -> Exception: + """The `kind` of error `call(*args)` raises. A call that skips or ends + without one FAILS the test, as `fail_where_the_caller_ends` has it.""" + with pytest.raises(kind) as raised: + fail_where_the_caller_ends(call, *args) + return raised.value + + +def raised_for_every_errno(disk: Disk, call, *args) -> None: + """Fail unless `call(*args)` raises what `disk` answers, whichever of + `EVERY_OTHER_ERRNO` it answers, as Python makes it (`PermissionError` for + EACCES). It stops at the first errno that does not, naming it, so a + caller that goes on past a refused name runs its whole test once and not + 129 times.""" + for code in EVERY_OTHER_ERRNO: + disk.code = code + try: + error = the_error_of(OSError, call, *args) + except pytest.fail.Exception as failed: + pytest.fail(f"errno {code} ({errno.errorcode[code]}): " + f"{failed.msg}") + assert error.errno == code, (code, error) + assert type(error) is type(OSError(code, "")), (code, error) # --- the three tests that ask the helper ------------------------------------ @@ -143,27 +194,28 @@ def test_a_disk_that_refuses_the_probe_with_eilseq_skips_the_test( @pytest.mark.usefixtures("a_system_that_carries_the_byte") -@pytest.mark.parametrize("code, kind", REFUSALS) @pytest.mark.parametrize("call", ASK_THE_HELPER) -def test_a_disk_that_refuses_the_probe_for_any_other_reason_fails_the_test( - tmp_path, monkeypatch, call, code, kind): - """A full disk, a read-only one or a folder nobody can write in: the - helper raises it, so the test FAILS, and no run reads it as a skip.""" - a_disk_that_refuses(monkeypatch, code, the_probe) - assert the_error_of(kind, call, tmp_path).errno == code +def test_a_disk_that_refuses_the_probe_with_any_other_errno_fails_the_test( + tmp_path, monkeypatch, call): + """A full disk, a read-only one, a folder nobody can write in, a failing + drive, a quota: the helper raises each, so the test FAILS, and no run reads + it as a skip.""" + disk = a_disk_that_refuses(monkeypatch, errno.EACCES, the_probe) + raised_for_every_errno(disk, call, tmp_path) @pytest.mark.usefixtures("a_disk_that_takes_the_probe") -@pytest.mark.parametrize("code, kind", REFUSALS) @pytest.mark.parametrize("call", THEN_MAKE_THEIR_OWN) -def test_a_disk_that_refuses_the_tests_own_directory_fails_the_test( - tmp_path, monkeypatch, call, code, kind): +def test_the_cli_reference_and_doctor_tests_fail_where_the_disk_refuses_their_own_directory( + tmp_path, monkeypatch, call): """The probe is made and removed, and then the directory the test means - to use is refused: that is the test's own, raised, where it was a skip. The - rows ask the real disk for the probe first, and skip where it answers - EILSEQ: they would FAIL there on a system that holds no defect.""" - a_disk_that_refuses(monkeypatch, code, anything_but_the_probe) - assert the_error_of(kind, call, tmp_path).errno == code + to use is refused with any errno: that is the test's own, raised, where it + was a skip. The rows ask the real disk for the probe first, and skip where + it answers EILSEQ: they would FAIL there on a system that holds no + defect.""" + disk = a_disk_that_refuses(monkeypatch, errno.EACCES, + anything_but_the_probe) + raised_for_every_errno(disk, call, tmp_path) @pytest.mark.parametrize("call", ASK_THE_HELPER) @@ -186,18 +238,22 @@ def test_where_fsencode_makes_no_byte_of_the_surrogate_the_test_skips_first( def test_make_file_skips_where_the_disk_answers_eilseq(tmp_path, monkeypatch): """macOS (emulated): APFS refuses a file name that is not UTF-8 with `Illegal byte sequence`, which is the disk's answer, so the file is not - written and the test that asked is skipped.""" + written and the test that asked is skipped, on a reason that says it is + the filesystem.""" a_disk_that_refuses(monkeypatch, errno.EILSEQ) target = tmp_path / non_utf8_git_reads.NOT_UTF8 reason = skip_message(non_utf8_git_reads.make_file, target) assert reason is not None, "make_file went on past the disk's EILSEQ" assert "filesystem" in reason, reason + assert "EILSEQ" in reason, reason def test_make_file_skips_where_the_file_name_encoding_cannot_spell_the_name( tmp_path, monkeypatch): """ASCII under `LC_ALL=C` (#757): the write raises `UnicodeEncodeError` - before the disk is asked, which is the system's answer.""" + before the disk is asked, which is the system's answer, and the reason + says it is the encoding: the name is UTF-8, and the disk is not + involved.""" def write_bytes(self, data): raise UnicodeEncodeError("ascii", self.name, 0, 1, "ordinal not in range(128)") @@ -205,35 +261,67 @@ def write_bytes(self, data): target = tmp_path / non_utf8_git_reads.UTF8 reason = skip_message(non_utf8_git_reads.make_file, target) assert reason is not None, "make_file raised where the encoding says no" - assert "filesystem" in reason, reason + assert "file name encoding" in reason, reason + assert "filesystem" not in reason, reason @pytest.mark.usefixtures("a_system_that_carries_the_byte") @pytest.mark.parametrize("name", [non_utf8_git_reads.NOT_UTF8, non_utf8_git_reads.UTF8], ids=["not-utf-8", "utf-8"]) -@pytest.mark.parametrize("code, kind", REFUSALS) def test_make_file_raises_any_other_error_of_the_disk( - tmp_path, monkeypatch, code, kind, name): - """A full disk (ENOSPC), a read-only one (EROFS) or a folder nobody can - write in (EACCES) is the test's own, for a name that is not UTF-8 and for - one that is: raised as it was, and never a skip that hides it.""" - a_disk_that_refuses(monkeypatch, code) - error = the_error_of(kind, non_utf8_git_reads.make_file, tmp_path / name) - assert error.errno == code + tmp_path, monkeypatch, name): + """A full disk, a read-only one, a folder nobody can write in or a failing + drive is the test's own, for a name that is not UTF-8 and for one that is: + raised as it was, and never a skip that hides it, whichever errno it is.""" + disk = a_disk_that_refuses(monkeypatch, errno.EACCES) + raised_for_every_errno(disk, non_utf8_git_reads.make_file, tmp_path / name) # --- the U+202E directory --------------------------------------------------- -@pytest.mark.parametrize("code, kind", REFUSALS) +#: U+202E RIGHT-TO-LEFT OVERRIDE, built from its code point: typed raw it +#: would reorder the line it is on. +BIDI = chr(0x202E) +BIDI_TEST = (template_values + .test_execute_refuses_a_local_source_directory_holding_a_bidi_control) +BYTE_TEST = (template_values + .test_execute_refuses_a_local_source_directory_that_is_not_utf8) + + def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( - tmp_path, monkeypatch, code, kind): + tmp_path, monkeypatch): """U+202E is valid text: no probe is asked, and the directory is made with - no guard, so what the disk answers to it is raised. It was a skip that said - "no directory here is named ..." for a full disk or a read-only one.""" - a_disk_that_refuses(monkeypatch, code) - error = the_error_of( - kind, template_values - .test_execute_refuses_a_local_source_directory_holding_a_bidi_control, - tmp_path) - assert error.errno == code + no guard, so what the disk answers to it is raised, whichever errno it is. + It was a skip that said "no directory here is named ..." for a full disk + or a read-only one.""" + disk = a_disk_that_refuses(monkeypatch, errno.EACCES) + raised_for_every_errno(disk, BIDI_TEST, tmp_path) + + +def test_a_failure_of_git_under_the_bidi_directory_fails_the_test( + tmp_path, monkeypatch): + """The source repository `refused_then_moved` makes in the directory: + where git cannot (a disk that filled after the `mkdir`), the failure is + raised, as the name is valid text and not the cause. It was a skip that + said "git will not work under ...". Skipped where this system's file name + encoding cannot spell U+202E (ASCII under `LC_ALL=C`): the directory is + really made here, and `mkdir` raises `UnicodeEncodeError` first.""" + non_utf8_git_reads.skip_where_the_environment_cannot_carry(BIDI) + monkeypatch.setattr(template_values, "make_source_repo", + a_source_git_cannot_make) + error = the_error_of(AssertionError, BIDI_TEST, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +def test_a_failure_of_git_under_the_byte_directory_still_skips_the_test( + tmp_path, monkeypatch): + """The control, and the known limit: #757 kept the second skip for a name + that is not UTF-8, which git may not work under, so a disk that filled + after the `mkdir` still reads as SKIPPED there.""" + monkeypatch.setattr(template_values, "make_source_repo", + a_source_git_cannot_make) + reason = skip_message(BYTE_TEST, tmp_path) + assert reason is not None, "the test went on past git's failure" + assert "git will not work under" in reason, reason From 9ba530a916363bd898869dc8ad7bf5d870bafe6c Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 20:40:13 +0000 Subject: [PATCH 08/13] tests: `raised_for_every_errno` passes the errno's label to the failure and does not catch it, so a failing errno still names itself and the test has no `try` for python:S8714 (#834) Fix round 2 on #845, after lane 4's re-check of efb17273 (READY, no MUST). SonarCloud reported python:S8714 on the `except pytest.fail.Exception` of `raised_for_every_errno`, which caught a failure only to fail again with the errno's name in front. `fail_where_the_caller_ends` takes a keyword `said`, put in front of its message, and `the_error_of` threads it through; the loop passes `said=f"errno {code} ({errno.errorcode[code]}): "`. The loop has no `try`. The message a failing errno gives is the same as before, `errno 1 (EPERM): the caller ended SKIPPED: ...`, and the sweep still stops at the first errno that fails. The three other callers of `fail_where_the_caller_ends` pass no `said` and are unchanged. One test holds it: a caller that skips where it should raise fails at the first errno with the label in front of the message, and is asked once. It passes on efb17273 too, as the message is kept; it fails where the label is dropped at any of the three places it is passed, and where the sweep goes on past a failure. The U+202E test under a C locale (the re-check's NIT 2) is #854 and is not changed here. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_non_utf8_name_callers.py | 8 ++++-- tests/test_non_utf8_oserror_sites.py | 43 +++++++++++++++++++++------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 13aa5518..76db45df 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -113,12 +113,14 @@ def mkdir(path, *args, **kwargs): return mkdir -def fail_where_the_caller_ends(call, *args) -> None: +def fail_where_the_caller_ends(call, *args, said: str = "") -> None: """Fail where `call(*args)` ends without an error, SKIPPED or not: the caller was to raise. A `pytest.raises` block may hold one call and no - more (python:S5778), so the failure is made here and not beside it.""" + more (python:S5778), so the failure is made here and not beside it. + `said` goes in front of the message, for a caller that asks the same + question many times and says which one failed (#834).""" reason = skip_message(call, *args) - pytest.fail("the caller ended " + pytest.fail(said + "the caller ended " + (f"SKIPPED: {reason}" if reason else "without an error")) diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index 8a88254c..cf8afa0c 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -154,27 +154,25 @@ def write_bytes(self, data): return disk -def the_error_of(kind, call, *args) -> Exception: +def the_error_of(kind, call, *args, said: str = "") -> Exception: """The `kind` of error `call(*args)` raises. A call that skips or ends - without one FAILS the test, as `fail_where_the_caller_ends` has it.""" + without one FAILS the test, as `fail_where_the_caller_ends` has it, with + `said` in front of the message.""" with pytest.raises(kind) as raised: - fail_where_the_caller_ends(call, *args) + fail_where_the_caller_ends(call, *args, said=said) return raised.value def raised_for_every_errno(disk: Disk, call, *args) -> None: """Fail unless `call(*args)` raises what `disk` answers, whichever of `EVERY_OTHER_ERRNO` it answers, as Python makes it (`PermissionError` for - EACCES). It stops at the first errno that does not, naming it, so a - caller that goes on past a refused name runs its whole test once and not - 129 times.""" + EACCES). It stops at the first errno that does not, naming it in the + failure, so a caller that goes on past a refused name runs its whole + test once and not 129 times.""" for code in EVERY_OTHER_ERRNO: disk.code = code - try: - error = the_error_of(OSError, call, *args) - except pytest.fail.Exception as failed: - pytest.fail(f"errno {code} ({errno.errorcode[code]}): " - f"{failed.msg}") + error = the_error_of(OSError, call, *args, + said=f"errno {code} ({errno.errorcode[code]}): ") assert error.errno == code, (code, error) assert type(error) is type(OSError(code, "")), (code, error) @@ -325,3 +323,26 @@ def test_a_failure_of_git_under_the_byte_directory_still_skips_the_test( reason = skip_message(BYTE_TEST, tmp_path) assert reason is not None, "the test went on past git's failure" assert "git will not work under" in reason, reason + + +# --- the sweep itself ------------------------------------------------------- + +def test_the_sweep_names_the_first_errno_that_fails_and_stops_there(): + """A caller that skips where it should raise fails the test at the first + errno, with that errno named in front of the message, and is not asked + the other 128: the label is passed in and not made by catching a failure + (python:S8714), so a failure that drops it, or a sweep that goes on, shows + here.""" + asked = [] + + def hides_the_error(tmp_path): + asked.append(tmp_path) + pytest.skip("hidden") + + first = EVERY_OTHER_ERRNO[0] + with pytest.raises(pytest.fail.Exception) as failed: + raised_for_every_errno(Disk(first), hides_the_error, "a path") + assert failed.value.msg == ( + f"errno {first} ({errno.errorcode[first]}): " + "the caller ended SKIPPED: hidden") + assert asked == ["a path"] From 8f6a38e3e4076f9484812516d8b58ade85cc5a26 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sat, 17 Oct 2026 21:47:25 +0000 Subject: [PATCH 09/13] tests: the sweep's own test builds its `Disk` before the `pytest.raises` block, so the block holds one call and python:S5778 no longer reports it (#834) Fix round 3 on #845, after lane 4's re-check of 9ba530a9 (READY, no MUST). SonarCloud reported python:S5778 (MAJOR) on `test_the_sweep_names_the_first_errno_that_fails_and_stops_there`: its `pytest.raises(pytest.fail.Exception)` block held two calls that could raise, `Disk(first)` and `raised_for_every_errno(...)`. `disk = Disk(first)` is built before the block, and the block holds `raised_for_every_errno(disk, hides_the_error, "a path")` alone. The assertions after it read the same exception, `failed.value.msg`, and the list of calls the caller was asked. Nothing else changes. Lane: openreposhape-5 (openRepoShape-5) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_non_utf8_oserror_sites.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index cf8afa0c..181ed792 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -340,8 +340,9 @@ def hides_the_error(tmp_path): pytest.skip("hidden") first = EVERY_OTHER_ERRNO[0] + disk = Disk(first) with pytest.raises(pytest.fail.Exception) as failed: - raised_for_every_errno(Disk(first), hides_the_error, "a path") + raised_for_every_errno(disk, hides_the_error, "a path") assert failed.value.msg == ( f"errno {first} ({errno.errorcode[first]}): " "the caller ended SKIPPED: hidden") From 40531e1997ffe8518eb1f1061309714d9eedc7e3 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 02:44:22 +0000 Subject: [PATCH 10/13] tests: the six places that skipped on a failed git call ask the system first and let git's failure fail the test, and the U+202E test asks the file name encoding before its `mkdir`, so a full disk reads as a failure, not SKIPPED, and a C locale skips that test instead of erroring (#853, #854) #853, line 1: `refused_then_moved`, `test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts` and `test_a_source_directory_that_is_not_utf8_is_shown_spelled` no longer wrap `make_source_repo` in `except AssertionError: pytest.skip(...)`. They already ask `skip_where_a_name_cannot_be_not_utf_8` before the `mkdir` (#757), and git works under any directory a system carries, so a git that fails there (a disk that filled after the `mkdir`) fails the test. `refused_then_moved` loses #834's `skip_where_git_fails`, which only the byte-name test still used. #853, line 2, and the sixth site: `with_default_branch`, `moved_to_a_branch_that_is_not_utf8` and `remembered_on` (added by #830 after the issue's scan) ask `skip_where_no_ref_can_be_named` before their first git call and drop their `except (AssertionError, UnicodeError, OSError)`. A name that is not UTF-8 asks the helper of the directory the source is in; a name that is text asks only whether the file name encoding can spell it, since `subprocess` puts every argument through `os.fsencode`. The skip after `git symbolic-ref` stays. #854: the U+202E test asks `skip_where_the_environment_cannot_carry` before its `mkdir`. The helper moves from `tests/test_non_utf8_git_reads.py` to `tests/conftest.py` with its callers, and its reason now reads "this system's file name encoding (...) cannot spell ...", as `make_file`'s does. `tests/test_non_utf8_git_failures.py` (new) calls each place with git made to fail, and with macOS, Windows and an ASCII encoding emulated. It also scans every `tests/*.py` for an `except` that names `AssertionError` and holds a `pytest.skip`. #845's control test that held the byte-name skip as a known limit is gone, and its two U+202E tests skip under an ASCII encoding. Co-Authored-By: Claude Opus 5.5 --- tests/conftest.py | 19 ++ tests/test_adopt_git_reads.py | 83 +++--- tests/test_adopt_plan_escapes.py | 8 +- tests/test_adopt_template_values.py | 44 ++-- tests/test_non_utf8_git_failures.py | 362 +++++++++++++++++++++++++++ tests/test_non_utf8_git_reads.py | 17 +- tests/test_non_utf8_name_callers.py | 15 +- tests/test_non_utf8_oserror_sites.py | 37 ++- 8 files changed, 475 insertions(+), 110 deletions(-) create mode 100644 tests/test_non_utf8_git_failures.py diff --git a/tests/conftest.py b/tests/conftest.py index 1ddd93cb..db342b94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1201,6 +1201,25 @@ def skip_message(call, *args) -> str | None: return None +def skip_where_the_environment_cannot_carry(value: str) -> None: + """Skip where this system's file name encoding cannot encode `value`, as + under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of + a file name (`make_file` cannot write `caf\\u00e9.yml`, and `mkdir` cannot + make `\\u202ecaf`), of an environment value and of a command-line + argument: `subprocess` puts each one through `os.fsencode` before it + starts the child, so a `user.name` of `Jos\\u00e9` or a branch + `main\\u00a0` raises `UnicodeEncodeError` in the test and not in the tool + (#757, #853, #854). Text that is ASCII, or a lone surrogate that + `os.fsencode` gives back as its byte, is carried. Asked through the `os` + module at call time, so a test emulates ASCII with + `monkeypatch.setattr(os, "fsencode", ...)`.""" + try: + os.fsencode(value) + except UnicodeEncodeError: + pytest.skip(f"this system's file name encoding " + f"({sys.getfilesystemencoding()}) cannot spell {value!r}") + + # --- the `gh` guard (#296) --------------------------------------------------- # # NO TEST REACHES THE OPERATOR'S `gh`, and the doctor's `machine` row is still diff --git a/tests/test_adopt_git_reads.py b/tests/test_adopt_git_reads.py index a8059442..fc86e7f6 100644 --- a/tests/test_adopt_git_reads.py +++ b/tests/test_adopt_git_reads.py @@ -44,7 +44,8 @@ import pytest from conftest import (REPO, appended_to_command_scope, git, make_source_repo, - skip_where_a_name_cannot_be_not_utf_8) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry) from test_adopt_template_values import (ADOPT_REFUSED, PLAN_FILE, PLAN_OK, PROJECT, REMOTES, SPLIT_BRANCH, VERIFIED, WORK, adopted, answer, @@ -462,15 +463,12 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): at a path that is text has the old path as its origin, so `execute --source ` ended in this issue's traceback; it now adopts. Skipped where no directory can be named with the byte 0xE9 (Windows, - macOS's APFS) or git will not work in one.""" + macOS's APFS); a failure of git under it fails the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) parent.mkdir() - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - pytest.skip(f"git will not work under {parent.name!r} here: {exc}") + source = make_source_repo(parent / "Thing") clone = tmp_path / "clone" git("clone", "-q", "--", str(source), str(clone), cwd=tmp_path) runs = {"plan": planned(tmp_path, source)} @@ -522,6 +520,25 @@ def test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path): LOST_REFUSAL = b"REFUSED plan-tracking-branch-missing: " +def skip_where_no_ref_can_be_named(source: Path, branch: str) -> None: + """Skip where this system cannot give `source` a ref named `branch`, + asked before any git call makes one (#853). A name that is not UTF-8 + asks `skip_where_a_name_cannot_be_not_utf_8` of the directory `source` + is in, on the disk its ref files go to: Windows' `os.fsencode` first, + then macOS's EILSEQ. A name that is text asks only whether this + system's file name encoding can spell it, as a command line must + (`subprocess` puts each argument through `os.fsencode`; ASCII under + `LC_ALL=C` cannot), since every disk git runs on holds text. What git + does after it is the test's own: a git call that fails, on a full disk + or a directory it cannot write in, fails the test.""" + try: + branch.encode("utf-8") + except UnicodeEncodeError: + skip_where_a_name_cannot_be_not_utf_8(source.parent) + else: + skip_where_the_environment_cannot_carry(branch) + + def with_default_branch(source: Path, where: str, branch: str = BRANCH) -> Path: """`source`, on `main`, with `branch` (`BRANCH` unless given) as its @@ -531,23 +548,21 @@ def with_default_branch(source: Path, where: str, source: a plan that read the wrong branch would adopt the wrong commit, and a `--tracking-branch main` names a branch it has. - Skipped where no branch can be named so (the byte 0xE9 on Windows or - macOS's APFS), or git here does not give the name back as it was - written.""" + Skipped where this system cannot name the branch + (`skip_where_no_ref_can_be_named`: the byte 0xE9 on Windows or macOS's + APFS, or text an ASCII file name encoding cannot spell), asked before + git is, and where git here does not give the name back as it was + written. A git call between the two that fails fails the test (#853).""" ref = "refs/remotes/origin/HEAD" if where == ORIGIN_HEAD else "HEAD" written = branch.encode("utf-8", "surrogateescape") - try: - git("switch", "-q", "-c", branch, cwd=source) - git("commit", "-q", "--allow-empty", "-m", "One ahead", cwd=source) - if where == ORIGIN_HEAD: - git("switch", "-q", "main", cwd=source) - for name in (branch, "main"): - git("update-ref", f"refs/remotes/origin/{name}", name, - cwd=source) - git("symbolic-ref", ref, f"refs/remotes/origin/{branch}", - cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no branch here is named {written!r}: {exc!r}") + skip_where_no_ref_can_be_named(source, branch) + git("switch", "-q", "-c", branch, cwd=source) + git("commit", "-q", "--allow-empty", "-m", "One ahead", cwd=source) + if where == ORIGIN_HEAD: + git("switch", "-q", "main", cwd=source) + for name in (branch, "main"): + git("update-ref", f"refs/remotes/origin/{name}", name, cwd=source) + git("symbolic-ref", ref, f"refs/remotes/origin/{branch}", cwd=source) read = subprocess.run(["git", "symbolic-ref", ref], cwd=str(source), capture_output=True, check=False) if not read.stdout.rstrip(b"\r\n").endswith(b"/" + written): @@ -559,15 +574,14 @@ def moved_to_a_branch_that_is_not_utf8(base: Path) -> Path: """A source on `main`, planned and answered, whose `main` is then renamed `BRANCH`: a plan that tracks a branch the source no longer has, and a source whose HEAD is on one that is not UTF-8. Skipped as - `with_default_branch` is.""" + `with_default_branch` is, before anything is made; a failure of git + fails the test (#853).""" + skip_where_no_ref_can_be_named(base / "Thing", BRANCH) source = make_source_repo(base / "Thing") made = planned(base, source) assert made.returncode == 0, said(made) answer(base) - try: - git("branch", "-m", "main", BRANCH, cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no branch here is named {NOT_UTF8_BRANCH!r}: {exc!r}") + git("branch", "-m", "main", BRANCH, cwd=source) return source @@ -720,15 +734,14 @@ def test_plan_reads_a_default_branch_ending_in_white_space_whole(tmp_path, def remembered_on(source: Path, branch: str) -> Path: """`source` with `origin/HEAD` naming `origin/`, which it has fetched, and no branch `branch` of its own, as `git clone -b main` of a - repository whose default branch is `branch` leaves it. Skipped as - `with_default_branch` is, where git refuses or alters the name.""" - try: - git("update-ref", f"refs/remotes/origin/{branch}", "refs/heads/main", - cwd=source) - git("symbolic-ref", "refs/remotes/origin/HEAD", - f"refs/remotes/origin/{branch}", cwd=source) - except (AssertionError, UnicodeError, OSError) as exc: - pytest.skip(f"no ref here is named {branch!r}: {exc!r}") + repository whose default branch is `branch` leaves it. Skipped where + this system cannot name the ref, asked before git is, as + `with_default_branch` is; a failure of git fails the test (#853).""" + skip_where_no_ref_can_be_named(source, branch) + git("update-ref", f"refs/remotes/origin/{branch}", "refs/heads/main", + cwd=source) + git("symbolic-ref", "refs/remotes/origin/HEAD", + f"refs/remotes/origin/{branch}", cwd=source) return source diff --git a/tests/test_adopt_plan_escapes.py b/tests/test_adopt_plan_escapes.py index c6041a72..d94446ec 100644 --- a/tests/test_adopt_plan_escapes.py +++ b/tests/test_adopt_plan_escapes.py @@ -398,15 +398,13 @@ def test_a_source_directory_that_is_not_utf8_is_shown_spelled(adopter, so did `check`'s `WHAT WILL HAPPEN, against @`, and each raised `UnicodeEncodeError` on a strict UTF-8 stdout after the plan was written. Both show it spelled now. Skipped where no directory can be named with - the byte 0xE9 (Windows, macOS's APFS) or git will not work in one.""" + the byte 0xE9 (Windows, macOS's APFS); a failure of git under it fails + the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) parent.mkdir() - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - pytest.skip(f"git will not work under b'caf\\xe9' here: {exc}") + source = make_source_repo(parent / "Thing") shown = adopter._spelled(str(source.resolve())) assert "\\udce9" in shown plan = tmp_path / PLAN_FILE diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index 991687f5..cf6b6952 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -51,7 +51,8 @@ from conftest import (ADOPT, REPO, WINDOWS_SKIP, blank_unnamed_pin_sources, make_source_repo, resolve, - skip_where_a_name_cannot_be_not_utf_8) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry) sys.path.insert(0, str(REPO / "scripts")) import shape_materialize # noqa: E402 @@ -246,8 +247,7 @@ def test_plan_names_the_elector_it_took_from_git_config(tmp_path): b"again") -def refused_then_moved(base: Path, parent: Path, reason: bytes, - skip_where_git_fails: bool = True) -> None: +def refused_then_moved(base: Path, parent: Path, reason: bytes) -> None: """A source in the directory `parent`: `plan` and `check` pass it, as #247 made them, and `execute` refuses its path as the assembly root's clone URL, giving `reason`, before any leg exists. Then the remediation @@ -255,17 +255,12 @@ def refused_then_moved(base: Path, parent: Path, reason: bytes, `plan --source `, `check` and `execute` run again -- ends `adoption verified`. - A source repository that git cannot make under `parent` skips the test - where `skip_where_git_fails` is true, for a name that is not UTF-8, which - git may not work under. A name that is valid text asks for it false, and - the failure is raised: the cause is the disk or git, never the name - (#834).""" - try: - source = make_source_repo(parent / "Thing") - except AssertionError as exc: - if not skip_where_git_fails: - raise - pytest.skip(f"git will not work under {parent.name!r} here: {exc}") + The source repository is made under `parent` with no guard: the caller + has asked whether this system can carry the name, and git works under + any directory a system carries, so a failure of git there (a disk that + filled after the `mkdir`) is the test's own and is raised, for a name + that is not UTF-8 (#853) as for one that is valid text (#834).""" + source = make_source_repo(parent / "Thing") runs = plan_check_execute(base, source) assert_no_traceback_and_no_leg(base, runs) assert runs["plan"].returncode == 0, said(runs["plan"]) @@ -300,8 +295,8 @@ def test_execute_refuses_a_local_source_directory_that_is_not_utf8(tmp_path): once said to clone the source to a UTF-8 path and pass that with --source, which ended in a traceback: the clone's `origin` is the old path, which `_remote_repository` reads as UTF-8. Skipped where no - directory can be named with the byte 0xE9 (Windows, macOS's APFS) or - git will not work in one.""" + directory can be named with the byte 0xE9 (Windows, macOS's APFS); a + failure of git under it fails the test (#853).""" skip_where_a_name_cannot_be_not_utf_8(tmp_path) parent = Path(os.fsdecode(os.path.join(os.fsencode(tmp_path), b"caf\xe9"))) @@ -315,15 +310,18 @@ def test_execute_refuses_a_local_source_directory_holding_a_bidi_control( """A source under a directory whose name begins with U+202E RIGHT-TO-LEFT OVERRIDE: on main `execute` adopted it, and the shape's README showed the rest of its `git clone` line reversed. Now it is refused as the - byte is, and moving the source adopts. Nothing here is guarded: U+202E - is valid text, which a POSIX disk, NTFS and APFS hold and git works - under (reasoned, not run, on the last two), so an error of the disk when - the directory is made, and a failure of git when the source is, are the - test's own and are raised, not skipped (#834).""" + byte is, and moving the source adopts. Skipped where this system's file + name encoding cannot spell U+202E (ASCII under `LC_ALL=C`), which is + asked before the `mkdir`, where it raised `UnicodeEncodeError` (#854). + Nothing else is guarded: U+202E is valid text, which a POSIX disk, NTFS + and APFS hold and git works under (reasoned, not run, on the last two), + so an error of the disk when the directory is made, and a failure of git + when the source is, are the test's own and are raised, not skipped + (#834).""" parent = tmp_path / "\u202ecaf" + skip_where_the_environment_cannot_carry(parent.name) parent.mkdir() - refused_then_moved(tmp_path, parent, b"a bidirectional control", - skip_where_git_fails=False) + refused_then_moved(tmp_path, parent, b"a bidirectional control") # --- `check` and `execute`: a plan value edited by hand ---------------------- diff --git a/tests/test_non_utf8_git_failures.py b/tests/test_non_utf8_git_failures.py new file mode 100644 index 00000000..e0e5cee4 --- /dev/null +++ b/tests/test_non_utf8_git_failures.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: Apache-2.0 +"""A test that needs a directory, or a branch, named with a byte that is not +UTF-8 asks the system once whether it can carry the name, and a failure of the +git call that follows fails the test (#853, #461). + +Six places turned a failure of git into `pytest.skip`. The issue's scan of +`tests/*.py` found five; the sixth, `remembered_on`, came with #830 (#797) +after it: + +- `refused_then_moved` (`tests/test_adopt_template_values.py`), + `test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts` + (`tests/test_adopt_git_reads.py`) and + `test_a_source_directory_that_is_not_utf8_is_shown_spelled` + (`tests/test_adopt_plan_escapes.py`) skipped "git will not work under ..." + on an `AssertionError` of `make_source_repo` under the directory `caf` and + byte 0xE9; +- `with_default_branch`, `moved_to_a_branch_that_is_not_utf8` and + `remembered_on` (`tests/test_adopt_git_reads.py`) skipped "no branch here + is named ..." on an `AssertionError`, a `UnicodeError` or an `OSError` of + the git calls that make the branch, and the first and the last are called + with a name that is text too (`main` and U+00A0, `ma`, U+2028 and `in`). + +A git process that fails raises `AssertionError` (`conftest.git`, and +`make_source_repo`'s own commits), so a disk that filled after the `mkdir`, or +a directory git could no longer write in, read as SKIPPED, and no run fails +on a skip. The answers that mean "this system cannot carry the name" are +asked before git now, and every git call after them is unguarded: for a +directory, `skip_where_a_name_cannot_be_not_utf_8`, which the three tests +already asked (#757); for a branch, `skip_where_no_ref_can_be_named`, which +asks that helper for a name that is not UTF-8 and the file name encoding for +a name that is text. The skip after `git symbolic-ref` stays: whether git +gives the name back as it was written is the system's answer, and no +exception is behind it. + +#854 is the C-locale half: under `PYTHONUTF8=0 LC_ALL=C PYTHONCOERCECLOCALE=0` +the file name encoding is ASCII, and the U+202E test's `mkdir` raised +`UnicodeEncodeError`, a `ValueError` its old `except OSError` never caught, +so the test ended in an error. It asks the encoding before the `mkdir` now. + +Each place is called here as a plain function, with git made to fail as a +full disk makes `git commit` fail, or with a system emulated that CI does not +run (CI is Linux only, #597; no Windows or macOS run was made): + +- git fails: the test FAILS with git's error, for a name that is not UTF-8 + and for one that is text; +- macOS's EILSEQ on the probe, or Windows' `os.fsencode` (a surrogate is three + bytes): a branch that is not UTF-8 is skipped on the helper's reason before + git is called and before anything is made; a branch that is text asks + neither and goes on to git; +- an ASCII file name encoding: a branch that is text, and the U+202E test, are + skipped on the encoding's reason, before git and before the `mkdir`; +- every `tests/*.py`, read as #853's scan reads it: no `except` that names + `AssertionError` holds a `pytest.skip`. +""" + +from __future__ import annotations + +import ast +import errno +import os +from textwrap import dedent + +import pytest + +import test_adopt_git_reads as git_reads +import test_adopt_plan_escapes as plan_escapes +import test_adopt_template_values as template_values +from conftest import (REPO, skip_message, + skip_where_the_environment_cannot_carry, + windows_fsencode) +# The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a +# module, which its directory test asks for. +from test_adopt_plan_escapes import adopter # noqa: F401 - fixture +# The fixtures and the emulations of #757's callers file, and the failure of +# git and the way of asking for an error of #834's sites file. +from test_non_utf8_name_callers import ( # noqa: F401 - fixtures + a_disk_that_takes_the_probe, a_system_that_carries_the_byte, + ascii_fsencode, refusing) +from test_non_utf8_oserror_sites import (BIDI_TEST, GIT_FAILED, + a_source_git_cannot_make, + the_error_of) + +# --- a source repository under a directory that is not UTF-8 ---------------- + +#: The three tests that make a source repository under the directory `caf` and +#: byte 0xE9, each with the module whose `make_source_repo` it calls, called +#: with a `tmp_path` and the `adopter` fixture. +UNDER_A_BYTE_DIRECTORY = [ + pytest.param( + template_values, + lambda tmp_path, adopter: template_values + .test_execute_refuses_a_local_source_directory_that_is_not_utf8( + tmp_path), + id="adopt-template-values"), + pytest.param( + git_reads, + lambda tmp_path, adopter: git_reads + .test_a_utf8_clone_of_a_source_whose_path_is_not_utf8_adopts(tmp_path), + id="adopt-git-reads"), + pytest.param( + plan_escapes, + lambda tmp_path, adopter: plan_escapes + .test_a_source_directory_that_is_not_utf8_is_shown_spelled( + adopter, tmp_path), + id="adopt-plan-escapes"), +] + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +@pytest.mark.parametrize("module, call", UNDER_A_BYTE_DIRECTORY) +def test_a_failure_of_git_under_the_byte_directory_fails_the_test( + tmp_path, monkeypatch, adopter, module, call): + """The directory is made, and then git cannot make the source repository + in it, as on a disk that filled after the `mkdir`: the test FAILS with + git's error, where it was SKIPPED as "git will not work under ...". The + rows ask the real disk for the probe first, and skip where it answers + EILSEQ, as the test does before git is reached.""" + monkeypatch.setattr(module, "make_source_repo", a_source_git_cannot_make) + error = the_error_of(AssertionError, call, tmp_path, adopter) + assert str(error) == GIT_FAILED + + +# --- a branch or a ref that is not UTF-8, or is text ------------------------ + +#: The three helpers of `tests/test_adopt_git_reads.py` that make a branch or a +#: ref with git, given a name that is not UTF-8 (`BRANCH`, `caf` and byte +#: 0xE9), each called with a `tmp_path`. The source is `tmp_path / "Thing"`; +#: only `moved_to_a_branch_that_is_not_utf8` makes it itself. +NOT_UTF_8_NAMES = [ + pytest.param( + lambda base: git_reads.with_default_branch(base / "Thing", + git_reads.ORIGIN_HEAD), + id="with-default-branch-origin-head"), + pytest.param( + lambda base: git_reads.with_default_branch(base / "Thing", "head"), + id="with-default-branch-head"), + pytest.param(git_reads.moved_to_a_branch_that_is_not_utf8, + id="moved-to-a-branch-that-is-not-utf8"), + pytest.param( + lambda base: git_reads.remembered_on(base / "Thing", git_reads.BRANCH), + id="remembered-on"), +] +#: The two of them that are also given a name that is text, with that name. +TEXT_NAMES = [ + pytest.param( + lambda base: git_reads.with_default_branch( + base / "Thing", git_reads.ORIGIN_HEAD, git_reads.NBSP_BRANCH), + git_reads.NBSP_BRANCH, id="with-default-branch-nbsp"), + pytest.param( + lambda base: git_reads.remembered_on(base / "Thing", + git_reads.LINE_SEPARATED), + git_reads.LINE_SEPARATED, id="remembered-on-u2028"), +] + + +def git_fails(monkeypatch) -> None: + """From here on every git call the module makes itself fails as + `conftest.git` fails where `git commit` cannot write an object. Only the + module's own name is replaced: `make_source_repo` and the tool's runs + still reach the real git.""" + monkeypatch.setattr(git_reads, "git", a_source_git_cannot_make) + + +@pytest.mark.usefixtures("a_disk_that_takes_the_probe") +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_a_failure_of_git_making_a_branch_that_is_not_utf8_fails_the_test( + tmp_path, monkeypatch, call): + """The system carries the name, so the helper goes on to git, and git's + failure reaches the test as a failure, where it was SKIPPED as "no + branch here is named ...". The rows ask the real disk for the probe + first, as the helper does.""" + git_fails(monkeypatch) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_a_failure_of_git_making_a_branch_that_is_text_fails_the_test( + tmp_path, monkeypatch, call, name): + """No system refuses a name that is text, so every skip there hid a + failure. Skipped where this system's file name encoding cannot spell the + name (ASCII under `LC_ALL=C`): the helper skips there first.""" + skip_where_the_environment_cannot_carry(name) + git_fails(monkeypatch) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.usefixtures("a_system_that_carries_the_byte") +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_a_disk_that_refuses_the_probe_with_eilseq_skips_before_git( + tmp_path, monkeypatch, call): + """macOS (emulated): the probe is `Illegal byte sequence`, so the helper + skips, on its reason, before any git call (each would fail here) and + before anything is made under `tmp_path`.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + reason = skip_message(call, tmp_path) + assert reason is not None, "the helper went on past the probe's EILSEQ" + assert "EILSEQ" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("call", NOT_UTF_8_NAMES) +def test_where_fsencode_makes_no_byte_of_the_surrogate_the_branch_skips_first( + tmp_path, monkeypatch, call): + """Windows (emulated): `os.fsencode` gives the surrogate three bytes, so + no argument can carry the byte. The helper skips on that reason before + the disk and git are asked, and nothing is made under `tmp_path`.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, "the helper went on where fsencode makes none" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_a_branch_that_is_text_asks_nothing_of_a_byte( + tmp_path, monkeypatch, call, name): + """Neither macOS's EILSEQ to a name that is not UTF-8 nor Windows' + `os.fsencode` of a surrogate is the answer for a name that is text, so + with both in place the helper goes on to git, and git's failure is the + test's. Skipped where the file name encoding cannot spell the name, as + the test above is.""" + skip_where_the_environment_cannot_carry(name) + git_fails(monkeypatch) + monkeypatch.setattr(os, "mkdir", refusing(errno.EILSEQ, probe=True)) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + error = the_error_of(AssertionError, call, tmp_path) + assert str(error) == GIT_FAILED + + +@pytest.mark.parametrize("call, name", TEXT_NAMES) +def test_where_the_file_name_encoding_is_ascii_a_branch_that_is_text_skips( + tmp_path, monkeypatch, call, name): + """`LC_ALL=C` (emulated): `subprocess` puts each argument through + `os.fsencode`, which cannot spell U+00A0 or U+2028 in ASCII, so the + helper skips on the encoding's reason before git is called, where the + git call raised `UnicodeEncodeError` in the test.""" + git_fails(monkeypatch) + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, f"{name!r} went on to git under ASCII" + assert "file name encoding" in reason, reason + + +# --- the U+202E directory under an ASCII file name encoding (#854) ---------- + +def test_where_the_file_name_encoding_cannot_spell_u202e_the_bidi_test_skips( + tmp_path, monkeypatch): + """`LC_ALL=C` (emulated as the real one answers): `os.fsencode` is ASCII, + and `os.mkdir` raises `UnicodeEncodeError` for a name ASCII cannot + encode. The test asks the encoding before its `mkdir` and skips on the + encoding's reason, worded as `make_file`'s is, and makes nothing; it + ended in that `UnicodeEncodeError`.""" + real_mkdir = os.mkdir + + def mkdir(path, *args, **kwargs): + ascii_fsencode(path) + return real_mkdir(path, *args, **kwargs) + + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + monkeypatch.setattr(os, "mkdir", mkdir) + reason = skip_message(BIDI_TEST, tmp_path) + assert reason is not None, "the test went on past an ASCII encoding" + assert "this system's file name encoding" in reason, reason + assert "cannot spell" in reason, reason + assert list(tmp_path.iterdir()) == [] + + +# --- no test skips on an `AssertionError` ----------------------------------- + +def names_in(node: ast.expr | None) -> set[str]: + """The names an `except` clause catches: one, or each of a tuple.""" + if isinstance(node, ast.Tuple): + return {name for element in node.elts for name in names_in(element)} + if isinstance(node, ast.Name): + return {node.id} + if isinstance(node, ast.Attribute): + return {node.attr} + return set() + + +def is_pytest_skip(node: ast.AST) -> bool: + return (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "skip" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "pytest") + + +def skips_on_an_assertion_error(source: str) -> list[int]: + """The line of each `except` in `source` that names `AssertionError`, + alone or in a tuple, and holds a call of `pytest.skip` anywhere in its + body: #853's scan.""" + return [node.lineno for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.ExceptHandler) + and "AssertionError" in names_in(node.type) + and any(is_pytest_skip(inner) for inner in ast.walk(node))] + + +def test_no_test_skips_on_an_assertion_error(): + """`conftest.git` and `make_source_repo` raise `AssertionError` for a git + process that fails, whatever made it fail, so an `except` that names it + and skips turns a full disk into SKIPPED. Where a system cannot carry a + name, a test asks it first and skips on that answer; the git call after + it is unguarded. Every `tests/*.py`, this file and `conftest.py` + included.""" + found = {} + for path in sorted((REPO / "tests").glob("*.py")): + lines = skips_on_an_assertion_error(path.read_text(encoding="utf-8")) + if lines: + found[path.name] = lines + assert found == {}, found + + +@pytest.mark.parametrize("source, lines", [ + pytest.param(""" + try: + make() + except AssertionError as exc: + pytest.skip(f"git will not work here: {exc}") + """, [4], id="alone"), + pytest.param(""" + try: + make() + except (AssertionError, UnicodeError, OSError) as exc: + pytest.skip(f"no branch here is named: {exc!r}") + """, [4], id="in-a-tuple"), + pytest.param(""" + try: + make() + except AssertionError: + if not asked: + raise + pytest.skip("git will not work here") + """, [4], id="behind-a-raise"), + pytest.param(""" + try: + make() + except AssertionError: + raise + """, [], id="no-skip"), + pytest.param(""" + try: + make() + except OSError: + pytest.skip("another error") + """, [], id="another-error"), + pytest.param(""" + if not read.endswith(written): + pytest.skip("git here does not give the name back") + """, [], id="a-skip-on-an-answer"), +]) +def test_the_scan_finds_a_skip_on_an_assertion_error_and_nothing_else( + source, lines): + """A skip behind an `if` in the `except` is found too, as #845's + `skip_where_git_fails` had it; an `except` that does not skip, one of + another error, and the skip after `git symbolic-ref`, which answers a + question and catches nothing, are not.""" + assert skips_on_an_assertion_error(dedent(source)) == lines diff --git a/tests/test_non_utf8_git_reads.py b/tests/test_non_utf8_git_reads.py index a53d6547..29728bbc 100644 --- a/tests/test_non_utf8_git_reads.py +++ b/tests/test_non_utf8_git_reads.py @@ -55,6 +55,7 @@ from conftest import (ORG, PROJECT, REPO, appended_to_command_scope, cloned_just_pushed, git, run_script, skip_message, skip_where_fsencode_gives_a_surrogate_no_byte, + skip_where_the_environment_cannot_carry, windows_fsencode) from test_bump_leg import branched, bump @@ -513,22 +514,6 @@ def test_a_clean_standard_prints_no_warning(standard, tmp_path, run, warning, LEGACY_NAME = "Jos\udce9" -def skip_where_the_environment_cannot_carry(value: str) -> None: - """Skip where this system's file name encoding cannot encode `value`, as - under `LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0` (ASCII). It is asked of - a file name (`make_file` cannot write `caf\\u00e9.yml`) and of an - environment value: `subprocess` puts each one through `os.fsencode` - before it starts the child, so a `user.name` of `Jos\\u00e9` raises - `UnicodeEncodeError` in the test and not in the tool (#757). Text that is - ASCII, or a lone surrogate that `os.fsencode` gives back as its byte, is - carried.""" - try: - os.fsencode(value) - except UnicodeEncodeError: - pytest.skip(f"this system's file name encoding " - f"({sys.getfilesystemencoding()}) cannot carry {value!r}") - - REFUSED = "REFUSED family-unwritable-value: " #: How the refusal names the creator when no flag supplied it: `_manifest_flags` #: (#307), so that the two refusals agree on who is meant. diff --git a/tests/test_non_utf8_name_callers.py b/tests/test_non_utf8_name_callers.py index 76db45df..dd6c7386 100644 --- a/tests/test_non_utf8_name_callers.py +++ b/tests/test_non_utf8_name_callers.py @@ -53,7 +53,9 @@ import test_non_utf8_git_reads as non_utf8_git_reads import test_stdout_encoding as stdout_encoding from conftest import (NOT_UTF_8_PROBE, skip_message, - skip_where_a_name_cannot_be_not_utf_8, windows_fsencode) + skip_where_a_name_cannot_be_not_utf_8, + skip_where_the_environment_cannot_carry, + windows_fsencode) # The fixture of `test_adopt_plan_escapes.py`, `adopt-project.py` loaded as a # module, which two of its tests ask for. from test_adopt_plan_escapes import adopter # noqa: F401 - fixture @@ -285,8 +287,7 @@ def test_a_write_the_disk_refuses_still_fails_the_make_file_test( skip on `make_file`'s own "filesystem" reason would end it SKIPPED, and no run fails on that; before #834 it did skip, and the test failed on its assert that the call is not a skip.""" - non_utf8_git_reads.skip_where_the_environment_cannot_carry( - non_utf8_git_reads.UTF8) + skip_where_the_environment_cannot_carry(non_utf8_git_reads.UTF8) def write_bytes(self, data): raise OSError(code, os.strerror(code), str(self)) @@ -301,8 +302,7 @@ def write_bytes(self, data): def test_a_user_name_the_encoding_cannot_carry_skips_where_it_is_ascii( monkeypatch, value): monkeypatch.setattr(os, "fsencode", ascii_fsencode) - reason = skip_message( - non_utf8_git_reads.skip_where_the_environment_cannot_carry, value) + reason = skip_message(skip_where_the_environment_cannot_carry, value) assert reason is not None, f"{value!r} was carried by an ASCII encoding" assert "file name encoding" in reason, reason @@ -315,9 +315,8 @@ def test_a_user_name_the_encoding_carries_is_not_skipped_even_where_it_is_ascii( """ASCII text, a lone surrogate that is a byte, and the control characters are the environment's own: the tests of them still run.""" monkeypatch.setattr(os, "fsencode", ascii_fsencode) - assert skip_message( - non_utf8_git_reads.skip_where_the_environment_cannot_carry, - value) is None + assert skip_message(skip_where_the_environment_cannot_carry, + value) is None @pytest.mark.parametrize("test, args", [ diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index 181ed792..913ccd49 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -41,9 +41,11 @@ EDQUOT, fails here; - Windows, by an `os.fsencode` that gives a surrogate three bytes: the three tests that ask the helper skip before the disk is touched; -- git failing under the U+202E directory fails the test, and under the - directory named with a byte that is not UTF-8 still skips it (#757 kept that - second skip, and it is a known limit: a full disk skips there). +- git failing under the U+202E directory fails the test. Under the directory + named with a byte that is not UTF-8 it fails the test too since #853, which + dropped the second skip #757 kept there: + `tests/test_non_utf8_git_failures.py` holds that, for each test that makes + a source repository under such a directory. """ from __future__ import annotations @@ -59,7 +61,8 @@ import test_cli_reference as cli_reference import test_doctor_validator_verdicts as doctor_verdicts import test_non_utf8_git_reads as non_utf8_git_reads -from conftest import skip_message, windows_fsencode +from conftest import (skip_message, skip_where_the_environment_cannot_carry, + windows_fsencode) # The fixtures and the two functions of #757's callers file, which ask the same # questions of the same helper. from test_non_utf8_name_callers import ( # noqa: F401 - fixtures @@ -283,8 +286,6 @@ def test_make_file_raises_any_other_error_of_the_disk( BIDI = chr(0x202E) BIDI_TEST = (template_values .test_execute_refuses_a_local_source_directory_holding_a_bidi_control) -BYTE_TEST = (template_values - .test_execute_refuses_a_local_source_directory_that_is_not_utf8) def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( @@ -292,7 +293,10 @@ def test_a_disk_that_refuses_the_bidi_directory_fails_the_test( """U+202E is valid text: no probe is asked, and the directory is made with no guard, so what the disk answers to it is raised, whichever errno it is. It was a skip that said "no directory here is named ..." for a full disk - or a read-only one.""" + or a read-only one. Skipped where this system's file name encoding cannot + spell U+202E (ASCII under `LC_ALL=C`): the test skips there before it + asks the disk (#854).""" + skip_where_the_environment_cannot_carry(BIDI) disk = a_disk_that_refuses(monkeypatch, errno.EACCES) raised_for_every_errno(disk, BIDI_TEST, tmp_path) @@ -303,28 +307,15 @@ def test_a_failure_of_git_under_the_bidi_directory_fails_the_test( where git cannot (a disk that filled after the `mkdir`), the failure is raised, as the name is valid text and not the cause. It was a skip that said "git will not work under ...". Skipped where this system's file name - encoding cannot spell U+202E (ASCII under `LC_ALL=C`): the directory is - really made here, and `mkdir` raises `UnicodeEncodeError` first.""" - non_utf8_git_reads.skip_where_the_environment_cannot_carry(BIDI) + encoding cannot spell U+202E (ASCII under `LC_ALL=C`): the test skips + there before it makes the directory (#854).""" + skip_where_the_environment_cannot_carry(BIDI) monkeypatch.setattr(template_values, "make_source_repo", a_source_git_cannot_make) error = the_error_of(AssertionError, BIDI_TEST, tmp_path) assert str(error) == GIT_FAILED -@pytest.mark.usefixtures("a_disk_that_takes_the_probe") -def test_a_failure_of_git_under_the_byte_directory_still_skips_the_test( - tmp_path, monkeypatch): - """The control, and the known limit: #757 kept the second skip for a name - that is not UTF-8, which git may not work under, so a disk that filled - after the `mkdir` still reads as SKIPPED there.""" - monkeypatch.setattr(template_values, "make_source_repo", - a_source_git_cannot_make) - reason = skip_message(BYTE_TEST, tmp_path) - assert reason is not None, "the test went on past git's failure" - assert "git will not work under" in reason, reason - - # --- the sweep itself ------------------------------------------------------- def test_the_sweep_names_the_first_errno_that_fails_and_stops_there(): From 60836f5cfcae6a82df33309d9f958fa56b877b6a Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 04:03:53 +0000 Subject: [PATCH 11/13] tests: the three tests of the byte directory have Windows rows that hold them to asking before their `mkdir`, and the scan finds a `pytest.skip` in a bare, `Exception` or `BaseException` handler too (#853, review of #869) - `test_where_fsencode_makes_no_byte_the_byte_directory_test_skips_first` (`tests/test_non_utf8_git_failures.py`), one row for each of the three tests over `UNDER_A_BYTE_DIRECTORY`: with `os.fsencode` as Windows has it, each test skips on the helper's reason and makes nothing under `tmp_path`. A test that asked after its `mkdir` passed every row before. - `skips_on_an_assertion_error` is `skips_that_catch_an_assertion_error`: it finds an `except` that catches an `AssertionError` without naming it as well, so an `except Exception: pytest.skip(...)` around a git call fails the scan. No open pull request's `tests/` holds one. - `tests/test_non_utf8_oserror_sites.py`'s docstring says the U+202E test asks the file name encoding before its `mkdir` (#854). Co-Authored-By: Claude Opus 5.5 --- tests/test_non_utf8_git_failures.py | 99 +++++++++++++++++++++++----- tests/test_non_utf8_oserror_sites.py | 2 +- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/tests/test_non_utf8_git_failures.py b/tests/test_non_utf8_git_failures.py index e0e5cee4..6a7286e8 100644 --- a/tests/test_non_utf8_git_failures.py +++ b/tests/test_non_utf8_git_failures.py @@ -43,14 +43,19 @@ - git fails: the test FAILS with git's error, for a name that is not UTF-8 and for one that is text; -- macOS's EILSEQ on the probe, or Windows' `os.fsencode` (a surrogate is three - bytes): a branch that is not UTF-8 is skipped on the helper's reason before - git is called and before anything is made; a branch that is text asks - neither and goes on to git; +- Windows' `os.fsencode` (a surrogate is three bytes): the three tests of the + byte directory are skipped on the helper's reason before their `mkdir` and + before git, and make nothing (macOS's EILSEQ on the probe is held for them + by #757's rows in `tests/test_non_utf8_name_callers.py`); +- macOS's EILSEQ on the probe, or Windows' `os.fsencode`: a branch that is + not UTF-8 is skipped on the helper's reason before git is called and + before anything is made; a branch that is text asks neither and goes on + to git; - an ASCII file name encoding: a branch that is text, and the U+202E test, are skipped on the encoding's reason, before git and before the `mkdir`; -- every `tests/*.py`, read as #853's scan reads it: no `except` that names - `AssertionError` holds a `pytest.skip`. +- every `tests/*.py`, read as #853's scan reads it and wider: no `except` + that catches an `AssertionError` (one that names it, `Exception` or + `BaseException`, or a bare one) holds a `pytest.skip`. """ from __future__ import annotations @@ -120,6 +125,22 @@ def test_a_failure_of_git_under_the_byte_directory_fails_the_test( assert str(error) == GIT_FAILED +@pytest.mark.parametrize("module, call", UNDER_A_BYTE_DIRECTORY) +def test_where_fsencode_makes_no_byte_the_byte_directory_test_skips_first( + tmp_path, monkeypatch, adopter, module, call): + """Windows (emulated): `os.fsencode` gives the surrogate three bytes, so + the test skips on the helper's reason before it makes its directory or + asks git (which would fail here), and nothing is made under `tmp_path`. + A test that asked after its `mkdir` would leave the directory here, and + on Windows raise at `os.fsdecode` of the name instead of skipping.""" + monkeypatch.setattr(module, "make_source_repo", a_source_git_cannot_make) + monkeypatch.setattr(os, "fsencode", windows_fsencode) + reason = skip_message(call, tmp_path, adopter) + assert reason is not None, "the test went on where fsencode makes none" + assert "os.fsencode" in reason, reason + assert list(tmp_path.iterdir()) == [] + + # --- a branch or a ref that is not UTF-8, or is text ------------------------ #: The three helpers of `tests/test_adopt_git_reads.py` that make a branch or a @@ -290,26 +311,42 @@ def is_pytest_skip(node: ast.AST) -> bool: and node.func.value.id == "pytest") -def skips_on_an_assertion_error(source: str) -> list[int]: - """The line of each `except` in `source` that names `AssertionError`, - alone or in a tuple, and holds a call of `pytest.skip` anywhere in its - body: #853's scan.""" +#: What an `except` that catches an `AssertionError` names: the error itself +#: or a class above it. A bare `except` catches it too. +CATCH_AN_ASSERTION_ERROR = {"AssertionError", "Exception", "BaseException"} + + +def catches_an_assertion_error(handler: ast.ExceptHandler) -> bool: + """A bare `except`, or one that names `AssertionError`, `Exception` or + `BaseException`, alone or in a tuple.""" + return (handler.type is None + or bool(names_in(handler.type) & CATCH_AN_ASSERTION_ERROR)) + + +def skips_that_catch_an_assertion_error(source: str) -> list[int]: + """The line of each `except` in `source` that catches an + `AssertionError` and holds a call of `pytest.skip` anywhere in its body: + #853's scan, of an `except` that names `AssertionError`, and with it one + that catches the error without naming it (`Exception`, `BaseException` + or a bare `except`), which would hide a failed git call as well.""" return [node.lineno for node in ast.walk(ast.parse(source)) if isinstance(node, ast.ExceptHandler) - and "AssertionError" in names_in(node.type) + and catches_an_assertion_error(node) and any(is_pytest_skip(inner) for inner in ast.walk(node))] def test_no_test_skips_on_an_assertion_error(): """`conftest.git` and `make_source_repo` raise `AssertionError` for a git - process that fails, whatever made it fail, so an `except` that names it - and skips turns a full disk into SKIPPED. Where a system cannot carry a + process that fails, whatever made it fail, so an `except` that catches it + and skips turns a full disk into SKIPPED, whether it names the error, + `Exception` or `BaseException`, or nothing. Where a system cannot carry a name, a test asks it first and skips on that answer; the git call after it is unguarded. Every `tests/*.py`, this file and `conftest.py` included.""" found = {} for path in sorted((REPO / "tests").glob("*.py")): - lines = skips_on_an_assertion_error(path.read_text(encoding="utf-8")) + lines = skips_that_catch_an_assertion_error( + path.read_text(encoding="utf-8")) if lines: found[path.name] = lines assert found == {}, found @@ -336,12 +373,36 @@ def test_no_test_skips_on_an_assertion_error(): raise pytest.skip("git will not work here") """, [4], id="behind-a-raise"), + pytest.param(""" + try: + make() + except Exception as exc: + pytest.skip(f"no branch here is named: {exc!r}") + """, [4], id="exception"), + pytest.param(""" + try: + make() + except (OSError, BaseException): + pytest.skip("anything at all") + """, [4], id="base-exception-in-a-tuple"), + pytest.param(""" + try: + make() + except: + pytest.skip("anything at all") + """, [4], id="bare"), pytest.param(""" try: make() except AssertionError: raise """, [], id="no-skip"), + pytest.param(""" + try: + make() + except Exception: + raise + """, [], id="exception-no-skip"), pytest.param(""" try: make() @@ -356,7 +417,9 @@ def test_no_test_skips_on_an_assertion_error(): def test_the_scan_finds_a_skip_on_an_assertion_error_and_nothing_else( source, lines): """A skip behind an `if` in the `except` is found too, as #845's - `skip_where_git_fails` had it; an `except` that does not skip, one of - another error, and the skip after `git symbolic-ref`, which answers a - question and catches nothing, are not.""" - assert skips_on_an_assertion_error(dedent(source)) == lines + `skip_where_git_fails` had it, and so is one in an `except` that catches + the error without naming it: `Exception`, `BaseException`, or a bare + `except`. An `except` that does not skip, one of another error, and the + skip after `git symbolic-ref`, which answers a question and catches + nothing, are not.""" + assert skips_that_catch_an_assertion_error(dedent(source)) == lines diff --git a/tests/test_non_utf8_oserror_sites.py b/tests/test_non_utf8_oserror_sites.py index 913ccd49..428828dd 100644 --- a/tests/test_non_utf8_oserror_sites.py +++ b/tests/test_non_utf8_oserror_sites.py @@ -26,7 +26,7 @@ and raises the rest. `make_file` skips on EILSEQ and on a `UnicodeError` (an ASCII file name encoding, #757), each on a reason of its own, and raises the rest. The U+202E test makes its directory, and its source repository, with no -guard. +guard but the file name encoding's question, asked before its `mkdir` (#854). Each is called here as a plain function, with the disk answering as it would on a system this suite does not run on (CI is Linux only, #597), and asked what From bcf6952eed13357dd5788066c6d1b0bd82d0e926 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 04:03:54 +0000 Subject: [PATCH 12/13] tests: the six template-value tests that raised `UnicodeEncodeError` under a C locale ask the file name encoding first and skip there, as #854's U+202E test does Under `PYTHONUTF8=0 LC_ALL=C PYTHONCOERCECLOCALE=0` the file name encoding is ASCII. Three rows of `test_plan_refuses_a_flag_no_file_can_hold` (U+2028, U+202E, U+2066) and both rows of `test_a_valid_non_ascii_elector_still_adopts` raised at `subprocess.run` in `adopted`, which cannot put the value on `plan`'s command line there; `test_a_utf8_source_directory_still_adopts_with_local_remotes` raised at the `mkdir` of `caf` and U+00E9 in `make_source_repo`. Each now calls `skip_where_the_environment_cannot_carry` first. Under UTF-8 none skips. Co-Authored-By: Claude Opus 5.5 --- tests/test_adopt_template_values.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/test_adopt_template_values.py b/tests/test_adopt_template_values.py index cf6b6952..c2944c07 100644 --- a/tests/test_adopt_template_values.py +++ b/tests/test_adopt_template_values.py @@ -198,7 +198,11 @@ def test_plan_refuses_a_flag_no_file_can_hold(tmp_path, flag, value): `plan ok`, and `execute` raised in `write_lf` after both legs were pushed (a surrogate), or adopted with the line break, the tab or the bidi control in the manifest and the README. Now `plan` refuses, and - writes no plan.""" + writes no plan. Skipped where this system's file name encoding cannot + spell `value` (U+2028, U+202E or U+2066 in ASCII under `LC_ALL=C`), + asked first: `subprocess` cannot put it on `plan`'s command line there, + and raised `UnicodeEncodeError` in the test, not in the tool.""" + skip_where_the_environment_cannot_carry(value) source = make_source_repo(tmp_path / "Thing") runs = plan_check_execute(tmp_path, source, flag, value) assert_no_traceback_and_no_leg(tmp_path, runs) @@ -645,7 +649,13 @@ def test_a_valid_non_ascii_elector_still_adopts(tmp_path, elected_by): The manifest is asked through the reader, not for a line: since #250 a YAML template's value is written as `double_quoted` spells it, so the - line holds U+200C as an escape there, and as it stands before.""" + line holds U+200C as an escape there, and as it stands before. + + Skipped where this system's file name encoding cannot spell the name + (ASCII under `LC_ALL=C`), asked first: `subprocess` cannot put it on + `plan`'s command line there, and raised `UnicodeEncodeError` in the + test, not in the tool.""" + skip_where_the_environment_cannot_carry(elected_by) source = make_source_repo(tmp_path / "Thing") runs = plan_check_execute(tmp_path, source, "--elected-by", elected_by) assert_no_traceback(runs) @@ -664,8 +674,13 @@ def test_a_valid_non_ascii_elector_still_adopts(tmp_path, elected_by): def test_a_utf8_source_directory_still_adopts_with_local_remotes(tmp_path): """A source under a directory `café`: its path is the clone URL, which a file holds, so it adopts, and the shape's README (beside the source's - own) names it.""" - source = make_source_repo(tmp_path / "café" / "Thing") + own) names it. Skipped where this system's file name encoding cannot + spell the directory's name (ASCII under `LC_ALL=C`), asked before the + directory is made, where `mkdir` raised `UnicodeEncodeError` in the + test.""" + parent = tmp_path / "caf\u00e9" + skip_where_the_environment_cannot_carry(parent.name) + source = make_source_repo(parent / "Thing") runs = plan_check_execute(tmp_path, source) assert_no_traceback(runs) assert runs["check"].returncode == 0, said(runs["check"]) From 2572a717b6a85d7ac246a8c28428374f63cc6802 Mon Sep 17 00:00:00 2001 From: Brett Heap Date: Sun, 18 Oct 2026 04:44:27 +0000 Subject: [PATCH 13/13] tests: windows_fsencode takes a path as os.fsencode does, and six rows hold the C-locale skips of test_adopt_template_values.py in CI (#853, #854) Fix round 2 of #869, RV869b's NIT 1 and NIT 2. NIT 1: windows_fsencode now calls os.fspath first, so a test that asks os.fsencode(tmp_path) under the emulation no longer fails with an AttributeError before the question it holds; the rows of test_non_utf8_git_failures.py that ask after a mkdir now fail on their own empty-tmp_path assertion. A row in test_non_utf8_name_helpers.py holds it. NIT 2: six emulated rows in test_non_utf8_git_failures.py (os.fsencode as ASCII, make_source_repo failing) hold the six tests of test_adopt_template_values.py that skip under LC_ALL=C: each must skip on the file name encoding with nothing made under tmp_path. CI runs UTF-8, so a test that asked after make_source_repo or adopted passed there before. Co-Authored-By: Claude Sonnet 5.5 Co-Authored-By: Claude Opus 5.5 --- tests/conftest.py | 8 ++-- tests/test_non_utf8_git_failures.py | 59 ++++++++++++++++++++++++++++- tests/test_non_utf8_name_helpers.py | 9 +++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index db342b94..45b0292e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1177,12 +1177,14 @@ def skip_where_a_name_cannot_be_not_utf_8(directory: Path) -> None: probe.rmdir() -def windows_fsencode(text: str | bytes) -> bytes: +def windows_fsencode(text: str | bytes | os.PathLike) -> bytes: """`os.fsencode` as Windows has it: UTF-8 with `surrogatepass`, so every - lone surrogate is the three bytes of a sequence that is not UTF-8. Bytes - go through as they are, as `os.fsencode` passes them on every system. For + lone surrogate is the three bytes of a sequence that is not UTF-8. A path + is taken as the real `os.fsencode` takes it (`os.fspath` first), and + bytes go through as they are, as it passes them on every system. For `monkeypatch.setattr(os, "fsencode", windows_fsencode)`, which puts the guard above in front of a test that Linux would let through.""" + text = os.fspath(text) if isinstance(text, bytes): return text return text.encode("utf-8", "surrogatepass") diff --git a/tests/test_non_utf8_git_failures.py b/tests/test_non_utf8_git_failures.py index 6a7286e8..6f657c21 100644 --- a/tests/test_non_utf8_git_failures.py +++ b/tests/test_non_utf8_git_failures.py @@ -36,6 +36,9 @@ the file name encoding is ASCII, and the U+202E test's `mkdir` raised `UnicodeEncodeError`, a `ValueError` its old `except OSError` never caught, so the test ended in an error. It asks the encoding before the `mkdir` now. +The six tests of `tests/test_adopt_template_values.py` that ask it before +`make_source_repo` (a flag, an elector or a directory that ASCII cannot +spell) are held for CI, which runs UTF-8, by rows that emulate ASCII. Each place is called here as a plain function, with git made to fail as a full disk makes `git commit` fail, or with a system emulated that CI does not @@ -52,7 +55,10 @@ before anything is made; a branch that is text asks neither and goes on to git; - an ASCII file name encoding: a branch that is text, and the U+202E test, are - skipped on the encoding's reason, before git and before the `mkdir`; + skipped on the encoding's reason, before git and before the `mkdir`; so + are the six tests of `tests/test_adopt_template_values.py` that ask the + encoding first, before `make_source_repo` (which would fail here) and + before anything is made under `tmp_path`; - every `tests/*.py`, read as #853's scan reads it and wider: no `except` that catches an `AssertionError` (one that names it, `Exception` or `BaseException`, or a bare one) holds a `pytest.skip`. @@ -290,6 +296,57 @@ def mkdir(path, *args, **kwargs): assert list(tmp_path.iterdir()) == [] +#: The six tests of `tests/test_adopt_template_values.py` that skip where the +#: file name encoding cannot spell their value (#854), each called with a +#: `tmp_path`: the flag test with the three values ASCII cannot spell (U+2028, +#: U+202E and U+2066), the elector test with `Jos\u00e9` and a name joined +#: with U+200C, and the test of a source under a directory `caf\u00e9`. +WHERE_ASCII_CANNOT_SPELL = [ + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--elected-by", "Jos\u2028e"), + id="flag-u2028"), + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--elected-by", "\u202eevil"), + id="flag-u202e"), + pytest.param( + lambda base: template_values.test_plan_refuses_a_flag_no_file_can_hold( + base, "--reference", "ref \u2066x\u2069 y"), + id="flag-u2066"), + pytest.param( + lambda base: template_values + .test_a_valid_non_ascii_elector_still_adopts(base, "Jos\u00e9"), + id="elector-accented"), + pytest.param( + lambda base: template_values + .test_a_valid_non_ascii_elector_still_adopts(base, "Ali\u200cReza"), + id="elector-zwnj"), + pytest.param( + template_values + .test_a_utf8_source_directory_still_adopts_with_local_remotes, + id="cafe-directory"), +] + + +@pytest.mark.parametrize("call", WHERE_ASCII_CANNOT_SPELL) +def test_where_the_file_name_encoding_is_ascii_the_template_values_tests_skip( + tmp_path, monkeypatch, call): + """`LC_ALL=C` (emulated): `os.fsencode` is ASCII, so each test skips on + the encoding's reason before `make_source_repo` (which fails here, as git + fails on a full disk) and before any directory is made, and makes nothing + under `tmp_path`. CI runs UTF-8, where a test that asked after either + would still pass; these rows fail it there, as the real `LC_ALL=C` run + does.""" + monkeypatch.setattr(template_values, "make_source_repo", + a_source_git_cannot_make) + monkeypatch.setattr(os, "fsencode", ascii_fsencode) + reason = skip_message(call, tmp_path) + assert reason is not None, "the test went on under an ASCII encoding" + assert "file name encoding" in reason, reason + assert list(tmp_path.iterdir()) == [] + + # --- no test skips on an `AssertionError` ----------------------------------- def names_in(node: ast.expr | None) -> set[str]: diff --git a/tests/test_non_utf8_name_helpers.py b/tests/test_non_utf8_name_helpers.py index 47ae0d13..8594c6b6 100644 --- a/tests/test_non_utf8_name_helpers.py +++ b/tests/test_non_utf8_name_helpers.py @@ -90,6 +90,15 @@ def test_windows_fsencode_passes_bytes_through_as_os_fsencode_does(): assert windows_fsencode(data) == data +def test_windows_fsencode_takes_a_path_as_os_fsencode_does(): + """`os.fsencode` takes a path-like object (`os.fspath` first), and a test + asks it for one (`os.fsencode(tmp_path)`), so the emulation must not fail + on a `Path` with an `AttributeError`, which would end such a test before + the question it is written to hold. A surrogate in a path is the three + bytes of one in text.""" + assert windows_fsencode(Path("caf\udcff")) == b"caf\xed\xb3\xbf" + + # --- the system's question: what `os.fsencode` makes of the surrogate -------- @LINUX_ONLY