repo_shape.py, validate-manifest.py: a bracket in a plain flow scalar is refused and a name holding a lone surrogate is a finding, so pins: [a, b]] is not the pin b] and name: "a\ud83db" no longer kills make validate (#301, #280)

scripts/repo_shape.py

@@ -461,15 +461,46 @@ def _scalar(raw: str) -> Any:
     return text
 
 
+def _refuse_flow_indicator(text: str) -> None:
+    """Refuse a PLAIN scalar of a flow collection that holds `[`, `]`, `{` or
+    `}` (#301); a quoted one is `_scalar`'s to judge, and its brackets are
+    text.
+
+    In YAML a plain scalar inside a flow collection ENDS at any of the four,
+    as it ends at the `,` `_split_flow` splits at, so a scalar holding one is
+    never the text it looks like, and PyYAML refuses the document. `k: [a]]`
+    closes the sequence after `a` and leaves a `]` nothing opened, and
+    `k: [x [a, b]]` opens a sequence while an item is still being read.
+    `_split_flow` counts brackets to find its commas and never asked whether
+    the count went below zero or ended above it, so the rest was read as
+    text: `['a]']`, `['x [a, b]']`, and the key of `k: {a"]:}` as `a"]`
+    since #284 stopped reading that `"` as a quote. Refused, a stray
+    bracket typed into `pins: [a, b]]` is a `yaml-unparsable` naming it, not
+    the pin `b]`.
+    """
+    if text.startswith(('"', "'")):
+        return
+    held = next((char for char in text if char in "[]{}"), None)
+    if held is not None:
+        raise YamlError(f"a plain scalar in a flow collection holds `{held}`, "
+                        f"which ends it in YAML: {text!r}")
+
+
 def _flow_value(text: str) -> Any:
     """A flow entry is itself a flow collection when it OPENS with `[` or `{`.
 
     AN EMPTY ENTRY OPENS NOTHING (#252): `{a: }` and `{a:}` give this `''`,
     and `'' in "[{"` is True for any string, so the test used to run `_flow`
     on it and refuse `{a: }` as `not a flow collection: ''`, a text nobody
     wrote. `_scalar('')` is the null a block `a:` reads.
+
+    ONE THAT OPENS NOTHING HOLDS NO BRACKET OR BRACE (#301): a plain item or
+    value holding one is refused by `_refuse_flow_indicator`.
     """
-    return _flow(text) if text.startswith(("[", "{")) else _scalar(text)
+    if text.startswith(("[", "{")):
+        return _flow(text)
+    _refuse_flow_indicator(text)
+    return _scalar(text)
 
 
 def _flow(text: str) -> Any:
@@ -506,10 +537,19 @@ def _split_key_flow(text: str) -> tuple[str, str] | None:
     names its line, and a flow collection has none to name, so this names the
     entry. A QUOTED key, `"? a"`, does not open with the `?`, and `?a` is not
     the indicator, which needs a space after it: both read as they did.
+
+    A PLAIN KEY HOLDS NO `]` OR `}` EITHER (#301). `_split_key` refuses a
+    `[` or `{` later in a key, and in a flow a closer ends the key as surely,
+    so `{a"]:}` read as the key `a"]` and `{]:}` as `]`, where PyYAML
+    refuses both. `_refuse_flow_indicator` refuses them; a quoted key is
+    still text.
     """
     if text.startswith("? "):
         raise YamlError(f"explicit keys are not supported: {text!r}")
-    return _split_key(text)
+    kv = _split_key(text)
+    if kv is not None and not text.startswith(('"', "'")):
+        _refuse_flow_indicator(kv[0])
+    return kv
 
 
 def _split_flow(body: str) -> list[str]:

templates/assembly-root/scripts/validate-manifest.py

@@ -378,9 +378,32 @@ def _name_finding(manifest: dict) -> list[str]:
     """`name`.
 
     Split from `_findings` for #132.
+
+    A NAME HOLDING A LONE SURROGATE IS A FINDING (#280). The double-quoted
+    escape `"a\\ud83db"` reads as one, as PyYAML's own Python reader reads it
+    (libyaml refuses the file). The reader keeps that escape on purpose: it
+    is how an adoption plan spells a path byte that is not UTF-8 (#199). A
+    display name is text a person chose, and no UTF-8 file, commit message
+    or console can hold a surrogate, so this file's `manifest ok` line
+    raised `UnicodeEncodeError` on it; `make validate` and `shape-doctor.py`
+    reported that traceback, while `bump-leg.py` and `update-shape.py apply`
+    passed the name, and the bump wrote it into its commit as `a?b`. Every
+    launcher now meets this finding instead. A pair of `\\u` escapes is two
+    surrogates here, as it is to PyYAML, so the finding asks for the
+    character itself or its one `\\U` escape.
     """
-    if not isinstance(manifest.get("name"), str) or not manifest.get("name").strip():
-        return [_finding("manifest-name", f"name is {manifest.get('name')!r}")]
+    name = manifest.get("name")
+    if not isinstance(name, str) or not name.strip():
+        return [_finding("manifest-name", f"name is {name!r}")]
+    surrogate = next((char for char in name if "\ud800" <= char <= "\udfff"),
+                     None)
+    if surrogate is not None:
+        return [_finding(
+            "manifest-name",
+            f"name is {name!r}, which holds U+{ord(surrogate):04X}, a lone "
+            "surrogate: no UTF-8 file, commit message or console can hold "
+            "it. Write each character as itself, or as one `\\U` escape "
+            "(`\\U0001F600`, not the pair `\\uD83D\\uDE00`).")]
     return []
 
 

tests/test_manifest_validator.py

@@ -3,6 +3,10 @@
 
 from __future__ import annotations
 
+import os
+import subprocess
+import sys
+
 import pytest
 
 from conftest import run_script
@@ -326,6 +330,73 @@
     assert "manifest ok" not in result.stdout
 
 
+# --- a display name no UTF-8 text can hold (#280) ---------------------------
+
+def rename(project, escape: str) -> None:
+    """Put `name: "<escape>"` in place of the scaffolded `name: "Atlas"`, as
+    BYTES: `escape` is the YAML spelling, backslash and all, and the file
+    stays ASCII and keeps whatever line endings it has."""
+    manifest = project / "project.yaml"
+    data = manifest.read_bytes()
+    assert data.count(b'\nname: "Atlas"') == 1, "fixture drift: the name line"
+    manifest.write_bytes(data.replace(b'\nname: "Atlas"',
+                                      f'\nname: "{escape}"'.encode("ascii")))
+
+
+def validate_bytes(project, **env: str):
+    """`(exit code, stdout, stderr)` as BYTES, so no codec of this process
+    reads them, with `env` set over this process's environment."""
+    proc = subprocess.run(
+        [sys.executable, str(project / "scripts" / "validate-manifest.py")],
+        cwd=str(project), capture_output=True, stdin=subprocess.DEVNULL,
+        env={**os.environ, **env}, check=False)
+    return proc.returncode, proc.stdout, proc.stderr
+
+
+@pytest.mark.parametrize("escape, code_point", [
+    pytest.param("a\\ud83db", "U+D83D", id="the-issue-row-high-surrogate"),
+    pytest.param("caf\\udce9", "U+DCE9", id="the-byte-surrogateescape-keeps"),
+    pytest.param("a\\ud800b", "U+D800", id="the-first-surrogate"),
+    pytest.param("a\\udfffb", "U+DFFF", id="the-last-surrogate"),
+    pytest.param("\\ud83d\\ude00", "U+D83D", id="a-pair-is-two-surrogates"),
+])
+@pytest.mark.parametrize("ioencoding", [None, "utf-8"])
+def test_a_name_holding_a_lone_surrogate_is_a_finding(project, escape,
+                                                      code_point, ioencoding):
+    """On c7c02fa the reader took the escape, the validator accepted the
+    name, and its `manifest ok` line raised `UnicodeEncodeError`: exit 1 and
+    a traceback, under a pipe and under `PYTHONIOENCODING=utf-8` (the
+    doctor's) alike. It is a finding now, on stderr, with the name escaped
+    by `repr`, before that line is reached."""
+    rename(project, escape)
+    env = {} if ioencoding is None else {"PYTHONIOENCODING": ioencoding}
+    code, stdout, stderr = validate_bytes(project, **env)
+    shown = (stdout + stderr).decode("utf-8", "backslashreplace")
+    assert code == 1, shown
+    assert b"Traceback" not in stderr and b"UnicodeEncodeError" not in stderr, \
+        shown
+    assert b"manifest ok" not in stdout, shown
+    finding = (f"FINDING manifest-name: name is '{escape}', which holds "
+               f"{code_point}, a lone surrogate").encode("ascii")
+    assert finding in stderr, shown
+    assert b"1 finding(s)" in stderr, "the name is the only finding"
+
+
+@pytest.mark.parametrize("escape", [
+    pytest.param("\\U0001F600", id="an-astral-character-as-one-escape"),
+    pytest.param("\\u00e9t\\u00e9", id="a-bmp-character-as-an-escape"),
+    pytest.param("\\ue000", id="private-use-just-past-the-surrogates"),
+    pytest.param("\\ud7ff", id="the-last-character-before-them"),
+])
+def test_a_name_spelled_without_a_surrogate_is_still_ok(project, escape):
+    """What the finding must not take: every other code point, spelled as an
+    escape, including the two either side of U+D800..U+DFFF."""
+    rename(project, escape)
+    code, stdout, stderr = validate_bytes(project, PYTHONIOENCODING="utf-8")
+    assert code == 0, stderr.decode("utf-8", "backslashreplace")
+    assert stdout.startswith(b"manifest ok: "), stdout
+
+
 # --- the ruling: a chain of pins may reach the referent (2026-09-05) --------
 #
 # "elect the shape for both, follow the pin chain, no family yet" — Brett Heap,

tests/test_validator_child_encoding.py

@@ -30,6 +30,10 @@
 - the environment helpers themselves, each one handed to a real child that
   prints the name, so a handler a child does not know fails here; and a guard
   that no tool runs the interpreter without one.
+
+One name no stdout can hold at all, a lone surrogate (#280), is refused by the
+validator as a finding, and the two tools, the doctor and `make validate` are
+each run on it.
 """
 
 from __future__ import annotations
@@ -345,6 +349,110 @@
     assert b"pins ok" in stdout, stdout
 
 
+# --- a display name no UTF-8 text can hold (#280) ---------------------------
+#
+# The escape `"a\ud83db"` reads as a string holding the lone surrogate U+D83D,
+# as PyYAML's own Python reader reads it. No UTF-8 stream can hold one, so the
+# validator's `manifest ok` line raised: `make validate` and the doctor met a
+# traceback, while the two callers above, which hand their child
+# `backslashreplace` (#254), passed the project. The validator now reports the
+# name as a finding, and all four launchers stop at it.
+
+#: The display name as the YAML spells it, and the finding the project's own
+#: validator reports for it.
+SURROGATE_ESCAPE = "a\\ud83db"
+SURROGATE_FINDING = (b"FINDING manifest-name: name is 'a\\ud83db', which "
+                     b"holds U+D83D, a lone surrogate")
+
+
+def rename_to_a_lone_surrogate(project: Path) -> str:
+    """Rename the display name to `SURROGATE_ESCAPE` and commit it on its own
+    branch; returns the sha it left HEAD at. Edited as bytes."""
+    manifest = project / "project.yaml"
+    body = manifest.read_bytes()
+    line = f'\nname: "{PROJECT}"'.encode("ascii")
+    assert body.count(line) == 1, body
+    manifest.write_bytes(body.replace(
+        line, f'\nname: "{SURROGATE_ESCAPE}"'.encode("ascii")))
+    git("checkout", "-q", "-b", WORKING_BRANCH, cwd=project)
+    git("-c", "user.name=t", "-c", "user.email=t@t.invalid", "commit", "-q",
+        "-am", "A display name holding a lone surrogate", cwd=project)
+    return git("rev-parse", "HEAD", cwd=project).stdout.strip()
+
+
+def assert_stopped_at_the_finding(code: int, stdout: bytes, stderr: bytes,
+                                  exit_code: int, refusal: bytes = b"") -> None:
+    """The launcher exits `exit_code` with the validator's finding in its
+    output, `refusal` named where it has one, and no traceback anywhere."""
+    both = stdout + stderr
+    shown = both.decode("utf-8", "backslashreplace")
+    assert code == exit_code, shown
+    assert SURROGATE_FINDING in both, shown
+    assert refusal in both, shown
+    assert b"Traceback" not in both and b"UnicodeEncodeError" not in both, \
+        shown
+    assert b"manifest ok" not in both, shown
+
+
+def test_a_real_bump_of_a_project_whose_name_holds_a_surrogate_is_refused(
+        project, scaffolded, advanced, tmp_path):
+    """On c7c02fa this bump exited 0: its child printed `manifest ok:
+    a\\ud83db`, and the commit it made read `Bump spec leg to <sha> in a?b`,
+    the name lost to the `replace` its message is written with. It is the
+    validator's finding now, and every byte is rolled back."""
+    before = rename_to_a_lone_surrogate(project)
+    pinned = git("rev-parse", "HEAD:spec", cwd=project).stdout.strip()
+    code, stdout, stderr = run_in(
+        BUMP, *bump_arguments(project, scaffolded, advanced),
+        encoding="utf-8:strict", tmp_path=tmp_path)
+    assert_stopped_at_the_finding(code, stdout, stderr, 2,
+                                  b"bump-leg-validators-red")
+    assert git("rev-parse", "HEAD", cwd=project).stdout.strip() == before
+    assert git("rev-parse", "HEAD:spec", cwd=project).stdout.strip() == pinned
+    assert git("status", "--porcelain", cwd=project).stdout == ""
+
+
+def test_an_apply_on_a_project_whose_name_holds_a_surrogate_is_refused(
+        upstream_and_project, shape_root, tmp_path):
+    """On c7c02fa `apply` exited 0 and printed `manifest ok: a\\ud83db`. The
+    validator it runs is the one B wrote, which carries the finding."""
+    before = rename_to_a_lone_surrogate(shape_root)
+    copy = (shape_root / CHANGED).read_bytes()
+    code, stdout, stderr = run_in(
+        UPDATE, *update_arguments(shape_root, upstream_and_project),
+        encoding="utf-8:strict", tmp_path=tmp_path)
+    assert_stopped_at_the_finding(code, stdout, stderr, 2,
+                                  b"update-validators-red")
+    assert git("rev-parse", "HEAD", cwd=shape_root).stdout.strip() == before
+    assert (shape_root / CHANGED).read_bytes() == copy, "rolled back"
+    assert git("status", "--porcelain", cwd=shape_root).stdout == ""
+
+
+def test_the_doctor_quotes_the_finding_for_a_name_holding_a_surrogate(
+        project, tmp_path):
+    """On c7c02fa the `manifest` row quoted the traceback's last line,
+    `UnicodeEncodeError: 'utf-8' codec can't encode character '\\ud83d'`."""
+    rename_to_a_lone_surrogate(project)
+    code, stdout, stderr = run_in(
+        REPO / "shape-doctor.py", "--root", str(project),
+        encoding="utf-8:strict", tmp_path=tmp_path)
+    assert_stopped_at_the_finding(code, stdout, stderr, 1)
+    row = [line for line in stdout.splitlines() if SURROGATE_FINDING in line]
+    assert len(row) == 1 and row[0].split()[:2] == [b"FINDING", b"manifest"], \
+        stdout.decode("utf-8", "backslashreplace")
+    assert b"INVALID (manifest)" in stdout
+
+
+@pytest.mark.skipif(shutil.which("make") is None, reason="needs `make` on PATH")
+def test_make_validate_stops_at_the_finding_for_a_surrogate(project, tmp_path):
+    """On c7c02fa `make validate` stopped on the validator's traceback."""
+    rename_to_a_lone_surrogate(project)
+    proc = subprocess.run(["make", "validate"], cwd=str(project),
+                          capture_output=True, stdin=subprocess.DEVNULL,
+                          env=child_environ(tmp_path, None), check=False)
+    assert_stopped_at_the_finding(proc.returncode, proc.stdout, proc.stderr, 2)
+
+
 # --- the environment each tool builds ---------------------------------------
 
 #: What a child prints: `manifest ok: 日本`, spelled with `chr` so the program