Both roots' scripts/bootstrap.py: main escapes what stdout cannot encode and spells each value it prints, so name: "a\ud83db" or 日本 under PYTHONIOENCODING=ascii no longer ends make bootstrap in a traceback (#391)

scripts/repo_shape.py

@@ -27,6 +27,9 @@
   - `Refusal` — the fail-closed exception every validator raises, carrying a
     remediation string, because a refusal that names what is wrong without
     naming what to run puts the exit in tribal memory instead of the message.
+  - `spelled` and `escape_unencodable_stdout` — how both roots'
+    `bootstrap.py` print a person's text: what a terminal must not be handed
+    raw is escaped, and a stdout that cannot encode it never raises (#391).
 
 This file is COPIED into a scaffolded assembly root by `scaffold-project.py`
 and its sha256 is recorded in that project's `contracts/shape-pin.yaml`, so an
@@ -41,6 +44,7 @@
 import re
 import subprocess
 import sys
+import unicodedata
 from pathlib import Path
 from typing import Any
 
@@ -2514,3 +2518,99 @@ def find_repo_root(start: Path) -> Path:
 def die(exc: Refusal, stream=sys.stderr) -> int:
     print(str(exc), file=stream)
     return 2
+
+
+# ---------------------------------------------------------------------------
+# Printing a person's text (#391)
+# ---------------------------------------------------------------------------
+#
+# BOTH ROOTS' `bootstrap.py` print what a manifest, a register row or the
+# disk says, before any validator has had its say about it. Each imports
+# these two from here, as it imports the YAML reader: this file is the one
+# library a project and a family holder carry beside it.
+
+#: The characters `spelled` writes as an escape, besides the bidirectional
+#: controls below: the controls (Cc), the surrogates (Cs), and the line and
+#: paragraph separators (Zl, Zp). Nothing else is escaped, `spelled` says why.
+ESCAPED_CATEGORIES = frozenset(("Cc", "Cs", "Zl", "Zp"))
+#: The bidirectional controls: U+061C, U+200E and U+200F, the embeddings and
+#: overrides U+202A to U+202E, and the isolates U+2066 to U+2069.
+ESCAPED_BIDI_CONTROLS = frozenset(
+    "\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
+
+
+def escape_unencodable_stdout() -> None:
+    """A character stdout cannot encode is printed escaped, never raised on (#391).
+
+    WHAT WAS WRONG. Each root's `bootstrap.py` `main` printed the manifest's
+    `name` with a plain `print` before any step ran, so `name: "a\\ud83db"` (a
+    lone surrogate, which no UTF-8 stdout can write) or `name: "日本"` on a
+    stdout that is not UTF-8 (`PYTHONIOENCODING=ascii`, or the ANSI code page
+    Windows gives a stdout that is not the console) ended `make bootstrap` in a
+    `UnicodeEncodeError` traceback one step before `validate-manifest.py`, the
+    validator that names the problem, was reached. Their own wording raised on
+    such a stdout too, wherever it carries a dash or a middle dot.
+
+    A `strict` or `surrogateescape` stdout, the two handlers that raise, is
+    reconfigured ONCE, at the start of each root's `main`, to
+    `backslashreplace`, the handler stderr already has: what the stdout holds
+    is written as it was, and what it cannot hold as `\\u65e5`. Any other
+    handler, one somebody chose, is left alone, and the ENCODING is never
+    changed. `scripts/validate-pins.py` does the same for its report (#295).
+    The tools run from a checkout of the standard use
+    `shape_materialize.escape_what_stdout_cannot_encode` (#214), which no
+    project carries. This module is what every project and every family holder
+    carries beside its `bootstrap.py`, so both roots import this function and
+    `spelled` from here, where each is written once.
+
+    GUARDED, because failing to set a handler is no reason to refuse the run: a
+    stream a caller replaced may have no `reconfigure`, a closed one raises
+    `ValueError`, and `pythonw` has no stdout at all.
+    """
+    if getattr(sys.stdout, "errors", None) in ("strict", "surrogateescape"):
+        try:
+            sys.stdout.reconfigure(errors="backslashreplace")
+        except (AttributeError, ValueError, OSError):
+            pass
+
+
+def spelled(value: object) -> str:
+    """`value` as ONE line of text, with what a terminal must not be handed
+    written as escapes (#391).
+
+    What `escape_unencodable_stdout` cannot do. A raw ESC or CR is encodable on
+    every stdout, so it reaches the terminal as the control it is, and a
+    manifest's `name` or a register row is a person's text. EXACTLY these
+    characters are written as the escape Python spells it (`\\x1b`, `\\r`,
+    `\\u2028`, `\\ud83d`, `\\u202e`), and no others:
+
+    - the controls (Unicode category Cc: C0, DEL and C1), which move a cursor,
+      clear a screen or end a line;
+    - the surrogates (Cs), which no UTF-8 text holds, so a lone one is named
+      whatever handler the stdout has;
+    - the line and paragraph separators (Zl and Zp, U+2028 and U+2029), which
+      end a line in some viewers, so the value would not stay on its line;
+    - the bidirectional controls (U+061C, U+200E, U+200F, U+202A to U+202E and
+      U+2066 to U+2069), which reorder the text around them so that a name
+      reads as another name.
+
+    EVERY OTHER CHARACTER PRINTS AS IT IS, as it did before this function
+    existed: the zero-width non-joiner a Persian or Indic name needs, the
+    zero-width joiner that makes an emoji sequence, the ideographic space of a
+    Japanese full name, the no-break space, the soft hyphen, and any other
+    format, space or unassigned character. `str.isprintable` is not the test:
+    it refuses all of those, which `validate-manifest.py` accepts and a UTF-8
+    terminal shows correctly, and what it counts as unassigned moves with the
+    interpreter's Unicode database. A backslash prints as it is too, so a name
+    holding the four characters `\\x1b` prints as one holding ESC does.
+
+    Each root's `bootstrap.py` applies it to every value it prints that it
+    read from the manifest, a register row or the disk, and to none of its
+    own wording.
+    """
+    return "".join(
+        char.encode("unicode_escape").decode("ascii")
+        if (char in ESCAPED_BIDI_CONTROLS
+            or unicodedata.category(char) in ESCAPED_CATEGORIES)
+        else char
+        for char in str(value))

templates/assembly-root/scripts/bootstrap.py

@@ -55,7 +55,8 @@
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from repo_shape import (  # noqa: E402
-    Refusal, find_repo_root, git_out, load_yaml, recorded_gitlink,
+    Refusal, escape_unencodable_stdout, find_repo_root, git_out, load_yaml,
+    recorded_gitlink, spelled,
 )
 
 DEGRADE_LINE = "authority is not wallet-carried in this org"
@@ -104,12 +105,15 @@ def _short(sha: str | None) -> str:
 def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None:
     """Step (a) for one leg. Prints; never raises; never moves the pin."""
     path = str(leg.get("path") or "")
-    role = leg.get("role")
     sub = root / path
+    # What is PRINTED of the leg is spelled (#391): `role`, `path` and `branch`
+    # are the manifest's text, read before any validator has had its say.
+    where = f"[{spelled(leg.get('role'))}] {spelled(path)}"
+    named = spelled(branch)
     if not (sub / ".git").exists():
-        print(f"  [{role}] {path}: NOT INITIALIZED. Run "
-              f"`git submodule update --init {path}` — the validators below "
-              f"will refuse until you do.")
+        print(f"  {where}: NOT INITIALIZED. Run "
+              f"`git submodule update --init {spelled(path)}` — the "
+              f"validators below will refuse until you do.")
         return
     pin = recorded_gitlink(root, path)
     local = _try_git(["rev-parse", "--verify", "--quiet",
@@ -119,30 +123,30 @@ def checkout_tracking_branch(root: Path, leg: dict, branch: str) -> None:
 
     if local is None:
         if pin is None:
-            print(f"  [{role}] {path}: no gitlink recorded; leaving as is")
+            print(f"  {where}: no gitlink recorded; leaving as is")
             return
         if _try_git(["checkout", "-q", "-b", branch, pin], sub) is None:
-            print(f"  [{role}] {path}: could not create branch {branch!r} at "
+            print(f"  {where}: could not create branch {branch!r} at "
                   f"the pin {_short(pin)}; leaving detached")
             return
         if remote is not None:
             _try_git(["branch", "-q", f"--set-upstream-to=origin/{branch}",
                       branch], sub)
         state = "at the pin"
         if remote is not None and remote != pin:
-            state = (f"at the pin; origin/{branch} is {_short(remote)}, which "
+            state = (f"at the pin; origin/{named} is {_short(remote)}, which "
                      f"the pin deliberately does NOT follow")
-        print(f"  [{role}] {path}: branch {branch} created at "
+        print(f"  {where}: branch {named} created at "
               f"{_short(pin)} — {state}")
         return
 
     if local == pin:
         _try_git(["checkout", "-q", branch], sub)
-        print(f"  [{role}] {path}: on {branch} at {_short(pin)} "
+        print(f"  {where}: on {named} at {_short(pin)} "
               f"(branch tip == pin)")
         return
 
-    print(f"  [{role}] {path}: PIN {_short(pin)} != branch {branch} tip "
+    print(f"  {where}: PIN {_short(pin)} != branch {named} tip "
           f"{_short(local)}")
     print("          not moving an existing branch. The checkout stays at "
           "the pin; the recorded pin is authoritative and advancing it is an "
@@ -236,7 +240,8 @@ def _load_register_rows(register_path: Path) -> list[dict] | None:
     try:
         data = load_yaml(register_path)
     except Refusal as exc:
-        print(f"    unreadable, so nothing is claimed from it: {exc.detail}")
+        print("    unreadable, so nothing is claimed from it: "
+              f"{spelled(exc.detail)}")
         return None
     rows = []
     if isinstance(data, dict):
@@ -258,11 +263,12 @@ def _row_report_line(row: dict, repositories: set[str],
               if _matches(o, repositories, prefixes)]
     if not objects:
         return None
-    holder = row.get("holder_ref") or row.get("holder") or "?"
-    act = row.get("act") or row.get("acts") or "?"
-    state = row.get("state") or "?"
-    expires = row.get("expires_at") or "-"
-    return (f"    {holder} · {act} · {', '.join(objects)} · "
+    # A register row is a file's text (#391): each value is spelled.
+    holder = spelled(row.get("holder_ref") or row.get("holder") or "?")
+    act = spelled(row.get("act") or row.get("acts") or "?")
+    state = spelled(row.get("state") or "?")
+    expires = spelled(row.get("expires_at") or "-")
+    return (f"    {holder} · {act} · {', '.join(map(spelled, objects))} · "
            f"{state} · expires {expires}")
 
 
@@ -277,7 +283,8 @@ def _report_one_register(register_path: Path, root: Path,
     """
     # POSIX, on every platform: this line names a file in the repository,
     # and git and the reader both spell it that way.
-    print(f"  register: {register_path.relative_to(root).as_posix()}")
+    print("  register: "
+          f"{spelled(register_path.relative_to(root).as_posix())}")
     rows = _load_register_rows(register_path)
     if rows is None:
         return
@@ -378,6 +385,7 @@ def _run_neutral_validators(root: Path, manifest: dict | None,
 
 
 def main(argv: list[str] | None = None) -> int:
+    escape_unencodable_stdout()
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     parser.add_argument("--root", type=Path, default=None)
     parser.add_argument("--branch", default=None,
@@ -397,7 +405,7 @@ def main(argv: list[str] | None = None) -> int:
     sub_legs = [leg for leg in legs if leg.get("role") != "assembly"]
 
     name = (manifest or {}).get("name") or root.name
-    print(f"bootstrap: {name} ({root})")
+    print(f"bootstrap: {spelled(name)} ({spelled(root)})")
 
     print("\n(a) legs on tracking branches, pins untouched")
     _checkout_all_tracking_branches(root, sub_legs, manifest, args.branch)

templates/family-root/scripts/bootstrap.py

@@ -53,7 +53,9 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
-from repo_shape import Refusal, find_repo_root, load_yaml  # noqa: E402
+from repo_shape import (  # noqa: E402
+    Refusal, escape_unencodable_stdout, find_repo_root, load_yaml, spelled,
+)
 
 MANIFEST = "family.yaml"
 DEFAULT_MEMBERS_DIR = "members"
@@ -133,11 +135,14 @@ def member_path(root: Path, row: dict) -> Path:
 
 def run_member_target(root: Path, row: dict, target: str) -> str | None:
     """(b) One member's own `make <target>`. Returns a failure line, or None."""
-    project = str(row.get("project") or "?")
+    # A member row is the family manifest's text (#391): what is PRINTED of it,
+    # and of `--members-make`, is spelled. The path is built from the raw row.
+    project = spelled(row.get("project") or "?")
     path = member_path(root, row)
+    named = spelled(target)
     if not (path / ".git").exists():
-        print(f"  [{project}] {path.name}: NOT CHECKED OUT; skipped. See the "
-              "fetch above.")
+        print(f"  [{project}] {spelled(path.name)}: NOT CHECKED OUT; skipped. "
+              "See the fetch above.")
         return None
     if not (path / "Makefile").is_file():
         print(f"  [{project}] no Makefile; skipped. A member without one is "
@@ -148,16 +153,17 @@ def run_member_target(root: Path, row: dict, target: str) -> str | None:
     if not make:
         print(f"  [{project}] `make` is not on PATH; skipped.")
         return None
-    print(f"  --- {project}: make {target} ---")
+    print(f"  --- {project}: make {named} ---")
     sys.stdout.flush()
     proc = subprocess.run([make, target], cwd=str(path), check=False)
     sys.stdout.flush()
     if proc.returncode != 0:
-        return f"{project} (make {target} exited {proc.returncode})"
+        return f"{project} (make {named} exited {proc.returncode})"
     return None
 
 
 def main(argv: list[str] | None = None) -> int:
+    escape_unencodable_stdout()
     parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
     parser.add_argument("--root", type=Path, default=None)
     parser.add_argument("--no-fetch", action="store_true",
@@ -177,15 +183,16 @@ def main(argv: list[str] | None = None) -> int:
         print(str(exc), file=sys.stderr)
         return 2
 
-    print(f"bootstrap: {name} ({root}) — {len(rows)} member(s)")
+    print(f"bootstrap: {spelled(name)} ({spelled(root)}) — "
+          f"{len(rows)} member(s)")
 
     print("\n(a) members and their legs")
     if args.no_fetch:
         print("  skipped (--no-fetch)")
     else:
         fetch_members(root)
 
-    print(f"\n(b) each member's own `make {args.members_make}`")
+    print(f"\n(b) each member's own `make {spelled(args.members_make)}`")
     failed: list[str] = []
     if args.skip_members:
         print("  skipped (--skip-members)")