adopt-project.py: plan's table pads each name by the columns a terminal gives it, not the code points it holds, so a wide or combining name's leg column lines up under leg with every other row's (#1034)

adopt-project.py

@@ -228,6 +228,7 @@
 import subprocess
 import sys
 import tempfile
+import unicodedata
 from pathlib import Path, PurePosixPath
 from typing import Callable, NamedTuple
 
@@ -6383,10 +6384,16 @@ def _print_entries(entries: list[Entry]) -> None:
     stood 3 characters right of its heading. The rule id is a path
     policy's, which `--path-policy` may name and nothing asks to be
     printable, so it is `_shown` as a question is.
+
+    AND IT IS PADDED BY THE COLUMNS IT TAKES, not the code points it holds
+    (#1034, `_padded`). `\\u540d\\u524d.md`, two wide characters, takes 7
+    columns and was padded as 5, so its row stood 2 columns right of its
+    headings; `e\\u0301.md`, whose acute is drawn over the `e`, takes 4 and
+    was padded as 5, 1 column left.
     """
     print(f"\n{'path':<34} {'leg':<10} {'conf':<7} {'files':>6} {'rule'}")
     for entry in entries:
-        print(f"{_printed(_shown(entry.path)):<34} "
+        print(f"{_padded(_printed(_shown(entry.path)), 34)} "
               f"{(entry.leg or 'REVIEW'):<10} "
               f"{entry.confidence:<7} {entry.files:>6} "
               f"{_shown(str(entry.rule))}")
@@ -6408,6 +6415,55 @@ def _printed(text: str) -> str:
         return text
 
 
+#: The general categories a terminal gives no column of their own (#1034):
+#: a non-spacing mark and an enclosing one, which it draws over the
+#: character before them, and a format character, which it does not draw.
+#: `_shown` spells every format character, so none reaches `plan`'s table
+#: as it is.
+ZERO_COLUMN_CATEGORIES = frozenset(("Mn", "Me", "Cf"))
+#: The East Asian widths a terminal gives two columns: wide and fullwidth.
+TWO_COLUMN_WIDTHS = frozenset(("W", "F"))
+#: The Hangul medial vowels and final consonants, which a terminal gives no
+#: column of their own, as glibc's `wcwidth` gives them none (#1034): U+1160
+#: to U+11FF in the block Hangul Jamo, and U+D7B0 to U+D7FF in Hangul Jamo
+#: Extended-B. Their category is Lo and their East Asian width N, so neither
+#: set above names them, but a terminal draws each inside the syllable its
+#: leading consonant begins: U+1112 U+1161 U+11AB, one syllable spelled in
+#: its three jamo, takes the 2 columns of U+1112.
+MEDIAL_AND_FINAL_JAMO = frozenset(map(chr, range(0x1160, 0x1200)))
+MEDIAL_AND_FINAL_JAMO_EXTENDED_B = frozenset(map(chr, range(0xD7B0, 0xD800)))
+
+
+def _columns(text: str) -> int:
+    """How many columns `text` takes on a terminal (#1034): none for a
+    character of `ZERO_COLUMN_CATEGORIES` or a Hangul medial vowel or final
+    consonant (`MEDIAL_AND_FINAL_JAMO`, `MEDIAL_AND_FINAL_JAMO_EXTENDED_B`),
+    two for one `unicodedata.east_asian_width` calls wide or fullwidth, and
+    one for any other, an ambiguous one among them, as a terminal outside
+    an East Asian locale draws it.
+
+    THE CATEGORY IS ASKED FIRST: the voiced sound mark U+3099, which a
+    decomposed `\\u304c` ends in, is wide, and is drawn over the kana
+    before it. AND IT IS THE CATEGORY, not `unicodedata.combining`, whose
+    canonical combining class is 0 for marks a terminal draws over the
+    character before them all the same: the Thai vowel sign U+0E35 and
+    the variation selector U+FE0F among them."""
+    return sum(
+        0 if (unicodedata.category(char) in ZERO_COLUMN_CATEGORIES
+              or char in MEDIAL_AND_FINAL_JAMO
+              or char in MEDIAL_AND_FINAL_JAMO_EXTENDED_B)
+        else 2 if unicodedata.east_asian_width(char) in TWO_COLUMN_WIDTHS
+        else 1
+        for char in text)
+
+
+def _padded(text: str, columns: int) -> str:
+    """`text` and the spaces that make it `columns` columns wide on a
+    terminal (`_columns`), as `f"{text:<{columns}}"` makes it that many
+    code points. A text wider than that is `text` as it is, as there."""
+    return text + " " * (columns - _columns(text))
+
+
 def _print_summary(entries: list[Entry]) -> None:
     print()
     for leg, row in summary_of(entries).items():

tests/test_adopt_spelled_names.py

@@ -28,6 +28,14 @@
   right of their headings.
 
 Every row here fails on c7a3ff16's `adopt-project.py`.
+
+And measured on 961640c6 (#1034): `_print_entries` padded each name by the
+code points it holds, not the columns a terminal gives it. On a UTF-8
+stdout `\\u540d\\u524d.md` took 7 columns and was padded as 5, so its
+row's leg stood 2 columns right of `leg`, and `e\\u0301.md` took 4 and
+stood 1 column left. `\\u1112\\u1161\\u11ab.md`, one syllable spelled in
+its three jamo, took 5 and was padded as 6: 1 column left too. The rows
+after `_columns`'s fail on 961640c6's.
 """
 
 from __future__ import annotations
@@ -526,3 +534,122 @@ def test_plan_under_the_c_locale_keeps_each_column_under_its_heading(
     heading, rows = table_of(written.stdout.decode("ascii", "replace"))
     assert "caf\\xe9.md" in [row.split()[0] for row in rows], rows
     assert_each_column_under_its_heading(heading, rows)
+
+
+# --- plan's table: a name padded by the columns it takes (#1034) ----------
+
+#: Each name and the columns a terminal gives it, counted by hand: two for a
+#: wide or fullwidth character, none for a non-spacing or enclosing mark, a
+#: format character or a Hangul medial vowel or final consonant, and one for
+#: any other, a halfwidth, an ambiguous one or a spacing mark among them.
+COLUMNS = {
+    "README.md": 9,
+    "\u540d\u524d.md": 7,
+    "\uff21\uff22.md": 7,
+    "\uff76.md": 4,
+    "\u00e9.md": 4,
+    "e\u0301.md": 4,
+    "\u0e17\u0e35.md": 4,
+    "\u0915\u093f.md": 5,
+    "\u304b\u3099.md": 5,
+    "\u1112\u1161\u11ab.md": 5,
+    "\u1100\ud7b0\ud7cb.md": 5,
+    "a\u20ddb": 2,
+    "a\u200bb": 2,
+    "caf\\xe9.md": 10,
+}
+
+
+@pytest.mark.parametrize("name", list(COLUMNS),
+                         ids=["ascii", "wide", "fullwidth", "halfwidth",
+                              "ambiguous", "combining-acute",
+                              "thai-mark-of-class-0", "spacing-mark",
+                              "wide-voiced-mark", "conjoining-jamo",
+                              "conjoining-jamo-extended-b",
+                              "enclosing-mark", "zero-width-space",
+                              "as-an-ascii-stdout-escapes-it"])
+def test_a_name_takes_the_columns_a_terminal_gives_it(adopter, name):
+    """`_columns`, which `plan`'s table pads a name by. The Thai vowel sign
+    U+0E35 is a mark whose combining class is 0, and the voiced sound mark
+    U+3099 a mark East Asian width calls wide: each takes no column. The
+    Devanagari vowel sign U+093F is a spacing mark (Mc), which takes one.
+    U+1161 and U+11AB, the vowel and final consonant of a syllable spelled
+    in its jamo, and U+D7B0 and U+D7CB of Hangul Jamo Extended-B, are
+    letters (Lo) East Asian width calls neutral, which a terminal draws
+    inside the syllable: each takes no column, as glibc's `wcwidth` gives
+    it."""
+    assert adopter._columns(name) == COLUMNS[name]
+
+
+def test_a_name_wider_than_its_column_is_padded_by_no_space(adopter):
+    """`_padded` adds no space to a name wider than the columns it pads
+    to, as `:<34` adds none to a name of more than 34 code points: ten of
+    `\\u540d\\u524d` and `.md` take 43 columns, past the 34 of `plan`'s
+    table. A pad of at least one space, or of the 9 columns it runs over,
+    would push that row's leg further right."""
+    name = "\u540d\u524d" * 10 + ".md"
+    assert adopter._columns(name) == 43
+    assert adopter._padded(name, 34) == name
+
+
+def leg_columns(rows: list[str]) -> list[int]:
+    """The column each row's leg begins at on a terminal: the columns its
+    name takes, as `COLUMNS` counts them or one to each character of an
+    ASCII name, then the spaces after it. No name here holds a space."""
+    columns = []
+    for row in rows:
+        name, _, rest = row.partition(" ")
+        if name in COLUMNS:
+            width = COLUMNS[name]
+        else:
+            assert name.isascii(), (name, rows)
+            width = len(name)
+        columns.append(width + 1 + len(rest) - len(rest.lstrip(" ")))
+    return columns
+
+
+def test_the_table_pads_a_wide_and_a_combining_name_by_their_columns(
+        adopter, monkeypatch):
+    """`_print_entries` on a UTF-8 stdout, which prints each name as it is:
+    every row's leg begins at column 35, under `leg`. On 961640c6 the two
+    wide names' legs stood 2 columns right of it and `e\\u0301.md`'s 1
+    left."""
+    names = ["README.md", "\u540d\u524d.md", "\uff21\uff22.md",
+             "e\u0301.md", "\u304b\u3099.md"]
+    stream = io.TextIOWrapper(io.BytesIO(), encoding=UTF8, newline="\n")
+    monkeypatch.setattr(sys, "stdout", stream)
+    adopter._print_entries([
+        adopter.Entry(name, adopter.Verdict("spec", "extension-majority",
+                                            "Why.", "high"), 1, 1)
+        for name in names])
+    stream.flush()
+    heading, rows = table_of(stream.buffer.getvalue().decode(UTF8) + "\n")
+    assert heading.index("leg") == 35, heading
+    assert [row.split()[0] for row in rows] == names
+    assert leg_columns(rows) == [35] * len(names), rows
+
+
+def test_plan_on_a_utf_8_stdout_keeps_each_column_under_its_heading(
+        tmp_path):
+    """THE ISSUE'S TABLE: `plan` with a UTF-8 stdout over a source holding
+    `\\u540d\\u524d.md`, `\\uff21\\uff22.md`, `e\\u0301.md` and
+    `\\u1112\\u1161\\u11ab.md`, one syllable spelled in its three jamo. On
+    961640c6 the first two rows' legs stood 2 columns right of `leg`, and
+    the last two 1 left."""
+    env = blank_unnamed_pin_sources(
+        {**os.environ, "LANES_LANE": "", "PYTHONIOENCODING": UTF8}, {})
+    source = make_source_repo(tmp_path / "Thing",
+                              {"README.md": "# Thing\n",
+                               "\u540d\u524d.md": "x\n",
+                               "\uff21\uff22.md": "x\n",
+                               "e\u0301.md": "x\n",
+                               "\u1112\u1161\u11ab.md": "x\n",
+                               "src/app.py": "VALUE = 1\n"}, edits=())
+    written = planned(source, tmp_path / "adoption-plan.yaml", env=env)
+    assert written.returncode == 0, written.stderr + written.stdout
+    heading, rows = table_of(written.stdout.decode(UTF8))
+    assert heading.index("leg") == 35, heading
+    assert {"\u540d\u524d.md", "\uff21\uff22.md", "e\u0301.md",
+            "\u1112\u1161\u11ab.md"} <= {
+        row.split()[0] for row in rows}, rows
+    assert leg_columns(rows) == [35] * len(rows), rows