From 9d34b8f4d854042bbf1ce9a93b310c0e72321f68 Mon Sep 17 00:00:00 2001 From: Peng Ding Date: Thu, 22 Oct 2026 17:13:27 -0500 Subject: [PATCH 1/3] test(tabulate): compare against the reference's len() width path The reference uses wcwidth for column widths when it is installed; our default is len(). The guarded import only hid the repo's own tabulate/ from the reference, so with wcwidth as a transitive dependency four unicode/CJK comparisons measured two different width rules and failed (#228). Hide wcwidth for the reference import and assert it stayed hidden. --- tabulate/test_tabulate_correctness.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tabulate/test_tabulate_correctness.py b/tabulate/test_tabulate_correctness.py index bfcd9cbb..103da65c 100644 --- a/tabulate/test_tabulate_correctness.py +++ b/tabulate/test_tabulate_correctness.py @@ -22,6 +22,16 @@ _cached_tabulate = sys.modules.pop("tabulate", None) +# The reference switches to ``wcwidth`` for column widths when it is +# installed, while this module's default is ``len(s)`` (``WIDE_CHARS_MODE`` +# is opt-in). Hide ``wcwidth`` from the reference import — ``None`` in +# ``sys.modules`` makes the import fail cleanly — so both sides compute +# widths the same way regardless of the environment. Without this the +# unicode/CJK cases differ as soon as ``wcwidth`` is a transitive dependency. +_cached_wcwidth = sys.modules.get("wcwidth") +_cached_wcwidth_present = "wcwidth" in sys.modules +sys.modules["wcwidth"] = None # type: ignore[assignment] + try: import tabulate as _ref_tabulate_mod @@ -35,10 +45,26 @@ sys.modules.pop("tabulate", None) if _cached_tabulate is not None: sys.modules["tabulate"] = _cached_tabulate + if _cached_wcwidth_present: + sys.modules["wcwidth"] = _cached_wcwidth # type: ignore[assignment] + else: + sys.modules.pop("wcwidth", None) sys.path.insert(0, _this_dir) from tabulate import tabulate as our_tabulate # noqa: E402 + +def test_reference_width_mode_is_len() -> None: + """The comparisons below assume the reference counts ``len(s)``. + + Our default is ``len(s)``; the reference switches to ``wcwidth`` when + that package is importable. If this fails, the guard above stopped + working and the unicode/CJK cases compare two different width rules + rather than two implementations. + """ + assert getattr(_ref_tabulate_mod, "wcwidth", None) is None + + # ── Helpers ── From 19f124727b33f0f732d438bde5825f713b3eb5e6 Mon Sep 17 00:00:00 2001 From: Peng Ding Date: Thu, 22 Oct 2026 17:20:48 -0500 Subject: [PATCH 2/3] fix(tabulate): combining marks are zero-width in WIDE_CHARS_MODE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review note on #229: hiding wcwidth made the default comparison environment-independent but dropped the only coverage of the wide-character width path. Adding it back exposed a real gap — WIDE_CHARS_MODE counted a combining accent as one column, so 'café' measured five where wcwidth reports four. Combining marks now contribute zero width, and a parametrised test compares WIDE_CHARS_MODE=True against the reference imported with wcwidth visible. --- tabulate/tabulate.py | 7 ++- tabulate/test_tabulate_correctness.py | 61 +++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tabulate/tabulate.py b/tabulate/tabulate.py index 95271a37..45c17f5e 100644 --- a/tabulate/tabulate.py +++ b/tabulate/tabulate.py @@ -52,12 +52,17 @@ def _visible_width(s: str) -> int: This function respects the ``WIDE_CHARS_MODE`` flag. When the flag is ``False`` (the default, matching the reference *tabulate* library), - width equals ``len(s)``. + width equals ``len(s)``. When it is ``True``, East Asian wide and + fullwidth characters count as two columns and combining marks as zero, + which is what ``wcwidth`` reports. """ if not WIDE_CHARS_MODE: return len(s) w = 0 for ch in s: + if unicodedata.combining(ch): + # Zero-width, as wcwidth reports: "café" is four columns. + continue eaw = unicodedata.east_asian_width(ch) if eaw in ("W", "F"): w += 2 diff --git a/tabulate/test_tabulate_correctness.py b/tabulate/test_tabulate_correctness.py index 103da65c..ea4763d1 100644 --- a/tabulate/test_tabulate_correctness.py +++ b/tabulate/test_tabulate_correctness.py @@ -51,6 +51,7 @@ sys.modules.pop("wcwidth", None) sys.path.insert(0, _this_dir) +import tabulate as _our_module # noqa: E402 from tabulate import tabulate as our_tabulate # noqa: E402 @@ -65,6 +66,66 @@ def test_reference_width_mode_is_len() -> None: assert getattr(_ref_tabulate_mod, "wcwidth", None) is None +def _reference_using_wcwidth(): + """Import the reference again with ``wcwidth`` visible. + + The module-level import hides ``wcwidth`` so that both sides use + ``len(s)``; this reproduces the configuration a user of the reference + has, where it detects wide characters automatically. Skips when + ``wcwidth`` is not installed. + """ + pytest.importorskip("wcwidth") + saved_path = sys.path[:] + saved_tabulate = sys.modules.pop("tabulate", None) + try: + sys.path = [ + p + for p in sys.path + if os.path.abspath(p or os.curdir) + not in ( + os.path.abspath(_this_dir), + os.path.abspath(os.path.join(_this_dir, "..")), + ) + ] + import tabulate as ref + + return ref + finally: + sys.path = saved_path + sys.modules.pop("tabulate", None) + if saved_tabulate is not None: + sys.modules["tabulate"] = saved_tabulate + + +@pytest.mark.parametrize( + ("data", "kwargs"), + [ + ([["café", 1], ["élève", 2]], {}), + ([["你好", 1], ["世界", 2]], {"headers": ["Text", "Num"]}), + ([["东京Tokyo", 1], ["NYC", 2]], {"headers": ["City", "ID"]}), + ([["\U0001f600", "smile"]], {}), + ], +) +def test_wide_chars_mode_matches_the_wcwidth_reference(data, kwargs) -> None: + """``WIDE_CHARS_MODE`` reproduces what the reference does with wcwidth. + + Hiding ``wcwidth`` above makes the default-mode comparison + environment-independent, but it also means nothing else exercises the + wide-character width path. This pins it against the configuration the + reference has when the package is installed. + """ + ref = _reference_using_wcwidth() + if getattr(ref, "wcwidth", None) is None: + pytest.skip("the reference did not import wcwidth") + + previous = _our_module.WIDE_CHARS_MODE + _our_module.WIDE_CHARS_MODE = True + try: + assert _our_module.tabulate(data, **kwargs) == ref.tabulate(data, **kwargs) + finally: + _our_module.WIDE_CHARS_MODE = previous + + # ── Helpers ── From 644ba8527cc7c708542c623b7ba3f708d29dcbfd Mon Sep 17 00:00:00 2001 From: Peng Ding Date: Thu, 22 Oct 2026 17:48:56 -0500 Subject: [PATCH 3/3] fix(tabulate): widen the zero-width rule and narrow the claim Milo's review: unicodedata.combining() is zero for format characters, enclosing marks and Hangul jamo, so eight shapes still diverged from wcwidth while the docstring claimed parity. Zero-width now covers Mn/Me/Cf plus the Hangul medial/final jamo ranges, which matches wcwidth on every per-character shape measured; emoji presentation sequences (ZWJ clusters, keycaps) stay per-component and are documented as the one divergence. Tests pin both halves. --- tabulate/tabulate.py | 34 ++++++++++++++++--- tabulate/test_tabulate_correctness.py | 48 +++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/tabulate/tabulate.py b/tabulate/tabulate.py index 45c17f5e..8f85e419 100644 --- a/tabulate/tabulate.py +++ b/tabulate/tabulate.py @@ -47,21 +47,45 @@ # ── CJK / visible-width helpers ────────────────────────────────────────────── +# Categories that occupy no column of their own: combining marks (Mn), +# enclosing marks (Me) — the keycap U+20E3 — and format characters (Cf), +# which include the zero-width space and joiner (U+200B, U+200D) and the +# combining grapheme joiner (U+034F). +_ZERO_WIDTH_CATEGORIES = frozenset({"Mn", "Me", "Cf"}) + +# Hangul medial (jungseong) and final (jongseong) jamo compose with the +# leading jamo and add no column. +_ZERO_WIDTH_RANGES = ((0x1160, 0x11FF), (0xD7B0, 0xD7FF)) + + +def _is_zero_width(ch: str) -> bool: + """Return True when *ch* occupies no column of its own.""" + if unicodedata.category(ch) in _ZERO_WIDTH_CATEGORIES: + return True + cp = ord(ch) + return any(lo <= cp <= hi for lo, hi in _ZERO_WIDTH_RANGES) + + def _visible_width(s: str) -> int: """Return the visible width of *s*, counting CJK / wide chars as 2. This function respects the ``WIDE_CHARS_MODE`` flag. When the flag is ``False`` (the default, matching the reference *tabulate* library), - width equals ``len(s)``. When it is ``True``, East Asian wide and - fullwidth characters count as two columns and combining marks as zero, - which is what ``wcwidth`` reports. + width equals ``len(s)``. + + When it is ``True``, East Asian wide and fullwidth characters count as + two columns, and zero-width marks (combining, enclosing and format + characters) and Hangul medial/final jamo as zero — the per-character + widths ``wcwidth`` reports. Emoji presentation sequences are counted + per component, which is where the two can differ: a ZWJ family emoji is + six columns here and two under ``wcwidth`` (it measures grapheme + clusters), and a keycap sequence is one here and two there. """ if not WIDE_CHARS_MODE: return len(s) w = 0 for ch in s: - if unicodedata.combining(ch): - # Zero-width, as wcwidth reports: "café" is four columns. + if _is_zero_width(ch): continue eaw = unicodedata.east_asian_width(ch) if eaw in ("W", "F"): diff --git a/tabulate/test_tabulate_correctness.py b/tabulate/test_tabulate_correctness.py index ea4763d1..c09f658b 100644 --- a/tabulate/test_tabulate_correctness.py +++ b/tabulate/test_tabulate_correctness.py @@ -126,6 +126,54 @@ def test_wide_chars_mode_matches_the_wcwidth_reference(data, kwargs) -> None: _our_module.WIDE_CHARS_MODE = previous +@pytest.mark.parametrize( + "text", + [ + "café", # combining acute + "élève", # precomposed + "你好", # CJK, wide + "\U0001f600", # emoji, wide + "a​b", # zero-width space + "a͏b", # combining grapheme joiner, ccc=0 + "क्ष", # Devanagari virama cluster + "각", # Hangul leading + medial + final jamo + ], +) +def test_zero_width_shapes_match_wcwidth(text: str) -> None: + """Per-character widths agree with ``wcwidth.wcswidth``.""" + wcwidth = pytest.importorskip("wcwidth") + previous = _our_module.WIDE_CHARS_MODE + _our_module.WIDE_CHARS_MODE = True + try: + assert _our_module._visible_width(text) == wcwidth.wcswidth(text) + finally: + _our_module.WIDE_CHARS_MODE = previous + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("\U0001f468‍\U0001f469‍\U0001f467", 6), # family, ZWJ + ("1️⃣", 1), # keycap sequence + ], +) +def test_emoji_presentation_sequences_count_per_component( + text: str, expected: int +) -> None: + """Documented divergence: ``wcwidth`` measures grapheme clusters. + + A ZWJ family emoji is 2 columns under ``wcwidth`` and 6 here; a keycap + is 2 there and 1 here. Pinning the per-component count keeps the + docstring honest. + """ + previous = _our_module.WIDE_CHARS_MODE + _our_module.WIDE_CHARS_MODE = True + try: + assert _our_module._visible_width(text) == expected + finally: + _our_module.WIDE_CHARS_MODE = previous + + # ── Helpers ──