From 121ea0238e8b3ff4b9892370d9212235f29a0d48 Mon Sep 17 00:00:00 2001 From: Wolfvin Date: Fri, 3 Jul 2026 09:00:53 +0000 Subject: [PATCH] fix(cli): force UTF-8 stdout/stderr at startup, no PYTHONUTF8 needed (closes #179) Windows without PYTHONUTF8=1 crashes with UnicodeEncodeError when commands emit Unicode chars like '\u2192' in trace paths (e.g. 'sidepanel.ts \u2192 auth/google-auth-cache.ts'). Default Windows stdout encoding (cp1252) cannot represent those chars. Fix: scripts/codelens.py wraps sys.stdout/sys.stderr as UTF-8 TextIOWrapper at import time, before any user-code imports. Behaviour: - Linux/macOS: no-op (encoding already utf-8) - pytest capsys: no-op (encoding is utf-8) - Windows without PYTHONUTF8=1: wrap with errors='replace' so unencodable bytes become '?' instead of crashing - Streams without .buffer (IDLE REPL, custom capture): skip gracefully - Old stream's buffer is detach()-ed before new wrapper is built, so GC of old wrapper does not close the underlying buffer Tests: tests/test_codelens.py::TestForceUtf8Stdio -- 7 regression tests covering all 4 scenarios + end-to-end test for the exact trace pattern from the issue. --- scripts/codelens.py | 64 ++++++++++++++ tests/test_codelens.py | 195 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+) diff --git a/scripts/codelens.py b/scripts/codelens.py index fbdc62ce..3b8256cd 100755 --- a/scripts/codelens.py +++ b/scripts/codelens.py @@ -68,6 +68,70 @@ """ import sys +import io + + +def _force_utf8_stdio() -> None: + """Force UTF-8 encoding on ``sys.stdout`` and ``sys.stderr``. + + Without this, Windows crashes with ``UnicodeEncodeError`` when commands + emit Unicode characters like ``'\u2192'`` (the ``\u2192`` arrow used in + trace paths such as ``sidepanel.ts \u2192 auth/google-auth-cache.ts``). + The default Windows stdout encoding (``cp1252`` or similar) cannot + represent those characters (issue #179). + + Behaviour: + + - On Linux/macOS ``sys.stdout.encoding`` is already ``utf-8`` \u2014 no-op. + - Under pytest's ``capsys`` capture, ``encoding`` is also ``utf-8`` \u2014 no-op. + - On Windows without ``PYTHONUTF8=1``, both streams are re-wrapped as + ``TextIOWrapper(buffer, encoding='utf-8', errors='replace')``. The + ``errors='replace'`` policy means any byte that still cannot be + rendered by the terminal becomes ``?`` instead of raising. + - Streams without a ``.buffer`` attribute (IDLE REPL, some custom + capture objects) are skipped to avoid breaking those environments. + + The old stream's buffer is ``detach()``-ed before the new wrapper is + built, so that garbage-collection of the old ``TextIOWrapper`` does + not close the underlying buffer (which would make the new wrapper + unusable with ``ValueError: I/O operation on closed file``). + + Runs unconditionally at module import time, before any user-code + imports that might emit Unicode output. + """ + for _name in ('stdout', 'stderr'): + _stream = getattr(sys, _name, None) + if _stream is None: + continue + _encoding = (getattr(_stream, 'encoding', None) or '').lower().replace('-', '') + if _encoding == 'utf8': + continue + _buffer = getattr(_stream, 'buffer', None) + if _buffer is None: + continue + # Detach the buffer from the old wrapper so its __del__/close + # does not close the buffer out from under the new wrapper. + # Best-effort: some custom stream classes lack detach() or + # raise on it; in that case we proceed anyway (the new wrapper + # is still created and the old wrapper's close, if any, will + # simply no-op on a NULLed buffer). + _detach = getattr(_stream, 'detach', None) + if callable(_detach): + try: + _detach() + except Exception: + pass + setattr( + sys, + _name, + io.TextIOWrapper(_buffer, encoding='utf-8', errors='replace'), + ) + + +# Run before any other imports that might emit Unicode output to stdout/stderr. +_force_utf8_stdio() + + import os import json import argparse diff --git a/tests/test_codelens.py b/tests/test_codelens.py index e5d65df8..7a631811 100644 --- a/tests/test_codelens.py +++ b/tests/test_codelens.py @@ -24,6 +24,7 @@ _suggest_fix, resolve_workspace, _SEVERITY_ORDER, + _force_utf8_stdio, ) @@ -656,5 +657,199 @@ def test_none_arg_returns_valid_path(self): self.assertTrue(os.path.isdir(result)) +# ─── _force_utf8_stdio Tests (issue #179) ───────────────────── + + +class TestForceUtf8Stdio(unittest.TestCase): + """Regression tests for issue #179. + + On Windows without ``PYTHONUTF8=1``, the default stdout encoding + (``cp1252`` or similar) cannot represent characters like ``\u2192`` + (the ``\u2192`` arrow used in trace paths such as + ``sidepanel.ts \u2192 auth/google-auth-cache.ts``), causing + ``UnicodeEncodeError`` crashes. + + ``_force_utf8_stdio`` wraps ``sys.stdout`` / ``sys.stderr`` as + UTF-8 ``TextIOWrapper`` so this no longer happens. It must be a + no-op on Linux/macOS (encoding already utf-8) and under pytest + capsys (encoding also utf-8), and must not crash on streams + without a ``.buffer`` attribute (IDLE REPL, custom capture). + """ + + _ARROW = "\u2192" # '→', used in trace output paths + _ARROW_UTF8_BYTES = b"\xe2\x86\x92" + + def setUp(self): + self._orig_stdout = sys.stdout + self._orig_stderr = sys.stderr + + def tearDown(self): + sys.stdout = self._orig_stdout + sys.stderr = self._orig_stderr + + def test_function_is_callable(self): + """Sanity: codelens module exposes _force_utf8_stdio as a callable.""" + self.assertTrue(callable(_force_utf8_stdio)) + + def test_wraps_non_utf8_stdout(self): + """cp1252-encoded stdout is rewrapped as UTF-8. + + Reproduces the issue #179 bug scenario: with cp1252 stdout, + writing the arrow character crashes. After _force_utf8_stdio, + it succeeds and the underlying buffer receives UTF-8 bytes. + """ + import io + buf = io.BytesIO() + cp1252_stream = io.TextIOWrapper(buf, encoding="cp1252") + sys.stdout = cp1252_stream + sys.stderr = cp1252_stream # also exercise stderr path + + # Sanity: the bug exists pre-fix + with self.assertRaises(UnicodeEncodeError): + cp1252_stream.write(self._ARROW) + + _force_utf8_stdio() + + # Post-fix: encoding is utf-8 and arrow writes without crashing + encoding = (sys.stdout.encoding or "").lower().replace("-", "") + self.assertEqual(encoding, "utf8") + sys.stdout.write(self._ARROW) + sys.stdout.flush() + self.assertIn(self._ARROW_UTF8_BYTES, buf.getvalue()) + + def test_noop_when_stream_already_utf8(self): + """If sys.stdout is already utf-8, _force_utf8_stdio is a no-op. + + This guards Linux/macOS behaviour (and pytest capsys, which also + reports utf-8) — the stream object must be left untouched so we + don't break test capture or replace a working stream with an + equivalent-but-different wrapper. + """ + import io + utf8_stream = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") + sys.stdout = utf8_stream + sys.stderr = utf8_stream + + _force_utf8_stdio() + + self.assertIs(sys.stdout, utf8_stream) + self.assertIs(sys.stderr, utf8_stream) + + def test_noop_for_stream_without_buffer(self): + """Streams without a .buffer attribute are skipped, not crashed. + + Some environments (IDLE REPL, custom capture objects) replace + sys.stdout with an object that doesn't expose the underlying + binary buffer. The fix must not crash those — it just leaves + them alone. + """ + class BufferlessStream: + encoding = "cp1252" + + def write(self, s): + pass + + def flush(self): + pass + + bufferless = BufferlessStream() + sys.stdout = bufferless + sys.stderr = bufferless + + # Must not raise + _force_utf8_stdio() + + # Stream is left untouched (cannot wrap without .buffer) + self.assertIs(sys.stdout, bufferless) + self.assertIs(sys.stderr, bufferless) + + def test_handles_stream_with_none_encoding(self): + """Streams with encoding=None are wrapped (treated as non-utf8). + + Some custom streams report encoding=None. The fix should + treat this as 'not utf-8' and attempt to wrap (subject to + the .buffer check). + """ + import io + buf = io.BytesIO() + none_enc_stream = io.TextIOWrapper(buf, encoding="utf-8") + # Force encoding attribute to None via subclass to simulate + # environments that don't populate it. + class _NoneEncStream: + buffer = buf + encoding = None + + def write(self, s): + pass + + def flush(self): + pass + + none_stream = _NoneEncStream() + sys.stdout = none_stream + sys.stderr = none_stream + + _force_utf8_stdio() + + # Stream should have been replaced with a UTF-8 wrapper + self.assertIsNot(sys.stdout, none_stream) + encoding = (sys.stdout.encoding or "").lower().replace("-", "") + self.assertEqual(encoding, "utf8") + + def test_writes_unicode_arrow_to_replaced_stream(self): + """End-to-end: after _force_utf8_stdio, the arrow character used + in trace output reaches the underlying buffer as UTF-8 bytes. + + This is the actual user-facing scenario from issue #179: a + Windows user runs ``codelens trace`` and the path string + ``foo.ts \u2192 bar.ts`` must not crash the CLI. + """ + import io + buf = io.BytesIO() + # Start with cp1252 (simulating Windows without PYTHONUTF8=1) + sys.stdout = io.TextIOWrapper(buf, encoding="cp1252") + + _force_utf8_stdio() + + # The exact pattern the trace command emits + sys.stdout.write("sidepanel.ts \u2192 auth/google-auth-cache.ts\n") + sys.stdout.flush() + + written = buf.getvalue() + self.assertIn(self._ARROW_UTF8_BYTES, written) + # Round-trip: bytes decode back to the original string + self.assertEqual( + written.decode("utf-8"), + "sidepanel.ts \u2192 auth/google-auth-cache.ts\n", + ) + + def test_both_stdout_and_stderr_wrapped(self): + """Both sys.stdout and sys.stderr get the UTF-8 treatment. + + stderr matters because the CLI writes error/diagnostic messages + there too (e.g. ``[CodeLens] Warning: ...`` lines), and on + Windows those would also crash if they contained Unicode. + """ + import io + out_buf = io.BytesIO() + err_buf = io.BytesIO() + sys.stdout = io.TextIOWrapper(out_buf, encoding="cp1252") + sys.stderr = io.TextIOWrapper(err_buf, encoding="cp1252") + + _force_utf8_stdio() + + out_enc = (sys.stdout.encoding or "").lower().replace("-", "") + err_enc = (sys.stderr.encoding or "").lower().replace("-", "") + self.assertEqual(out_enc, "utf8") + self.assertEqual(err_enc, "utf8") + + sys.stdout.write("out \u2192 arrow\n") + sys.stderr.write("err \u2192 arrow\n") + sys.stdout.flush() + sys.stderr.flush() + self.assertIn(self._ARROW_UTF8_BYTES, out_buf.getvalue()) + self.assertIn(self._ARROW_UTF8_BYTES, err_buf.getvalue()) + + if __name__ == "__main__": unittest.main()