Skip to content

Commit b1e530e

Browse files
gh-154582: Fix test_invalid_utf8_arg in non-UTF-8 multibyte locales (GH-154584)
Arbitrary bytes round-trip through surrogateescape only in UTF-8 and single-byte encodings, not in a stateful multibyte encoding such as EUC-JP. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 41a087a commit b1e530e

1 file changed

Lines changed: 22 additions & 3 deletions

File tree

Lib/test/test_cmd_line.py

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Most tests are executed with environment variables ignored
33
# See test_cmd_line_script.py for testing of script execution
44

5+
import locale
56
import os
67
import re
78
import subprocess
@@ -369,9 +370,27 @@ def run_no_utf8_mode(arg):
369370
)
370371
test_args = [valid_utf8, invalid_utf8]
371372

372-
for run_cmd in (run_default, run_c_locale, run_utf8_mode,
373-
run_no_utf8_mode):
374-
with self.subTest(run_cmd=run_cmd):
373+
for run_cmd, encoding in (
374+
(run_default, sys.getfilesystemencoding()),
375+
(run_c_locale, None),
376+
(run_utf8_mode, None),
377+
(run_no_utf8_mode, locale.getencoding())
378+
):
379+
with self.subTest(run_cmd=run_cmd.__name__):
380+
# Arbitrary bytes round-trip through surrogateescape only in
381+
# UTF-8 and single-byte encodings, not in a multibyte encoding
382+
# such as EUC-JP.
383+
if encoding is not None:
384+
try:
385+
lossless = len(bytes(range(256)).decode(
386+
encoding, 'surrogateescape')) == 256
387+
except UnicodeError:
388+
lossless = False
389+
else:
390+
lossless = True
391+
if not lossless:
392+
self.skipTest(f'{encoding} cannot losslessly '
393+
f'round-trip arbitrary bytes')
375394
for arg in test_args:
376395
proc = run_cmd(arg)
377396
self.assertEqual(proc.stdout.rstrip(), ascii(arg))

0 commit comments

Comments
 (0)