diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index d4923a31aef835..9626e0eee6af0d 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -3,6 +3,7 @@ # Unit test for multibytecodec itself # +import _codecs import _multibytecodec import codecs import io @@ -252,6 +253,91 @@ def test_subinterp(self): class Test_IncrementalDecoder(unittest.TestCase): + def test_final_decode_incomplete_sequence(self): + cases = [ + ('euc_jp', b'\x8f\x00'), + ('euc_jis_2004', b'\x8f\x00'), + ('euc_jisx0213', b'\x8f\x00'), + ('gb18030', b'\x806'), + ('gb18030', b'\x81\x30\x81'), + ] + for encoding, data in cases: + for errors in ('surrogateescape', 'replace', 'ignore', + 'backslashreplace'): + expected = data.decode(encoding, errors) + for split in range(len(data) + 1): + with self.subTest(encoding=encoding, data=data, + errors=errors, split=split): + decoder = codecs.getincrementaldecoder(encoding)(errors) + text = decoder.decode(data[:split]) + text += decoder.decode(data[split:], final=True) + self.assertEqual(text, expected) + self.assertEqual(decoder.getstate()[0], b'') + self.assertEqual(decoder.decode(b'', final=True), '') + + def test_final_decode_incomplete_sequence_strict(self): + for encoding in ('euc_jp', 'euc_jis_2004', 'euc_jisx0213', 'gb18030'): + with self.subTest(encoding=encoding): + decoder = codecs.getincrementaldecoder(encoding)() + self.assertEqual(decoder.decode(b'\x81'), '') + state = decoder.getstate() + with self.assertRaises(UnicodeDecodeError): + decoder.decode(b'', final=True) + self.assertEqual(decoder.getstate(), state) + decoder.errors = 'surrogateescape' + self.assertEqual(decoder.decode(b'', final=True), '\udc81') + self.assertEqual(decoder.getstate()[0], b'') + + def test_final_decode_partial_error_handler(self): + def replace_one(exc): + return ('?', exc.start + 1) + + errors = 'test.multibytecodec.replace_one' + codecs.register_error(errors, replace_one) + self.addCleanup(_codecs._unregister_error, errors) + for encoding, data, expected in ( + ('euc_jp', b'\x8f\x81', '??'), + ('gb18030', b'\x81\x30\x81', '?0?'), + ): + with self.subTest(encoding=encoding): + decoder = codecs.getincrementaldecoder(encoding)(errors) + self.assertEqual(decoder.decode(data, final=True), expected) + self.assertEqual(decoder.getstate()[0], b'') + + def test_final_decode_partial_error_handler_failure(self): + def replace_one_then_raise(exc): + if exc.start: + raise exc + return ('?', exc.start + 1) + + errors = 'test.multibytecodec.replace_one_then_raise' + codecs.register_error(errors, replace_one_then_raise) + self.addCleanup(_codecs._unregister_error, errors) + decoder = codecs.getincrementaldecoder('euc_jp')(errors) + self.assertEqual(decoder.decode(b'\x8f\x81'), '') + state = decoder.getstate() + with self.assertRaises(UnicodeDecodeError): + decoder.decode(b'', final=True) + self.assertEqual(decoder.getstate(), state) + decoder.errors = 'surrogateescape' + self.assertEqual(decoder.decode(b'', final=True), '\udc8f\udc81') + self.assertEqual(decoder.getstate()[0], b'') + + def test_textio_incomplete_sequence(self): + for encoding, data in ( + ('euc_jp', b'\x8f\x00'), + ('euc_jis_2004', b'\x8f\x00'), + ('euc_jisx0213', b'\x8f\x00'), + ('gb18030', b'\x806'), + ('gb18030', b'\x81\x30\x81'), + ): + with self.subTest(encoding=encoding, data=data): + with io.TextIOWrapper(io.BytesIO(data), encoding=encoding, + errors='surrogateescape') as stream: + self.assertEqual(stream.read(), + data.decode(encoding, 'surrogateescape')) + self.assertEqual(stream.read(), '') + def test_iso2022_split_escape(self): raw = b'\x1b$0000000B' decoder = codecs.getincrementaldecoder('iso2022_jp')('replace') diff --git a/Misc/NEWS.d/next/Library/2026-10-25-11-00-00.gh-issue-160307.Qe7mJ2.rst b/Misc/NEWS.d/next/Library/2026-10-25-11-00-00.gh-issue-160307.Qe7mJ2.rst new file mode 100644 index 00000000000000..ed7cb6fcad38ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-25-11-00-00.gh-issue-160307.Qe7mJ2.rst @@ -0,0 +1,4 @@ +Fix incomplete final decoding with the ``surrogateescape`` error handler in +the ``euc_jp``, ``euc_jis_2004``, ``euc_jisx0213`` and ``gb18030`` codecs. +Trailing input after an escaped byte is now decoded instead of remaining +buffered, including when reading through :class:`io.TextIOWrapper`. diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index f513182cb70260..c6ef25eff07efc 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -1217,7 +1217,7 @@ _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDeco if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) goto errorexit; - if (final && buf.inbuf < buf.inbuf_end) { + while (final && buf.inbuf < buf.inbuf_end) { if (multibytecodec_decerror(self->codec, &self->state, &buf, self->errors, MBERR_TOOFEW)) { /* recover the original pending buffer */ @@ -1225,6 +1225,9 @@ _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDeco self->pendingsize = origpending; goto errorexit; } + /* The error handler may have consumed only part of the input. */ + if (decoder_feed_buffer(STATEFUL_DCTX(self), &buf)) + goto errorexit; } if (buf.inbuf < buf.inbuf_end) { /* pending sequence still exists */