From c41270ea722743ddccdab2c636237f74e96b89a2 Mon Sep 17 00:00:00 2001 From: Julian Soreavis <263610811+soreavis@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:12:38 +0200 Subject: [PATCH 1/2] gh-43521: Add an opt-in gzip Content-Encoding handler to urllib.request Add urllib.request.HTTPGzipHandler, an opt-in handler that requests gzip-compressed responses and transparently decodes them. The handler is not one of the default build_opener() handlers, so urlopen()'s behaviour is unchanged unless it is added explicitly. When the request has no Accept-Encoding header, the handler adds "Accept-Encoding: gzip"; if the response then carries "Content-Encoding: gzip", the body is decoded incrementally as it is read and the Content-Encoding and Content-Length headers, which describe the compressed body, are removed. A request that already carries an Accept-Encoding header is left untouched, so a caller that means to handle the compressed response itself still receives it. Only gzip is handled, and the zlib module is required. --- Doc/library/urllib.request.rst | 25 +++ Doc/whatsnew/3.16.rst | 10 ++ Lib/test/test_urllib2.py | 143 ++++++++++++++++++ Lib/urllib/request.py | 109 ++++++++++++- ...6-07-19-12-00-00.gh-issue-43521.aYO2xG.rst | 5 + 5 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 95e4d2627c8b23b..83d50f9e368f656 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -512,6 +512,31 @@ The following classes are provided: Process HTTP error responses. +.. class:: HTTPGzipHandler() + + Request and transparently decode gzip-compressed responses. + + This handler is not one of the handlers installed by :func:`build_opener` + by default; pass it explicitly to opt in:: + + opener = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + with opener.open("http://www.example.com/") as response: + body = response.read() + + Unless the request already has an :mailheader:`Accept-Encoding` header, + the handler adds ``Accept-Encoding: gzip``. If the response is then sent + with ``Content-Encoding: gzip``, its body is decoded as it is read and the + :mailheader:`Content-Encoding` and :mailheader:`Content-Length` headers, + which describe the compressed body, are removed. If the request already + carries an :mailheader:`Accept-Encoding` header, the response is returned + unchanged, so a caller that asked for compressed data can decode it itself. + + Only ``gzip`` is supported. The :mod:`zlib` module is required; + constructing the handler without it raises :exc:`ImportError`. + + .. versionadded:: next + + .. _request-objects: Request Objects diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index c06e6930ac78bff..e5e03fcbf1b8879 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -539,6 +539,16 @@ tkinter with no unit suffix to :class:`int` or :class:`float`. (Contributed by Serhiy Storchaka in :gh:`153513`.) +urllib.request +-------------- + +* Add :class:`~urllib.request.HTTPGzipHandler`, an opt-in handler that + requests gzip-compressed responses (by adding an ``Accept-Encoding: gzip`` + header) and transparently decodes them. It is not installed by + :func:`~urllib.request.build_opener` by default; add it explicitly to opt + in. + (Contributed by Julian Soreavis in :gh:`43521`.) + xml --- diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index d2fd111f6d9de02..82527bbde076552 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1893,6 +1893,149 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") +class FakeGzipResponse: + # Minimal stand-in for the http.client.HTTPResponse that a response + # processor sees: headers plus a readable, closable body. + def __init__(self, data, headers, code=200): + self._file = io.BytesIO(data) + self.headers = headers + self.code = code + self.msg = "OK" + self.url = "http://example.com/" + self.closed = False + + def read(self, amt=-1): + return self._file.read(amt) + + def close(self): + self.closed = True + self._file.close() + + +class HTTPGzipHandlerTests(unittest.TestCase): + + body = b"The quick brown fox jumps over the lazy dog.\n" * 200 + + def make_response(self, data, content_encoding="gzip", content_length=True): + import email.message + headers = email.message.Message() + if content_encoding is not None: + headers["Content-Encoding"] = content_encoding + if content_length: + headers["Content-Length"] = str(len(data)) + headers["Content-Type"] = "text/plain" + return FakeGzipResponse(data, headers) + + def gzip_bytes(self, data): + import gzip + return gzip.compress(data) + + def opened_request(self, **kwargs): + h = urllib.request.HTTPGzipHandler() + req = Request("http://example.com/", **kwargs) + h.http_request(req) + return h, req + + def test_not_installed_by_default(self): + # Opt-in: build_opener() does not install it, but accepts it explicitly. + o = urllib.request.build_opener() + self.assertFalse(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + o = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + self.assertTrue(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + + def test_request_adds_accept_encoding(self): + h, req = self.opened_request() + self.assertEqual(req.get_header("Accept-encoding"), "gzip") + + def test_request_keeps_user_accept_encoding(self): + h, req = self.opened_request(headers={"Accept-Encoding": "identity"}) + self.assertEqual(req.get_header("Accept-encoding"), "identity") + + def test_response_is_decoded(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), self.body) + + def test_response_headers_stripped(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + info = result.info() + self.assertIsNone(info.get("Content-Encoding")) + self.assertIsNone(info.get("Content-Length")) + self.assertEqual(info.get("Content-Type"), "text/plain") + self.assertEqual(result.status, 200) + result.close() + + def test_response_streamed_in_chunks(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + chunks = [] + while chunk := result.read(64): + self.assertLessEqual(len(chunk), 64) + chunks.append(chunk) + self.assertEqual(b"".join(chunks), self.body) + result.close() + + def test_multiple_gzip_members_decoded(self): + # A response may be several concatenated gzip members; all of them + # must be decoded, not just the first. + h, req = self.opened_request() + comp = self.gzip_bytes(b"AAA") + self.gzip_bytes(b"BBB") + response = self.make_response(comp) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), b"AAABBB") + + def test_trailing_junk_after_member_ignored(self): + # Bytes after the final member that are not another gzip member are + # tolerated: the body decodes without error and is not corrupted. + h, req = self.opened_request() + comp = self.gzip_bytes(self.body) + b"trailing garbage" + response = self.make_response(comp) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), self.body) + + def test_user_accept_encoding_passes_through(self): + # A caller-supplied Accept-Encoding is left to the caller to decode: + # the gzipped response is returned untouched. + h, req = self.opened_request(headers={"Accept-Encoding": "gzip"}) + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + self.assertIs(result, response) + self.assertEqual(result.headers.get("Content-Encoding"), "gzip") + + def test_non_gzip_response_passes_through(self): + h, req = self.opened_request() + response = self.make_response(self.body, content_encoding=None) + result = h.http_response(req, response) + self.assertIs(result, response) + + def test_truncated_stream_raises(self): + h, req = self.opened_request() + comp = self.gzip_bytes(self.body) + response = self.make_response(comp[:len(comp) // 2]) + result = h.http_response(req, response) + with self.assertRaises(EOFError): + result.read() + result.close() + + def test_close_propagates(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + result.close() + self.assertTrue(response.closed) + + def test_requires_zlib(self): + with mock.patch.object(urllib.request, "zlib", None): + with self.assertRaises(ImportError): + urllib.request.HTTPGzipHandler() + + class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 660301fef612588..c10648558bc338c 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -84,6 +84,7 @@ import base64 import bisect import contextlib +import copy import email import hashlib import http.client @@ -113,6 +114,11 @@ else: _have_ssl = True +try: + import zlib +except ImportError: + zlib = None + __all__ = [ # Classes 'Request', 'OpenerDirector', 'BaseHandler', 'HTTPDefaultErrorHandler', @@ -122,7 +128,7 @@ 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler', 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler', 'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler', - 'UnknownHandler', 'HTTPErrorProcessor', + 'UnknownHandler', 'HTTPErrorProcessor', 'HTTPGzipHandler', # Functions 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', @@ -1392,6 +1398,107 @@ def http_response(self, request, response): https_request = http_request https_response = http_response + +class _GzipReader(io.BufferedIOBase): + """Incrementally decode a gzip Content-Encoding response body.""" + + def __init__(self, fp): + self._fp = fp + self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) + self._buffer = b'' + self._leftover = b'' + self._got_data = False + self._eof = False + + def readable(self): + return True + + def _refill(self): + while True: + raw = self._leftover or self._fp.read(io.DEFAULT_BUFFER_SIZE) + self._leftover = b'' + if not raw: + self._eof = True + data = self._decompressor.flush() + if self._got_data and not self._decompressor.eof: + raise EOFError('compressed response ended before the ' + 'end-of-stream marker was reached') + return data + self._got_data = True + if self._decompressor.eof: + # The previous member ended; the remaining bytes start the + # next concatenated member. Decode it with a fresh + # decompressor, and tolerate trailing bytes that are not + # another member, as the reader did before. + self._decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) + try: + data = self._decompressor.decompress(raw) + except zlib.error: + self._eof = True + return b'' + else: + data = self._decompressor.decompress(raw) + self._leftover = self._decompressor.unused_data + if data: + return data + + def read(self, size=-1): + if size is None: + size = -1 + while not self._eof and (size < 0 or len(self._buffer) < size): + self._buffer += self._refill() + if size < 0: + data, self._buffer = self._buffer, b'' + else: + data, self._buffer = self._buffer[:size], self._buffer[size:] + return data + + def close(self): + try: + fp = self._fp + if fp is not None: + self._fp = None + fp.close() + finally: + super().close() + + +class HTTPGzipHandler(BaseHandler): + """Request and transparently decode gzip Content-Encoding responses. + + This handler is not installed by build_opener() by default; add it + explicitly to opt in. It sends Accept-Encoding: gzip unless the request + already carries an Accept-Encoding header, and decodes the response only + when it added that header, so a caller supplying its own Accept-Encoding + still receives the raw response. + """ + + def __init__(self): + if zlib is None: + raise ImportError('the zlib module is required for gzip decoding') + + def http_request(self, request): + if not request.has_header('Accept-encoding'): + request.add_unredirected_header('Accept-encoding', 'gzip') + request._gzip_injected = True + return request + + def http_response(self, request, response): + if (getattr(request, '_gzip_injected', False) and + response.headers.get('Content-Encoding', '').lower() == 'gzip'): + headers = copy.copy(response.headers) + del headers['Content-Encoding'] + del headers['Content-Length'] + result = addinfourl(_GzipReader(response), headers, + response.url, response.code) + result.msg = response.msg + return result + return response + + https_request = http_request + https_response = http_response + + class UnknownHandler(BaseHandler): def unknown_open(self, req): type = req.type diff --git a/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst new file mode 100644 index 000000000000000..bacfaf32f1a8153 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst @@ -0,0 +1,5 @@ +Add :class:`urllib.request.HTTPGzipHandler`, an opt-in handler that sends an +``Accept-Encoding: gzip`` request header and transparently decodes +gzip-compressed responses. It is not one of the default +:func:`~urllib.request.build_opener` handlers, so it must be added +explicitly. From 17d1b5038ad3ddc12ae11401eea321c82bc979e0 Mon Sep 17 00:00:00 2001 From: Julian Soreavis <263610811+soreavis@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:54:29 +0100 Subject: [PATCH 2/2] gh-43521: Skip the HTTPGzipHandler tests when zlib is unavailable HTTPGzipHandler() raises ImportError without zlib, so on builds that lack it (such as the MSan CI job) twelve of the tests errored instead of being skipped. --- Lib/test/test_urllib2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 071eaeee09cbf79..4866af77c012dbe 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1968,6 +1968,7 @@ def close(self): self._file.close() +@support.requires_zlib() class HTTPGzipHandlerTests(unittest.TestCase): body = b"The quick brown fox jumps over the lazy dog.\n" * 200