From f2a7516dbadd4a8492be3c3c72c46253b2cccbcf Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Fri, 23 Oct 2026 22:35:01 +0000 Subject: [PATCH 1/2] gh-160535: Keep the ZstdDict alive until the zstd context is freed The garbage collector could release a ZstdCompressor's dictionary, or the digested dictionaries cached by the ZstdDict, while worker threads of the compression context were still reading them. They are now only released on deallocation, after the context has been freed. ANT-2026-CT3P1CSJ --- Lib/test/test_zstd.py | 38 +++++++++++++++++++ ...-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst | 4 ++ Modules/_zstd/compressor.c | 10 +---- Modules/_zstd/decompressor.c | 11 +----- Modules/_zstd/zstddict.c | 16 +++----- 5 files changed, 51 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 372326a26c71ea..60f41ead76b867 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -7,10 +7,12 @@ import os import unittest import tempfile +import textwrap import threading from test import support from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok from test.support import threading_helper from test.support import _1M @@ -1571,6 +1573,42 @@ def test_len(self): self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + @unittest.skipUnless(SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + @support.requires_subprocess() + def test_collect_cycle_with_busy_compressor(self): + # The dictionary must outlive the compression context when both are + # collected by the garbage collector while a worker thread is still + # using the dictionary. Whether the failure is detected depends on + # the memory allocator, so use a fresh process for each case. + code = textwrap.dedent(""" + import gc, random, sys + from compression.zstd import (CompressionParameter, + ZstdCompressor, ZstdDict) + + class Cycle: + pass + + job_size = 1 << 20 + content = random.randbytes(1 << 10) * (1 << 8) + zd = ZstdDict(content, is_raw=True) + options = {CompressionParameter.compression_level: 10, + CompressionParameter.nb_workers: 1, + CompressionParameter.job_size: job_size} + c = ZstdCompressor(options=options, + zstd_dict=getattr(zd, sys.argv[1])) + cycle = Cycle() + cycle.cycle = cycle + cycle.compressor = c + # Returns while the worker thread is compressing the first job. + c.compress(random.randbytes(1 << 10) * ((job_size >> 10) + 64)) + del zd, c, cycle + gc.collect() + """) + for attr in 'as_digested_dict', 'as_prefix': + with self.subTest(attr=attr): + assert_python_ok('-c', code, attr) + class FileTestCase(unittest.TestCase): def setUp(self): self.DECOMPRESSED_42 = b'a'*42 diff --git a/Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst new file mode 100644 index 00000000000000..ea2082716d64ac --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst @@ -0,0 +1,4 @@ +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 894568fce28d0d..e88fa2bb206c89 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -770,13 +770,8 @@ ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) return 0; } -static int -ZstdCompressor_clear(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - Py_CLEAR(self->dict); - return 0; -} +/* No tp_clear: the compression context (and its worker threads) uses the + dictionary until ZstdCompressor_dealloc() has freed the context. */ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, @@ -785,7 +780,6 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_members, ZstdCompressor_members}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, - {Py_tp_clear, ZstdCompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index ca9ea7e3b8f99b..2ed13cb38797b7 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -693,14 +693,8 @@ ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) return 0; } -static int -ZstdDecompressor_clear(PyObject *ob) -{ - ZstdDecompressor *self = ZstdDecompressor_CAST(ob); - Py_CLEAR(self->dict); - Py_CLEAR(self->unused_data); - return 0; -} +/* No tp_clear: the decompression context references the dictionary until + ZstdDecompressor_dealloc() has freed the context. */ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, @@ -710,7 +704,6 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, - {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index e1b9d998e697fb..11ea7619976bf9 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -240,16 +240,11 @@ ZstdDict_length(PyObject *ob) static int ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { - ZstdDict *self = ZstdDict_CAST(ob); - Py_VISIT(self->c_dicts); - return 0; -} - -static int -ZstdDict_clear(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->c_dicts); + /* c_dicts is not visited and there is no tp_clear: it only maps ints + to capsules, and the ZSTD_CDict instances they own must stay alive + until ZstdDict_dealloc() as a compression context being collected + together with this object may still use them. */ + Py_VISIT(Py_TYPE(ob)); return 0; } @@ -262,7 +257,6 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, - {Py_tp_clear, ZstdDict_clear}, {0, 0} }; From fc2e18d826a33393f02776ffc1212c0adf15170a Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 25 Oct 2026 00:16:37 +0000 Subject: [PATCH 2/2] Visit the type in the compressor and decompressor traverse; move NEWS to Library --- .../2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst | 0 Modules/_zstd/compressor.c | 1 + Modules/_zstd/decompressor.c | 1 + 3 files changed, 2 insertions(+) rename Misc/NEWS.d/next/{Security => Library}/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst (100%) diff --git a/Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst similarity index 100% rename from Misc/NEWS.d/next/Security/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst rename to Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index e88fa2bb206c89..19e12ed4af7479 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -766,6 +766,7 @@ static int ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdCompressor *self = ZstdCompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 2ed13cb38797b7..454736d687a430 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -689,6 +689,7 @@ static int ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDecompressor *self = ZstdDecompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; }