From df32f2bdacbcbbd42e992fce2f7e6197969ee53c Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:31:45 -0700 Subject: [PATCH 1/2] gh-160535: Keep the ZstdDict alive until the zstd context is freed (GH-160536) The garbage collector could release a ZstdCompressor's dictionary, or the digested dictionaries cached by the ZstdDict, while worker threads of the compression context were still reading them. They are now only released on deallocation, after the context has been freed. ANT-2026-CT3P1CSJ + Visit the type in the compressor and decompressor traverse; move NEWS to Library (cherry picked from commit b949033ae8b22ff77e798a08f4ecc6eaf4cf0d8a) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_zstd.py | 38 +++++++++++++++++++ ...-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst | 4 ++ Modules/_zstd/compressor.c | 11 ++---- Modules/_zstd/decompressor.c | 12 ++---- Modules/_zstd/zstddict.c | 16 +++----- 5 files changed, 53 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 6de63d840898ac..b5bc37428f90e1 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -7,9 +7,11 @@ import os import unittest import tempfile +import textwrap import threading from test.support.import_helper import import_module +from test.support.script_helper import assert_python_ok from test.support import threading_helper from test.support import _1M from test.support import Py_GIL_DISABLED @@ -1549,6 +1551,42 @@ def test_len(self): self.assertEqual(len(TRAINED_DICT), len(TRAINED_DICT.dict_content)) self.assertIn(str(len(TRAINED_DICT)), str(TRAINED_DICT)) + @unittest.skipUnless(SUPPORT_MULTITHREADING, + "zstd build doesn't support multi-threaded compression") + @support.requires_subprocess() + def test_collect_cycle_with_busy_compressor(self): + # The dictionary must outlive the compression context when both are + # collected by the garbage collector while a worker thread is still + # using the dictionary. Whether the failure is detected depends on + # the memory allocator, so use a fresh process for each case. + code = textwrap.dedent(""" + import gc, random, sys + from compression.zstd import (CompressionParameter, + ZstdCompressor, ZstdDict) + + class Cycle: + pass + + job_size = 1 << 20 + content = random.randbytes(1 << 10) * (1 << 8) + zd = ZstdDict(content, is_raw=True) + options = {CompressionParameter.compression_level: 10, + CompressionParameter.nb_workers: 1, + CompressionParameter.job_size: job_size} + c = ZstdCompressor(options=options, + zstd_dict=getattr(zd, sys.argv[1])) + cycle = Cycle() + cycle.cycle = cycle + cycle.compressor = c + # Returns while the worker thread is compressing the first job. + c.compress(random.randbytes(1 << 10) * ((job_size >> 10) + 64)) + del zd, c, cycle + gc.collect() + """) + for attr in 'as_digested_dict', 'as_prefix': + with self.subTest(attr=attr): + assert_python_ok('-c', code, attr) + class FileTestCase(unittest.TestCase): def setUp(self): self.DECOMPRESSED_42 = b'a'*42 diff --git a/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst new file mode 100644 index 00000000000000..ea2082716d64ac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-23-22-34-55.gh-issue-160535.Zk3dGc.rst @@ -0,0 +1,4 @@ +Fix a crash in :mod:`compression.zstd` when the garbage collector frees a +:class:`~compression.zstd.ZstdCompressor` that uses worker threads together +with its :class:`~compression.zstd.ZstdDict` before the compression of +already submitted data has finished. diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index fa31d3a0e90abd..02162c5c9a6f58 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -765,17 +765,13 @@ static int ZstdCompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdCompressor *self = ZstdCompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdCompressor_clear(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - Py_CLEAR(self->dict); - return 0; -} +/* No tp_clear: the compression context (and its worker threads) uses the + dictionary until ZstdCompressor_dealloc() has freed the context. */ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, @@ -784,7 +780,6 @@ static PyType_Slot zstdcompressor_slots[] = { {Py_tp_members, ZstdCompressor_members}, {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, - {Py_tp_clear, ZstdCompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 85b67d46886489..08918629446cd6 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -688,18 +688,13 @@ static int ZstdDecompressor_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDecompressor *self = ZstdDecompressor_CAST(ob); + Py_VISIT(Py_TYPE(ob)); Py_VISIT(self->dict); return 0; } -static int -ZstdDecompressor_clear(PyObject *ob) -{ - ZstdDecompressor *self = ZstdDecompressor_CAST(ob); - Py_CLEAR(self->dict); - Py_CLEAR(self->unused_data); - return 0; -} +/* No tp_clear: the decompression context references the dictionary until + ZstdDecompressor_dealloc() has freed the context. */ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, @@ -709,7 +704,6 @@ static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_getset, ZstdDecompressor_getset}, {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, - {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index e1b9d998e697fb..11ea7619976bf9 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -240,16 +240,11 @@ ZstdDict_length(PyObject *ob) static int ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { - ZstdDict *self = ZstdDict_CAST(ob); - Py_VISIT(self->c_dicts); - return 0; -} - -static int -ZstdDict_clear(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->c_dicts); + /* c_dicts is not visited and there is no tp_clear: it only maps ints + to capsules, and the ZSTD_CDict instances they own must stay alive + until ZstdDict_dealloc() as a compression context being collected + together with this object may still use them. */ + Py_VISIT(Py_TYPE(ob)); return 0; } @@ -262,7 +257,6 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, - {Py_tp_clear, ZstdDict_clear}, {0, 0} }; From 48b1c9ddd3e305af73e602997a4660510eba3119 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 25 Oct 2026 05:40:28 +0000 Subject: [PATCH 2/2] Import test.support for requires_subprocess() --- Lib/test/test_zstd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index b5bc37428f90e1..b62b125aaecde3 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -10,6 +10,7 @@ import textwrap import threading +from test import support from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok from test.support import threading_helper