From 9ba7fbefa2217592043d8bbf79fc74be096c5126 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 24 Oct 2026 18:58:23 +0000 Subject: [PATCH 1/2] gh-153568: Use a managed dict for AST nodes AST nodes had an explicit `__dict__`` slot, so every node built by `ast.parse` allocated its own dict. With Py_TPFLAGS_MANAGED_DICT, node attributes are stored as inline values with shared keys instead. `ast.parse` is ~15.8% faster, walking trees is ~15.9% faster, and the memory used by the AST trees is reduced by ~51.6%. --- Lib/test/test_ast/test_ast.py | 7 +++++++ Lib/test/test_sys.py | 2 +- ...6-10-23-23-45-00.gh-issue-153568.astinl.rst | 2 ++ Parser/asdl_c.py | 18 +++++------------- Python/Python-ast.c | 18 +++++------------- 5 files changed, 20 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index b1483ad6bf2d40..08d4cfb9ae6c8a 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -103,6 +103,13 @@ def test_AST_objects(self): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_nodes_have_inline_values(self): + tree = ast.parse("x = f(y, z=1)") + for node in ast.walk(tree): + with self.subTest(node=type(node).__name__): + self.assertTrue(_testinternalcapi.has_inline_values(node)) + def test_AST_fields_NULL_check(self): # See: https://github.com/python/cpython/issues/126105 old_value = ast.AST._fields diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3de66a5508e69a..201c209d821415 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2042,7 +2042,7 @@ def test_pythontypes(self): check = self.check_sizeof # _ast.AST import _ast - check(_ast.Module(), size('3P')) + check(_ast.Module(), size('2P')) try: raise TypeError except TypeError as e: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst new file mode 100644 index 00000000000000..a043d20b7a3408 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-23-45-00.gh-issue-153568.astinl.rst @@ -0,0 +1,2 @@ +Speed up :func:`ast.parse` and reduce the memory used by AST nodes by storing +node attributes as inline values instead of a separate per-node dictionary. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 2b83e97353bbf1..edbb749f5ee231 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -913,7 +913,6 @@ def visitModule(self, mod): typedef struct { PyObject_HEAD - PyObject *dict; } AST_object; static void @@ -923,7 +922,7 @@ def visitModule(self, mod): /* bpo-31095: UnTrack is needed before calling any callbacks */ PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); freefunc free_func = PyType_GetSlot(tp, Py_tp_free); assert(free_func != NULL); free_func(self); @@ -935,15 +934,13 @@ def visitModule(self, mod): { AST_object *self = (AST_object*)op; Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int ast_clear(PyObject *op) { - AST_object *self = (AST_object*)op; - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -1419,11 +1416,6 @@ def visitModule(self, mod): return result; } -static PyMemberDef ast_type_members[] = { - {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY}, - {NULL} /* Sentinel */ -}; - static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, {"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS | METH_KEYWORDS, @@ -1668,7 +1660,6 @@ def visitModule(self, mod): {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, {Py_tp_clear, ast_clear}, - {Py_tp_members, ast_type_members}, {Py_tp_methods, ast_type_methods}, {Py_tp_getset, ast_type_getsets}, {Py_tp_init, ast_type_init}, @@ -1682,7 +1673,8 @@ def visitModule(self, mod): "ast.AST", sizeof(AST_object), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_MANAGED_DICT, AST_type_slots }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 0d89f7e87f1fa2..2c020e6c2070fc 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1414,7 +1414,6 @@ add_ast_annotations(struct ast_state *state) typedef struct { PyObject_HEAD - PyObject *dict; } AST_object; static void @@ -1424,7 +1423,7 @@ ast_dealloc(PyObject *op) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); freefunc free_func = PyType_GetSlot(tp, Py_tp_free); assert(free_func != NULL); free_func(self); @@ -1436,15 +1435,13 @@ ast_traverse(PyObject *op, visitproc visit, void *arg) { AST_object *self = (AST_object*)op; Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->dict); - return 0; + return PyObject_VisitManagedDict(op, visit, arg); } static int ast_clear(PyObject *op) { - AST_object *self = (AST_object*)op; - Py_CLEAR(self->dict); + PyObject_ClearManagedDict(op); return 0; } @@ -1920,11 +1917,6 @@ ast_type_replace(PyObject *self, PyObject *args, PyObject *kwargs) return result; } -static PyMemberDef ast_type_members[] = { - {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY}, - {NULL} /* Sentinel */ -}; - static PyMethodDef ast_type_methods[] = { {"__reduce__", ast_type_reduce, METH_NOARGS, NULL}, {"__replace__", _PyCFunction_CAST(ast_type_replace), METH_VARARGS | METH_KEYWORDS, @@ -2169,7 +2161,6 @@ static PyType_Slot AST_type_slots[] = { {Py_tp_setattro, PyObject_GenericSetAttr}, {Py_tp_traverse, ast_traverse}, {Py_tp_clear, ast_clear}, - {Py_tp_members, ast_type_members}, {Py_tp_methods, ast_type_methods}, {Py_tp_getset, ast_type_getsets}, {Py_tp_init, ast_type_init}, @@ -2183,7 +2174,8 @@ static PyType_Spec AST_type_spec = { "ast.AST", sizeof(AST_object), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_MANAGED_DICT, AST_type_slots }; From 1a69802082a1795740b38b9be9116070f58449c4 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 24 Oct 2026 19:07:40 +0000 Subject: [PATCH 2/2] Add news blurb --- .../2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst new file mode 100644 index 00000000000000..98628f6a74208f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-24-19-07-34.gh-issue-153568.6sVdqs.rst @@ -0,0 +1,3 @@ +Speed up :func:`ast.parse` by ~15.8% and reduce the memory used by ~51.6% by +storing node attributes as inline values instead of in a per-node +dictionary.