From 6a5c49b9198be405455b53fde2944119f541761d Mon Sep 17 00:00:00 2001 From: Brij <97006829+brijkapadia@users.noreply.github.com> Date: Thu, 2 Jul 2026 12:33:21 -0400 Subject: [PATCH 1/6] gh-152666: Avoid reference counting of code objects --- Include/cpython/funcobject.h | 1 + Include/internal/pycore_interpframe.h | 6 +++--- Lib/test/test_capi/test_function.py | 22 ++++++++++++++++++++++ Modules/_testcapi/function.c | 14 ++++++++++++++ Objects/funcobject.c | 17 +++++++++++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 9e1599a76485646..352cdcfd5ad5f93 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -43,6 +43,7 @@ typedef struct { PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ + PyObject *func_old_codes; vectorcallfunc vectorcall; /* Version number for use by specializer. * Can set to non-zero when we want to specialize. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 9809cd292995f0b..c22a94d6dbbf0ed 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -132,7 +132,7 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - dest->f_executable = PyStackRef_MakeHeapSafe(src->f_executable); + dest->f_executable = PyStackRef_Borrow(src->f_executable); // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: dest->previous = NULL; @@ -191,7 +191,7 @@ _PyFrame_Initialize( { frame->previous = previous; frame->f_funcobj = func; - frame->f_executable = PyStackRef_FromPyObjectNew(code); + frame->f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)code); PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); frame->f_builtins = func_obj->func_builtins; frame->f_globals = func_obj->func_globals; @@ -424,7 +424,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int assert(tstate->datastack_top < tstate->datastack_limit); frame->previous = previous; frame->f_funcobj = PyStackRef_None; - frame->f_executable = PyStackRef_FromPyObjectNew(code); + frame->f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index c1a278e5d4da916..eb976a97c81dc3e 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -325,6 +325,28 @@ def annofn(arg: int) -> str: with self.assertRaises(SystemError): _testcapi.function_get_annotations(None) + def test_function_old_codes(self): + def f(): + pass + + def g(): + pass + + def h(): + pass + + old_codes = _testcapi.function_get_old_codes(f) + self.assertIsNone(old_codes) + + f.__code__ = g.__code__ + old_codes = _testcapi.function_get_old_codes(f) + self.assertIsInstance(old_codes, list) + self.assertEqual(len(old_codes), 1) + + f.__code__ = h.__code__ + old_codes = _testcapi.function_get_old_codes(f) + self.assertEqual(len(old_codes), 2) + # TODO: test PyFunction_New() # TODO: test PyFunction_NewWithQualName() # TODO: test PyFunction_SetVectorcall() diff --git a/Modules/_testcapi/function.c b/Modules/_testcapi/function.c index 40767adbd3f14a7..bc790a63254b436 100644 --- a/Modules/_testcapi/function.c +++ b/Modules/_testcapi/function.c @@ -130,6 +130,19 @@ function_get_annotations(PyObject *self, PyObject *func) } +static PyObject * +function_get_old_codes(PyObject *self, PyObject *func) +{ + PyFunctionObject *func_o = (PyFunctionObject *) func; + PyObject *old_codes = func_o->func_old_codes; + if (old_codes == NULL) { + Py_RETURN_NONE; + } + + return Py_NewRef(old_codes); +} + + static PyMethodDef test_methods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, @@ -141,6 +154,7 @@ static PyMethodDef test_methods[] = { {"function_get_closure", function_get_closure, METH_O, NULL}, {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"function_get_annotations", function_get_annotations, METH_O, NULL}, + {"function_get_old_codes", function_get_old_codes, METH_O, NULL}, {NULL}, }; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 0fffd36ad462dab..591d0d4c47d5016 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -145,6 +145,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; + op->func_old_codes = NULL; // NOTE: functions created via FrameConstructor do not use deferred // reference counting because they are typically not part of cycles // nor accessed by multiple threads. @@ -223,6 +224,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; + op->func_old_codes = NULL; if (((code_obj->co_flags & CO_NESTED) == 0) || (code_obj->co_flags & CO_METHOD)) { // Use deferred reference counting for top-level functions, but not @@ -686,6 +688,19 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); _PyFunction_ClearVersion(op); + if (op->func_old_codes == NULL) { + op->func_old_codes = PyList_New(0); + if (op->func_old_codes == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + if (PyList_Append(op->func_old_codes, op->func_code) < 0) { + PyErr_NoMemory(); + return -1; + } + Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } @@ -1114,6 +1129,7 @@ func_clear(PyObject *self) Py_CLEAR(op->func_annotations); Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); + Py_CLEAR(op->func_old_codes); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. // However, name and qualname could be str subclasses, so they @@ -1169,6 +1185,7 @@ func_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); + Py_VISIT(f->func_old_codes); return 0; } From ead5d4f4060f71581240f15195f914816502ba0b Mon Sep 17 00:00:00 2001 From: Brij <97006829+brijkapadia@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:48:54 -0400 Subject: [PATCH 2/6] fix tests --- Lib/test/test_sys.py | 2 +- Objects/funcobject.c | 2 -- Objects/genobject.c | 5 ----- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1773633730ea001..fd1fa40b73e2149 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1703,7 +1703,7 @@ def func(): check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('16Pi')) + check(func, size('17Pi')) class c(): @staticmethod def foo(): diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 591d0d4c47d5016..1d898ba71f42685 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -691,13 +691,11 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) if (op->func_old_codes == NULL) { op->func_old_codes = PyList_New(0); if (op->func_old_codes == NULL) { - PyErr_NoMemory(); return -1; } } if (PyList_Append(op->func_old_codes, op->func_code) < 0) { - PyErr_NoMemory(); return -1; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 3cdc06733363d3e..5b6aba60a8bdb37 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -93,11 +93,6 @@ gen_traverse(PyObject *self, visitproc visit, void *arg) return err; } } - else { - // We still need to visit the code object when the frame is cleared to - // ensure that it's kept alive if the reference is deferred. - _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); - } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ Py_VISIT(gen->gi_exc_state.exc_value); From ebad4138cb0c4e1b8bad7a4158d5a3a090d35366 Mon Sep 17 00:00:00 2001 From: Brij <97006829+brijkapadia@users.noreply.github.com> Date: Sat, 4 Jul 2026 09:59:28 -0400 Subject: [PATCH 3/6] Update generator logic --- Include/cpython/funcobject.h | 2 +- Include/internal/pycore_interpframe.h | 7 +++++++ Include/internal/pycore_interpframe_structs.h | 2 +- Modules/_testinternalcapi/test_cases.c.h | 3 +-- Objects/genobject.c | 8 ++++++-- Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 3 +-- Python/generated_cases.c.h | 3 +-- 8 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 352cdcfd5ad5f93..010a549a069bcce 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -43,7 +43,7 @@ typedef struct { PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ - PyObject *func_old_codes; + PyObject *func_old_codes; /* NULL or list of past code objects */ vectorcallfunc vectorcall; /* Version number for use by specializer. * Can set to non-zero when we want to specialize. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index c22a94d6dbbf0ed..4e3fdff0875aa32 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -160,6 +160,13 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * } } +/* Generator frames need a strong reference to the code object */ +static inline void _PyFrame_CopyForGenerators(_PyInterpreterFrame *old_frame, _PyInterpreterFrame *gen_frame) { + _PyFrame_Copy(old_frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + gen_frame->f_executable = PyStackRef_MakeHeapSafe(gen_frame->f_executable); +} + #ifdef Py_GIL_DISABLED static inline void _PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 4d267e35504b94d..a22bffc070da7fb 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -27,7 +27,7 @@ enum _frameowner { }; struct _PyInterpreterFrame { - _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ + _PyStackRef f_executable; /* Borrowed reference (code object or None) */ struct _PyInterpreterFrame *previous; _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 5f2b1ae5d978aab..18336b13147ca9b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -11555,10 +11555,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Objects/genobject.c b/Objects/genobject.c index 5b6aba60a8bdb37..fd982eee8187185 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -93,6 +93,11 @@ gen_traverse(PyObject *self, visitproc visit, void *arg) return err; } } + else { + // We still need to visit the code object when the frame is cleared to + // ensure that it's kept alive if the reference is deferred. + _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); + } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ Py_VISIT(gen->gi_exc_state.exc_value); @@ -1171,11 +1176,10 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, assert(f->f_frame->frame_obj == NULL); assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); _PyInterpreterFrame *frame = &gen->gi_iframe; - _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); + _PyFrame_CopyForGenerators((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); f->f_frame = frame; - frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); gen->gi_weakreflist = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8ac397081e27384..d45fc8f9fd9f46f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5854,10 +5854,9 @@ dummy_func( SAVE_STACK(); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 96dea2d2a6d5bc1..e33108c9bc8f3ca 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20714,10 +20714,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc95179ccaab17c..3b5bac69a4fbc5d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11552,10 +11552,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); From 8170840b3d136ed89acc62cf066363cc85ce49bd Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:04:05 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst new file mode 100644 index 000000000000000..f2d261a9179ac85 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst @@ -0,0 +1,2 @@ +Avoid reference counting of :class:`code` objects when creating and destroying +frames by having functions retain a list of all past :class:`code` objects. From 8d5e2aaaac01019fe01eae01d5154e69d7bedd6d Mon Sep 17 00:00:00 2001 From: Brij <97006829+brijkapadia@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:07:12 -0400 Subject: [PATCH 5/6] improve formatting/fix lint errors --- Include/cpython/funcobject.h | 2 +- Include/internal/pycore_interpframe.h | 4 +++- Include/internal/pycore_interpframe_structs.h | 4 +++- .../2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 010a549a069bcce..e8a985ca37a35ff 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -43,7 +43,7 @@ typedef struct { PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ - PyObject *func_old_codes; /* NULL or list of past code objects */ + PyObject *func_old_codes; /* List of past code objects or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. * Can set to non-zero when we want to specialize. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 4e3fdff0875aa32..2ea4811080b35c0 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -161,7 +161,9 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * } /* Generator frames need a strong reference to the code object */ -static inline void _PyFrame_CopyForGenerators(_PyInterpreterFrame *old_frame, _PyInterpreterFrame *gen_frame) { +static inline void +_PyFrame_CopyForGenerators(_PyInterpreterFrame *old_frame, _PyInterpreterFrame *gen_frame) +{ _PyFrame_Copy(old_frame, gen_frame); gen_frame->owner = FRAME_OWNED_BY_GENERATOR; gen_frame->f_executable = PyStackRef_MakeHeapSafe(gen_frame->f_executable); diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index a22bffc070da7fb..5f534214cbe0bc6 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -27,7 +27,9 @@ enum _frameowner { }; struct _PyInterpreterFrame { - _PyStackRef f_executable; /* Borrowed reference (code object or None) */ + /* Borrowed reference (code object or None) */ + /* Strong reference for generators */ + _PyStackRef f_executable; struct _PyInterpreterFrame *previous; _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst index f2d261a9179ac85..299f4961d2c13bc 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst @@ -1,2 +1,3 @@ -Avoid reference counting of :class:`code` objects when creating and destroying +Avoid reference counting of :class:`code` objects when creating and destroying frames by having functions retain a list of all past :class:`code` objects. + From fece1aed69258dd960d09ee93d4692f369796630 Mon Sep 17 00:00:00 2001 From: Brij Kapadia <97006829+brijkapadia@users.noreply.github.com> Date: Mon, 6 Jul 2026 06:43:45 -0400 Subject: [PATCH 6/6] Update Include/internal/pycore_interpframe_structs.h Co-authored-by: Mark Shannon --- Include/internal/pycore_interpframe_structs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 5f534214cbe0bc6..9854141a8ad3ef4 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -29,7 +29,7 @@ enum _frameowner { struct _PyInterpreterFrame { /* Borrowed reference (code object or None) */ /* Strong reference for generators */ - _PyStackRef f_executable; + PyObject *f_executable; struct _PyInterpreterFrame *previous; _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */