diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 9e1599a76485646..e8a985ca37a35ff 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -43,6 +43,7 @@ typedef struct { PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_annotate; /* Callable to fill the annotations dictionary */ PyObject *func_typeparams; /* Tuple of active type variables or NULL */ + PyObject *func_old_codes; /* List of past code objects or NULL */ vectorcallfunc vectorcall; /* Version number for use by specializer. * Can set to non-zero when we want to specialize. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 9809cd292995f0b..2ea4811080b35c0 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -132,7 +132,7 @@ _PyFrame_NumSlotsForCodeObject(PyCodeObject *code) static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) { - dest->f_executable = PyStackRef_MakeHeapSafe(src->f_executable); + dest->f_executable = PyStackRef_Borrow(src->f_executable); // Don't leave a dangling pointer to the old frame when creating generators // and coroutines: dest->previous = NULL; @@ -160,6 +160,15 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * } } +/* Generator frames need a strong reference to the code object */ +static inline void +_PyFrame_CopyForGenerators(_PyInterpreterFrame *old_frame, _PyInterpreterFrame *gen_frame) +{ + _PyFrame_Copy(old_frame, gen_frame); + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + gen_frame->f_executable = PyStackRef_MakeHeapSafe(gen_frame->f_executable); +} + #ifdef Py_GIL_DISABLED static inline void _PyFrame_InitializeTLBC(PyThreadState *tstate, _PyInterpreterFrame *frame, @@ -191,7 +200,7 @@ _PyFrame_Initialize( { frame->previous = previous; frame->f_funcobj = func; - frame->f_executable = PyStackRef_FromPyObjectNew(code); + frame->f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)code); PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); frame->f_builtins = func_obj->func_builtins; frame->f_globals = func_obj->func_globals; @@ -424,7 +433,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int assert(tstate->datastack_top < tstate->datastack_limit); frame->previous = previous; frame->f_funcobj = PyStackRef_None; - frame->f_executable = PyStackRef_FromPyObjectNew(code); + frame->f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)code); #ifdef Py_DEBUG frame->f_builtins = NULL; frame->f_globals = NULL; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 4d267e35504b94d..5f534214cbe0bc6 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -27,7 +27,9 @@ enum _frameowner { }; struct _PyInterpreterFrame { - _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ + /* Borrowed reference (code object or None) */ + /* Strong reference for generators */ + _PyStackRef f_executable; struct _PyInterpreterFrame *previous; _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index c1a278e5d4da916..eb976a97c81dc3e 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -325,6 +325,28 @@ def annofn(arg: int) -> str: with self.assertRaises(SystemError): _testcapi.function_get_annotations(None) + def test_function_old_codes(self): + def f(): + pass + + def g(): + pass + + def h(): + pass + + old_codes = _testcapi.function_get_old_codes(f) + self.assertIsNone(old_codes) + + f.__code__ = g.__code__ + old_codes = _testcapi.function_get_old_codes(f) + self.assertIsInstance(old_codes, list) + self.assertEqual(len(old_codes), 1) + + f.__code__ = h.__code__ + old_codes = _testcapi.function_get_old_codes(f) + self.assertEqual(len(old_codes), 2) + # TODO: test PyFunction_New() # TODO: test PyFunction_NewWithQualName() # TODO: test PyFunction_SetVectorcall() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1773633730ea001..fd1fa40b73e2149 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1703,7 +1703,7 @@ def func(): check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P')) # function def func(): pass - check(func, size('16Pi')) + check(func, size('17Pi')) class c(): @staticmethod def foo(): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst new file mode 100644 index 000000000000000..299f4961d2c13bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-04-17-04-03.gh-issue-152666.Vo5wJv.rst @@ -0,0 +1,3 @@ +Avoid reference counting of :class:`code` objects when creating and destroying +frames by having functions retain a list of all past :class:`code` objects. + diff --git a/Modules/_testcapi/function.c b/Modules/_testcapi/function.c index 40767adbd3f14a7..bc790a63254b436 100644 --- a/Modules/_testcapi/function.c +++ b/Modules/_testcapi/function.c @@ -130,6 +130,19 @@ function_get_annotations(PyObject *self, PyObject *func) } +static PyObject * +function_get_old_codes(PyObject *self, PyObject *func) +{ + PyFunctionObject *func_o = (PyFunctionObject *) func; + PyObject *old_codes = func_o->func_old_codes; + if (old_codes == NULL) { + Py_RETURN_NONE; + } + + return Py_NewRef(old_codes); +} + + static PyMethodDef test_methods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, @@ -141,6 +154,7 @@ static PyMethodDef test_methods[] = { {"function_get_closure", function_get_closure, METH_O, NULL}, {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, {"function_get_annotations", function_get_annotations, METH_O, NULL}, + {"function_get_old_codes", function_get_old_codes, METH_O, NULL}, {NULL}, }; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 5f2b1ae5d978aab..18336b13147ca9b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -11555,10 +11555,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 0fffd36ad462dab..1d898ba71f42685 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -145,6 +145,7 @@ _PyFunction_FromConstructor(PyFrameConstructor *constr) op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; + op->func_old_codes = NULL; // NOTE: functions created via FrameConstructor do not use deferred // reference counting because they are typically not part of cycles // nor accessed by multiple threads. @@ -223,6 +224,7 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname op->func_typeparams = NULL; op->vectorcall = _PyFunction_Vectorcall; op->func_version = FUNC_VERSION_UNSET; + op->func_old_codes = NULL; if (((code_obj->co_flags & CO_NESTED) == 0) || (code_obj->co_flags & CO_METHOD)) { // Use deferred reference counting for top-level functions, but not @@ -686,6 +688,17 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) handle_func_event(PyFunction_EVENT_MODIFY_CODE, op, value); _PyFunction_ClearVersion(op); + if (op->func_old_codes == NULL) { + op->func_old_codes = PyList_New(0); + if (op->func_old_codes == NULL) { + return -1; + } + } + + if (PyList_Append(op->func_old_codes, op->func_code) < 0) { + return -1; + } + Py_XSETREF(op->func_code, Py_NewRef(value)); return 0; } @@ -1114,6 +1127,7 @@ func_clear(PyObject *self) Py_CLEAR(op->func_annotations); Py_CLEAR(op->func_annotate); Py_CLEAR(op->func_typeparams); + Py_CLEAR(op->func_old_codes); // Don't Py_CLEAR(op->func_code), since code is always required // to be non-NULL. Similarly, name and qualname shouldn't be NULL. // However, name and qualname could be str subclasses, so they @@ -1169,6 +1183,7 @@ func_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(f->func_annotate); Py_VISIT(f->func_typeparams); Py_VISIT(f->func_qualname); + Py_VISIT(f->func_old_codes); return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 3cdc06733363d3e..fd982eee8187185 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1176,11 +1176,10 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, assert(f->f_frame->frame_obj == NULL); assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); _PyInterpreterFrame *frame = &gen->gi_iframe; - _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); + _PyFrame_CopyForGenerators((_PyInterpreterFrame *)f->_f_frame_data, frame); gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); f->f_frame = frame; - frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); gen->gi_weakreflist = NULL; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8ac397081e27384..d45fc8f9fd9f46f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5854,10 +5854,9 @@ dummy_func( SAVE_STACK(); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 96dea2d2a6d5bc1..e33108c9bc8f3ca 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -20714,10 +20714,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index cc95179ccaab17c..3b5bac69a4fbc5d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11552,10 +11552,9 @@ _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); + _PyFrame_CopyForGenerators(frame, gen_frame); assert(frame->frame_obj == NULL); gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev);