From b4a6d81f538909566ff3dd18be41f460fe7a0271 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 01:36:00 -0400 Subject: [PATCH 01/13] Initial implementation. --- Python/codegen.c | 98 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 19 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index e65c308617df5ee..24fef9c72b141f3 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -233,12 +233,18 @@ typedef enum { ITERATOR_ON_STACK = 2, } IterStackPosition; +static int +codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation); +static int +codegen_setcomp_impl(compiler *c, expr_ty e, bool avoid_creation); +static int +codegen_dictcomp_impl(compiler *c, expr_ty e, bool avoid_creation); static int codegen_sync_comprehension_generator( compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type, - IterStackPosition iter_pos); + IterStackPosition iter_pos, bool avoid_creation); static int codegen_async_comprehension_generator( compiler *c, location loc, @@ -3084,6 +3090,12 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value) return SUCCESS; } + if (value->kind == ListComp_kind) { + /* Optimization: Don't bother creating structures if they won't be + * used. */ + return codegen_listcomp_impl(c, value, /*avoid_creation=*/true); + } + VISIT(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ return SUCCESS; @@ -4541,7 +4553,7 @@ codegen_comprehension_generator(compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type, - IterStackPosition iter_pos) + IterStackPosition iter_pos, bool avoid_creation) { comprehension_ty gen; gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); @@ -4552,7 +4564,7 @@ codegen_comprehension_generator(compiler *c, location loc, } else { return codegen_sync_comprehension_generator( c, loc, generators, gen_index, depth, elt, val, type, - iter_pos); + iter_pos, avoid_creation); } } @@ -4561,7 +4573,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type, - IterStackPosition iter_pos) + IterStackPosition iter_pos, bool avoid_creation) { /* generate code for the iterator, then each of the ifs, and then write to the element */ @@ -4629,7 +4641,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, RETURN_IF_ERROR( codegen_comprehension_generator(c, loc, generators, gen_index, depth, - elt, val, type, ITERABLE_IN_LOCAL)); + elt, val, type, ITERABLE_IN_LOCAL, avoid_creation)); } location elt_loc = LOC(elt); @@ -4639,6 +4651,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, /* comprehension specific code */ switch (type) { case COMP_GENEXP: + assert(!avoid_creation); if (elt->kind == Starred_kind) { NEW_JUMP_TARGET_LABEL(c, unpack_start); NEW_JUMP_TARGET_LABEL(c, unpack_end); @@ -4660,6 +4673,11 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } break; case COMP_LISTCOMP: + if (avoid_creation) { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + break; + } if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1); @@ -4670,6 +4688,11 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } break; case COMP_SETCOMP: + if (avoid_creation) { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + break; + } if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, SET_UPDATE, depth + 1); @@ -4680,6 +4703,11 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } break; case COMP_DICTCOMP: + if (avoid_creation) { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + break; + } if (val == NULL) { /* unpacking (**) case */ VISIT(c, expr, elt); @@ -4773,7 +4801,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, RETURN_IF_ERROR( codegen_comprehension_generator(c, loc, generators, gen_index, depth, - elt, val, type, 0)); + elt, val, type, 0, false)); } location elt_loc = LOC(elt); @@ -4996,7 +5024,7 @@ pop_inlined_comprehension_state(compiler *c, location loc, static int codegen_comprehension(compiler *c, expr_ty e, int type, identifier name, asdl_comprehension_seq *generators, expr_ty elt, - expr_ty val) + expr_ty val, bool avoid_creation) { PyCodeObject *co = NULL; _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; @@ -5070,13 +5098,17 @@ codegen_comprehension(compiler *c, expr_ty e, int type, goto error_in_scope; } - ADDOP_I(c, loc, op, 0); - if (is_inlined) { - ADDOP_I(c, loc, SWAP, 2); + if (!avoid_creation) { + ADDOP_I(c, loc, op, 0); + if (is_inlined) { + ADDOP_I(c, loc, SWAP, 2); + } + } else { + ADDOP_I(c, loc, COPY, 1); } } if (codegen_comprehension_generator(c, loc, generators, 0, 0, - elt, val, type, iter_state) < 0) { + elt, val, type, iter_state, avoid_creation) < 0) { goto error_in_scope; } @@ -5084,6 +5116,9 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (pop_inlined_comprehension_state(c, loc, &inline_state)) { goto error; } + if (avoid_creation) { + ADDOP(c, loc, POP_TOP); + } return SUCCESS; } @@ -5118,6 +5153,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type, ADD_YIELD_FROM(c, loc, 1); } + assert(!avoid_creation); + return SUCCESS; error_in_scope: if (!is_inlined) { @@ -5133,44 +5170,67 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } static int -codegen_genexp(compiler *c, expr_ty e) +codegen_genexp_impl(compiler *c, expr_ty e, bool avoid_creation) { assert(e->kind == GeneratorExp_kind); _Py_DECLARE_STR(anon_genexpr, ""); return codegen_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr), e->v.GeneratorExp.generators, - e->v.GeneratorExp.elt, NULL); + e->v.GeneratorExp.elt, NULL, avoid_creation); } static int -codegen_listcomp(compiler *c, expr_ty e) +codegen_genexp(compiler *c, expr_ty e) +{ + return codegen_genexp_impl(c, e, false); +} + +static int +codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation) { assert(e->kind == ListComp_kind); _Py_DECLARE_STR(anon_listcomp, ""); return codegen_comprehension(c, e, COMP_LISTCOMP, &_Py_STR(anon_listcomp), e->v.ListComp.generators, - e->v.ListComp.elt, NULL); + e->v.ListComp.elt, NULL, avoid_creation); } static int -codegen_setcomp(compiler *c, expr_ty e) +codegen_listcomp(compiler *c, expr_ty e) +{ + return codegen_listcomp_impl(c, e, false); +} + +static int +codegen_setcomp_impl(compiler *c, expr_ty e, bool avoid_creation) { assert(e->kind == SetComp_kind); _Py_DECLARE_STR(anon_setcomp, ""); return codegen_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp), e->v.SetComp.generators, - e->v.SetComp.elt, NULL); + e->v.SetComp.elt, NULL, avoid_creation); } +static int +codegen_setcomp(compiler *c, expr_ty e) +{ + return codegen_setcomp_impl(c, e, false); +} static int -codegen_dictcomp(compiler *c, expr_ty e) +codegen_dictcomp_impl(compiler *c, expr_ty e, bool avoid_creation) { assert(e->kind == DictComp_kind); _Py_DECLARE_STR(anon_dictcomp, ""); return codegen_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp), e->v.DictComp.generators, - e->v.DictComp.key, e->v.DictComp.value); + e->v.DictComp.key, e->v.DictComp.value, avoid_creation); +} + +static int +codegen_dictcomp(compiler *c, expr_ty e) +{ + return codegen_dictcomp_impl(c, e, false); } From 77251a2816d65d0f1a70d91c4fe8ed5c09591ae0 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 01:49:17 -0400 Subject: [PATCH 02/13] Apply the optimization to sets and dicts. --- Python/codegen.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/codegen.c b/Python/codegen.c index 24fef9c72b141f3..285f9e26e5b487d 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3096,6 +3096,14 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value) return codegen_listcomp_impl(c, value, /*avoid_creation=*/true); } + if (value->kind == SetComp_kind) { + return codegen_setcomp_impl(c, value, /*avoid_creation=*/true); + } + + if (value->kind == DictComp_kind) { + return codegen_dictcomp_impl(c, value, /*avoid_creation=*/true); + } + VISIT(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ return SUCCESS; From d80384a6df1777c22cc499c0d33531a1fbb37927 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 02:25:23 -0400 Subject: [PATCH 03/13] Tests. --- Lib/test/test_compile.py | 34 ++++++++++++++--------------- Lib/test/test_compiler_codegen.py | 36 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9edbca3c383b43d..cce244d093f0dd6 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2063,7 +2063,7 @@ def test_multiline_async_generator_expression(self): def test_multiline_list_comprehension(self): snippet = textwrap.dedent("""\ - [(x, + _ = [(x, 2*x) for x in [1,2,3] if (x > 0 @@ -2073,14 +2073,14 @@ def test_multiline_list_comprehension(self): compiled_code, _ = self.check_positions_against_ast(snippet) self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', - line=1, end_line=2, column=1, end_column=8, occurrence=1) + line=1, end_line=2, column=5, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=1, end_line=2, column=1, end_column=8, occurrence=1) + line=1, end_line=2, column=5, end_column=8, occurrence=1) def test_multiline_async_list_comprehension(self): snippet = textwrap.dedent("""\ async def f(): - [(x, + _ = [(x, 2*x) async for x in [1,2,3] if (x > 0 @@ -2093,15 +2093,15 @@ async def f(): compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND', - line=2, end_line=3, column=5, end_column=12, occurrence=1) + line=2, end_line=3, column=9, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=2, end_line=3, column=5, end_column=12, occurrence=1) + line=2, end_line=3, column=9, end_column=12, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=2, end_line=7, column=4, end_column=36, occurrence=1) + line=2, end_line=2, column=4, end_column=5, occurrence=1) def test_multiline_set_comprehension(self): snippet = textwrap.dedent("""\ - {(x, + _ = {(x, 2*x) for x in [1,2,3] if (x > 0 @@ -2111,9 +2111,9 @@ def test_multiline_set_comprehension(self): compiled_code, _ = self.check_positions_against_ast(snippet) self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', - line=1, end_line=2, column=1, end_column=8, occurrence=1) + line=1, end_line=2, column=5, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=1, end_line=2, column=1, end_column=8, occurrence=1) + line=1, end_line=2, column=5, end_column=8, occurrence=1) def test_multiline_async_set_comprehension(self): snippet = textwrap.dedent("""\ @@ -2139,7 +2139,7 @@ async def f(): def test_multiline_dict_comprehension(self): snippet = textwrap.dedent("""\ - {x: + _ = {x: 2*x for x in [1,2,3] if (x > 0 @@ -2149,14 +2149,14 @@ def test_multiline_dict_comprehension(self): compiled_code, _ = self.check_positions_against_ast(snippet) self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', - line=1, end_line=2, column=1, end_column=7, occurrence=1) + line=1, end_line=2, column=5, end_column=7, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=1, end_line=2, column=1, end_column=7, occurrence=1) + line=1, end_line=2, column=5, end_column=7, occurrence=1) def test_multiline_async_dict_comprehension(self): snippet = textwrap.dedent("""\ async def f(): - {x: + _ = {x: 2*x async for x in [1,2,3] if (x > 0 @@ -2169,11 +2169,11 @@ async def f(): compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', - line=2, end_line=3, column=5, end_column=11, occurrence=1) + line=2, end_line=3, column=9, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=2, end_line=3, column=5, end_column=11, occurrence=1) + line=2, end_line=3, column=9, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=2, end_line=7, column=4, end_column=36, occurrence=1) + line=2, end_line=2, column=4, end_column=5, occurrence=1) def test_matchcase_sequence(self): snippet = textwrap.dedent("""\ diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 36058854a41d63e..172b2362e97bc3a 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -192,3 +192,39 @@ def test_frozenset_optimization(self): ('RETURN_VALUE', None) ] self.codegen_test(snippet, expected) + + def test_no_target_comp_optimization(self): + snippet = "[i for i in range(10)]" + expected = [ + ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_NAME', 0), + ('PUSH_NULL', None), + ('LOAD_CONST', 0), + ('CALL', 1), + ('LOAD_FAST_AND_CLEAR', 0), + ('SWAP', 2), + ('SETUP_FINALLY', 20), + ('COPY', 1), + ('GET_ITER', 0), + ('FOR_ITER', 16), + ('STORE_FAST', 0), + ('LOAD_FAST', 0), + ('POP_TOP', None), + ('JUMP', 11), + ('END_FOR', None), + ('POP_ITER', None), + ('POP_BLOCK', None), + ('JUMP_NO_INTERRUPT', 25), + ('SWAP', 2), + ('POP_TOP', None), + ('SWAP', 2), + ('STORE_FAST_MAYBE_NULL', 0), + ('RERAISE', 0), + ('SWAP', 2), + ('STORE_FAST_MAYBE_NULL', 0), + ('POP_TOP', None), + ('LOAD_CONST', 1), + ('RETURN_VALUE', None), + ] + self.codegen_test(snippet, expected) From d2749ae988f1048dd8b4a41c2ad2f2276ba47ae7 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 02:42:40 -0400 Subject: [PATCH 04/13] Add blurb. --- .../2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst new file mode 100644 index 000000000000000..d4ef1ec17915cb7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst @@ -0,0 +1,2 @@ +Avoid creating :class:`list`, :class:`set`, and :class:`dict` objects in +comprehensions when the comprehension is not used as a value. From cf1bb55633bd49eb3eef46609fcd4e689b08a32c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 03:49:17 -0400 Subject: [PATCH 05/13] Remove dict and set stuff. This was a breaking change because hash() was no longer called on the elements. --- Lib/test/test_compile.py | 20 ++++----- ...-06-22-02-42-38.gh-issue-151907.6Tol_J.rst | 4 +- Python/codegen.c | 41 ++----------------- 3 files changed, 16 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index cce244d093f0dd6..757abf5ffa01b87 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -2101,7 +2101,7 @@ async def f(): def test_multiline_set_comprehension(self): snippet = textwrap.dedent("""\ - _ = {(x, + {(x, 2*x) for x in [1,2,3] if (x > 0 @@ -2111,9 +2111,9 @@ def test_multiline_set_comprehension(self): compiled_code, _ = self.check_positions_against_ast(snippet) self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'SET_ADD', - line=1, end_line=2, column=5, end_column=8, occurrence=1) + line=1, end_line=2, column=1, end_column=8, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=1, end_line=2, column=5, end_column=8, occurrence=1) + line=1, end_line=2, column=1, end_column=8, occurrence=1) def test_multiline_async_set_comprehension(self): snippet = textwrap.dedent("""\ @@ -2139,7 +2139,7 @@ async def f(): def test_multiline_dict_comprehension(self): snippet = textwrap.dedent("""\ - _ = {x: + {x: 2*x for x in [1,2,3] if (x > 0 @@ -2149,14 +2149,14 @@ def test_multiline_dict_comprehension(self): compiled_code, _ = self.check_positions_against_ast(snippet) self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', - line=1, end_line=2, column=5, end_column=7, occurrence=1) + line=1, end_line=2, column=1, end_column=7, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=1, end_line=2, column=5, end_column=7, occurrence=1) + line=1, end_line=2, column=1, end_column=7, occurrence=1) def test_multiline_async_dict_comprehension(self): snippet = textwrap.dedent("""\ async def f(): - _ = {x: + {x: 2*x async for x in [1,2,3] if (x > 0 @@ -2169,11 +2169,11 @@ async def f(): compiled_code = g['f'].__code__ self.assertIsInstance(compiled_code, types.CodeType) self.assertOpcodeSourcePositionIs(compiled_code, 'MAP_ADD', - line=2, end_line=3, column=9, end_column=11, occurrence=1) + line=2, end_line=3, column=5, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD', - line=2, end_line=3, column=9, end_column=11, occurrence=1) + line=2, end_line=3, column=5, end_column=11, occurrence=1) self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE', - line=2, end_line=2, column=4, end_column=5, occurrence=1) + line=2, end_line=7, column=4, end_column=36, occurrence=1) def test_matchcase_sequence(self): snippet = textwrap.dedent("""\ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst index d4ef1ec17915cb7..cca300f6d0e2f42 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst @@ -1,2 +1,2 @@ -Avoid creating :class:`list`, :class:`set`, and :class:`dict` objects in -comprehensions when the comprehension is not used as a value. +Avoid creating :class:`list` objects in comprehensions when the comprehension +is not used as a value. diff --git a/Python/codegen.c b/Python/codegen.c index 285f9e26e5b487d..9eb4e8bd791c20a 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -235,10 +235,6 @@ typedef enum { static int codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation); -static int -codegen_setcomp_impl(compiler *c, expr_ty e, bool avoid_creation); -static int -codegen_dictcomp_impl(compiler *c, expr_ty e, bool avoid_creation); static int codegen_sync_comprehension_generator( compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, @@ -3096,14 +3092,6 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value) return codegen_listcomp_impl(c, value, /*avoid_creation=*/true); } - if (value->kind == SetComp_kind) { - return codegen_setcomp_impl(c, value, /*avoid_creation=*/true); - } - - if (value->kind == DictComp_kind) { - return codegen_dictcomp_impl(c, value, /*avoid_creation=*/true); - } - VISIT(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ return SUCCESS; @@ -4696,11 +4684,6 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } break; case COMP_SETCOMP: - if (avoid_creation) { - VISIT(c, expr, elt); - ADDOP(c, elt_loc, POP_TOP); - break; - } if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, SET_UPDATE, depth + 1); @@ -4711,11 +4694,6 @@ codegen_sync_comprehension_generator(compiler *c, location loc, } break; case COMP_DICTCOMP: - if (avoid_creation) { - VISIT(c, expr, elt); - ADDOP(c, elt_loc, POP_TOP); - break; - } if (val == NULL) { /* unpacking (**) case */ VISIT(c, expr, elt); @@ -5210,35 +5188,24 @@ codegen_listcomp(compiler *c, expr_ty e) } static int -codegen_setcomp_impl(compiler *c, expr_ty e, bool avoid_creation) +codegen_setcomp(compiler *c, expr_ty e) { assert(e->kind == SetComp_kind); _Py_DECLARE_STR(anon_setcomp, ""); return codegen_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp), e->v.SetComp.generators, - e->v.SetComp.elt, NULL, avoid_creation); + e->v.SetComp.elt, NULL, /*avoid_creation=*/false); } -static int -codegen_setcomp(compiler *c, expr_ty e) -{ - return codegen_setcomp_impl(c, e, false); -} static int -codegen_dictcomp_impl(compiler *c, expr_ty e, bool avoid_creation) +codegen_dictcomp(compiler *c, expr_ty e) { assert(e->kind == DictComp_kind); _Py_DECLARE_STR(anon_dictcomp, ""); return codegen_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp), e->v.DictComp.generators, - e->v.DictComp.key, e->v.DictComp.value, avoid_creation); -} - -static int -codegen_dictcomp(compiler *c, expr_ty e) -{ - return codegen_dictcomp_impl(c, e, false); + e->v.DictComp.key, e->v.DictComp.value, /*avoid_creation=*/false); } From 1988b9083bed3bec521d5ed24a8f5ac2dda371f5 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 14:54:39 -0400 Subject: [PATCH 06/13] Add tests for hashing errors on dict and set comprehensions. --- Lib/test/test_dictcomps.py | 8 ++++++++ Lib/test/test_setcomps.py | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Lib/test/test_dictcomps.py b/Lib/test/test_dictcomps.py index a7a46216787437e..9873e1f80aa631d 100644 --- a/Lib/test/test_dictcomps.py +++ b/Lib/test/test_dictcomps.py @@ -165,6 +165,14 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_hash_error(self): + class Unhashable: + def __hash__(self): + 0/0 + + with self.assertRaises(ZeroDivisionError): + {unhashable: 1 for unhashable in [Unhashable()]} + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index 6fc5bb74036c5fa..27748ebd997a620 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -188,6 +188,18 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_hash_error(self): + class Unhashable: + def __hash__(self): + 0/0 + + with self.assertRaises(ZeroDivisionError): + {unhashable for unhashable in [Unhashable()]} + + +if __name__ == "__main__": + unittest.main() + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): From c01058992d7ab9c030f526a13a83c14d227bfadf Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 22 Jun 2026 14:58:23 -0400 Subject: [PATCH 07/13] Add a test ensuring that side effects are retained. --- Lib/test/test_listcomps.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index cf3796d94808015..b73124c16e9d94f 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -793,6 +793,21 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_optimization_with_side_effects(self): + # List comprehensions that aren't used as a value are optimized + # to avoid creating a list. Ensure that side effects are still + # retained when this happens. + with self.assertRaises(ZeroDivisionError): + [0/0 for _ in [1]] + + count = 0 + def increment(): + nonlocal count + count += 1 + + [increment() for _ in range(5)] + self.assertEqual(count, 5) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): From c1aa7851ce6b01a75cca151a2a2af0bdb7b78d0a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 23 Jun 2026 16:26:38 -0400 Subject: [PATCH 08/13] Fix async list comprehensions. --- Lib/test/test_listcomps.py | 23 +++++++++++++++++++++++ Python/codegen.c | 26 ++++++++++++++------------ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index b73124c16e9d94f..0d2423905d40c9a 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -808,6 +808,29 @@ def increment(): [increment() for _ in range(5)] self.assertEqual(count, 5) + def test_async_optimization_with_side_effects(self): + import asyncio + + async def gen1(aiterator): + with self.assertRaises(ZeroDivisionError): + [0/0 async for _ in aiterator] + + async def gen2(aiterator): + [increment() async for _ in aiterator] + + async def numbers(): + for i in range(5): + yield i + + count = 0 + def increment(): + nonlocal count + count += 1 + + asyncio.run(gen1(numbers())) + asyncio.run(gen2(numbers())) + self.assertEqual(count, 5) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Python/codegen.c b/Python/codegen.c index 9eb4e8bd791c20a..7eb6a5315dc4706 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -247,7 +247,7 @@ static int codegen_async_comprehension_generator( asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type, - IterStackPosition iter_pos); + IterStackPosition iter_pos, bool avoid_creation); static int codegen_pattern(compiler *, pattern_ty, pattern_context *); static int codegen_match(compiler *, stmt_ty); @@ -4556,7 +4556,7 @@ codegen_comprehension_generator(compiler *c, location loc, if (gen->is_async) { return codegen_async_comprehension_generator( c, loc, generators, gen_index, depth, elt, val, type, - iter_pos); + iter_pos, avoid_creation); } else { return codegen_sync_comprehension_generator( c, loc, generators, gen_index, depth, elt, val, type, @@ -4737,7 +4737,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, int depth, expr_ty elt, expr_ty val, int type, - IterStackPosition iter_pos) + IterStackPosition iter_pos, bool avoid_creation) { NEW_JUMP_TARGET_LABEL(c, start); NEW_JUMP_TARGET_LABEL(c, send); @@ -4787,7 +4787,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, RETURN_IF_ERROR( codegen_comprehension_generator(c, loc, generators, gen_index, depth, - elt, val, type, 0, false)); + elt, val, type, 0, avoid_creation)); } location elt_loc = LOC(elt); @@ -4796,6 +4796,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, /* comprehension specific code */ switch (type) { case COMP_GENEXP: + assert(!avoid_creation); if (elt->kind == Starred_kind) { NEW_JUMP_TARGET_LABEL(c, unpack_start); NEW_JUMP_TARGET_LABEL(c, unpack_end); @@ -4817,6 +4818,11 @@ codegen_async_comprehension_generator(compiler *c, location loc, } break; case COMP_LISTCOMP: + if (avoid_creation) { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + break; + } if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1); @@ -4827,6 +4833,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, } break; case COMP_SETCOMP: + assert(!avoid_creation); if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, SET_UPDATE, depth + 1); @@ -4837,6 +4844,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, } break; case COMP_DICTCOMP: + assert(!avoid_creation); if (val == NULL) { /* unpacking (**) case */ VISIT(c, expr, elt); @@ -5156,19 +5164,13 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } static int -codegen_genexp_impl(compiler *c, expr_ty e, bool avoid_creation) +codegen_genexp(compiler *c, expr_ty e) { assert(e->kind == GeneratorExp_kind); _Py_DECLARE_STR(anon_genexpr, ""); return codegen_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr), e->v.GeneratorExp.generators, - e->v.GeneratorExp.elt, NULL, avoid_creation); -} - -static int -codegen_genexp(compiler *c, expr_ty e) -{ - return codegen_genexp_impl(c, e, false); + e->v.GeneratorExp.elt, NULL, false); } static int From a750ed556693a19557cbcc1a067791b150c46f14 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 24 Jun 2026 11:24:53 -0400 Subject: [PATCH 09/13] Don't use asyncio for testing async for. Avoids an "env-changed" error with regrtest. --- Lib/test/test_listcomps.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 0d2423905d40c9a..f64ef5a55fc42fc 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -809,8 +809,6 @@ def increment(): self.assertEqual(count, 5) def test_async_optimization_with_side_effects(self): - import asyncio - async def gen1(aiterator): with self.assertRaises(ZeroDivisionError): [0/0 async for _ in aiterator] @@ -827,8 +825,14 @@ def increment(): nonlocal count count += 1 - asyncio.run(gen1(numbers())) - asyncio.run(gen2(numbers())) + def exhaust(coro): + try: + coro.send(None) + except StopIteration: + pass + + exhaust(gen1(numbers())) + exhaust(gen2(numbers())) self.assertEqual(count, 5) __test__ = {'doctests' : doctests} From cdbcfadd014976d99e5129292ad1862f7e2cf26d Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 30 Jun 2026 13:27:28 -0400 Subject: [PATCH 10/13] Update Lib/test/test_compiler_codegen.py Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/test/test_compiler_codegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 172b2362e97bc3a..85c8526c049c0e4 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -193,7 +193,7 @@ def test_frozenset_optimization(self): ] self.codegen_test(snippet, expected) - def test_no_target_comp_optimization(self): + def test_comp_without_target_optimization(self): snippet = "[i for i in range(10)]" expected = [ ('RESUME', 0), From 5d9ccf3a8973bb69456d1945cf835c84e459a0d9 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 1 Jul 2026 14:49:32 -0400 Subject: [PATCH 11/13] Use a more generic implementation. --- Python/codegen.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 7eb6a5315dc4706..4b37b97a581574f 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -200,6 +200,7 @@ static int codegen_nameop(compiler *, location, identifier, expr_context_ty); static int codegen_visit_stmt(compiler *, stmt_ty); static int codegen_visit_keyword(compiler *, keyword_ty); static int codegen_visit_expr(compiler *, expr_ty); +static int codegen_visit_unused_expr(compiler *, expr_ty); static int codegen_augassign(compiler *, stmt_ty); static int codegen_annassign(compiler *, stmt_ty); static int codegen_subscript(compiler *, expr_ty); @@ -233,8 +234,6 @@ typedef enum { ITERATOR_ON_STACK = 2, } IterStackPosition; -static int -codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation); static int codegen_sync_comprehension_generator( compiler *c, location loc, asdl_comprehension_seq *generators, int gen_index, @@ -477,6 +476,9 @@ codegen_addop_j(instr_sequence *seq, location loc, } \ } while (0) +#define VISIT_UNUSED(C, TYPE, V) \ + RETURN_IF_ERROR(codegen_visit_unused_ ## TYPE((C), (V))) + static int codegen_call_exit_with_nones(compiler *c, location loc) { @@ -3086,13 +3088,7 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value) return SUCCESS; } - if (value->kind == ListComp_kind) { - /* Optimization: Don't bother creating structures if they won't be - * used. */ - return codegen_listcomp_impl(c, value, /*avoid_creation=*/true); - } - - VISIT(c, expr, value); + VISIT_UNUSED(c, expr, value); ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */ return SUCCESS; } @@ -5110,9 +5106,6 @@ codegen_comprehension(compiler *c, expr_ty e, int type, if (pop_inlined_comprehension_state(c, loc, &inline_state)) { goto error; } - if (avoid_creation) { - ADDOP(c, loc, POP_TOP); - } return SUCCESS; } @@ -5174,7 +5167,7 @@ codegen_genexp(compiler *c, expr_ty e) } static int -codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation) +codegen_listcomp(compiler *c, expr_ty e, bool avoid_creation) { assert(e->kind == ListComp_kind); _Py_DECLARE_STR(anon_listcomp, ""); @@ -5183,12 +5176,6 @@ codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation) e->v.ListComp.elt, NULL, avoid_creation); } -static int -codegen_listcomp(compiler *c, expr_ty e) -{ - return codegen_listcomp_impl(c, e, false); -} - static int codegen_setcomp(compiler *c, expr_ty e) { @@ -5455,7 +5442,7 @@ codegen_with(compiler *c, stmt_ty s) } static int -codegen_visit_expr(compiler *c, expr_ty e) +codegen_visit_expr_impl(compiler *c, expr_ty e, bool result_is_unused) { if (Py_EnterRecursiveCall(" during compilation")) { return ERROR; @@ -5498,7 +5485,7 @@ codegen_visit_expr(compiler *c, expr_ty e) case GeneratorExp_kind: return codegen_genexp(c, e); case ListComp_kind: - return codegen_listcomp(c, e); + return codegen_listcomp(c, e, result_is_unused); case SetComp_kind: return codegen_setcomp(c, e); case DictComp_kind: @@ -5608,6 +5595,18 @@ codegen_visit_expr(compiler *c, expr_ty e) return SUCCESS; } +static int +codegen_visit_expr(compiler *c, expr_ty e) +{ + return codegen_visit_expr_impl(c, e, false); +} + +static int +codegen_visit_unused_expr(compiler *c, expr_ty e) +{ + return codegen_visit_expr_impl(c, e, true); +} + static bool is_constant_slice(expr_ty s) { From ad24c1955c3a3ed12db51ac8d1950279ed17ea6f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 2 Jul 2026 12:20:51 -0400 Subject: [PATCH 12/13] Fix the optimization for starred comprehensions. --- Lib/test/test_listcomps.py | 27 ++++++++++++++++++++ Python/codegen.c | 52 ++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index f64ef5a55fc42fc..fca9acbc6b1ef6c 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -835,6 +835,33 @@ def exhaust(coro): exhaust(gen2(numbers())) self.assertEqual(count, 5) + def test_optimization_with_starred_unpack(self): + with self.assertRaises(TypeError): + [*i for i in [1, 2, 3]] + + async def coro(): + async def gen(): + yield 1 + + with self.assertRaises(TypeError): + [*i async for i in gen()] + + c = coro() + while True: + try: + c.send(None) + except StopIteration: + break + + count = 0 + def weird(): + nonlocal count + count += 1 + yield 0 + + [*weird() for _ in range(5)] + self.assertEqual(count, 5) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Python/codegen.c b/Python/codegen.c index 4b37b97a581574f..11a8926738bd934 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4560,6 +4560,28 @@ codegen_comprehension_generator(compiler *c, location loc, } } +static int +codegen_starred_comprehension(compiler *c, location loc, expr_ty value, bool discard) +{ + NEW_JUMP_TARGET_LABEL(c, unpack_start); + NEW_JUMP_TARGET_LABEL(c, unpack_end); + VISIT(c, expr, value); + ADDOP_I(c, loc, GET_ITER, 0); + USE_LABEL(c, unpack_start); + ADDOP_JUMP(c, loc, FOR_ITER, unpack_end); + if (discard) { + ADDOP(c, loc, POP_TOP); + } else { + ADDOP_YIELD(c, loc); + ADDOP(c, loc, POP_TOP); + } + ADDOP_JUMP(c, NO_LOCATION, JUMP, unpack_start); + USE_LABEL(c, unpack_end); + ADDOP(c, NO_LOCATION, END_FOR); + ADDOP(c, NO_LOCATION, POP_ITER); + return SUCCESS; +} + static int codegen_sync_comprehension_generator(compiler *c, location loc, asdl_comprehension_seq *generators, @@ -4645,18 +4667,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, case COMP_GENEXP: assert(!avoid_creation); if (elt->kind == Starred_kind) { - NEW_JUMP_TARGET_LABEL(c, unpack_start); - NEW_JUMP_TARGET_LABEL(c, unpack_end); - VISIT(c, expr, elt->v.Starred.value); - ADDOP_I(c, elt_loc, GET_ITER, 0); - USE_LABEL(c, unpack_start); - ADDOP_JUMP(c, elt_loc, FOR_ITER, unpack_end); - ADDOP_YIELD(c, elt_loc); - ADDOP(c, elt_loc, POP_TOP); - ADDOP_JUMP(c, NO_LOCATION, JUMP, unpack_start); - USE_LABEL(c, unpack_end); - ADDOP(c, NO_LOCATION, END_FOR); - ADDOP(c, NO_LOCATION, POP_ITER); + RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); } else { VISIT(c, expr, elt); @@ -4666,8 +4677,12 @@ codegen_sync_comprehension_generator(compiler *c, location loc, break; case COMP_LISTCOMP: if (avoid_creation) { - VISIT(c, expr, elt); - ADDOP(c, elt_loc, POP_TOP); + if (elt->kind == Starred_kind) { + RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/true)); + } else { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + } break; } if (elt->kind == Starred_kind) { @@ -4815,10 +4830,15 @@ codegen_async_comprehension_generator(compiler *c, location loc, break; case COMP_LISTCOMP: if (avoid_creation) { - VISIT(c, expr, elt); - ADDOP(c, elt_loc, POP_TOP); + if (elt->kind == Starred_kind) { + RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); + } else { + VISIT(c, expr, elt); + ADDOP(c, elt_loc, POP_TOP); + } break; } + if (elt->kind == Starred_kind) { VISIT(c, expr, elt->v.Starred.value); ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1); From 9fe67180ebf6941ea4e2c3496e72e0e77376b0fb Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 2 Jul 2026 13:31:44 -0400 Subject: [PATCH 13/13] Use a better name. --- Python/codegen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 11a8926738bd934..b3c68af276d3f2e 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4561,7 +4561,7 @@ codegen_comprehension_generator(compiler *c, location loc, } static int -codegen_starred_comprehension(compiler *c, location loc, expr_ty value, bool discard) +codegen_unpack_starred(compiler *c, location loc, expr_ty value, bool discard) { NEW_JUMP_TARGET_LABEL(c, unpack_start); NEW_JUMP_TARGET_LABEL(c, unpack_end); @@ -4667,7 +4667,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, case COMP_GENEXP: assert(!avoid_creation); if (elt->kind == Starred_kind) { - RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); + RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); } else { VISIT(c, expr, elt); @@ -4678,7 +4678,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc, case COMP_LISTCOMP: if (avoid_creation) { if (elt->kind == Starred_kind) { - RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/true)); + RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*discard=*/true)); } else { VISIT(c, expr, elt); ADDOP(c, elt_loc, POP_TOP); @@ -4831,7 +4831,7 @@ codegen_async_comprehension_generator(compiler *c, location loc, case COMP_LISTCOMP: if (avoid_creation) { if (elt->kind == Starred_kind) { - RETURN_IF_ERROR(codegen_starred_comprehension(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); + RETURN_IF_ERROR(codegen_unpack_starred(c, elt_loc, elt->v.Starred.value, /*discard=*/false)); } else { VISIT(c, expr, elt); ADDOP(c, elt_loc, POP_TOP);