Skip to content

gh-151907: Avoid creating temporary objects in some comprehensions #151908

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZeroIntensity wants to merge 7 commits into
base: main
Choose a base branch
from
Open

gh-151907: Avoid creating temporary objects in some comprehensions #151908

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b4a6d81
Initial implementation.
ZeroIntensity Jun 22, 2026
77251a2
Apply the optimization to sets and dicts.
ZeroIntensity Jun 22, 2026
d80384a
Tests.
ZeroIntensity Jun 22, 2026
d2749ae
Add blurb.
ZeroIntensity Jun 22, 2026
cf1bb55
Remove dict and set stuff.
ZeroIntensity Jun 22, 2026
1988b90
Add tests for hashing errors on dict and set comprehensions.
ZeroIntensity Jun 22, 2026
c010589
Add a test ensuring that side effects are retained.
ZeroIntensity Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions Lib/test/test_compile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,7 +2063,7 @@ def test_multiline_async_generator_expression(self):

def test_multiline_list_comprehension(self):
snippet = textwrap.dedent("""\
[(x,
_ = [(x,
2*x)
for x
in [1,2,3] if (x > 0
Expand All @@ -2073,14 +2073,14 @@ def test_multiline_list_comprehension(self):
compiled_code, _ = self.check_positions_against_ast(snippet)
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
line=1, end_line=2, column=5, end_column=8, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=1, end_line=2, column=1, end_column=8, occurrence=1)
line=1, end_line=2, column=5, end_column=8, occurrence=1)

def test_multiline_async_list_comprehension(self):
snippet = textwrap.dedent("""\
async def f():
[(x,
_ = [(x,
2*x)
async for x
in [1,2,3] if (x > 0
Expand All @@ -2093,11 +2093,11 @@ async def f():
compiled_code = g['f'].__code__
self.assertIsInstance(compiled_code, types.CodeType)
self.assertOpcodeSourcePositionIs(compiled_code, 'LIST_APPEND',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
line=2, end_line=3, column=9, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'JUMP_BACKWARD',
line=2, end_line=3, column=5, end_column=12, occurrence=1)
line=2, end_line=3, column=9, end_column=12, occurrence=1)
self.assertOpcodeSourcePositionIs(compiled_code, 'RETURN_VALUE',
line=2, end_line=7, column=4, end_column=36, occurrence=1)
line=2, end_line=2, column=4, end_column=5, occurrence=1)

def test_multiline_set_comprehension(self):
snippet = textwrap.dedent("""\
Expand Down
36 changes: 36 additions & 0 deletions Lib/test/test_compiler_codegen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,3 +192,39 @@ def test_frozenset_optimization(self):
('RETURN_VALUE', None)
]
self.codegen_test(snippet, expected)

def test_no_target_comp_optimization(self):

@sergey-miryanov sergey-miryanov Jun 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe you need to add a test where the loop body has a side effect. IIUC, the body with thread.join() should be turned into a method call, rather than just pass.

snippet = "[i for i in range(10)]"
expected = [
('RESUME', 0),
('ANNOTATIONS_PLACEHOLDER', None),
('LOAD_NAME', 0),
('PUSH_NULL', None),
('LOAD_CONST', 0),
('CALL', 1),
('LOAD_FAST_AND_CLEAR', 0),
('SWAP', 2),
('SETUP_FINALLY', 20),
('COPY', 1),
('GET_ITER', 0),
('FOR_ITER', 16),
('STORE_FAST', 0),
('LOAD_FAST', 0),
('POP_TOP', None),
('JUMP', 11),
('END_FOR', None),
('POP_ITER', None),
('POP_BLOCK', None),
('JUMP_NO_INTERRUPT', 25),
('SWAP', 2),
('POP_TOP', None),
('SWAP', 2),
('STORE_FAST_MAYBE_NULL', 0),
('RERAISE', 0),
('SWAP', 2),
('STORE_FAST_MAYBE_NULL', 0),
('POP_TOP', None),
('LOAD_CONST', 1),
('RETURN_VALUE', None),
]
self.codegen_test(snippet, expected)
8 changes: 8 additions & 0 deletions Lib/test/test_dictcomps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,14 @@ def iter_raises():
self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
expected)

def test_hash_error(self):
class Unhashable:
def __hash__(self):
0/0

with self.assertRaises(ZeroDivisionError):
{unhashable: 1 for unhashable in [Unhashable()]}


if __name__ == "__main__":
unittest.main()
15 changes: 15 additions & 0 deletions Lib/test/test_listcomps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,21 @@ def iter_raises():
self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
expected)

def test_optimization_with_side_effects(self):
# List comprehensions that aren't used as a value are optimized
# to avoid creating a list. Ensure that side effects are still
# retained when this happens.
with self.assertRaises(ZeroDivisionError):
[0/0 for _ in [1]]

count = 0
def increment():
nonlocal count
count += 1

[increment() for _ in range(5)]
self.assertEqual(count, 5)

__test__ = {'doctests' : doctests}

def load_tests(loader, tests, pattern):
Expand Down
12 changes: 12 additions & 0 deletions Lib/test/test_setcomps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,18 @@ def iter_raises():
self.assertEqual(f.line[f.colno - indent : f.end_colno - indent],
expected)

def test_hash_error(self):
class Unhashable:
def __hash__(self):
0/0

with self.assertRaises(ZeroDivisionError):
{unhashable for unhashable in [Unhashable()]}


if __name__ == "__main__":
unittest.main()

__test__ = {'doctests' : doctests}

def load_tests(loader, tests, pattern):
Expand Down
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-02-42-38.gh-issue-151907.6Tol_J.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Avoid creating :class:`list` objects in comprehensions when the comprehension
is not used as a value.
69 changes: 52 additions & 17 deletions Python/codegen.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,12 +233,14 @@ typedef enum {
ITERATOR_ON_STACK = 2,
} IterStackPosition;

static int
codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation);
static int codegen_sync_comprehension_generator(
compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
IterStackPosition iter_pos);
IterStackPosition iter_pos, bool avoid_creation);

static int codegen_async_comprehension_generator(
compiler *c, location loc,
Expand Down Expand Up @@ -3084,6 +3086,12 @@ codegen_stmt_expr(compiler *c, location loc, expr_ty value)
return SUCCESS;
}

if (value->kind == ListComp_kind) {
/* Optimization: Don't bother creating structures if they won't be
* used. */
return codegen_listcomp_impl(c, value, /*avoid_creation=*/true);
}

VISIT(c, expr, value);
ADDOP(c, NO_LOCATION, POP_TOP); /* artificial */
return SUCCESS;
Expand Down Expand Up @@ -4541,7 +4549,7 @@ codegen_comprehension_generator(compiler *c, location loc,
asdl_comprehension_seq *generators, int gen_index,
int depth,
expr_ty elt, expr_ty val, int type,
IterStackPosition iter_pos)
IterStackPosition iter_pos, bool avoid_creation)
{
comprehension_ty gen;
gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
Expand All @@ -4552,7 +4560,7 @@ codegen_comprehension_generator(compiler *c, location loc,
} else {
return codegen_sync_comprehension_generator(
c, loc, generators, gen_index, depth, elt, val, type,
iter_pos);
iter_pos, avoid_creation);
}
}

Expand All @@ -4561,7 +4569,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
asdl_comprehension_seq *generators,
int gen_index, int depth,
expr_ty elt, expr_ty val, int type,
IterStackPosition iter_pos)
IterStackPosition iter_pos, bool avoid_creation)
{
/* generate code for the iterator, then each of the ifs,
and then write to the element */
Expand Down Expand Up @@ -4629,7 +4637,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
RETURN_IF_ERROR(
codegen_comprehension_generator(c, loc,
generators, gen_index, depth,
elt, val, type, ITERABLE_IN_LOCAL));
elt, val, type, ITERABLE_IN_LOCAL, avoid_creation));
}

location elt_loc = LOC(elt);
Expand All @@ -4639,6 +4647,7 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
/* comprehension specific code */
switch (type) {
case COMP_GENEXP:
assert(!avoid_creation);
if (elt->kind == Starred_kind) {
NEW_JUMP_TARGET_LABEL(c, unpack_start);
NEW_JUMP_TARGET_LABEL(c, unpack_end);
Expand All @@ -4660,6 +4669,11 @@ codegen_sync_comprehension_generator(compiler *c, location loc,
}
break;
case COMP_LISTCOMP:
if (avoid_creation) {
VISIT(c, expr, elt);
ADDOP(c, elt_loc, POP_TOP);
break;
}

@iritkatriel iritkatriel Jun 22, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we instead just wrap the LIST_APPEND in if !avoid_creation?

@ZeroIntensity ZeroIntensity Jun 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we'd also need to wrap the LIST_EXTEND with that too; I opted for this approach to avoid duplicating that logic.

But do we need the separate VISIT path for the Starred case at all? I might be able to refactor it to look like this:

ISIT(c, expr, elt);
if (elt->kind == Starred_kind) {
    op = LIST_EXTEND;
}
else {
    op = LIST_APPEND;
}
if (!avoid_creation) {
    ADDOP_I(c, elt_loc, op, depth + 1);
}

if (elt->kind == Starred_kind) {
VISIT(c, expr, elt->v.Starred.value);
ADDOP_I(c, elt_loc, LIST_EXTEND, depth + 1);
Expand Down Expand Up @@ -4773,7 +4787,7 @@ codegen_async_comprehension_generator(compiler *c, location loc,
RETURN_IF_ERROR(
codegen_comprehension_generator(c, loc,
generators, gen_index, depth,
elt, val, type, 0));
elt, val, type, 0, false));
}

location elt_loc = LOC(elt);
Expand Down Expand Up @@ -4996,7 +5010,7 @@ pop_inlined_comprehension_state(compiler *c, location loc,
static int
codegen_comprehension(compiler *c, expr_ty e, int type,
identifier name, asdl_comprehension_seq *generators, expr_ty elt,
expr_ty val)
expr_ty val, bool avoid_creation)
{
PyCodeObject *co = NULL;
_PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL};
Expand Down Expand Up @@ -5070,20 +5084,27 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
goto error_in_scope;
}

ADDOP_I(c, loc, op, 0);
if (is_inlined) {
ADDOP_I(c, loc, SWAP, 2);
if (!avoid_creation) {
ADDOP_I(c, loc, op, 0);
if (is_inlined) {
ADDOP_I(c, loc, SWAP, 2);
}
} else {
ADDOP_I(c, loc, COPY, 1);
}
}
if (codegen_comprehension_generator(c, loc, generators, 0, 0,
elt, val, type, iter_state) < 0) {
elt, val, type, iter_state, avoid_creation) < 0) {
goto error_in_scope;
}

if (is_inlined) {
if (pop_inlined_comprehension_state(c, loc, &inline_state)) {
goto error;
}
if (avoid_creation) {
ADDOP(c, loc, POP_TOP);
}
return SUCCESS;
}

Expand Down Expand Up @@ -5118,6 +5139,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
ADD_YIELD_FROM(c, loc, 1);
}

assert(!avoid_creation);

return SUCCESS;
error_in_scope:
if (!is_inlined) {
Expand All @@ -5133,23 +5156,35 @@ codegen_comprehension(compiler *c, expr_ty e, int type,
}

static int
codegen_genexp(compiler *c, expr_ty e)
codegen_genexp_impl(compiler *c, expr_ty e, bool avoid_creation)
{
assert(e->kind == GeneratorExp_kind);
_Py_DECLARE_STR(anon_genexpr, "<genexpr>");
return codegen_comprehension(c, e, COMP_GENEXP, &_Py_STR(anon_genexpr),
e->v.GeneratorExp.generators,
e->v.GeneratorExp.elt, NULL);
e->v.GeneratorExp.elt, NULL, avoid_creation);
}

static int
codegen_listcomp(compiler *c, expr_ty e)
codegen_genexp(compiler *c, expr_ty e)
{
return codegen_genexp_impl(c, e, false);
}

static int
codegen_listcomp_impl(compiler *c, expr_ty e, bool avoid_creation)
{
assert(e->kind == ListComp_kind);
_Py_DECLARE_STR(anon_listcomp, "<listcomp>");
return codegen_comprehension(c, e, COMP_LISTCOMP, &_Py_STR(anon_listcomp),
e->v.ListComp.generators,
e->v.ListComp.elt, NULL);
e->v.ListComp.elt, NULL, avoid_creation);
}

static int
codegen_listcomp(compiler *c, expr_ty e)
{
return codegen_listcomp_impl(c, e, false);
}

static int
Expand All @@ -5159,7 +5194,7 @@ codegen_setcomp(compiler *c, expr_ty e)
_Py_DECLARE_STR(anon_setcomp, "<setcomp>");
return codegen_comprehension(c, e, COMP_SETCOMP, &_Py_STR(anon_setcomp),
e->v.SetComp.generators,
e->v.SetComp.elt, NULL);
e->v.SetComp.elt, NULL, /*avoid_creation=*/false);
}


Expand All @@ -5170,7 +5205,7 @@ codegen_dictcomp(compiler *c, expr_ty e)
_Py_DECLARE_STR(anon_dictcomp, "<dictcomp>");
return codegen_comprehension(c, e, COMP_DICTCOMP, &_Py_STR(anon_dictcomp),
e->v.DictComp.generators,
e->v.DictComp.key, e->v.DictComp.value);
e->v.DictComp.key, e->v.DictComp.value, /*avoid_creation=*/false);
}


Expand Down
Loading